From 46fd5069c99849c9b1ee1c0e432d7301c83f834a Mon Sep 17 00:00:00 2001 From: bin Date: Wed, 17 Nov 2021 22:50:15 +0800 Subject: [PATCH 01/66] docs: update using-SPDK-vhostuser-and-kata.md Use `ctr` instead of `Docker`. Fixes: #3054 Signed-off-by: bin --- .../using-SPDK-vhostuser-and-kata.md | 35 ++++++++++++++----- 1 file changed, 26 insertions(+), 9 deletions(-) diff --git a/docs/use-cases/using-SPDK-vhostuser-and-kata.md b/docs/use-cases/using-SPDK-vhostuser-and-kata.md index 4cef647ea..ff6905a72 100644 --- a/docs/use-cases/using-SPDK-vhostuser-and-kata.md +++ b/docs/use-cases/using-SPDK-vhostuser-and-kata.md @@ -1,4 +1,4 @@ -# Setup to run SPDK vhost-user devices with Kata Containers and Docker* +# Setup to run SPDK vhost-user devices with Kata Containers > **Note:** This guide only applies to QEMU, since the vhost-user storage > device is only available for QEMU now. The enablement work on other @@ -222,26 +222,43 @@ minor `0` should be created for it, in order to be recognized by Kata runtime: $ sudo mknod /var/run/kata-containers/vhost-user/block/devices/vhostblk0 b 241 0 ``` -> **Note:** The enablement of vhost-user block device in Kata containers -> is supported by Kata Containers `1.11.0-alpha1` or newer. -> Make sure you have updated your Kata containers before evaluation. - ## Launch a Kata container with SPDK vhost-user block device -To use `vhost-user-blk` device, use Docker to pass a host `vhost-user-blk` -device to the container. In docker, `--device=HOST-DIR:CONTAINER-DIR` is used +To use `vhost-user-blk` device, use `ctr` to pass a host `vhost-user-blk` +device to the container. In your `config.json`, you should use `devices` to pass a host device to the container. -For example: +For example (only `vhost-user-blk` listed): + +```json +{ + "linux": { + "devices": [ + { + "path": "/dev/vda", + "type": "b", + "major": 241, + "minor": 0, + "fileMode": 420, + "uid": 0, + "gid": 0 + } + ] + } +} +``` + +With `rootfs` provisioned under `bundle` directory, you can run your SPDK container: ```bash -$ sudo docker run --runtime kata-runtime --device=/var/run/kata-containers/vhost-user/block/devices/vhostblk0:/dev/vda -it busybox sh +$ sudo ctr run -d --runtime io.containerd.run.kata.v2 --config bundle/config.json spdk_container ``` Example of performing I/O operations on the `vhost-user-blk` device inside container: ``` +$ sudo ctr t exec --exec-id 1 -t spdk_container sh / # ls -l /dev/vda brw-r--r-- 1 root root 254, 0 Jan 20 03:54 /dev/vda / # dd if=/dev/vda of=/tmp/ddtest bs=4k count=20 From 6955d1442f413ae90e62e73b46f49420ed116049 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fabiano=20Fid=C3=AAncio?= Date: Thu, 18 Nov 2021 09:36:13 +0100 Subject: [PATCH 02/66] kata-deploy: Add back stable & latest tags MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit stable-2.3 was the first time we branched the repo since 43a72d76e2520ae044335b9f1dbf54c12363f54b was merged. One bit that I didn't notice while working on this, regardless of being warned by @amshinde (sorry!), was that the change would happen on `main` branch, rather than on the branched `stable-2.3` one. In my mind, the workflow was: * we branch. * we do the changes, including removing the files. * we tag a release. However, the workflow actually is: * we do the changes, including removing the files. * we branch. * we tag a release. A better way to deal with this has to be figured out before 2.4.0 is out, but for now let's just re-add the files back. Fixes: #3067 Signed-off-by: Fabiano Fidêncio --- .../base/kata-cleanup-stable.yaml | 46 +++++++++++++ .../kata-cleanup/base/kata-cleanup.yaml | 2 +- .../kata-deploy/base/kata-deploy-stable.yaml | 69 +++++++++++++++++++ .../kata-deploy/base/kata-deploy.yaml | 2 +- 4 files changed, 117 insertions(+), 2 deletions(-) create mode 100644 tools/packaging/kata-deploy/kata-cleanup/base/kata-cleanup-stable.yaml create mode 100644 tools/packaging/kata-deploy/kata-deploy/base/kata-deploy-stable.yaml diff --git a/tools/packaging/kata-deploy/kata-cleanup/base/kata-cleanup-stable.yaml b/tools/packaging/kata-deploy/kata-cleanup/base/kata-cleanup-stable.yaml new file mode 100644 index 000000000..f1d9d0a2f --- /dev/null +++ b/tools/packaging/kata-deploy/kata-cleanup/base/kata-cleanup-stable.yaml @@ -0,0 +1,46 @@ +--- +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: kubelet-kata-cleanup + namespace: kube-system +spec: + selector: + matchLabels: + name: kubelet-kata-cleanup + template: + metadata: + labels: + name: kubelet-kata-cleanup + spec: + serviceAccountName: kata-label-node + nodeSelector: + katacontainers.io/kata-runtime: cleanup + containers: + - name: kube-kata-cleanup + image: quay.io/kata-containers/kata-deploy:stable + imagePullPolicy: Always + command: [ "bash", "-c", "/opt/kata-artifacts/scripts/kata-deploy.sh reset" ] + env: + - name: NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + securityContext: + privileged: false + volumeMounts: + - name: dbus + mountPath: /var/run/dbus + - name: systemd + mountPath: /run/systemd + volumes: + - name: dbus + hostPath: + path: /var/run/dbus + - name: systemd + hostPath: + path: /run/systemd + updateStrategy: + rollingUpdate: + maxUnavailable: 1 + type: RollingUpdate diff --git a/tools/packaging/kata-deploy/kata-cleanup/base/kata-cleanup.yaml b/tools/packaging/kata-deploy/kata-cleanup/base/kata-cleanup.yaml index fe49f2bc2..851e958a4 100644 --- a/tools/packaging/kata-deploy/kata-cleanup/base/kata-cleanup.yaml +++ b/tools/packaging/kata-deploy/kata-cleanup/base/kata-cleanup.yaml @@ -18,7 +18,7 @@ spec: katacontainers.io/kata-runtime: cleanup containers: - name: kube-kata-cleanup - image: quay.io/kata-containers/kata-deploy:2.3.0-rc0 + image: quay.io/kata-containers/kata-deploy:latest imagePullPolicy: Always command: [ "bash", "-c", "/opt/kata-artifacts/scripts/kata-deploy.sh reset" ] env: diff --git a/tools/packaging/kata-deploy/kata-deploy/base/kata-deploy-stable.yaml b/tools/packaging/kata-deploy/kata-deploy/base/kata-deploy-stable.yaml new file mode 100644 index 000000000..346e4c0ee --- /dev/null +++ b/tools/packaging/kata-deploy/kata-deploy/base/kata-deploy-stable.yaml @@ -0,0 +1,69 @@ +--- +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: kata-deploy + namespace: kube-system +spec: + selector: + matchLabels: + name: kata-deploy + template: + metadata: + labels: + name: kata-deploy + spec: + serviceAccountName: kata-label-node + containers: + - name: kube-kata + image: quay.io/kata-containers/kata-deploy:stable + imagePullPolicy: Always + lifecycle: + preStop: + exec: + command: ["bash", "-c", "/opt/kata-artifacts/scripts/kata-deploy.sh cleanup"] + command: [ "bash", "-c", "/opt/kata-artifacts/scripts/kata-deploy.sh install" ] + env: + - name: NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + securityContext: + privileged: false + volumeMounts: + - name: crio-conf + mountPath: /etc/crio/ + - name: containerd-conf + mountPath: /etc/containerd/ + - name: kata-artifacts + mountPath: /opt/kata/ + - name: dbus + mountPath: /var/run/dbus + - name: systemd + mountPath: /run/systemd + - name: local-bin + mountPath: /usr/local/bin/ + volumes: + - name: crio-conf + hostPath: + path: /etc/crio/ + - name: containerd-conf + hostPath: + path: /etc/containerd/ + - name: kata-artifacts + hostPath: + path: /opt/kata/ + type: DirectoryOrCreate + - name: dbus + hostPath: + path: /var/run/dbus + - name: systemd + hostPath: + path: /run/systemd + - name: local-bin + hostPath: + path: /usr/local/bin/ + updateStrategy: + rollingUpdate: + maxUnavailable: 1 + type: RollingUpdate diff --git a/tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml b/tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml index cc3638e57..a03a56b84 100644 --- a/tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml +++ b/tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml @@ -16,7 +16,7 @@ spec: serviceAccountName: kata-label-node containers: - name: kube-kata - image: quay.io/kata-containers/kata-deploy:2.3.0-rc0 + image: quay.io/kata-containers/kata-deploy:latest imagePullPolicy: Always lifecycle: preStop: From 3c9ae7fb4b93a33a8c2c9b39a99dac541bc859ee Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fabiano=20Fid=C3=AAncio?= Date: Thu, 18 Nov 2021 12:49:17 +0100 Subject: [PATCH 03/66] kata-deploy: Ensure we test HEAD with `/test_kata_deploy` MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Is the past few releases we ended up hitting issues that could be easily avoided if `/test_kata_deploy` would use HEAD instead of a specific tarball. By the end of the day, we want to ensure kata-deploy works, but before we cut a release we also want to ensure that the binaries used in that release are in a good shape. If we don't do that we end up either having to roll a release back, or to cut a second release in a really short time (and that's time consuming). Note: there's code duplication here that could and should be avoided,b but I sincerely would prefer treating it in a different PR. Fixes: #3001 Signed-off-by: Fabiano Fidêncio --- .github/workflows/kata-deploy-test.yaml | 111 +++++++++++++++++------- 1 file changed, 78 insertions(+), 33 deletions(-) diff --git a/.github/workflows/kata-deploy-test.yaml b/.github/workflows/kata-deploy-test.yaml index febd4fbee..343fc6701 100644 --- a/.github/workflows/kata-deploy-test.yaml +++ b/.github/workflows/kata-deploy-test.yaml @@ -5,46 +5,91 @@ on: name: test-kata-deploy jobs: - create-and-test-container: - if: | - github.event.issue.pull_request - && github.event_name == 'issue_comment' - && github.event.action == 'created' - && startsWith(github.event.comment.body, '/test_kata_deploy') + build-asset: + runs-on: ubuntu-latest + strategy: + matrix: + asset: + - cloud-hypervisor + - firecracker + - kernel + - qemu + - rootfs-image + - rootfs-initrd + - shim-v2 + steps: + - uses: actions/checkout@v2 + - name: Install docker + run: | + curl -fsSL https://test.docker.com -o test-docker.sh + sh test-docker.sh + + - name: Build ${{ matrix.asset }} + run: | + make "${KATA_ASSET}-tarball" + build_dir=$(readlink -f build) + # store-artifact does not work with symlink + sudo cp -r "${build_dir}" "kata-build" + env: + KATA_ASSET: ${{ matrix.asset }} + TAR_OUTPUT: ${{ matrix.asset }}.tar.gz + + - name: store-artifact ${{ matrix.asset }} + uses: actions/upload-artifact@v2 + with: + name: kata-artifacts + path: kata-build/kata-static-${{ matrix.asset }}.tar.xz + if-no-files-found: error + + create-kata-tarball: + runs-on: ubuntu-latest + needs: build-asset + steps: + - uses: actions/checkout@v2 + - name: get-artifacts + uses: actions/download-artifact@v2 + with: + name: kata-artifacts + path: kata-artifacts + - name: merge-artifacts + run: | + ./tools/packaging/kata-deploy/local-build/kata-deploy-merge-builds.sh kata-artifacts + - name: store-artifacts + uses: actions/upload-artifact@v2 + with: + name: kata-static-tarball + path: kata-static.tar.xz + + kata-deploy: + needs: create-kata-tarball runs-on: ubuntu-latest steps: - - name: get-PR-ref - id: get-PR-ref - run: | - ref=$(cat $GITHUB_EVENT_PATH | jq -r '.issue.pull_request.url' | sed 's#^.*\/pulls#refs\/pull#' | sed 's#$#\/merge#') - echo "reference for PR: " ${ref} - echo "##[set-output name=pr-ref;]${ref}" - - - name: check out - uses: actions/checkout@v2 + - uses: actions/checkout@v2 + - name: get-kata-tarball + uses: actions/download-artifact@v2 with: - ref: ${{ steps.get-PR-ref.outputs.pr-ref }} - - - name: build-container-image - id: build-container-image + name: kata-static-tarball + - name: build-and-push-kata-deploy-ci + id: build-and-push-kata-deploy-ci run: | - PR_SHA=$(git log --format=format:%H -n1) - VERSION="2.0.0" - ARTIFACT_URL="https://github.com/kata-containers/kata-containers/releases/download/${VERSION}/kata-static-${VERSION}-x86_64.tar.xz" - wget "${ARTIFACT_URL}" -O tools/packaging/kata-deploy/kata-static.tar.xz - docker build --build-arg KATA_ARTIFACTS=kata-static.tar.xz -t katadocker/kata-deploy-ci:${PR_SHA} -t quay.io/kata-containers/kata-deploy-ci:${PR_SHA} ./tools/packaging/kata-deploy - docker login -u ${{ secrets.DOCKER_USERNAME }} -p ${{ secrets.DOCKER_PASSWORD }} - docker push katadocker/kata-deploy-ci:$PR_SHA - docker login -u ${{ secrets.QUAY_DEPLOYER_USERNAME }} -p ${{ secrets.QUAY_DEPLOYER_PASSWORD }} quay.io - docker push quay.io/kata-containers/kata-deploy-ci:$PR_SHA - echo "##[set-output name=pr-sha;]${PR_SHA}" - + tag=$(echo $GITHUB_REF | cut -d/ -f3-) + pushd $GITHUB_WORKSPACE + git checkout $tag + pkg_sha=$(git rev-parse HEAD) + popd + mv kata-static.tar.xz $GITHUB_WORKSPACE/tools/packaging/kata-deploy/kata-static.tar.xz + docker build --build-arg KATA_ARTIFACTS=kata-static.tar.xz -t quay.io/kata-containers/kata-deploy-ci:$pkg_sha $GITHUB_WORKSPACE/tools/packaging/kata-deploy + docker login -u ${{ secrets.QUAY_DEPLOYER_USERNAME }} -p ${{ secrets.QUAY_DEPLOYER_PASSWORD }} quay.io + docker push quay.io/kata-containers/kata-deploy-ci:$pkg_sha + mkdir -p packaging/kata-deploy + ln -s $GITHUB_WORKSPACE/tools/packaging/kata-deploy/action packaging/kata-deploy/action + echo "::set-output name=PKG_SHA::${pkg_sha}" - name: test-kata-deploy-ci-in-aks - uses: ./tools/packaging/kata-deploy/action + uses: ./packaging/kata-deploy/action with: - packaging-sha: ${{ steps.build-container-image.outputs.pr-sha }} + packaging-sha: ${{steps.build-and-push-kata-deploy-ci.outputs.PKG_SHA}} env: - PKG_SHA: ${{ steps.build-container-image.outputs.pr-sha }} + PKG_SHA: ${{steps.build-and-push-kata-deploy-ci.outputs.PKG_SHA}} AZ_APPID: ${{ secrets.AZ_APPID }} AZ_PASSWORD: ${{ secrets.AZ_PASSWORD }} AZ_SUBSCRIPTION_ID: ${{ secrets.AZ_SUBSCRIPTION_ID }} From 2b6dfe414a4c592200ccfcc6426708d4a781a9d4 Mon Sep 17 00:00:00 2001 From: Eric Ernst Date: Tue, 9 Nov 2021 10:56:39 -0800 Subject: [PATCH 04/66] watchers: don't dereference symlinks when copying files The current implementation just copies the file, dereferencing any simlinks in the process. This results in symlinks no being preserved, and a change in layout relative to the mount that we are making watchable. What we want is something like "cp -d" This isn't available in a crate, so let's go ahead and introduce a copy function which will create a symlink with same relative path if the source file is a symlink. Regular files are handled with the standard fs::copy. Introduce a unit test to verify symlinks are now handled appropriately. Fixes: #2950 Signed-off-by: Eric Ernst --- src/agent/src/watcher.rs | 109 ++++++++++++++++++++++++++++++++++++--- 1 file changed, 101 insertions(+), 8 deletions(-) diff --git a/src/agent/src/watcher.rs b/src/agent/src/watcher.rs index b111aa166..187d85ccc 100644 --- a/src/agent/src/watcher.rs +++ b/src/agent/src/watcher.rs @@ -79,6 +79,16 @@ impl Drop for Storage { } } +async fn copy(from: impl AsRef, to: impl AsRef) -> Result<()> { + // if source is a symlink, just create new symlink with same link source + if fs::symlink_metadata(&from).await?.file_type().is_symlink() { + fs::symlink(fs::read_link(&from).await?, to).await?; + } else { + fs::copy(from, to).await?; + } + Ok(()) +} + impl Storage { async fn new(storage: protos::Storage) -> Result { let entry = Storage { @@ -110,19 +120,13 @@ impl Storage { dest_file_path }; - debug!( - logger, - "Copy from {} to {}", - source_file_path.display(), - dest_file_path.display() - ); - fs::copy(&source_file_path, &dest_file_path) + copy(&source_file_path, &dest_file_path) .await .with_context(|| { format!( "Copy from {} to {} failed", source_file_path.display(), - dest_file_path.display() + dest_file_path.display(), ) })?; @@ -843,6 +847,95 @@ mod tests { } } + #[tokio::test] + async fn test_copy() { + // prepare tmp src/destination + let source_dir = tempfile::tempdir().unwrap(); + let dest_dir = tempfile::tempdir().unwrap(); + + // verify copy of a regular file + let src_file = source_dir.path().join("file.txt"); + let dst_file = dest_dir.path().join("file.txt"); + fs::write(&src_file, "foo").unwrap(); + copy(&src_file, &dst_file).await.unwrap(); + // verify destination: + assert!(!fs::symlink_metadata(dst_file) + .unwrap() + .file_type() + .is_symlink()); + + // verify copy of a symlink + let src_symlink_file = source_dir.path().join("symlink_file.txt"); + let dst_symlink_file = dest_dir.path().join("symlink_file.txt"); + tokio::fs::symlink(&src_file, &src_symlink_file) + .await + .unwrap(); + copy(src_symlink_file, &dst_symlink_file).await.unwrap(); + // verify destination: + assert!(fs::symlink_metadata(&dst_symlink_file) + .unwrap() + .file_type() + .is_symlink()); + assert_eq!(fs::read_link(&dst_symlink_file).unwrap(), src_file); + assert_eq!(fs::read_to_string(&dst_symlink_file).unwrap(), "foo") + } + + #[tokio::test] + async fn watch_directory_with_symlinks() { + // Prepare source directory: + // ./tmp/.data/file.txt + // ./tmp/1.txt -> ./tmp/.data/file.txt + let source_dir = tempfile::tempdir().unwrap(); + fs::create_dir_all(source_dir.path().join(".data")).unwrap(); + fs::write(source_dir.path().join(".data/file.txt"), "two").unwrap(); + tokio::fs::symlink( + source_dir.path().join(".data/file.txt"), + source_dir.path().join("1.txt"), + ) + .await + .unwrap(); + + let dest_dir = tempfile::tempdir().unwrap(); + + let mut entry = Storage::new(protos::Storage { + source: source_dir.path().display().to_string(), + mount_point: dest_dir.path().display().to_string(), + ..Default::default() + }) + .await + .unwrap(); + + let logger = slog::Logger::root(slog::Discard, o!()); + + assert_eq!(entry.scan(&logger).await.unwrap(), 2); + + // Should copy no files since nothing is changed since last check + assert_eq!(entry.scan(&logger).await.unwrap(), 0); + + // Should copy 1 file + thread::sleep(Duration::from_secs(1)); + fs::write(source_dir.path().join(".data/file.txt"), "updated").unwrap(); + assert_eq!(entry.scan(&logger).await.unwrap(), 2); + assert_eq!( + fs::read_to_string(dest_dir.path().join(".data/file.txt")).unwrap(), + "updated" + ); + assert_eq!( + fs::read_to_string(dest_dir.path().join("1.txt")).unwrap(), + "updated" + ); + + // Verify that resulting 1.txt is a symlink: + assert!(tokio::fs::symlink_metadata(dest_dir.path().join("1.txt")) + .await + .unwrap() + .file_type() + .is_symlink()); + + // Should copy no new files after copy happened + assert_eq!(entry.scan(&logger).await.unwrap(), 0); + } + #[tokio::test] async fn watch_directory() { // Prepare source directory: From 296e76f8eea4c070642b496e134f8c10f6872b4c Mon Sep 17 00:00:00 2001 From: Eric Ernst Date: Mon, 15 Nov 2021 14:46:09 -0800 Subject: [PATCH 05/66] watchers: handle symlinked directories, dir removal - Even a directory could be a symlink - check for this. This is very common when using configmaps/secrets - Add unit test to better mimic a configmap, configmap update - We would never remove directories before. Let's ensure that these are added to the watched_list, and verify in unit tests - Update unit tests which exercise maximum number of files per entry. There's a change in behavior now that we consider directories/symlinks watchable as well. For these tests, it means we support one less file in a watchable mount. Signed-off-by: Eric Ernst --- src/agent/src/watcher.rs | 206 ++++++++++++++++++++++++++++++--------- 1 file changed, 159 insertions(+), 47 deletions(-) diff --git a/src/agent/src/watcher.rs b/src/agent/src/watcher.rs index 187d85ccc..fd0f9fe86 100644 --- a/src/agent/src/watcher.rs +++ b/src/agent/src/watcher.rs @@ -49,7 +49,7 @@ struct Storage { /// the source becomes too large, either in number of files (>16) or total size (>1MB). watch: bool, - /// The list of files to watch from the source mount point and updated in the target one. + /// The list of files, directories, symlinks to watch from the source mount point and updated in the target one. watched_files: HashMap, } @@ -80,9 +80,13 @@ impl Drop for Storage { } async fn copy(from: impl AsRef, to: impl AsRef) -> Result<()> { - // if source is a symlink, just create new symlink with same link source if fs::symlink_metadata(&from).await?.file_type().is_symlink() { - fs::symlink(fs::read_link(&from).await?, to).await?; + // if source is a symlink, create new symlink with same link source. If + // the symlink exists, remove and create new one: + if fs::symlink_metadata(&to).await.is_ok() { + fs::remove_file(&to).await?; + } + fs::symlink(fs::read_link(&from).await?, &to).await?; } else { fs::copy(from, to).await?; } @@ -103,6 +107,16 @@ impl Storage { async fn update_target(&self, logger: &Logger, source_path: impl AsRef) -> Result<()> { let source_file_path = source_path.as_ref(); + // if we are creating a directory: just create it, nothing more to do + if source_file_path.symlink_metadata()?.file_type().is_dir() { + fs::create_dir_all(source_file_path) + .await + .with_context(|| { + format!("Unable to mkdir all for {}", source_file_path.display()) + })? + } + + // Assume we are dealing with either a file or a symlink now: let dest_file_path = if self.source_mount_point.is_file() { // Simple file to file copy // Assume target mount is a file path @@ -139,7 +153,7 @@ impl Storage { let mut remove_list = Vec::new(); let mut updated_files: Vec = Vec::new(); - // Remove deleted files for tracking list + // Remove deleted files for tracking list. self.watched_files.retain(|st, _| { if st.exists() { true @@ -151,10 +165,19 @@ impl Storage { // Delete from target for path in remove_list { - // File has been deleted, remove it from target mount let target = self.make_target_path(path)?; - debug!(logger, "Removing file from mount: {}", target.display()); - let _ = fs::remove_file(target).await; + // The target may be a directory or a file. If it is a directory that is removed, + // we'll remove all files under that directory as well. Because of this, there's a + // chance the target (a subdirectory or file under a prior removed target) was already + // removed. Make sure we check if the target exists before checking the metadata, and + // don't return an error if the remove fails + if target.exists() && target.symlink_metadata()?.file_type().is_dir() { + debug!(logger, "Removing a directory: {}", target.display()); + let _ = fs::remove_dir_all(target).await; + } else { + debug!(logger, "Removing a file: {}", target.display()); + let _ = fs::remove_file(target).await; + } } // Scan new & changed files @@ -186,15 +209,16 @@ impl Storage { let mut size: u64 = 0; debug!(logger, "Scanning path: {}", path.display()); - if path.is_file() { - let metadata = path - .metadata() - .with_context(|| format!("Failed to query metadata for: {}", path.display()))?; + let metadata = path + .symlink_metadata() + .with_context(|| format!("Failed to query metadata for: {}", path.display()))?; - let modified = metadata - .modified() - .with_context(|| format!("Failed to get modified date for: {}", path.display()))?; + let modified = metadata + .modified() + .with_context(|| format!("Failed to get modified date for: {}", path.display()))?; + // Treat files and symlinks the same: + if path.is_file() || metadata.file_type().is_symlink() { size += metadata.len(); // Insert will return old entry if any @@ -216,6 +240,16 @@ impl Storage { } ); } else { + // Handling regular directories - check to see if this directory is already being tracked, and + // track if not: + if self + .watched_files + .insert(path.to_path_buf(), modified) + .is_none() + { + update_list.push(path.to_path_buf()); + } + // Scan dir recursively let mut entries = fs::read_dir(path) .await @@ -616,7 +650,7 @@ mod tests { .unwrap(); // setup storage3: many files, but still watchable - for i in 1..MAX_ENTRIES_PER_STORAGE + 1 { + for i in 1..MAX_ENTRIES_PER_STORAGE { fs::write(src3_path.join(format!("{}.txt", i)), "original").unwrap(); } @@ -678,7 +712,7 @@ mod tests { std::fs::read_dir(entries.0[3].target_mount_point.as_path()) .unwrap() .count(), - MAX_ENTRIES_PER_STORAGE + MAX_ENTRIES_PER_STORAGE - 1 ); // Add two files to storage 0, verify it is updated without needing to run check: @@ -748,7 +782,7 @@ mod tests { std::fs::read_dir(entries.0[3].target_mount_point.as_path()) .unwrap() .count(), - MAX_ENTRIES_PER_STORAGE + 1 + MAX_ENTRIES_PER_STORAGE ); // verify that we can remove files as well, but that it isn't observed until check is run @@ -826,15 +860,20 @@ mod tests { fs::remove_file(source_dir.path().join("big.txt")).unwrap(); fs::remove_file(source_dir.path().join("too-big.txt")).unwrap(); - // Up to 16 files should be okay: - for i in 1..MAX_ENTRIES_PER_STORAGE + 1 { + assert_eq!(entry.scan(&logger).await.unwrap(), 0); + + // Up to 15 files should be okay (can watch 15 files + 1 directory) + for i in 1..MAX_ENTRIES_PER_STORAGE { fs::write(source_dir.path().join(format!("{}.txt", i)), "original").unwrap(); } - assert_eq!(entry.scan(&logger).await.unwrap(), MAX_ENTRIES_PER_STORAGE); + assert_eq!( + entry.scan(&logger).await.unwrap(), + MAX_ENTRIES_PER_STORAGE - 1 + ); - // 17 files is too many: - fs::write(source_dir.path().join("17.txt"), "updated").unwrap(); + // 16 files wll be too many: + fs::write(source_dir.path().join("16.txt"), "updated").unwrap(); thread::sleep(Duration::from_secs(1)); // Expect to receive a MountTooManyFiles error @@ -881,19 +920,67 @@ mod tests { } #[tokio::test] - async fn watch_directory_with_symlinks() { - // Prepare source directory: - // ./tmp/.data/file.txt - // ./tmp/1.txt -> ./tmp/.data/file.txt + async fn watch_directory_verify_dir_removal() { let source_dir = tempfile::tempdir().unwrap(); - fs::create_dir_all(source_dir.path().join(".data")).unwrap(); - fs::write(source_dir.path().join(".data/file.txt"), "two").unwrap(); - tokio::fs::symlink( - source_dir.path().join(".data/file.txt"), - source_dir.path().join("1.txt"), - ) + let dest_dir = tempfile::tempdir().unwrap(); + + let mut entry = Storage::new(protos::Storage { + source: source_dir.path().display().to_string(), + mount_point: dest_dir.path().display().to_string(), + ..Default::default() + }) .await .unwrap(); + let logger = slog::Logger::root(slog::Discard, o!()); + + // create a path we'll remove later + fs::create_dir_all(source_dir.path().join("tmp")).unwrap(); + fs::write(&source_dir.path().join("tmp/test-file"), "foo").unwrap(); + assert_eq!(entry.scan(&logger).await.unwrap(), 3); // root, ./tmp, test-file + + // Verify expected directory, file: + assert_eq!( + std::fs::read_dir(dest_dir.path().join("tmp")) + .unwrap() + .count(), + 1 + ); + assert_eq!(std::fs::read_dir(&dest_dir).unwrap().count(), 1); + + // Now, remove directory, and verify that the directory (and its file) are removed: + fs::remove_dir_all(source_dir.path().join("tmp")).unwrap(); + thread::sleep(Duration::from_secs(1)); + assert_eq!(entry.scan(&logger).await.unwrap(), 0); + + assert_eq!(std::fs::read_dir(&dest_dir).unwrap().count(), 0); + + assert_eq!(entry.scan(&logger).await.unwrap(), 0); + } + + #[tokio::test] + async fn watch_directory_with_symlinks() { + // Prepare source directory: + // ..2021_10_29_03_10_48.161654083/file.txt + // ..data -> ..2021_10_29_03_10_48.161654083 + // file.txt -> ..data/file.txt + + let source_dir = tempfile::tempdir().unwrap(); + let actual_dir = source_dir.path().join("..2021_10_29_03_10_48.161654083"); + let actual_file = actual_dir.join("file.txt"); + let sym_dir = source_dir.path().join("..data"); + let sym_file = source_dir.path().join("file.txt"); + + // create backing file/path + fs::create_dir_all(&actual_dir).unwrap(); + fs::write(&actual_file, "two").unwrap(); + + // create indirection symlink directory tha points to actual_dir: + tokio::fs::symlink(&actual_dir, &sym_dir).await.unwrap(); + + // create presented data file symlink: + tokio::fs::symlink(sym_dir.join("file.txt"), sym_file) + .await + .unwrap(); let dest_dir = tempfile::tempdir().unwrap(); @@ -907,26 +994,31 @@ mod tests { let logger = slog::Logger::root(slog::Discard, o!()); - assert_eq!(entry.scan(&logger).await.unwrap(), 2); + assert_eq!(entry.scan(&logger).await.unwrap(), 5); // Should copy no files since nothing is changed since last check assert_eq!(entry.scan(&logger).await.unwrap(), 0); - // Should copy 1 file + // now what, what is updated? + fs::write(actual_file, "updated").unwrap(); thread::sleep(Duration::from_secs(1)); - fs::write(source_dir.path().join(".data/file.txt"), "updated").unwrap(); - assert_eq!(entry.scan(&logger).await.unwrap(), 2); + assert_eq!(entry.scan(&logger).await.unwrap(), 1); assert_eq!( - fs::read_to_string(dest_dir.path().join(".data/file.txt")).unwrap(), - "updated" - ); - assert_eq!( - fs::read_to_string(dest_dir.path().join("1.txt")).unwrap(), + fs::read_to_string(dest_dir.path().join("file.txt")).unwrap(), "updated" ); - // Verify that resulting 1.txt is a symlink: - assert!(tokio::fs::symlink_metadata(dest_dir.path().join("1.txt")) + // Verify that resulting file.txt is a symlink: + assert!( + tokio::fs::symlink_metadata(dest_dir.path().join("file.txt")) + .await + .unwrap() + .file_type() + .is_symlink() + ); + + // Verify that .data directory is a symlink: + assert!(tokio::fs::symlink_metadata(&dest_dir.path().join("..data")) .await .unwrap() .file_type() @@ -934,6 +1026,25 @@ mod tests { // Should copy no new files after copy happened assert_eq!(entry.scan(&logger).await.unwrap(), 0); + + // Now, simulate configmap update. + // - create a new actual dir/file, + // - update the symlink directory to point to this one + // - remove old dir/file + let new_actual_dir = source_dir.path().join("..2021_10_31_03_10_48.161654083"); + let new_actual_file = new_actual_dir.join("file.txt"); + fs::create_dir_all(&new_actual_dir).unwrap(); + fs::write(&new_actual_file, "new configmap").unwrap(); + + tokio::fs::remove_file(&sym_dir).await.unwrap(); + tokio::fs::symlink(&new_actual_dir, &sym_dir).await.unwrap(); + tokio::fs::remove_dir_all(&actual_dir).await.unwrap(); + + assert_eq!(entry.scan(&logger).await.unwrap(), 3); // file, file-dir, symlink + assert_eq!( + fs::read_to_string(dest_dir.path().join("file.txt")).unwrap(), + "new configmap" + ); } #[tokio::test] @@ -958,7 +1069,7 @@ mod tests { let logger = slog::Logger::root(slog::Discard, o!()); - assert_eq!(entry.scan(&logger).await.unwrap(), 2); + assert_eq!(entry.scan(&logger).await.unwrap(), 5); // Should copy no files since nothing is changed since last check assert_eq!(entry.scan(&logger).await.unwrap(), 0); @@ -1028,8 +1139,9 @@ mod tests { let logger = slog::Logger::root(slog::Discard, o!()); - assert_eq!(entry.scan(&logger).await.unwrap(), 1); - assert_eq!(entry.watched_files.len(), 1); + // expect the root directory and the file: + assert_eq!(entry.scan(&logger).await.unwrap(), 2); + assert_eq!(entry.watched_files.len(), 2); assert!(target_file.exists()); assert!(entry.watched_files.contains_key(&source_file)); @@ -1039,7 +1151,7 @@ mod tests { assert_eq!(entry.scan(&logger).await.unwrap(), 0); - assert_eq!(entry.watched_files.len(), 0); + assert_eq!(entry.watched_files.len(), 1); assert!(!target_file.exists()); } From 94b7936f5122d90250da6a7db77dc6c4f4b666d9 Mon Sep 17 00:00:00 2001 From: David Gibson Date: Tue, 2 Nov 2021 16:32:04 +1100 Subject: [PATCH 06/66] agent/device: Use nix::sys::stat::{major,minor} instead of libc::* update_spec_devices() includes an unsafe block, in order to call the libc functions to get the major and minor numbers from a device ID. However, the nix crate already has a safe wrapper for this function, which we use in other places in the file. Signed-off-by: David Gibson --- src/agent/src/device.rs | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/src/agent/src/device.rs b/src/agent/src/device.rs index df0221c74..e01b7d3df 100644 --- a/src/agent/src/device.rs +++ b/src/agent/src/device.rs @@ -3,7 +3,6 @@ // SPDX-License-Identifier: Apache-2.0 // -use libc::{c_uint, major, minor}; use nix::sys::stat; use regex::Regex; use std::collections::HashMap; @@ -450,9 +449,6 @@ fn update_spec_device( vm_path: &str, final_path: &str, ) -> Result<()> { - let major_id: c_uint; - let minor_id: c_uint; - // If no container_path is provided, we won't be able to match and // update the device in the OCI spec device list. This is an error. if host_path.is_empty() { @@ -470,10 +466,8 @@ fn update_spec_device( let meta = fs::metadata(vm_path)?; let dev_id = meta.rdev(); - unsafe { - major_id = major(dev_id); - minor_id = minor(dev_id); - } + let major_id = stat::major(dev_id); + let minor_id = stat::minor(dev_id); info!( sl!(), From 0c51da3dd0a04a25b8d923f72d99e2b1fff3bfb7 Mon Sep 17 00:00:00 2001 From: David Gibson Date: Thu, 4 Nov 2021 16:47:45 +1100 Subject: [PATCH 07/66] agent/device: Correct misleading error message in update_spec_device() This error is returned if we have information for a device from the runtime, but a matching device does not appear in the OCI spec. However, the name for the device we print is the name from the VM, rather than the name from the container which is what we actually expect in the spec. Signed-off-by: David Gibson --- src/agent/src/device.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/agent/src/device.rs b/src/agent/src/device.rs index e01b7d3df..fe79b4441 100644 --- a/src/agent/src/device.rs +++ b/src/agent/src/device.rs @@ -511,8 +511,8 @@ fn update_spec_device( Ok(()) } else { Err(anyhow!( - "Should have found a matching device {} in the spec", - vm_path + "Should have found a device {} in the spec", + host_path )) } } From 57541315db19be2c43a3d3adb751344204ee8628 Mon Sep 17 00:00:00 2001 From: David Gibson Date: Fri, 29 Oct 2021 14:12:47 +1100 Subject: [PATCH 08/66] agent/device: Correct misleading parameter name in update_spec_device() update_spec_device() takes a 'host_path' parameter which it uses to locate the device to correct in the OCI spec. Although this will usually be the path of the device on the host, it doesn't have to be - a traditional runtime like runc would create a device node of that name in the container with the given (host) major and minor numbers. To clarify that, rename it to 'container_path'. We also update the block comment to explain the distinctions more carefully. Finally we update some variable names in tests to match. Signed-off-by: David Gibson --- src/agent/src/device.rs | 29 ++++++++++++++++------------- 1 file changed, 16 insertions(+), 13 deletions(-) diff --git a/src/agent/src/device.rs b/src/agent/src/device.rs index fe79b4441..1aef6e00b 100644 --- a/src/agent/src/device.rs +++ b/src/agent/src/device.rs @@ -437,22 +437,25 @@ fn scan_scsi_bus(scsi_addr: &str) -> Result<()> { // update_spec_device updates the device list in the OCI spec to make // it include details appropriate for the VM, instead of the host. It -// is given the host path to the device (to locate the device in the -// original OCI spec) and the VM path which it uses to determine the -// VM major/minor numbers, and the final path with which to present -// the device in the (inner) container +// is given: +// container_path: the path to the device in the original OCI spec +// vm_path: the path to the equivalent device in the VM (used to +// determine the correct major/minor numbers) +// final_path: a new path to update the device to in the "inner" +// container specification (usually the same as +// container_path) #[instrument] fn update_spec_device( spec: &mut Spec, devidx: &DevIndex, - host_path: &str, + container_path: &str, vm_path: &str, final_path: &str, ) -> Result<()> { // If no container_path is provided, we won't be able to match and // update the device in the OCI spec device list. This is an error. - if host_path.is_empty() { - return Err(anyhow!("Host path cannot empty for device")); + if container_path.is_empty() { + return Err(anyhow!("Container path cannot be empty for device")); } let linux = spec @@ -474,7 +477,7 @@ fn update_spec_device( "update_spec_device(): vm_path={}, major: {}, minor: {}\n", vm_path, major_id, minor_id ); - if let Some(idxdata) = devidx.0.get(host_path) { + if let Some(idxdata) = devidx.0.get(container_path) { let dev = &mut linux.devices[idxdata.idx]; let host_major = dev.major; let host_minor = dev.minor; @@ -486,7 +489,7 @@ fn update_spec_device( info!( sl!(), "change the device from path: {} major: {} minor: {} to vm device path: {} major: {} minor: {}", - host_path, + container_path, host_major, host_minor, dev.path, @@ -512,7 +515,7 @@ fn update_spec_device( } else { Err(anyhow!( "Should have found a device {} in the spec", - host_path + container_path )) } } @@ -1107,14 +1110,14 @@ mod tests { let guest_major = stat::major(null_rdev) as i64; let guest_minor = stat::minor(null_rdev) as i64; - let host_path = "/dev/host"; + let container_path = "/dev/original"; let host_major: i64 = 99; let host_minor: i64 = 99; let mut spec = Spec { linux: Some(Linux { devices: vec![oci::LinuxDevice { - path: host_path.to_string(), + path: container_path.to_string(), r#type: "c".to_string(), major: host_major, minor: host_minor, @@ -1129,7 +1132,7 @@ mod tests { let vm_path = "/dev/null"; let final_path = "/dev/final"; - let res = update_spec_device(&mut spec, &devidx, host_path, vm_path, final_path); + let res = update_spec_device(&mut spec, &devidx, container_path, vm_path, final_path); assert!(res.is_ok()); let specdevices = &spec.linux.as_ref().unwrap().devices; From 2029eeebca1b24ba5b2501e6055d912f838de1e5 Mon Sep 17 00:00:00 2001 From: David Gibson Date: Tue, 2 Nov 2021 11:07:51 +1100 Subject: [PATCH 09/66] agent/device: Improve update_spec_device() final_path handling update_spec_device() takes a 'final_path' parameter which gives the name the device should be given in the "inner" OCI spec. We need this for VFIO devices where the name the payload sees needs to match the VM's IOMMU groups. However, in all other cases (for now, and maybe forever), this is the same as the original 'container_path' given in the input OCI spec. To make this clearer and simplify callers, make this parameter an Option, and only update the device name if it is non-None. Additionally, update_spec_device() needs to call to_string() on update_path to get an owned version. Rust convention[0] is to let the caller decide whether it should copy, or just give an existing owned version to the function. Change from &str to String to allow that; it doesn't buy us anything right now, but will make some things a little nicer in future. [0] https://rust-lang.github.io/api-guidelines/flexibility.html?highlight=clone#caller-decides-where-to-copy-and-place-data-c-caller-control Signed-off-by: David Gibson --- src/agent/src/device.rs | 88 ++++++++++++++++------------------------- 1 file changed, 33 insertions(+), 55 deletions(-) diff --git a/src/agent/src/device.rs b/src/agent/src/device.rs index 1aef6e00b..238d55640 100644 --- a/src/agent/src/device.rs +++ b/src/agent/src/device.rs @@ -450,7 +450,7 @@ fn update_spec_device( devidx: &DevIndex, container_path: &str, vm_path: &str, - final_path: &str, + final_path: Option, ) -> Result<()> { // If no container_path is provided, we won't be able to match and // update the device in the OCI spec device list. This is an error. @@ -484,7 +484,9 @@ fn update_spec_device( dev.major = major_id as i64; dev.minor = minor_id as i64; - dev.path = final_path.to_string(); + if let Some(final_path) = final_path { + dev.path = final_path; + } info!( sl!(), @@ -533,13 +535,7 @@ async fn virtiommio_blk_device_handler( return Err(anyhow!("Invalid path for virtio mmio blk device")); } - update_spec_device( - spec, - devidx, - &device.container_path, - &device.vm_path, - &device.container_path, - ) + update_spec_device(spec, devidx, &device.container_path, &device.vm_path, None) } // device.Id should be a PCI path string @@ -555,13 +551,7 @@ async fn virtio_blk_device_handler( dev.vm_path = get_virtio_blk_pci_device_name(sandbox, &pcipath).await?; - update_spec_device( - spec, - devidx, - &dev.container_path, - &dev.vm_path, - &dev.container_path, - ) + update_spec_device(spec, devidx, &dev.container_path, &dev.vm_path, None) } // device.id should be a CCW path string @@ -606,13 +596,7 @@ async fn virtio_scsi_device_handler( ) -> Result<()> { let mut dev = device.clone(); dev.vm_path = get_scsi_device_name(sandbox, &device.id).await?; - update_spec_device( - spec, - devidx, - &dev.container_path, - &dev.vm_path, - &dev.container_path, - ) + update_spec_device(spec, devidx, &dev.container_path, &dev.vm_path, None) } #[instrument] @@ -626,13 +610,7 @@ async fn virtio_nvdimm_device_handler( return Err(anyhow!("Invalid path for nvdimm device")); } - update_spec_device( - spec, - devidx, - &device.container_path, - &device.vm_path, - &device.container_path, - ) + update_spec_device(spec, devidx, &device.container_path, &device.vm_path, None) } fn split_vfio_option(opt: &str) -> Option<(&str, &str)> { @@ -695,7 +673,13 @@ async fn vfio_device_handler( let group = group.unwrap(); let vmpath = get_vfio_device_name(sandbox, group).await?; - update_spec_device(spec, devidx, &device.container_path, &vmpath, &vmpath)?; + update_spec_device( + spec, + devidx, + &device.container_path, + &vmpath, + Some(vmpath.to_string()), + )?; } Ok(()) @@ -845,20 +829,20 @@ mod tests { let container_path = ""; let vm_path = ""; let devidx = DevIndex::new(&spec); - let res = update_spec_device(&mut spec, &devidx, container_path, vm_path, container_path); + let res = update_spec_device(&mut spec, &devidx, container_path, vm_path, None); assert!(res.is_err()); // linux is empty let container_path = "/dev/null"; let devidx = DevIndex::new(&spec); - let res = update_spec_device(&mut spec, &devidx, container_path, vm_path, container_path); + let res = update_spec_device(&mut spec, &devidx, container_path, vm_path, None); assert!(res.is_err()); spec.linux = Some(Linux::default()); // linux.devices is empty let devidx = DevIndex::new(&spec); - let res = update_spec_device(&mut spec, &devidx, container_path, vm_path, container_path); + let res = update_spec_device(&mut spec, &devidx, container_path, vm_path, None); assert!(res.is_err()); spec.linux.as_mut().unwrap().devices = vec![oci::LinuxDevice { @@ -870,14 +854,14 @@ mod tests { // vm_path empty let devidx = DevIndex::new(&spec); - let res = update_spec_device(&mut spec, &devidx, container_path, vm_path, container_path); + let res = update_spec_device(&mut spec, &devidx, container_path, vm_path, None); assert!(res.is_err()); let vm_path = "/dev/null"; // guest and host path are not the same let devidx = DevIndex::new(&spec); - let res = update_spec_device(&mut spec, &devidx, container_path, vm_path, container_path); + let res = update_spec_device(&mut spec, &devidx, container_path, vm_path, None); assert!( res.is_err(), "container_path={:?} vm_path={:?} spec={:?}", @@ -890,7 +874,7 @@ mod tests { // spec.linux.resources is empty let devidx = DevIndex::new(&spec); - let res = update_spec_device(&mut spec, &devidx, container_path, vm_path, container_path); + let res = update_spec_device(&mut spec, &devidx, container_path, vm_path, None); assert!(res.is_ok()); // update both devices and cgroup lists @@ -911,7 +895,7 @@ mod tests { }); let devidx = DevIndex::new(&spec); - let res = update_spec_device(&mut spec, &devidx, container_path, vm_path, container_path); + let res = update_spec_device(&mut spec, &devidx, container_path, vm_path, None); assert!(res.is_ok()); } @@ -991,13 +975,7 @@ mod tests { assert_eq!(Some(host_major_b), specresources.devices[1].major); assert_eq!(Some(host_minor_b), specresources.devices[1].minor); - let res = update_spec_device( - &mut spec, - &devidx, - container_path_a, - vm_path_a, - container_path_a, - ); + let res = update_spec_device(&mut spec, &devidx, container_path_a, vm_path_a, None); assert!(res.is_ok()); let specdevices = &spec.linux.as_ref().unwrap().devices; @@ -1012,13 +990,7 @@ mod tests { assert_eq!(Some(host_major_b), specresources.devices[1].major); assert_eq!(Some(host_minor_b), specresources.devices[1].minor); - let res = update_spec_device( - &mut spec, - &devidx, - container_path_b, - vm_path_b, - container_path_b, - ); + let res = update_spec_device(&mut spec, &devidx, container_path_b, vm_path_b, None); assert!(res.is_ok()); let specdevices = &spec.linux.as_ref().unwrap().devices; @@ -1093,7 +1065,7 @@ mod tests { assert_eq!(Some(host_major), specresources.devices[1].major); assert_eq!(Some(host_minor), specresources.devices[1].minor); - let res = update_spec_device(&mut spec, &devidx, container_path, vm_path, container_path); + let res = update_spec_device(&mut spec, &devidx, container_path, vm_path, None); assert!(res.is_ok()); // Only the char device, not the block device should be updated @@ -1130,9 +1102,15 @@ mod tests { let devidx = DevIndex::new(&spec); let vm_path = "/dev/null"; - let final_path = "/dev/final"; + let final_path = "/dev/new"; - let res = update_spec_device(&mut spec, &devidx, container_path, vm_path, final_path); + let res = update_spec_device( + &mut spec, + &devidx, + container_path, + vm_path, + Some(final_path.to_string()), + ); assert!(res.is_ok()); let specdevices = &spec.linux.as_ref().unwrap().devices; From e7beed543066619b3ae2eebe4c4b16f6a6622479 Mon Sep 17 00:00:00 2001 From: David Gibson Date: Thu, 4 Nov 2021 15:18:04 +1100 Subject: [PATCH 10/66] agent/device: Remove unneeded clone() from several device handlers virtio_blk_device_handler(), virtio_blk_ccw_device_handler() and virtio_scsi_device_handler() all take a clone of their 'device' parameter. They appear to do this in order to get a mutable copy in which they can update the vm_path field. However, the copy is dropped at the end of the function, so the only thing that's used in it is the vm_path field passed to update_spec_device() afterwards. We can avoid the clone by just using a local variable for the vm_path. Signed-off-by: David Gibson --- src/agent/src/device.rs | 20 ++++++++------------ 1 file changed, 8 insertions(+), 12 deletions(-) diff --git a/src/agent/src/device.rs b/src/agent/src/device.rs index 238d55640..afae70dd9 100644 --- a/src/agent/src/device.rs +++ b/src/agent/src/device.rs @@ -546,12 +546,10 @@ async fn virtio_blk_device_handler( sandbox: &Arc>, devidx: &DevIndex, ) -> Result<()> { - let mut dev = device.clone(); let pcipath = pci::Path::from_str(&device.id)?; + let vm_path = get_virtio_blk_pci_device_name(sandbox, &pcipath).await?; - dev.vm_path = get_virtio_blk_pci_device_name(sandbox, &pcipath).await?; - - update_spec_device(spec, devidx, &dev.container_path, &dev.vm_path, None) + update_spec_device(spec, devidx, &device.container_path, &vm_path, None) } // device.id should be a CCW path string @@ -563,15 +561,14 @@ async fn virtio_blk_ccw_device_handler( sandbox: &Arc>, devidx: &DevIndex, ) -> Result<()> { - let mut dev = device.clone(); let ccw_device = ccw::Device::from_str(&device.id)?; - dev.vm_path = get_virtio_blk_ccw_device_name(sandbox, &ccw_device).await?; + let vm_path = get_virtio_blk_ccw_device_name(sandbox, &ccw_device).await?; update_spec_device( spec, devidx, - &dev.container_path, - &dev.vm_path, - &dev.container_path, + &device.container_path, + &vm_path, + &device.container_path, ) } @@ -594,9 +591,8 @@ async fn virtio_scsi_device_handler( sandbox: &Arc>, devidx: &DevIndex, ) -> Result<()> { - let mut dev = device.clone(); - dev.vm_path = get_scsi_device_name(sandbox, &device.id).await?; - update_spec_device(spec, devidx, &dev.container_path, &dev.vm_path, None) + let vm_path = get_scsi_device_name(sandbox, &device.id).await?; + update_spec_device(spec, devidx, &device.container_path, &vm_path, None) } #[instrument] From 46a4020e9e5831a588e61a4cc8dadb6efa233e7a Mon Sep 17 00:00:00 2001 From: David Gibson Date: Thu, 4 Nov 2021 15:37:41 +1100 Subject: [PATCH 11/66] agent/device: Types to represent update for a device in the OCI spec Currently update_spec_device() takes parameters 'vm_path' and 'final_path' to give it the information it needs to update a single device in the OCI spec for the guest. This bundles these parameters into a single structure type describing the updates to a single device. This doesn't accomplish much immediately, but will allow a number of further cleanups. At the same time we change the representation of vm_path from a Unicode string to a std::path::Path, which is a bit more natural since we are performing file operations on it. Signed-off-by: David Gibson --- src/agent/src/device.rs | 187 ++++++++++++++++++++++++++++++++-------- 1 file changed, 153 insertions(+), 34 deletions(-) diff --git a/src/agent/src/device.rs b/src/agent/src/device.rs index afae70dd9..d68f53327 100644 --- a/src/agent/src/device.rs +++ b/src/agent/src/device.rs @@ -11,7 +11,7 @@ use std::fmt; use std::fs; use std::os::unix::ffi::OsStrExt; use std::os::unix::fs::MetadataExt; -use std::path::Path; +use std::path::{Path, PathBuf}; use std::str::FromStr; use std::sync::Arc; use tokio::sync::Mutex; @@ -435,22 +435,74 @@ fn scan_scsi_bus(scsi_addr: &str) -> Result<()> { Ok(()) } +#[derive(Debug, Clone)] +struct DevNumUpdate { + // the path to the equivalent device in the VM (used to determine + // the correct major/minor numbers) + vm_path: PathBuf, +} + +impl DevNumUpdate { + fn from_vm_path>(vm_path: T) -> Result { + Ok(DevNumUpdate { + vm_path: vm_path.as_ref().to_owned(), + }) + } +} + +// Represents the device-node and resource related updates to the OCI +// spec needed for a particular device +#[derive(Debug, Clone)] +struct DevUpdate { + num: DevNumUpdate, + // an optional new path to update the device to in the "inner" container + // specification + final_path: Option, +} + +impl DevUpdate { + fn from_vm_path>(vm_path: T, final_path: String) -> Result { + Ok(DevUpdate { + final_path: Some(final_path), + ..DevNumUpdate::from_vm_path(vm_path)?.into() + }) + } +} + +impl From for DevUpdate { + fn from(num: DevNumUpdate) -> Self { + DevUpdate { + num, + final_path: None, + } + } +} + +// Represents the updates to the OCI spec needed for a particular device +#[derive(Debug, Clone, Default)] +struct SpecUpdate { + dev: Option, +} + +impl> From for SpecUpdate { + fn from(dev: T) -> Self { + SpecUpdate { + dev: Some(dev.into()), + } + } +} + // update_spec_device updates the device list in the OCI spec to make // it include details appropriate for the VM, instead of the host. It // is given: // container_path: the path to the device in the original OCI spec -// vm_path: the path to the equivalent device in the VM (used to -// determine the correct major/minor numbers) -// final_path: a new path to update the device to in the "inner" -// container specification (usually the same as -// container_path) +// update: information on changes to make to the device #[instrument] fn update_spec_device( spec: &mut Spec, devidx: &DevIndex, container_path: &str, - vm_path: &str, - final_path: Option, + update: DevUpdate, ) -> Result<()> { // If no container_path is provided, we won't be able to match and // update the device in the OCI spec device list. This is an error. @@ -463,18 +515,24 @@ fn update_spec_device( .as_mut() .ok_or_else(|| anyhow!("Spec didn't container linux field"))?; - if !Path::new(vm_path).exists() { - return Err(anyhow!("vm_path:{} doesn't exist", vm_path)); + if !update.num.vm_path.exists() { + return Err(anyhow!( + "vm_path:{} doesn't exist", + update.num.vm_path.display() + )); } - let meta = fs::metadata(vm_path)?; + let meta = fs::metadata(&update.num.vm_path)?; let dev_id = meta.rdev(); let major_id = stat::major(dev_id); let minor_id = stat::minor(dev_id); info!( sl!(), - "update_spec_device(): vm_path={}, major: {}, minor: {}\n", vm_path, major_id, minor_id + "update_spec_device(): vm_path={}, major: {}, minor: {}\n", + update.num.vm_path.display(), + major_id, + minor_id ); if let Some(idxdata) = devidx.0.get(container_path) { @@ -484,7 +542,7 @@ fn update_spec_device( dev.major = major_id as i64; dev.minor = minor_id as i64; - if let Some(final_path) = final_path { + if let Some(final_path) = update.final_path { dev.path = final_path; } @@ -535,7 +593,12 @@ async fn virtiommio_blk_device_handler( return Err(anyhow!("Invalid path for virtio mmio blk device")); } - update_spec_device(spec, devidx, &device.container_path, &device.vm_path, None) + update_spec_device( + spec, + devidx, + &device.container_path, + DevNumUpdate::from_vm_path(&device.vm_path)?.into(), + ) } // device.Id should be a PCI path string @@ -549,7 +612,12 @@ async fn virtio_blk_device_handler( let pcipath = pci::Path::from_str(&device.id)?; let vm_path = get_virtio_blk_pci_device_name(sandbox, &pcipath).await?; - update_spec_device(spec, devidx, &device.container_path, &vm_path, None) + update_spec_device( + spec, + devidx, + &device.container_path, + DevNumUpdate::from_vm_path(vm_path)?.into(), + ) } // device.id should be a CCW path string @@ -563,12 +631,12 @@ async fn virtio_blk_ccw_device_handler( ) -> Result<()> { let ccw_device = ccw::Device::from_str(&device.id)?; let vm_path = get_virtio_blk_ccw_device_name(sandbox, &ccw_device).await?; + update_spec_device( spec, devidx, &device.container_path, - &vm_path, - &device.container_path, + DevNumUpdate::from_vm_path(vm_path)?.into(), ) } @@ -592,7 +660,13 @@ async fn virtio_scsi_device_handler( devidx: &DevIndex, ) -> Result<()> { let vm_path = get_scsi_device_name(sandbox, &device.id).await?; - update_spec_device(spec, devidx, &device.container_path, &vm_path, None) + + update_spec_device( + spec, + devidx, + &device.container_path, + DevNumUpdate::from_vm_path(vm_path)?.into(), + ) } #[instrument] @@ -606,7 +680,12 @@ async fn virtio_nvdimm_device_handler( return Err(anyhow!("Invalid path for nvdimm device")); } - update_spec_device(spec, devidx, &device.container_path, &device.vm_path, None) + update_spec_device( + spec, + devidx, + &device.container_path, + DevNumUpdate::from_vm_path(&device.vm_path)?.into(), + ) } fn split_vfio_option(opt: &str) -> Option<(&str, &str)> { @@ -667,14 +746,13 @@ async fn vfio_device_handler( if vfio_in_guest { // If there are any devices at all, logic above ensures that group is not None let group = group.unwrap(); - let vmpath = get_vfio_device_name(sandbox, group).await?; + let vm_path = get_vfio_device_name(sandbox, group).await?; update_spec_device( spec, devidx, &device.container_path, - &vmpath, - Some(vmpath.to_string()), + DevUpdate::from_vm_path(&vm_path, vm_path.clone())?, )?; } @@ -825,20 +903,35 @@ mod tests { let container_path = ""; let vm_path = ""; let devidx = DevIndex::new(&spec); - let res = update_spec_device(&mut spec, &devidx, container_path, vm_path, None); + let res = update_spec_device( + &mut spec, + &devidx, + container_path, + DevNumUpdate::from_vm_path(vm_path).unwrap().into(), + ); assert!(res.is_err()); // linux is empty let container_path = "/dev/null"; let devidx = DevIndex::new(&spec); - let res = update_spec_device(&mut spec, &devidx, container_path, vm_path, None); + let res = update_spec_device( + &mut spec, + &devidx, + container_path, + DevNumUpdate::from_vm_path(vm_path).unwrap().into(), + ); assert!(res.is_err()); spec.linux = Some(Linux::default()); // linux.devices is empty let devidx = DevIndex::new(&spec); - let res = update_spec_device(&mut spec, &devidx, container_path, vm_path, None); + let res = update_spec_device( + &mut spec, + &devidx, + container_path, + DevNumUpdate::from_vm_path(vm_path).unwrap().into(), + ); assert!(res.is_err()); spec.linux.as_mut().unwrap().devices = vec![oci::LinuxDevice { @@ -850,14 +943,24 @@ mod tests { // vm_path empty let devidx = DevIndex::new(&spec); - let res = update_spec_device(&mut spec, &devidx, container_path, vm_path, None); + let res = update_spec_device( + &mut spec, + &devidx, + container_path, + DevNumUpdate::from_vm_path(vm_path).unwrap().into(), + ); assert!(res.is_err()); let vm_path = "/dev/null"; // guest and host path are not the same let devidx = DevIndex::new(&spec); - let res = update_spec_device(&mut spec, &devidx, container_path, vm_path, None); + let res = update_spec_device( + &mut spec, + &devidx, + container_path, + DevNumUpdate::from_vm_path(vm_path).unwrap().into(), + ); assert!( res.is_err(), "container_path={:?} vm_path={:?} spec={:?}", @@ -870,7 +973,12 @@ mod tests { // spec.linux.resources is empty let devidx = DevIndex::new(&spec); - let res = update_spec_device(&mut spec, &devidx, container_path, vm_path, None); + let res = update_spec_device( + &mut spec, + &devidx, + container_path, + DevNumUpdate::from_vm_path(vm_path).unwrap().into(), + ); assert!(res.is_ok()); // update both devices and cgroup lists @@ -891,7 +999,12 @@ mod tests { }); let devidx = DevIndex::new(&spec); - let res = update_spec_device(&mut spec, &devidx, container_path, vm_path, None); + let res = update_spec_device( + &mut spec, + &devidx, + container_path, + DevNumUpdate::from_vm_path(vm_path).unwrap().into(), + ); assert!(res.is_ok()); } @@ -971,7 +1084,8 @@ mod tests { assert_eq!(Some(host_major_b), specresources.devices[1].major); assert_eq!(Some(host_minor_b), specresources.devices[1].minor); - let res = update_spec_device(&mut spec, &devidx, container_path_a, vm_path_a, None); + let update_a = DevNumUpdate::from_vm_path(vm_path_a).unwrap().into(); + let res = update_spec_device(&mut spec, &devidx, container_path_a, update_a); assert!(res.is_ok()); let specdevices = &spec.linux.as_ref().unwrap().devices; @@ -986,7 +1100,8 @@ mod tests { assert_eq!(Some(host_major_b), specresources.devices[1].major); assert_eq!(Some(host_minor_b), specresources.devices[1].minor); - let res = update_spec_device(&mut spec, &devidx, container_path_b, vm_path_b, None); + let update_b = DevNumUpdate::from_vm_path(vm_path_b).unwrap().into(); + let res = update_spec_device(&mut spec, &devidx, container_path_b, update_b); assert!(res.is_ok()); let specdevices = &spec.linux.as_ref().unwrap().devices; @@ -1061,7 +1176,12 @@ mod tests { assert_eq!(Some(host_major), specresources.devices[1].major); assert_eq!(Some(host_minor), specresources.devices[1].minor); - let res = update_spec_device(&mut spec, &devidx, container_path, vm_path, None); + let res = update_spec_device( + &mut spec, + &devidx, + container_path, + DevNumUpdate::from_vm_path(vm_path).unwrap().into(), + ); assert!(res.is_ok()); // Only the char device, not the block device should be updated @@ -1104,8 +1224,7 @@ mod tests { &mut spec, &devidx, container_path, - vm_path, - Some(final_path.to_string()), + DevUpdate::from_vm_path(vm_path, final_path.to_string()).unwrap(), ); assert!(res.is_ok()); From f10e8c816562d76c79af3d573b8148ec3a4a7ae1 Mon Sep 17 00:00:00 2001 From: David Gibson Date: Fri, 29 Oct 2021 14:36:07 +1100 Subject: [PATCH 12/66] agent/device: Batch changes to the OCI specification As we process container devices in the agent, we repeatedly call update_spec_device() to adjust the OCI spec as necessary for differences between the host and the VM. This means that for the whole of a pretty complex call graph, the spec is in a partially-updated state - neither fully as it was on the host, not fully as it will be for the container within the VM. Worse, it's not discernable from the contents itself which parts of the spec have already been updated and which have not. We used to have real bugs because of this, until the DevIndex structure was introduced, but that means a whole, fairly complex, parallel data structure needs to be passed around this call graph just to keep track of the state we're in. Start simplifying this by having the device handler functions not directly update the spec, but instead return an update structure describing the change they need. Once all the devices are added, add_devices() will process all the updates as a batch. Note that collecting the updates in a HashMap, rather than a simple Vec doesn't make a lot of sense in the current code, but will reduce churn in future changes which make use of it. Signed-off-by: David Gibson --- src/agent/src/device.rs | 117 ++++++++++++---------------------------- 1 file changed, 34 insertions(+), 83 deletions(-) diff --git a/src/agent/src/device.rs b/src/agent/src/device.rs index d68f53327..66ee9d9fa 100644 --- a/src/agent/src/device.rs +++ b/src/agent/src/device.rs @@ -585,39 +585,25 @@ fn update_spec_device( #[instrument] async fn virtiommio_blk_device_handler( device: &Device, - spec: &mut Spec, _sandbox: &Arc>, - devidx: &DevIndex, -) -> Result<()> { +) -> Result { if device.vm_path.is_empty() { return Err(anyhow!("Invalid path for virtio mmio blk device")); } - update_spec_device( - spec, - devidx, - &device.container_path, - DevNumUpdate::from_vm_path(&device.vm_path)?.into(), - ) + Ok(DevNumUpdate::from_vm_path(&device.vm_path)?.into()) } // device.Id should be a PCI path string #[instrument] async fn virtio_blk_device_handler( device: &Device, - spec: &mut Spec, sandbox: &Arc>, - devidx: &DevIndex, -) -> Result<()> { +) -> Result { let pcipath = pci::Path::from_str(&device.id)?; let vm_path = get_virtio_blk_pci_device_name(sandbox, &pcipath).await?; - update_spec_device( - spec, - devidx, - &device.container_path, - DevNumUpdate::from_vm_path(vm_path)?.into(), - ) + Ok(DevNumUpdate::from_vm_path(vm_path)?.into()) } // device.id should be a CCW path string @@ -625,29 +611,17 @@ async fn virtio_blk_device_handler( #[instrument] async fn virtio_blk_ccw_device_handler( device: &Device, - spec: &mut Spec, sandbox: &Arc>, - devidx: &DevIndex, -) -> Result<()> { +) -> Result { let ccw_device = ccw::Device::from_str(&device.id)?; let vm_path = get_virtio_blk_ccw_device_name(sandbox, &ccw_device).await?; - update_spec_device( - spec, - devidx, - &device.container_path, - DevNumUpdate::from_vm_path(vm_path)?.into(), - ) + Ok(DevNumUpdate::from_vm_path(vm_path)?.into()) } #[cfg(not(target_arch = "s390x"))] #[instrument] -async fn virtio_blk_ccw_device_handler( - _: &Device, - _: &mut Spec, - _: &Arc>, - _: &DevIndex, -) -> Result<()> { +async fn virtio_blk_ccw_device_handler(_: &Device, _: &Arc>) -> Result { Err(anyhow!("CCW is only supported on s390x")) } @@ -655,37 +629,23 @@ async fn virtio_blk_ccw_device_handler( #[instrument] async fn virtio_scsi_device_handler( device: &Device, - spec: &mut Spec, sandbox: &Arc>, - devidx: &DevIndex, -) -> Result<()> { +) -> Result { let vm_path = get_scsi_device_name(sandbox, &device.id).await?; - update_spec_device( - spec, - devidx, - &device.container_path, - DevNumUpdate::from_vm_path(vm_path)?.into(), - ) + Ok(DevNumUpdate::from_vm_path(vm_path)?.into()) } #[instrument] async fn virtio_nvdimm_device_handler( device: &Device, - spec: &mut Spec, _sandbox: &Arc>, - devidx: &DevIndex, -) -> Result<()> { +) -> Result { if device.vm_path.is_empty() { return Err(anyhow!("Invalid path for nvdimm device")); } - update_spec_device( - spec, - devidx, - &device.container_path, - DevNumUpdate::from_vm_path(&device.vm_path)?.into(), - ) + Ok(DevNumUpdate::from_vm_path(&device.vm_path)?.into()) } fn split_vfio_option(opt: &str) -> Option<(&str, &str)> { @@ -703,12 +663,7 @@ fn split_vfio_option(opt: &str) -> Option<(&str, &str)> { // Each option should have the form "DDDD:BB:DD.F=" // DDDD:BB:DD.F is the device's PCI address in the host // is a PCI path to the device in the guest (see pci.rs) -async fn vfio_device_handler( - device: &Device, - spec: &mut Spec, - sandbox: &Arc>, - devidx: &DevIndex, -) -> Result<()> { +async fn vfio_device_handler(device: &Device, sandbox: &Arc>) -> Result { let vfio_in_guest = device.field_type != DRIVER_VFIO_GK_TYPE; let mut group = None; @@ -743,20 +698,15 @@ async fn vfio_device_handler( } } - if vfio_in_guest { + Ok(if vfio_in_guest { // If there are any devices at all, logic above ensures that group is not None let group = group.unwrap(); let vm_path = get_vfio_device_name(sandbox, group).await?; - update_spec_device( - spec, - devidx, - &device.container_path, - DevUpdate::from_vm_path(&vm_path, vm_path.clone())?, - )?; - } - - Ok(()) + DevUpdate::from_vm_path(&vm_path, vm_path.clone())?.into() + } else { + SpecUpdate::default() + }) } impl DevIndex { @@ -790,22 +740,25 @@ pub async fn add_devices( spec: &mut Spec, sandbox: &Arc>, ) -> Result<()> { - let devidx = DevIndex::new(spec); + let mut dev_updates = HashMap::<&str, DevUpdate>::with_capacity(devices.len()); for device in devices.iter() { - add_device(device, spec, sandbox, &devidx).await?; + let update = add_device(device, sandbox).await?; + if let Some(dev_update) = update.dev { + dev_updates.insert(&device.container_path, dev_update); + } + } + + let devidx = DevIndex::new(spec); + for (container_path, update) in dev_updates.drain() { + update_spec_device(spec, &devidx, container_path, update)?; } Ok(()) } #[instrument] -async fn add_device( - device: &Device, - spec: &mut Spec, - sandbox: &Arc>, - devidx: &DevIndex, -) -> Result<()> { +async fn add_device(device: &Device, sandbox: &Arc>) -> Result { // log before validation to help with debugging gRPC protocol version differences. info!(sl!(), "device-id: {}, device-type: {}, device-vm-path: {}, device-container-path: {}, device-options: {:?}", device.id, device.field_type, device.vm_path, device.container_path, device.options); @@ -823,14 +776,12 @@ async fn add_device( } match device.field_type.as_str() { - DRIVER_BLK_TYPE => virtio_blk_device_handler(device, spec, sandbox, devidx).await, - DRIVER_BLK_CCW_TYPE => virtio_blk_ccw_device_handler(device, spec, sandbox, devidx).await, - DRIVER_MMIO_BLK_TYPE => virtiommio_blk_device_handler(device, spec, sandbox, devidx).await, - DRIVER_NVDIMM_TYPE => virtio_nvdimm_device_handler(device, spec, sandbox, devidx).await, - DRIVER_SCSI_TYPE => virtio_scsi_device_handler(device, spec, sandbox, devidx).await, - DRIVER_VFIO_GK_TYPE | DRIVER_VFIO_TYPE => { - vfio_device_handler(device, spec, sandbox, devidx).await - } + DRIVER_BLK_TYPE => virtio_blk_device_handler(device, sandbox).await, + DRIVER_BLK_CCW_TYPE => virtio_blk_ccw_device_handler(device, sandbox).await, + DRIVER_MMIO_BLK_TYPE => virtiommio_blk_device_handler(device, sandbox).await, + DRIVER_NVDIMM_TYPE => virtio_nvdimm_device_handler(device, sandbox).await, + DRIVER_SCSI_TYPE => virtio_scsi_device_handler(device, sandbox).await, + DRIVER_VFIO_GK_TYPE | DRIVER_VFIO_TYPE => vfio_device_handler(device, sandbox).await, _ => Err(anyhow!("Unknown device type {}", device.field_type)), } } From f4982130e16ae5add3d5c119d5e522f3d0aed001 Mon Sep 17 00:00:00 2001 From: David Gibson Date: Fri, 5 Nov 2021 16:04:20 +1100 Subject: [PATCH 13/66] agent/device: Check for conflicting device updates For each device in the OCI spec we need to update it to reflect the guest rather than the host. We do this with additional device information provided by the runtime. There should only be one update for each device though, if there are multiple, something has gone horribly wrong. Detect and report this situation, for safety. Signed-off-by: David Gibson --- src/agent/src/device.rs | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/src/agent/src/device.rs b/src/agent/src/device.rs index 66ee9d9fa..a8c49ecd9 100644 --- a/src/agent/src/device.rs +++ b/src/agent/src/device.rs @@ -745,7 +745,15 @@ pub async fn add_devices( for device in devices.iter() { let update = add_device(device, sandbox).await?; if let Some(dev_update) = update.dev { - dev_updates.insert(&device.container_path, dev_update); + if dev_updates + .insert(&device.container_path, dev_update) + .is_some() + { + return Err(anyhow!( + "Conflicting device updates for {}", + &device.container_path + )); + } } } From d6a3ebc496df78b0a9ef8ab711ad7c4aa923effe Mon Sep 17 00:00:00 2001 From: David Gibson Date: Thu, 4 Nov 2021 16:43:45 +1100 Subject: [PATCH 14/66] agent/device: Obtain guest major/minor numbers when creating DevNumUpdate Currently the DevNumUpdate structure is created with a path to a device node in the VM, which is then used by update_spec_device(). However the only piece of information that update_spec_device() actually needs is the VM side major and minor numbers for the device. We can determine those when we create the DevNumUpdate structure. This means we detect errors earlier and as a bonus we don't need to make a copy of the vm path string. Since that change requires updating 2 of the log statements, we take the opportunity to update all the log statements to structured style. Signed-off-by: David Gibson --- src/agent/src/device.rs | 85 +++++++++++++++++++---------------------- 1 file changed, 39 insertions(+), 46 deletions(-) diff --git a/src/agent/src/device.rs b/src/agent/src/device.rs index a8c49ecd9..06cdc4b5f 100644 --- a/src/agent/src/device.rs +++ b/src/agent/src/device.rs @@ -11,7 +11,7 @@ use std::fmt; use std::fs; use std::os::unix::ffi::OsStrExt; use std::os::unix::fs::MetadataExt; -use std::path::{Path, PathBuf}; +use std::path::Path; use std::str::FromStr; use std::sync::Arc; use tokio::sync::Mutex; @@ -437,15 +437,26 @@ fn scan_scsi_bus(scsi_addr: &str) -> Result<()> { #[derive(Debug, Clone)] struct DevNumUpdate { - // the path to the equivalent device in the VM (used to determine - // the correct major/minor numbers) - vm_path: PathBuf, + // the major and minor numbers for the device within the guest + guest_major: i64, + guest_minor: i64, } impl DevNumUpdate { fn from_vm_path>(vm_path: T) -> Result { + let vm_path = vm_path.as_ref(); + + if !vm_path.exists() { + return Err(anyhow!("VM device path {:?} doesn't exist", vm_path)); + } + + let devid = fs::metadata(vm_path)?.rdev(); + let guest_major = stat::major(devid) as i64; + let guest_minor = stat::minor(devid) as i64; + Ok(DevNumUpdate { - vm_path: vm_path.as_ref().to_owned(), + guest_major, + guest_minor, }) } } @@ -515,24 +526,12 @@ fn update_spec_device( .as_mut() .ok_or_else(|| anyhow!("Spec didn't container linux field"))?; - if !update.num.vm_path.exists() { - return Err(anyhow!( - "vm_path:{} doesn't exist", - update.num.vm_path.display() - )); - } - - let meta = fs::metadata(&update.num.vm_path)?; - let dev_id = meta.rdev(); - let major_id = stat::major(dev_id); - let minor_id = stat::minor(dev_id); - info!( sl!(), - "update_spec_device(): vm_path={}, major: {}, minor: {}\n", - update.num.vm_path.display(), - major_id, - minor_id + "update_spec_device() considering device"; + "container_path" => container_path, + "guest_major" => update.num.guest_major, + "guest_minor" => update.num.guest_minor, ); if let Some(idxdata) = devidx.0.get(container_path) { @@ -540,21 +539,21 @@ fn update_spec_device( let host_major = dev.major; let host_minor = dev.minor; - dev.major = major_id as i64; - dev.minor = minor_id as i64; + dev.major = update.num.guest_major; + dev.minor = update.num.guest_minor; if let Some(final_path) = update.final_path { dev.path = final_path; } info!( sl!(), - "change the device from path: {} major: {} minor: {} to vm device path: {} major: {} minor: {}", - container_path, - host_major, - host_minor, - dev.path, - dev.major, - dev.minor, + "update_spec_device(): updating device"; + "container_path" => container_path, + "host_major" => host_major, + "host_minor" => host_minor, + "updated_path" => &dev.path, + "guest_major" => dev.major, + "guest_minor" => dev.minor, ); // Resources must be updated since they are used to identify @@ -563,12 +562,14 @@ fn update_spec_device( // unwrap is safe, because residx would be empty if there // were no resources let res = &mut linux.resources.as_mut().unwrap().devices[*ridx]; - res.major = Some(major_id as i64); - res.minor = Some(minor_id as i64); + res.major = Some(update.num.guest_major); + res.minor = Some(update.num.guest_minor); info!( sl!(), - "set resources for device major: {} minor: {}\n", major_id, minor_id + "set resources for resource"; + "guest_major" => update.num.guest_major, + "guest_minor" => update.num.guest_minor, ); } Ok(()) @@ -858,9 +859,13 @@ mod tests { let (major, minor) = (7, 2); let mut spec = Spec::default(); + // vm_path empty + let update = DevNumUpdate::from_vm_path(""); + assert!(update.is_err()); + // container_path empty let container_path = ""; - let vm_path = ""; + let vm_path = "/dev/null"; let devidx = DevIndex::new(&spec); let res = update_spec_device( &mut spec, @@ -900,18 +905,6 @@ mod tests { ..oci::LinuxDevice::default() }]; - // vm_path empty - let devidx = DevIndex::new(&spec); - let res = update_spec_device( - &mut spec, - &devidx, - container_path, - DevNumUpdate::from_vm_path(vm_path).unwrap().into(), - ); - assert!(res.is_err()); - - let vm_path = "/dev/null"; - // guest and host path are not the same let devidx = DevIndex::new(&spec); let res = update_spec_device( From 084538d33407f5c681db56647c45cc3d6b61c907 Mon Sep 17 00:00:00 2001 From: David Gibson Date: Thu, 4 Nov 2021 19:58:52 +1100 Subject: [PATCH 15/66] agent/device: Change update_spec_device to handle multiple devices at once update_spec_device() adjusts the OCI spec for device differences between the host and guest. It is called repeatedly for each device we need to alter. These calls are now all in a single loop in add_devices(), so it makes more sense to move the loop into a renamed update_spec_devices() and process all the fixups in one call. Signed-off-by: David Gibson --- src/agent/src/device.rs | 221 +++++++++++++++++++++------------------- 1 file changed, 114 insertions(+), 107 deletions(-) diff --git a/src/agent/src/device.rs b/src/agent/src/device.rs index 06cdc4b5f..1f4ed0b3d 100644 --- a/src/agent/src/device.rs +++ b/src/agent/src/device.rs @@ -503,82 +503,83 @@ impl> From for SpecUpdate { } } -// update_spec_device updates the device list in the OCI spec to make +// update_spec_devices updates the device list in the OCI spec to make // it include details appropriate for the VM, instead of the host. It -// is given: +// is given a map of (container_path => update) where: // container_path: the path to the device in the original OCI spec // update: information on changes to make to the device #[instrument] -fn update_spec_device( +fn update_spec_devices( spec: &mut Spec, devidx: &DevIndex, - container_path: &str, - update: DevUpdate, + updates: HashMap<&str, DevUpdate>, ) -> Result<()> { - // If no container_path is provided, we won't be able to match and - // update the device in the OCI spec device list. This is an error. - if container_path.is_empty() { - return Err(anyhow!("Container path cannot be empty for device")); - } - - let linux = spec - .linux - .as_mut() - .ok_or_else(|| anyhow!("Spec didn't container linux field"))?; - - info!( - sl!(), - "update_spec_device() considering device"; - "container_path" => container_path, - "guest_major" => update.num.guest_major, - "guest_minor" => update.num.guest_minor, - ); - - if let Some(idxdata) = devidx.0.get(container_path) { - let dev = &mut linux.devices[idxdata.idx]; - let host_major = dev.major; - let host_minor = dev.minor; - - dev.major = update.num.guest_major; - dev.minor = update.num.guest_minor; - if let Some(final_path) = update.final_path { - dev.path = final_path; + for (container_path, update) in updates { + // If no container_path is provided, we won't be able to match and + // update the device in the OCI spec device list. This is an error. + if container_path.is_empty() { + return Err(anyhow!("Container path cannot be empty for device")); } + let linux = spec + .linux + .as_mut() + .ok_or_else(|| anyhow!("Spec didn't container linux field"))?; + info!( sl!(), - "update_spec_device(): updating device"; + "update_spec_devices() considering device"; "container_path" => container_path, - "host_major" => host_major, - "host_minor" => host_minor, - "updated_path" => &dev.path, - "guest_major" => dev.major, - "guest_minor" => dev.minor, + "guest_major" => update.num.guest_major, + "guest_minor" => update.num.guest_minor, ); - // Resources must be updated since they are used to identify - // the device in the devices cgroup. - for ridx in &idxdata.residx { - // unwrap is safe, because residx would be empty if there - // were no resources - let res = &mut linux.resources.as_mut().unwrap().devices[*ridx]; - res.major = Some(update.num.guest_major); - res.minor = Some(update.num.guest_minor); + if let Some(idxdata) = devidx.0.get(container_path) { + let dev = &mut linux.devices[idxdata.idx]; + let host_major = dev.major; + let host_minor = dev.minor; + + dev.major = update.num.guest_major; + dev.minor = update.num.guest_minor; + if let Some(final_path) = update.final_path { + dev.path = final_path; + } info!( sl!(), - "set resources for resource"; - "guest_major" => update.num.guest_major, - "guest_minor" => update.num.guest_minor, + "update_spec_devices(): updating device"; + "container_path" => container_path, + "host_major" => host_major, + "host_minor" => host_minor, + "updated_path" => &dev.path, + "guest_major" => dev.major, + "guest_minor" => dev.minor, ); + + // Resources must be updated since they are used to identify + // the device in the devices cgroup. + for ridx in &idxdata.residx { + // unwrap is safe, because residx would be empty if there + // were no resources + let res = &mut linux.resources.as_mut().unwrap().devices[*ridx]; + res.major = Some(update.num.guest_major); + res.minor = Some(update.num.guest_minor); + + info!( + sl!(), + "set resources for resource"; + "guest_major" => update.num.guest_major, + "guest_minor" => update.num.guest_minor, + ); + } + } else { + return Err(anyhow!( + "Should have found a device {} in the spec", + container_path + )); } - Ok(()) - } else { - Err(anyhow!( - "Should have found a device {} in the spec", - container_path - )) } + Ok(()) } // device.Id should be the predicted device name (vda, vdb, ...) @@ -759,11 +760,7 @@ pub async fn add_devices( } let devidx = DevIndex::new(spec); - for (container_path, update) in dev_updates.drain() { - update_spec_device(spec, &devidx, container_path, update)?; - } - - Ok(()) + update_spec_devices(spec, &devidx, dev_updates) } #[instrument] @@ -831,6 +828,7 @@ mod tests { use super::*; use crate::uevent::spawn_test_watcher; use oci::Linux; + use std::iter::FromIterator; use tempfile::tempdir; #[test] @@ -855,7 +853,7 @@ mod tests { } #[test] - fn test_update_spec_device() { + fn test_update_spec_devices() { let (major, minor) = (7, 2); let mut spec = Spec::default(); @@ -867,22 +865,26 @@ mod tests { let container_path = ""; let vm_path = "/dev/null"; let devidx = DevIndex::new(&spec); - let res = update_spec_device( + let res = update_spec_devices( &mut spec, &devidx, - container_path, - DevNumUpdate::from_vm_path(vm_path).unwrap().into(), + HashMap::from_iter(vec![( + container_path, + DevNumUpdate::from_vm_path(vm_path).unwrap().into(), + )]), ); assert!(res.is_err()); // linux is empty let container_path = "/dev/null"; let devidx = DevIndex::new(&spec); - let res = update_spec_device( + let res = update_spec_devices( &mut spec, &devidx, - container_path, - DevNumUpdate::from_vm_path(vm_path).unwrap().into(), + HashMap::from_iter(vec![( + container_path, + DevNumUpdate::from_vm_path(vm_path).unwrap().into(), + )]), ); assert!(res.is_err()); @@ -890,11 +892,13 @@ mod tests { // linux.devices is empty let devidx = DevIndex::new(&spec); - let res = update_spec_device( + let res = update_spec_devices( &mut spec, &devidx, - container_path, - DevNumUpdate::from_vm_path(vm_path).unwrap().into(), + HashMap::from_iter(vec![( + container_path, + DevNumUpdate::from_vm_path(vm_path).unwrap().into(), + )]), ); assert!(res.is_err()); @@ -907,11 +911,13 @@ mod tests { // guest and host path are not the same let devidx = DevIndex::new(&spec); - let res = update_spec_device( + let res = update_spec_devices( &mut spec, &devidx, - container_path, - DevNumUpdate::from_vm_path(vm_path).unwrap().into(), + HashMap::from_iter(vec![( + container_path, + DevNumUpdate::from_vm_path(vm_path).unwrap().into(), + )]), ); assert!( res.is_err(), @@ -925,11 +931,13 @@ mod tests { // spec.linux.resources is empty let devidx = DevIndex::new(&spec); - let res = update_spec_device( + let res = update_spec_devices( &mut spec, &devidx, - container_path, - DevNumUpdate::from_vm_path(vm_path).unwrap().into(), + HashMap::from_iter(vec![( + container_path, + DevNumUpdate::from_vm_path(vm_path).unwrap().into(), + )]), ); assert!(res.is_ok()); @@ -951,17 +959,19 @@ mod tests { }); let devidx = DevIndex::new(&spec); - let res = update_spec_device( + let res = update_spec_devices( &mut spec, &devidx, - container_path, - DevNumUpdate::from_vm_path(vm_path).unwrap().into(), + HashMap::from_iter(vec![( + container_path, + DevNumUpdate::from_vm_path(vm_path).unwrap().into(), + )]), ); assert!(res.is_ok()); } #[test] - fn test_update_spec_device_guest_host_conflict() { + fn test_update_spec_devices_guest_host_conflict() { let null_rdev = fs::metadata("/dev/null").unwrap().rdev(); let zero_rdev = fs::metadata("/dev/zero").unwrap().rdev(); let full_rdev = fs::metadata("/dev/full").unwrap().rdev(); @@ -1036,24 +1046,17 @@ mod tests { assert_eq!(Some(host_major_b), specresources.devices[1].major); assert_eq!(Some(host_minor_b), specresources.devices[1].minor); - let update_a = DevNumUpdate::from_vm_path(vm_path_a).unwrap().into(); - let res = update_spec_device(&mut spec, &devidx, container_path_a, update_a); - assert!(res.is_ok()); - - let specdevices = &spec.linux.as_ref().unwrap().devices; - assert_eq!(guest_major_a, specdevices[0].major); - assert_eq!(guest_minor_a, specdevices[0].minor); - assert_eq!(host_major_b, specdevices[1].major); - assert_eq!(host_minor_b, specdevices[1].minor); - - let specresources = spec.linux.as_ref().unwrap().resources.as_ref().unwrap(); - assert_eq!(Some(guest_major_a), specresources.devices[0].major); - assert_eq!(Some(guest_minor_a), specresources.devices[0].minor); - assert_eq!(Some(host_major_b), specresources.devices[1].major); - assert_eq!(Some(host_minor_b), specresources.devices[1].minor); - - let update_b = DevNumUpdate::from_vm_path(vm_path_b).unwrap().into(); - let res = update_spec_device(&mut spec, &devidx, container_path_b, update_b); + let updates = HashMap::from_iter(vec![ + ( + container_path_a, + DevNumUpdate::from_vm_path(vm_path_a).unwrap().into(), + ), + ( + container_path_b, + DevNumUpdate::from_vm_path(vm_path_b).unwrap().into(), + ), + ]); + let res = update_spec_devices(&mut spec, &devidx, updates); assert!(res.is_ok()); let specdevices = &spec.linux.as_ref().unwrap().devices; @@ -1070,7 +1073,7 @@ mod tests { } #[test] - fn test_update_spec_device_char_block_conflict() { + fn test_update_spec_devices_char_block_conflict() { let null_rdev = fs::metadata("/dev/null").unwrap().rdev(); let guest_major = stat::major(null_rdev) as i64; @@ -1128,11 +1131,13 @@ mod tests { assert_eq!(Some(host_major), specresources.devices[1].major); assert_eq!(Some(host_minor), specresources.devices[1].minor); - let res = update_spec_device( + let res = update_spec_devices( &mut spec, &devidx, - container_path, - DevNumUpdate::from_vm_path(vm_path).unwrap().into(), + HashMap::from_iter(vec![( + container_path, + DevNumUpdate::from_vm_path(vm_path).unwrap().into(), + )]), ); assert!(res.is_ok()); @@ -1145,7 +1150,7 @@ mod tests { } #[test] - fn test_update_spec_device_final_path() { + fn test_update_spec_devices_final_path() { let null_rdev = fs::metadata("/dev/null").unwrap().rdev(); let guest_major = stat::major(null_rdev) as i64; let guest_minor = stat::minor(null_rdev) as i64; @@ -1172,11 +1177,13 @@ mod tests { let vm_path = "/dev/null"; let final_path = "/dev/new"; - let res = update_spec_device( + let res = update_spec_devices( &mut spec, &devidx, - container_path, - DevUpdate::from_vm_path(vm_path, final_path.to_string()).unwrap(), + HashMap::from_iter(vec![( + container_path, + DevUpdate::from_vm_path(vm_path, final_path.to_string()).unwrap(), + )]), ); assert!(res.is_ok()); From c855a312f011a85c180811dc3a73ab3114cf83be Mon Sep 17 00:00:00 2001 From: David Gibson Date: Thu, 4 Nov 2021 20:46:10 +1100 Subject: [PATCH 16/66] agent/device: Make DevIndex local to update_spec_devices() The DevIndex data structure keeps track of devices in the OCI specification. We used to carry it around to quite a lot of functions, but it's now used only within update_spec_devices(). That means we can simplify things a bit by just open coding the maps we need, rather than declaring a special type. Signed-off-by: David Gibson --- src/agent/src/device.rs | 93 ++++++++++++----------------------------- 1 file changed, 26 insertions(+), 67 deletions(-) diff --git a/src/agent/src/device.rs b/src/agent/src/device.rs index 1f4ed0b3d..3ab58d66b 100644 --- a/src/agent/src/device.rs +++ b/src/agent/src/device.rs @@ -52,15 +52,6 @@ pub const DRIVER_VFIO_GK_TYPE: &str = "vfio-gk"; // container as a VFIO device node pub const DRIVER_VFIO_TYPE: &str = "vfio"; -#[derive(Debug)] -struct DevIndexEntry { - idx: usize, - residx: Vec, -} - -#[derive(Debug)] -struct DevIndex(HashMap); - #[instrument] pub fn online_device(path: &str) -> Result<()> { fs::write(path, "1")?; @@ -509,11 +500,27 @@ impl> From for SpecUpdate { // container_path: the path to the device in the original OCI spec // update: information on changes to make to the device #[instrument] -fn update_spec_devices( - spec: &mut Spec, - devidx: &DevIndex, - updates: HashMap<&str, DevUpdate>, -) -> Result<()> { +fn update_spec_devices(spec: &mut Spec, updates: HashMap<&str, DevUpdate>) -> Result<()> { + let linux = spec + .linux + .as_mut() + .ok_or_else(|| anyhow!("Spec didn't contain linux field"))?; + + let mut devidx = HashMap::)>::new(); + + for (i, d) in linux.devices.iter().enumerate() { + let mut residx = Vec::new(); + + if let Some(linuxres) = linux.resources.as_ref() { + for (j, r) in linuxres.devices.iter().enumerate() { + if r.r#type == d.r#type && r.major == Some(d.major) && r.minor == Some(d.minor) { + residx.push(j); + } + } + } + devidx.insert(d.path.clone(), (i, residx)); + } + for (container_path, update) in updates { // If no container_path is provided, we won't be able to match and // update the device in the OCI spec device list. This is an error. @@ -521,11 +528,6 @@ fn update_spec_devices( return Err(anyhow!("Container path cannot be empty for device")); } - let linux = spec - .linux - .as_mut() - .ok_or_else(|| anyhow!("Spec didn't container linux field"))?; - info!( sl!(), "update_spec_devices() considering device"; @@ -534,8 +536,8 @@ fn update_spec_devices( "guest_minor" => update.num.guest_minor, ); - if let Some(idxdata) = devidx.0.get(container_path) { - let dev = &mut linux.devices[idxdata.idx]; + if let Some(idxdata) = devidx.get(container_path) { + let dev = &mut linux.devices[idxdata.0]; let host_major = dev.major; let host_minor = dev.minor; @@ -558,7 +560,7 @@ fn update_spec_devices( // Resources must be updated since they are used to identify // the device in the devices cgroup. - for ridx in &idxdata.residx { + for ridx in &idxdata.1 { // unwrap is safe, because residx would be empty if there // were no resources let res = &mut linux.resources.as_mut().unwrap().devices[*ridx]; @@ -711,31 +713,6 @@ async fn vfio_device_handler(device: &Device, sandbox: &Arc>) -> }) } -impl DevIndex { - fn new(spec: &Spec) -> DevIndex { - let mut map = HashMap::new(); - - if let Some(linux) = spec.linux.as_ref() { - for (i, d) in linux.devices.iter().enumerate() { - let mut residx = Vec::new(); - - if let Some(linuxres) = linux.resources.as_ref() { - for (j, r) in linuxres.devices.iter().enumerate() { - if r.r#type == d.r#type - && r.major == Some(d.major) - && r.minor == Some(d.minor) - { - residx.push(j); - } - } - } - map.insert(d.path.clone(), DevIndexEntry { idx: i, residx }); - } - } - DevIndex(map) - } -} - #[instrument] pub async fn add_devices( devices: &[Device], @@ -759,8 +736,7 @@ pub async fn add_devices( } } - let devidx = DevIndex::new(spec); - update_spec_devices(spec, &devidx, dev_updates) + update_spec_devices(spec, dev_updates) } #[instrument] @@ -864,10 +840,8 @@ mod tests { // container_path empty let container_path = ""; let vm_path = "/dev/null"; - let devidx = DevIndex::new(&spec); let res = update_spec_devices( &mut spec, - &devidx, HashMap::from_iter(vec![( container_path, DevNumUpdate::from_vm_path(vm_path).unwrap().into(), @@ -877,10 +851,8 @@ mod tests { // linux is empty let container_path = "/dev/null"; - let devidx = DevIndex::new(&spec); let res = update_spec_devices( &mut spec, - &devidx, HashMap::from_iter(vec![( container_path, DevNumUpdate::from_vm_path(vm_path).unwrap().into(), @@ -891,10 +863,8 @@ mod tests { spec.linux = Some(Linux::default()); // linux.devices is empty - let devidx = DevIndex::new(&spec); let res = update_spec_devices( &mut spec, - &devidx, HashMap::from_iter(vec![( container_path, DevNumUpdate::from_vm_path(vm_path).unwrap().into(), @@ -910,10 +880,8 @@ mod tests { }]; // guest and host path are not the same - let devidx = DevIndex::new(&spec); let res = update_spec_devices( &mut spec, - &devidx, HashMap::from_iter(vec![( container_path, DevNumUpdate::from_vm_path(vm_path).unwrap().into(), @@ -930,10 +898,8 @@ mod tests { spec.linux.as_mut().unwrap().devices[0].path = container_path.to_string(); // spec.linux.resources is empty - let devidx = DevIndex::new(&spec); let res = update_spec_devices( &mut spec, - &devidx, HashMap::from_iter(vec![( container_path, DevNumUpdate::from_vm_path(vm_path).unwrap().into(), @@ -958,10 +924,8 @@ mod tests { ..oci::LinuxResources::default() }); - let devidx = DevIndex::new(&spec); let res = update_spec_devices( &mut spec, - &devidx, HashMap::from_iter(vec![( container_path, DevNumUpdate::from_vm_path(vm_path).unwrap().into(), @@ -1020,7 +984,6 @@ mod tests { }), ..Spec::default() }; - let devidx = DevIndex::new(&spec); let container_path_a = "/dev/a"; let vm_path_a = "/dev/zero"; @@ -1056,7 +1019,7 @@ mod tests { DevNumUpdate::from_vm_path(vm_path_b).unwrap().into(), ), ]); - let res = update_spec_devices(&mut spec, &devidx, updates); + let res = update_spec_devices(&mut spec, updates); assert!(res.is_ok()); let specdevices = &spec.linux.as_ref().unwrap().devices; @@ -1120,7 +1083,6 @@ mod tests { }), ..Spec::default() }; - let devidx = DevIndex::new(&spec); let container_path = "/dev/char"; let vm_path = "/dev/null"; @@ -1133,7 +1095,6 @@ mod tests { let res = update_spec_devices( &mut spec, - &devidx, HashMap::from_iter(vec![( container_path, DevNumUpdate::from_vm_path(vm_path).unwrap().into(), @@ -1172,14 +1133,12 @@ mod tests { }), ..Spec::default() }; - let devidx = DevIndex::new(&spec); let vm_path = "/dev/null"; let final_path = "/dev/new"; let res = update_spec_devices( &mut spec, - &devidx, HashMap::from_iter(vec![( container_path, DevUpdate::from_vm_path(vm_path, final_path.to_string()).unwrap(), From 89ff7000380d0456c9a879125723b457873a0fc5 Mon Sep 17 00:00:00 2001 From: David Gibson Date: Fri, 5 Nov 2021 16:37:22 +1100 Subject: [PATCH 17/66] agent/device: Remove unnecessary check for empty container_path update_spec_devices() explicitly checks for being called with an empty container path and fails. We have a unit test to verify this behaviour. But while an empty container_path probably does mean something has gone wrong elsewhere, that's also true of any number of other bad paths. Having an empty string here doesn't prevent what we're doing in this function making sense - we can compare it to the strings in the OCI spec perfectly well (though more likely we simply won't find it there). So, there's no real reason to check this one particular odd case. Signed-off-by: David Gibson --- src/agent/src/device.rs | 19 +------------------ 1 file changed, 1 insertion(+), 18 deletions(-) diff --git a/src/agent/src/device.rs b/src/agent/src/device.rs index 3ab58d66b..16bae152c 100644 --- a/src/agent/src/device.rs +++ b/src/agent/src/device.rs @@ -522,12 +522,6 @@ fn update_spec_devices(spec: &mut Spec, updates: HashMap<&str, DevUpdate>) -> Re } for (container_path, update) in updates { - // If no container_path is provided, we won't be able to match and - // update the device in the OCI spec device list. This is an error. - if container_path.is_empty() { - return Err(anyhow!("Container path cannot be empty for device")); - } - info!( sl!(), "update_spec_devices() considering device"; @@ -837,20 +831,9 @@ mod tests { let update = DevNumUpdate::from_vm_path(""); assert!(update.is_err()); - // container_path empty - let container_path = ""; - let vm_path = "/dev/null"; - let res = update_spec_devices( - &mut spec, - HashMap::from_iter(vec![( - container_path, - DevNumUpdate::from_vm_path(vm_path).unwrap().into(), - )]), - ); - assert!(res.is_err()); - // linux is empty let container_path = "/dev/null"; + let vm_path = "/dev/null"; let res = update_spec_devices( &mut spec, HashMap::from_iter(vec![( From b60622786d0275028fedf4ae4645cf840cecf3fc Mon Sep 17 00:00:00 2001 From: David Gibson Date: Fri, 5 Nov 2021 17:28:27 +1100 Subject: [PATCH 18/66] agent/device: Correct misleading comment on test case We have a test case commented as testing the case where linux.devices is empty in the OCI spec. While it's true that linux.devices is empth in this example, the reason it fails isn't specifically because it's empty but because it doesn't contain a device for the update we're trying to apply. Signed-off-by: David Gibson --- src/agent/src/device.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/agent/src/device.rs b/src/agent/src/device.rs index 16bae152c..0994a3e2d 100644 --- a/src/agent/src/device.rs +++ b/src/agent/src/device.rs @@ -845,7 +845,7 @@ mod tests { spec.linux = Some(Linux::default()); - // linux.devices is empty + // linux.devices doesn't contain the updated device let res = update_spec_devices( &mut spec, HashMap::from_iter(vec![( From 4530e7df294f1832a1a134615ac32e975fe2ce12 Mon Sep 17 00:00:00 2001 From: David Gibson Date: Fri, 5 Nov 2021 17:12:30 +1100 Subject: [PATCH 19/66] agent/device: Use simpler structure in update_spec_devices() update_spec_devices() takes a bunch of updates for the device entries in the OCI spec and applies them, adjusting things in both the linux.devices and linux.resources.devices sections of the spec. It's important that each entry in the spec only be updated once. Currently we ensure this by first creating an index of where the entries are, then consulting that as we apply each update, so that earlier updates don't cause us to incorrectly detect an entry as being relevant to a later update. This method works, but it's quite awkward. This inverts the loop structure in update_spec_devices() to make this clearer. Instead of stepping through each update and finding the relevant entries in the spec to change, we step through each entry in the spec and find the relevant update. This makes it structurally clear that we're only updating each entry once. Signed-off-by: David Gibson --- src/agent/src/device.rs | 125 +++++++++++++++++++++------------------- 1 file changed, 65 insertions(+), 60 deletions(-) diff --git a/src/agent/src/device.rs b/src/agent/src/device.rs index 0994a3e2d..8297ccb4e 100644 --- a/src/agent/src/device.rs +++ b/src/agent/src/device.rs @@ -500,81 +500,86 @@ impl> From for SpecUpdate { // container_path: the path to the device in the original OCI spec // update: information on changes to make to the device #[instrument] -fn update_spec_devices(spec: &mut Spec, updates: HashMap<&str, DevUpdate>) -> Result<()> { +fn update_spec_devices(spec: &mut Spec, mut updates: HashMap<&str, DevUpdate>) -> Result<()> { let linux = spec .linux .as_mut() .ok_or_else(|| anyhow!("Spec didn't contain linux field"))?; + let mut res_updates = HashMap::<(&str, i64, i64), DevNumUpdate>::with_capacity(updates.len()); - let mut devidx = HashMap::)>::new(); - - for (i, d) in linux.devices.iter().enumerate() { - let mut residx = Vec::new(); - - if let Some(linuxres) = linux.resources.as_ref() { - for (j, r) in linuxres.devices.iter().enumerate() { - if r.r#type == d.r#type && r.major == Some(d.major) && r.minor == Some(d.minor) { - residx.push(j); - } - } - } - devidx.insert(d.path.clone(), (i, residx)); - } - - for (container_path, update) in updates { - info!( - sl!(), - "update_spec_devices() considering device"; - "container_path" => container_path, - "guest_major" => update.num.guest_major, - "guest_minor" => update.num.guest_minor, - ); - - if let Some(idxdata) = devidx.get(container_path) { - let dev = &mut linux.devices[idxdata.0]; - let host_major = dev.major; - let host_minor = dev.minor; - - dev.major = update.num.guest_major; - dev.minor = update.num.guest_minor; - if let Some(final_path) = update.final_path { - dev.path = final_path; - } + for specdev in &mut linux.devices { + if let Some(update) = updates.remove(specdev.path.as_str()) { + let host_major = specdev.major; + let host_minor = specdev.minor; info!( sl!(), - "update_spec_devices(): updating device"; - "container_path" => container_path, + "update_spec_devices() updating device"; + "container_path" => &specdev.path, + "type" => &specdev.r#type, "host_major" => host_major, "host_minor" => host_minor, - "updated_path" => &dev.path, - "guest_major" => dev.major, - "guest_minor" => dev.minor, + "guest_major" => update.num.guest_major, + "guest_minor" => update.num.guest_minor, + "final_path" => update.final_path.as_ref(), ); - // Resources must be updated since they are used to identify - // the device in the devices cgroup. - for ridx in &idxdata.1 { - // unwrap is safe, because residx would be empty if there - // were no resources - let res = &mut linux.resources.as_mut().unwrap().devices[*ridx]; - res.major = Some(update.num.guest_major); - res.minor = Some(update.num.guest_minor); - - info!( - sl!(), - "set resources for resource"; - "guest_major" => update.num.guest_major, - "guest_minor" => update.num.guest_minor, - ); + specdev.major = update.num.guest_major; + specdev.minor = update.num.guest_minor; + if let Some(final_path) = update.final_path { + specdev.path = final_path; + } + + if res_updates + .insert( + (specdev.r#type.as_str(), host_major, host_minor), + update.num, + ) + .is_some() + { + return Err(anyhow!( + "Conflicting resource updates for host_major={} host_minor={}", + host_major, + host_minor + )); } - } else { - return Err(anyhow!( - "Should have found a device {} in the spec", - container_path - )); } } + + // Make sure we applied all of our updates + if !updates.is_empty() { + return Err(anyhow!( + "Missing devices in OCI spec: {:?}", + updates + .keys() + .map(|d| format!("{:?}", d)) + .collect::>() + .join(" ") + )); + } + + if let Some(resources) = linux.resources.as_mut() { + for r in &mut resources.devices { + if let (Some(host_major), Some(host_minor)) = (r.major, r.minor) { + if let Some(update) = res_updates.get(&(r.r#type.as_str(), host_major, host_minor)) + { + info!( + sl!(), + "update_spec_devices() updating resource"; + "type" => &r.r#type, + "host_major" => host_major, + "host_minor" => host_minor, + "guest_major" => update.guest_major, + "guest_minor" => update.guest_minor, + ); + + r.major = Some(update.guest_major); + r.minor = Some(update.guest_minor); + } + } + } + } + Ok(()) } From 78dff468bff95d33b0042cb6e3aef2fc8d2d2f14 Mon Sep 17 00:00:00 2001 From: David Gibson Date: Wed, 10 Nov 2021 13:07:25 +1100 Subject: [PATCH 20/66] agent/device: Adjust PCIDEVICE_* container environment variables for VM The k8s SR-IOV plugin, when it assigns a VFIO device to a container, adds an variable of the form PCIDEVICE_ to the container's environment, so that the payload knows which device is which. The contents of the variable gives the PCI address of the device to use. Kata allows VFIO devices to be passed in to a Kata container, however it runs within a VM which has a different PCI topology. In order for the payload to find the right device, the environment variables therefore need to be converted to list the guest PCI addresses instead of the host PCI addresses. fixes #2897 Signed-off-by: David Gibson --- src/agent/src/device.rs | 116 +++++++++++++++++++++++++++++++++++++--- src/agent/src/pci.rs | 4 +- 2 files changed, 112 insertions(+), 8 deletions(-) diff --git a/src/agent/src/device.rs b/src/agent/src/device.rs index 8297ccb4e..c22a65795 100644 --- a/src/agent/src/device.rs +++ b/src/agent/src/device.rs @@ -22,7 +22,7 @@ use crate::linux_abi::*; use crate::pci; use crate::sandbox::Sandbox; use crate::uevent::{wait_for_uevent, Uevent, UeventMatcher}; -use anyhow::{anyhow, Result}; +use anyhow::{anyhow, Context, Result}; use oci::{LinuxDeviceCgroup, LinuxResources, Spec}; use protocols::agent::Device; use tracing::instrument; @@ -484,12 +484,15 @@ impl From for DevUpdate { #[derive(Debug, Clone, Default)] struct SpecUpdate { dev: Option, + // optional corrections for PCI addresses + pci: Vec<(pci::Address, pci::Address)>, } impl> From for SpecUpdate { fn from(dev: T) -> Self { SpecUpdate { dev: Some(dev.into()), + pci: Vec::new(), } } } @@ -583,6 +586,43 @@ fn update_spec_devices(spec: &mut Spec, mut updates: HashMap<&str, DevUpdate>) - Ok(()) } +// update_spec_pci PCI addresses in the OCI spec to be guest addresses +// instead of host addresses. It is given a map of (host address => +// guest address) +#[instrument] +fn update_spec_pci(spec: &mut Spec, updates: HashMap) -> Result<()> { + // Correct PCI addresses in the environment + if let Some(process) = spec.process.as_mut() { + for envvar in process.env.iter_mut() { + let eqpos = envvar + .find('=') + .ok_or_else(|| anyhow!("Malformed OCI env entry {:?}", envvar))?; + + let (name, eqval) = envvar.split_at(eqpos); + let val = &eqval[1..]; + + if !name.starts_with("PCIDEVICE_") { + continue; + } + + let mut guest_addrs = Vec::::new(); + + for host_addr in val.split(',') { + let host_addr = pci::Address::from_str(host_addr) + .with_context(|| format!("Can't parse {} environment variable", name))?; + let guest_addr = updates + .get(&host_addr) + .ok_or_else(|| anyhow!("Unable to translate host PCI address {}", host_addr))?; + guest_addrs.push(format!("{}", guest_addr)); + } + + envvar.replace_range(eqpos + 1.., guest_addrs.join(",").as_str()); + } + } + + Ok(()) +} + // device.Id should be the predicted device name (vda, vdb, ...) // device.VmPath already provides a way to send it in #[instrument] @@ -668,11 +708,14 @@ fn split_vfio_option(opt: &str) -> Option<(&str, &str)> { // is a PCI path to the device in the guest (see pci.rs) async fn vfio_device_handler(device: &Device, sandbox: &Arc>) -> Result { let vfio_in_guest = device.field_type != DRIVER_VFIO_GK_TYPE; + let mut pci_fixups = Vec::<(pci::Address, pci::Address)>::new(); let mut group = None; for opt in device.options.iter() { - let (_, pcipath) = + let (host, pcipath) = split_vfio_option(opt).ok_or_else(|| anyhow!("Malformed VFIO option {:?}", opt))?; + let host = + pci::Address::from_str(host).context("Bad host PCI address in VFIO option {:?}")?; let pcipath = pci::Path::from_str(pcipath)?; let guestdev = wait_for_pci_device(sandbox, &pcipath).await?; @@ -698,17 +741,24 @@ async fn vfio_device_handler(device: &Device, sandbox: &Arc>) -> } group = devgroup; + + pci_fixups.push((host, guestdev)); } } - Ok(if vfio_in_guest { + let dev_update = if vfio_in_guest { // If there are any devices at all, logic above ensures that group is not None let group = group.unwrap(); let vm_path = get_vfio_device_name(sandbox, group).await?; - DevUpdate::from_vm_path(&vm_path, vm_path.clone())?.into() + Some(DevUpdate::from_vm_path(&vm_path, vm_path.clone())?) } else { - SpecUpdate::default() + None + }; + + Ok(SpecUpdate { + dev: dev_update, + pci: pci_fixups, }) } @@ -719,6 +769,7 @@ pub async fn add_devices( sandbox: &Arc>, ) -> Result<()> { let mut dev_updates = HashMap::<&str, DevUpdate>::with_capacity(devices.len()); + let mut pci_updates = HashMap::::new(); for device in devices.iter() { let update = add_device(device, sandbox).await?; @@ -732,6 +783,17 @@ pub async fn add_devices( &device.container_path )); } + + for (host, guest) in update.pci { + if let Some(other_guest) = pci_updates.insert(host, guest) { + return Err(anyhow!( + "Conflicting guest address for host device {} ({} versus {})", + host, + guest, + other_guest + )); + } + } } } @@ -802,7 +864,7 @@ pub fn update_device_cgroup(spec: &mut Spec) -> Result<()> { mod tests { use super::*; use crate::uevent::spawn_test_watcher; - use oci::Linux; + use oci::{Linux, Process}; use std::iter::FromIterator; use tempfile::tempdir; @@ -1140,6 +1202,48 @@ mod tests { assert_eq!(final_path, specdevices[0].path); } + #[test] + fn test_update_spec_pci() { + let example_map = [ + // Each is a host,guest pair of pci addresses + ("0000:1a:01.0", "0000:01:01.0"), + ("0000:1b:02.0", "0000:01:02.0"), + // This one has the same host address as guest address + // above, to test that we're not double-translating + ("0000:01:01.0", "ffff:02:1f.7"), + ]; + + let mut spec = Spec { + process: Some(Process { + env: vec![ + "PCIDEVICE_x=0000:1a:01.0,0000:1b:02.0".to_string(), + "PCIDEVICE_y=0000:01:01.0".to_string(), + "NOTAPCIDEVICE_blah=abcd:ef:01.0".to_string(), + ], + ..Process::default() + }), + ..Spec::default() + }; + + let pci_fixups = example_map + .iter() + .map(|(h, g)| { + ( + pci::Address::from_str(h).unwrap(), + pci::Address::from_str(g).unwrap(), + ) + }) + .collect(); + + let res = update_spec_pci(&mut spec, pci_fixups); + assert!(res.is_ok()); + + let env = &spec.process.as_ref().unwrap().env; + assert_eq!(env[0], "PCIDEVICE_x=0000:01:01.0,0000:01:02.0"); + assert_eq!(env[1], "PCIDEVICE_y=ffff:02:1f.7"); + assert_eq!(env[2], "NOTAPCIDEVICE_blah=abcd:ef:01.0"); + } + #[test] fn test_pcipath_to_sysfs() { let testdir = tempdir().expect("failed to create tmpdir"); diff --git a/src/agent/src/pci.rs b/src/agent/src/pci.rs index 25ad9a6a0..4cb6b521a 100644 --- a/src/agent/src/pci.rs +++ b/src/agent/src/pci.rs @@ -20,7 +20,7 @@ const FUNCTION_MAX: u8 = (1 << FUNCTION_BITS) - 1; // Represents a PCI function's slot (a.k.a. device) and function // numbers, giving its location on a single logical bus -#[derive(Copy, Clone, Debug, PartialEq, Eq)] +#[derive(Copy, Clone, Debug, PartialEq, Eq, Hash)] pub struct SlotFn(u8); impl SlotFn { @@ -94,7 +94,7 @@ impl fmt::Display for SlotFn { } } -#[derive(Copy, Clone, Debug, PartialEq, Eq)] +#[derive(Copy, Clone, Debug, PartialEq, Eq, Hash)] pub struct Address { domain: u16, bus: u8, From b5dfcf26538d6262a016698550f11ed15b742090 Mon Sep 17 00:00:00 2001 From: Eric Ernst Date: Thu, 18 Nov 2021 13:49:54 -0800 Subject: [PATCH 21/66] watcher: tests: ensure there is 20ms delay between fs writes We noticed s390x test failures on several of the watcher unit tests. Discovered that on s390 in particular, if we update a file in quick sucecssion, the time stampe on the file would not be unique between the writes. Through testing, we observe that a 20 millisecond delay is very reliable for being able to observe the timestamp update. Let's ensure we have this delay between writes for our tests so our tests are more reliable. In "the real world" we'll be polling for changes every 2 seconds, and frequency of filesystem updates will be on order of minutes and days, rather that microseconds. Fixes: #2946 Signed-off-by: Eric Ernst --- src/agent/src/watcher.rs | 45 ++++++++++++++++++++++++++++++++-------- 1 file changed, 36 insertions(+), 9 deletions(-) diff --git a/src/agent/src/watcher.rs b/src/agent/src/watcher.rs index fd0f9fe86..bb3fb1641 100644 --- a/src/agent/src/watcher.rs +++ b/src/agent/src/watcher.rs @@ -660,6 +660,9 @@ mod tests { ..Default::default() }; + // delay 20 ms between writes to files in order to ensure filesystem timestamps are unique + thread::sleep(Duration::from_millis(20)); + entries .add(std::iter::once(storage0), &logger) .await @@ -730,6 +733,9 @@ mod tests { "updated" ); + // delay 20 ms between writes to files in order to ensure filesystem timestamps are unique + thread::sleep(Duration::from_millis(20)); + // // Prepare for second check: update mount sources // @@ -970,20 +976,27 @@ mod tests { let sym_dir = source_dir.path().join("..data"); let sym_file = source_dir.path().join("file.txt"); + let relative_to_dir = PathBuf::from("..2021_10_29_03_10_48.161654083"); + // create backing file/path fs::create_dir_all(&actual_dir).unwrap(); fs::write(&actual_file, "two").unwrap(); - // create indirection symlink directory tha points to actual_dir: - tokio::fs::symlink(&actual_dir, &sym_dir).await.unwrap(); + // create indirection symlink directory that points to the directory that holds the actual file: + tokio::fs::symlink(&relative_to_dir, &sym_dir) + .await + .unwrap(); // create presented data file symlink: - tokio::fs::symlink(sym_dir.join("file.txt"), sym_file) + tokio::fs::symlink(PathBuf::from("..data/file.txt"), sym_file) .await .unwrap(); let dest_dir = tempfile::tempdir().unwrap(); + // delay 20 ms between writes to files in order to ensure filesystem timestamps are unique + thread::sleep(Duration::from_millis(20)); + let mut entry = Storage::new(protos::Storage { source: source_dir.path().display().to_string(), mount_point: dest_dir.path().display().to_string(), @@ -1001,8 +1014,12 @@ mod tests { // now what, what is updated? fs::write(actual_file, "updated").unwrap(); - thread::sleep(Duration::from_secs(1)); + + // delay 20 ms between writes to files in order to ensure filesystem timestamps are unique + thread::sleep(Duration::from_millis(20)); + assert_eq!(entry.scan(&logger).await.unwrap(), 1); + assert_eq!( fs::read_to_string(dest_dir.path().join("file.txt")).unwrap(), "updated" @@ -1031,13 +1048,15 @@ mod tests { // - create a new actual dir/file, // - update the symlink directory to point to this one // - remove old dir/file - let new_actual_dir = source_dir.path().join("..2021_10_31_03_10_48.161654083"); + let new_actual_dir = source_dir.path().join("..2021_10_31"); let new_actual_file = new_actual_dir.join("file.txt"); fs::create_dir_all(&new_actual_dir).unwrap(); fs::write(&new_actual_file, "new configmap").unwrap(); tokio::fs::remove_file(&sym_dir).await.unwrap(); - tokio::fs::symlink(&new_actual_dir, &sym_dir).await.unwrap(); + tokio::fs::symlink(PathBuf::from("..2021_10_31"), &sym_dir) + .await + .unwrap(); tokio::fs::remove_dir_all(&actual_dir).await.unwrap(); assert_eq!(entry.scan(&logger).await.unwrap(), 3); // file, file-dir, symlink @@ -1057,6 +1076,9 @@ mod tests { fs::create_dir_all(source_dir.path().join("A/B")).unwrap(); fs::write(source_dir.path().join("A/B/1.txt"), "two").unwrap(); + // delay 20 ms between writes to files in order to ensure filesystem timestamps are unique + thread::sleep(Duration::from_millis(20)); + let dest_dir = tempfile::tempdir().unwrap(); let mut entry = Storage::new(protos::Storage { @@ -1074,8 +1096,6 @@ mod tests { // Should copy no files since nothing is changed since last check assert_eq!(entry.scan(&logger).await.unwrap(), 0); - // Should copy 1 file - thread::sleep(Duration::from_secs(1)); fs::write(source_dir.path().join("A/B/1.txt"), "updated").unwrap(); assert_eq!(entry.scan(&logger).await.unwrap(), 1); assert_eq!( @@ -1083,6 +1103,9 @@ mod tests { "updated" ); + // delay 20 ms between writes to files in order to ensure filesystem timestamps are unique + thread::sleep(Duration::from_millis(20)); + // Should copy no new files after copy happened assert_eq!(entry.scan(&logger).await.unwrap(), 0); @@ -1113,7 +1136,9 @@ mod tests { assert_eq!(entry.scan(&logger).await.unwrap(), 1); - thread::sleep(Duration::from_secs(1)); + // delay 20 ms between writes to files in order to ensure filesystem timestamps are unique + thread::sleep(Duration::from_millis(20)); + fs::write(&source_file, "two").unwrap(); assert_eq!(entry.scan(&logger).await.unwrap(), 1); assert_eq!(fs::read_to_string(&dest_file).unwrap(), "two"); @@ -1197,6 +1222,8 @@ mod tests { watcher.mount(&logger).await.unwrap(); assert!(is_mounted(WATCH_MOUNT_POINT_PATH).unwrap()); + thread::sleep(Duration::from_millis(20)); + watcher.cleanup(); assert!(!is_mounted(WATCH_MOUNT_POINT_PATH).unwrap()); } From a0e0e18639e6a4bbaa347bf6360b5940db350ecb Mon Sep 17 00:00:00 2001 From: Eric Ernst Date: Thu, 11 Nov 2021 16:12:24 -0800 Subject: [PATCH 22/66] hypervisors: introduce pkg to unbreak vc/persist dependency Initial hypervisors pkg, with just basic state types defined. Fixes: #2883 Signed-off-by: Eric Ernst --- .../pkg/hypervisors/hypervisor_state.go | 50 +++++++++++++++++++ 1 file changed, 50 insertions(+) create mode 100644 src/runtime/pkg/hypervisors/hypervisor_state.go diff --git a/src/runtime/pkg/hypervisors/hypervisor_state.go b/src/runtime/pkg/hypervisors/hypervisor_state.go new file mode 100644 index 000000000..6dae6222c --- /dev/null +++ b/src/runtime/pkg/hypervisors/hypervisor_state.go @@ -0,0 +1,50 @@ +// Copyright (c) 2019 Huawei Corporation +// +// SPDX-License-Identifier: Apache-2.0 +// + +package hypervisors + +// Bridge is a bridge where devices can be hot plugged +type Bridge struct { + // DeviceAddr contains information about devices plugged and its address in the bridge + DeviceAddr map[uint32]string + + // Type is the type of the bridge (pci, pcie, etc) + Type string + + //ID is used to identify the bridge in the hypervisor + ID string + + // Addr is the PCI/e slot of the bridge + Addr int +} + +// CPUDevice represents a CPU device which was hot-added in a running VM +type CPUDevice struct { + // ID is used to identify this CPU in the hypervisor options. + ID string +} + +type HypervisorState struct { + BlockIndexMap map[int]struct{} + + // Type of hypervisor, E.g. qemu/firecracker/acrn. + Type string + UUID string + // clh sepcific: refer to 'virtcontainers/clh.go:CloudHypervisorState' + APISocket string + + // Belows are qemu specific + // Refs: virtcontainers/qemu.go:QemuState + Bridges []Bridge + // HotpluggedCPUs is the list of CPUs that were hot-added + HotpluggedVCPUs []CPUDevice + + HotpluggedMemory int + VirtiofsdPid int + Pid int + PCIeRootPort int + + HotplugVFIOOnRootBus bool +} From c28e5a780782c473f3b8f86a29c1a729d1e3d5b4 Mon Sep 17 00:00:00 2001 From: Eric Ernst Date: Wed, 17 Nov 2021 15:38:26 -0800 Subject: [PATCH 23/66] acrn: remove dependency on sandbox, persistapi datatypes Today, acrn relies on sandbox level information, as well as a store provided by common parts of the hypervisor. As we cleanup the abstractions within our runtime, we need to ensure that there aren't cross dependencies between the sandbox, the persistence logic and the hypervisor. Ensure that ACRN still compiles, but remove the setSandbox usage as well as persist driver setup. Signed-off-by: Eric Ernst --- src/runtime/virtcontainers/acrn.go | 54 +++++++++++++++--------------- 1 file changed, 27 insertions(+), 27 deletions(-) diff --git a/src/runtime/virtcontainers/acrn.go b/src/runtime/virtcontainers/acrn.go index fafc734a5..db0f6491a 100644 --- a/src/runtime/virtcontainers/acrn.go +++ b/src/runtime/virtcontainers/acrn.go @@ -7,7 +7,6 @@ package virtcontainers import ( "context" - "encoding/json" "fmt" "os" "os/exec" @@ -20,6 +19,7 @@ import ( "github.com/pkg/errors" "github.com/sirupsen/logrus" + hv "github.com/kata-containers/kata-containers/src/runtime/pkg/hypervisors" "github.com/kata-containers/kata-containers/src/runtime/pkg/katautils/katatrace" "github.com/kata-containers/kata-containers/src/runtime/pkg/uuid" "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/device/config" @@ -39,11 +39,13 @@ var acrnTracingTags = map[string]string{ // Since ACRN is using the store in a quite abnormal way, let's first draw it back from store to here +/* // UUIDPathSuffix is the suffix used for uuid storage const ( UUIDPathSuffix = "uuid" uuidFile = "uuid.json" ) +*/ // ACRN currently supports only known UUIDs for security // reasons (FuSa). When launching VM, only these pre-defined @@ -312,7 +314,7 @@ func (a *Acrn) setup(ctx context.Context, id string, hypervisorConfig *Hyperviso // The path might already exist, but in case of VM templating, // we have to create it since the sandbox has not created it yet. - if err = os.MkdirAll(filepath.Join(a.store.RunStoragePath(), id), DirMode); err != nil { + if err = os.MkdirAll(filepath.Join(a.config.RunStorePath, id), DirMode); err != nil { return err } @@ -438,7 +440,7 @@ func (a *Acrn) StartVM(ctx context.Context, timeoutSecs int) error { a.Logger().WithField("default-kernel-parameters", formatted).Debug() } - vmPath := filepath.Join(a.store.RunVMStoragePath(), a.id) + vmPath := filepath.Join(a.config.VMStorePath, a.id) err := os.MkdirAll(vmPath, DirMode) if err != nil { return err @@ -634,7 +636,7 @@ func (a *Acrn) GetVMConsole(ctx context.Context, id string) (string, string, err span, _ := katatrace.Trace(ctx, a.Logger(), "GetVMConsole", acrnTracingTags, map[string]string{"sandbox_id": a.id}) defer span.End() - consoleURL, err := utils.BuildSocketPath(a.store.RunVMStoragePath(), id, acrnConsoleSocket) + consoleURL, err := utils.BuildSocketPath(a.config.VMStorePath, id, acrnConsoleSocket) if err != nil { return consoleProtoUnix, "", err } @@ -698,14 +700,14 @@ func (a *Acrn) toGrpc(ctx context.Context) ([]byte, error) { return nil, errors.New("acrn is not supported by VM cache") } -func (a *Acrn) Save() (s persistapi.HypervisorState) { +func (a *Acrn) Save() (s hv.HypervisorState) { s.Pid = a.state.PID s.Type = string(AcrnHypervisor) s.UUID = a.state.UUID return } -func (a *Acrn) Load(s persistapi.HypervisorState) { +func (a *Acrn) Load(s hv.HypervisorState) { a.state.PID = s.Pid a.state.UUID = s.UUID } @@ -719,7 +721,7 @@ func (a *Acrn) Check() error { } func (a *Acrn) GenerateSocket(id string) (interface{}, error) { - return generateVMSocket(id, a.store.RunVMStoragePath()) + return generateVMSocket(id, a.config.VMStorePath) } // GetACRNUUIDBytes returns UUID bytes that is used for VM creation @@ -782,38 +784,36 @@ func (a *Acrn) GetMaxSupportedACRNVM() (uint8, error) { } func (a *Acrn) storeInfo() error { - relPath := filepath.Join(UUIDPathSuffix, uuidFile) + /* + relPath := filepath.Join(UUIDPathSuffix, uuidFile) - jsonOut, err := json.Marshal(a.info) - if err != nil { - return fmt.Errorf("Could not marshal data: %s", err) - } + jsonOut, err := json.Marshal(a.info) + if err != nil { + return fmt.Errorf("Could not marshal data: %s", err) + } - if err := a.store.GlobalWrite(relPath, jsonOut); err != nil { - return fmt.Errorf("failed to write uuid to file: %v", err) - } + if err := a.store.GlobalWrite(relPath, jsonOut); err != nil { + return fmt.Errorf("failed to write uuid to file: %v", err) + }*/ return nil } func (a *Acrn) loadInfo() error { - relPath := filepath.Join(UUIDPathSuffix, uuidFile) + /* + relPath := filepath.Join(UUIDPathSuffix, uuidFile) + data, err := a.store.GlobalRead(relPath) + if err != nil { + return fmt.Errorf("failed to read uuid from file: %v", err) + } - data, err := a.store.GlobalRead(relPath) - if err != nil { - return fmt.Errorf("failed to read uuid from file: %v", err) - } + if err := json.Unmarshal(data, &a.info); err != nil { + return fmt.Errorf("failed to unmarshal uuid info: %v", err) + }*/ - if err := json.Unmarshal(data, &a.info); err != nil { - return fmt.Errorf("failed to unmarshal uuid info: %v", err) - } return nil } func (a *Acrn) IsRateLimiterBuiltin() bool { return false } - -func (a *Acrn) setSandbox(sandbox *Sandbox) { - a.sandbox = sandbox -} From 34f23de512f4499b7e8edf46159dc16ce19d2196 Mon Sep 17 00:00:00 2001 From: Eric Ernst Date: Tue, 9 Nov 2021 11:31:44 -0800 Subject: [PATCH 24/66] vc: hypervisor: Remove need to get shared address from sandbox Add shared path as part of the hypervisor config Signed-off-by: Eric Ernst --- src/runtime/pkg/katautils/create.go | 3 +++ src/runtime/virtcontainers/clh.go | 2 +- src/runtime/virtcontainers/hypervisor.go | 13 ++++++++----- src/runtime/virtcontainers/kata_agent.go | 2 +- src/runtime/virtcontainers/qemu.go | 2 +- 5 files changed, 14 insertions(+), 8 deletions(-) diff --git a/src/runtime/pkg/katautils/create.go b/src/runtime/pkg/katautils/create.go index fc593c671..dd7056214 100644 --- a/src/runtime/pkg/katautils/create.go +++ b/src/runtime/pkg/katautils/create.go @@ -120,6 +120,9 @@ func CreateSandbox(ctx context.Context, vci vc.VC, ociSpec specs.Spec, runtimeCo return nil, vc.Process{}, err } + // setup shared path in hypervisor config: + sandboxConfig.HypervisorConfig.SharedPath = vc.GetSharePath(containerID) + if err := checkForFIPS(&sandboxConfig); err != nil { return nil, vc.Process{}, err } diff --git a/src/runtime/virtcontainers/clh.go b/src/runtime/virtcontainers/clh.go index 9236c832c..315cb599f 100644 --- a/src/runtime/virtcontainers/clh.go +++ b/src/runtime/virtcontainers/clh.go @@ -226,7 +226,7 @@ func (clh *cloudHypervisor) CreateVM(ctx context.Context, id string, networkNS N clh.Logger().WithField("function", "CreateVM").Info("Sandbox already exist, loading from state") clh.virtiofsd = &virtiofsd{ PID: clh.state.VirtiofsdPID, - sourcePath: filepath.Join(getSharePath(clh.id)), + sourcePath: hypervisorConfig.SharedPath, debug: clh.config.Debug, socketPath: virtiofsdSocketPath, } diff --git a/src/runtime/virtcontainers/hypervisor.go b/src/runtime/virtcontainers/hypervisor.go index 0186c397e..bc22a1711 100644 --- a/src/runtime/virtcontainers/hypervisor.go +++ b/src/runtime/virtcontainers/hypervisor.go @@ -315,13 +315,19 @@ type HypervisorConfig struct { EntropySource string // Shared file system type: - // - virtio-9p (default) - // - virtio-fs + // - virtio-9p + // - virtio-fs (default) SharedFS string + // Path for filesystem sharing + SharedPath string + // VirtioFSDaemon is the virtio-fs vhost-user daemon path VirtioFSDaemon string + // VirtioFSCache cache mode for fs version cache or "none" + VirtioFSCache string + // File based memory backend root directory FileBackedMemRootDir string @@ -342,9 +348,6 @@ type HypervisorConfig struct { // SELinux label for the VM SELinuxProcessLabel string - // VirtioFSCache cache mode for fs version cache or "none" - VirtioFSCache string - // HypervisorPathList is the list of hypervisor paths names allowed in annotations HypervisorPathList []string diff --git a/src/runtime/virtcontainers/kata_agent.go b/src/runtime/virtcontainers/kata_agent.go index 3562c9216..b604917b2 100644 --- a/src/runtime/virtcontainers/kata_agent.go +++ b/src/runtime/virtcontainers/kata_agent.go @@ -162,7 +162,7 @@ var kataHostSharedDir = func() string { // 2. /run/kata-containers/shared/sandboxes/$sbx_id/mounts/ is bind mounted readonly to /run/kata-containers/shared/sandboxes/$sbx_id/shared/, so guest cannot modify it // // 3. host-guest shared files/directories are mounted one-level under /run/kata-containers/shared/sandboxes/$sbx_id/mounts/ and thus present to guest at one level under /run/kata-containers/shared/sandboxes/$sbx_id/shared/ -func getSharePath(id string) string { +func GetSharePath(id string) string { return filepath.Join(kataHostSharedDir(), id, "shared") } diff --git a/src/runtime/virtcontainers/qemu.go b/src/runtime/virtcontainers/qemu.go index 29cd0d173..8db487a80 100644 --- a/src/runtime/virtcontainers/qemu.go +++ b/src/runtime/virtcontainers/qemu.go @@ -655,7 +655,7 @@ func (q *qemu) CreateVM(ctx context.Context, id string, networkNS NetworkNamespa q.virtiofsd = &virtiofsd{ path: q.config.VirtioFSDaemon, - sourcePath: filepath.Join(getSharePath(q.id)), + sourcePath: hypervisorConfig.SharedPath, socketPath: virtiofsdSocketPath, extraArgs: q.config.VirtioFSExtraArgs, debug: q.config.Debug, From 4c2883f7e25615a789ce232236b225e2571248ca Mon Sep 17 00:00:00 2001 From: Eric Ernst Date: Thu, 11 Nov 2021 15:26:10 -0800 Subject: [PATCH 25/66] vc: hypervisor: remove dependency on persist API Today the hypervisor code in vc relies on persist pkg for two things: 1. To get the VM/run store path on the host filesystem, 2. For type definition of the Load/Save functions of the hypervisor interface. For (1), we can simply remove the store interface from the hypervisor config and replace it with just the path, since this is all we really need. When we create a NewHypervisor structure, outside of the hypervisor, we can populate this path. For (2), rather than have the persist pkg define the structure, let's let the hypervisor code (soon to be pkg) define the structure. persist API already needs to call into hypervisor anyway; let's allow us to define the structure. We'll probably want to look at following similar pattern for other parts of vc that we want to make independent of the persist API. In doing this, we started an initial hypervisors pkg, to hold these types (avoid a circular dependency between virtcontainers and persist pkg). Next step will be to remove all other dependencies and move the hypervisor specific code into this pkg, and out of virtcontaienrs. Signed-off-by: Eric Ernst --- src/runtime/cmd/kata-runtime/kata-env_test.go | 4 +- src/runtime/virtcontainers/acrn_test.go | 12 ++++- src/runtime/virtcontainers/clh.go | 21 ++++---- src/runtime/virtcontainers/clh_test.go | 22 ++++++-- src/runtime/virtcontainers/fc.go | 6 +-- src/runtime/virtcontainers/hypervisor.go | 31 +++++------ src/runtime/virtcontainers/kata_agent.go | 8 +-- src/runtime/virtcontainers/kata_agent_test.go | 2 +- src/runtime/virtcontainers/mock_hypervisor.go | 6 +-- src/runtime/virtcontainers/persist.go | 3 +- .../virtcontainers/persist/api/hypervisor.go | 50 ----------------- .../virtcontainers/persist/api/sandbox.go | 6 ++- src/runtime/virtcontainers/qemu.go | 44 +++++++-------- src/runtime/virtcontainers/qemu_test.go | 53 ++++++++++++++----- src/runtime/virtcontainers/sandbox.go | 4 +- .../virtcontainers/virtcontainers_test.go | 2 +- 16 files changed, 133 insertions(+), 141 deletions(-) delete mode 100644 src/runtime/virtcontainers/persist/api/hypervisor.go diff --git a/src/runtime/cmd/kata-runtime/kata-env_test.go b/src/runtime/cmd/kata-runtime/kata-env_test.go index 507fd0325..793c3df42 100644 --- a/src/runtime/cmd/kata-runtime/kata-env_test.go +++ b/src/runtime/cmd/kata-runtime/kata-env_test.go @@ -1052,11 +1052,13 @@ func TestGetHypervisorInfoSocket(t *testing.T) { {vc.QemuHypervisor, false}, } + config.HypervisorConfig.VMStorePath = "/foo" + config.HypervisorConfig.RunStorePath = "/bar" + for i, details := range hypervisors { msg := fmt.Sprintf("hypervisor[%d]: %+v", i, details) config.HypervisorType = details.hType - info, err := getHypervisorInfo(config) assert.NoError(err, msg) diff --git a/src/runtime/virtcontainers/acrn_test.go b/src/runtime/virtcontainers/acrn_test.go index f3d46b7b8..bb19b45a7 100644 --- a/src/runtime/virtcontainers/acrn_test.go +++ b/src/runtime/virtcontainers/acrn_test.go @@ -199,11 +199,15 @@ func TestAcrnGetSandboxConsole(t *testing.T) { assert.NoError(err) a := &Acrn{ - ctx: context.Background(), + ctx: context.Background(), + config: HypervisorConfig{ + VMStorePath: store.RunVMStoragePath(), + RunStorePath: store.RunStoragePath(), + }, store: store, } sandboxID := "testSandboxID" - expected := filepath.Join(a.store.RunVMStoragePath(), sandboxID, consoleSocket) + expected := filepath.Join(store.RunVMStoragePath(), sandboxID, consoleSocket) proto, result, err := a.GetVMConsole(a.ctx, sandboxID) assert.NoError(err) @@ -219,6 +223,10 @@ func TestAcrnCreateVM(t *testing.T) { a := &Acrn{ store: store, + config: HypervisorConfig{ + VMStorePath: store.RunVMStoragePath(), + RunStorePath: store.RunStoragePath(), + }, } sandbox := &Sandbox{ diff --git a/src/runtime/virtcontainers/clh.go b/src/runtime/virtcontainers/clh.go index 315cb599f..e4ca0a006 100644 --- a/src/runtime/virtcontainers/clh.go +++ b/src/runtime/virtcontainers/clh.go @@ -20,12 +20,12 @@ import ( "time" "github.com/containerd/console" - persistapi "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/persist/api" chclient "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/pkg/cloud-hypervisor/client" "github.com/opencontainers/selinux/go-selinux/label" "github.com/pkg/errors" log "github.com/sirupsen/logrus" + hv "github.com/kata-containers/kata-containers/src/runtime/pkg/hypervisors" "github.com/kata-containers/kata-containers/src/runtime/pkg/katautils/katatrace" "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/device/config" "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/types" @@ -157,7 +157,6 @@ func (s *CloudHypervisorState) reset() { } type cloudHypervisor struct { - store persistapi.PersistDriver console console.Console virtiofsd Virtiofsd APIClient clhClient @@ -342,7 +341,7 @@ func (clh *cloudHypervisor) CreateVM(ctx context.Context, id string, networkNS N clh.virtiofsd = &virtiofsd{ path: clh.config.VirtioFSDaemon, - sourcePath: filepath.Join(getSharePath(clh.id)), + sourcePath: filepath.Join(GetSharePath(clh.id)), socketPath: virtiofsdSocketPath, extraArgs: clh.config.VirtioFSExtraArgs, debug: clh.config.Debug, @@ -374,7 +373,7 @@ func (clh *cloudHypervisor) StartVM(ctx context.Context, timeout int) error { clh.Logger().WithField("function", "StartVM").Info("starting Sandbox") - vmPath := filepath.Join(clh.store.RunVMStoragePath(), clh.id) + vmPath := filepath.Join(clh.config.VMStorePath, clh.id) err := os.MkdirAll(vmPath, DirMode) if err != nil { return err @@ -747,7 +746,7 @@ func (clh *cloudHypervisor) toGrpc(ctx context.Context) ([]byte, error) { return nil, errors.New("cloudHypervisor is not supported by VM cache") } -func (clh *cloudHypervisor) Save() (s persistapi.HypervisorState) { +func (clh *cloudHypervisor) Save() (s hv.HypervisorState) { s.Pid = clh.state.PID s.Type = string(ClhHypervisor) s.VirtiofsdPid = clh.state.VirtiofsdPID @@ -755,7 +754,7 @@ func (clh *cloudHypervisor) Save() (s persistapi.HypervisorState) { return } -func (clh *cloudHypervisor) Load(s persistapi.HypervisorState) { +func (clh *cloudHypervisor) Load(s hv.HypervisorState) { clh.state.PID = s.Pid clh.state.VirtiofsdPID = s.VirtiofsdPid clh.state.apiSocket = s.APISocket @@ -893,15 +892,15 @@ func (clh *cloudHypervisor) GenerateSocket(id string) (interface{}, error) { } func (clh *cloudHypervisor) virtioFsSocketPath(id string) (string, error) { - return utils.BuildSocketPath(clh.store.RunVMStoragePath(), id, virtioFsSocket) + return utils.BuildSocketPath(clh.config.VMStorePath, id, virtioFsSocket) } func (clh *cloudHypervisor) vsockSocketPath(id string) (string, error) { - return utils.BuildSocketPath(clh.store.RunVMStoragePath(), id, clhSocket) + return utils.BuildSocketPath(clh.config.VMStorePath, id, clhSocket) } func (clh *cloudHypervisor) apiSocketPath(id string) (string, error) { - return utils.BuildSocketPath(clh.store.RunVMStoragePath(), id, clhAPISocket) + return utils.BuildSocketPath(clh.config.VMStorePath, id, clhAPISocket) } func (clh *cloudHypervisor) waitVMM(timeout uint) error { @@ -1213,7 +1212,7 @@ func (clh *cloudHypervisor) cleanupVM(force bool) error { } // Cleanup vm path - dir := filepath.Join(clh.store.RunVMStoragePath(), clh.id) + dir := filepath.Join(clh.config.VMStorePath, clh.id) // If it's a symlink, remove both dir and the target. link, err := filepath.EvalSymlinks(dir) @@ -1242,7 +1241,7 @@ func (clh *cloudHypervisor) cleanupVM(force bool) error { } if clh.config.VMid != "" { - dir = filepath.Join(clh.store.RunStoragePath(), clh.config.VMid) + dir = filepath.Join(clh.config.VMStorePath, clh.config.VMid) if err := os.RemoveAll(dir); err != nil { if !force { return err diff --git a/src/runtime/virtcontainers/clh_test.go b/src/runtime/virtcontainers/clh_test.go index 54718dd57..b0de2ae9b 100644 --- a/src/runtime/virtcontainers/clh_test.go +++ b/src/runtime/virtcontainers/clh_test.go @@ -203,7 +203,10 @@ func TestCloudHypervisorCleanupVM(t *testing.T) { assert.NoError(err, "persist.GetDriver() unexpected error") clh := &cloudHypervisor{ - store: store, + config: HypervisorConfig{ + VMStorePath: store.RunVMStoragePath(), + RunStorePath: store.RunStoragePath(), + }, } err = clh.cleanupVM(true) @@ -214,7 +217,7 @@ func TestCloudHypervisorCleanupVM(t *testing.T) { err = clh.cleanupVM(true) assert.NoError(err, "persist.GetDriver() unexpected error") - dir := filepath.Join(clh.store.RunVMStoragePath(), clh.id) + dir := filepath.Join(store.RunVMStoragePath(), clh.id) os.MkdirAll(dir, os.ModePerm) err = clh.cleanupVM(false) @@ -235,9 +238,11 @@ func TestClhCreateVM(t *testing.T) { store, err := persist.GetDriver() assert.NoError(err) + clhConfig.VMStorePath = store.RunVMStoragePath() + clhConfig.RunStorePath = store.RunStoragePath() + clh := &cloudHypervisor{ config: clhConfig, - store: store, } sandbox := &Sandbox{ @@ -261,11 +266,13 @@ func TestClooudHypervisorStartSandbox(t *testing.T) { store, err := persist.GetDriver() assert.NoError(err) + clhConfig.VMStorePath = store.RunVMStoragePath() + clhConfig.RunStorePath = store.RunStoragePath() + clh := &cloudHypervisor{ config: clhConfig, APIClient: &clhClientMock{}, virtiofsd: &virtiofsdMock{}, - store: store, } err = clh.StartVM(context.Background(), 10) @@ -379,6 +386,11 @@ func TestClhGenerateSocket(t *testing.T) { clh, ok := hypervisor.(*cloudHypervisor) assert.True(ok) + clh.config = HypervisorConfig{ + VMStorePath: "/foo", + RunStorePath: "/bar", + } + clh.addVSock(1, "path") s, err := clh.GenerateSocket("c") @@ -391,7 +403,7 @@ func TestClhGenerateSocket(t *testing.T) { assert.NotEmpty(hvsock.UdsPath) // Path must be absolute - assert.True(strings.HasPrefix(hvsock.UdsPath, "/")) + assert.True(strings.HasPrefix(hvsock.UdsPath, "/"), "failed: socket path: %s", hvsock.UdsPath) assert.NotZero(hvsock.Port) } diff --git a/src/runtime/virtcontainers/fc.go b/src/runtime/virtcontainers/fc.go index 1b4c055cc..fa4e56ffa 100644 --- a/src/runtime/virtcontainers/fc.go +++ b/src/runtime/virtcontainers/fc.go @@ -22,9 +22,9 @@ import ( "syscall" "time" + hv "github.com/kata-containers/kata-containers/src/runtime/pkg/hypervisors" "github.com/kata-containers/kata-containers/src/runtime/pkg/katautils/katatrace" "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/device/config" - persistapi "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/persist/api" "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/pkg/firecracker/client" models "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/pkg/firecracker/client/models" ops "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/pkg/firecracker/client/operations" @@ -1226,13 +1226,13 @@ func (fc *firecracker) toGrpc(ctx context.Context) ([]byte, error) { return nil, errors.New("firecracker is not supported by VM cache") } -func (fc *firecracker) Save() (s persistapi.HypervisorState) { +func (fc *firecracker) Save() (s hv.HypervisorState) { s.Pid = fc.info.PID s.Type = string(FirecrackerHypervisor) return } -func (fc *firecracker) Load(s persistapi.HypervisorState) { +func (fc *firecracker) Load(s hv.HypervisorState) { fc.info.PID = s.Pid } diff --git a/src/runtime/virtcontainers/hypervisor.go b/src/runtime/virtcontainers/hypervisor.go index bc22a1711..f446a078b 100644 --- a/src/runtime/virtcontainers/hypervisor.go +++ b/src/runtime/virtcontainers/hypervisor.go @@ -14,9 +14,8 @@ import ( "strconv" "strings" + hv "github.com/kata-containers/kata-containers/src/runtime/pkg/hypervisors" "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/device/config" - "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/persist" - persistapi "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/persist/api" "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/types" "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/utils" ) @@ -185,28 +184,18 @@ func (hType *HypervisorType) String() string { } } -// NewHypervisor returns an hypervisor from and hypervisor type. +// NewHypervisor returns an hypervisor from a hypervisor type. func NewHypervisor(hType HypervisorType) (Hypervisor, error) { - store, err := persist.GetDriver() - if err != nil { - return nil, err - } switch hType { case QemuHypervisor: - return &qemu{ - store: store, - }, nil + return &qemu{}, nil case FirecrackerHypervisor: return &firecracker{}, nil case AcrnHypervisor: - return &Acrn{ - store: store, - }, nil + return &Acrn{}, nil case ClhHypervisor: - return &cloudHypervisor{ - store: store, - }, nil + return &cloudHypervisor{}, nil case MockHypervisor: return &mockHypervisor{}, nil default: @@ -345,6 +334,12 @@ type HypervisorConfig struct { // VMid is "" if the hypervisor is not created by the factory. VMid string + // VMStorePath is the location on disk where VM information will persist + VMStorePath string + + // VMStorePath is the location on disk where runtime information will persist + RunStorePath string + // SELinux label for the VM SELinuxProcessLabel string @@ -934,8 +929,8 @@ type Hypervisor interface { toGrpc(ctx context.Context) ([]byte, error) Check() error - Save() persistapi.HypervisorState - Load(persistapi.HypervisorState) + Save() hv.HypervisorState + Load(hv.HypervisorState) // generate the socket to communicate the host and guest GenerateSocket(id string) (interface{}, error) diff --git a/src/runtime/virtcontainers/kata_agent.go b/src/runtime/virtcontainers/kata_agent.go index b604917b2..6cecbd452 100644 --- a/src/runtime/virtcontainers/kata_agent.go +++ b/src/runtime/virtcontainers/kata_agent.go @@ -356,7 +356,7 @@ func (k *kataAgent) setupSandboxBindMounts(ctx context.Context, sandbox *Sandbox // Create subdirectory in host shared path for sandbox mounts sandboxMountDir := filepath.Join(getMountPath(sandbox.id), sandboxMountsDir) - sandboxShareDir := filepath.Join(getSharePath(sandbox.id), sandboxMountsDir) + sandboxShareDir := filepath.Join(GetSharePath(sandbox.id), sandboxMountsDir) if err := os.MkdirAll(sandboxMountDir, DirMode); err != nil { return fmt.Errorf("Creating sandbox shared mount directory: %v: %w", sandboxMountDir, err) } @@ -473,7 +473,7 @@ func (k *kataAgent) setupSharedPath(ctx context.Context, sandbox *Sandbox) (err defer span.End() // create shared path structure - sharePath := getSharePath(sandbox.id) + sharePath := GetSharePath(sandbox.id) mountPath := getMountPath(sandbox.id) if err := os.MkdirAll(sharePath, sharedDirMode); err != nil { return err @@ -509,7 +509,7 @@ func (k *kataAgent) createSandbox(ctx context.Context, sandbox *Sandbox) error { if err := k.setupSharedPath(ctx, sandbox); err != nil { return err } - return k.configure(ctx, sandbox.hypervisor, sandbox.id, getSharePath(sandbox.id), sandbox.config.AgentConfig) + return k.configure(ctx, sandbox.hypervisor, sandbox.id, GetSharePath(sandbox.id), sandbox.config.AgentConfig) } func cmdToKataProcess(cmd types.Cmd) (process *grpc.Process, err error) { @@ -2198,7 +2198,7 @@ func (k *kataAgent) cleanup(ctx context.Context, s *Sandbox) { } // Unmount shared path - path := getSharePath(s.id) + path := GetSharePath(s.id) k.Logger().WithField("path", path).Infof("Cleanup agent") if err := syscall.Unmount(path, syscall.MNT_DETACH|UmountNoFollow); err != nil { k.Logger().WithError(err).Errorf("failed to unmount vm share path %s", path) diff --git a/src/runtime/virtcontainers/kata_agent_test.go b/src/runtime/virtcontainers/kata_agent_test.go index 6e329919e..48c0f6a73 100644 --- a/src/runtime/virtcontainers/kata_agent_test.go +++ b/src/runtime/virtcontainers/kata_agent_test.go @@ -1158,7 +1158,7 @@ func TestSandboxBindMount(t *testing.T) { assert.Nil(err) defer os.RemoveAll(dir) - sharePath := getSharePath(sandbox.id) + sharePath := GetSharePath(sandbox.id) mountPath := getMountPath(sandbox.id) err = os.MkdirAll(sharePath, DirMode) diff --git a/src/runtime/virtcontainers/mock_hypervisor.go b/src/runtime/virtcontainers/mock_hypervisor.go index 2c132f0ba..dc4b11ad6 100644 --- a/src/runtime/virtcontainers/mock_hypervisor.go +++ b/src/runtime/virtcontainers/mock_hypervisor.go @@ -10,7 +10,7 @@ import ( "errors" "os" - persistapi "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/persist/api" + hv "github.com/kata-containers/kata-containers/src/runtime/pkg/hypervisors" "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/types" ) @@ -130,11 +130,11 @@ func (m *mockHypervisor) toGrpc(ctx context.Context) ([]byte, error) { return nil, errors.New("mockHypervisor is not supported by VM cache") } -func (m *mockHypervisor) Save() (s persistapi.HypervisorState) { +func (m *mockHypervisor) Save() (s hv.HypervisorState) { return } -func (m *mockHypervisor) Load(s persistapi.HypervisorState) {} +func (m *mockHypervisor) Load(s hv.HypervisorState) {} func (m *mockHypervisor) Check() error { return nil diff --git a/src/runtime/virtcontainers/persist.go b/src/runtime/virtcontainers/persist.go index b8fc2c8ab..3fe45351d 100644 --- a/src/runtime/virtcontainers/persist.go +++ b/src/runtime/virtcontainers/persist.go @@ -8,6 +8,7 @@ package virtcontainers import ( "errors" + hv "github.com/kata-containers/kata-containers/src/runtime/pkg/hypervisors" "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/device/api" exp "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/experimental" "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/persist" @@ -315,7 +316,7 @@ func (c *Container) loadContState(cs persistapi.ContainerState) { } } -func (s *Sandbox) loadHypervisor(hs persistapi.HypervisorState) { +func (s *Sandbox) loadHypervisor(hs hv.HypervisorState) { s.hypervisor.Load(hs) } diff --git a/src/runtime/virtcontainers/persist/api/hypervisor.go b/src/runtime/virtcontainers/persist/api/hypervisor.go deleted file mode 100644 index b2d41000f..000000000 --- a/src/runtime/virtcontainers/persist/api/hypervisor.go +++ /dev/null @@ -1,50 +0,0 @@ -// Copyright (c) 2019 Huawei Corporation -// -// SPDX-License-Identifier: Apache-2.0 -// - -package persistapi - -// Bridge is a bridge where devices can be hot plugged -type Bridge struct { - // DeviceAddr contains information about devices plugged and its address in the bridge - DeviceAddr map[uint32]string - - // Type is the type of the bridge (pci, pcie, etc) - Type string - - //ID is used to identify the bridge in the hypervisor - ID string - - // Addr is the PCI/e slot of the bridge - Addr int -} - -// CPUDevice represents a CPU device which was hot-added in a running VM -type CPUDevice struct { - // ID is used to identify this CPU in the hypervisor options. - ID string -} - -type HypervisorState struct { - BlockIndexMap map[int]struct{} - - // Type of hypervisor, E.g. qemu/firecracker/acrn. - Type string - UUID string - // clh sepcific: refer to 'virtcontainers/clh.go:CloudHypervisorState' - APISocket string - - // Belows are qemu specific - // Refs: virtcontainers/qemu.go:QemuState - Bridges []Bridge - // HotpluggedCPUs is the list of CPUs that were hot-added - HotpluggedVCPUs []CPUDevice - - HotpluggedMemory int - VirtiofsdPid int - Pid int - PCIeRootPort int - - HotplugVFIOOnRootBus bool -} diff --git a/src/runtime/virtcontainers/persist/api/sandbox.go b/src/runtime/virtcontainers/persist/api/sandbox.go index 1398cc20f..09196637c 100644 --- a/src/runtime/virtcontainers/persist/api/sandbox.go +++ b/src/runtime/virtcontainers/persist/api/sandbox.go @@ -6,6 +6,10 @@ package persistapi +import ( + hv "github.com/kata-containers/kata-containers/src/runtime/pkg/hypervisors" +) + // ============= sandbox level resources ============= // AgentState save agent state data @@ -38,7 +42,7 @@ type SandboxState struct { OverheadCgroupPath string // HypervisorState saves hypervisor specific data - HypervisorState HypervisorState + HypervisorState hv.HypervisorState // AgentState saves state data of agent AgentState AgentState diff --git a/src/runtime/virtcontainers/qemu.go b/src/runtime/virtcontainers/qemu.go index 8db487a80..1c1073b24 100644 --- a/src/runtime/virtcontainers/qemu.go +++ b/src/runtime/virtcontainers/qemu.go @@ -31,11 +31,11 @@ import ( "github.com/sirupsen/logrus" "golang.org/x/sys/unix" + hv "github.com/kata-containers/kata-containers/src/runtime/pkg/hypervisors" "github.com/kata-containers/kata-containers/src/runtime/pkg/katautils/katatrace" pkgUtils "github.com/kata-containers/kata-containers/src/runtime/pkg/utils" "github.com/kata-containers/kata-containers/src/runtime/pkg/uuid" "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/device/config" - persistapi "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/persist/api" "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/types" vcTypes "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/types" "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/utils" @@ -67,18 +67,12 @@ type qmpChannel struct { sync.Mutex } -// CPUDevice represents a CPU device which was hot-added in a running VM -type CPUDevice struct { - // ID is used to identify this CPU in the hypervisor options. - ID string -} - // QemuState keeps Qemu's state type QemuState struct { UUID string Bridges []types.Bridge // HotpluggedCPUs is the list of CPUs that were hot-added - HotpluggedVCPUs []CPUDevice + HotpluggedVCPUs []hv.CPUDevice HotpluggedMemory int VirtiofsdPid int PCIeRootPort int @@ -92,8 +86,6 @@ type qemu struct { virtiofsd Virtiofsd - store persistapi.PersistDriver - ctx context.Context // fds is a list of file descriptors inherited by QEMU process @@ -276,7 +268,7 @@ func (q *qemu) setup(ctx context.Context, id string, hypervisorConfig *Hyperviso // The path might already exist, but in case of VM templating, // we have to create it since the sandbox has not created it yet. - if err = utils.MkdirAllWithInheritedOwner(filepath.Join(q.store.RunStoragePath(), id), DirMode); err != nil { + if err = utils.MkdirAllWithInheritedOwner(filepath.Join(q.config.RunStorePath, id), DirMode); err != nil { return err } } @@ -331,7 +323,7 @@ func (q *qemu) memoryTopology() (govmmQemu.Memory, error) { } func (q *qemu) qmpSocketPath(id string) (string, error) { - return utils.BuildSocketPath(q.store.RunVMStoragePath(), id, qmpSocket) + return utils.BuildSocketPath(q.config.VMStorePath, id, qmpSocket) } func (q *qemu) getQemuMachine() (govmmQemu.Machine, error) { @@ -618,7 +610,7 @@ func (q *qemu) CreateVM(ctx context.Context, id string, networkNS NetworkNamespa GlobalParam: "kvm-pit.lost_tick_policy=discard", Bios: firmwarePath, PFlash: pflash, - PidFile: filepath.Join(q.store.RunVMStoragePath(), q.id, "pid"), + PidFile: filepath.Join(q.config.VMStorePath, q.id, "pid"), } qemuConfig.Devices, qemuConfig.Bios, err = q.arch.appendProtectionDevice(qemuConfig.Devices, firmwarePath) @@ -666,7 +658,7 @@ func (q *qemu) CreateVM(ctx context.Context, id string, networkNS NetworkNamespa } func (q *qemu) vhostFSSocketPath(id string) (string, error) { - return utils.BuildSocketPath(q.store.RunVMStoragePath(), id, vhostFSSocket) + return utils.BuildSocketPath(q.config.VMStorePath, id, vhostFSSocket) } func (q *qemu) setupVirtiofsd(ctx context.Context) (err error) { @@ -795,7 +787,7 @@ func (q *qemu) StartVM(ctx context.Context, timeout int) error { q.fds = []*os.File{} }() - vmPath := filepath.Join(q.store.RunVMStoragePath(), q.id) + vmPath := filepath.Join(q.config.VMStorePath, q.id) err := utils.MkdirAllWithInheritedOwner(vmPath, DirMode) if err != nil { return err @@ -1002,7 +994,7 @@ func (q *qemu) StopVM(ctx context.Context, waitOnly bool) error { func (q *qemu) cleanupVM() error { // Cleanup vm path - dir := filepath.Join(q.store.RunVMStoragePath(), q.id) + dir := filepath.Join(q.config.VMStorePath, q.id) // If it's a symlink, remove both dir and the target. // This can happen when vm template links a sandbox to a vm. @@ -1023,7 +1015,7 @@ func (q *qemu) cleanupVM() error { } if q.config.VMid != "" { - dir = filepath.Join(q.store.RunStoragePath(), q.config.VMid) + dir = filepath.Join(q.config.RunStorePath, q.config.VMid) if err := os.RemoveAll(dir); err != nil { q.Logger().WithError(err).WithField("path", dir).Warnf("failed to remove vm path") } @@ -1149,7 +1141,7 @@ func (q *qemu) dumpSandboxMetaInfo(dumpSavePath string) { dumpStatePath := filepath.Join(dumpSavePath, "state") // copy state from /run/vc/sbs to memory dump directory - statePath := filepath.Join(q.store.RunStoragePath(), q.id) + statePath := filepath.Join(q.config.RunStorePath, q.id) command := []string{"/bin/cp", "-ar", statePath, dumpStatePath} q.Logger().WithField("command", command).Info("try to Save sandbox state") if output, err := pkgUtils.RunCommandFull(command, true); err != nil { @@ -1822,7 +1814,7 @@ func (q *qemu) hotplugAddCPUs(amount uint32) (uint32, error) { } // a new vCPU was added, update list of hotplugged vCPUs and Check if all vCPUs were added - q.state.HotpluggedVCPUs = append(q.state.HotpluggedVCPUs, CPUDevice{cpuID}) + q.state.HotpluggedVCPUs = append(q.state.HotpluggedVCPUs, hv.CPUDevice{ID: cpuID}) hotpluggedVCPUs++ if hotpluggedVCPUs == amount { // All vCPUs were hotplugged @@ -2030,7 +2022,7 @@ func (q *qemu) GetVMConsole(ctx context.Context, id string) (string, string, err span, _ := katatrace.Trace(ctx, q.Logger(), "GetVMConsole", qemuTracingTags, map[string]string{"sandbox_id": q.id}) defer span.End() - consoleURL, err := utils.BuildSocketPath(q.store.RunVMStoragePath(), id, consoleSocket) + consoleURL, err := utils.BuildSocketPath(q.config.VMStorePath, id, consoleSocket) if err != nil { return consoleProtoUnix, "", err } @@ -2469,7 +2461,7 @@ func (q *qemu) toGrpc(ctx context.Context) ([]byte, error) { return json.Marshal(&qp) } -func (q *qemu) Save() (s persistapi.HypervisorState) { +func (q *qemu) Save() (s hv.HypervisorState) { // If QEMU isn't even running, there isn't any state to Save if q.stopped { @@ -2488,7 +2480,7 @@ func (q *qemu) Save() (s persistapi.HypervisorState) { s.PCIeRootPort = q.state.PCIeRootPort for _, bridge := range q.arch.getBridges() { - s.Bridges = append(s.Bridges, persistapi.Bridge{ + s.Bridges = append(s.Bridges, hv.Bridge{ DeviceAddr: bridge.Devices, Type: string(bridge.Type), ID: bridge.ID, @@ -2497,14 +2489,14 @@ func (q *qemu) Save() (s persistapi.HypervisorState) { } for _, cpu := range q.state.HotpluggedVCPUs { - s.HotpluggedVCPUs = append(s.HotpluggedVCPUs, persistapi.CPUDevice{ + s.HotpluggedVCPUs = append(s.HotpluggedVCPUs, hv.CPUDevice{ ID: cpu.ID, }) } return } -func (q *qemu) Load(s persistapi.HypervisorState) { +func (q *qemu) Load(s hv.HypervisorState) { q.state.UUID = s.UUID q.state.HotpluggedMemory = s.HotpluggedMemory q.state.HotplugVFIOOnRootBus = s.HotplugVFIOOnRootBus @@ -2516,7 +2508,7 @@ func (q *qemu) Load(s persistapi.HypervisorState) { } for _, cpu := range s.HotpluggedVCPUs { - q.state.HotpluggedVCPUs = append(q.state.HotpluggedVCPUs, CPUDevice{ + q.state.HotpluggedVCPUs = append(q.state.HotpluggedVCPUs, hv.CPUDevice{ ID: cpu.ID, }) } @@ -2543,7 +2535,7 @@ func (q *qemu) Check() error { } func (q *qemu) GenerateSocket(id string) (interface{}, error) { - return generateVMSocket(id, q.store.RunVMStoragePath()) + return generateVMSocket(id, q.config.VMStorePath) } func (q *qemu) IsRateLimiterBuiltin() bool { diff --git a/src/runtime/virtcontainers/qemu_test.go b/src/runtime/virtcontainers/qemu_test.go index deeb482dc..9104a6892 100644 --- a/src/runtime/virtcontainers/qemu_test.go +++ b/src/runtime/virtcontainers/qemu_test.go @@ -78,7 +78,10 @@ func TestQemuCreateVM(t *testing.T) { store, err := persist.GetDriver() assert.NoError(err) q := &qemu{ - store: store, + config: HypervisorConfig{ + VMStorePath: store.RunVMStoragePath(), + RunStorePath: store.RunStoragePath(), + }, } sandbox := &Sandbox{ ctx: context.Background(), @@ -94,7 +97,7 @@ func TestQemuCreateVM(t *testing.T) { assert.NoError(err) // Create parent dir path for hypervisor.json - parentDir := filepath.Join(q.store.RunStoragePath(), sandbox.id) + parentDir := filepath.Join(store.RunStoragePath(), sandbox.id) assert.NoError(os.MkdirAll(parentDir, DirMode)) err = q.CreateVM(context.Background(), sandbox.id, NetworkNamespace{}, &sandbox.config.HypervisorConfig) @@ -110,7 +113,10 @@ func TestQemuCreateVMMissingParentDirFail(t *testing.T) { store, err := persist.GetDriver() assert.NoError(err) q := &qemu{ - store: store, + config: HypervisorConfig{ + VMStorePath: store.RunVMStoragePath(), + RunStorePath: store.RunStoragePath(), + }, } sandbox := &Sandbox{ ctx: context.Background(), @@ -126,7 +132,7 @@ func TestQemuCreateVMMissingParentDirFail(t *testing.T) { assert.NoError(err) // Ensure parent dir path for hypervisor.json does not exist. - parentDir := filepath.Join(q.store.RunStoragePath(), sandbox.id) + parentDir := filepath.Join(store.RunStoragePath(), sandbox.id) assert.NoError(os.RemoveAll(parentDir)) err = q.CreateVM(context.Background(), sandbox.id, NetworkNamespace{}, &sandbox.config.HypervisorConfig) @@ -192,7 +198,10 @@ func TestQemuKnobs(t *testing.T) { assert.NoError(err) q := &qemu{ - store: sandbox.store, + config: HypervisorConfig{ + VMStorePath: sandbox.store.RunVMStoragePath(), + RunStorePath: sandbox.store.RunStoragePath(), + }, } err = q.CreateVM(context.Background(), sandbox.id, NetworkNamespace{}, &sandbox.config.HypervisorConfig) assert.NoError(err) @@ -325,11 +334,14 @@ func TestQemuGetSandboxConsole(t *testing.T) { store, err := persist.GetDriver() assert.NoError(err) q := &qemu{ - ctx: context.Background(), - store: store, + ctx: context.Background(), + config: HypervisorConfig{ + VMStorePath: store.RunVMStoragePath(), + RunStorePath: store.RunStoragePath(), + }, } sandboxID := "testSandboxID" - expected := filepath.Join(q.store.RunVMStoragePath(), sandboxID, consoleSocket) + expected := filepath.Join(store.RunVMStoragePath(), sandboxID, consoleSocket) proto, result, err := q.GetVMConsole(q.ctx, sandboxID) assert.NoError(err) @@ -460,7 +472,10 @@ func TestQemuFileBackedMem(t *testing.T) { assert.NoError(err) q := &qemu{ - store: sandbox.store, + config: HypervisorConfig{ + VMStorePath: sandbox.store.RunVMStoragePath(), + RunStorePath: sandbox.store.RunStoragePath(), + }, } sandbox.config.HypervisorConfig.SharedFS = config.VirtioFS err = q.CreateVM(context.Background(), sandbox.id, NetworkNamespace{}, &sandbox.config.HypervisorConfig) @@ -475,7 +490,10 @@ func TestQemuFileBackedMem(t *testing.T) { assert.NoError(err) q = &qemu{ - store: sandbox.store, + config: HypervisorConfig{ + VMStorePath: sandbox.store.RunVMStoragePath(), + RunStorePath: sandbox.store.RunStoragePath(), + }, } sandbox.config.HypervisorConfig.BootToBeTemplate = true sandbox.config.HypervisorConfig.SharedFS = config.VirtioFS @@ -491,7 +509,10 @@ func TestQemuFileBackedMem(t *testing.T) { assert.NoError(err) q = &qemu{ - store: sandbox.store, + config: HypervisorConfig{ + VMStorePath: sandbox.store.RunVMStoragePath(), + RunStorePath: sandbox.store.RunStoragePath(), + }, } sandbox.config.HypervisorConfig.FileBackedMemRootDir = "/tmp/xyzabc" err = q.CreateVM(context.Background(), sandbox.id, NetworkNamespace{}, &sandbox.config.HypervisorConfig) @@ -505,7 +526,10 @@ func TestQemuFileBackedMem(t *testing.T) { assert.NoError(err) q = &qemu{ - store: sandbox.store, + config: HypervisorConfig{ + VMStorePath: sandbox.store.RunVMStoragePath(), + RunStorePath: sandbox.store.RunStoragePath(), + }, } sandbox.config.HypervisorConfig.EnableVhostUserStore = true sandbox.config.HypervisorConfig.HugePages = true @@ -518,7 +542,10 @@ func TestQemuFileBackedMem(t *testing.T) { assert.NoError(err) q = &qemu{ - store: sandbox.store, + config: HypervisorConfig{ + VMStorePath: sandbox.store.RunVMStoragePath(), + RunStorePath: sandbox.store.RunStoragePath(), + }, } sandbox.config.HypervisorConfig.EnableVhostUserStore = true sandbox.config.HypervisorConfig.HugePages = false diff --git a/src/runtime/virtcontainers/sandbox.go b/src/runtime/virtcontainers/sandbox.go index c9e07947b..207ba1d81 100644 --- a/src/runtime/virtcontainers/sandbox.go +++ b/src/runtime/virtcontainers/sandbox.go @@ -530,7 +530,6 @@ func newSandbox(ctx context.Context, sandboxConfig SandboxConfig, factory Factor if s.store, err = persist.GetDriver(); err != nil || s.store == nil { return nil, fmt.Errorf("failed to get fs persist driver: %v", err) } - defer func() { if retErr != nil { s.Logger().WithError(retErr).Error("Create new sandbox failed") @@ -538,6 +537,9 @@ func newSandbox(ctx context.Context, sandboxConfig SandboxConfig, factory Factor } }() + sandboxConfig.HypervisorConfig.VMStorePath = s.store.RunVMStoragePath() + sandboxConfig.HypervisorConfig.RunStorePath = s.store.RunStoragePath() + spec := s.GetPatchedOCISpec() if spec != nil && spec.Process.SelinuxLabel != "" { sandboxConfig.HypervisorConfig.SELinuxProcessLabel = spec.Process.SelinuxLabel diff --git a/src/runtime/virtcontainers/virtcontainers_test.go b/src/runtime/virtcontainers/virtcontainers_test.go index 56b27b1c5..5772dd60a 100644 --- a/src/runtime/virtcontainers/virtcontainers_test.go +++ b/src/runtime/virtcontainers/virtcontainers_test.go @@ -60,7 +60,7 @@ var testHyperstartTtySocket = "" func cleanUp() { os.RemoveAll(fs.MockRunStoragePath()) os.RemoveAll(fs.MockRunVMStoragePath()) - syscall.Unmount(getSharePath(testSandboxID), syscall.MNT_DETACH|UmountNoFollow) + syscall.Unmount(GetSharePath(testSandboxID), syscall.MNT_DETACH|UmountNoFollow) os.RemoveAll(testDir) os.MkdirAll(testDir, DirMode) From 2227c46c25530ce12916bf3d7ac9e6ba46fad003 Mon Sep 17 00:00:00 2001 From: Eric Ernst Date: Thu, 11 Nov 2021 16:46:14 -0800 Subject: [PATCH 26/66] vc: hypervisor: use our own logger This'll end up moving to hypervisors pkg, but let's stop using virtLog, instead introduce hvLogger. Fixes: #2884 Signed-off-by: Eric Ernst --- src/runtime/virtcontainers/api.go | 2 +- src/runtime/virtcontainers/clh.go | 2 +- src/runtime/virtcontainers/hypervisor.go | 20 ++++++++++++++------ src/runtime/virtcontainers/qemu.go | 4 ++-- src/runtime/virtcontainers/qemu_amd64.go | 4 ++-- src/runtime/virtcontainers/qemu_arch_base.go | 2 +- src/runtime/virtcontainers/qemu_arm64.go | 2 +- src/runtime/virtcontainers/qemu_ppc64le.go | 4 ++-- src/runtime/virtcontainers/qemu_s390x.go | 2 +- src/runtime/virtcontainers/virtiofsd.go | 2 +- 10 files changed, 26 insertions(+), 18 deletions(-) diff --git a/src/runtime/virtcontainers/api.go b/src/runtime/virtcontainers/api.go index 67e7608ac..14cfbb109 100644 --- a/src/runtime/virtcontainers/api.go +++ b/src/runtime/virtcontainers/api.go @@ -35,7 +35,7 @@ var virtLog = logrus.WithField("source", "virtcontainers") func SetLogger(ctx context.Context, logger *logrus.Entry) { fields := virtLog.Data virtLog = logger.WithFields(fields) - + SetHypervisorLogger(virtLog) // TODO: this will move to hypervisors pkg deviceApi.SetLogger(virtLog) compatoci.SetLogger(virtLog) deviceConfig.SetLogger(virtLog) diff --git a/src/runtime/virtcontainers/clh.go b/src/runtime/virtcontainers/clh.go index e4ca0a006..7fad500f5 100644 --- a/src/runtime/virtcontainers/clh.go +++ b/src/runtime/virtcontainers/clh.go @@ -813,7 +813,7 @@ func (clh *cloudHypervisor) AddDevice(ctx context.Context, devInfo interface{}, //########################################################################### func (clh *cloudHypervisor) Logger() *log.Entry { - return virtLog.WithField("subsystem", "cloudHypervisor") + return hvLogger.WithField("subsystem", "cloudHypervisor") } // Adds all capabilities supported by cloudHypervisor implementation of hypervisor interface diff --git a/src/runtime/virtcontainers/hypervisor.go b/src/runtime/virtcontainers/hypervisor.go index f446a078b..06a0385c9 100644 --- a/src/runtime/virtcontainers/hypervisor.go +++ b/src/runtime/virtcontainers/hypervisor.go @@ -18,6 +18,8 @@ import ( "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/device/config" "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/types" "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/utils" + + "github.com/sirupsen/logrus" ) // HypervisorType describes an hypervisor type. @@ -45,14 +47,10 @@ const ( // MockHypervisor is a mock hypervisor for testing purposes MockHypervisor HypervisorType = "mock" -) -const ( procMemInfo = "/proc/meminfo" procCPUInfo = "/proc/cpuinfo" -) -const ( defaultVCPUs = 1 // 2 GiB defaultMemSzMiB = 2048 @@ -73,6 +71,10 @@ const ( MinHypervisorMemory = 256 ) +var ( + hvLogger = logrus.WithField("source", "virtcontainers/hypervisor") +) + // In some architectures the maximum number of vCPUs depends on the number of physical cores. var defaultMaxQemuVCPUs = MaxQemuVCPUs() @@ -143,6 +145,12 @@ type MemoryDevice struct { Probe bool } +// SetHypervisorLogger sets up a logger for the hypervisor part of this pkg +func SetHypervisorLogger(logger *logrus.Entry) { + fields := hvLogger.Data + hvLogger = logger.WithFields(fields) +} + // Set sets an hypervisor type based on the input string. func (hType *HypervisorType) Set(value string) error { switch value { @@ -604,7 +612,7 @@ func (conf *HypervisorConfig) AddCustomAsset(a *types.Asset) error { return fmt.Errorf("Invalid %s at %s", a.Type(), a.Path()) } - virtLog.Debugf("Using custom %v asset %s", a.Type(), a.Path()) + hvLogger.Debugf("Using custom %v asset %s", a.Type(), a.Path()) if conf.customAssets == nil { conf.customAssets = make(map[types.AssetType]*types.Asset) @@ -872,7 +880,7 @@ func RunningOnVMM(cpuInfoPath string) (bool, error) { return flags["hypervisor"], nil } - virtLog.WithField("arch", runtime.GOARCH).Info("Unable to know if the system is running inside a VM") + hvLogger.WithField("arch", runtime.GOARCH).Info("Unable to know if the system is running inside a VM") return false, nil } diff --git a/src/runtime/virtcontainers/qemu.go b/src/runtime/virtcontainers/qemu.go index 1c1073b24..d0b3bd5c5 100644 --- a/src/runtime/virtcontainers/qemu.go +++ b/src/runtime/virtcontainers/qemu.go @@ -141,7 +141,7 @@ type qmpLogger struct { func newQMPLogger() qmpLogger { return qmpLogger{ - logger: virtLog.WithField("subsystem", "qmp"), + logger: hvLogger.WithField("subsystem", "qmp"), } } @@ -163,7 +163,7 @@ func (l qmpLogger) Errorf(format string, v ...interface{}) { // Logger returns a logrus logger appropriate for logging qemu messages func (q *qemu) Logger() *logrus.Entry { - return virtLog.WithField("subsystem", "qemu") + return hvLogger.WithField("subsystem", "qemu") } func (q *qemu) kernelParameters() string { diff --git a/src/runtime/virtcontainers/qemu_amd64.go b/src/runtime/virtcontainers/qemu_amd64.go index 689575ab5..c1464809a 100644 --- a/src/runtime/virtcontainers/qemu_amd64.go +++ b/src/runtime/virtcontainers/qemu_amd64.go @@ -169,7 +169,7 @@ func (q *qemuAmd64) cpuModel() string { // VMX is not migratable yet. // issue: https://github.com/kata-containers/runtime/issues/1750 if q.vmFactory { - virtLog.WithField("subsystem", "qemuAmd64").Warn("VMX is not migratable yet: turning it off") + hvLogger.WithField("subsystem", "qemuAmd64").Warn("VMX is not migratable yet: turning it off") cpuModel += ",vmx=off" } @@ -200,7 +200,7 @@ func (q *qemuAmd64) enableProtection() error { if err != nil { return err } - logger := virtLog.WithFields(logrus.Fields{ + logger := hvLogger.WithFields(logrus.Fields{ "subsystem": "qemuAmd64", "machine": q.qemuMachine, "kernel-params-debug": q.kernelParamsDebug, diff --git a/src/runtime/virtcontainers/qemu_arch_base.go b/src/runtime/virtcontainers/qemu_arch_base.go index 426da6bac..97cd6eb83 100644 --- a/src/runtime/virtcontainers/qemu_arch_base.go +++ b/src/runtime/virtcontainers/qemu_arch_base.go @@ -846,6 +846,6 @@ func (q *qemuArchBase) setPFlash(p []string) { // append protection device func (q *qemuArchBase) appendProtectionDevice(devices []govmmQemu.Device, firmware string) ([]govmmQemu.Device, string, error) { - virtLog.WithField("arch", runtime.GOARCH).Warnf("Confidential Computing has not been implemented for this architecture") + hvLogger.WithField("arch", runtime.GOARCH).Warnf("Confidential Computing has not been implemented for this architecture") return devices, firmware, nil } diff --git a/src/runtime/virtcontainers/qemu_arm64.go b/src/runtime/virtcontainers/qemu_arm64.go index 2cd869a8c..452493ce1 100644 --- a/src/runtime/virtcontainers/qemu_arm64.go +++ b/src/runtime/virtcontainers/qemu_arm64.go @@ -171,6 +171,6 @@ func (q *qemuArm64) enableProtection() error { func (q *qemuArm64) appendProtectionDevice(devices []govmmQemu.Device, firmware string) ([]govmmQemu.Device, string, error) { err := q.enableProtection() - virtLog.WithField("arch", runtime.GOARCH).Warnf("%v", err) + hvLogger.WithField("arch", runtime.GOARCH).Warnf("%v", err) return devices, firmware, err } diff --git a/src/runtime/virtcontainers/qemu_ppc64le.go b/src/runtime/virtcontainers/qemu_ppc64le.go index 51c9003ba..00fec3529 100644 --- a/src/runtime/virtcontainers/qemu_ppc64le.go +++ b/src/runtime/virtcontainers/qemu_ppc64le.go @@ -51,7 +51,7 @@ var supportedQemuMachine = govmmQemu.Machine{ // Logger returns a logrus logger appropriate for logging qemu messages func (q *qemuPPC64le) Logger() *logrus.Entry { - return virtLog.WithField("subsystem", "qemuPPC64le") + return hvLogger.WithField("subsystem", "qemuPPC64le") } // MaxQemuVCPUs returns the maximum number of vCPUs supported @@ -141,7 +141,7 @@ func (q *qemuPPC64le) enableProtection() error { q.qemuMachine.Options += "," } q.qemuMachine.Options += fmt.Sprintf("confidential-guest-support=%s", pefID) - virtLog.WithFields(logrus.Fields{ + hvLogger.WithFields(logrus.Fields{ "subsystem": "qemuPPC64le", "machine": q.qemuMachine, "kernel-params": q.kernelParams, diff --git a/src/runtime/virtcontainers/qemu_s390x.go b/src/runtime/virtcontainers/qemu_s390x.go index b7611decc..d6c013156 100644 --- a/src/runtime/virtcontainers/qemu_s390x.go +++ b/src/runtime/virtcontainers/qemu_s390x.go @@ -324,7 +324,7 @@ func (q *qemuS390x) enableProtection() error { q.qemuMachine.Options += "," } q.qemuMachine.Options += fmt.Sprintf("confidential-guest-support=%s", secExecID) - virtLog.WithFields(logrus.Fields{ + hvLogger.WithFields(logrus.Fields{ "subsystem": logSubsystem, "machine": q.qemuMachine}). Info("Enabling guest protection with Secure Execution") diff --git a/src/runtime/virtcontainers/virtiofsd.go b/src/runtime/virtcontainers/virtiofsd.go index d35399bea..baaec862f 100644 --- a/src/runtime/virtcontainers/virtiofsd.go +++ b/src/runtime/virtcontainers/virtiofsd.go @@ -216,7 +216,7 @@ func (v *virtiofsd) valid() error { } func (v *virtiofsd) Logger() *log.Entry { - return virtLog.WithField("subsystem", "virtiofsd") + return hvLogger.WithField("subsystem", "virtiofsd") } func (v *virtiofsd) kill(ctx context.Context) (err error) { From ce92cadc7d491226cd502d86982e50b7a404613f Mon Sep 17 00:00:00 2001 From: Eric Ernst Date: Thu, 11 Nov 2021 17:50:13 -0800 Subject: [PATCH 27/66] vc: hypervisor: remove setSandbox The hypervisor interface implementation should not know a thing about sandboxes. Fixes: #2882 Signed-off-by: Eric Ernst --- src/runtime/virtcontainers/clh.go | 3 --- src/runtime/virtcontainers/fc.go | 3 --- src/runtime/virtcontainers/hypervisor.go | 2 -- src/runtime/virtcontainers/mock_hypervisor.go | 3 --- src/runtime/virtcontainers/qemu.go | 3 --- src/runtime/virtcontainers/sandbox.go | 2 -- 6 files changed, 16 deletions(-) diff --git a/src/runtime/virtcontainers/clh.go b/src/runtime/virtcontainers/clh.go index 7fad500f5..a3cdf02f6 100644 --- a/src/runtime/virtcontainers/clh.go +++ b/src/runtime/virtcontainers/clh.go @@ -1271,6 +1271,3 @@ func (clh *cloudHypervisor) vmInfo() (chclient.VmInfo, error) { func (clh *cloudHypervisor) IsRateLimiterBuiltin() bool { return false } - -func (clh *cloudHypervisor) setSandbox(sandbox *Sandbox) { -} diff --git a/src/runtime/virtcontainers/fc.go b/src/runtime/virtcontainers/fc.go index fa4e56ffa..c6dffb337 100644 --- a/src/runtime/virtcontainers/fc.go +++ b/src/runtime/virtcontainers/fc.go @@ -1274,6 +1274,3 @@ func revertBytes(num uint64) uint64 { } return 1024*revertBytes(a) + b } - -func (fc *firecracker) setSandbox(sandbox *Sandbox) { -} diff --git a/src/runtime/virtcontainers/hypervisor.go b/src/runtime/virtcontainers/hypervisor.go index 06a0385c9..715b4793c 100644 --- a/src/runtime/virtcontainers/hypervisor.go +++ b/src/runtime/virtcontainers/hypervisor.go @@ -945,6 +945,4 @@ type Hypervisor interface { // check if hypervisor supports built-in rate limiter. IsRateLimiterBuiltin() bool - - setSandbox(sandbox *Sandbox) } diff --git a/src/runtime/virtcontainers/mock_hypervisor.go b/src/runtime/virtcontainers/mock_hypervisor.go index dc4b11ad6..111707fd9 100644 --- a/src/runtime/virtcontainers/mock_hypervisor.go +++ b/src/runtime/virtcontainers/mock_hypervisor.go @@ -149,6 +149,3 @@ func (m *mockHypervisor) GenerateSocket(id string) (interface{}, error) { func (m *mockHypervisor) IsRateLimiterBuiltin() bool { return false } - -func (m *mockHypervisor) setSandbox(sandbox *Sandbox) { -} diff --git a/src/runtime/virtcontainers/qemu.go b/src/runtime/virtcontainers/qemu.go index d0b3bd5c5..9b78fb24e 100644 --- a/src/runtime/virtcontainers/qemu.go +++ b/src/runtime/virtcontainers/qemu.go @@ -2541,6 +2541,3 @@ func (q *qemu) GenerateSocket(id string) (interface{}, error) { func (q *qemu) IsRateLimiterBuiltin() bool { return false } - -func (q *qemu) setSandbox(sandbox *Sandbox) { -} diff --git a/src/runtime/virtcontainers/sandbox.go b/src/runtime/virtcontainers/sandbox.go index 207ba1d81..6f3956216 100644 --- a/src/runtime/virtcontainers/sandbox.go +++ b/src/runtime/virtcontainers/sandbox.go @@ -525,8 +525,6 @@ func newSandbox(ctx context.Context, sandboxConfig SandboxConfig, factory Factor swapDevices: []*config.BlockDrive{}, } - hypervisor.setSandbox(s) - if s.store, err = persist.GetDriver(); err != nil || s.store == nil { return nil, fmt.Errorf("failed to get fs persist driver: %v", err) } From ce0693d6dc7c248d62979edecb0cf9a06bef8f38 Mon Sep 17 00:00:00 2001 From: bin Date: Mon, 22 Nov 2021 18:16:32 +0800 Subject: [PATCH 28/66] agent: clear cargo test warnings Function parameters in test config is not used. This commit will add under score before variable name in test config. Fixes: #3091 Signed-off-by: bin --- src/agent/rustjail/src/mount.rs | 64 ++++++++++++++++++++++++--------- 1 file changed, 48 insertions(+), 16 deletions(-) diff --git a/src/agent/rustjail/src/mount.rs b/src/agent/rustjail/src/mount.rs index 7d1f7bc8f..7de987400 100644 --- a/src/agent/rustjail/src/mount.rs +++ b/src/agent/rustjail/src/mount.rs @@ -112,6 +112,7 @@ lazy_static! { } #[inline(always)] +#[cfg(not(test))] pub fn mount< P1: ?Sized + NixPath, P2: ?Sized + NixPath, @@ -124,21 +125,42 @@ pub fn mount< flags: MsFlags, data: Option<&P4>, ) -> std::result::Result<(), nix::Error> { - #[cfg(not(test))] - return mount::mount(source, target, fstype, flags, data); - #[cfg(test)] - return Ok(()); + mount::mount(source, target, fstype, flags, data) } #[inline(always)] +#[cfg(test)] +pub fn mount< + P1: ?Sized + NixPath, + P2: ?Sized + NixPath, + P3: ?Sized + NixPath, + P4: ?Sized + NixPath, +>( + _source: Option<&P1>, + _target: &P2, + _fstype: Option<&P3>, + _flags: MsFlags, + _data: Option<&P4>, +) -> std::result::Result<(), nix::Error> { + Ok(()) +} + +#[inline(always)] +#[cfg(not(test))] pub fn umount2( target: &P, flags: MntFlags, ) -> std::result::Result<(), nix::Error> { - #[cfg(not(test))] - return mount::umount2(target, flags); - #[cfg(test)] - return Ok(()); + mount::umount2(target, flags) +} + +#[inline(always)] +#[cfg(test)] +pub fn umount2( + _target: &P, + _flags: MntFlags, +) -> std::result::Result<(), nix::Error> { + Ok(()) } pub fn init_rootfs( @@ -450,14 +472,20 @@ fn mount_cgroups( Ok(()) } +#[cfg(not(test))] fn pivot_root( new_root: &P1, put_old: &P2, ) -> anyhow::Result<(), nix::Error> { - #[cfg(not(test))] - return unistd::pivot_root(new_root, put_old); - #[cfg(test)] - return Ok(()); + unistd::pivot_root(new_root, put_old) +} + +#[cfg(test)] +fn pivot_root( + _new_root: &P1, + _put_old: &P2, +) -> anyhow::Result<(), nix::Error> { + Ok(()) } pub fn pivot_rootfs(path: &P) -> Result<()> { @@ -582,11 +610,15 @@ fn parse_mount_table() -> Result> { } #[inline(always)] +#[cfg(not(test))] fn chroot(path: &P) -> Result<(), nix::Error> { - #[cfg(not(test))] - return unistd::chroot(path); - #[cfg(test)] - return Ok(()); + unistd::chroot(path) +} + +#[inline(always)] +#[cfg(test)] +fn chroot(_path: &P) -> Result<(), nix::Error> { + Ok(()) } pub fn ms_move_root(rootfs: &str) -> Result { From a7c08aa4b6f3192ebf5e25bf14f79e55139da20e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fabiano=20Fid=C3=AAncio?= Date: Mon, 22 Nov 2021 18:29:26 +0100 Subject: [PATCH 29/66] workflows: Add back the checks for running test-kata-deploy MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Commit 3c9ae7f made /test_kata_deploy run against HEAD, but it also mistakenly removed all the checks that ensure /test_kata_deploy only runs when explicitly called. Mea culpa on this, and let's add the tests back. Fixes: #3101 Signed-off-by: Fabiano Fidêncio --- .github/workflows/kata-deploy-test.yaml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.github/workflows/kata-deploy-test.yaml b/.github/workflows/kata-deploy-test.yaml index 343fc6701..548b30959 100644 --- a/.github/workflows/kata-deploy-test.yaml +++ b/.github/workflows/kata-deploy-test.yaml @@ -6,6 +6,11 @@ name: test-kata-deploy jobs: build-asset: + if: | + github.event.issue.pull_request + && github.event_name == 'issue_comment' + && github.event.action == 'created' + && startsWith(github.event.comment.body, '/test_kata_deploy') runs-on: ubuntu-latest strategy: matrix: From 0c6c0735ecc04c28cca5dc76a10f8be570f8e4d4 Mon Sep 17 00:00:00 2001 From: "wangyongchao.bj" Date: Tue, 23 Nov 2021 09:44:05 +0800 Subject: [PATCH 30/66] agent: fixed the `make optimize` bug The unrecognized option: 'deny-warnings' args caused `make optimize` failed. Fixed the Makefile of the agent project, make sure the `make optimize` command execute correctly. This PR modify the rustc args from '--deny-warnings' to '--deny warnings'. Fixes: #3104 Signed-off-by: wangyongchao.bj --- src/agent/Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/agent/Makefile b/src/agent/Makefile index 8078bb9df..bd647d448 100644 --- a/src/agent/Makefile +++ b/src/agent/Makefile @@ -114,7 +114,7 @@ $(GENERATED_FILES): %: %.in ##TARGET optimize: optimized build optimize: $(SOURCES) | show-summary show-header - @RUSTFLAGS="-C link-arg=-s $(EXTRA_RUSTFLAGS) --deny-warnings" cargo build --target $(TRIPLE) --$(BUILD_TYPE) $(EXTRA_RUSTFEATURES) + @RUSTFLAGS="-C link-arg=-s $(EXTRA_RUSTFLAGS) --deny warnings" cargo build --target $(TRIPLE) --$(BUILD_TYPE) $(EXTRA_RUSTFEATURES) ##TARGET clippy: run clippy linter clippy: $(GENERATED_CODE) From 45d76407aac9c7750e58158dcde0bf20a8af18b5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fabiano=20Fid=C3=AAncio?= Date: Tue, 23 Nov 2021 13:21:37 +0100 Subject: [PATCH 31/66] kata-deploy: Don't mention arch specific binaries in the README MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Although the binary name of the shipped binary is `qemu-system-x86_64`, and we only ship kata-deploy for `x86_64`, we better leaving the architecture specific name out of our README file. Signed-off-by: Fabiano Fidêncio --- tools/packaging/kata-deploy/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/packaging/kata-deploy/README.md b/tools/packaging/kata-deploy/README.md index 9f54f0a72..cf4286e8b 100644 --- a/tools/packaging/kata-deploy/README.md +++ b/tools/packaging/kata-deploy/README.md @@ -165,7 +165,7 @@ This image contains all the necessary artifacts for running Kata Containers, all from the [Kata Containers release page](https://github.com/kata-containers/kata-containers/releases). Host artifacts: -* `cloud-hypervisor`, `firecracker`, `qemu-system-x86_64`, and supporting binaries +* `cloud-hypervisor`, `firecracker`, `qemu`, and supporting binaries * `containerd-shim-kata-v2` * `kata-collect-data.sh` * `kata-runtime` From 143fb278029a0aa4845243c5bc0a1dec90cdc0fc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fabiano=20Fid=C3=AAncio?= Date: Tue, 23 Nov 2021 13:24:42 +0100 Subject: [PATCH 32/66] kata-deploy: Use the default notation for "Note" MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Let's use the default GitHub notation for notes in documentation, as describe here: https://github.com/kata-containers/kata-containers/blob/main/docs/Documentation-Requirements.md#notes Signed-off-by: Fabiano Fidêncio Suggested-by: James O. D. Hunt --- tools/packaging/kata-deploy/README.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tools/packaging/kata-deploy/README.md b/tools/packaging/kata-deploy/README.md index cf4286e8b..b6d455adc 100644 --- a/tools/packaging/kata-deploy/README.md +++ b/tools/packaging/kata-deploy/README.md @@ -4,8 +4,8 @@ and artifacts required to run Kata Containers, as well as reference DaemonSets, which can be utilized to install Kata Containers on a running Kubernetes cluster. -Note, installation through DaemonSets successfully installs `katacontainers.io/kata-runtime` on -a node only if it uses either containerd or CRI-O CRI-shims. +> **Note**: installation through DaemonSets successfully installs `katacontainers.io/kata-runtime` +> on a node only if it uses either containerd or CRI-O CRI-shims. ## Kubernetes quick start @@ -24,8 +24,8 @@ $ kubectl apply -f https://raw.githubusercontent.com/kata-containers/kata-contai The stable image refers to the last stable releases content. -Note that if you use a tagged version of the repo, the stable image does match that version. -For instance, if you use the 2.2.1 tagged version of the kata-deploy.yaml file, then the version 2.2.1 of the kata runtime will be deployed. +> **Note:** if you use a tagged version of the repo, the stable image does match that version. +> For instance, if you use the 2.2.1 tagged version of the kata-deploy.yaml file, then the version 2.2.1 of the kata runtime will be deployed. ```sh $ kubectl apply -f https://raw.githubusercontent.com/kata-containers/kata-containers/main/tools/packaging/kata-deploy/kata-rbac/base/kata-rbac.yaml From acece84906fa13ee05f501cf116fe315cdd1d2b7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fabiano=20Fid=C3=AAncio?= Date: Tue, 23 Nov 2021 13:27:35 +0100 Subject: [PATCH 33/66] docs: Use the default notation for "Note" on install README MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Let's use the default GitHub notation for notes in documentation, as describe here: https://github.com/kata-containers/kata-containers/blob/main/docs/Documentation-Requir Signed-off-by: Fabiano Fidêncio Suggested-by: James O. D. Hunt --- docs/install/README.md | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/docs/install/README.md b/docs/install/README.md index 7b5cd0229..65cab6115 100644 --- a/docs/install/README.md +++ b/docs/install/README.md @@ -12,8 +12,8 @@ Containers. Packaged installation methods uses your distribution's native package format (such as RPM or DEB). -*Note:* We encourage installation methods that provides automatic updates, it ensures security updates and bug fixes are -easily applied. +> **Note:** We encourage installation methods that provides automatic updates, it ensures security updates and bug fixes are +> easily applied. | Installation method | Description | Automatic updates | Use case | |------------------------------------------------------|---------------------------------------------------------------------|-------------------|----------------------------------------------------------| @@ -48,9 +48,9 @@ Follow the [containerd installation guide](container-manager/containerd/containe ## Build from source installation -*Note:* Power users who decide to build from sources should be aware of the -implications of using an unpackaged system which will not be automatically -updated as new [releases](../Stable-Branch-Strategy.md) are made available. +> **Note:** Power users who decide to build from sources should be aware of the +> implications of using an unpackaged system which will not be automatically +> updated as new [releases](../Stable-Branch-Strategy.md) are made available. [Building from sources](../Developer-Guide.md#initial-setup) allows power users who are comfortable building software from source to use the latest component From 705687dc42a75471976dad94951a94ad34857dd1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jo=C3=A3o=20Vanzuita?= Date: Tue, 23 Nov 2021 13:46:17 +0100 Subject: [PATCH 34/66] docs: Add kata-deploy as part of the install docs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This PR links the kata-deloy installation instructions to the docs/install folder. Fixes: #2450 Signed-off-by: João Vanzuita Signed-off-by: Fabiano Fidêncio --- docs/install/README.md | 24 +++++++++++++++++------- 1 file changed, 17 insertions(+), 7 deletions(-) diff --git a/docs/install/README.md b/docs/install/README.md index 65cab6115..9ad55f0f2 100644 --- a/docs/install/README.md +++ b/docs/install/README.md @@ -15,13 +15,23 @@ Packaged installation methods uses your distribution's native package format (su > **Note:** We encourage installation methods that provides automatic updates, it ensures security updates and bug fixes are > easily applied. -| Installation method | Description | Automatic updates | Use case | -|------------------------------------------------------|---------------------------------------------------------------------|-------------------|----------------------------------------------------------| -| [Using official distro packages](#official-packages) | Kata packages provided by Linux distributions official repositories | yes | Recommended for most users. | -| [Using snap](#snap-installation) | Easy to install | yes | Good alternative to official distro packages. | -| [Automatic](#automatic-installation) | Run a single command to install a full system | **No!** | For those wanting the latest release quickly. | -| [Manual](#manual-installation) | Follow a guide step-by-step to install a working system | **No!** | For those who want the latest release with more control. | -| [Build from source](#build-from-source-installation) | Build the software components manually | **No!** | Power users and developers only. | +| Installation method | Description | Automatic updates | Use case | +|------------------------------------------------------|----------------------------------------------------------------------------------------------|-------------------|-----------------------------------------------------------------------------------------------| +| [Using kata-deploy](#kata-deploy-installation) | The preferred way to deploy the Kata Containers distributed binaries on a Kubernetes cluster | **No!** | Best way to give it a try on kata-containers on an already up and running Kubernetes cluster. | +| [Using official distro packages](#official-packages) | Kata packages provided by Linux distributions official repositories | yes | Recommended for most users. | +| [Using snap](#snap-installation) | Easy to install | yes | Good alternative to official distro packages. | +| [Automatic](#automatic-installation) | Run a single command to install a full system | **No!** | For those wanting the latest release quickly. | +| [Manual](#manual-installation) | Follow a guide step-by-step to install a working system | **No!** | For those who want the latest release with more control. | +| [Build from source](#build-from-source-installation) | Build the software components manually | **No!** | Power users and developers only. | + +### Kata Deploy Installation + +Kata Deploy provides a Dockerfile, which contains all of the binaries and +artifacts required to run Kata Containers, as well as reference DaemonSets, +which can be utilized to install Kata Containers on a running Kubernetes +cluster. + +[Use Kata Deploy](/tools/packaging/kata-deploy/README.md) to install Kata Containers on a Kubernetes Cluster. ### Official packages From ddc68131df05265fb4aa56f45b88c9492009bc81 Mon Sep 17 00:00:00 2001 From: bin Date: Wed, 24 Nov 2021 15:08:18 +0800 Subject: [PATCH 35/66] runtime: delete netmon Netmon is not used anymore. Fixes: #3112 Signed-off-by: bin --- src/runtime/.gitignore | 1 - src/runtime/Makefile | 23 +- src/runtime/cmd/kata-runtime/kata-env.go | 32 - src/runtime/cmd/kata-runtime/kata-env_test.go | 69 -- src/runtime/cmd/netmon/netmon.go | 683 ----------------- src/runtime/cmd/netmon/netmon_test.go | 701 ------------------ src/runtime/config/configuration-acrn.toml.in | 16 - src/runtime/config/configuration-clh.toml.in | 17 - src/runtime/config/configuration-fc.toml.in | 16 - src/runtime/config/configuration-qemu.toml.in | 16 - .../pkg/containerd-shim-v2/create_test.go | 2 - src/runtime/pkg/katatestutils/utils.go | 6 - .../pkg/katautils/config-settings.go.in | 2 - src/runtime/pkg/katautils/config.go | 32 - src/runtime/pkg/katautils/config_test.go | 30 - src/runtime/pkg/oci/utils.go | 7 - .../documentation/api/1.0/api.md | 1 - src/runtime/virtcontainers/netmon.go | 96 --- src/runtime/virtcontainers/netmon_test.go | 61 -- src/runtime/virtcontainers/network.go | 2 - src/runtime/virtcontainers/persist.go | 2 - .../virtcontainers/persist/api/network.go | 1 - src/runtime/virtcontainers/sandbox.go | 53 -- src/runtime/virtcontainers/sandbox_test.go | 28 - 24 files changed, 2 insertions(+), 1895 deletions(-) delete mode 100644 src/runtime/cmd/netmon/netmon.go delete mode 100644 src/runtime/cmd/netmon/netmon_test.go delete mode 100644 src/runtime/virtcontainers/netmon.go delete mode 100644 src/runtime/virtcontainers/netmon_test.go diff --git a/src/runtime/.gitignore b/src/runtime/.gitignore index d155740b7..ffbf8a755 100644 --- a/src/runtime/.gitignore +++ b/src/runtime/.gitignore @@ -11,7 +11,6 @@ config-generated.go /pkg/containerd-shim-v2/monitor_address /data/kata-collect-data.sh /kata-monitor -/kata-netmon /kata-runtime /pkg/katautils/config-settings.go /virtcontainers/hack/virtc/virtc diff --git a/src/runtime/Makefile b/src/runtime/Makefile index a8f7ef070..9d1080780 100644 --- a/src/runtime/Makefile +++ b/src/runtime/Makefile @@ -55,11 +55,6 @@ RUNTIME_OUTPUT = $(CURDIR)/$(TARGET) RUNTIME_DIR = $(CLI_DIR)/$(TARGET) BINLIST += $(TARGET) -NETMON_DIR = $(CLI_DIR)/netmon -NETMON_TARGET = $(PROJECT_TYPE)-netmon -NETMON_RUNTIME_OUTPUT = $(CURDIR)/$(NETMON_TARGET) -BINLIBEXECLIST += $(NETMON_TARGET) - DESTDIR ?= / ifeq ($(PREFIX),) @@ -142,9 +137,6 @@ ACRNVALIDHYPERVISORPATHS := [\"$(ACRNPATH)\"] ACRNCTLPATH := $(ACRNBINDIR)/$(ACRNCTLCMD) ACRNVALIDCTLPATHS := [\"$(ACRNCTLPATH)\"] -NETMONCMD := $(BIN_PREFIX)-netmon -NETMONPATH := $(PKGLIBEXECDIR)/$(NETMONCMD) - # Default number of vCPUs DEFVCPUS := 1 # Default maximum number of vCPUs @@ -416,7 +408,6 @@ USER_VARS += PROJECT_PREFIX USER_VARS += PROJECT_TAG USER_VARS += PROJECT_TYPE USER_VARS += PROJECT_URL -USER_VARS += NETMONPATH USER_VARS += QEMUBINDIR USER_VARS += QEMUCMD USER_VARS += QEMUPATH @@ -509,7 +500,7 @@ define SHOW_ARCH $(shell printf "\\t%s%s\\\n" "$(1)" $(if $(filter $(ARCH),$(1))," (default)","")) endef -all: runtime containerd-shim-v2 netmon monitor +all: runtime containerd-shim-v2 monitor # Targets that depend on .git-commit can use $(shell cat .git-commit) to get a # git revision string. They will only be rebuilt if the revision string @@ -525,11 +516,6 @@ containerd-shim-v2: $(SHIMV2_OUTPUT) monitor: $(MONITOR_OUTPUT) -netmon: $(NETMON_RUNTIME_OUTPUT) - -$(NETMON_RUNTIME_OUTPUT): $(SOURCES) VERSION - $(QUIET_BUILD)(cd $(NETMON_DIR) && go build $(BUILDFLAGS) -o $@ -ldflags "-X main.version=$(VERSION)" $(KATA_LDFLAGS)) - runtime: $(RUNTIME_OUTPUT) $(CONFIGS) .DEFAULT: default @@ -638,15 +624,13 @@ coverage: go test -v -mod=vendor -covermode=atomic -coverprofile=coverage.txt ./... go tool cover -html=coverage.txt -o coverage.html -install: all install-runtime install-containerd-shim-v2 install-monitor install-netmon +install: all install-runtime install-containerd-shim-v2 install-monitor install-bin: $(BINLIST) $(QUIET_INST)$(foreach f,$(BINLIST),$(call INSTALL_EXEC,$f,$(BINDIR))) install-runtime: runtime install-scripts install-completions install-configs install-bin -install-netmon: install-bin-libexec - install-containerd-shim-v2: $(SHIMV2) $(QUIET_INST)$(call INSTALL_EXEC,$<,$(BINDIR)) @@ -678,7 +662,6 @@ clean: $(QUIET_CLEAN)rm -f \ $(CONFIGS) \ $(GENERATED_FILES) \ - $(NETMON_TARGET) \ $(MONITOR) \ $(SHIMV2) \ $(TARGET) \ @@ -706,9 +689,7 @@ show-usage: show-header @printf "\tgenerate-config : create configuration file.\n" @printf "\tinstall : install everything.\n" @printf "\tinstall-containerd-shim-v2 : only install containerd shim v2 files.\n" - @printf "\tinstall-netmon : only install netmon files.\n" @printf "\tinstall-runtime : only install runtime files.\n" - @printf "\tnetmon : only build netmon.\n" @printf "\truntime : only build runtime.\n" @printf "\tshow-arches : show supported architectures (ARCH variable values).\n" @printf "\tshow-summary : show install locations.\n" diff --git a/src/runtime/cmd/kata-runtime/kata-env.go b/src/runtime/cmd/kata-runtime/kata-env.go index 2fb8f08ba..b1421fa00 100644 --- a/src/runtime/cmd/kata-runtime/kata-env.go +++ b/src/runtime/cmd/kata-runtime/kata-env.go @@ -140,14 +140,6 @@ type HostInfo struct { SupportVSocks bool } -// NetmonInfo stores netmon details -type NetmonInfo struct { - Path string - Version VersionInfo - Debug bool - Enable bool -} - // EnvInfo collects all information that will be displayed by the // env command. // @@ -159,7 +151,6 @@ type EnvInfo struct { Initrd InitrdInfo Hypervisor HypervisorInfo Runtime RuntimeInfo - Netmon NetmonInfo Host HostInfo Agent AgentInfo } @@ -276,26 +267,6 @@ func getMemoryInfo() MemoryInfo { } } -func getNetmonInfo(config oci.RuntimeConfig) NetmonInfo { - netmonConfig := config.NetmonConfig - - var netmonVersionInfo VersionInfo - if version, err := getCommandVersion(netmonConfig.Path); err != nil { - netmonVersionInfo = unknownVersionInfo - } else { - netmonVersionInfo = constructVersionInfo(version) - } - - netmon := NetmonInfo{ - Version: netmonVersionInfo, - Path: netmonConfig.Path, - Debug: netmonConfig.Debug, - Enable: netmonConfig.Enable, - } - - return netmon -} - func getCommandVersion(cmd string) (string, error) { return utils.RunCommand([]string{cmd, "--version"}) } @@ -364,8 +335,6 @@ func getEnvInfo(configFile string, config oci.RuntimeConfig) (env EnvInfo, err e return EnvInfo{}, err } - netmon := getNetmonInfo(config) - agent, err := getAgentInfo(config) if err != nil { return EnvInfo{}, err @@ -398,7 +367,6 @@ func getEnvInfo(configFile string, config oci.RuntimeConfig) (env EnvInfo, err e Initrd: initrd, Agent: agent, Host: host, - Netmon: netmon, } return env, nil diff --git a/src/runtime/cmd/kata-runtime/kata-env_test.go b/src/runtime/cmd/kata-runtime/kata-env_test.go index 507fd0325..178ef8b3b 100644 --- a/src/runtime/cmd/kata-runtime/kata-env_test.go +++ b/src/runtime/cmd/kata-runtime/kata-env_test.go @@ -32,7 +32,6 @@ import ( ) var ( - testNetmonVersion = "netmon version 0.1" testHypervisorVersion = "QEMU emulator version 2.7.0+git.741f430a96-6.1, Copyright (c) 2003-2016 Fabrice Bellard and the QEMU Project developers" ) @@ -41,7 +40,6 @@ var ( enableVirtioFS = false runtimeDebug = false runtimeTrace = false - netmonDebug = false agentDebug = false agentTrace = false ) @@ -83,7 +81,6 @@ func makeRuntimeConfig(prefixDir string) (configFile string, config oci.RuntimeC imagePath := filepath.Join(prefixDir, "image") kernelParams := "foo=bar xyz" machineType := "machineType" - netmonPath := filepath.Join(prefixDir, "netmon") disableBlock := true blockStorageDriver := "virtio-scsi" enableIOThreads := true @@ -107,11 +104,6 @@ func makeRuntimeConfig(prefixDir string) (configFile string, config oci.RuntimeC } } - err = makeVersionBinary(netmonPath, testNetmonVersion) - if err != nil { - return "", oci.RuntimeConfig{}, err - } - err = makeVersionBinary(hypervisorPath, testHypervisorVersion) if err != nil { return "", oci.RuntimeConfig{}, err @@ -130,7 +122,6 @@ func makeRuntimeConfig(prefixDir string) (configFile string, config oci.RuntimeC ImagePath: imagePath, KernelParams: kernelParams, MachineType: machineType, - NetmonPath: netmonPath, LogPath: logPath, DefaultGuestHookPath: hypConfig.GuestHookPath, DisableBlock: disableBlock, @@ -146,7 +137,6 @@ func makeRuntimeConfig(prefixDir string) (configFile string, config oci.RuntimeC HypervisorDebug: hypervisorDebug, RuntimeDebug: runtimeDebug, RuntimeTrace: runtimeTrace, - NetmonDebug: netmonDebug, AgentDebug: agentDebug, AgentTrace: agentTrace, SharedFS: sharedFS, @@ -169,15 +159,6 @@ func makeRuntimeConfig(prefixDir string) (configFile string, config oci.RuntimeC return configFile, config, nil } -func getExpectedNetmonDetails(config oci.RuntimeConfig) (NetmonInfo, error) { - return NetmonInfo{ - Version: constructVersionInfo(testNetmonVersion), - Path: config.NetmonConfig.Path, - Debug: config.NetmonConfig.Debug, - Enable: config.NetmonConfig.Enable, - }, nil -} - func getExpectedAgentDetails(config oci.RuntimeConfig) (AgentInfo, error) { agentConfig := config.AgentConfig @@ -352,11 +333,6 @@ func getExpectedSettings(config oci.RuntimeConfig, tmpdir, configFile string) (E return EnvInfo{}, err } - netmon, err := getExpectedNetmonDetails(config) - if err != nil { - return EnvInfo{}, err - } - hypervisor := getExpectedHypervisor(config) kernel := getExpectedKernel(config) image := getExpectedImage(config) @@ -369,7 +345,6 @@ func getExpectedSettings(config oci.RuntimeConfig, tmpdir, configFile string) (E Kernel: kernel, Agent: agent, Host: host, - Netmon: netmon, } return env, nil @@ -612,50 +587,6 @@ func TestEnvGetRuntimeInfo(t *testing.T) { assert.Equal(t, expectedRuntime, runtime) } -func TestEnvGetNetmonInfo(t *testing.T) { - tmpdir, err := ioutil.TempDir("", "") - if err != nil { - panic(err) - } - defer os.RemoveAll(tmpdir) - - _, config, err := makeRuntimeConfig(tmpdir) - assert.NoError(t, err) - - expectedNetmon, err := getExpectedNetmonDetails(config) - assert.NoError(t, err) - - netmon := getNetmonInfo(config) - assert.NoError(t, err) - - assert.Equal(t, expectedNetmon, netmon) -} - -func TestEnvGetNetmonInfoNoVersion(t *testing.T) { - tmpdir, err := ioutil.TempDir("", "") - if err != nil { - panic(err) - } - defer os.RemoveAll(tmpdir) - - _, config, err := makeRuntimeConfig(tmpdir) - assert.NoError(t, err) - - expectedNetmon, err := getExpectedNetmonDetails(config) - assert.NoError(t, err) - - // remove the netmon ensuring its version cannot be queried - err = os.Remove(config.NetmonConfig.Path) - assert.NoError(t, err) - - expectedNetmon.Version = unknownVersionInfo - - netmon := getNetmonInfo(config) - assert.NoError(t, err) - - assert.Equal(t, expectedNetmon, netmon) -} - func TestEnvGetAgentInfo(t *testing.T) { tmpdir, err := ioutil.TempDir("", "") if err != nil { diff --git a/src/runtime/cmd/netmon/netmon.go b/src/runtime/cmd/netmon/netmon.go deleted file mode 100644 index 21798a153..000000000 --- a/src/runtime/cmd/netmon/netmon.go +++ /dev/null @@ -1,683 +0,0 @@ -// Copyright (c) 2018 Intel Corporation -// -// SPDX-License-Identifier: Apache-2.0 -// - -package main - -import ( - "encoding/json" - "errors" - "flag" - "fmt" - "io/ioutil" - "log/syslog" - "net" - "os" - "os/exec" - "os/signal" - "path/filepath" - "strings" - "syscall" - "time" - - "github.com/kata-containers/kata-containers/src/runtime/pkg/signals" - pbTypes "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/pkg/agent/protocols" - "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/utils" - - "github.com/sirupsen/logrus" - lSyslog "github.com/sirupsen/logrus/hooks/syslog" - "github.com/vishvananda/netlink" - "golang.org/x/sys/unix" -) - -const ( - netmonName = "kata-netmon" - - kataCmd = "kata-network" - kataCLIAddIfaceCmd = "add-iface" - kataCLIDelIfaceCmd = "del-iface" - kataCLIUpdtRoutesCmd = "update-routes" - - kataSuffix = "kata" - - // sharedFile is the name of the file that will be used to share - // the data between this process and the kata-runtime process - // responsible for updating the network. - sharedFile = "shared.json" - storageFilePerm = os.FileMode(0640) - storageDirPerm = os.FileMode(0750) -) - -var ( - // version is the netmon version. This variable is populated at build time. - version = "unknown" - - // For simplicity the code will only focus on IPv4 addresses for now. - netlinkFamily = netlink.FAMILY_ALL - - storageParentPath = "/var/run/kata-containers/netmon/sbs" -) - -type netmonParams struct { - sandboxID string - runtimePath string - logLevel string - debug bool -} - -type netmon struct { - netIfaces map[int]pbTypes.Interface - - linkUpdateCh chan netlink.LinkUpdate - linkDoneCh chan struct{} - - rtUpdateCh chan netlink.RouteUpdate - rtDoneCh chan struct{} - - netHandler *netlink.Handle - - storagePath string - sharedFile string - - netmonParams -} - -var netmonLog = logrus.New() - -func printVersion() { - fmt.Printf("%s version %s\n", netmonName, version) -} - -const componentDescription = `is a network monitoring process that is intended to be started in the -appropriate network namespace so that it can listen to any event related to -link and routes. Whenever a new interface or route is created/updated, it is -responsible for calling into the kata-runtime CLI to ask for the actual -creation/update of the given interface or route. -` - -func printComponentDescription() { - fmt.Printf("\n%s %s\n", netmonName, componentDescription) -} - -func parseOptions() netmonParams { - var version, help bool - - params := netmonParams{} - - flag.BoolVar(&help, "h", false, "describe component usage") - flag.BoolVar(&help, "help", false, "") - flag.BoolVar(¶ms.debug, "d", false, "enable debug mode") - flag.BoolVar(&version, "v", false, "display program version and exit") - flag.BoolVar(&version, "version", false, "") - flag.StringVar(¶ms.sandboxID, "s", "", "sandbox id (required)") - flag.StringVar(¶ms.runtimePath, "r", "", "runtime path (required)") - flag.StringVar(¶ms.logLevel, "log", "warn", - "log messages above specified level: debug, warn, error, fatal or panic") - - flag.Parse() - - if help { - printComponentDescription() - flag.PrintDefaults() - os.Exit(0) - } - - if version { - printVersion() - os.Exit(0) - } - - if params.sandboxID == "" { - fmt.Fprintf(os.Stderr, "Error: sandbox id is empty, one must be provided\n") - flag.PrintDefaults() - os.Exit(1) - } - - if params.runtimePath == "" { - fmt.Fprintf(os.Stderr, "Error: runtime path is empty, one must be provided\n") - flag.PrintDefaults() - os.Exit(1) - } - - return params -} - -func newNetmon(params netmonParams) (*netmon, error) { - handler, err := netlink.NewHandle(netlinkFamily) - if err != nil { - return nil, err - } - - n := &netmon{ - netmonParams: params, - storagePath: filepath.Join(storageParentPath, params.sandboxID), - sharedFile: filepath.Join(storageParentPath, params.sandboxID, sharedFile), - netIfaces: make(map[int]pbTypes.Interface), - linkUpdateCh: make(chan netlink.LinkUpdate), - linkDoneCh: make(chan struct{}), - rtUpdateCh: make(chan netlink.RouteUpdate), - rtDoneCh: make(chan struct{}), - netHandler: handler, - } - - if err := os.MkdirAll(n.storagePath, storageDirPerm); err != nil { - return nil, err - } - - return n, nil -} - -func (n *netmon) cleanup() { - os.RemoveAll(n.storagePath) - n.netHandler.Close() - close(n.linkDoneCh) - close(n.rtDoneCh) -} - -// setupSignalHandler sets up signal handling, starting a go routine to deal -// with signals as they arrive. -func (n *netmon) setupSignalHandler() { - signals.SetLogger(n.logger()) - - sigCh := make(chan os.Signal, 8) - - for _, sig := range signals.HandledSignals() { - signal.Notify(sigCh, sig) - } - - go func() { - for { - sig := <-sigCh - - nativeSignal, ok := sig.(syscall.Signal) - if !ok { - err := errors.New("unknown signal") - netmonLog.WithError(err).WithField("signal", sig.String()).Error() - continue - } - - if signals.FatalSignal(nativeSignal) { - netmonLog.WithField("signal", sig).Error("received fatal signal") - signals.Die(nil) - } else if n.debug && signals.NonFatalSignal(nativeSignal) { - netmonLog.WithField("signal", sig).Debug("handling signal") - signals.Backtrace() - } - } - }() -} - -func (n *netmon) logger() *logrus.Entry { - fields := logrus.Fields{ - "name": netmonName, - "pid": os.Getpid(), - "source": "netmon", - } - - if n.sandboxID != "" { - fields["sandbox"] = n.sandboxID - } - - return netmonLog.WithFields(fields) -} - -func (n *netmon) setupLogger() error { - level, err := logrus.ParseLevel(n.logLevel) - if err != nil { - return err - } - - netmonLog.SetLevel(level) - - netmonLog.Formatter = &logrus.TextFormatter{TimestampFormat: time.RFC3339Nano} - - hook, err := lSyslog.NewSyslogHook("", "", syslog.LOG_INFO|syslog.LOG_USER, netmonName) - if err != nil { - return err - } - - netmonLog.AddHook(hook) - - announceFields := logrus.Fields{ - "runtime-path": n.runtimePath, - "debug": n.debug, - "log-level": n.logLevel, - } - - n.logger().WithFields(announceFields).Info("announce") - - return nil -} - -func (n *netmon) listenNetlinkEvents() error { - if err := netlink.LinkSubscribe(n.linkUpdateCh, n.linkDoneCh); err != nil { - return err - } - - return netlink.RouteSubscribe(n.rtUpdateCh, n.rtDoneCh) -} - -// convertInterface converts a link and its IP addresses as defined by netlink -// package, into the Interface structure format expected by kata-runtime to -// describe an interface and its associated IP addresses. -func convertInterface(linkAttrs *netlink.LinkAttrs, linkType string, addrs []netlink.Addr) pbTypes.Interface { - if linkAttrs == nil { - netmonLog.Warn("Link attributes are nil") - return pbTypes.Interface{} - } - - var ipAddrs []*pbTypes.IPAddress - - for _, addr := range addrs { - if addr.IPNet == nil { - continue - } - - netMask, _ := addr.Mask.Size() - - ipAddr := &pbTypes.IPAddress{ - Address: addr.IP.String(), - Mask: fmt.Sprintf("%d", netMask), - } - - if addr.IP.To4() != nil { - ipAddr.Family = utils.ConvertNetlinkFamily(netlink.FAMILY_V4) - } else { - ipAddr.Family = utils.ConvertNetlinkFamily(netlink.FAMILY_V6) - } - - ipAddrs = append(ipAddrs, ipAddr) - } - - iface := pbTypes.Interface{ - Device: linkAttrs.Name, - Name: linkAttrs.Name, - IPAddresses: ipAddrs, - Mtu: uint64(linkAttrs.MTU), - HwAddr: linkAttrs.HardwareAddr.String(), - Type: linkType, - } - - netmonLog.WithField("interface", iface).Debug("Interface converted") - - return iface -} - -// convertRoutes converts a list of routes as defined by netlink package, -// into a list of Route structure format expected by kata-runtime to -// describe a set of routes. -func convertRoutes(netRoutes []netlink.Route) []pbTypes.Route { - var routes []pbTypes.Route - - for _, netRoute := range netRoutes { - dst := "" - - if netRoute.Protocol == unix.RTPROT_KERNEL { - continue - } - - if netRoute.Dst != nil { - dst = netRoute.Dst.String() - if netRoute.Dst.IP.To4() != nil || netRoute.Dst.IP.To16() != nil { - dst = netRoute.Dst.String() - } else { - netmonLog.WithField("destination", netRoute.Dst.IP.String()).Warn("Unexpected network address format") - } - } - - src := "" - if netRoute.Src != nil { - if netRoute.Src.To4() != nil || netRoute.Src.To16() != nil { - src = netRoute.Src.String() - } else { - netmonLog.WithField("source", netRoute.Src.String()).Warn("Unexpected network address format") - } - } - - gw := "" - if netRoute.Gw != nil { - if netRoute.Gw.To4() != nil || netRoute.Gw.To16() != nil { - gw = netRoute.Gw.String() - } else { - netmonLog.WithField("gateway", netRoute.Gw.String()).Warn("Unexpected network address format") - } - } - - dev := "" - iface, err := net.InterfaceByIndex(netRoute.LinkIndex) - if err == nil { - dev = iface.Name - } - - route := pbTypes.Route{ - Dest: dst, - Gateway: gw, - Device: dev, - Source: src, - Scope: uint32(netRoute.Scope), - } - - routes = append(routes, route) - } - - netmonLog.WithField("routes", routes).Debug("Routes converted") - - return routes -} - -// scanNetwork lists all the interfaces it can find inside the current -// network namespace, and store them in-memory to keep track of them. -func (n *netmon) scanNetwork() error { - links, err := n.netHandler.LinkList() - if err != nil { - return err - } - - for _, link := range links { - addrs, err := n.netHandler.AddrList(link, netlinkFamily) - if err != nil { - return err - } - - linkAttrs := link.Attrs() - if linkAttrs == nil { - continue - } - - iface := convertInterface(linkAttrs, link.Type(), addrs) - n.netIfaces[linkAttrs.Index] = iface - } - - n.logger().Debug("Network scanned") - - return nil -} - -func (n *netmon) storeDataToSend(data interface{}) error { - // Marshal the data structure into a JSON bytes array. - jsonArray, err := json.Marshal(data) - if err != nil { - return err - } - - // Store the JSON bytes array at the specified path. - return ioutil.WriteFile(n.sharedFile, jsonArray, storageFilePerm) -} - -func (n *netmon) execKataCmd(subCmd string) error { - execCmd := exec.Command(n.runtimePath, kataCmd, subCmd, n.sandboxID, n.sharedFile) - - n.logger().WithField("command", execCmd).Debug("Running runtime command") - - // Make use of Run() to ensure the kata-runtime process has correctly - // terminated before to go further. - if err := execCmd.Run(); err != nil { - return err - } - - // Remove the shared file after the command returned. At this point - // we know the content of the file is not going to be used anymore, - // and the file path can be reused for further commands. - return os.Remove(n.sharedFile) -} - -func (n *netmon) addInterfaceCLI(iface pbTypes.Interface) error { - if err := n.storeDataToSend(iface); err != nil { - return err - } - - return n.execKataCmd(kataCLIAddIfaceCmd) -} - -func (n *netmon) delInterfaceCLI(iface pbTypes.Interface) error { - if err := n.storeDataToSend(iface); err != nil { - return err - } - - return n.execKataCmd(kataCLIDelIfaceCmd) -} - -func (n *netmon) updateRoutesCLI(routes []pbTypes.Route) error { - if err := n.storeDataToSend(routes); err != nil { - return err - } - - return n.execKataCmd(kataCLIUpdtRoutesCmd) -} - -func (n *netmon) updateRoutes() error { - // Get all the routes. - netlinkRoutes, err := n.netHandler.RouteList(nil, netlinkFamily) - if err != nil { - return err - } - - // Translate them into Route structures. - routes := convertRoutes(netlinkRoutes) - - // Update the routes through the Kata CLI. - return n.updateRoutesCLI(routes) -} - -func (n *netmon) handleRTMNewAddr(ev netlink.LinkUpdate) error { - n.logger().Debug("Interface update not supported") - return nil -} - -func (n *netmon) handleRTMDelAddr(ev netlink.LinkUpdate) error { - n.logger().Debug("Interface update not supported") - return nil -} - -func (n *netmon) handleRTMNewLink(ev netlink.LinkUpdate) error { - // NEWLINK might be a lot of different things. We're interested in - // adding the interface (both to our list and by calling into the - // Kata CLI API) only if this has the flags UP and RUNNING, meaning - // we don't expect any further change on the interface, and that we - // are ready to add it. - - linkAttrs := ev.Link.Attrs() - if linkAttrs == nil { - n.logger().Warn("The link attributes are nil") - return nil - } - - // First, ignore if the interface name contains "kata". This way we - // are preventing from adding interfaces created by Kata Containers. - if strings.HasSuffix(linkAttrs.Name, kataSuffix) { - n.logger().Debugf("Ignore the interface %s because found %q", - linkAttrs.Name, kataSuffix) - return nil - } - - // Check if the interface exist in the internal list. - if _, exist := n.netIfaces[int(ev.Index)]; exist { - n.logger().Debugf("Ignoring interface %s because already exist", - linkAttrs.Name) - return nil - } - - // Now, check if the interface has been enabled to UP and RUNNING. - if (ev.Flags&unix.IFF_UP) != unix.IFF_UP || - (ev.Flags&unix.IFF_RUNNING) != unix.IFF_RUNNING { - n.logger().Debugf("Ignore the interface %s because not UP and RUNNING", - linkAttrs.Name) - return nil - } - - // Get the list of IP addresses associated with this interface. - addrs, err := n.netHandler.AddrList(ev.Link, netlinkFamily) - if err != nil { - return err - } - - // Convert the interfaces in the appropriate structure format. - iface := convertInterface(linkAttrs, ev.Link.Type(), addrs) - - // Add the interface through the Kata CLI. - if err := n.addInterfaceCLI(iface); err != nil { - return err - } - - // Add the interface to the internal list. - n.netIfaces[linkAttrs.Index] = iface - - // Complete by updating the routes. - return n.updateRoutes() -} - -func (n *netmon) handleRTMDelLink(ev netlink.LinkUpdate) error { - // It can only delete if identical interface is found in the internal - // list of interfaces. Otherwise, the deletion will be ignored. - linkAttrs := ev.Link.Attrs() - if linkAttrs == nil { - n.logger().Warn("Link attributes are nil") - return nil - } - - // First, ignore if the interface name contains "kata". This way we - // are preventing from deleting interfaces created by Kata Containers. - if strings.Contains(linkAttrs.Name, kataSuffix) { - n.logger().Debugf("Ignore the interface %s because found %q", - linkAttrs.Name, kataSuffix) - return nil - } - - // Check if the interface exist in the internal list. - iface, exist := n.netIfaces[int(ev.Index)] - if !exist { - n.logger().Debugf("Ignoring interface %s because not found", - linkAttrs.Name) - return nil - } - - if err := n.delInterfaceCLI(iface); err != nil { - return err - } - - // Delete the interface from the internal list. - delete(n.netIfaces, linkAttrs.Index) - - // Complete by updating the routes. - return n.updateRoutes() -} - -func (n *netmon) handleRTMNewRoute(ev netlink.RouteUpdate) error { - // Add the route through updateRoutes(), only if the route refer to an - // interface that already exists in the internal list of interfaces. - if _, exist := n.netIfaces[ev.Route.LinkIndex]; !exist { - n.logger().Debugf("Ignoring route %+v since interface %d not found", - ev.Route, ev.Route.LinkIndex) - return nil - } - - return n.updateRoutes() -} - -func (n *netmon) handleRTMDelRoute(ev netlink.RouteUpdate) error { - // Remove the route through updateRoutes(), only if the route refer to - // an interface that already exists in the internal list of interfaces. - return n.updateRoutes() -} - -func (n *netmon) handleLinkEvent(ev netlink.LinkUpdate) error { - n.logger().Debug("handleLinkEvent: netlink event received") - - switch ev.Header.Type { - case unix.NLMSG_DONE: - n.logger().Debug("NLMSG_DONE") - return nil - case unix.NLMSG_ERROR: - n.logger().Error("NLMSG_ERROR") - return fmt.Errorf("Error while listening on netlink socket") - case unix.RTM_NEWADDR: - n.logger().Debug("RTM_NEWADDR") - return n.handleRTMNewAddr(ev) - case unix.RTM_DELADDR: - n.logger().Debug("RTM_DELADDR") - return n.handleRTMDelAddr(ev) - case unix.RTM_NEWLINK: - n.logger().Debug("RTM_NEWLINK") - return n.handleRTMNewLink(ev) - case unix.RTM_DELLINK: - n.logger().Debug("RTM_DELLINK") - return n.handleRTMDelLink(ev) - default: - n.logger().Warnf("Unknown msg type %v", ev.Header.Type) - } - - return nil -} - -func (n *netmon) handleRouteEvent(ev netlink.RouteUpdate) error { - n.logger().Debug("handleRouteEvent: netlink event received") - - switch ev.Type { - case unix.RTM_NEWROUTE: - n.logger().Debug("RTM_NEWROUTE") - return n.handleRTMNewRoute(ev) - case unix.RTM_DELROUTE: - n.logger().Debug("RTM_DELROUTE") - return n.handleRTMDelRoute(ev) - default: - n.logger().Warnf("Unknown msg type %v", ev.Type) - } - - return nil -} - -func (n *netmon) handleEvents() (err error) { - for { - select { - case ev := <-n.linkUpdateCh: - if err = n.handleLinkEvent(ev); err != nil { - return err - } - case ev := <-n.rtUpdateCh: - if err = n.handleRouteEvent(ev); err != nil { - return err - } - } - } -} - -func main() { - // Parse parameters. - params := parseOptions() - - // Create netmon handler. - n, err := newNetmon(params) - if err != nil { - netmonLog.WithError(err).Fatal("newNetmon()") - os.Exit(1) - } - defer n.cleanup() - - // Init logger. - if err := n.setupLogger(); err != nil { - netmonLog.WithError(err).Fatal("setupLogger()") - os.Exit(1) - } - - // Setup signal handlers - n.setupSignalHandler() - - // Scan the current interfaces. - if err := n.scanNetwork(); err != nil { - n.logger().WithError(err).Fatal("scanNetwork()") - os.Exit(1) - } - - // Subscribe to the link listener. - if err := n.listenNetlinkEvents(); err != nil { - n.logger().WithError(err).Fatal("listenNetlinkEvents()") - os.Exit(1) - } - - // Go into the main loop. - if err := n.handleEvents(); err != nil { - n.logger().WithError(err).Fatal("handleEvents()") - os.Exit(1) - } -} diff --git a/src/runtime/cmd/netmon/netmon_test.go b/src/runtime/cmd/netmon/netmon_test.go deleted file mode 100644 index 7e059c737..000000000 --- a/src/runtime/cmd/netmon/netmon_test.go +++ /dev/null @@ -1,701 +0,0 @@ -// Copyright (c) 2018 Intel Corporation -// -// SPDX-License-Identifier: Apache-2.0 -// - -package main - -import ( - "encoding/json" - "fmt" - "io/ioutil" - "net" - "os" - "os/exec" - "path/filepath" - "reflect" - "runtime" - "testing" - - ktu "github.com/kata-containers/kata-containers/src/runtime/pkg/katatestutils" - pbTypes "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/pkg/agent/protocols" - "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/utils" - - "github.com/sirupsen/logrus" - "github.com/stretchr/testify/assert" - "github.com/vishvananda/netlink" - "github.com/vishvananda/netns" - "golang.org/x/sys/unix" -) - -const ( - testSandboxID = "123456789" - testRuntimePath = "/foo/bar/test-runtime" - testLogLevel = "info" - testStorageParentPath = "/tmp/netmon" - testSharedFile = "foo-shared.json" - testWrongNetlinkFamily = -1 - testIfaceName = "test_eth0" - testMTU = 12345 - testHwAddr = "02:00:ca:fe:00:48" - testIPAddress = "192.168.0.15" - testIPAddressWithMask = "192.168.0.15/32" - testIP6Address = "2001:db8:1::242:ac11:2" - testIP6AddressWithMask = "2001:db8:1::/64" - testScope = 1 - testTxQLen = -1 - testIfaceIndex = 5 -) - -func skipUnlessRoot(t *testing.T) { - tc := ktu.NewTestConstraint(false) - - if tc.NotValid(ktu.NeedRoot()) { - t.Skip("Test disabled as requires root user") - } -} - -func TestNewNetmon(t *testing.T) { - skipUnlessRoot(t) - - // Override storageParentPath - savedStorageParentPath := storageParentPath - storageParentPath = testStorageParentPath - defer func() { - storageParentPath = savedStorageParentPath - }() - - params := netmonParams{ - sandboxID: testSandboxID, - runtimePath: testRuntimePath, - debug: true, - logLevel: testLogLevel, - } - - expected := &netmon{ - netmonParams: params, - storagePath: filepath.Join(storageParentPath, params.sandboxID), - sharedFile: filepath.Join(storageParentPath, params.sandboxID, sharedFile), - } - - os.RemoveAll(expected.storagePath) - - got, err := newNetmon(params) - assert.Nil(t, err) - assert.True(t, reflect.DeepEqual(expected.netmonParams, got.netmonParams), - "Got %+v\nExpected %+v", got.netmonParams, expected.netmonParams) - assert.True(t, reflect.DeepEqual(expected.storagePath, got.storagePath), - "Got %+v\nExpected %+v", got.storagePath, expected.storagePath) - assert.True(t, reflect.DeepEqual(expected.sharedFile, got.sharedFile), - "Got %+v\nExpected %+v", got.sharedFile, expected.sharedFile) - - _, err = os.Stat(got.storagePath) - assert.Nil(t, err) - - os.RemoveAll(got.storagePath) -} - -func TestNewNetmonErrorWrongFamilyType(t *testing.T) { - // Override netlinkFamily - savedNetlinkFamily := netlinkFamily - netlinkFamily = testWrongNetlinkFamily - defer func() { - netlinkFamily = savedNetlinkFamily - }() - - n, err := newNetmon(netmonParams{}) - assert.NotNil(t, err) - assert.Nil(t, n) -} - -func TestCleanup(t *testing.T) { - skipUnlessRoot(t) - - // Override storageParentPath - savedStorageParentPath := storageParentPath - storageParentPath = testStorageParentPath - defer func() { - storageParentPath = savedStorageParentPath - }() - - handler, err := netlink.NewHandle(netlinkFamily) - assert.Nil(t, err) - - n := &netmon{ - storagePath: filepath.Join(storageParentPath, testSandboxID), - linkDoneCh: make(chan struct{}), - rtDoneCh: make(chan struct{}), - netHandler: handler, - } - - err = os.MkdirAll(n.storagePath, storageDirPerm) - assert.Nil(t, err) - _, err = os.Stat(n.storagePath) - assert.Nil(t, err) - - n.cleanup() - - _, err = os.Stat(n.storagePath) - assert.NotNil(t, err) - _, ok := (<-n.linkDoneCh) - assert.False(t, ok) - _, ok = (<-n.rtDoneCh) - assert.False(t, ok) -} - -func TestLogger(t *testing.T) { - fields := logrus.Fields{ - "name": netmonName, - "pid": os.Getpid(), - "source": "netmon", - "sandbox": testSandboxID, - } - - expected := netmonLog.WithFields(fields) - - n := &netmon{ - netmonParams: netmonParams{ - sandboxID: testSandboxID, - }, - } - - got := n.logger() - assert.True(t, reflect.DeepEqual(*expected, *got), - "Got %+v\nExpected %+v", *got, *expected) -} - -func TestConvertInterface(t *testing.T) { - hwAddr, err := net.ParseMAC(testHwAddr) - assert.Nil(t, err) - - addrs := []netlink.Addr{ - { - IPNet: &net.IPNet{ - IP: net.ParseIP(testIPAddress), - }, - }, - { - IPNet: &net.IPNet{ - IP: net.ParseIP(testIP6Address), - }, - }, - } - - linkAttrs := &netlink.LinkAttrs{ - Name: testIfaceName, - MTU: testMTU, - HardwareAddr: hwAddr, - } - - linkType := "link_type_test" - - expected := pbTypes.Interface{ - Device: testIfaceName, - Name: testIfaceName, - Mtu: uint64(testMTU), - HwAddr: testHwAddr, - IPAddresses: []*pbTypes.IPAddress{ - { - Family: utils.ConvertNetlinkFamily(netlink.FAMILY_V4), - Address: testIPAddress, - Mask: "0", - }, - { - Family: utils.ConvertNetlinkFamily(netlink.FAMILY_V6), - Address: testIP6Address, - Mask: "0", - }, - }, - Type: linkType, - } - - got := convertInterface(linkAttrs, linkType, addrs) - - assert.True(t, reflect.DeepEqual(expected, got), - "Got %+v\nExpected %+v", got, expected) -} - -func TestConvertRoutes(t *testing.T) { - ip, ipNet, err := net.ParseCIDR(testIPAddressWithMask) - assert.Nil(t, err) - assert.NotNil(t, ipNet) - - _, ip6Net, err := net.ParseCIDR(testIP6AddressWithMask) - assert.Nil(t, err) - assert.NotNil(t, ipNet) - - routes := []netlink.Route{ - { - Dst: ipNet, - Src: ip, - Gw: ip, - LinkIndex: -1, - Scope: testScope, - }, - { - Dst: ip6Net, - Src: nil, - Gw: nil, - LinkIndex: -1, - Scope: testScope, - }, - } - - expected := []pbTypes.Route{ - { - Dest: testIPAddressWithMask, - Gateway: testIPAddress, - Source: testIPAddress, - Scope: uint32(testScope), - }, - { - Dest: testIP6AddressWithMask, - Gateway: "", - Source: "", - Scope: uint32(testScope), - }, - } - - got := convertRoutes(routes) - assert.True(t, reflect.DeepEqual(expected, got), - "Got %+v\nExpected %+v", got, expected) -} - -type testTeardownNetwork func() - -func testSetupNetwork(t *testing.T) testTeardownNetwork { - skipUnlessRoot(t) - - // new temporary namespace so we don't pollute the host - // lock thread since the namespace is thread local - runtime.LockOSThread() - var err error - ns, err := netns.New() - if err != nil { - t.Fatal("Failed to create newns", ns) - } - - return func() { - ns.Close() - runtime.UnlockOSThread() - } -} - -func testCreateDummyNetwork(t *testing.T, handler *netlink.Handle) (int, pbTypes.Interface) { - hwAddr, err := net.ParseMAC(testHwAddr) - assert.Nil(t, err) - - link := &netlink.Dummy{ - LinkAttrs: netlink.LinkAttrs{ - MTU: testMTU, - TxQLen: testTxQLen, - Name: testIfaceName, - HardwareAddr: hwAddr, - }, - } - - err = handler.LinkAdd(link) - assert.Nil(t, err) - err = handler.LinkSetUp(link) - assert.Nil(t, err) - - attrs := link.Attrs() - assert.NotNil(t, attrs) - - addrs, err := handler.AddrList(link, netlinkFamily) - assert.Nil(t, err) - - var ipAddrs []*pbTypes.IPAddress - - // Scan addresses for ipv6 link local address which is automatically assigned - for _, addr := range addrs { - if addr.IPNet == nil { - continue - } - - netMask, _ := addr.Mask.Size() - - ipAddr := &pbTypes.IPAddress{ - Address: addr.IP.String(), - Mask: fmt.Sprintf("%d", netMask), - } - - if addr.IP.To4() != nil { - ipAddr.Family = utils.ConvertNetlinkFamily(netlink.FAMILY_V4) - } else { - ipAddr.Family = utils.ConvertNetlinkFamily(netlink.FAMILY_V6) - } - - ipAddrs = append(ipAddrs, ipAddr) - } - - iface := pbTypes.Interface{ - Device: testIfaceName, - Name: testIfaceName, - Mtu: uint64(testMTU), - HwAddr: testHwAddr, - Type: link.Type(), - IPAddresses: ipAddrs, - } - - return attrs.Index, iface -} - -func TestScanNetwork(t *testing.T) { - tearDownNetworkCb := testSetupNetwork(t) - defer tearDownNetworkCb() - - handler, err := netlink.NewHandle(netlinkFamily) - assert.Nil(t, err) - assert.NotNil(t, handler) - defer handler.Close() - - idx, expected := testCreateDummyNetwork(t, handler) - - n := &netmon{ - netIfaces: make(map[int]pbTypes.Interface), - netHandler: handler, - } - - err = n.scanNetwork() - assert.Nil(t, err) - assert.True(t, reflect.DeepEqual(expected, n.netIfaces[idx]), - "Got %+v\nExpected %+v", n.netIfaces[idx], expected) -} - -func TestStoreDataToSend(t *testing.T) { - var got pbTypes.Interface - - expected := pbTypes.Interface{ - Device: testIfaceName, - Name: testIfaceName, - Mtu: uint64(testMTU), - HwAddr: testHwAddr, - } - - n := &netmon{ - sharedFile: filepath.Join(testStorageParentPath, testSharedFile), - } - - err := os.MkdirAll(testStorageParentPath, storageDirPerm) - defer os.RemoveAll(testStorageParentPath) - assert.Nil(t, err) - - err = n.storeDataToSend(expected) - assert.Nil(t, err) - - // Check the file has been created, check the content, and delete it. - _, err = os.Stat(n.sharedFile) - assert.Nil(t, err) - byteArray, err := ioutil.ReadFile(n.sharedFile) - assert.Nil(t, err) - err = json.Unmarshal(byteArray, &got) - assert.Nil(t, err) - assert.True(t, reflect.DeepEqual(expected, got), - "Got %+v\nExpected %+v", got, expected) -} - -func TestExecKataCmdSuccess(t *testing.T) { - trueBinPath, err := exec.LookPath("true") - assert.Nil(t, err) - assert.NotEmpty(t, trueBinPath) - - params := netmonParams{ - runtimePath: trueBinPath, - } - - n := &netmon{ - netmonParams: params, - sharedFile: filepath.Join(testStorageParentPath, testSharedFile), - } - - err = os.MkdirAll(testStorageParentPath, storageDirPerm) - assert.Nil(t, err) - defer os.RemoveAll(testStorageParentPath) - - file, err := os.Create(n.sharedFile) - assert.Nil(t, err) - assert.NotNil(t, file) - file.Close() - - _, err = os.Stat(n.sharedFile) - assert.Nil(t, err) - - err = n.execKataCmd("") - assert.Nil(t, err) - _, err = os.Stat(n.sharedFile) - assert.NotNil(t, err) -} - -func TestExecKataCmdFailure(t *testing.T) { - falseBinPath, err := exec.LookPath("false") - assert.Nil(t, err) - assert.NotEmpty(t, falseBinPath) - - params := netmonParams{ - runtimePath: falseBinPath, - } - - n := &netmon{ - netmonParams: params, - } - - err = n.execKataCmd("") - assert.NotNil(t, err) -} - -func TestActionsCLI(t *testing.T) { - trueBinPath, err := exec.LookPath("true") - assert.Nil(t, err) - assert.NotEmpty(t, trueBinPath) - - params := netmonParams{ - runtimePath: trueBinPath, - } - - n := &netmon{ - netmonParams: params, - sharedFile: filepath.Join(testStorageParentPath, testSharedFile), - } - - err = os.MkdirAll(testStorageParentPath, storageDirPerm) - assert.Nil(t, err) - defer os.RemoveAll(testStorageParentPath) - - // Test addInterfaceCLI - err = n.addInterfaceCLI(pbTypes.Interface{}) - assert.Nil(t, err) - - // Test delInterfaceCLI - err = n.delInterfaceCLI(pbTypes.Interface{}) - assert.Nil(t, err) - - // Test updateRoutesCLI - err = n.updateRoutesCLI([]pbTypes.Route{}) - assert.Nil(t, err) - - tearDownNetworkCb := testSetupNetwork(t) - defer tearDownNetworkCb() - - handler, err := netlink.NewHandle(netlinkFamily) - assert.Nil(t, err) - assert.NotNil(t, handler) - defer handler.Close() - - n.netHandler = handler - - // Test updateRoutes - err = n.updateRoutes() - assert.Nil(t, err) - - // Test handleRTMDelRoute - err = n.handleRTMDelRoute(netlink.RouteUpdate{}) - assert.Nil(t, err) -} - -func TestHandleRTMNewAddr(t *testing.T) { - n := &netmon{} - - err := n.handleRTMNewAddr(netlink.LinkUpdate{}) - assert.Nil(t, err) -} - -func TestHandleRTMDelAddr(t *testing.T) { - n := &netmon{} - - err := n.handleRTMDelAddr(netlink.LinkUpdate{}) - assert.Nil(t, err) -} - -func TestHandleRTMNewLink(t *testing.T) { - n := &netmon{} - ev := netlink.LinkUpdate{ - Link: &netlink.Dummy{}, - } - - // LinkAttrs is nil - err := n.handleRTMNewLink(ev) - assert.Nil(t, err) - - // Link name contains "kata" suffix - ev = netlink.LinkUpdate{ - Link: &netlink.Dummy{ - LinkAttrs: netlink.LinkAttrs{ - Name: "foo_kata", - }, - }, - } - err = n.handleRTMNewLink(ev) - assert.Nil(t, err) - - // Interface already exist in list - n.netIfaces = make(map[int]pbTypes.Interface) - n.netIfaces[testIfaceIndex] = pbTypes.Interface{} - ev = netlink.LinkUpdate{ - Link: &netlink.Dummy{ - LinkAttrs: netlink.LinkAttrs{ - Name: "foo0", - }, - }, - } - ev.Index = testIfaceIndex - err = n.handleRTMNewLink(ev) - assert.Nil(t, err) - - // Flags are not up and running - n.netIfaces = make(map[int]pbTypes.Interface) - ev = netlink.LinkUpdate{ - Link: &netlink.Dummy{ - LinkAttrs: netlink.LinkAttrs{ - Name: "foo0", - }, - }, - } - ev.Index = testIfaceIndex - err = n.handleRTMNewLink(ev) - assert.Nil(t, err) - - // Invalid link - n.netIfaces = make(map[int]pbTypes.Interface) - ev = netlink.LinkUpdate{ - Link: &netlink.Dummy{ - LinkAttrs: netlink.LinkAttrs{ - Name: "foo0", - }, - }, - } - ev.Index = testIfaceIndex - ev.Flags = unix.IFF_UP | unix.IFF_RUNNING - handler, err := netlink.NewHandle(netlinkFamily) - assert.Nil(t, err) - assert.NotNil(t, handler) - defer handler.Close() - n.netHandler = handler - err = n.handleRTMNewLink(ev) - assert.NotNil(t, err) -} - -func TestHandleRTMDelLink(t *testing.T) { - n := &netmon{} - ev := netlink.LinkUpdate{ - Link: &netlink.Dummy{}, - } - - // LinkAttrs is nil - err := n.handleRTMDelLink(ev) - assert.Nil(t, err) - - // Link name contains "kata" suffix - ev = netlink.LinkUpdate{ - Link: &netlink.Dummy{ - LinkAttrs: netlink.LinkAttrs{ - Name: "foo_kata", - }, - }, - } - err = n.handleRTMDelLink(ev) - assert.Nil(t, err) - - // Interface does not exist in list - n.netIfaces = make(map[int]pbTypes.Interface) - ev = netlink.LinkUpdate{ - Link: &netlink.Dummy{ - LinkAttrs: netlink.LinkAttrs{ - Name: "foo0", - }, - }, - } - ev.Index = testIfaceIndex - err = n.handleRTMDelLink(ev) - assert.Nil(t, err) -} - -func TestHandleRTMNewRouteIfaceNotFound(t *testing.T) { - n := &netmon{ - netIfaces: make(map[int]pbTypes.Interface), - } - - err := n.handleRTMNewRoute(netlink.RouteUpdate{}) - assert.Nil(t, err) -} - -func TestHandleLinkEvent(t *testing.T) { - n := &netmon{} - ev := netlink.LinkUpdate{} - - // Unknown event - err := n.handleLinkEvent(ev) - assert.Nil(t, err) - - // DONE event - ev.Header.Type = unix.NLMSG_DONE - err = n.handleLinkEvent(ev) - assert.Nil(t, err) - - // ERROR event - ev.Header.Type = unix.NLMSG_ERROR - err = n.handleLinkEvent(ev) - assert.NotNil(t, err) - - // NEWADDR event - ev.Header.Type = unix.RTM_NEWADDR - err = n.handleLinkEvent(ev) - assert.Nil(t, err) - - // DELADDR event - ev.Header.Type = unix.RTM_DELADDR - err = n.handleLinkEvent(ev) - assert.Nil(t, err) - - // NEWLINK event - ev.Header.Type = unix.RTM_NEWLINK - ev.Link = &netlink.Dummy{} - err = n.handleLinkEvent(ev) - assert.Nil(t, err) - - // DELLINK event - ev.Header.Type = unix.RTM_DELLINK - ev.Link = &netlink.Dummy{} - err = n.handleLinkEvent(ev) - assert.Nil(t, err) -} - -func TestHandleRouteEvent(t *testing.T) { - n := &netmon{} - ev := netlink.RouteUpdate{} - - // Unknown event - err := n.handleRouteEvent(ev) - assert.Nil(t, err) - - // RTM_NEWROUTE event - ev.Type = unix.RTM_NEWROUTE - err = n.handleRouteEvent(ev) - assert.Nil(t, err) - - trueBinPath, err := exec.LookPath("true") - assert.Nil(t, err) - assert.NotEmpty(t, trueBinPath) - - n.runtimePath = trueBinPath - n.sharedFile = filepath.Join(testStorageParentPath, testSharedFile) - - err = os.MkdirAll(testStorageParentPath, storageDirPerm) - assert.Nil(t, err) - defer os.RemoveAll(testStorageParentPath) - - tearDownNetworkCb := testSetupNetwork(t) - defer tearDownNetworkCb() - - handler, err := netlink.NewHandle(netlinkFamily) - assert.Nil(t, err) - assert.NotNil(t, handler) - defer handler.Close() - - n.netHandler = handler - - // RTM_DELROUTE event - ev.Type = unix.RTM_DELROUTE - err = n.handleRouteEvent(ev) - assert.Nil(t, err) -} diff --git a/src/runtime/config/configuration-acrn.toml.in b/src/runtime/config/configuration-acrn.toml.in index 69b203703..f9fb39e23 100644 --- a/src/runtime/config/configuration-acrn.toml.in +++ b/src/runtime/config/configuration-acrn.toml.in @@ -147,21 +147,6 @@ block_device_driver = "@DEFBLOCKSTORAGEDRIVER_ACRN@" # (default: 30) #dial_timeout = 30 -[netmon] -# If enabled, the network monitoring process gets started when the -# sandbox is created. This allows for the detection of some additional -# network being added to the existing network namespace, after the -# sandbox has been created. -# (default: disabled) -#enable_netmon = true - -# Specify the path to the netmon binary. -path = "@NETMONPATH@" - -# If enabled, netmon messages will be sent to the system log -# (default: disabled) -#enable_debug = true - [runtime] # If enabled, the runtime will log additional debug messages to the # system log @@ -217,7 +202,6 @@ disable_guest_seccomp=@DEFDISABLEGUESTSECCOMP@ # If enabled, the runtime will not create a network namespace for shim and hypervisor processes. # This option may have some potential impacts to your host. It should only be used when you know what you're doing. -# `disable_new_netns` conflicts with `enable_netmon` # `disable_new_netns` conflicts with `internetworking_model=bridged` and `internetworking_model=macvtap`. It works only # with `internetworking_model=none`. The tap device will be in the host network namespace and can connect to a bridge # (like OVS) directly. diff --git a/src/runtime/config/configuration-clh.toml.in b/src/runtime/config/configuration-clh.toml.in index 2dc8a8322..a016e0975 100644 --- a/src/runtime/config/configuration-clh.toml.in +++ b/src/runtime/config/configuration-clh.toml.in @@ -169,22 +169,6 @@ block_device_driver = "virtio-blk" # (default: 30) #dial_timeout = 30 -[netmon] -# If enabled, the network monitoring process gets started when the -# sandbox is created. This allows for the detection of some additional -# network being added to the existing network namespace, after the -# sandbox has been created. -# (default: disabled) -#enable_netmon = true - -# Specify the path to the netmon binary. -path = "@NETMONPATH@" - -# If enabled, netmon messages will be sent to the system log -# (default: disabled) -#enable_debug = true - - [runtime] # If enabled, the runtime will log additional debug messages to the # system log @@ -240,7 +224,6 @@ disable_guest_seccomp=@DEFDISABLEGUESTSECCOMP@ # If enabled, the runtime will not create a network namespace for shim and hypervisor processes. # This option may have some potential impacts to your host. It should only be used when you know what you're doing. -# `disable_new_netns` conflicts with `enable_netmon` # `disable_new_netns` conflicts with `internetworking_model=bridged` and `internetworking_model=macvtap`. It works only # with `internetworking_model=none`. The tap device will be in the host network namespace and can connect to a bridge # (like OVS) directly. diff --git a/src/runtime/config/configuration-fc.toml.in b/src/runtime/config/configuration-fc.toml.in index d03e452f1..8454a0a3d 100644 --- a/src/runtime/config/configuration-fc.toml.in +++ b/src/runtime/config/configuration-fc.toml.in @@ -282,21 +282,6 @@ kernel_modules=[] # (default: 30) #dial_timeout = 30 -[netmon] -# If enabled, the network monitoring process gets started when the -# sandbox is created. This allows for the detection of some additional -# network being added to the existing network namespace, after the -# sandbox has been created. -# (default: disabled) -#enable_netmon = true - -# Specify the path to the netmon binary. -path = "@NETMONPATH@" - -# If enabled, netmon messages will be sent to the system log -# (default: disabled) -#enable_debug = true - [runtime] # If enabled, the runtime will log additional debug messages to the # system log @@ -345,7 +330,6 @@ disable_guest_seccomp=@DEFDISABLEGUESTSECCOMP@ # If enabled, the runtime will not create a network namespace for shim and hypervisor processes. # This option may have some potential impacts to your host. It should only be used when you know what you're doing. -# `disable_new_netns` conflicts with `enable_netmon` # `disable_new_netns` conflicts with `internetworking_model=tcfilter` and `internetworking_model=macvtap`. It works only # with `internetworking_model=none`. The tap device will be in the host network namespace and can connect to a bridge # (like OVS) directly. diff --git a/src/runtime/config/configuration-qemu.toml.in b/src/runtime/config/configuration-qemu.toml.in index 5bc8e9cc3..883dc6789 100644 --- a/src/runtime/config/configuration-qemu.toml.in +++ b/src/runtime/config/configuration-qemu.toml.in @@ -458,21 +458,6 @@ kernel_modules=[] # (default: 30) #dial_timeout = 30 -[netmon] -# If enabled, the network monitoring process gets started when the -# sandbox is created. This allows for the detection of some additional -# network being added to the existing network namespace, after the -# sandbox has been created. -# (default: disabled) -#enable_netmon = true - -# Specify the path to the netmon binary. -path = "@NETMONPATH@" - -# If enabled, netmon messages will be sent to the system log -# (default: disabled) -#enable_debug = true - [runtime] # If enabled, the runtime will log additional debug messages to the # system log @@ -521,7 +506,6 @@ disable_guest_seccomp=@DEFDISABLEGUESTSECCOMP@ # If enabled, the runtime will not create a network namespace for shim and hypervisor processes. # This option may have some potential impacts to your host. It should only be used when you know what you're doing. -# `disable_new_netns` conflicts with `enable_netmon` # `disable_new_netns` conflicts with `internetworking_model=tcfilter` and `internetworking_model=macvtap`. It works only # with `internetworking_model=none`. The tap device will be in the host network namespace and can connect to a bridge # (like OVS) directly. diff --git a/src/runtime/pkg/containerd-shim-v2/create_test.go b/src/runtime/pkg/containerd-shim-v2/create_test.go index 5ad96f149..7b6f9f57a 100644 --- a/src/runtime/pkg/containerd-shim-v2/create_test.go +++ b/src/runtime/pkg/containerd-shim-v2/create_test.go @@ -328,7 +328,6 @@ func createAllRuntimeConfigFiles(dir, hypervisor string) (config string, err err kernelParams := "foo=bar xyz" imagePath := path.Join(dir, "image") shimPath := path.Join(dir, "shim") - netmonPath := path.Join(dir, "netmon") logDir := path.Join(dir, "logs") logPath := path.Join(logDir, "runtime.log") machineType := "machineType" @@ -349,7 +348,6 @@ func createAllRuntimeConfigFiles(dir, hypervisor string) (config string, err err KernelParams: kernelParams, MachineType: machineType, ShimPath: shimPath, - NetmonPath: netmonPath, LogPath: logPath, DisableBlock: disableBlockDevice, BlockDeviceDriver: blockDeviceDriver, diff --git a/src/runtime/pkg/katatestutils/utils.go b/src/runtime/pkg/katatestutils/utils.go index 617943a03..c5d79e6c4 100644 --- a/src/runtime/pkg/katatestutils/utils.go +++ b/src/runtime/pkg/katatestutils/utils.go @@ -215,7 +215,6 @@ type RuntimeConfigOptions struct { KernelParams string MachineType string ShimPath string - NetmonPath string LogPath string BlockDeviceDriver string SharedFS string @@ -237,7 +236,6 @@ type RuntimeConfigOptions struct { RuntimeDebug bool RuntimeTrace bool ShimDebug bool - NetmonDebug bool AgentDebug bool AgentTrace bool EnablePprof bool @@ -331,10 +329,6 @@ func MakeRuntimeConfigFileData(config RuntimeConfigOptions) string { enable_debug = ` + strconv.FormatBool(config.AgentDebug) + ` enable_tracing = ` + strconv.FormatBool(config.AgentTrace) + ` - [netmon] - path = "` + config.NetmonPath + `" - enable_debug = ` + strconv.FormatBool(config.NetmonDebug) + ` - [runtime] enable_debug = ` + strconv.FormatBool(config.RuntimeDebug) + ` enable_tracing = ` + strconv.FormatBool(config.RuntimeTrace) + ` diff --git a/src/runtime/pkg/katautils/config-settings.go.in b/src/runtime/pkg/katautils/config-settings.go.in index bd793e23f..8f2ae6bfd 100644 --- a/src/runtime/pkg/katautils/config-settings.go.in +++ b/src/runtime/pkg/katautils/config-settings.go.in @@ -97,5 +97,3 @@ const defaultVMCacheEndpoint string = "/var/run/kata-containers/cache.sock" // Default config file used by stateless systems. var defaultRuntimeConfiguration = "@CONFIG_PATH@" - -var defaultNetmonPath = "/usr/libexec/kata-containers/kata-netmon" diff --git a/src/runtime/pkg/katautils/config.go b/src/runtime/pkg/katautils/config.go index 1e3baca9b..5f9b7f6fb 100644 --- a/src/runtime/pkg/katautils/config.go +++ b/src/runtime/pkg/katautils/config.go @@ -56,7 +56,6 @@ type tomlConfig struct { Agent map[string]agent Runtime runtime Image image - Netmon netmon Factory factory } @@ -162,12 +161,6 @@ type agent struct { DialTimeout uint32 `toml:"dial_timeout"` } -type netmon struct { - Path string `toml:"path"` - Debug bool `toml:"enable_debug"` - Enable bool `toml:"enable_netmon"` -} - func (h hypervisor) path() (string, error) { p := h.Path @@ -506,22 +499,6 @@ func (a agent) kernelModules() []string { return a.KernelModules } -func (n netmon) enable() bool { - return n.Enable -} - -func (n netmon) path() string { - if n.Path == "" { - return defaultNetmonPath - } - - return n.Path -} - -func (n netmon) debug() bool { - return n.Debug -} - func newFirecrackerHypervisorConfig(h hypervisor) (vc.HypervisorConfig, error) { hypervisor, err := h.path() if err != nil { @@ -1014,12 +991,6 @@ func updateRuntimeConfig(configPath string, tomlConf tomlConfig, config *oci.Run } config.FactoryConfig = fConfig - config.NetmonConfig = vc.NetmonConfig{ - Path: tomlConf.Netmon.path(), - Debug: tomlConf.Netmon.debug(), - Enable: tomlConf.Netmon.enable(), - } - err = SetKernelParams(config) if err != nil { return err @@ -1262,9 +1233,6 @@ func checkConfig(config oci.RuntimeConfig) error { // Because it is an expert option and conflicts with some other common configs. func checkNetNsConfig(config oci.RuntimeConfig) error { if config.DisableNewNetNs { - if config.NetmonConfig.Enable { - return fmt.Errorf("config disable_new_netns conflicts with enable_netmon") - } if config.InterNetworkModel != vc.NetXConnectNoneModel { return fmt.Errorf("config disable_new_netns only works with 'none' internetworking_model") } diff --git a/src/runtime/pkg/katautils/config_test.go b/src/runtime/pkg/katautils/config_test.go index acf92efec..0c596d84e 100644 --- a/src/runtime/pkg/katautils/config_test.go +++ b/src/runtime/pkg/katautils/config_test.go @@ -29,7 +29,6 @@ var ( hypervisorDebug = false runtimeDebug = false runtimeTrace = false - netmonDebug = false agentDebug = false agentTrace = false enablePprof = true @@ -74,7 +73,6 @@ func createAllRuntimeConfigFiles(dir, hypervisor string) (config testRuntimeConf kernelPath := path.Join(dir, "kernel") kernelParams := "foo=bar xyz" imagePath := path.Join(dir, "image") - netmonPath := path.Join(dir, "netmon") logDir := path.Join(dir, "logs") logPath := path.Join(logDir, "runtime.log") machineType := "machineType" @@ -95,7 +93,6 @@ func createAllRuntimeConfigFiles(dir, hypervisor string) (config testRuntimeConf ImagePath: imagePath, KernelParams: kernelParams, MachineType: machineType, - NetmonPath: netmonPath, LogPath: logPath, DefaultGuestHookPath: defaultGuestHookPath, DisableBlock: disableBlockDevice, @@ -111,7 +108,6 @@ func createAllRuntimeConfigFiles(dir, hypervisor string) (config testRuntimeConf HypervisorDebug: hypervisorDebug, RuntimeDebug: runtimeDebug, RuntimeTrace: runtimeTrace, - NetmonDebug: netmonDebug, AgentDebug: agentDebug, AgentTrace: agentTrace, SharedFS: sharedFS, @@ -180,12 +176,6 @@ func createAllRuntimeConfigFiles(dir, hypervisor string) (config testRuntimeConf LongLiveConn: true, } - netmonConfig := vc.NetmonConfig{ - Path: netmonPath, - Debug: false, - Enable: false, - } - factoryConfig := oci.FactoryConfig{ TemplatePath: defaultTemplatePath, VMCacheEndpoint: defaultVMCacheEndpoint, @@ -197,7 +187,6 @@ func createAllRuntimeConfigFiles(dir, hypervisor string) (config testRuntimeConf AgentConfig: agentConfig, - NetmonConfig: netmonConfig, DisableNewNetNs: disableNewNetNs, EnablePprof: enablePprof, JaegerEndpoint: jaegerEndpoint, @@ -493,7 +482,6 @@ func TestMinimalRuntimeConfig(t *testing.T) { defaultHypervisorPath = hypervisorPath jailerPath := path.Join(dir, "jailer") defaultJailerPath = jailerPath - netmonPath := path.Join(dir, "netmon") imagePath := path.Join(dir, "image.img") initrdPath := path.Join(dir, "initrd.img") @@ -535,8 +523,6 @@ func TestMinimalRuntimeConfig(t *testing.T) { [agent.kata] debug_console_enabled=true kernel_modules=["a", "b", "c"] - [netmon] - path = "` + netmonPath + `" ` orgVHostVSockDevicePath := utils.VHostVSockDevicePath @@ -561,11 +547,6 @@ func TestMinimalRuntimeConfig(t *testing.T) { t.Error(err) } - err = createEmptyFile(netmonPath) - if err != nil { - t.Error(err) - } - _, config, err := LoadConfiguration(configPath, false) if err != nil { t.Fatal(err) @@ -597,12 +578,6 @@ func TestMinimalRuntimeConfig(t *testing.T) { KernelModules: []string{"a", "b", "c"}, } - expectedNetmonConfig := vc.NetmonConfig{ - Path: netmonPath, - Debug: false, - Enable: false, - } - expectedFactoryConfig := oci.FactoryConfig{ TemplatePath: defaultTemplatePath, VMCacheEndpoint: defaultVMCacheEndpoint, @@ -614,8 +589,6 @@ func TestMinimalRuntimeConfig(t *testing.T) { AgentConfig: expectedAgentConfig, - NetmonConfig: expectedNetmonConfig, - FactoryConfig: expectedFactoryConfig, } err = SetKernelParams(&expectedConfig) @@ -1553,9 +1526,6 @@ func TestCheckNetNsConfig(t *testing.T) { config := oci.RuntimeConfig{ DisableNewNetNs: true, - NetmonConfig: vc.NetmonConfig{ - Enable: true, - }, } err := checkNetNsConfig(config) assert.Error(err) diff --git a/src/runtime/pkg/oci/utils.go b/src/runtime/pkg/oci/utils.go index bbef21e9b..ec1266854 100644 --- a/src/runtime/pkg/oci/utils.go +++ b/src/runtime/pkg/oci/utils.go @@ -109,7 +109,6 @@ type RuntimeConfig struct { FactoryConfig FactoryConfig HypervisorConfig vc.HypervisorConfig - NetmonConfig vc.NetmonConfig AgentConfig vc.KataAgentConfig //Determines how the VM should be connected to the @@ -314,12 +313,6 @@ func networkConfig(ocispec specs.Spec, config RuntimeConfig) (vc.NetworkConfig, netConf.InterworkingModel = config.InterNetworkModel netConf.DisableNewNetNs = config.DisableNewNetNs - netConf.NetmonConfig = vc.NetmonConfig{ - Path: config.NetmonConfig.Path, - Debug: config.NetmonConfig.Debug, - Enable: config.NetmonConfig.Enable, - } - return netConf, nil } diff --git a/src/runtime/virtcontainers/documentation/api/1.0/api.md b/src/runtime/virtcontainers/documentation/api/1.0/api.md index 4cec38137..403a71280 100644 --- a/src/runtime/virtcontainers/documentation/api/1.0/api.md +++ b/src/runtime/virtcontainers/documentation/api/1.0/api.md @@ -366,7 +366,6 @@ type NetworkConfig struct { NetNSPath string NetNsCreated bool DisableNewNetNs bool - NetmonConfig NetmonConfig InterworkingModel NetInterworkingModel } ``` diff --git a/src/runtime/virtcontainers/netmon.go b/src/runtime/virtcontainers/netmon.go deleted file mode 100644 index 58483df22..000000000 --- a/src/runtime/virtcontainers/netmon.go +++ /dev/null @@ -1,96 +0,0 @@ -// Copyright (c) 2018 Intel Corporation -// -// SPDX-License-Identifier: Apache-2.0 -// - -package virtcontainers - -import ( - "fmt" - "os/exec" - "syscall" - - "github.com/sirupsen/logrus" -) - -// NetmonConfig is the structure providing specific configuration -// for the network monitor. -type NetmonConfig struct { - Path string - Debug bool - Enable bool -} - -// netmonParams is the structure providing specific parameters needed -// for the execution of the network monitor binary. -type netmonParams struct { - netmonPath string - logLevel string - runtime string - sandboxID string - debug bool -} - -func netmonLogger() *logrus.Entry { - return virtLog.WithField("subsystem", "netmon") -} - -func prepareNetMonParams(params netmonParams) ([]string, error) { - if params.netmonPath == "" { - return []string{}, fmt.Errorf("Netmon path is empty") - } - if params.runtime == "" { - return []string{}, fmt.Errorf("Netmon runtime path is empty") - } - if params.sandboxID == "" { - return []string{}, fmt.Errorf("Netmon sandbox ID is empty") - } - - args := []string{params.netmonPath, - "-r", params.runtime, - "-s", params.sandboxID, - } - - if params.debug { - args = append(args, "-d") - } - if params.logLevel != "" { - args = append(args, []string{"-log", params.logLevel}...) - } - - return args, nil -} - -func startNetmon(params netmonParams) (int, error) { - args, err := prepareNetMonParams(params) - if err != nil { - return -1, err - } - - cmd := exec.Command(args[0], args[1:]...) - if err := cmd.Start(); err != nil { - return -1, err - } - - return cmd.Process.Pid, nil -} - -func stopNetmon(pid int) error { - if pid <= 0 { - return nil - } - - sig := syscall.SIGKILL - - netmonLogger().WithFields( - logrus.Fields{ - "netmon-pid": pid, - "netmon-signal": sig, - }).Info("Stopping netmon") - - if err := syscall.Kill(pid, sig); err != nil && err != syscall.ESRCH { - return err - } - - return nil -} diff --git a/src/runtime/virtcontainers/netmon_test.go b/src/runtime/virtcontainers/netmon_test.go deleted file mode 100644 index 0a7f3bb53..000000000 --- a/src/runtime/virtcontainers/netmon_test.go +++ /dev/null @@ -1,61 +0,0 @@ -// Copyright (c) 2018 Intel Corporation -// -// SPDX-License-Identifier: Apache-2.0 -// - -package virtcontainers - -import ( - "reflect" - "testing" - - "github.com/stretchr/testify/assert" -) - -const ( - testNetmonPath = "/foo/bar/netmon" - testRuntimePath = "/foo/bar/runtime" -) - -func TestNetmonLogger(t *testing.T) { - got := netmonLogger() - expected := virtLog.WithField("subsystem", "netmon") - assert.True(t, reflect.DeepEqual(expected, got), - "Got %+v\nExpected %+v", got, expected) -} - -func TestPrepareNetMonParams(t *testing.T) { - // Empty netmon path - params := netmonParams{} - got, err := prepareNetMonParams(params) - assert.NotNil(t, err) - assert.Equal(t, got, []string{}) - - // Empty runtime path - params.netmonPath = testNetmonPath - got, err = prepareNetMonParams(params) - assert.NotNil(t, err) - assert.Equal(t, got, []string{}) - - // Empty sandbox ID - params.runtime = testRuntimePath - got, err = prepareNetMonParams(params) - assert.NotNil(t, err) - assert.Equal(t, got, []string{}) - - // Successful case - params.sandboxID = testSandboxID - got, err = prepareNetMonParams(params) - assert.Nil(t, err) - expected := []string{testNetmonPath, - "-r", testRuntimePath, - "-s", testSandboxID} - assert.True(t, reflect.DeepEqual(expected, got), - "Got %+v\nExpected %+v", got, expected) -} - -func TestStopNetmon(t *testing.T) { - pid := -1 - err := stopNetmon(pid) - assert.Nil(t, err) -} diff --git a/src/runtime/virtcontainers/network.go b/src/runtime/virtcontainers/network.go index 0af47f46a..2bb39a600 100644 --- a/src/runtime/virtcontainers/network.go +++ b/src/runtime/virtcontainers/network.go @@ -178,7 +178,6 @@ type NetworkInterfacePair struct { // NetworkConfig is the network configuration related to a network. type NetworkConfig struct { NetNSPath string - NetmonConfig NetmonConfig InterworkingModel NetInterworkingModel NetNsCreated bool DisableNewNetNs bool @@ -193,7 +192,6 @@ type NetworkNamespace struct { NetNsPath string Endpoints []Endpoint NetNsCreated bool - NetmonPID int } func createLink(netHandle *netlink.Handle, name string, expectedLink netlink.Link, queues int) (netlink.Link, []*os.File, error) { diff --git a/src/runtime/virtcontainers/persist.go b/src/runtime/virtcontainers/persist.go index b8fc2c8ab..e52c6315e 100644 --- a/src/runtime/virtcontainers/persist.go +++ b/src/runtime/virtcontainers/persist.go @@ -164,7 +164,6 @@ func (s *Sandbox) dumpAgent(ss *persistapi.SandboxState) { func (s *Sandbox) dumpNetwork(ss *persistapi.SandboxState) { ss.Network = persistapi.NetworkInfo{ NetNsPath: s.networkNS.NetNsPath, - NetmonPID: s.networkNS.NetmonPID, NetNsCreated: s.networkNS.NetNsCreated, } for _, e := range s.networkNS.Endpoints { @@ -367,7 +366,6 @@ func (c *Container) loadContProcess(cs persistapi.ContainerState) { func (s *Sandbox) loadNetwork(netInfo persistapi.NetworkInfo) { s.networkNS = NetworkNamespace{ NetNsPath: netInfo.NetNsPath, - NetmonPID: netInfo.NetmonPID, NetNsCreated: netInfo.NetNsCreated, } diff --git a/src/runtime/virtcontainers/persist/api/network.go b/src/runtime/virtcontainers/persist/api/network.go index f1c1cc2f8..51c3aac62 100644 --- a/src/runtime/virtcontainers/persist/api/network.go +++ b/src/runtime/virtcontainers/persist/api/network.go @@ -98,6 +98,5 @@ type NetworkEndpoint struct { type NetworkInfo struct { NetNsPath string Endpoints []NetworkEndpoint - NetmonPID int NetNsCreated bool } diff --git a/src/runtime/virtcontainers/sandbox.go b/src/runtime/virtcontainers/sandbox.go index c9e07947b..74e7f7f73 100644 --- a/src/runtime/virtcontainers/sandbox.go +++ b/src/runtime/virtcontainers/sandbox.go @@ -778,40 +778,6 @@ func (s *Sandbox) Delete(ctx context.Context) error { return s.store.Destroy(s.id) } -func (s *Sandbox) startNetworkMonitor(ctx context.Context) error { - span, ctx := katatrace.Trace(ctx, s.Logger(), "startNetworkMonitor", sandboxTracingTags, map[string]string{"sandbox_id": s.id}) - defer span.End() - - binPath, err := os.Executable() - if err != nil { - return err - } - - logLevel := "info" - if s.config.NetworkConfig.NetmonConfig.Debug { - logLevel = "debug" - } - - params := netmonParams{ - netmonPath: s.config.NetworkConfig.NetmonConfig.Path, - debug: s.config.NetworkConfig.NetmonConfig.Debug, - logLevel: logLevel, - runtime: binPath, - sandboxID: s.id, - } - - return s.network.Run(ctx, s.networkNS.NetNsPath, func() error { - pid, err := startNetmon(params) - if err != nil { - return err - } - - s.networkNS.NetmonPID = pid - - return nil - }) -} - func (s *Sandbox) createNetwork(ctx context.Context) error { if s.config.NetworkConfig.DisableNewNetNs || s.config.NetworkConfig.NetNSPath == "" { @@ -838,18 +804,11 @@ func (s *Sandbox) createNetwork(ctx context.Context) error { } s.networkNS.Endpoints = endpoints - - if s.config.NetworkConfig.NetmonConfig.Enable { - if err := s.startNetworkMonitor(ctx); err != nil { - return err - } - } } return nil } func (s *Sandbox) postCreatedNetwork(ctx context.Context) error { - return s.network.PostAdd(ctx, &s.networkNS, s.factory != nil) } @@ -857,12 +816,6 @@ func (s *Sandbox) removeNetwork(ctx context.Context) error { span, ctx := katatrace.Trace(ctx, s.Logger(), "removeNetwork", sandboxTracingTags, map[string]string{"sandbox_id": s.id}) defer span.End() - if s.config.NetworkConfig.NetmonConfig.Enable { - if err := stopNetmon(s.networkNS.NetmonPID); err != nil { - return err - } - } - return s.network.Remove(ctx, &s.networkNS, s.hypervisor) } @@ -1218,12 +1171,6 @@ func (s *Sandbox) startVM(ctx context.Context) (err error) { } s.networkNS.Endpoints = endpoints - - if s.config.NetworkConfig.NetmonConfig.Enable { - if err := s.startNetworkMonitor(ctx); err != nil { - return err - } - } } s.Logger().Info("VM started") diff --git a/src/runtime/virtcontainers/sandbox_test.go b/src/runtime/virtcontainers/sandbox_test.go index 435a13da3..3d6f88e21 100644 --- a/src/runtime/virtcontainers/sandbox_test.go +++ b/src/runtime/virtcontainers/sandbox_test.go @@ -10,7 +10,6 @@ import ( "fmt" "io/ioutil" "os" - "os/exec" "path" "path/filepath" "strings" @@ -1300,33 +1299,6 @@ func TestGetNetNs(t *testing.T) { assert.Equal(t, netNs, expected) } -func TestStartNetworkMonitor(t *testing.T) { - if os.Getuid() != 0 { - t.Skip("Test disabled as requires root user") - } - trueBinPath, err := exec.LookPath("true") - assert.Nil(t, err) - assert.NotEmpty(t, trueBinPath) - - s := &Sandbox{ - id: testSandboxID, - config: &SandboxConfig{ - NetworkConfig: NetworkConfig{ - NetmonConfig: NetmonConfig{ - Path: trueBinPath, - }, - }, - }, - networkNS: NetworkNamespace{ - NetNsPath: fmt.Sprintf("/proc/%d/task/%d/ns/net", os.Getpid(), unix.Gettid()), - }, - ctx: context.Background(), - } - - err = s.startNetworkMonitor(context.Background()) - assert.Nil(t, err) -} - func TestSandboxStopStopped(t *testing.T) { s := &Sandbox{ ctx: context.Background(), From c7349d0bf1fd8a14c84f446270f492bc8a2d473f Mon Sep 17 00:00:00 2001 From: "James O. D. Hunt" Date: Mon, 22 Nov 2021 14:21:24 +0000 Subject: [PATCH 36/66] agent-ctl: Simplify error handling Replace `ok_or().map_err()` combinations with the simpler `ok_or_else()` construct. Signed-off-by: James O. D. Hunt --- tools/agent-ctl/src/main.rs | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/tools/agent-ctl/src/main.rs b/tools/agent-ctl/src/main.rs index 00519c932..8dac09cfd 100644 --- a/tools/agent-ctl/src/main.rs +++ b/tools/agent-ctl/src/main.rs @@ -134,16 +134,14 @@ fn make_examples_text(program_name: &str) -> String { fn connect(name: &str, global_args: clap::ArgMatches) -> Result<()> { let args = global_args .subcommand_matches("connect") - .ok_or("BUG: missing sub-command arguments".to_string()) - .map_err(|e| anyhow!(e))?; + .ok_or_else(|| anyhow!("BUG: missing sub-command arguments"))?; let interactive = args.is_present("interactive"); let ignore_errors = args.is_present("ignore-errors"); let server_address = args .value_of("server-address") - .ok_or("need server adddress".to_string()) - .map_err(|e| anyhow!(e))? + .ok_or_else(|| anyhow!("need server adddress"))? .to_string(); let mut commands: Vec<&str> = Vec::new(); @@ -151,8 +149,7 @@ fn connect(name: &str, global_args: clap::ArgMatches) -> Result<()> { if !interactive { commands = args .values_of("cmd") - .ok_or("need commands to send to the server".to_string()) - .map_err(|e| anyhow!(e))? + .ok_or_else(|| anyhow!("need commands to send to the server"))? .collect(); } @@ -304,8 +301,7 @@ fn real_main() -> Result<()> { let subcmd = args .subcommand_name() - .ok_or("need sub-command".to_string()) - .map_err(|e| anyhow!(e))?; + .ok_or_else(|| anyhow!("need sub-command"))?; match subcmd { "generate-cid" => { From 46e459584dff14bc1b8459547ac379d1ac764b6d Mon Sep 17 00:00:00 2001 From: "James O. D. Hunt" Date: Mon, 22 Nov 2021 14:21:41 +0000 Subject: [PATCH 37/66] agent-ctl: Simplify main Make the `main()` function simpler. Signed-off-by: James O. D. Hunt --- tools/agent-ctl/src/main.rs | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/tools/agent-ctl/src/main.rs b/tools/agent-ctl/src/main.rs index 8dac09cfd..168194f06 100644 --- a/tools/agent-ctl/src/main.rs +++ b/tools/agent-ctl/src/main.rs @@ -324,11 +324,8 @@ fn real_main() -> Result<()> { } fn main() { - match real_main() { - Err(e) => { - eprintln!("ERROR: {}", e); - exit(1); - } - _ => (), - }; + if let Err(e) = real_main() { + eprintln!("ERROR: {}", e); + exit(1); + } } From 35db75baa1a89c987ac9e7580ba7d7706cbf87f9 Mon Sep 17 00:00:00 2001 From: "James O. D. Hunt" Date: Mon, 22 Nov 2021 14:22:35 +0000 Subject: [PATCH 38/66] agent-ctl: Remove redundant returns Remove a number of redundant `return`'s. Signed-off-by: James O. D. Hunt --- tools/agent-ctl/src/main.rs | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/tools/agent-ctl/src/main.rs b/tools/agent-ctl/src/main.rs index 168194f06..a4b11915e 100644 --- a/tools/agent-ctl/src/main.rs +++ b/tools/agent-ctl/src/main.rs @@ -306,19 +306,17 @@ fn real_main() -> Result<()> { match subcmd { "generate-cid" => { println!("{}", utils::random_container_id()); - return Ok(()); + Ok(()) } "generate-sid" => { println!("{}", utils::random_sandbox_id()); - return Ok(()); + Ok(()) } "examples" => { println!("{}", make_examples_text(name)); - return Ok(()); - } - "connect" => { - return connect(name, args); + Ok(()) } + "connect" => connect(name, args), _ => return Err(anyhow!(format!("invalid sub-command: {:?}", subcmd))), } } From 09abcd4dc6c51b78c55ef2c2f4f61f09ed380a14 Mon Sep 17 00:00:00 2001 From: "James O. D. Hunt" Date: Mon, 22 Nov 2021 14:23:23 +0000 Subject: [PATCH 39/66] agent-ctl: Remove some unwrap and expect calls Replace some `unwrap()` and `expect()` calls with code to return the error to the caller. Signed-off-by: James O. D. Hunt --- tools/agent-ctl/src/main.rs | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/tools/agent-ctl/src/main.rs b/tools/agent-ctl/src/main.rs index a4b11915e..88c12e984 100644 --- a/tools/agent-ctl/src/main.rs +++ b/tools/agent-ctl/src/main.rs @@ -153,8 +153,9 @@ fn connect(name: &str, global_args: clap::ArgMatches) -> Result<()> { .collect(); } - // Cannot fail as a default has been specified - let log_level_name = global_args.value_of("log-level").unwrap(); + let log_level_name = global_args + .value_of("log-level") + .ok_or_else(|| anyhow!("cannot get log level"))?; let log_level = logging::level_name_to_slog_level(log_level_name).map_err(|e| anyhow!(e))?; @@ -166,10 +167,10 @@ fn connect(name: &str, global_args: clap::ArgMatches) -> Result<()> { None => 0, }; - let hybrid_vsock_port: u64 = args + let hybrid_vsock_port = args .value_of("hybrid-vsock-port") - .ok_or("Need Hybrid VSOCK port number") - .map(|p| p.parse::().unwrap()) + .ok_or_else(|| anyhow!("Need Hybrid VSOCK port number"))? + .parse::() .map_err(|e| anyhow!("VSOCK port number must be an integer: {:?}", e))?; let bundle_dir = args.value_of("bundle-dir").unwrap_or("").to_string(); @@ -215,7 +216,7 @@ fn real_main() -> Result<()> { .long("log-level") .short("l") .help("specific log level") - .default_value(logging::slog_level_to_level_name(DEFAULT_LOG_LEVEL).unwrap()) + .default_value(logging::slog_level_to_level_name(DEFAULT_LOG_LEVEL).map_err(|e| anyhow!(e))?) .possible_values(&logging::get_log_levels()) .takes_value(true) .required(false), From a7d1c70c4bced794750a7d7f3778f873d0c1943c Mon Sep 17 00:00:00 2001 From: "James O. D. Hunt" Date: Mon, 15 Nov 2021 10:57:54 +0000 Subject: [PATCH 40/66] agent: Improve baremount Change `baremount()` to accept `Path` values rather than string values since: - `Path` is more natural given the function deals with paths. - This minimises the caller having to convert between string and `Path` types, which simplifies the surrounding code. Signed-off-by: James O. D. Hunt --- src/agent/src/mount.rs | 48 ++++++++++++++++++-------------------- src/agent/src/namespace.rs | 9 +++---- src/agent/src/rpc.rs | 21 ++++++++++++----- src/agent/src/sandbox.rs | 6 ++++- src/agent/src/watcher.rs | 8 +++---- 5 files changed, 52 insertions(+), 40 deletions(-) diff --git a/src/agent/src/mount.rs b/src/agent/src/mount.rs index 3d55f874f..fb92311bc 100644 --- a/src/agent/src/mount.rs +++ b/src/agent/src/mount.rs @@ -139,8 +139,8 @@ pub const STORAGE_HANDLER_LIST: &[&str] = &[ #[instrument] pub fn baremount( - source: &str, - destination: &str, + source: &Path, + destination: &Path, fs_type: &str, flags: MsFlags, options: &str, @@ -148,11 +148,11 @@ pub fn baremount( ) -> Result<()> { let logger = logger.new(o!("subsystem" => "baremount")); - if source.is_empty() { + if source.as_os_str().is_empty() { return Err(anyhow!("need mount source")); } - if destination.is_empty() { + if destination.as_os_str().is_empty() { return Err(anyhow!("need mount destination")); } @@ -444,16 +444,19 @@ fn mount_storage(logger: &Logger, storage: &Storage) -> Result<()> { let options_vec = options_vec.iter().map(String::as_str).collect(); let (flags, options) = parse_mount_flags_and_options(options_vec); + let source = Path::new(&storage.source); + let mount_point = Path::new(&storage.mount_point); + info!(logger, "mounting storage"; - "mount-source:" => storage.source.as_str(), - "mount-destination" => storage.mount_point.as_str(), + "mount-source" => source.display(), + "mount-destination" => mount_point.display(), "mount-fstype" => storage.fstype.as_str(), "mount-options" => options.as_str(), ); baremount( - storage.source.as_str(), - storage.mount_point.as_str(), + source, + mount_point, storage.fstype.as_str(), flags, options.as_str(), @@ -579,7 +582,10 @@ fn mount_to_rootfs(logger: &Logger, m: &InitMount) -> Result<()> { fs::create_dir_all(Path::new(m.dest)).context("could not create directory")?; - baremount(m.src, m.dest, m.fstype, flags, &options, logger).or_else(|e| { + let source = Path::new(m.src); + let dest = Path::new(m.dest); + + baremount(source, dest, m.fstype, flags, &options, logger).or_else(|e| { if m.src != "dev" { return Err(e); } @@ -937,14 +943,10 @@ mod tests { std::fs::create_dir_all(d).expect("failed to created directory"); } - let result = baremount( - &src_filename, - &dest_filename, - d.fs_type, - d.flags, - d.options, - &logger, - ); + let src = Path::new(&src_filename); + let dest = Path::new(&dest_filename); + + let result = baremount(src, dest, d.fs_type, d.flags, d.options, &logger); let msg = format!("{}: result: {:?}", msg, result); @@ -1021,15 +1023,11 @@ mod tests { .unwrap_or_else(|_| panic!("failed to create directory {}", d)); } + let src = Path::new(mnt_src_filename); + let dest = Path::new(mnt_dest_filename); + // Create an actual mount - let result = baremount( - mnt_src_filename, - mnt_dest_filename, - "bind", - MsFlags::MS_BIND, - "", - &logger, - ); + let result = baremount(src, dest, "bind", MsFlags::MS_BIND, "", &logger); assert!(result.is_ok(), "mount for test setup failed"); let tests = &[ diff --git a/src/agent/src/namespace.rs b/src/agent/src/namespace.rs index 061370a46..c821a0acb 100644 --- a/src/agent/src/namespace.rs +++ b/src/agent/src/namespace.rs @@ -104,7 +104,10 @@ impl Namespace { if let Err(err) = || -> Result<()> { let origin_ns_path = get_current_thread_ns_path(ns_type.get()); - File::open(Path::new(&origin_ns_path))?; + let source = Path::new(&origin_ns_path); + let destination = new_ns_path.as_path(); + + File::open(&source)?; // Create a new netns on the current thread. let cf = ns_type.get_flags(); @@ -115,8 +118,6 @@ impl Namespace { nix::unistd::sethostname(hostname.unwrap())?; } // Bind mount the new namespace from the current thread onto the mount point to persist it. - let source: &str = origin_ns_path.as_str(); - let destination: &str = new_ns_path.as_path().to_str().unwrap_or("none"); let mut flags = MsFlags::empty(); @@ -131,7 +132,7 @@ impl Namespace { baremount(source, destination, "none", flags, "", &logger).map_err(|e| { anyhow!( - "Failed to mount {} to {} with err:{:?}", + "Failed to mount {:?} to {:?} with err:{:?}", source, destination, e diff --git a/src/agent/src/rpc.rs b/src/agent/src/rpc.rs index 778da07fe..4cf58830e 100644 --- a/src/agent/src/rpc.rs +++ b/src/agent/src/rpc.rs @@ -1607,10 +1607,13 @@ async fn do_add_swap(sandbox: &Arc>, req: &AddSwapRequest) -> Res // - container rootfs bind mounted at ///rootfs // - modify container spec root to point to ///rootfs fn setup_bundle(cid: &str, spec: &mut Spec) -> Result { - if spec.root.is_none() { + let spec_root = if let Some(sr) = &spec.root { + sr + } else { return Err(nix::Error::Sys(Errno::EINVAL).into()); - } - let spec_root = spec.root.as_ref().unwrap(); + }; + + let spec_root_path = Path::new(&spec_root.path); let bundle_path = Path::new(CONTAINER_BASE).join(cid); let config_path = bundle_path.join("config.json"); @@ -1618,15 +1621,21 @@ fn setup_bundle(cid: &str, spec: &mut Spec) -> Result { fs::create_dir_all(&rootfs_path)?; baremount( - &spec_root.path, - rootfs_path.to_str().unwrap(), + spec_root_path, + &rootfs_path, "bind", MsFlags::MS_BIND, "", &sl!(), )?; + + let rootfs_path_name = rootfs_path + .to_str() + .ok_or_else(|| anyhow!("failed to convert rootfs to unicode"))? + .to_string(); + spec.root = Some(Root { - path: rootfs_path.to_str().unwrap().to_owned(), + path: rootfs_path_name, readonly: spec_root.readonly, }); diff --git a/src/agent/src/sandbox.rs b/src/agent/src/sandbox.rs index ddc18c5c9..526464014 100644 --- a/src/agent/src/sandbox.rs +++ b/src/agent/src/sandbox.rs @@ -458,10 +458,14 @@ mod tests { use slog::Logger; use std::fs::{self, File}; use std::os::unix::fs::PermissionsExt; + use std::path::Path; use tempfile::Builder; fn bind_mount(src: &str, dst: &str, logger: &Logger) -> Result<(), Error> { - baremount(src, dst, "bind", MsFlags::MS_BIND, "", logger) + let src_path = Path::new(src); + let dst_path = Path::new(dst); + + baremount(src_path, dst_path, "bind", MsFlags::MS_BIND, "", logger) } use serial_test::serial; diff --git a/src/agent/src/watcher.rs b/src/agent/src/watcher.rs index bb3fb1641..b3cd3f832 100644 --- a/src/agent/src/watcher.rs +++ b/src/agent/src/watcher.rs @@ -366,8 +366,8 @@ impl SandboxStorages { } match baremount( - entry.source_mount_point.to_str().unwrap(), - entry.target_mount_point.to_str().unwrap(), + entry.source_mount_point.as_path(), + entry.target_mount_point.as_path(), "bind", MsFlags::MS_BIND, "bind", @@ -477,8 +477,8 @@ impl BindWatcher { fs::create_dir_all(WATCH_MOUNT_POINT_PATH).await?; baremount( - "tmpfs", - WATCH_MOUNT_POINT_PATH, + Path::new("tmpfs"), + Path::new(WATCH_MOUNT_POINT_PATH), "tmpfs", MsFlags::empty(), "", From 351cef7b6a698f21af700ececf716b9d849e0049 Mon Sep 17 00:00:00 2001 From: "James O. D. Hunt" Date: Mon, 15 Nov 2021 10:59:18 +0000 Subject: [PATCH 41/66] agent: Remove unwrap from verify_cid() Improved the `verify_cid()` function that validates container ID's by removing the need for an `unwrap()`. Signed-off-by: James O. D. Hunt --- src/agent/src/rpc.rs | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/src/agent/src/rpc.rs b/src/agent/src/rpc.rs index 4cf58830e..bcabecddf 100644 --- a/src/agent/src/rpc.rs +++ b/src/agent/src/rpc.rs @@ -111,11 +111,18 @@ pub struct AgentService { // ^[a-zA-Z0-9][a-zA-Z0-9_.-]+$ // fn verify_cid(id: &str) -> Result<()> { - let valid = id.len() > 1 - && id.chars().next().unwrap().is_alphanumeric() - && id - .chars() - .all(|c| (c.is_alphanumeric() || ['.', '-', '_'].contains(&c))); + let mut chars = id.chars(); + + let valid = match chars.next() { + Some(first) + if first.is_alphanumeric() + && id.len() > 1 + && chars.all(|c| c.is_alphanumeric() || ['.', '-', '_'].contains(&c)) => + { + true + } + _ => false, + }; match valid { true => Ok(()), From adab64349c1f66c580a6d44b99225bcb62f5aff1 Mon Sep 17 00:00:00 2001 From: "James O. D. Hunt" Date: Mon, 22 Nov 2021 14:32:37 +0000 Subject: [PATCH 42/66] agent: Remove some unwrap and expect calls Replace some `unwrap()` and `expect()` calls with code to return the error to the caller. Fixes: #3011. Signed-off-by: James O. D. Hunt --- src/agent/src/device.rs | 86 +++++++++---------- src/agent/src/main.rs | 8 +- src/agent/src/metrics.rs | 32 ++++--- src/agent/src/mount.rs | 21 +++-- src/agent/src/rpc.rs | 179 ++++++++++++++++++++++----------------- src/agent/src/util.rs | 14 ++- 6 files changed, 184 insertions(+), 156 deletions(-) diff --git a/src/agent/src/device.rs b/src/agent/src/device.rs index c22a65795..7c73dd2ca 100644 --- a/src/agent/src/device.rs +++ b/src/agent/src/device.rs @@ -11,7 +11,7 @@ use std::fmt; use std::fs; use std::os::unix::ffi::OsStrExt; use std::os::unix::fs::MetadataExt; -use std::path::Path; +use std::path::{Path, PathBuf}; use std::str::FromStr; use std::sync::Arc; use tokio::sync::Mutex; @@ -157,20 +157,22 @@ pub fn pcipath_to_sysfs(root_bus_sysfs: &str, pcipath: &pci::Path) -> Result = fs::read_dir(&bridgebuspath)?.collect(); - if files.len() != 1 { - return Err(anyhow!( - "Expected exactly one PCI bus in {}, got {} instead", - bridgebuspath, - files.len() - )); - } - - // unwrap is safe, because of the length test above - let busfile = files.pop().unwrap()?; - bus = busfile - .file_name() - .into_string() - .map_err(|e| anyhow!("Bad filename under {}: {:?}", &bridgebuspath, e))?; + match files.pop() { + Some(busfile) if files.is_empty() => { + bus = busfile? + .file_name() + .into_string() + .map_err(|e| anyhow!("Bad filename under {}: {:?}", &bridgebuspath, e))?; + } + _ => { + return Err(anyhow!( + "Expected exactly one PCI bus in {}, got {} instead", + bridgebuspath, + // Adjust to original value as we've already popped + files.len() + 1 + )); + } + }; } Ok(relpath) @@ -218,8 +220,9 @@ impl VirtioBlkPciMatcher { fn new(relpath: &str) -> VirtioBlkPciMatcher { let root_bus = create_pci_root_bus_path(); let re = format!(r"^{}{}/virtio[0-9]+/block/", root_bus, relpath); + VirtioBlkPciMatcher { - rex: Regex::new(&re).unwrap(), + rex: Regex::new(&re).expect("BUG: failed to compile VirtioBlkPciMatcher regex"), } } } @@ -257,7 +260,7 @@ impl VirtioBlkCCWMatcher { root_bus_path, device ); VirtioBlkCCWMatcher { - rex: Regex::new(&re).unwrap(), + rex: Regex::new(&re).expect("BUG: failed to compile VirtioBlkCCWMatcher regex"), } } } @@ -413,12 +416,15 @@ fn scan_scsi_bus(scsi_addr: &str) -> Result<()> { for entry in fs::read_dir(SYSFS_SCSI_HOST_PATH)? { let host = entry?.file_name(); - let scan_path = format!( - "{}/{}/{}", - SYSFS_SCSI_HOST_PATH, - host.to_str().unwrap(), - "scan" - ); + + let host_str = host.to_str().ok_or_else(|| { + anyhow!( + "failed to convert directory entry to unicode for file {:?}", + host + ) + })?; + + let scan_path = PathBuf::from(&format!("{}/{}/{}", SYSFS_SCSI_HOST_PATH, host_str, "scan")); fs::write(scan_path, &scan_data)?; } @@ -722,25 +728,17 @@ async fn vfio_device_handler(device: &Device, sandbox: &Arc>) -> if vfio_in_guest { pci_driver_override(SYSFS_BUS_PCI_PATH, guestdev, "vfio-pci")?; - let devgroup = pci_iommu_group(SYSFS_BUS_PCI_PATH, guestdev)?; - if devgroup.is_none() { - // Devices must have an IOMMU group to be usable via VFIO - return Err(anyhow!("{} has no IOMMU group", guestdev)); + // Devices must have an IOMMU group to be usable via VFIO + let devgroup = pci_iommu_group(SYSFS_BUS_PCI_PATH, guestdev)? + .ok_or_else(|| anyhow!("{} has no IOMMU group", guestdev))?; + + if let Some(g) = group { + if g != devgroup { + return Err(anyhow!("{} is not in guest IOMMU group {}", guestdev, g)); + } } - if group.is_some() && group != devgroup { - // If PCI devices associated with the same VFIO device - // (and therefore group) in the host don't end up in - // the same group in the guest, something has gone - // horribly wrong - return Err(anyhow!( - "{} is not in guest IOMMU group {}", - guestdev, - group.unwrap() - )); - } - - group = devgroup; + group = Some(devgroup); pci_fixups.push((host, guestdev)); } @@ -748,7 +746,8 @@ async fn vfio_device_handler(device: &Device, sandbox: &Arc>) -> let dev_update = if vfio_in_guest { // If there are any devices at all, logic above ensures that group is not None - let group = group.unwrap(); + let group = group.ok_or_else(|| anyhow!("failed to get VFIO group: {:?}"))?; + let vm_path = get_vfio_device_name(sandbox, group).await?; Some(DevUpdate::from_vm_path(&vm_path, vm_path.clone())?) @@ -844,11 +843,8 @@ pub fn update_device_cgroup(spec: &mut Spec) -> Result<()> { .as_mut() .ok_or_else(|| anyhow!("Spec didn't container linux field"))?; - if linux.resources.is_none() { - linux.resources = Some(LinuxResources::default()); - } + let resources = linux.resources.get_or_insert(LinuxResources::default()); - let resources = linux.resources.as_mut().unwrap(); resources.devices.push(LinuxDeviceCgroup { allow: false, major: Some(major), diff --git a/src/agent/src/main.rs b/src/agent/src/main.rs index c2ae8c769..d1745fb01 100644 --- a/src/agent/src/main.rs +++ b/src/agent/src/main.rs @@ -113,10 +113,10 @@ async fn create_logger_task(rfd: RawFd, vsock_port: u32, shutdown: Receiver Result { AGENT_SCRAPE_COUNT.inc(); // update agent process metrics - update_agent_metrics(); + update_agent_metrics()?; // update guest os metrics update_guest_metrics(); @@ -84,23 +85,26 @@ pub fn get_metrics(_: &protocols::agent::GetMetricsRequest) -> Result { let mut buffer = Vec::new(); let encoder = TextEncoder::new(); - encoder.encode(&metric_families, &mut buffer).unwrap(); + encoder.encode(&metric_families, &mut buffer)?; - Ok(String::from_utf8(buffer).unwrap()) + Ok(String::from_utf8(buffer)?) } #[instrument] -fn update_agent_metrics() { +fn update_agent_metrics() -> Result<()> { let me = procfs::process::Process::myself(); - if let Err(err) = me { - error!(sl!(), "failed to create process instance: {:?}", err); - return; - } + let me = match me { + Ok(p) => p, + Err(e) => { + // FIXME: return Ok for all errors? + warn!(sl!(), "failed to create process instance: {:?}", e); - let me = me.unwrap(); + return Ok(()); + } + }; - let tps = procfs::ticks_per_second().unwrap(); + let tps = procfs::ticks_per_second()?; // process total time AGENT_TOTAL_TIME.set((me.stat.utime + me.stat.stime) as f64 / (tps as f64)); @@ -109,7 +113,7 @@ fn update_agent_metrics() { AGENT_TOTAL_VM.set(me.stat.vsize as f64); // Total resident set - let page_size = procfs::page_size().unwrap() as f64; + let page_size = procfs::page_size()? as f64; AGENT_TOTAL_RSS.set(me.stat.rss as f64 * page_size); // io @@ -132,11 +136,11 @@ fn update_agent_metrics() { } match me.status() { - Err(err) => { - info!(sl!(), "failed to get process status: {:?}", err); - } + Err(err) => error!(sl!(), "failed to get process status: {:?}", err), Ok(status) => set_gauge_vec_proc_status(&AGENT_PROC_STATUS, &status), } + + return Ok(()); } #[instrument] diff --git a/src/agent/src/mount.rs b/src/agent/src/mount.rs index fb92311bc..20fd0494c 100644 --- a/src/agent/src/mount.rs +++ b/src/agent/src/mount.rs @@ -628,8 +628,7 @@ pub fn get_mount_fs_type_from_file(mount_file: &str, mount_point: &str) -> Resul let file = File::open(mount_file)?; let reader = BufReader::new(file); - let re = Regex::new(format!("device .+ mounted on {} with fstype (.+)", mount_point).as_str()) - .unwrap(); + let re = Regex::new(format!("device .+ mounted on {} with fstype (.+)", mount_point).as_str())?; // Read the file line by line using the lines() iterator from std::io::BufRead. for (_index, line) in reader.lines().enumerate() { @@ -707,20 +706,21 @@ pub fn get_cgroup_mounts( } } - if fields[0].is_empty() { + let subsystem_name = fields[0]; + + if subsystem_name.is_empty() { continue; } - if fields[0] == "devices" { + if subsystem_name == "devices" { has_device_cgroup = true; } - if let Some(value) = CGROUPS.get(&fields[0]) { - let key = CGROUPS.keys().find(|&&f| f == fields[0]).unwrap(); + if let Some((key, value)) = CGROUPS.get_key_value(subsystem_name) { cg_mounts.push(InitMount { fstype: "cgroup", src: "cgroup", - dest: *value, + dest: value, options: vec!["nosuid", "nodev", "noexec", "relatime", key], }); } @@ -773,10 +773,9 @@ fn ensure_destination_file_exists(path: &Path) -> Result<()> { return Err(anyhow!("{:?} exists but is not a regular file", path)); } - // The only way parent() can return None is if the path is /, - // which always exists, so the test above will already have caught - // it, thus the unwrap() is safe - let dir = path.parent().unwrap(); + let dir = path + .parent() + .ok_or_else(|| anyhow!("failed to find parent path for {:?}", path))?; fs::create_dir_all(dir).context(format!("create_dir_all {:?}", dir))?; diff --git a/src/agent/src/rpc.rs b/src/agent/src/rpc.rs index bcabecddf..87cdc4f84 100644 --- a/src/agent/src/rpc.rs +++ b/src/agent/src/rpc.rs @@ -183,7 +183,7 @@ impl AgentService { update_device_cgroup(&mut oci)?; // Append guest hooks - append_guest_hooks(&s, &mut oci); + append_guest_hooks(&s, &mut oci)?; // write spec to bundle path, hooks might // read ocispec @@ -205,21 +205,14 @@ impl AgentService { LinuxContainer::new(cid.as_str(), CONTAINER_BASE, opts, &sl!())?; let pipe_size = AGENT_CONFIG.read().await.container_pipe_size; - let p = if oci.process.is_some() { - Process::new( - &sl!(), - oci.process.as_ref().unwrap(), - cid.as_str(), - true, - pipe_size, - )? + + let p = if let Some(p) = oci.process { + Process::new(&sl!(), &p, cid.as_str(), true, pipe_size)? } else { info!(sl!(), "no process configurations!"); return Err(anyhow!(nix::Error::from_errno(nix::errno::Errno::EINVAL))); }; - ctr.start(p).await?; - s.update_shared_pidns(&ctr)?; s.add_container(ctr); info!(sl!(), "created container!"); @@ -241,11 +234,17 @@ impl AgentService { ctr.exec()?; + if sid == cid { + return Ok(()); + } + // start oom event loop - if sid != cid && ctr.cgroup_manager.is_some() { - let cg_path = ctr.cgroup_manager.as_ref().unwrap().get_cg_path("memory"); - if cg_path.is_some() { - let rx = notifier::notify_oom(cid.as_str(), cg_path.unwrap()).await?; + if let Some(ref ctr) = ctr.cgroup_manager { + let cg_path = ctr.get_cg_path("memory"); + + if let Some(cg_path) = cg_path { + let rx = notifier::notify_oom(cid.as_str(), cg_path.to_string()).await?; + s.run_oom_event_monitor(rx, cid.clone()).await; } } @@ -345,14 +344,13 @@ impl AgentService { let s = self.sandbox.clone(); let mut sandbox = s.lock().await; - let process = if req.process.is_some() { - req.process.as_ref().unwrap() - } else { - return Err(anyhow!(nix::Error::from_errno(nix::errno::Errno::EINVAL))); - }; + let process = req + .process + .into_option() + .ok_or_else(|| anyhow!(nix::Error::from_errno(nix::errno::Errno::EINVAL)))?; let pipe_size = AGENT_CONFIG.read().await.container_pipe_size; - let ocip = rustjail::process_grpc_to_oci(process); + let ocip = rustjail::process_grpc_to_oci(&process); let p = Process::new(&sl!(), &ocip, exec_id.as_str(), false, pipe_size)?; let ctr = sandbox @@ -385,7 +383,12 @@ impl AgentService { let p = find_process(&mut sandbox, cid.as_str(), eid.as_str(), init)?; - let mut signal = Signal::try_from(req.signal as i32).unwrap(); + let mut signal = Signal::try_from(req.signal as i32).map_err(|e| { + anyhow!(e).context(format!( + "failed to convert {:?} to signal (container-id: {}, exec-id: {})", + req.signal, cid, eid + )) + })?; // For container initProcess, if it hasn't installed handler for "SIGTERM" signal, // it will ignore the "SIGTERM" signal sent to it, thus send it "SIGKILL" signal @@ -444,7 +447,11 @@ impl AgentService { Some(p) => p, None => { // Lost race, pick up exit code from channel - resp.status = exit_recv.recv().await.unwrap(); + resp.status = exit_recv + .recv() + .await + .ok_or_else(|| anyhow!("Failed to receive exit code"))?; + return Ok(resp); } }; @@ -486,7 +493,7 @@ impl AgentService { } }; - let writer = writer.unwrap(); + let writer = writer.ok_or_else(|| anyhow!("cannot get writer"))?; writer.lock().await.write_all(req.data.as_slice()).await?; let mut resp = WriteStreamResponse::new(); @@ -528,7 +535,7 @@ impl AgentService { return Err(anyhow!(nix::Error::from_errno(nix::errno::Errno::EINVAL))); } - let reader = reader.unwrap(); + let reader = reader.ok_or_else(|| anyhow!("cannot get stream reader"))?; tokio::select! { _ = term_exit_notifier.notified() => { @@ -646,8 +653,8 @@ impl protocols::agent_ttrpc::AgentService for AgentService { let resp = Empty::new(); - if res.is_some() { - let oci_res = rustjail::resources_grpc_to_oci(&res.unwrap()); + if let Some(res) = res.as_ref() { + let oci_res = rustjail::resources_grpc_to_oci(res); match ctr.set(oci_res) { Err(e) => { return Err(ttrpc_error(ttrpc::Code::INTERNAL, e.to_string())); @@ -807,25 +814,24 @@ impl protocols::agent_ttrpc::AgentService for AgentService { ) })?; - if p.term_master.is_none() { + if let Some(fd) = p.term_master { + unsafe { + let win = winsize { + ws_row: req.row as c_ushort, + ws_col: req.column as c_ushort, + ws_xpixel: 0, + ws_ypixel: 0, + }; + + let err = libc::ioctl(fd, TIOCSWINSZ, &win); + Errno::result(err).map(drop).map_err(|e| { + ttrpc_error(ttrpc::Code::INTERNAL, format!("ioctl error: {:?}", e)) + })?; + } + } else { return Err(ttrpc_error(ttrpc::Code::UNAVAILABLE, "no tty".to_string())); } - let fd = p.term_master.unwrap(); - unsafe { - let win = winsize { - ws_row: req.row as c_ushort, - ws_col: req.column as c_ushort, - ws_xpixel: 0, - ws_ypixel: 0, - }; - - let err = libc::ioctl(fd, TIOCSWINSZ, &win); - Errno::result(err) - .map(drop) - .map_err(|e| ttrpc_error(ttrpc::Code::INTERNAL, format!("ioctl error: {:?}", e)))?; - } - Ok(Empty::new()) } @@ -1027,12 +1033,25 @@ impl protocols::agent_ttrpc::AgentService for AgentService { let mut sandbox = s.lock().await; // destroy all containers, clean up, notify agent to exit // etc. - sandbox.destroy().await.unwrap(); + sandbox + .destroy() + .await + .map_err(|e| ttrpc_error(ttrpc::Code::INTERNAL, e.to_string()))?; // Close get_oom_event connection, // otherwise it will block the shutdown of ttrpc. sandbox.event_tx.take(); - sandbox.sender.take().unwrap().send(1).unwrap(); + sandbox + .sender + .take() + .ok_or_else(|| { + ttrpc_error( + ttrpc::Code::INTERNAL, + "failed to get sandbox sender channel".to_string(), + ) + })? + .send(1) + .map_err(|e| ttrpc_error(ttrpc::Code::INTERNAL, e.to_string()))?; Ok(Empty::new()) } @@ -1291,11 +1310,7 @@ fn get_memory_info(block_size: bool, hotplug: bool) -> Result<(u64, bool)> { match stat::stat(SYSFS_MEMORY_HOTPLUG_PROBE_PATH) { Ok(_) => plug = true, Err(e) => { - info!( - sl!(), - "hotplug memory error: {}", - e.as_errno().unwrap().desc() - ); + info!(sl!(), "hotplug memory error: {:?}", e); match e { nix::Error::Sys(errno) => match errno { Errno::ENOENT => plug = false, @@ -1371,7 +1386,7 @@ fn find_process<'a>( ctr.get_process(eid).map_err(|_| anyhow!("Invalid exec id")) } -pub fn start(s: Arc>, server_address: &str) -> TtrpcServer { +pub fn start(s: Arc>, server_address: &str) -> Result { let agent_service = Box::new(AgentService { sandbox: s }) as Box; @@ -1386,14 +1401,13 @@ pub fn start(s: Arc>, server_address: &str) -> TtrpcServer { let hservice = protocols::health_ttrpc::create_health(health_worker); let server = TtrpcServer::new() - .bind(server_address) - .unwrap() + .bind(server_address)? .register_service(aservice) .register_service(hservice); info!(sl!(), "ttRPC server started"; "address" => server_address); - server + Ok(server) } // This function updates the container namespaces configuration based on the @@ -1438,24 +1452,28 @@ fn update_container_namespaces( // the create_sandbox request or create_container request. // Else set this to empty string so that a new pid namespace is // created for the container. - if sandbox_pidns && sandbox.sandbox_pidns.is_some() { - pid_ns.path = String::from(sandbox.sandbox_pidns.as_ref().unwrap().path.as_str()); + if sandbox_pidns { + if let Some(ref pidns) = &sandbox.sandbox_pidns { + pid_ns.path = String::from(pidns.path.as_str()); + } else { + return Err(anyhow!("failed to get sandbox pidns")); + } } linux.namespaces.push(pid_ns); Ok(()) } -fn append_guest_hooks(s: &Sandbox, oci: &mut Spec) { - if s.hooks.is_none() { - return; +fn append_guest_hooks(s: &Sandbox, oci: &mut Spec) -> Result<()> { + if let Some(ref guest_hooks) = s.hooks { + let mut hooks = oci.hooks.take().unwrap_or_default(); + hooks.prestart.append(&mut guest_hooks.prestart.clone()); + hooks.poststart.append(&mut guest_hooks.poststart.clone()); + hooks.poststop.append(&mut guest_hooks.poststop.clone()); + oci.hooks = Some(hooks); } - let guest_hooks = s.hooks.as_ref().unwrap(); - let mut hooks = oci.hooks.take().unwrap_or_default(); - hooks.prestart.append(&mut guest_hooks.prestart.clone()); - hooks.poststart.append(&mut guest_hooks.poststart.clone()); - hooks.poststop.append(&mut guest_hooks.poststop.clone()); - oci.hooks = Some(hooks); + + Ok(()) } // Check is the container process installed the @@ -1545,7 +1563,7 @@ fn do_copy_file(req: &CopyFileRequest) -> Result<()> { PathBuf::from("/") }; - fs::create_dir_all(dir.to_str().unwrap()).or_else(|e| { + fs::create_dir_all(&dir).or_else(|e| { if e.kind() != std::io::ErrorKind::AlreadyExists { return Err(e); } @@ -1553,10 +1571,7 @@ fn do_copy_file(req: &CopyFileRequest) -> Result<()> { Ok(()) })?; - std::fs::set_permissions( - dir.to_str().unwrap(), - std::fs::Permissions::from_mode(req.dir_mode), - )?; + std::fs::set_permissions(&dir, std::fs::Permissions::from_mode(req.dir_mode))?; let mut tmpfile = path.clone(); tmpfile.set_extension("tmp"); @@ -1565,10 +1580,10 @@ fn do_copy_file(req: &CopyFileRequest) -> Result<()> { .write(true) .create(true) .truncate(false) - .open(tmpfile.to_str().unwrap())?; + .open(&tmpfile)?; file.write_all_at(req.data.as_slice(), req.offset as u64)?; - let st = stat::stat(tmpfile.to_str().unwrap())?; + let st = stat::stat(&tmpfile)?; if st.st_size != req.file_size { return Ok(()); @@ -1577,7 +1592,7 @@ fn do_copy_file(req: &CopyFileRequest) -> Result<()> { file.set_permissions(std::fs::Permissions::from_mode(req.file_mode))?; unistd::chown( - tmpfile.to_str().unwrap(), + &tmpfile, Some(Uid::from_raw(req.uid as u32)), Some(Gid::from_raw(req.gid as u32)), )?; @@ -1646,10 +1661,18 @@ fn setup_bundle(cid: &str, spec: &mut Spec) -> Result { readonly: spec_root.readonly, }); - let _ = spec.save(config_path.to_str().unwrap()); + let _ = spec.save( + config_path + .to_str() + .ok_or_else(|| anyhow!("cannot convert path to unicode"))?, + ); let olddir = unistd::getcwd().context("cannot getcwd")?; - unistd::chdir(bundle_path.to_str().unwrap())?; + unistd::chdir( + bundle_path + .to_str() + .ok_or_else(|| anyhow!("cannot convert bundle path to unicode"))?, + )?; Ok(olddir) } @@ -1682,8 +1705,8 @@ fn load_kernel_module(module: &protocols::agent::KernelModule) -> Result<()> { match status.code() { Some(code) => { - let std_out: String = String::from_utf8(output.stdout).unwrap(); - let std_err: String = String::from_utf8(output.stderr).unwrap(); + let std_out = String::from_utf8_lossy(&output.stdout); + let std_err = String::from_utf8_lossy(&output.stderr); let msg = format!( "load_kernel_module return code: {} stdout:{} stderr:{}", code, std_out, std_err @@ -1746,7 +1769,7 @@ mod tests { let mut oci = Spec { ..Default::default() }; - append_guest_hooks(&s, &mut oci); + append_guest_hooks(&s, &mut oci).unwrap(); assert_eq!(s.hooks, oci.hooks); } diff --git a/src/agent/src/util.rs b/src/agent/src/util.rs index 0e262e7ee..1be52f730 100644 --- a/src/agent/src/util.rs +++ b/src/agent/src/util.rs @@ -3,7 +3,7 @@ // SPDX-License-Identifier: Apache-2.0 // -use anyhow::Result; +use anyhow::{anyhow, Result}; use futures::StreamExt; use std::io; use std::io::ErrorKind; @@ -64,8 +64,12 @@ pub fn get_vsock_incoming(fd: RawFd) -> Incoming { #[instrument] pub async fn get_vsock_stream(fd: RawFd) -> Result { - let stream = get_vsock_incoming(fd).next().await.unwrap()?; - Ok(stream) + let stream = get_vsock_incoming(fd) + .next() + .await + .ok_or_else(|| anyhow!("cannot handle incoming vsock connection"))?; + + Ok(stream?) } #[cfg(test)] @@ -124,7 +128,9 @@ mod tests { let mut vec_locked = vec_ref.lock(); - let v = vec_locked.as_deref_mut().unwrap(); + let v = vec_locked + .as_deref_mut() + .map_err(|e| std::io::Error::new(std::io::ErrorKind::Other, e.to_string()))?; std::io::Write::flush(v) } From bd3217daeb6d07120096a96be21fb2c05724008b Mon Sep 17 00:00:00 2001 From: "James O. D. Hunt" Date: Wed, 24 Nov 2021 11:42:04 +0000 Subject: [PATCH 43/66] agent: Remove redundant returns Remove an unnecessary `return` statement identified by clippy. Signed-off-by: James O. D. Hunt --- src/agent/src/metrics.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/agent/src/metrics.rs b/src/agent/src/metrics.rs index a125531ea..b32dd4487 100644 --- a/src/agent/src/metrics.rs +++ b/src/agent/src/metrics.rs @@ -140,7 +140,7 @@ fn update_agent_metrics() -> Result<()> { Ok(status) => set_gauge_vec_proc_status(&AGENT_PROC_STATUS, &status), } - return Ok(()); + Ok(()) } #[instrument] From 75bb340137e3938a3e44eef7586213be4d345c3f Mon Sep 17 00:00:00 2001 From: Binbin Zhang Date: Mon, 22 Nov 2021 20:23:31 +0800 Subject: [PATCH 44/66] shimv2/service: fix defer funtions never run with os.Exit() os.Exit() will terminate program immediately, the defer functions won't be executed, so we add defer functions again before os.Exit(). Refer to https://pkg.go.dev/os#Exit Fixes: #3059 Signed-off-by: Binbin Zhang --- src/runtime/pkg/containerd-shim-v2/service.go | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/runtime/pkg/containerd-shim-v2/service.go b/src/runtime/pkg/containerd-shim-v2/service.go index 29cd259f9..cafbb1907 100644 --- a/src/runtime/pkg/containerd-shim-v2/service.go +++ b/src/runtime/pkg/containerd-shim-v2/service.go @@ -953,6 +953,12 @@ func (s *service) Shutdown(ctx context.Context, r *taskAPI.ShutdownRequest) (_ * // exited when shimv2 terminated. Thus here to do the last cleanup of the hypervisor. syscall.Kill(int(s.hpid), syscall.SIGKILL) + // os.Exit() will terminate program immediately, the defer functions won't be executed, + // so we add defer functions again before os.Exit(). + // Refer to https://pkg.go.dev/os#Exit + shimLog.WithField("container", r.ID).Debug("Shutdown() end") + rpcDurationsHistogram.WithLabelValues("shutdown").Observe(float64(time.Since(start).Nanoseconds() / int64(time.Millisecond))) + os.Exit(0) // This will never be called, but this is only there to make sure the From 31f6c2c2eacfbaa311b7f37c6c2744b982e57c32 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fabiano=20Fid=C3=AAncio?= Date: Sun, 21 Nov 2021 11:33:30 +0100 Subject: [PATCH 45/66] tools: Update comments about the kata-deploy yaml changes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The comments were mentioning kata-deploy-base files while it really should mention kata-deploy-stable files. While here, I've also added a missing '"' to one of the tags. Signed-off-by: Fabiano Fidêncio --- .../release/update-repository-version.sh | 44 +++++++++---------- 1 file changed, 22 insertions(+), 22 deletions(-) diff --git a/tools/packaging/release/update-repository-version.sh b/tools/packaging/release/update-repository-version.sh index cded448cc..6a5a3e7ff 100755 --- a/tools/packaging/release/update-repository-version.sh +++ b/tools/packaging/release/update-repository-version.sh @@ -117,37 +117,37 @@ bump_repo() { # 1) [main] ------> [main] NO-OP # "alpha0" "alpha1" # - # +----------------+----------------+ - # | from | to | - # -----------------+----------------+----------------+ - # kata-deploy | "latest" | "latest" | - # -----------------+----------------+----------------+ - # kata-deploy-base | "stable | "stable" | - # -----------------+----------------+----------------+ + # +----------------+----------------+ + # | from | to | + # -------------------+----------------+----------------+ + # kata-deploy | "latest" | "latest" | + # -------------------+----------------+----------------+ + # kata-deploy-stable | "stable | "stable" | + # -------------------+----------------+----------------+ # # # 2) [main] ------> [stable] Update kata-deploy and - # "alpha2" "rc0" get rid of kata-deploy-base + # "alpha2" "rc0" get rid of kata-deploy-stable # - # +----------------+----------------+ - # | from | to | - # -----------------+----------------+----------------+ - # kata-deploy | "latest" | "rc0" | - # -----------------+----------------+----------------+ - # kata-deploy-base | "stable" | REMOVED | - # -----------------+----------------+----------------+ + # +----------------+----------------+ + # | from | to | + # -------------------+----------------+----------------+ + # kata-deploy | "latest" | "rc0" | + # -------------------+----------------+----------------+ + # kata-deploy-stable | "stable" | REMOVED | + # -------------------+----------------+----------------+ # # # 3) [stable] ------> [stable] Update kata-deploy # "x.y.z" "x.y.(z+1)" # - # +----------------+----------------+ - # | from | to | - # -----------------+----------------+----------------+ - # kata-deploy | "x.y.z" | "x.y.(z+1)" | - # -----------------+----------------+----------------+ - # kata-deploy-base | NON-EXISTENT | NON-EXISTENT | - # -----------------+----------------+----------------+ + # +----------------+----------------+ + # | from | to | + # -------------------+----------------+----------------+ + # kata-deploy | "x.y.z" | "x.y.(z+1)" | + # -------------------+----------------+----------------+ + # kata-deploy-stable | NON-EXISTENT | NON-EXISTENT | + # -------------------+----------------+----------------+ info "Updating kata-deploy / kata-cleanup image tags" if [ "${target_branch}" == "main" ] && [[ "${new_version}" =~ "rc" ]]; then From edca82924268887fb94bd0ebed1392f90636d7d8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fabiano=20Fid=C3=AAncio?= Date: Sun, 21 Nov 2021 11:07:44 +0100 Subject: [PATCH 46/66] tools: Rewrite the logic around kata-deploy changes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit We can simplify the code a little bit, as at least now we group common operationr together. Hopefully this will improve the maintainability and the readability of the code. Signed-off-by: Fabiano Fidêncio --- .../release/update-repository-version.sh | 32 ++++++++++--------- 1 file changed, 17 insertions(+), 15 deletions(-) diff --git a/tools/packaging/release/update-repository-version.sh b/tools/packaging/release/update-repository-version.sh index 6a5a3e7ff..2fec951d2 100755 --- a/tools/packaging/release/update-repository-version.sh +++ b/tools/packaging/release/update-repository-version.sh @@ -150,24 +150,26 @@ bump_repo() { # -------------------+----------------+----------------+ info "Updating kata-deploy / kata-cleanup image tags" - if [ "${target_branch}" == "main" ] && [[ "${new_version}" =~ "rc" ]]; then - # case 2) - ## change the "latest" tag to the "#{new_version}" one - sed -i "s#quay.io/kata-containers/kata-deploy:latest#quay.io/kata-containers/kata-deploy:${new_version}#g" tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml - sed -i "s#quay.io/kata-containers/kata-deploy:latest#quay.io/kata-containers/kata-deploy:${new_version}#g" tools/packaging/kata-deploy/kata-cleanup/base/kata-cleanup.yaml + local version_to_replace="${current_version}" + local replacement="${new_version}" + if [ "${target_branch}" == "main" ]; then + if [[ "${new_version}" =~ "rc" ]]; then + ## this is the case 2) where we remove te kata-deploy / kata-cleanup stable files + git rm tools/packaging/kata-deploy/kata-deploy/base/kata-deploy-stable.yaml + git rm tools/packaging/kata-deploy/kata-cleanup/base/kata-cleanup-stable.yaml + else + ## this is the case 1) where we just do nothing + replacement="latest" + fi - git diff + version_to_replace="latest" + fi - git add tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml - git add tools/packaging/kata-deploy/kata-cleanup/base/kata-cleanup.yaml + if [ "${version_to_replace}" != "${replacement}" ]; then + ## this covers case 2) and 3), as on both of them we have changes on kata-deploy / kata-cleanup files + sed -i "s#quay.io/kata-containers/kata-deploy:${version_to_replace}#quay.io/kata-containers/kata-deploy:${replacement}#g" tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml + sed -i "s#quay.io/kata-containers/kata-deploy:${version_to_replace}#quay.io/kata-containers/kata-deploy:${replacement}#g" tools/packaging/kata-deploy/kata-cleanup/base/kata-cleanup.yaml - ## and remove the kata-deploy & kata-cleanup stable yaml files - git rm tools/packaging/kata-deploy/kata-deploy/base/kata-deploy-stable.yaml - git rm tools/packaging/kata-deploy/kata-cleanup/base/kata-cleanup-stable.yaml - elif [[ "${target_branch}" =~ "stable" ]]; then - # case 3) - sed -i "s#quay.io/kata-containers/kata-deploy:${current_version}#quay.io/kata-containers/kata-deploy:${new_version}#g" tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml - sed -i "s#quay.io/kata-containers/kata-deploy:${current_version}#quay.io/kata-containers/kata-deploy:${new_version}#g" tools/packaging/kata-deploy/kata-cleanup/base/kata-cleanup.yaml git diff git add tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml From ac958a3073b382a4fdfd8ddb8f6d2569303feb2f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fabiano=20Fid=C3=AAncio?= Date: Tue, 23 Nov 2021 10:36:20 +0100 Subject: [PATCH 47/66] tools: Use vars for the yaml files used in the update repo script MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Instead of always writing the full path of some files, let's just create some vars and avoid both repetition (which is quite error prone) and too long lines (which makes the file not so easy to read). Signed-off-by: Fabiano Fidêncio --- .../release/update-repository-version.sh | 22 +++++++++++++------ 1 file changed, 15 insertions(+), 7 deletions(-) diff --git a/tools/packaging/release/update-repository-version.sh b/tools/packaging/release/update-repository-version.sh index 2fec951d2..eddee5834 100755 --- a/tools/packaging/release/update-repository-version.sh +++ b/tools/packaging/release/update-repository-version.sh @@ -90,6 +90,14 @@ bump_repo() { pushd "${repo}" >>/dev/null + local kata_deploy_dir="tools/packaging/kata-deploy" + local kata_deploy_base="${kata_deploy_dir}/kata-deploy/base" + local kata_cleanup_base="${kata_deploy_dir}/kata-cleanup/base" + local kata_deploy_yaml="${kata_deploy_base}/kata-deploy.yaml" + local kata_cleanup_yaml="${kata_cleanup_base}/kata-cleanup.yaml" + local kata_deploy_stable_yaml="${kata_deploy_base}/kata-deploy-stable.yaml" + local kata_cleanup_stable_yaml="${kata_cleanup_base}/kata-cleanup-stable.yaml" + branch="${new_version}-branch-bump" git fetch origin "${target_branch}" git checkout "origin/${target_branch}" -b "${branch}" @@ -155,25 +163,25 @@ bump_repo() { if [ "${target_branch}" == "main" ]; then if [[ "${new_version}" =~ "rc" ]]; then ## this is the case 2) where we remove te kata-deploy / kata-cleanup stable files - git rm tools/packaging/kata-deploy/kata-deploy/base/kata-deploy-stable.yaml - git rm tools/packaging/kata-deploy/kata-cleanup/base/kata-cleanup-stable.yaml + git rm "${kata_deploy_stable_yaml}" + git rm "${kata_cleanup_stable_yaml}" + else ## this is the case 1) where we just do nothing replacement="latest" fi - version_to_replace="latest" fi if [ "${version_to_replace}" != "${replacement}" ]; then ## this covers case 2) and 3), as on both of them we have changes on kata-deploy / kata-cleanup files - sed -i "s#quay.io/kata-containers/kata-deploy:${version_to_replace}#quay.io/kata-containers/kata-deploy:${replacement}#g" tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml - sed -i "s#quay.io/kata-containers/kata-deploy:${version_to_replace}#quay.io/kata-containers/kata-deploy:${replacement}#g" tools/packaging/kata-deploy/kata-cleanup/base/kata-cleanup.yaml + sed -i "s#quay.io/kata-containers/kata-deploy:${version_to_replace}#quay.io/kata-containers/kata-deploy:${new_version}#g" "${kata_deploy_yaml}" + sed -i "s#quay.io/kata-containers/kata-deploy:${version_to_replace}#quay.io/kata-containers/kata-deploy:${new_version}#g" "${kata_cleanup_yaml}" git diff - git add tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml - git add tools/packaging/kata-deploy/kata-cleanup/base/kata-cleanup.yaml + git add "${kata_deploy_yaml}" + git add "${kata_cleanup_yaml}" fi fi From c8e22daf6773828de4bfef965ffa9bcdbc85dcdf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fabiano=20Fid=C3=AAncio?= Date: Tue, 23 Nov 2021 10:41:06 +0100 Subject: [PATCH 48/66] tools: Use vars for the registry in the update repo script MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Similarly to what was done for the yaml files, let's use a var for representing the registry where our images will be pushed to and avoid repetition and too long lines. Signed-off-by: Fabiano Fidêncio --- tools/packaging/release/update-repository-version.sh | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/tools/packaging/release/update-repository-version.sh b/tools/packaging/release/update-repository-version.sh index eddee5834..856a22bff 100755 --- a/tools/packaging/release/update-repository-version.sh +++ b/tools/packaging/release/update-repository-version.sh @@ -157,6 +157,8 @@ bump_repo() { # kata-deploy-stable | NON-EXISTENT | NON-EXISTENT | # -------------------+----------------+----------------+ + local registry="quay.io/kata-containers/kata-deploy" + info "Updating kata-deploy / kata-cleanup image tags" local version_to_replace="${current_version}" local replacement="${new_version}" @@ -175,8 +177,8 @@ bump_repo() { if [ "${version_to_replace}" != "${replacement}" ]; then ## this covers case 2) and 3), as on both of them we have changes on kata-deploy / kata-cleanup files - sed -i "s#quay.io/kata-containers/kata-deploy:${version_to_replace}#quay.io/kata-containers/kata-deploy:${new_version}#g" "${kata_deploy_yaml}" - sed -i "s#quay.io/kata-containers/kata-deploy:${version_to_replace}#quay.io/kata-containers/kata-deploy:${new_version}#g" "${kata_cleanup_yaml}" + sed -i "s#${registry}:${version_to_replace}#${registry}:${new_version}#g" "${kata_deploy_yaml}" + sed -i "s#${registry}:${version_to_replace}#${registry}:${new_version}#g" "${kata_cleanup_yaml}" git diff From 36d73c96c836a1797f86910b5e4e063c99183700 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fabiano=20Fid=C3=AAncio?= Date: Sun, 21 Nov 2021 11:27:42 +0100 Subject: [PATCH 49/66] tools: Do the kata-deploy changes on its own commit MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Rather than doing the kata-deploy changes as part of the release bump commit, let's split those on its own changes, as it will both make the life of the reviewer less confusing and also allows us to start preparing the field for a possible automated revert of these changes, whenever it becomes needed. Signed-off-by: Fabiano Fidêncio --- .../release/update-repository-version.sh | 24 +++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/tools/packaging/release/update-repository-version.sh b/tools/packaging/release/update-repository-version.sh index 856a22bff..1cb0c02fa 100755 --- a/tools/packaging/release/update-repository-version.sh +++ b/tools/packaging/release/update-repository-version.sh @@ -62,6 +62,26 @@ get_changes() { git log --oneline "${current_version}..HEAD" --no-merges } +generate_kata_deploy_commit() { + local new_version=$1 + [ -n "$new_version" ] || die "no new version" + + printf "release: Adapt kata-deploy for %s" "${new_version}" + + printf "\n +kata-deploy files must be adapted to a new release. The cases where it +happens are when the release goes from -> to: +* main -> stable: + * kata-deploy / kata-cleanup: change from \"latest\" to \"rc0\" + * kata-deploy-stable / kata-cleanup-stable: are removed + +* stable -> stable: + * kata-deploy / kata-cleanup: bump the release to the new one. + +There are no changes when doing an alpha release, as the files on the +\"main\" branch always point to the \"latest\" and \"stable\" tags." +} + generate_commit() { local new_version=$1 local current_version=$2 @@ -184,6 +204,10 @@ bump_repo() { git add "${kata_deploy_yaml}" git add "${kata_cleanup_yaml}" + + info "Creating the commit with the kata-deploy changes" + local commit_msg="$(generate_kata_deploy_commit $new_version)" + git commit -s -m "${commit_msg}" fi fi From 76540dbdd14d72abc7cf8e26b0e617789a4cb9b8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fabiano=20Fid=C3=AAncio?= Date: Sun, 21 Nov 2021 12:11:16 +0100 Subject: [PATCH 50/66] tools: Automatically revert kata-deploy changes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit When branching the "stable-x.y" branch, we need to do some quite specific changes to kata-deploy / kata-cleanup files, such as: * changing the tags from "latest" to "stable-x.y". * removing the kata-deploy / kata-cleanup stable files. However, after the branching is done, we need to get the `main` repo to its original state, with the kata-deploy / kata-cleanup using the "latest" tag, and with the stable files present there, and this commit ensures that, during the release process, a new PR is automatically created with these changes. Fixes: #3069 Signed-off-by: Fabiano Fidêncio --- .../release/update-repository-version.sh | 36 +++++++++++++++++++ 1 file changed, 36 insertions(+) diff --git a/tools/packaging/release/update-repository-version.sh b/tools/packaging/release/update-repository-version.sh index 1cb0c02fa..624cac8d3 100755 --- a/tools/packaging/release/update-repository-version.sh +++ b/tools/packaging/release/update-repository-version.sh @@ -82,6 +82,18 @@ There are no changes when doing an alpha release, as the files on the \"main\" branch always point to the \"latest\" and \"stable\" tags." } +generate_revert_kata_deploy_commit() { + local new_version=$1 + [ -n "$new_version" ] || die "no new version" + + printf "release: Revert kata-deploy changes after %s release" "${new_version}" + + printf "\n +As %s has been released, let's switch the kata-deploy / kata-cleanup +tags back to \"latest\", and re-add the kata-deploy-stable and the +kata-cleanup-stable files." "${new_version}" +} + generate_commit() { local new_version=$1 local current_version=$2 @@ -208,6 +220,7 @@ bump_repo() { info "Creating the commit with the kata-deploy changes" local commit_msg="$(generate_kata_deploy_commit $new_version)" git commit -s -m "${commit_msg}" + local kata_deploy_commit="$(git rev-parse HEAD)" fi fi @@ -244,6 +257,29 @@ EOT out="" out=$("${hub_bin}" pull-request -b "${target_branch}" -F "${notes_file}" 2>&1) || echo "$out" | grep "A pull request already exists" fi + + if [ "${repo}" == "kata-containers" ] && [ "${target_branch}" == "main" ] && [[ "${new_version}" =~ "rc" ]]; then + reverting_kata_deploy_changes_branch="revert-kata-deploy-changes-after-${new_version}-release" + git checkout -b "${reverting_kata_deploy_changes_branch}" + + git revert --no-edit ${kata_deploy_commit} >>/dev/null + commit_msg="$(generate_revert_kata_deploy_commit $new_version)" + info "Creating the commit message reverting the kata-deploy changes" + git commit --amend -s -m "${commit_msg}" + + echo "${commit_msg}" >"${notes_file}" + echo "" >>"${notes_file}" + echo "Only merge this commit after ${new_version} release is successfully tagged!" >>"${notes_file}" + + if [[ ${PUSH} == "true" ]]; then + info "Push \"${reverting_kata_deploy_changes_branch}\" to fork" + ${hub_bin} push fork -f "${reverting_kata_deploy_changes_branch}" + info "Create \"${reverting_kata_deploy_changes_branch}\" PR" + out="" + out=$("${hub_bin}" pull-request -b "${target_branch}" -F "${notes_file}" 2>&1) || echo "$out" | grep "A pull request already exists" + fi + fi + popd >>/dev/null } From 85eb743f466651910abeee8db28b2945cd962998 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fabiano=20Fid=C3=AAncio?= Date: Tue, 23 Nov 2021 11:39:57 +0100 Subject: [PATCH 51/66] tools: Make hub usage slightly less fragile MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `grep`ing by a specific output, in a specific language, is quite fragile and could easily break `hub`. For now, let's work this around following James' suggestion of setting `LC_ALL=C LANG=C` when calling `hub`. > **Note**: I don't think we should invest much time on fixing `hub` > usage, as it'll be soon replaced by `gh`, see: > https://github.com/kata-containers/kata-containers/issues/3083 Signed-off-by: Fabiano Fidêncio --- tools/packaging/release/update-repository-version.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tools/packaging/release/update-repository-version.sh b/tools/packaging/release/update-repository-version.sh index 624cac8d3..5a01b1840 100755 --- a/tools/packaging/release/update-repository-version.sh +++ b/tools/packaging/release/update-repository-version.sh @@ -255,7 +255,7 @@ EOT ${hub_bin} push fork -f "${branch}" info "Create PR" out="" - out=$("${hub_bin}" pull-request -b "${target_branch}" -F "${notes_file}" 2>&1) || echo "$out" | grep "A pull request already exists" + out=$(LC_ALL=C LANG=C "${hub_bin}" pull-request -b "${target_branch}" -F "${notes_file}" 2>&1) || echo "$out" | grep "A pull request already exists" fi if [ "${repo}" == "kata-containers" ] && [ "${target_branch}" == "main" ] && [[ "${new_version}" =~ "rc" ]]; then @@ -276,7 +276,7 @@ EOT ${hub_bin} push fork -f "${reverting_kata_deploy_changes_branch}" info "Create \"${reverting_kata_deploy_changes_branch}\" PR" out="" - out=$("${hub_bin}" pull-request -b "${target_branch}" -F "${notes_file}" 2>&1) || echo "$out" | grep "A pull request already exists" + out=$(LC_ALL=C LANG=C "${hub_bin}" pull-request -b "${target_branch}" -F "${notes_file}" 2>&1) || echo "$out" | grep "A pull request already exists" fi fi From 5dbd752f8ff7bb271d8463a558df2102be5f5691 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fabiano=20Fid=C3=AAncio?= Date: Tue, 23 Nov 2021 11:45:16 +0100 Subject: [PATCH 52/66] tools: Remove the check for the VERSION file MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit All repos we release (https://github.com/kata-containers/kata-containers and https://github.com/kata-containers/tests) have a VERSION file. Keeping a check for it, although useful for a new repo, just complicates the use-case we currently deal with. While here, let's also anchor the '#' and potentially exclude blank lines, following James' suggestion. Signed-off-by: Fabiano Fidêncio --- .../release/update-repository-version.sh | 23 +++++-------------- 1 file changed, 6 insertions(+), 17 deletions(-) diff --git a/tools/packaging/release/update-repository-version.sh b/tools/packaging/release/update-repository-version.sh index 5a01b1840..c80277369 100755 --- a/tools/packaging/release/update-repository-version.sh +++ b/tools/packaging/release/update-repository-version.sh @@ -36,10 +36,6 @@ trap 'handle_error $LINENO' ERR get_changes() { local current_version=$1 [ -n "${current_version}" ] || die "current version not provided" - if [ "${current_version}" == "new" ];then - echo "Starting to version this repository" - return - fi # If for some reason there is not a tag this could fail # better fail and write the error in the PR @@ -134,20 +130,13 @@ bump_repo() { git fetch origin "${target_branch}" git checkout "origin/${target_branch}" -b "${branch}" - # All repos we build should have a VERSION file - if [ ! -f "VERSION" ]; then - current_version="new" - echo "${new_version}" >VERSION - else - current_version="$(grep -v '#' ./VERSION)" + local current_version="$(egrep -v '^(#|$)' ./VERSION)" - info "Updating VERSION file" - echo "${new_version}" >VERSION - if git diff --exit-code; then - info "${repo} already in version ${new_version}" - cat VERSION - return 0 - fi + info "Updating VERSION file" + echo "${new_version}" >VERSION + if git diff --exit-code; then + info "${repo} already in version ${new_version}" + return 0 fi if [ "${repo}" == "kata-containers" ]; then From 5ba2f52c7316db60b9d85911b2cb626720f99b2c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fabiano=20Fid=C3=AAncio?= Date: Tue, 23 Nov 2021 13:00:48 +0100 Subject: [PATCH 53/66] tools: Quote functions arguments in the update repos script MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Although this is not strictly needed, better be safe than sorry on those cases. Signed-off-by: Fabiano Fidêncio --- .../packaging/release/update-repository-version.sh | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/tools/packaging/release/update-repository-version.sh b/tools/packaging/release/update-repository-version.sh index c80277369..4703032cb 100755 --- a/tools/packaging/release/update-repository-version.sh +++ b/tools/packaging/release/update-repository-version.sh @@ -34,7 +34,7 @@ handle_error() { trap 'handle_error $LINENO' ERR get_changes() { - local current_version=$1 + local current_version="$1" [ -n "${current_version}" ] || die "current version not provided" # If for some reason there is not a tag this could fail @@ -59,7 +59,7 @@ get_changes() { } generate_kata_deploy_commit() { - local new_version=$1 + local new_version="$1" [ -n "$new_version" ] || die "no new version" printf "release: Adapt kata-deploy for %s" "${new_version}" @@ -79,7 +79,7 @@ There are no changes when doing an alpha release, as the files on the } generate_revert_kata_deploy_commit() { - local new_version=$1 + local new_version="$1" [ -n "$new_version" ] || die "no new version" printf "release: Revert kata-deploy changes after %s release" "${new_version}" @@ -91,8 +91,8 @@ kata-cleanup-stable files." "${new_version}" } generate_commit() { - local new_version=$1 - local current_version=$2 + local new_version="$1" + local current_version="$2" [ -n "$new_version" ] || die "no new version" [ -n "$current_version" ] || die "no current version" @@ -305,8 +305,8 @@ main(){ shift $((OPTIND - 1)) - new_version=${1:-} - target_branch=${2:-} + new_version="${1:-}" + target_branch="${2:-}" [ -n "${new_version}" ] || { echo "ERROR: no new version" && usage 1; } [ -n "${target_branch}" ] || die "no target branch" for repo in "${repos[@]}" From 6a0b7165ba013cf4413d8047fe84b8e1278053aa Mon Sep 17 00:00:00 2001 From: bin Date: Wed, 24 Nov 2021 21:45:07 +0800 Subject: [PATCH 54/66] agent: refactor find_process function and add test cases Delete redundant parameter init in find_process function and add test case for it. Fixes: #3117 Signed-off-by: bin --- src/agent/src/rpc.rs | 61 +++++++++++++-------------------------- src/agent/src/sandbox.rs | 62 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 82 insertions(+), 41 deletions(-) diff --git a/src/agent/src/rpc.rs b/src/agent/src/rpc.rs index 87cdc4f84..c4d164cd3 100644 --- a/src/agent/src/rpc.rs +++ b/src/agent/src/rpc.rs @@ -368,7 +368,6 @@ impl AgentService { let eid = req.exec_id.clone(); let s = self.sandbox.clone(); let mut sandbox = s.lock().await; - let mut init = false; info!( sl!(), @@ -377,11 +376,7 @@ impl AgentService { "exec-id" => eid.clone(), ); - if eid.is_empty() { - init = true; - } - - let p = find_process(&mut sandbox, cid.as_str(), eid.as_str(), init)?; + let p = sandbox.find_container_process(cid.as_str(), eid.as_str())?; let mut signal = Signal::try_from(req.signal as i32).map_err(|e| { anyhow!(e).context(format!( @@ -424,7 +419,7 @@ impl AgentService { let exit_rx = { let mut sandbox = s.lock().await; - let p = find_process(&mut sandbox, cid.as_str(), eid.as_str(), false)?; + let p = sandbox.find_container_process(cid.as_str(), eid.as_str())?; p.exit_watchers.push(exit_send); pid = p.pid; @@ -482,7 +477,7 @@ impl AgentService { let writer = { let s = self.sandbox.clone(); let mut sandbox = s.lock().await; - let p = find_process(&mut sandbox, cid.as_str(), eid.as_str(), false)?; + let p = sandbox.find_container_process(cid.as_str(), eid.as_str())?; // use ptmx io if p.term_master.is_some() { @@ -515,7 +510,7 @@ impl AgentService { let s = self.sandbox.clone(); let mut sandbox = s.lock().await; - let p = find_process(&mut sandbox, cid.as_str(), eid.as_str(), false)?; + let p = sandbox.find_container_process(cid.as_str(), eid.as_str())?; if p.term_master.is_some() { term_exit_notifier = p.term_exit_notifier.clone(); @@ -783,12 +778,14 @@ impl protocols::agent_ttrpc::AgentService for AgentService { let s = Arc::clone(&self.sandbox); let mut sandbox = s.lock().await; - let p = find_process(&mut sandbox, cid.as_str(), eid.as_str(), false).map_err(|e| { - ttrpc_error( - ttrpc::Code::INVALID_ARGUMENT, - format!("invalid argument: {:?}", e), - ) - })?; + let p = sandbox + .find_container_process(cid.as_str(), eid.as_str()) + .map_err(|e| { + ttrpc_error( + ttrpc::Code::INVALID_ARGUMENT, + format!("invalid argument: {:?}", e), + ) + })?; p.close_stdin(); @@ -807,12 +804,14 @@ impl protocols::agent_ttrpc::AgentService for AgentService { let eid = req.exec_id.clone(); let s = Arc::clone(&self.sandbox); let mut sandbox = s.lock().await; - let p = find_process(&mut sandbox, cid.as_str(), eid.as_str(), false).map_err(|e| { - ttrpc_error( - ttrpc::Code::UNAVAILABLE, - format!("invalid argument: {:?}", e), - ) - })?; + let p = sandbox + .find_container_process(cid.as_str(), eid.as_str()) + .map_err(|e| { + ttrpc_error( + ttrpc::Code::UNAVAILABLE, + format!("invalid argument: {:?}", e), + ) + })?; if let Some(fd) = p.term_master { unsafe { @@ -1366,26 +1365,6 @@ async fn read_stream(reader: Arc>>, l: usize) -> Resu Ok(content) } -fn find_process<'a>( - sandbox: &'a mut Sandbox, - cid: &'a str, - eid: &'a str, - init: bool, -) -> Result<&'a mut Process> { - let ctr = sandbox - .get_container(cid) - .ok_or_else(|| anyhow!("Invalid container id"))?; - - if init || eid.is_empty() { - return ctr - .processes - .get_mut(&ctr.init_process_pid) - .ok_or_else(|| anyhow!("cannot find init process!")); - } - - ctr.get_process(eid).map_err(|_| anyhow!("Invalid exec id")) -} - pub fn start(s: Arc>, server_address: &str) -> Result { let agent_service = Box::new(AgentService { sandbox: s }) as Box; diff --git a/src/agent/src/sandbox.rs b/src/agent/src/sandbox.rs index 526464014..e13d10e34 100644 --- a/src/agent/src/sandbox.rs +++ b/src/agent/src/sandbox.rs @@ -226,6 +226,21 @@ impl Sandbox { None } + pub fn find_container_process(&mut self, cid: &str, eid: &str) -> Result<&mut Process> { + let ctr = self + .get_container(cid) + .ok_or_else(|| anyhow!("Invalid container id"))?; + + if eid.is_empty() { + return ctr + .processes + .get_mut(&ctr.init_process_pid) + .ok_or_else(|| anyhow!("cannot find init process!")); + } + + ctr.get_process(eid).map_err(|_| anyhow!("Invalid exec id")) + } + #[instrument] pub async fn destroy(&mut self) -> Result<()> { for ctr in self.containers.values_mut() { @@ -454,6 +469,7 @@ mod tests { use nix::mount::MsFlags; use oci::{Linux, Root, Spec}; use rustjail::container::LinuxContainer; + use rustjail::process::Process; use rustjail::specconv::CreateOpts; use slog::Logger; use std::fs::{self, File}; @@ -785,4 +801,50 @@ mod tests { let ret = s.destroy().await; assert!(ret.is_ok()); } + + #[tokio::test] + async fn test_find_container_process() { + skip_if_not_root!(); + let logger = slog::Logger::root(slog::Discard, o!()); + let mut s = Sandbox::new(&logger).unwrap(); + let cid = "container-123"; + + let mut linux_container = create_linuxcontainer(); + linux_container.init_process_pid = 1; + linux_container.id = cid.to_string(); + // add init process + linux_container.processes.insert( + 1, + Process::new(&logger, &oci::Process::default(), "1", true, 1).unwrap(), + ); + // add exec process + linux_container.processes.insert( + 123, + Process::new(&logger, &oci::Process::default(), "exec-123", false, 1).unwrap(), + ); + + s.add_container(linux_container); + + // empty exec-id will return init process + let p = s.find_container_process(cid, ""); + assert!(p.is_ok(), "Expecting Ok, Got {:?}", p); + let p = p.unwrap(); + assert_eq!("1", p.exec_id, "exec_id should be 1"); + assert!(p.init, "init flag should be true"); + + // get exist exec-id will return the exec process + let p = s.find_container_process(cid, "exec-123"); + assert!(p.is_ok(), "Expecting Ok, Got {:?}", p); + let p = p.unwrap(); + assert_eq!("exec-123", p.exec_id, "exec_id should be exec-123"); + assert!(!p.init, "init flag should be false"); + + // get not exist exec-id will return error + let p = s.find_container_process(cid, "exec-456"); + assert!(p.is_err(), "Expecting Error, Got {:?}", p); + + // container does not exist + let p = s.find_container_process("not-exist-cid", ""); + assert!(p.is_err(), "Expecting Error, Got {:?}", p); + } } From bc9558149ca8e4ba6279a1b7ec2e9a7ef8387b7b Mon Sep 17 00:00:00 2001 From: "James O. D. Hunt" Date: Mon, 22 Nov 2021 16:28:07 +0000 Subject: [PATCH 55/66] docs: Move doc requirements section higher Move the documentation requirements document link up so that it appears immediately below the "How to Contribute" section. Signed-off-by: James O. D. Hunt --- docs/README.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/README.md b/docs/README.md index 5dedec378..fcb76cf34 100644 --- a/docs/README.md +++ b/docs/README.md @@ -52,6 +52,10 @@ Documents that help to understand and contribute to Kata Containers. * [How to contribute to Kata Containers](https://github.com/kata-containers/community/blob/master/CONTRIBUTING.md) * [Code of Conduct](../CODE_OF_CONDUCT.md) +## Help Improving the Documents + +* [Documentation Requirements](Documentation-Requirements.md) + ### Code Licensing * [Licensing](Licensing-strategy.md): About the licensing strategy of Kata Containers. @@ -61,10 +65,6 @@ Documents that help to understand and contribute to Kata Containers. * [Release strategy](Stable-Branch-Strategy.md) * [Release Process](Release-Process.md) -## Help Improving the Documents - -* [Documentation Requirements](Documentation-Requirements.md) - ## Website Changes If you have a suggestion for how we can improve the From cf360fad9224ac0a8d2d4709a8c973be0ab169f2 Mon Sep 17 00:00:00 2001 From: "James O. D. Hunt" Date: Tue, 23 Nov 2021 09:45:01 +0000 Subject: [PATCH 56/66] docs: Move unit test advice doc from tests repo Unit tests necessarily need to be maintained with the code they test so it makes sense to keep the Unit Test Advice document into the main repo since that is where the majority of unit tests reside. Note: The [`Unit-Test-Advice.md` file](https://github.com/kata-containers/tests/blob/main/Unit-Test-Advice.md) was copied from the `tests` repo when it's `HEAD` was https://github.com/kata-containers/tests/commit/38855f1f40120f4f009fa30061e9affc07930640. Signed-off-by: James O. D. Hunt --- docs/README.md | 4 + docs/Unit-Test-Advice.md | 333 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 337 insertions(+) create mode 100644 docs/Unit-Test-Advice.md diff --git a/docs/README.md b/docs/README.md index fcb76cf34..d1efc14ba 100644 --- a/docs/README.md +++ b/docs/README.md @@ -52,6 +52,10 @@ Documents that help to understand and contribute to Kata Containers. * [How to contribute to Kata Containers](https://github.com/kata-containers/community/blob/master/CONTRIBUTING.md) * [Code of Conduct](../CODE_OF_CONDUCT.md) +## Help Writing Unit Tests + +* [Unit Test Advice](Unit-Test-Advice.md) + ## Help Improving the Documents * [Documentation Requirements](Documentation-Requirements.md) diff --git a/docs/Unit-Test-Advice.md b/docs/Unit-Test-Advice.md new file mode 100644 index 000000000..88663c976 --- /dev/null +++ b/docs/Unit-Test-Advice.md @@ -0,0 +1,333 @@ +# Unit Test Advice + +* [Overview](#overview) +* [Assertions](#assertions) + * [golang assertions](#golang-assertions) + * [rust assertions](#rust-assertions) +* [Table driven tests](#table-driven-tests) + * [golang table driven tests](#golang-table-driven-tests) + * [rust table driven tests](#rust-table-driven-tests) +* [Temporary files](#temporary-files) + * [golang temporary files](#golang-temporary-files) + * [rust temporary files](#rust-temporary-files) +* [User running the test](#user-running-the-test) + * [running golang tests as different users](#running-golang-tests-as-different-users) + * [running rust tests as different users](#running-rust-tests-as-different-users) + +## Overview + +This document offers advice on writing a Unit Test (UT) in +[`golang`](https://golang.org) and [`rust`](https://www.rust-lang.org). + +## Assertions + +### golang assertions + +Use the `testify` assertions package to create a new assertion object as this +keeps the test code free from distracting `if` tests: + +```go +func TestSomething(t *testing.T) { + assert := assert.New(t) + + err := doSomething() + assert.NoError(err) +} +``` + +### rust assertions + +Use the standard set of `assert!()` macros. + +## Table driven tests + +Try to write tests using a table-based approach. This allows you to distill +the logic into a compact table (rather than spreading the tests across +multiple test functions). It also makes it easy to cover all the +interesting boundary conditions: + +### golang table driven tests + +Assume the following function: + +```go +// The function under test. +// +// Accepts a string and an integer and returns the +// result of sticking them together separated by a dash as a string. +func joinParamsWithDash(str string, num int) (string, error) { + if str == "" { + return "", errors.New("string cannot be blank") + } + + if num <= 0 { + return "", errors.New("number must be positive") + } + + return fmt.Sprintf("%s-%d", str, num), nil +} +``` + +A table driven approach to testing it: + +```go +import ( + "testing" + "github.com/stretchr/testify/assert" +) + +func TestJoinParamsWithDash(t *testing.T) { + assert := assert.New(t) + + // Type used to hold function parameters and expected results. + type testData struct { + param1 string + param2 int + expectedResult string + expectError bool + } + + // List of tests to run including the expected results + data := []testData{ + // Failure scenarios + {"", -1, "", true}, + {"", 0, "", true}, + {"", 1, "", true}, + {"foo", 0, "", true}, + {"foo", -1, "", true}, + + // Success scenarios + {"foo", 1, "foo-1", false}, + {"bar", 42, "bar-42", false}, + } + + // Run the tests + for i, d := range data { + // Create a test-specific string that is added to each assert + // call. It will be displayed if any assert test fails. + msg := fmt.Sprintf("test[%d]: %+v", i, d) + + // Call the function under test + result, err := joinParamsWithDash(d.param1, d.param2) + + // update the message for more information on failure + msg = fmt.Sprintf("%s, result: %q, err: %v", msg, result, err) + + if d.expectError { + assert.Error(err, msg) + + // If an error is expected, there is no point + // performing additional checks. + continue + } + + assert.NoError(err, msg) + assert.Equal(d.expectedResult, result, msg) + } +} +``` + +### rust table driven tests + +Assume the following function: + +```rust +// Convenience type to allow Result return types to only specify the type +// for the true case; failures are specified as static strings. +pub type Result = std::result::Result; + +// The function under test. +// +// Accepts a string and an integer and returns the +// result of sticking them together separated by a dash as a string. +fn join_params_with_dash(str: &str, num: i32) -> Result { + if str == "" { + return Err("string cannot be blank"); + } + + if num <= 0 { + return Err("number must be positive"); + } + + let result = format!("{}-{}", str, num); + + Ok(result) +} + +``` + +A table driven approach to testing it: + +```rust +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_join_params_with_dash() { + // This is a type used to record all details of the inputs + // and outputs of the function under test. + #[derive(Debug)] + struct TestData<'a> { + str: &'a str, + num: i32, + result: Result, + } + + // The tests can now be specified as a set of inputs and outputs + let tests = &[ + // Failure scenarios + TestData { + str: "", + num: 0, + result: Err("string cannot be blank"), + }, + TestData { + str: "foo", + num: -1, + result: Err("number must be positive"), + }, + + // Success scenarios + TestData { + str: "foo", + num: 42, + result: Ok("foo-42".to_string()), + }, + TestData { + str: "-", + num: 1, + result: Ok("--1".to_string()), + }, + ]; + + // Run the tests + for (i, d) in tests.iter().enumerate() { + // Create a string containing details of the test + let msg = format!("test[{}]: {:?}", i, d); + + // Call the function under test + let result = join_params_with_dash(d.str, d.num); + + // Update the test details string with the results of the call + let msg = format!("{}, result: {:?}", msg, result); + + // Perform the checks + if d.result.is_ok() { + assert!(result == d.result, msg); + continue; + } + + let expected_error = format!("{}", d.result.as_ref().unwrap_err()); + let actual_error = format!("{}", result.unwrap_err()); + assert!(actual_error == expected_error, msg); + } + } +} +``` + +## Temporary files + +Always delete temporary files on success. + +### golang temporary files + +```go +func TestSomething(t *testing.T) { + assert := assert.New(t) + + // Create a temporary directory + tmpdir, err := ioutil.TempDir("", "") + assert.NoError(err) + + // Delete it at the end of the test + defer os.RemoveAll(tmpdir) + + // Add test logic that will use the tmpdir here... +} +``` + +### rust temporary files + +Use the `tempfile` crate which allows files and directories to be deleted +automatically: + +```rust +#[cfg(test)] +mod tests { + use tempfile::tempdir; + + #[test] + fn test_something() { + + // Create a temporary directory (which will be deleted automatically + let dir = tempdir().expect("failed to create tmpdir"); + + let filename = dir.path().join("file.txt"); + + // create filename ... + } +} + +``` + +## User running the test + +[Unit tests are run *twice*](https://github.com/kata-containers/tests/blob/main/.ci/go-test.sh): + +- as the current user +- as the `root` user (if different to the current user) + +When writing a test consider which user should run it; even if the code the +test is exercising runs as `root`, it may be necessary to *only* run the test +as a non-`root` for the test to be meaningful. + +Some repositories already provide utility functions to skip a test: + +- if running as `root` +- if not running as `root` + +### running golang tests as different users + +The runtime repository has the most comprehensive set of skip abilities. See: + +- https://github.com/kata-containers/kata-containers/tree/main/src/runtime/pkg/katatestutils + +### running rust tests as different users + +One method is to use the `nix` crate along with some custom macros: + +``` +#[cfg(test)] +mod tests { + #[allow(unused_macros)] + macro_rules! skip_if_root { + () => { + if nix::unistd::Uid::effective().is_root() { + println!("INFO: skipping {} which needs non-root", module_path!()); + return; + } + }; + } + + #[allow(unused_macros)] + macro_rules! skip_if_not_root { + () => { + if !nix::unistd::Uid::effective().is_root() { + println!("INFO: skipping {} which needs root", module_path!()); + return; + } + }; + } + + #[test] + fn test_that_must_be_run_as_root() { + // Not running as the superuser, so skip. + skip_if_not_root!(); + + // Run test *iff* the user running the test is root + + // ... + } +} +``` From 597b239ef3ef78feb689eb60df1e71130efedd6e Mon Sep 17 00:00:00 2001 From: "James O. D. Hunt" Date: Tue, 23 Nov 2021 09:54:35 +0000 Subject: [PATCH 57/66] docs: Remove TOC in UT advice doc Remove the table of contents in the Unit Test Advice document since GitHub auto-generates these now. See: https://github.com/kata-containers/kata-containers/pull/2023 Signed-off-by: James O. D. Hunt --- docs/Unit-Test-Advice.md | 14 -------------- 1 file changed, 14 deletions(-) diff --git a/docs/Unit-Test-Advice.md b/docs/Unit-Test-Advice.md index 88663c976..8bdbfd4c6 100644 --- a/docs/Unit-Test-Advice.md +++ b/docs/Unit-Test-Advice.md @@ -1,19 +1,5 @@ # Unit Test Advice -* [Overview](#overview) -* [Assertions](#assertions) - * [golang assertions](#golang-assertions) - * [rust assertions](#rust-assertions) -* [Table driven tests](#table-driven-tests) - * [golang table driven tests](#golang-table-driven-tests) - * [rust table driven tests](#rust-table-driven-tests) -* [Temporary files](#temporary-files) - * [golang temporary files](#golang-temporary-files) - * [rust temporary files](#rust-temporary-files) -* [User running the test](#user-running-the-test) - * [running golang tests as different users](#running-golang-tests-as-different-users) - * [running rust tests as different users](#running-rust-tests-as-different-users) - ## Overview This document offers advice on writing a Unit Test (UT) in From c1111a1d2d7720cbf0087b384eeb29d5c3fee31c Mon Sep 17 00:00:00 2001 From: "James O. D. Hunt" Date: Tue, 23 Nov 2021 09:56:27 +0000 Subject: [PATCH 58/66] docs: Use leading caps for lang names in UT advice doc Use a capital letter when referring to Golang and Rust (and remove unnecessary backticks for Rust). > **Note:** > > We continue refer to "Go" as "Golang" since it's a common alias, > but, crucially, familiarity with this name makes searching for > information using this term possible: "Go" is too generic a word. Signed-off-by: James O. D. Hunt --- docs/Unit-Test-Advice.md | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/docs/Unit-Test-Advice.md b/docs/Unit-Test-Advice.md index 8bdbfd4c6..f4231dff7 100644 --- a/docs/Unit-Test-Advice.md +++ b/docs/Unit-Test-Advice.md @@ -3,11 +3,11 @@ ## Overview This document offers advice on writing a Unit Test (UT) in -[`golang`](https://golang.org) and [`rust`](https://www.rust-lang.org). +[Golang](https://golang.org) and [Rust](https://www.rust-lang.org). ## Assertions -### golang assertions +### Golang assertions Use the `testify` assertions package to create a new assertion object as this keeps the test code free from distracting `if` tests: @@ -21,7 +21,7 @@ func TestSomething(t *testing.T) { } ``` -### rust assertions +### Rust assertions Use the standard set of `assert!()` macros. @@ -32,7 +32,7 @@ the logic into a compact table (rather than spreading the tests across multiple test functions). It also makes it easy to cover all the interesting boundary conditions: -### golang table driven tests +### Golang table driven tests Assume the following function: @@ -113,7 +113,7 @@ func TestJoinParamsWithDash(t *testing.T) { } ``` -### rust table driven tests +### Rust table driven tests Assume the following function: @@ -216,7 +216,7 @@ mod tests { Always delete temporary files on success. -### golang temporary files +### Golang temporary files ```go func TestSomething(t *testing.T) { @@ -233,7 +233,7 @@ func TestSomething(t *testing.T) { } ``` -### rust temporary files +### Rust temporary files Use the `tempfile` crate which allows files and directories to be deleted automatically: @@ -273,13 +273,13 @@ Some repositories already provide utility functions to skip a test: - if running as `root` - if not running as `root` -### running golang tests as different users +### running Golang tests as a different user The runtime repository has the most comprehensive set of skip abilities. See: - https://github.com/kata-containers/kata-containers/tree/main/src/runtime/pkg/katatestutils -### running rust tests as different users +### running Rust tests as a different user One method is to use the `nix` crate along with some custom macros: From e8bb6b26660e745bd625fe14bc89634e2b930c8d Mon Sep 17 00:00:00 2001 From: "James O. D. Hunt" Date: Tue, 23 Nov 2021 10:20:39 +0000 Subject: [PATCH 59/66] docs: Correct repo name usage Change reference from "runtime repo" to "main repo" in unit test advice document. Signed-off-by: James O. D. Hunt --- docs/Unit-Test-Advice.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/Unit-Test-Advice.md b/docs/Unit-Test-Advice.md index f4231dff7..8d3b7b94e 100644 --- a/docs/Unit-Test-Advice.md +++ b/docs/Unit-Test-Advice.md @@ -275,7 +275,7 @@ Some repositories already provide utility functions to skip a test: ### running Golang tests as a different user -The runtime repository has the most comprehensive set of skip abilities. See: +The main repository has the most comprehensive set of skip abilities. See: - https://github.com/kata-containers/kata-containers/tree/main/src/runtime/pkg/katatestutils From 318b3f187bef9f073c3fe73de72f65c0818e31ff Mon Sep 17 00:00:00 2001 From: "James O. D. Hunt" Date: Tue, 23 Nov 2021 10:21:29 +0000 Subject: [PATCH 60/66] docs: No present continuous in UT advice doc Change some headings to avoid using the present continuous tense which should not be used for headings. Signed-off-by: James O. D. Hunt --- docs/Unit-Test-Advice.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/Unit-Test-Advice.md b/docs/Unit-Test-Advice.md index 8d3b7b94e..e88d8acf0 100644 --- a/docs/Unit-Test-Advice.md +++ b/docs/Unit-Test-Advice.md @@ -257,7 +257,7 @@ mod tests { ``` -## User running the test +## Test user [Unit tests are run *twice*](https://github.com/kata-containers/tests/blob/main/.ci/go-test.sh): @@ -273,13 +273,13 @@ Some repositories already provide utility functions to skip a test: - if running as `root` - if not running as `root` -### running Golang tests as a different user +### Run Golang tests as a different user The main repository has the most comprehensive set of skip abilities. See: - https://github.com/kata-containers/kata-containers/tree/main/src/runtime/pkg/katatestutils -### running Rust tests as a different user +### Run Rust tests as a different user One method is to use the `nix` crate along with some custom macros: From 9fed7d0bde7dbcce1f1128a9f5e1e6e52b4c0d52 Mon Sep 17 00:00:00 2001 From: "James O. D. Hunt" Date: Tue, 23 Nov 2021 10:25:16 +0000 Subject: [PATCH 61/66] docs: Mention anyhow for error handling in UT doc Add a comment stating that `anyhow` and `thiserror` should be used in real rust code, rather than the unwieldy default `Result` handling shown in the example. Signed-off-by: James O. D. Hunt --- docs/Unit-Test-Advice.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/Unit-Test-Advice.md b/docs/Unit-Test-Advice.md index e88d8acf0..07af3922f 100644 --- a/docs/Unit-Test-Advice.md +++ b/docs/Unit-Test-Advice.md @@ -120,6 +120,8 @@ Assume the following function: ```rust // Convenience type to allow Result return types to only specify the type // for the true case; failures are specified as static strings. +// XXX: This is an example. In real code use the "anyhow" and +// XXX: "thiserror" crates. pub type Result = std::result::Result; // The function under test. From fcf45b0c92d152ddfb222de32eaf8f9a0fe1f321 Mon Sep 17 00:00:00 2001 From: "James O. D. Hunt" Date: Tue, 23 Nov 2021 10:26:47 +0000 Subject: [PATCH 62/66] docs: Use more idiomatic rust string check Rather than comparing a string to a literal in the rust example, use `.is_empty()` as that approach is more idiomatic and preferred. Signed-off-by: James O. D. Hunt --- docs/Unit-Test-Advice.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/Unit-Test-Advice.md b/docs/Unit-Test-Advice.md index 07af3922f..cc46486a0 100644 --- a/docs/Unit-Test-Advice.md +++ b/docs/Unit-Test-Advice.md @@ -129,7 +129,7 @@ pub type Result = std::result::Result; // Accepts a string and an integer and returns the // result of sticking them together separated by a dash as a string. fn join_params_with_dash(str: &str, num: i32) -> Result { - if str == "" { + if str.is_empty() { return Err("string cannot be blank"); } From baf4f76d97da9c57f6a6fb0a22ed4cb69f47dbbb Mon Sep 17 00:00:00 2001 From: "James O. D. Hunt" Date: Tue, 23 Nov 2021 10:28:47 +0000 Subject: [PATCH 63/66] docs: More detail on running tests as different users Add some more detail to the unit test advice document about running tests as different users. Signed-off-by: James O. D. Hunt --- docs/Unit-Test-Advice.md | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/docs/Unit-Test-Advice.md b/docs/Unit-Test-Advice.md index cc46486a0..b877f8eb8 100644 --- a/docs/Unit-Test-Advice.md +++ b/docs/Unit-Test-Advice.md @@ -268,12 +268,9 @@ mod tests { When writing a test consider which user should run it; even if the code the test is exercising runs as `root`, it may be necessary to *only* run the test -as a non-`root` for the test to be meaningful. - -Some repositories already provide utility functions to skip a test: - -- if running as `root` -- if not running as `root` +as a non-`root` for the test to be meaningful. Add appropriate skip +guards around code that requires `root` and non-`root` so that the test +will run if the correct type of user is detected and skipped if not. ### Run Golang tests as a different user From d41c375c4f2a8a97016fa56ee9e27cf0b61b47f3 Mon Sep 17 00:00:00 2001 From: "James O. D. Hunt" Date: Tue, 23 Nov 2021 10:29:21 +0000 Subject: [PATCH 64/66] docs: Add more advice to the UT advice doc Add information to the unit test advice document on test strategies and the test environment. Signed-off-by: James O. D. Hunt --- docs/Unit-Test-Advice.md | 61 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 61 insertions(+) diff --git a/docs/Unit-Test-Advice.md b/docs/Unit-Test-Advice.md index b877f8eb8..52dce452b 100644 --- a/docs/Unit-Test-Advice.md +++ b/docs/Unit-Test-Advice.md @@ -5,6 +5,67 @@ This document offers advice on writing a Unit Test (UT) in [Golang](https://golang.org) and [Rust](https://www.rust-lang.org). +## General advice + +### Unit test strategies + +#### Positive and negative tests + +Always add positive tests (where success is expected) *and* negative +tests (where failure is expected). + +#### Boundary condition tests + +Try to add unit tests that exercise boundary conditions such as: + +- Missing values (`null` or `None`). +- Empty strings and huge strings. +- Empty (or uninitialised) complex data structures + (such as lists, vectors and hash tables). +- Common numeric values (such as `-1`, `0`, `1` and the minimum and + maximum values). + +#### Test unusual values + +Also always consider "unusual" input values such as: + +- String values containing spaces, Unicode characters, special + characters, escaped characters or null bytes. + + > **Note:** Consider these unusual values in prefix, infix and + > suffix position. + +- String values that cannot be converted into numeric values or which + contain invalid structured data (such as invalid JSON). + +#### Other types of tests + +If the code requires other forms of testing (such as stress testing, +fuzz testing and integration testing), raise a GitHub issue and +reference it on the issue you are using for the main work. This +ensures the test team are aware that a new test is required. + +### Test environment + +#### Create unique files and directories + +Ensure your tests do not write to a fixed file or directory. This can +cause problems when running multiple tests simultaneously and also +when running tests after a previous test run failure. + +#### Assume parallel testing + +Always assume your tests will be run *in parallel*. If this is +problematic for a test, force it to run in isolation using the +`serial_test` crate for Rust code for example. + +### Running + +Ensure you run the unit tests and they all pass before raising a PR. +Ideally do this on different distributions on different architectures +to maximise coverage (and so minimise surprises when your code runs in +the CI). + ## Assertions ### Golang assertions From aff32756081b472be0c7a5f1283c323e2d926df2 Mon Sep 17 00:00:00 2001 From: "James O. D. Hunt" Date: Tue, 23 Nov 2021 10:49:33 +0000 Subject: [PATCH 65/66] docs: Add a code PR advice document Add a document giving advice to code PR authors. Fixes: #3099. Signed-off-by: James O. D. Hunt --- docs/README.md | 4 + docs/code-pr-advice.md | 246 +++++++++++++++++++++++++++++++++++++++++ 2 files changed, 250 insertions(+) create mode 100644 docs/code-pr-advice.md diff --git a/docs/README.md b/docs/README.md index d1efc14ba..f5fd38eef 100644 --- a/docs/README.md +++ b/docs/README.md @@ -52,6 +52,10 @@ Documents that help to understand and contribute to Kata Containers. * [How to contribute to Kata Containers](https://github.com/kata-containers/community/blob/master/CONTRIBUTING.md) * [Code of Conduct](../CODE_OF_CONDUCT.md) +## Help Writing a Code PR + +* [Code PR advice](code-pr-advice.md). + ## Help Writing Unit Tests * [Unit Test Advice](Unit-Test-Advice.md) diff --git a/docs/code-pr-advice.md b/docs/code-pr-advice.md new file mode 100644 index 000000000..78413ccf2 --- /dev/null +++ b/docs/code-pr-advice.md @@ -0,0 +1,246 @@ +# Code PR Advice + +Before raising a PR containing code changes, we suggest you consider +the following to ensure a smooth and fast process. + +> **Note:** +> +> - All the advice in this document is optional. However, if the +> advice provided is not followed, there is no guarantee your PR +> will be merged. +> +> - All the check tools will be run automatically on your PR by the CI. +> However, if you run them locally first, there is a much better +> chance of a successful initial CI run. + +## Assumptions + +This document assumes you have already read (and in the case of the +code of conduct agreed to): + +- The [Kata Containers code of conduct](https://github.com/kata-containers/community/blob/main/CODE_OF_CONDUCT.md). +- The [Kata Containers contributing guide](https://github.com/kata-containers/community/blob/main/CONTRIBUTING.md). + +## Code + +### Architectures + +Do not write architecture-specific code if it is possible to write the +code generically. + +### General advice + +- Do not write code to impress: instead write code that is easy to read and understand. + +- Always consider which user will run the code. Try to minimise + the privileges the code requires. + +### Comments + +Always add comments if the intent of the code is not obvious. However, +try to avoid comments if the code could be made clearer (for example +by using more meaningful variable names). + +### Constants + +Don't embed magic numbers and strings in functions, particularly if +they are used repeatedly. + +Create constants at the top of the file instead. + +### Copyright and license + +Ensure all new files contain a copyright statement and an SPDX license +identifier in the comments at the top of the file. + +### FIXME and TODO + +If the code contains areas that are not fully implemented, make this +clear a comment which provides a link to a GitHub issue that provides +further information. + +Do not just rely on comments in this case though: if possible, return +a "`BUG: feature X not implemented see {bug-url}`" type error. + +### Functions + +- Keep functions relatively short (less than 100 lines is a good "rule of thumb"). + +- Document functions if the parameters, return value or general intent + of the function is not obvious. + +- Always return errors where possible. + + Do not discard error return values from the functions this function + calls. + +### Logging + +- Don't use multiple log calls when a single log call could be used. + +- Use structured logging where possible to allow + [standard tooling](https://github.com/kata-containers/tests/tree/main/cmd/log-parser) + be able to extract the log fields. + +### Names + +Give functions, macros and variables clear and meaningful names. + +### Structures + +#### Golang structures + +Unlike Rust, Go does not enforce that all structure members be set. +This has lead to numerous bugs in the past where code like the +following is used: + +```go +type Foo struct { + Key string + Value string +} + +// BUG: Key not set, but nobody noticed! ;( +let foo1 = Foo { + Value: "foo", +} +``` + +A much safer approach is to create a constructor function to enforce +integrity: + +```go +type Foo struct { + Key string + Value string +} + +func NewFoo(key, value string) (*Foo, error) { + if key == "" { + return nil, errors.New("Foo needs a key") + } + + if value == "" { + return nil, errors.New("Foo needs a value") + } + + return &Foo{ + Key: key, + Value: value, + }, nil +} + +func testFoo() error { + // BUG: Key not set, but nobody noticed! ;( + badFoo := Foo{Value: "value"} + + // Ok - the constructor performs needed validation + goodFoo, err := NewFoo("name", "value") + if err != nil { + return err + } + + return nil +``` + +> **Note:** +> +> The above is just an example. The *safest* approach would be to move +> `NewFoo()` into a separate package and make `Foo` and it's elements +> private. The compiler would then enforce the use of the constructor +> to guarantee correctly defined objects. + + +### Tracing + +Consider if the code needs to create a new +[trace span](https://github.com/kata-containers/kata-containers/blob/main/docs/tracing.md). + +Ensure any new trace spans added to the code are completed. + +## Tests + +### Unit tests + +Where possible, code changes should be accompanied by unit tests. + +Consider using the standard +[table-based approach](https://github.com/kata-containers/tests/blob/main/Unit-Test-Advice.md) +as it encourages you to make functions small and simple, and also +allows you to think about what types of value to test. + +### Other categories of test + +Raised a GitHub issue in the +[`tests`](https://github.com/kata-containers/tests) repository that +explains what sort of test is required along with as much detail as +possible. Ensure the original issue is referenced on the `tests` issue. + +### Unsafe code + +#### Rust language specifics + +Minimise the use of `unsafe` blocks in Rust code and since it is +potentially dangerous always write [unit tests][#unit-tests] +for this code where possible. + +`expect()` and `unwrap()` will cause the code to panic on error. +Prefer to return a `Result` on error rather than using these calls to +allow the caller to deal with the error condition. + +The table below lists the small number of cases where use of +`expect()` and `unwrap()` are permitted: + +| Area | Rationale for permitting | +|-|-| +| In test code (the `tests` module) | Panics will cause the test to fail, which is desirable. | +| `lazy_static!()` | This magic macro cannot "return" a value as it runs before `main()`. | +| `defer!()` | Similar to golang's `defer()` but doesn't allow the use of `?`. | +| `tokio::spawn(async move {})` | Cannot currently return a `Result` from an `async move` closure. | +| If an explicit test is performed before the `unwrap()` / `expect()` | *"Just about acceptable"*, but not ideal `[*]` | + + +`[*]` - There can lead to bad *future* code: consider what would +happen if the explicit test gets dropped in the future. This is easier +to happen if the test and the extraction of the value are two separate +operations. In summary, this strategy can introduce an insidious +maintenance issue. + +## Documentation + +### General requirements + +- All new features should be accompanied by documentation explaining: + + - What the new feature does + + - Why it is useful + + - How to use the feature + + - Any known issues or limitations + + Links should be provided to GitHub issues tracking the issues + +- The [documentation requirements document](Documentation-Requirements.md) + explains how the project formats documentation. + +### Markdown syntax + +Run the +[markdown checker](https://github.com/kata-containers/tests/tree/main/cmd/check-markdown) +on your documentation changes. + +### Spell check + +Run the +[spell checker](https://github.com/kata-containers/tests/tree/main/cmd/check-spelling) +on your documentation changes. + +## Finally + +You may wish to read the documentation that the +[Kata Review Team](https://github.com/kata-containers/community/blob/main/Rota-Process.md) use to help review PRs: + +- [PR review guide](https://github.com/kata-containers/community/blob/main/PR-Review-Guide.md). +- [documentation review process](https://github.com/kata-containers/community/blob/main/Documentation-Review-Process.md). From f0734f52c1325ad064cc9256fbbed4a4d4466778 Mon Sep 17 00:00:00 2001 From: "James O. D. Hunt" Date: Thu, 25 Nov 2021 14:44:07 +0000 Subject: [PATCH 66/66] docs: Remove extraneous whitespace Remove trailing whitespace in the unit test advice doc. Signed-off-by: James O. D. Hunt --- docs/Unit-Test-Advice.md | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/docs/Unit-Test-Advice.md b/docs/Unit-Test-Advice.md index 52dce452b..5fe8b9caf 100644 --- a/docs/Unit-Test-Advice.md +++ b/docs/Unit-Test-Advice.md @@ -222,7 +222,7 @@ mod tests { num: i32, result: Result, } - + // The tests can now be specified as a set of inputs and outputs let tests = &[ // Failure scenarios @@ -249,24 +249,24 @@ mod tests { result: Ok("--1".to_string()), }, ]; - + // Run the tests for (i, d) in tests.iter().enumerate() { // Create a string containing details of the test let msg = format!("test[{}]: {:?}", i, d); - + // Call the function under test let result = join_params_with_dash(d.str, d.num); - + // Update the test details string with the results of the call let msg = format!("{}, result: {:?}", msg, result); - + // Perform the checks if d.result.is_ok() { assert!(result == d.result, msg); continue; } - + let expected_error = format!("{}", d.result.as_ref().unwrap_err()); let actual_error = format!("{}", result.unwrap_err()); assert!(actual_error == expected_error, msg);