From 87d41b3dfaf10792826da0a3429bc70d629c8a95 Mon Sep 17 00:00:00 2001 From: Gabriela Cervantes Date: Mon, 31 Jul 2023 16:50:16 +0000 Subject: [PATCH 1/7] metrics: Add FIO test to gha for kata metrics CI This PR adds FIO test to gha for kata metrics CI. Fixes #7502 Signed-off-by: Gabriela Cervantes --- tests/metrics/storage/fio-k8s/fio-test-ci.sh | 85 ++++++++++++++++++++ 1 file changed, 85 insertions(+) create mode 100755 tests/metrics/storage/fio-k8s/fio-test-ci.sh diff --git a/tests/metrics/storage/fio-k8s/fio-test-ci.sh b/tests/metrics/storage/fio-k8s/fio-test-ci.sh new file mode 100755 index 000000000..3ae51bea4 --- /dev/null +++ b/tests/metrics/storage/fio-k8s/fio-test-ci.sh @@ -0,0 +1,85 @@ +#!/bin/bash +# +# Copyright (c) 2022-2023 Intel Corporation +# +# SPDX-License-Identifier: Apache-2.0 + +set -e + +# General env +SCRIPT_PATH=$(dirname "$(readlink -f "$0")") +source "${SCRIPT_PATH}/../../lib/common.bash" +FIO_PATH="${GOPATH}/src/${TEST_REPO}/metrics/storage/fio-k8s" +TEST_NAME="${TEST_NAME:-fio}" + +function main() { + cmds=("bc" "jq") + check_cmds "${cmds[@]}" + check_processes + init_env + + export KUBECONFIG="$HOME/.kube/config" + + pushd "${FIO_PATH}" + echo "INFO: Running K8S FIO test" + make test-ci + popd + + test_result_file="${FIO_PATH}/cmd/fiotest/test-results/kata/randrw-sync.job/output.json" + + metrics_json_init + local read_io=$(cat $test_result_file | grep io_bytes | head -1 | sed 's/[[:blank:]]//g' | cut -f2 -d ':' | cut -f1 -d ',') + local read_bw=$(cat $test_result_file | grep bw_bytes | head -1 | sed 's/[[:blank:]]//g' | cut -f2 -d ':' | cut -f1 -d ',') + local read_90_percentile=$(cat $test_result_file | grep 90.000000 | head -1 | sed 's/[[:blank:]]//g' | cut -f2 -d ':' | cut -f1 -d ',') + local read_95_percentile=$(cat $test_result_file | grep 95.000000 | head -1 | sed 's/[[:blank:]]//g' | cut -f2 -d ':' | cut -f1 -d ',') + local write_io=$(cat $test_result_file | grep io_bytes | head -2 | tail -1 | sed 's/[[:blank:]]//g' | cut -f2 -d ':' | cut -f1 -d ',') + local write_bw=$(cat $test_result_file | grep bw_bytes | head -2 | tail -1 | sed 's/[[:blank:]]//g' | cut -f2 -d ':' | cut -f1 -d ',') + local write_90_percentile=$(cat $test_result_file | grep 90.000000 | head -2 | tail -1 | sed 's/[[:blank:]]//g' | cut -f2 -d ':' | cut -f1 -d ',') + local write_95_percentile=$(cat $test_result_file | grep 95.000000 | head -2 | tail -1 | sed 's/[[:blank:]]//g' | cut -f2 -d ':' | cut -f1 -d ',') + + metrics_json_start_array + local json="$(cat << EOF + { + "readio": { + "Result" : $read_io, + "Units" : "bytes" + }, + "readbw": { + "Result" : $read_bw, + "Units" : "bytes/sec" + }, + "read90percentile": { + "Result" : $read_90_percentile, + "Units" : "ns" + }, + "read95percentile": { + "Result" : $read_95_percentile, + "Units" : "ns" + }, + "writeio": { + "Result" : $write_io, + "Units" : "bytes" + }, + "writebw": { + "Result" : $write_bw, + "Units" : "bytes/sec" + }, + "write90percentile": { + "Result" : $write_90_percentile, + "Units" : "ns" + }, + "write95percentile": { + "Result" : $write_95_percentile, + "Units" : "ns" + } + } +EOF +)" + metrics_json_add_array_element "$json" + metrics_json_end_array "Results" + metrics_json_save + + check_processes +} + +main "$@" From 69f05cf9e6fc700c57bc7e139c1729148fe15cbc Mon Sep 17 00:00:00 2001 From: Gabriela Cervantes Date: Mon, 31 Jul 2023 17:30:05 +0000 Subject: [PATCH 2/7] metrics: Add FIO general README This PR adds FIO general README information. Signed-off-by: Gabriela Cervantes --- tests/metrics/storage/fio-k8s/README.md | 26 +++++++++++++++++++++++++ 1 file changed, 26 insertions(+) create mode 100644 tests/metrics/storage/fio-k8s/README.md diff --git a/tests/metrics/storage/fio-k8s/README.md b/tests/metrics/storage/fio-k8s/README.md new file mode 100644 index 000000000..8daad8408 --- /dev/null +++ b/tests/metrics/storage/fio-k8s/README.md @@ -0,0 +1,26 @@ +# FIO test in Kubernetes + +This is an automation to run `fio` with Kubernetes. + +## Requirements: +- Kubernetes cluster running. +- Kata configured as `runtimeclass`. +The test is structured as follow: +- [fio-test]: Program wrapper to launch `fio` in a K8s pod. +- [pkg]: Library code that could be used for more `fio` automation. +- [configs]: Configuration files used by [fio-test]. +- [DAX-compare-test]: Script to run [fio-test] to generate `fio` data for Kata with/without `virtio-fs DAX` and K8s bare-metal runtime(`runc`). +- [report] Jupyter Notebook to create reports for data generated by [DAX-compare-test]. + +## Top-level Makefile targets +- `build`: Build `fio` metrics. +- `test`: quick test, used to verify changes in [fio-test]. +- `run`: Run `fio` metrics and generate reports. +- `test-report-interactive`: Run python notebook in `localhost:8888`, useful to edit the report. +- `test-report`: Generate report from data generated by `make test`. + +[fio-test]:cmd/fiotest +[configs]:configs +[pkg]:pkg +[report]:scripts/dax-compare-test/report +[DAX-compare-test]:scripts/dax-compare-test/README.md From 21f5b652331859abc528964759fc9d7ae525c38f Mon Sep 17 00:00:00 2001 From: Gabriela Cervantes Date: Mon, 31 Jul 2023 17:32:40 +0000 Subject: [PATCH 3/7] metrics: Add FIO information in storage general README This PR adds FIO information in storage general README. Signed-off-by: Gabriela Cervantes --- tests/metrics/storage/README.md | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/tests/metrics/storage/README.md b/tests/metrics/storage/README.md index b9d1cc68a..305c7e227 100644 --- a/tests/metrics/storage/README.md +++ b/tests/metrics/storage/README.md @@ -1,11 +1,27 @@ # Kata Containers storage I/O tests + The metrics tests in this directory are designed to be used to assess storage IO. + ## `Blogbench` test + The `blogbench` script is based on the `blogbench` program which is designed to emulate a busy blog server with a number of concurrent threads performing a mixture of reads, writes and rewrites. + ### Running the `blogbench` test + The `blogbench` test can be run by hand, for example: ``` $ cd metrics $ bash storage/blogbench.sh ``` +## `fio` test + +The `fio` test utilises the [fio tool](https://github.com/axboe/fio), configured +to perform measurements upon a single test file. + +The test configuration used by the script can be modified by setting a number of +environment variables to change or over-ride the test defaults. + +## DAX `virtio-fs` `fio` Kubernetes tests + +[Test](fio-k8s/README.md) to compare the use of DAX option in `virtio-fs`. From 8a584589ff39bbaba9796604323f86f8179482e6 Mon Sep 17 00:00:00 2001 From: Gabriela Cervantes Date: Mon, 31 Jul 2023 17:42:44 +0000 Subject: [PATCH 4/7] metrics: Add DAX FIO README This PR adds DAX FIO README information. Signed-off-by: Gabriela Cervantes --- .../metrics/storage/fio-k8s/scripts/README.md | 41 +++++++++++++++++++ 1 file changed, 41 insertions(+) create mode 100644 tests/metrics/storage/fio-k8s/scripts/README.md diff --git a/tests/metrics/storage/fio-k8s/scripts/README.md b/tests/metrics/storage/fio-k8s/scripts/README.md new file mode 100644 index 000000000..71cfdea84 --- /dev/null +++ b/tests/metrics/storage/fio-k8s/scripts/README.md @@ -0,0 +1,41 @@ +# FIO in Kubernetes + +This test run `fio` jobs to measure how Kata Containers work using virtio-fs DAX. The test works using Kubernetes. +The test has to run in a single node cluster, it is needed as the test modifies Kata configuration file. + +The `virtio-fs` options that this test will use are: + +* `cache mode` Only `auto`, this is the most compatible mode for most of the Kata use cases. Today this is default in Kata. +* `thread pool size` Restrict the number of worker threads per request queue, zero means no thread pool. +* `DAX` +``` +File contents can be mapped into a memory window on the host, allowing the guest to directly access data from the host page cache. This has several advantages: The guest page cache is bypassed, reducing the memory footprint. No communication is necessary +to access file contents, improving I/O performance. Shared file access is coherent between virtual machines on the same host even with mmap. +``` + +This test by default iterates over different `virtio-fs` configurations. + +| test name | DAX | thread pool size | cache mode | +|---------------------------|-----|------------------|------------| +| pool_0_cache_auto_no_DAX | no | 0 | auto | pool_0_cache_auto_DAX | yes | 0 | auto | + +The `fio` options used are: + +`ioengine`: How the IO requests are issued to the kernel. +* `libaio`: Supports async IO for both direct and buffered IO. +* `mmap`: File is memory mapped with mmap(2) and data copied to/from using memcpy(3). +`rw type`: Type of I/O pattern. +* `randread`: Random reads. +* `randrw`: Random mixed reads and writes. +* `randwrite`: Random writes. +* `read`: Sequential reads. +* `write`: Sequential writes. Additional notes: Some jobs contain a `multi` prefix. This means that the same job runs more than once at the same time using its own file. + +### Static `fio` values: +Some `fio` values are not modified over all the jobs. +`runtime`: Tell `fio` to terminate processing after the specified period of time(seconds). +`loops`: Run the specified number of iterations of this job. Used to repeat the same workload a given number of times. +`iodepth`: Number of I/O units to keep in flight against the file. Note that increasing `iodepth` beyond 1 will not affect synchronous `ioengine`. +`size`: The total size of file I/O for each thread of this job. +`direct`: If value is true, use non-buffered I/O. This is usually O_`DIRECT`. +`blocksize`: The block size in bytes used for I/O units From 8933d54428d012b6154e45a2d4ca3b8abb4d210b Mon Sep 17 00:00:00 2001 From: Gabriela Cervantes Date: Mon, 31 Jul 2023 17:51:11 +0000 Subject: [PATCH 5/7] metrics: Add FIO to gha run script This PR adds FIO to gha run script. Signed-off-by: Gabriela Cervantes --- tests/metrics/gha-run.sh | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/tests/metrics/gha-run.sh b/tests/metrics/gha-run.sh index 99ec5a1c1..3a663bf23 100755 --- a/tests/metrics/gha-run.sh +++ b/tests/metrics/gha-run.sh @@ -87,6 +87,14 @@ function run_test_tensorflow() { bash tests/metrics/machine_learning/tensorflow.sh 1 20 } +function run_test_fio() { + info "Running FIO test using ${KATA_HYPERVISOR} hypervisor" + # ToDo: remove the exit once the metrics workflow is stable + exit 0 + + bash storage/fio-k8s/fio-test-ci.sh +} + function main() { action="${1:-}" case "${action}" in @@ -97,6 +105,7 @@ function main() { run-test-memory-usage-inside-container) run_test_memory_usage_inside_container ;; run-test-blogbench) run_test_blogbench ;; run-test-tensorflow) run_test_tensorflow ;; + run-test-fio) run_test_fio ;; *) >&2 die "Invalid argument" ;; esac } From f7642480951564d0625665a07cd8bd6e4371f3e2 Mon Sep 17 00:00:00 2001 From: Gabriela Cervantes Date: Mon, 31 Jul 2023 17:54:03 +0000 Subject: [PATCH 6/7] gha: Add FIO test to run metrics yaml This PR adds FIO test to run metrics yaml. Signed-off-by: Gabriela Cervantes --- .github/workflows/run-metrics.yaml | 5 +++- tests/metrics/storage/fio-k8s/README.md | 6 ++++- tests/metrics/storage/fio-k8s/fio-test-ci.sh | 2 +- .../scripts/{ => dax-compare-test}/Makefile | 0 .../scripts/{ => dax-compare-test}/README.md | 24 ++++++++++++------- .../compare-virtiofsd-dax.sh | 0 6 files changed, 25 insertions(+), 12 deletions(-) rename tests/metrics/storage/fio-k8s/scripts/{ => dax-compare-test}/Makefile (100%) rename tests/metrics/storage/fio-k8s/scripts/{ => dax-compare-test}/README.md (60%) rename tests/metrics/storage/fio-k8s/scripts/{ => dax-compare-test}/compare-virtiofsd-dax.sh (100%) diff --git a/.github/workflows/run-metrics.yaml b/.github/workflows/run-metrics.yaml index f1ac9d61a..596a3dea4 100644 --- a/.github/workflows/run-metrics.yaml +++ b/.github/workflows/run-metrics.yaml @@ -49,9 +49,12 @@ jobs: - name: run tensorflow test run: bash tests/metrics/gha-run.sh run-test-tensorflow + - name: run fio test + run: bash tests/metrics/gha-run.sh run-test-fio + - name: make metrics tarball ${{ matrix.vmm }} run: bash tests/metrics/gha-run.sh make-tarball-results - + - name: archive metrics results ${{ matrix.vmm }} uses: actions/upload-artifact@v3 with: diff --git a/tests/metrics/storage/fio-k8s/README.md b/tests/metrics/storage/fio-k8s/README.md index 8daad8408..96798849a 100644 --- a/tests/metrics/storage/fio-k8s/README.md +++ b/tests/metrics/storage/fio-k8s/README.md @@ -3,9 +3,12 @@ This is an automation to run `fio` with Kubernetes. ## Requirements: + - Kubernetes cluster running. - Kata configured as `runtimeclass`. -The test is structured as follow: + +## Test structure: + - [fio-test]: Program wrapper to launch `fio` in a K8s pod. - [pkg]: Library code that could be used for more `fio` automation. - [configs]: Configuration files used by [fio-test]. @@ -13,6 +16,7 @@ The test is structured as follow: - [report] Jupyter Notebook to create reports for data generated by [DAX-compare-test]. ## Top-level Makefile targets + - `build`: Build `fio` metrics. - `test`: quick test, used to verify changes in [fio-test]. - `run`: Run `fio` metrics and generate reports. diff --git a/tests/metrics/storage/fio-k8s/fio-test-ci.sh b/tests/metrics/storage/fio-k8s/fio-test-ci.sh index 3ae51bea4..a05ca9bf0 100755 --- a/tests/metrics/storage/fio-k8s/fio-test-ci.sh +++ b/tests/metrics/storage/fio-k8s/fio-test-ci.sh @@ -9,7 +9,7 @@ set -e # General env SCRIPT_PATH=$(dirname "$(readlink -f "$0")") source "${SCRIPT_PATH}/../../lib/common.bash" -FIO_PATH="${GOPATH}/src/${TEST_REPO}/metrics/storage/fio-k8s" +FIO_PATH="${GOPATH}/src/github.com/kata-containers/kata-containers/tests/metrics/storage/fio-k8s" TEST_NAME="${TEST_NAME:-fio}" function main() { diff --git a/tests/metrics/storage/fio-k8s/scripts/Makefile b/tests/metrics/storage/fio-k8s/scripts/dax-compare-test/Makefile similarity index 100% rename from tests/metrics/storage/fio-k8s/scripts/Makefile rename to tests/metrics/storage/fio-k8s/scripts/dax-compare-test/Makefile diff --git a/tests/metrics/storage/fio-k8s/scripts/README.md b/tests/metrics/storage/fio-k8s/scripts/dax-compare-test/README.md similarity index 60% rename from tests/metrics/storage/fio-k8s/scripts/README.md rename to tests/metrics/storage/fio-k8s/scripts/dax-compare-test/README.md index 71cfdea84..c8b00f0f9 100644 --- a/tests/metrics/storage/fio-k8s/scripts/README.md +++ b/tests/metrics/storage/fio-k8s/scripts/dax-compare-test/README.md @@ -1,6 +1,6 @@ # FIO in Kubernetes -This test run `fio` jobs to measure how Kata Containers work using virtio-fs DAX. The test works using Kubernetes. +This test runs `fio` jobs to measure how Kata Containers work using virtio-fs DAX. The test works using Kubernetes. The test has to run in a single node cluster, it is needed as the test modifies Kata configuration file. The `virtio-fs` options that this test will use are: @@ -17,25 +17,31 @@ This test by default iterates over different `virtio-fs` configurations. | test name | DAX | thread pool size | cache mode | |---------------------------|-----|------------------|------------| -| pool_0_cache_auto_no_DAX | no | 0 | auto | pool_0_cache_auto_DAX | yes | 0 | auto | +| pool_0_cache_auto_no_DAX | no | 0 | auto | +| pool_0_cache_auto_DAX | yes | 0 | auto | The `fio` options used are: `ioengine`: How the IO requests are issued to the kernel. * `libaio`: Supports async IO for both direct and buffered IO. * `mmap`: File is memory mapped with mmap(2) and data copied to/from using memcpy(3). + `rw type`: Type of I/O pattern. * `randread`: Random reads. * `randrw`: Random mixed reads and writes. * `randwrite`: Random writes. * `read`: Sequential reads. -* `write`: Sequential writes. Additional notes: Some jobs contain a `multi` prefix. This means that the same job runs more than once at the same time using its own file. +* `write`: Sequential writes. + +Additional notes: Some jobs contain a `multi` prefix. This means that the same job runs more than once at the same time using its own file. ### Static `fio` values: + Some `fio` values are not modified over all the jobs. -`runtime`: Tell `fio` to terminate processing after the specified period of time(seconds). -`loops`: Run the specified number of iterations of this job. Used to repeat the same workload a given number of times. -`iodepth`: Number of I/O units to keep in flight against the file. Note that increasing `iodepth` beyond 1 will not affect synchronous `ioengine`. -`size`: The total size of file I/O for each thread of this job. -`direct`: If value is true, use non-buffered I/O. This is usually O_`DIRECT`. -`blocksize`: The block size in bytes used for I/O units + +* `runtime`: Tell `fio` to terminate processing after the specified period of time(seconds). +* `loops`: Run the specified number of iterations of this job. Used to repeat the same workload a given number of times. +* `iodepth`: Number of I/O units to keep in flight against the file. Note that increasing `iodepth` beyond 1 will not affect synchronous `ioengine`. +* `size`: The total size of file I/O for each thread of this job. +* `direct`: If value is true, use non-buffered I/O. This is usually O_`DIRECT`. +* `blocksize`: The block size in bytes used for I/O units. diff --git a/tests/metrics/storage/fio-k8s/scripts/compare-virtiofsd-dax.sh b/tests/metrics/storage/fio-k8s/scripts/dax-compare-test/compare-virtiofsd-dax.sh similarity index 100% rename from tests/metrics/storage/fio-k8s/scripts/compare-virtiofsd-dax.sh rename to tests/metrics/storage/fio-k8s/scripts/dax-compare-test/compare-virtiofsd-dax.sh From ad6e53c399b6ac768b0575ba6c59b8d3c0a90686 Mon Sep 17 00:00:00 2001 From: Gabriela Cervantes Date: Tue, 1 Aug 2023 23:34:15 +0000 Subject: [PATCH 7/7] metrics: Modify boot time values This PR modifies boot time values limit. Signed-off-by: Gabriela Cervantes --- .../ci_worker/checkmetrics-json-clh-kata-metric8.toml | 2 +- .../ci_worker/checkmetrics-json-qemu-kata-metric8.toml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/metrics/cmd/checkmetrics/ci_worker/checkmetrics-json-clh-kata-metric8.toml b/tests/metrics/cmd/checkmetrics/ci_worker/checkmetrics-json-clh-kata-metric8.toml index 9ca4d139b..a9199a9e0 100644 --- a/tests/metrics/cmd/checkmetrics/ci_worker/checkmetrics-json-clh-kata-metric8.toml +++ b/tests/metrics/cmd/checkmetrics/ci_worker/checkmetrics-json-clh-kata-metric8.toml @@ -17,7 +17,7 @@ description = "measure container lifecycle timings" checkvar = ".\"boot-times\".Results | .[] | .\"to-workload\".Result" checktype = "mean" midval = 0.69 -minpercent = 30.0 +minpercent = 40.0 maxpercent = 30.0 [[metric]] diff --git a/tests/metrics/cmd/checkmetrics/ci_worker/checkmetrics-json-qemu-kata-metric8.toml b/tests/metrics/cmd/checkmetrics/ci_worker/checkmetrics-json-qemu-kata-metric8.toml index 4860fa256..5437d2d6a 100644 --- a/tests/metrics/cmd/checkmetrics/ci_worker/checkmetrics-json-qemu-kata-metric8.toml +++ b/tests/metrics/cmd/checkmetrics/ci_worker/checkmetrics-json-qemu-kata-metric8.toml @@ -17,7 +17,7 @@ description = "measure container lifecycle timings" checkvar = ".\"boot-times\".Results | .[] | .\"to-workload\".Result" checktype = "mean" midval = 0.71 -minpercent = 30.0 +minpercent = 40.0 maxpercent = 30.0 [[metric]]