From 129713018c996bb4160bd7e2e882ed87de052001 Mon Sep 17 00:00:00 2001 From: Pekka Enberg Date: Thu, 26 Jun 2025 10:27:46 +0300 Subject: [PATCH] Bring back TPC-H benchmarks --- .github/workflows/rust_perf.yml | 120 ++++++++++++++++---------------- perf/tpc-h/run.sh | 4 +- 2 files changed, 62 insertions(+), 62 deletions(-) diff --git a/.github/workflows/rust_perf.yml b/.github/workflows/rust_perf.yml index 5323cb1d1..af8674480 100644 --- a/.github/workflows/rust_perf.yml +++ b/.github/workflows/rust_perf.yml @@ -100,63 +100,63 @@ jobs: never-fail: true nyrkio-public: true -# tpc-h-criterion: -# runs-on: blacksmith-4vcpu-ubuntu-2404 -# env: -# DB_FILE: "perf/tpc-h/TPC-H.db" -# steps: -# - uses: actions/checkout@v3 -# - uses: useblacksmith/rust-cache@v3 -# with: -# prefix-key: "v1-rust" # can be updated if we need to reset caches due to non-trivial change in the dependencies (for example, custom env var were set for single workspace project) -# -# - name: Cache TPC-H -# id: cache-primes -# uses: useblacksmith/cache@v5 -# with: -# path: ${{ env.DB_FILE }} -# key: tpc-h -# - name: Download TPC-H -# if: steps.cache-primes.outputs.cache-hit != 'true' -# env: -# DB_URL: "https://github.com/lovasoa/TPCH-sqlite/releases/download/v1.0/TPC-H.db" -# run: wget -O $DB_FILE --no-verbose $DB_URL -# -# - name: Bench -# run: cargo bench --bench tpc_h_benchmark 2>&1 | tee output.txt -# - name: Analyze benchmark result with Nyrkiö -# uses: nyrkio/change-detection@HEAD -# with: -# name: tpc-h -# tool: criterion -# output-file-path: output.txt -# -# # What to do if a change is immediately detected by Nyrkiö. -# # Note that smaller changes are only detected with delay, usually after a change -# # persisted over 2-7 commits. Go to nyrkiö.com to view those or configure alerts. -# # Note that Nyrkiö will find all changes, also improvements. This means fail-on-alert -# # on pull events isn't compatible with this workflow being required to pass branch protection. -# fail-on-alert: false -# comment-on-alert: true -# comment-always: false -# # Nyrkiö configuration -# # Get yours from https://nyrkio.com/docs/getting-started -# nyrkio-token: ${{ secrets.NYRKIO_JWT_TOKEN }} -# # HTTP requests will fail for all non-core contributors that don't have their own token. -# # Don't want that to spoil the build, so: -# never-fail: true -# # Make results and change points public, so that any oss contributor can see them -# nyrkio-public: true -# -# # parameters of the algorithm. Note: These are global, so we only set them once and for all. -# # Smaller p-value = less change points found. Larger p-value = more, but also more false positives. -# nyrkio-settings-pvalue: 0.0001 -# # Ignore changes smaller than this. -# nyrkio-settings-threshold: 0% -# -# tpc-h: -# runs-on: blacksmith-4vcpu-ubuntu-2404 -# steps: -# - uses: actions/checkout@v3 -# - name: TPC-H -# run: ./perf/tpc-h/benchmark.sh + tpc-h-criterion: + runs-on: blacksmith-4vcpu-ubuntu-2404 + env: + DB_FILE: "perf/tpc-h/TPC-H.db" + steps: + - uses: actions/checkout@v3 + - uses: useblacksmith/rust-cache@v3 + with: + prefix-key: "v1-rust" # can be updated if we need to reset caches due to non-trivial change in the dependencies (for example, custom env var were set for single workspace project) + + - name: Cache TPC-H + id: cache-primes + uses: useblacksmith/cache@v5 + with: + path: ${{ env.DB_FILE }} + key: tpc-h + - name: Download TPC-H + if: steps.cache-primes.outputs.cache-hit != 'true' + env: + DB_URL: "https://github.com/lovasoa/TPCH-sqlite/releases/download/v1.0/TPC-H.db" + run: wget -O $DB_FILE --no-verbose $DB_URL + + - name: Bench + run: cargo bench --bench tpc_h_benchmark 2>&1 | tee output.txt + - name: Analyze benchmark result with Nyrkiö + uses: nyrkio/change-detection@HEAD + with: + name: tpc-h + tool: criterion + output-file-path: output.txt + + # What to do if a change is immediately detected by Nyrkiö. + # Note that smaller changes are only detected with delay, usually after a change + # persisted over 2-7 commits. Go to nyrkiö.com to view those or configure alerts. + # Note that Nyrkiö will find all changes, also improvements. This means fail-on-alert + # on pull events isn't compatible with this workflow being required to pass branch protection. + fail-on-alert: false + comment-on-alert: true + comment-always: false + # Nyrkiö configuration + # Get yours from https://nyrkio.com/docs/getting-started + nyrkio-token: ${{ secrets.NYRKIO_JWT_TOKEN }} + # HTTP requests will fail for all non-core contributors that don't have their own token. + # Don't want that to spoil the build, so: + never-fail: true + # Make results and change points public, so that any oss contributor can see them + nyrkio-public: true + + # parameters of the algorithm. Note: These are global, so we only set them once and for all. + # Smaller p-value = less change points found. Larger p-value = more, but also more false positives. + nyrkio-settings-pvalue: 0.0001 + # Ignore changes smaller than this. + nyrkio-settings-threshold: 0% + + tpc-h: + runs-on: blacksmith-4vcpu-ubuntu-2404 + steps: + - uses: actions/checkout@v3 + - name: TPC-H + run: ./perf/tpc-h/benchmark.sh diff --git a/perf/tpc-h/run.sh b/perf/tpc-h/run.sh index 50ad16f60..d0a30650e 100755 --- a/perf/tpc-h/run.sh +++ b/perf/tpc-h/run.sh @@ -66,7 +66,7 @@ for query_file in $(ls "$QUERIES_DIR"/*.sql | sort -V); do # Clear caches before Limbo run clear_caches # Run Limbo - limbo_output=$( { time -p "$LIMBO_BIN" "$DB_FILE" --quiet --output-mode list "$(cat $query_file)" 2>&1; } 2>&1) + limbo_output=$( { time -p "$LIMBO_BIN" "$DB_FILE" --experimental-indexes --quiet --output-mode list "$(cat $query_file)" 2>&1; } 2>&1) limbo_non_time_lines=$(echo "$limbo_output" | grep -v -e "^real" -e "^user" -e "^sys") limbo_real_time=$(echo "$limbo_output" | grep "^real" | awk '{print $2}') echo "Running $query_name with SQLite3..." >&2 @@ -95,4 +95,4 @@ for query_file in $(ls "$QUERIES_DIR"/*.sql | sort -V); do done echo "-----------------------------------------------------------" -echo "TPC-H query timing comparison completed." \ No newline at end of file +echo "TPC-H query timing comparison completed."