From f977c5f0a4c2b41b869fdc476ff6ac2dcdb68a46 Mon Sep 17 00:00:00 2001 From: Henrik Ingo Date: Tue, 7 Oct 2025 15:38:04 +0300 Subject: [PATCH] =?UTF-8?q?Add=20Nightly=20versions=20of=20benchmarks=20th?= =?UTF-8?q?at=20run=20on=20Nyrki=C3=B6=20runners?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .github/workflows/perf_nightly.yml | 165 +++++++++++++++++++++++++++++ 1 file changed, 165 insertions(+) create mode 100644 .github/workflows/perf_nightly.yml diff --git a/.github/workflows/perf_nightly.yml b/.github/workflows/perf_nightly.yml new file mode 100644 index 000000000..ec2ba7ee0 --- /dev/null +++ b/.github/workflows/perf_nightly.yml @@ -0,0 +1,165 @@ +name: Nightly Benchmarks on Nyrkiö Runners (stability) + +on: + workflow_dispatch: ["main", "notmain", "master"] + schedule: + - branches: ["main"] + - cron: '24 6,11,14,20 * * *' + push: + # branches: ["main", "notmain", "master"] + branches: ["notmain"] + pull_request: + # branches: ["main", "notmain", "master"] + branches: ["notmain"] + +env: + CARGO_TERM_COLOR: never + +jobs: + bench: + runs-on: nyrkio_perf_server_2cpu_ubuntu2404 + timeout-minutes: 30 + steps: + - uses: actions/checkout@v3 + - uses: useblacksmith/setup-node@v5 + with: + node-version: 20 + # cache: 'npm' + # - name: Install dependencies + # run: npm install && npm run build + + - name: Bench + run: make bench-exclude-tpc-h 2>&1 | tee output.txt + - name: Analyze benchmark result with Nyrkiö + uses: nyrkio/change-detection@HEAD + with: + name: nightly/turso + tool: criterion + output-file-path: output.txt + + # What to do if a change is immediately detected by Nyrkiö. + # Note that smaller changes are only detected with delay, usually after a change + # persisted over 2-7 commits. Go to nyrkiö.com to view those or configure alerts. + # Note that Nyrkiö will find all changes, also improvements. This means fail-on-alert + # on pull events isn't compatible with this workflow being required to pass branch protection. + fail-on-alert: false + comment-on-alert: true + comment-always: true + # Nyrkiö configuration + # Get yours from https://nyrkio.com/docs/getting-started + nyrkio-token: ${{ secrets.NYRKIO_JWT_TOKEN }} + # HTTP requests will fail for all non-core contributors that don't have their own token. + # Don't want that to spoil the build, so: + never-fail: true + # Make results and change points public, so that any oss contributor can see them + nyrkio-public: true + + # parameters of the algorithm. Note: These are global, so we only set them once and for all. + # Smaller p-value = less change points found. Larger p-value = more, but also more false positives. + nyrkio-settings-pvalue: 0.0001 + # Ignore changes smaller than this. + nyrkio-settings-threshold: 0% + + clickbench: + runs-on: nyrkio_perf_server_2cpu_ubuntu2404 + timeout-minutes: 30 + steps: + - uses: actions/checkout@v3 + - uses: useblacksmith/setup-node@v5 + with: + node-version: 20 + + - name: Clickbench + run: make clickbench + + - name: Analyze TURSO result with Nyrkiö + uses: nyrkio/change-detection@HEAD + with: + name: nightly/clickbench/turso + tool: time + output-file-path: clickbench-tursodb.txt + # What to do if a change is immediately detected by Nyrkiö. + # Note that smaller changes are only detected with delay, usually after a change + # persisted over 2-7 commits. Go to nyrkiö.com to view those or configure alerts. + # Note that Nyrkiö will find all changes, also improvements. This means fail-on-alert + # on pull events isn't compatible with this workflow being required to pass branch protection. + fail-on-alert: false + comment-on-alert: true + comment-always: true + # Nyrkiö configuration + # Get yours from https://nyrkio.com/docs/getting-started + nyrkio-token: ${{ secrets.NYRKIO_JWT_TOKEN }} + # HTTP requests will fail for all non-core contributors that don't have their own token. + # Don't want that to spoil the build, so: + never-fail: true + # Make results and change points public, so that any oss contributor can see them + nyrkio-public: true + + - name: Analyze SQLITE3 result with Nyrkiö + uses: nyrkio/change-detection@HEAD + with: + name: clickbench/sqlite3 + tool: time + output-file-path: clickbench-sqlite3.txt + fail-on-alert: false + comment-on-alert: true + comment-always: false + nyrkio-token: ${{ secrets.NYRKIO_JWT_TOKEN }} + never-fail: true + nyrkio-public: true + + tpc-h-criterion: + runs-on: nyrkio_perf_server_2cpu_ubuntu2404 + timeout-minutes: 60 + env: + DB_FILE: "perf/tpc-h/TPC-H.db" + steps: + - uses: actions/checkout@v3 + - uses: useblacksmith/rust-cache@v3 + with: + prefix-key: "v1-rust" # can be updated if we need to reset caches due to non-trivial change in the dependencies (for example, custom env var were set for single workspace project) + + - name: Cache TPC-H + id: cache-primes + uses: useblacksmith/cache@v5 + with: + path: ${{ env.DB_FILE }} + key: tpc-h + - name: Download TPC-H + if: steps.cache-primes.outputs.cache-hit != 'true' + env: + DB_URL: "https://github.com/lovasoa/TPCH-sqlite/releases/download/v1.0/TPC-H.db" + run: wget -O $DB_FILE --no-verbose $DB_URL + + - name: Bench + run: cargo bench --bench tpc_h_benchmark 2>&1 | tee output.txt + - name: Analyze benchmark result with Nyrkiö + uses: nyrkio/change-detection@HEAD + with: + name: nightly/tpc-h + tool: criterion + output-file-path: output.txt + + # What to do if a change is immediately detected by Nyrkiö. + # Note that smaller changes are only detected with delay, usually after a change + # persisted over 2-7 commits. Go to nyrkiö.com to view those or configure alerts. + # Note that Nyrkiö will find all changes, also improvements. This means fail-on-alert + # on pull events isn't compatible with this workflow being required to pass branch protection. + fail-on-alert: false + comment-on-alert: true + comment-always: true + # Nyrkiö configuration + # Get yours from https://nyrkio.com/docs/getting-started + nyrkio-token: ${{ secrets.NYRKIO_JWT_TOKEN }} + # HTTP requests will fail for all non-core contributors that don't have their own token. + # Don't want that to spoil the build, so: + never-fail: true + # Make results and change points public, so that any oss contributor can see them + nyrkio-public: true + + # parameters of the algorithm. Note: These are global, so we only set them once and for all. + # Smaller p-value = less change points found. Larger p-value = more, but also more false positives. + nyrkio-settings-pvalue: 0.0001 + # Ignore changes smaller than this. + nyrkio-settings-threshold: 0% +