From b7ed531ef080419d79ebb2eefde2b3d11eb96c06 Mon Sep 17 00:00:00 2001 From: Henrik Ingo Date: Thu, 15 May 2025 03:54:38 +0300 Subject: [PATCH 1/5] =?UTF-8?q?Update=20Nyrki=C3=B6=20change=20detection?= =?UTF-8?q?=20to=20newest=20version?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit (Yes, I changed the name of the repo.) Also switch back to 'cargo' for the parser, which is the original upstream code. I created 'criterion' because I didn't realize cargo bench spitz half of the text to stdout the other half to stderr. --- .github/workflows/rust_perf.yml | 49 ++++++++------------------------- 1 file changed, 11 insertions(+), 38 deletions(-) diff --git a/.github/workflows/rust_perf.yml b/.github/workflows/rust_perf.yml index 1d6cd6f25..c4b764d41 100644 --- a/.github/workflows/rust_perf.yml +++ b/.github/workflows/rust_perf.yml @@ -22,18 +22,12 @@ jobs: # run: npm install && npm run build - name: Bench - run: cargo bench 2>&1 | tee output.txt - # - name: Bench (fake) - # run: | - # pwd - # ls - # cp .github/data/limbo-cargo-output.txt output.txt - + run: cargo bench | tee output.txt - name: Analyze benchmark result with Nyrkiö - uses: nyrkio/github-action-benchmark@HEAD + uses: nyrkio/change-detection@HEAD with: name: turso - tool: criterion + tool: cargo output-file-path: output.txt # What to do if a change is immediately detected by Nyrkiö. @@ -45,25 +39,20 @@ jobs: comment-on-alert: true comment-always: false # Nyrkiö configuration - nyrkio-enable: true # Get yours from https://nyrkio.com/docs/getting-started nyrkio-token: ${{ secrets.NYRKIO_JWT_TOKEN }} - # You may not want share the NYRKIO_JWT_TOKEN token with pull requests, for example. - # In that case this task would unnecessarily fail for random contributors. Don't want that: + # HTTP requests will fail for all non-core contributors that don't have their own token. + # Don't want that to spoil the build, so: never-fail: true # Make results and change points public, so that any oss contributor can see them nyrkio-public: true - nyrkio-api-root: https://nyrkio.com/api/v0 - # Make results and change points public, so that any oss contributor can see them + # parameters of the algorithm. Note: These are global, so we only set them once and for all. + # Smaller p-value = less change points found. Larger p-value = more, but also more false positives. nyrkio-settings-pvalue: 0.01% + # Ignore changes smaller than this. nyrkio-settings-threshold: 2% - # Old way... - # Explicitly set this to null. We don't want threshold based alerts today. - external-data-json-path: null - gh-repository: null - clickbench: runs-on: ubuntu-latest steps: @@ -76,7 +65,7 @@ jobs: run: make clickbench - name: Analyze LIMBO result with Nyrkiö - uses: nyrkio/github-action-benchmark@HEAD + uses: nyrkio/change-detection@HEAD with: name: clickbench/limbo tool: time @@ -90,24 +79,14 @@ jobs: comment-on-alert: true comment-always: false # Nyrkiö configuration - nyrkio-enable: true # Get yours from https://nyrkio.com/docs/getting-started nyrkio-token: ${{ secrets.NYRKIO_JWT_TOKEN }} - # You may not want share the NYRKIO_JWT_TOKEN token with pull requests, for example. - # In that case this task would unnecessarily fail for random contributors. Don't want that: + # HTTP requests will fail for all non-core contributors that don't have their own token. + # Don't want that to spoil the build, so: never-fail: true # Make results and change points public, so that any oss contributor can see them nyrkio-public: true - nyrkio-api-root: https://nyrkio.com/api/v0 - # Team support = results are visible and manageable to everyone in the same Github org - # nyrkio-org: tursodatabase - - # Old way... - # Explicitly set this to null. We don't want threshold based alerts today. - external-data-json-path: null - gh-repository: null - - name: Analyze SQLITE3 result with Nyrkiö uses: nyrkio/github-action-benchmark@HEAD with: @@ -117,12 +96,6 @@ jobs: fail-on-alert: false comment-on-alert: true comment-always: false - nyrkio-enable: true nyrkio-token: ${{ secrets.NYRKIO_JWT_TOKEN }} never-fail: true nyrkio-public: true - nyrkio-api-root: https://nyrkio.com/api/v0 - # nyrkio-org: tursodatabase - - external-data-json-path: null - gh-repository: null From e1330f5e620b3ab4a2d131b9a909813ec0371eea Mon Sep 17 00:00:00 2001 From: Henrik Ingo Date: Thu, 15 May 2025 04:17:40 +0300 Subject: [PATCH 2/5] fix: p-value is not a percentage --- .github/workflows/rust_perf.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/rust_perf.yml b/.github/workflows/rust_perf.yml index c4b764d41..847257a59 100644 --- a/.github/workflows/rust_perf.yml +++ b/.github/workflows/rust_perf.yml @@ -49,7 +49,7 @@ jobs: # parameters of the algorithm. Note: These are global, so we only set them once and for all. # Smaller p-value = less change points found. Larger p-value = more, but also more false positives. - nyrkio-settings-pvalue: 0.01% + nyrkio-settings-pvalue: 0.01 # Ignore changes smaller than this. nyrkio-settings-threshold: 2% From 070b38d04f412b63267faf999f786bf6b63a85b2 Mon Sep 17 00:00:00 2001 From: Henrik Ingo Date: Thu, 15 May 2025 05:06:24 +0300 Subject: [PATCH 3/5] =?UTF-8?q?Nyrki=C3=B6=20Github=20Action=20requires=20?= =?UTF-8?q?--output-format=20bencher?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .github/workflows/rust_perf.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/rust_perf.yml b/.github/workflows/rust_perf.yml index 847257a59..dcec2fc29 100644 --- a/.github/workflows/rust_perf.yml +++ b/.github/workflows/rust_perf.yml @@ -22,7 +22,7 @@ jobs: # run: npm install && npm run build - name: Bench - run: cargo bench | tee output.txt + run: cargo bench --output-format bencher | tee output.txt - name: Analyze benchmark result with Nyrkiö uses: nyrkio/change-detection@HEAD with: From 1029cf700b942d2b3d25dc8478ca4f5df2a80f18 Mon Sep 17 00:00:00 2001 From: Henrik Ingo Date: Thu, 15 May 2025 05:27:53 +0300 Subject: [PATCH 4/5] Upstream 'cargo' parser doesn't work, go back to 'criterion' --- .github/workflows/rust_perf.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/rust_perf.yml b/.github/workflows/rust_perf.yml index dcec2fc29..11ea9c9da 100644 --- a/.github/workflows/rust_perf.yml +++ b/.github/workflows/rust_perf.yml @@ -22,12 +22,12 @@ jobs: # run: npm install && npm run build - name: Bench - run: cargo bench --output-format bencher | tee output.txt + run: cargo bench 2>&1 | tee output.txt - name: Analyze benchmark result with Nyrkiö uses: nyrkio/change-detection@HEAD with: name: turso - tool: cargo + tool: criterion output-file-path: output.txt # What to do if a change is immediately detected by Nyrkiö. From ba9bab983ebac61c692efb77baf9634ba0c245e0 Mon Sep 17 00:00:00 2001 From: Henrik Ingo Date: Fri, 16 May 2025 01:36:37 +0300 Subject: [PATCH 5/5] =?UTF-8?q?Change=20Nyrki=C3=B6=20parameters=20to=20p?= =?UTF-8?q?=3D0.0001=20and=20min=3D0%?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This is my recommendation based on fiddling with your data. Seems to roughly halve the number of change points. --- .github/workflows/rust_perf.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/rust_perf.yml b/.github/workflows/rust_perf.yml index 11ea9c9da..fe8a3e98e 100644 --- a/.github/workflows/rust_perf.yml +++ b/.github/workflows/rust_perf.yml @@ -49,9 +49,9 @@ jobs: # parameters of the algorithm. Note: These are global, so we only set them once and for all. # Smaller p-value = less change points found. Larger p-value = more, but also more false positives. - nyrkio-settings-pvalue: 0.01 + nyrkio-settings-pvalue: 0.0001 # Ignore changes smaller than this. - nyrkio-settings-threshold: 2% + nyrkio-settings-threshold: 0% clickbench: runs-on: ubuntu-latest