diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index 6421e9016..67d6d7e23 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -88,9 +88,3 @@ jobs: - name: Test run: SQLITE_EXEC="sqlite3" make test-compat - clickbench: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 - - name: Clickbench - run: make clickbench diff --git a/.github/workflows/rust_perf.yml b/.github/workflows/rust_perf.yml index 7dc2bb111..12614dc4b 100644 --- a/.github/workflows/rust_perf.yml +++ b/.github/workflows/rust_perf.yml @@ -71,3 +71,46 @@ jobs: # Explicitly set this to null. We don't want threshold based alerts today. external-data-json-path: null gh-repository: null + + clickbench: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - uses: actions/setup-node@v4 + with: + node-version: 20 + + - name: Clickbench + run: make clickbench + + - name: Analyze benchmark result with Nyrkiö + uses: nyrkio/github-action-benchmark@HEAD + with: + name: clickbench + tool: time + output-file-path: [clickbench-limbo.txt, clickbench-sqlite3.txt] + # What to do if a change is immediately detected by Nyrkiö. + # Note that smaller changes are only detected with delay, usually after a change + # persisted over 2-7 commits. Go to nyrkiö.com to view those or configure alerts. + # Note that Nyrkiö will find all changes, also improvements. This means fail-on-alert + # on pull events isn't compatible with this workflow being required to pass branch protection. + fail-on-alert: false + comment-on-alert: true + comment-always: false + # Nyrkiö configuration + nyrkio-enable: true + # Get yours from https://nyrkio.com/docs/getting-started + nyrkio-token: ${{ secrets.NYRKIO_JWT_TOKEN }} + # You may not want share the NYRKIO_JWT_TOKEN token with pull requests, for example. + # In that case this task would unnecessarily fail for random contributors. Don't want that: + never-fail: true + # Make results and change points public, so that any oss contributor can see them + nyrkio-public: true + # Team support = results are visible and manageable to everyone in the same Github org + nyrkio-org: tursodatabase + + # Old way... + # Explicitly set this to null. We don't want threshold based alerts today. + external-data-json-path: null + gh-repository: null + diff --git a/perf/clickbench/run.sh b/perf/clickbench/run.sh index 6a317571b..ed7616f25 100755 --- a/perf/clickbench/run.sh +++ b/perf/clickbench/run.sh @@ -25,18 +25,20 @@ clear_caches() { # Clear caches once echo "The script might ask you to enter the password for sudo, in order to clear system caches." clear_caches +count=1; # Run the queries, skipping any that are commented out # Between each invocation to Limbo/Sqlite, purge the caches grep -v '^--' "$CLICKBENCH_DIR/queries.sql" | while read -r query; do - echo "$query"; + echo "$count $query" | tee -a "${REPO_ROOT}/clickbench-limbo.txt" | tee -a "${REPO_ROOT}/clickbench-sqlite3.txt" for _ in $(seq 1 $TRIES); do clear_caches echo "----limbo----" - time "$RELEASE_BUILD_DIR/limbo" --quiet "$CLICKBENCH_DIR/mydb" <<< "${query}" + time "$RELEASE_BUILD_DIR/limbo" --quiet "$CLICKBENCH_DIR/mydb" <<< "${query}" | tee -a "${REPO_ROOT}/clickbench-limbo.txt" clear_caches echo echo "----sqlite----" - time sqlite3 "$CLICKBENCH_DIR/mydb" <<< "${query}" + time sqlite3 "$CLICKBENCH_DIR/mydb" <<< "${query}" | tee -a "${REPO_ROOT}/clickbench-sqlite3.txt" done; + count=$(($count+1)) done;