Nyrkiö: Analyze limbo and sqlite3 clickbench results separately

.github/workflows/rust_perf.yml

@@ -22,17 +22,11 @@ jobs:
       # - name: Install dependencies
       #   run: npm install && npm run build
 
-      # - name: Bench
-      #   run: cargo bench  2>&1 | tee output.txt
-      - name: Bench (fake)
-        run: |
-            pwd
-            ls
-            cp .github/data/limbo-cargo-output.txt output.txt
+      - name: Bench
+        run: cargo bench  2>&1 | tee output.txt
 
       - name: Analyze benchmark result with Nyrkiö
-        uses: henrikingo/github-action-benchmark@HEAD
-        # uses: nyrkio/github-action-benchmark@HEAD
+        uses: nyrkio/github-action-benchmark@HEAD
 
         with:
           name: turso
@@ -56,13 +50,12 @@ jobs:
           # Make results and change points public, so that any oss contributor can see them
           nyrkio-public: true
 
-          nyrkio-api-root: https://staging.nyrkio.com/api/v0
+          nyrkio-api-root: https://nyrkio.com/api/v0
           # Make results and change points public, so that any oss contributor can see them
           nyrkio-settings-pvalue: 0.01%
-          nyrkio-settings-threshold: 1%
+          nyrkio-settings-threshold: 2%
           # Team support = results are visible and manageable to everyone in the same Github org
           # nyrkio-org: tursodatabase
-          # nyrkio-org: nyrkio
 
           # Old way...
           # Explicitly set this to null. We don't want threshold based alerts today.
@@ -71,6 +64,7 @@ jobs:
 
   clickbench:
     runs-on: ubuntu-latest
+    environment: test
     steps:
       - uses: actions/checkout@v3
       - uses: actions/setup-node@v4
@@ -80,12 +74,12 @@ jobs:
       - name: Clickbench
         run: make clickbench
 
-      - name: Analyze benchmark result with Nyrkiö
+      - name: Analyze LIMBO result with Nyrkiö
         uses: nyrkio/github-action-benchmark@HEAD
         with:
-          name: clickbench
+          name: clickbench/limbo
           tool: time
-          output-file-path: [clickbench-limbo.txt, clickbench-sqlite3.txt]
+          output-file-path: clickbench-limbo.txt
           # What to do if a change is immediately detected by Nyrkiö.
           # Note that smaller changes are only detected with delay, usually after a change
           # persisted over 2-7 commits. Go to nyrkiö.com to view those or configure alerts.
@@ -103,11 +97,32 @@ jobs:
           never-fail: true
           # Make results and change points public, so that any oss contributor can see them
           nyrkio-public: true
+
+          nyrkio-api-root: https://nyrkio.com/api/v0
           # Team support = results are visible and manageable to everyone in the same Github org
-          nyrkio-org: tursodatabase
+          # nyrkio-org: tursodatabase
 
           # Old way...
           # Explicitly set this to null. We don't want threshold based alerts today.
           external-data-json-path: null
           gh-repository: null
 
+      - name: Analyze SQLITE3 result with Nyrkiö
+        uses: nyrkio/github-action-benchmark@HEAD
+        with:
+          name: clickbench/sqlite3
+          tool: time
+          output-file-path: clickbench-sqlite3.txt
+          fail-on-alert: false
+          comment-on-alert: true
+          comment-always: false
+          nyrkio-enable: true
+          nyrkio-token: ${{ secrets.NYRKIO_JWT_TOKEN }}
+          never-fail: true
+          nyrkio-public: true
+          nyrkio-api-root: https://nyrkio.com/api/v0
+          # nyrkio-org: tursodatabase
+
+          external-data-json-path: null
+          gh-repository: null
+

perf/clickbench/run.sh

@@ -30,15 +30,17 @@ count=1;
 # Run the queries, skipping any that are commented out
 # Between each invocation to Limbo/Sqlite, purge the caches
 grep -v '^--' "$CLICKBENCH_DIR/queries.sql" | while read -r query; do
-    echo "$count $query" | tee -a "${REPO_ROOT}/clickbench-limbo.txt" | tee -a "${REPO_ROOT}/clickbench-sqlite3.txt"
+    echo "$count $query"
+    ((echo "$count $query") 2>&1) | tee -a clickbench-limbo.txt > /dev/null
+    ((echo "$count $query") 2>&1) | tee -a clickbench-sqlite3.txt >/dev/null
     for _ in $(seq 1 $TRIES); do
         clear_caches
         echo "----limbo----"
-        time "$RELEASE_BUILD_DIR/limbo" --quiet "$CLICKBENCH_DIR/mydb" <<< "${query}"  | tee -a "${REPO_ROOT}/clickbench-limbo.txt"
+        ((time "$RELEASE_BUILD_DIR/limbo" --quiet "$CLICKBENCH_DIR/mydb" <<< "${query}") 2>&1) | tee -a clickbench-limbo.txt
         clear_caches
         echo
         echo "----sqlite----"
-        time sqlite3 "$CLICKBENCH_DIR/mydb" <<< "${query}" | tee -a "${REPO_ROOT}/clickbench-sqlite3.txt"
+        ((time sqlite3 "$CLICKBENCH_DIR/mydb" <<< "${query}") 2>&1) | tee -a clickbench-sqlite3.txt
     done;
     count=$(($count+1))
 done;