turso/.github/workflows/rust_perf.yml

name: Rust Benchmarks+Nyrkiö

on:
  workflow_dispatch:
  push:
    branches: ["main", "master", "notmain"]
  pull_request:
    branches: ["main", "notmain", "master"]

env:
  CARGO_TERM_COLOR: never

jobs:
  bench:
    runs-on: ubuntu-latest
    timeout-minutes: 30
    steps:
      - uses: actions/checkout@v3
      - uses: useblacksmith/setup-node@v5
        with:
          node-version: 20
      #     cache: 'npm'
      # - name: Install dependencies
      #   run: npm install && npm run build

      - name: Bench
        run: make bench-exclude-tpc-h  2>&1 | tee output.txt
      - name: Analyze benchmark result with Nyrkiö
        uses: nyrkio/change-detection@HEAD
        with:
          name: turso
          tool: criterion
          output-file-path: output.txt

          # What to do if a change is immediately detected by Nyrkiö.
          # Note that smaller changes are only detected with delay, usually after a change
          # persisted over 2-7 commits. Go to nyrkiö.com to view those or configure alerts.
          # Note that Nyrkiö will find all changes, also improvements. This means fail-on-alert
          # on pull events isn't compatible with this workflow being required to pass branch protection.
          fail-on-alert: false
          comment-on-alert: true
          comment-always: true
          # Nyrkiö configuration
          # Get yours from https://nyrkio.com/docs/getting-started
          nyrkio-token: ${{ secrets.NYRKIO_JWT_TOKEN }}
          # HTTP requests will fail for all non-core contributors that don't have their own token.
          # Don't want that to spoil the build, so:
          never-fail: true
          # Make results and change points public, so that any oss contributor can see them
          nyrkio-public: true

          # parameters of the algorithm. Note: These are global, so we only set them once and for all.
          # Smaller p-value = less change points found. Larger p-value = more, but also more false positives.
          nyrkio-settings-pvalue: 0.0001
          # Ignore changes smaller than this.
          nyrkio-settings-threshold: 0%

  clickbench:
    runs-on: ubuntu-latest
    timeout-minutes: 30
    steps:
      - uses: actions/checkout@v3
      - uses: useblacksmith/setup-node@v5
        with:
          node-version: 20

      - name: Clickbench
        run: make clickbench

      - name: Analyze TURSO result with Nyrkiö
        uses: nyrkio/change-detection@HEAD
        with:
          name: clickbench/turso
          tool: time
          output-file-path: clickbench-tursodb.txt
          # What to do if a change is immediately detected by Nyrkiö.
          # Note that smaller changes are only detected with delay, usually after a change
          # persisted over 2-7 commits. Go to nyrkiö.com to view those or configure alerts.
          # Note that Nyrkiö will find all changes, also improvements. This means fail-on-alert
          # on pull events isn't compatible with this workflow being required to pass branch protection.
          fail-on-alert: false
          comment-on-alert: true
          comment-always: true
          # Nyrkiö configuration
          # Get yours from https://nyrkio.com/docs/getting-started
          nyrkio-token: ${{ secrets.NYRKIO_JWT_TOKEN }}
          # HTTP requests will fail for all non-core contributors that don't have their own token.
          # Don't want that to spoil the build, so:
          never-fail: true
          # Make results and change points public, so that any oss contributor can see them
          nyrkio-public: true

      - name: Analyze SQLITE3 result with Nyrkiö
        uses: nyrkio/change-detection@HEAD
        with:
          name: clickbench/sqlite3
          tool: time
          output-file-path: clickbench-sqlite3.txt
          fail-on-alert: false
          comment-on-alert: true
          comment-always: false
          nyrkio-token: ${{ secrets.NYRKIO_JWT_TOKEN }}
          never-fail: true
          nyrkio-public: true

  tpc-h-criterion:
    runs-on: ubuntu-latest
    timeout-minutes: 60
    env:
      DB_FILE: "perf/tpc-h/TPC-H.db"
    steps:
      - uses: actions/checkout@v3
      - uses: useblacksmith/rust-cache@v3
        with:
          prefix-key: "v1-rust" # can be updated if we need to reset caches due to non-trivial change in the dependencies (for example, custom env var were set for single workspace project)

      - name: Cache TPC-H
        id: cache-primes
        uses: useblacksmith/cache@v5
        with:
          path: ${{ env.DB_FILE }}
          key: tpc-h
      - name: Download TPC-H
        if: steps.cache-primes.outputs.cache-hit != 'true'
        env:
          DB_URL: "https://github.com/lovasoa/TPCH-sqlite/releases/download/v1.0/TPC-H.db"
        run: wget -O $DB_FILE --no-verbose $DB_URL

      - name: Bench
        run: cargo bench --bench tpc_h_benchmark  2>&1 | tee output.txt
      - name: Analyze benchmark result with Nyrkiö
        uses: nyrkio/change-detection@HEAD
        with:
          name: tpc-h
          tool: criterion
          output-file-path: output.txt

          # What to do if a change is immediately detected by Nyrkiö.
          # Note that smaller changes are only detected with delay, usually after a change
          # persisted over 2-7 commits. Go to nyrkiö.com to view those or configure alerts.
          # Note that Nyrkiö will find all changes, also improvements. This means fail-on-alert
          # on pull events isn't compatible with this workflow being required to pass branch protection.
          fail-on-alert: false
          comment-on-alert: true
          comment-always: true
          # Nyrkiö configuration
          # Get yours from https://nyrkio.com/docs/getting-started
          nyrkio-token: ${{ secrets.NYRKIO_JWT_TOKEN }}
          # HTTP requests will fail for all non-core contributors that don't have their own token.
          # Don't want that to spoil the build, so:
          never-fail: true
          # Make results and change points public, so that any oss contributor can see them
          nyrkio-public: true

          # parameters of the algorithm. Note: These are global, so we only set them once and for all.
          # Smaller p-value = less change points found. Larger p-value = more, but also more false positives.
          nyrkio-settings-pvalue: 0.0001
          # Ignore changes smaller than this.
          nyrkio-settings-threshold: 0%

  tpc-h:
    runs-on: ubuntu-latest
    timeout-minutes: 30
    steps:
      - uses: actions/checkout@v3
      - name: TPC-H
        run: ./perf/tpc-h/benchmark.sh

  vfs-bench-compile:
    runs-on: ubuntu-latest
    timeout-minutes: 30
    steps:
      - uses: actions/checkout@v3
      - uses: useblacksmith/rust-cache@v3
        with:
          prefix-key: "v1-rust" # can be updated if we need to reset caches due to non-trivial change in the dependencies (for example, custom env var were set for single workspace project)

      - name: Build
        run: cargo build --release --verbose

      - name: Install uv
        uses: useblacksmith/setup-uv@v4
        with:
          enable-cache: true

      - name: Set up Python
        run: uv python install

      - name: Install the project
        run: uv sync --package turso_test

      - name: Run benchmark
        run: uv run bench-vfs "SELECT 1;" 100