From a5de79beb6ce1856b7c0167082c26e477402b02b Mon Sep 17 00:00:00 2001
From: Reinier van der Leer <pwuts@agpt.co>
Date: Fri, 16 Feb 2024 17:41:58 +0100
Subject: [PATCH] ci(benchmark): Add nightly benchmark workflow

Added autogpts-benchmark-nightly.yml, which will run every night at 02:00 UTC with a selection of challenges.
---
 .../workflows/autogpts-benchmark-nightly.yml  | 71 +++++++++++++++++++
 1 file changed, 71 insertions(+)
 create mode 100644 .github/workflows/autogpts-benchmark-nightly.yml

diff --git a/.github/workflows/autogpts-benchmark-nightly.yml b/.github/workflows/autogpts-benchmark-nightly.yml
new file mode 100644
index 00000000..6c4d7461
--- /dev/null
+++ b/.github/workflows/autogpts-benchmark-nightly.yml
@@ -0,0 +1,71 @@
+name: AutoGPTs Nightly Benchmark
+
+on:
+  workflow_dispatch:
+  schedule:
+    - cron: '0 2 * * *'
+
+env:
+  TELEMETRY_ENVIRONMENT: autogpt-benchmark-ci
+  TELEMETRY_OPT_IN: ${{ github.event_name == 'push' && github.ref_name == 'master' }}
+
+jobs:
+  benchmark:
+    permissions:
+      contents: write
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        agent-name: [ autogpt ]
+      fail-fast: false
+    timeout-minutes: 120
+    env:
+      min-python-version: '3.10'
+      REPORTS_BRANCH: data/benchmark-reports
+      REPORTS_FOLDER: ${{ format('benchmark/reports/{0}', matrix.agent-name) }}
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+          submodules: true
+
+      - name: Set up Python ${{ env.min-python-version }}
+        uses: actions/setup-python@v5
+        with:
+          python-version: ${{ env.min-python-version }}
+
+      - name: Install Poetry
+        run: curl -sSL https://install.python-poetry.org | python -
+
+      - name: Prepare reports folder
+        run: mkdir -p ${{ env.REPORTS_FOLDER }}
+
+      - name: Benchmark ${{ matrix.agent-name }}
+        run: |
+          ./run agent start ${{ matrix.agent-name }}
+          cd autogpts/${{ matrix.agent-name }}
+          poetry run agbenchmark run -N 3 \
+            --test=ReadFile \
+            --test=BasicRetrieval --test=RevenueRetrieval2 \
+            --test=CombineCsv --test=LabelCsv --test=AnswerQuestionCombineCsv \
+            --test=UrlShortener --test=TicTacToe --test=Battleship \
+            --test=WebArenaTask_0 --test=WebArenaTask_21 --test=WebArenaTask_124 \
+            --test=WebArenaTask_134 --test=WebArenaTask_163
+        env:
+          AGENT_NAME: ${{ matrix.agent-name }}
+          OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
+          REQUESTS_CA_BUNDLE: /etc/ssl/certs/ca-certificates.crt
+          REPORTS_FOLDER: ${{ format('../../{0}', env.REPORTS_FOLDER) }}  # account for changed workdir
+
+      - name: Push reports to data branch
+        run: |
+          git config --global user.name 'GitHub Actions'
+          git config --global user.email 'github-actions@agpt.co'
+          git fetch origin ${{ env.REPORTS_BRANCH }}:${{ env.REPORTS_BRANCH }} \
+            && git checkout ${{ env.REPORTS_BRANCH }} \
+            || git checkout --orphan ${{ env.REPORTS_BRANCH }}
+          git reset --hard
+          git add ${{ env.REPORTS_FOLDER }}
+          git commit -m "Benchmark report for ${{ matrix.agent-name }} @ $(date +'%Y-%m-%d')" \
+            && git push origin ${{ env.REPORTS_BRANCH }}