Auto-GPT/.github/workflows/ci.yml

name: CI

on:
  workflow_dispatch:
    branches: [master]
  schedule:
    - cron: "0 8 * * *"
  push:
    branches: [master, ci-test*]
  pull_request:
    branches: [stable, master, release-*]

jobs:
  lint:
    runs-on: ubuntu-latest
    env:
      min-python-version: "3.10"

    steps:
      - name: Checkout repository
        uses: actions/checkout@v3
        with:
          fetch-depth: 0
          ref: ${{ github.event.pull_request.head.ref }}
          repository: ${{ github.event.pull_request.head.repo.full_name }}
          submodules: true

      - name: Set up Python ${{ env.min-python-version }}
        uses: actions/setup-python@v2
        with:
          python-version: ${{ env.min-python-version }}

      - id: get_date
        name: Get date
        run: echo "date=$(date +'%Y-%m-%d')" >> $GITHUB_OUTPUT

      - name: Install Poetry
        run: |
          curl -sSL https://install.python-poetry.org | python -

      - name: Install dependencies
        run: |
          poetry install

      - name: Lint with flake8
        run: poetry run flake8

      - name: Check black formatting
        run: poetry run  black . --check
        if: success() || failure()

      - name: Check isort formatting
        run: poetry run  isort . --check
        if: success() || failure()

      - name: Check mypy formatting
        run: poetry run mypy --ignore-missing-imports .
        if: success() || failure()

      - name: Check for unused imports and pass statements
        run: |
          cmd="poetry run autoflake --remove-all-unused-imports --recursive --ignore-init-module-imports --ignore-pass-after-docstring agbenchmark"
          $cmd --check || (echo "You have unused imports or pass statements, please run '${cmd} --in-place'" && exit 1)
        if: success() || failure()

  tests:
    name: "${{ matrix.agent-name }} (Cache: ${{ matrix.cache-enabled }})"
    runs-on: ubuntu-latest
    timeout-minutes: 10
    env:
      min-python-version: "3.10"
    strategy:
      fail-fast: false
      matrix:
        agent-name:
          - "gpt-engineer"
          - "smol-developer"
          - "Auto-GPT"
          - "mini-agi"
        cache-enabled: [ true, false ]

    steps:
      - name: Checkout repository
        uses: actions/checkout@v3
        with:
          fetch-depth: 0
          ref: ${{ github.event.pull_request.head.ref }}
          repository: ${{ github.event.pull_request.head.repo.full_name }}
          submodules: true

      - name: Set up Python ${{ env.min-python-version }}
        uses: actions/setup-python@v2
        with:
          python-version: ${{ env.min-python-version }}

      - id: get_date
        name: Get date
        run: echo "date=$(date +'%Y-%m-%d')" >> $GITHUB_OUTPUT

      - name: Install Poetry
        run: |
          curl -sSL https://install.python-poetry.org | python -

      - name: Install dependencies
        run: |
          poetry install
          poetry run agbenchmark start --mock
          poetry run agbenchmark start --mock --maintain
          poetry build

      - name: Run regression tests
        run: |
          cd agent/$AGENT_NAME
          if [ "$AGENT_NAME" == "gpt-engineer" ]; then
            make install
            source venv/bin/activate
          elif [ "$AGENT_NAME" == "Auto-GPT" ]; then
            python -m venv venv
            source venv/bin/activate
            pip install -r requirements.txt
          elif [ "$AGENT_NAME" == "mini-agi" ]; then
            python -m venv venv
            source venv/bin/activate
            pip install -r requirements.txt
            cp .env_example .env
          elif [ "$AGENT_NAME" == "smol-developer" ]; then
            python -m venv venv
            source venv/bin/activate
            pip install -r requirements.txt
          elif [ "$AGENT_NAME" == "SuperAGI" ]; then
            cp config_template.yaml config.yaml
            sed -i 's/OPENAI_API_KEY:.*/OPENAI_API_KEY: "'"${{ secrets.OPENAI_API_KEY }}"'"/' config.yaml
            docker-compose up -d --build
          else
            echo "Unknown agent name: $AGENT_NAME"
            exit 1
          fi

          pip install ../../dist/*.whl
          
          if [ "${GITHUB_EVENT_NAME}" == "pull_request" ]; then
            agbenchmark start --maintain --mock
            agbenchmark start --improve --mock
            agbenchmark start --mock
            agbenchmark start --mock --category=retrieval
            agbenchmark start --mock --category=interface
            agbenchmark start --mock --category=code
            agbenchmark start --mock --category=memory
            agbenchmark start --mock --category=iterate
          else
            curl -s https://raw.githubusercontent.com/Helicone/helicone/main/mitmproxy.sh | bash -s start
            agbenchmark start | echo "This command will always return a non zero exit code unless all the challenges are solved."
          fi
        env:
          GITHUB_EVENT_NAME: ${{ github.event_name }}
          OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
          AGENT_NAME: ${{ matrix.agent-name }}
          PROMPT_USER: false # For mini-agi. TODO: Remove this once mini-agi follows the standards.
          HELICONE_API_KEY: ${{ secrets.HELICONE_API_KEY }}
          REQUESTS_CA_BUNDLE: /etc/ssl/certs/ca-certificates.crt
          HELICONE_CACHE_ENABLED: ${{ matrix.cache-enabled }}
          HELICONE_PROPERTY_AGENT: ${{ matrix.agent-name }}
          REPORT_LOCATION:  ${{ matrix.cache-enabled == true && format('../../../benchmark_runs/{0}', matrix.agent-name) || '.' }}


      - name: Upload reports
        if: always()
        uses: actions/upload-artifact@v3
        with:
          name: ${{ matrix.agent-name }}
          path: benchmark_runs/${{ matrix.agent-name }}