Integrate with gpt engineer (#47)

.github/workflows/gpt-engineer.yml

@@ -0,0 +1,70 @@
+name: gpt-engineer Regression Test
+
+on:
+  workflow_dispatch:
+    branches: [ master ]
+  push:
+    branches: [ stable, master, ci-test* ]
+
+jobs:
+  regression-tests:
+    permissions:
+      pull-requests: write
+      contents: write
+    runs-on: ubuntu-latest
+    timeout-minutes: 30
+    strategy:
+      matrix:
+        python-version: ["3.10"]
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
+          ref: ${{ github.event.pull_request.head.ref }}
+          repository: ${{ github.event.pull_request.head.repo.full_name }}
+          submodules: true
+
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v2
+        with:
+          python-version: ${{ matrix.python-version }}
+
+      - id: get_date
+        name: Get date
+        run: echo "date=$(date +'%Y-%m-%d')" >> $GITHUB_OUTPUT
+
+      - name: Install Poetry
+        run: |
+          curl -sSL https://install.python-poetry.org | python -
+
+      - name: Set up Poetry cache
+        uses: actions/cache@v2
+        with:
+          path: |
+            ~/.cache/pypoetry
+            .venv
+          key: ${{ runner.os }}-poetry-${{ hashFiles('**/pyproject.toml') }}-${{ hashFiles('**/poetry.lock') }}-${{ steps.get_date.outputs.date }}
+
+      - name: Set up venv and install Python dependencies
+        run: |
+          poetry install --only main
+          poetry build
+
+      - name: Run regression tests
+        run: |
+          cd agent/gpt-engineer
+          make install
+          source venv/bin/activate
+          pip install ../../dist/agbenchmark-0.1.0-py3-none-any.whl
+          agbenchmark start --reg
+        env:
+          OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
+
+      - name: Upload logs as artifact
+        if: always()
+        uses: actions/upload-artifact@v3
+        with:
+          name: gpt-engineer-projects
+          path: agent/gpt-engineer/projects

.gitmodules

@@ -2,3 +2,7 @@
 	path = agent/Auto-GPT
 	url = https://github.com/Significant-Gravitas/Auto-GPT.git
 	branch = benchmark-integration
+[submodule "agent/gpt-engineer"]
+	path = agent/gpt-engineer
+	url = https://github.com/merwanehamadi/gpt-engineer.git
+	branch = benchmark-integration

agbenchmark/config.json

@@ -1,5 +0,0 @@
-{
-  "workspace": "autogpt/workspace/auto_gpt_workspace",
-  "func_path": "benchmarks.py",
-  "cutoff": 60
-}

agbenchmark/start_benchmark.py

@@ -12,9 +12,8 @@ load_dotenv()
 
 CURRENT_DIRECTORY = Path(__file__).resolve().parent
 
-new_path = CURRENT_DIRECTORY / "config.json"
 
-CONFIG_PATH = str(new_path.resolve())
+CONFIG_PATH = str(Path(os.getcwd()) / "config.json")
 
 REGRESSION_TESTS_PATH = str(Path(os.getcwd()) / "regression_tests.json")
 

config.json

@@ -0,0 +1,5 @@
+{
+  "workspace": "projects/my-new-project/workspace",
+  "func_path": "benchmarks.py",
+  "cutoff": 60
+}

poetry.lock

@@ -210,7 +210,6 @@ mypy-extensions = ">=0.4.3"
 pathspec = ">=0.9.0"
 platformdirs = ">=2"
 tomli = {version = ">=1.1.0", markers = "python_version < \"3.11\""}
-typing-extensions = {version = ">=3.10.0.0", markers = "python_version < \"3.10\""}
 
 [package.extras]
 colorama = ["colorama (>=0.4.3)"]
@@ -882,14 +881,14 @@ pytest = ">=3"
 
 [[package]]
 name = "python-dotenv"
-version = "1.0.0"
+version = "0.21.1"
 description = "Read key-value pairs from a .env file and set them as environment variables"
 category = "main"
 optional = false
-python-versions = ">=3.8"
+python-versions = ">=3.7"
 files = [
-    {file = "python-dotenv-1.0.0.tar.gz", hash = "sha256:a8df96034aae6d2d50a4ebe8216326c61c3eb64836776504fcca410e5937a3ba"},
-    {file = "python_dotenv-1.0.0-py3-none-any.whl", hash = "sha256:f5971a9226b701070a4bf2c38c89e5a3f0d64de8debda981d1db98583009122a"},
+    {file = "python-dotenv-0.21.1.tar.gz", hash = "sha256:1c93de8f636cde3ce377292818d0e440b6e45a82f215c3744979151fa8151c49"},
+    {file = "python_dotenv-0.21.1-py3-none-any.whl", hash = "sha256:41e12e0318bebc859fcc4d97d4db8d20ad21721a6aa5047dd59f090391cb549a"},
 ]
 
 [package.extras]
@@ -964,14 +963,14 @@ telegram = ["requests"]
 
 [[package]]
 name = "typing-extensions"
-version = "4.7.0"
+version = "4.7.1"
 description = "Backported and Experimental Type Hints for Python 3.7+"
 category = "main"
 optional = false
 python-versions = ">=3.7"
 files = [
-    {file = "typing_extensions-4.7.0-py3-none-any.whl", hash = "sha256:5d8c9dac95c27d20df12fb1d97b9793ab8b2af8a3a525e68c80e21060c161771"},
-    {file = "typing_extensions-4.7.0.tar.gz", hash = "sha256:935ccf31549830cda708b42289d44b6f74084d616a00be651601a4f968e77c82"},
+    {file = "typing_extensions-4.7.1-py3-none-any.whl", hash = "sha256:440d5dd3af93b060174bf433bccd69b0babc3b15b1a8dca43789fd7f61514b36"},
+    {file = "typing_extensions-4.7.1.tar.gz", hash = "sha256:b75ddc264f0ba5615db7ba217daeb99701ad295353c45f9e95963337ceeeffb2"},
 ]
 
 [[package]]
@@ -1083,4 +1082,4 @@ multidict = ">=4.0"
 [metadata]
 lock-version = "2.0"
 python-versions = "^3.10"
-content-hash = "7b5ef821765fd03ae347d42a62be71cb50e97b778544da90a06d35e1808f8ac3"
+content-hash = "44b5789494e73f3cb8bcb9d25daa62143e59352a246fd7724fdb3ad58c2560ae"

pyproject.toml

@@ -14,7 +14,7 @@ requests = "^2.31.0"
 openai = "^0.27.8"
 pydantic = "^1.10.9"
 pytest-depends = "^1.0.1"
-python-dotenv = "^1.0.0"
+python-dotenv = "^0.21.0"
 click = "^8.1.3"
 
 [tool.poetry.group.dev.dependencies]