From 101ffdbce03086b3ef5cd56ef46bff2e58f99783 Mon Sep 17 00:00:00 2001 From: merwanehamadi Date: Mon, 3 Jul 2023 11:53:28 -0700 Subject: [PATCH] Integrate with gpt engineer (#47) --- .github/workflows/gpt-engineer.yml | 70 +++++++++++++++++++ .gitmodules | 4 ++ .../retrieval/{Retrieval.py => retrieval.py} | 0 agbenchmark/config.json | 5 -- agbenchmark/start_benchmark.py | 3 +- agent/Auto-GPT | 2 +- agent/gpt-engineer | 1 + config.json | 5 ++ poetry.lock | 17 +++-- pyproject.toml | 2 +- 10 files changed, 91 insertions(+), 18 deletions(-) create mode 100644 .github/workflows/gpt-engineer.yml rename agbenchmark/challenges/retrieval/{Retrieval.py => retrieval.py} (100%) delete mode 100644 agbenchmark/config.json create mode 160000 agent/gpt-engineer create mode 100644 config.json diff --git a/.github/workflows/gpt-engineer.yml b/.github/workflows/gpt-engineer.yml new file mode 100644 index 00000000..e0dbac2f --- /dev/null +++ b/.github/workflows/gpt-engineer.yml @@ -0,0 +1,70 @@ +name: gpt-engineer Regression Test + +on: + workflow_dispatch: + branches: [ master ] + push: + branches: [ stable, master, ci-test* ] + +jobs: + regression-tests: + permissions: + pull-requests: write + contents: write + runs-on: ubuntu-latest + timeout-minutes: 30 + strategy: + matrix: + python-version: ["3.10"] + + steps: + - name: Checkout repository + uses: actions/checkout@v3 + with: + fetch-depth: 0 + ref: ${{ github.event.pull_request.head.ref }} + repository: ${{ github.event.pull_request.head.repo.full_name }} + submodules: true + + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v2 + with: + python-version: ${{ matrix.python-version }} + + - id: get_date + name: Get date + run: echo "date=$(date +'%Y-%m-%d')" >> $GITHUB_OUTPUT + + - name: Install Poetry + run: | + curl -sSL https://install.python-poetry.org | python - + + - name: Set up Poetry cache + uses: actions/cache@v2 + with: + path: | + ~/.cache/pypoetry + .venv + key: ${{ runner.os }}-poetry-${{ hashFiles('**/pyproject.toml') }}-${{ hashFiles('**/poetry.lock') }}-${{ steps.get_date.outputs.date }} + + - name: Set up venv and install Python dependencies + run: | + poetry install --only main + poetry build + + - name: Run regression tests + run: | + cd agent/gpt-engineer + make install + source venv/bin/activate + pip install ../../dist/agbenchmark-0.1.0-py3-none-any.whl + agbenchmark start --reg + env: + OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} + + - name: Upload logs as artifact + if: always() + uses: actions/upload-artifact@v3 + with: + name: gpt-engineer-projects + path: agent/gpt-engineer/projects diff --git a/.gitmodules b/.gitmodules index 2e3a86e5..b5b7ba24 100644 --- a/.gitmodules +++ b/.gitmodules @@ -2,3 +2,7 @@ path = agent/Auto-GPT url = https://github.com/Significant-Gravitas/Auto-GPT.git branch = benchmark-integration +[submodule "agent/gpt-engineer"] + path = agent/gpt-engineer + url = https://github.com/merwanehamadi/gpt-engineer.git + branch = benchmark-integration diff --git a/agbenchmark/challenges/retrieval/Retrieval.py b/agbenchmark/challenges/retrieval/retrieval.py similarity index 100% rename from agbenchmark/challenges/retrieval/Retrieval.py rename to agbenchmark/challenges/retrieval/retrieval.py diff --git a/agbenchmark/config.json b/agbenchmark/config.json deleted file mode 100644 index e1c5f154..00000000 --- a/agbenchmark/config.json +++ /dev/null @@ -1,5 +0,0 @@ -{ - "workspace": "autogpt/workspace/auto_gpt_workspace", - "func_path": "benchmarks.py", - "cutoff": 60 -} diff --git a/agbenchmark/start_benchmark.py b/agbenchmark/start_benchmark.py index 13e1af23..7489aa30 100644 --- a/agbenchmark/start_benchmark.py +++ b/agbenchmark/start_benchmark.py @@ -12,9 +12,8 @@ load_dotenv() CURRENT_DIRECTORY = Path(__file__).resolve().parent -new_path = CURRENT_DIRECTORY / "config.json" -CONFIG_PATH = str(new_path.resolve()) +CONFIG_PATH = str(Path(os.getcwd()) / "config.json") REGRESSION_TESTS_PATH = str(Path(os.getcwd()) / "regression_tests.json") diff --git a/agent/Auto-GPT b/agent/Auto-GPT index c29ec925..2e5eac51 160000 --- a/agent/Auto-GPT +++ b/agent/Auto-GPT @@ -1 +1 @@ -Subproject commit c29ec925fd9e24f219ef0f2884b08908cd66239b +Subproject commit 2e5eac51d06d495919d720d370c4d9efd49f4784 diff --git a/agent/gpt-engineer b/agent/gpt-engineer new file mode 160000 index 00000000..f91ac66b --- /dev/null +++ b/agent/gpt-engineer @@ -0,0 +1 @@ +Subproject commit f91ac66b8e8210760aaa0047f2ca11c52e55aaa5 diff --git a/config.json b/config.json new file mode 100644 index 00000000..652618e4 --- /dev/null +++ b/config.json @@ -0,0 +1,5 @@ +{ + "workspace": "projects/my-new-project/workspace", + "func_path": "benchmarks.py", + "cutoff": 60 +} diff --git a/poetry.lock b/poetry.lock index e05fc6c0..4eae340b 100644 --- a/poetry.lock +++ b/poetry.lock @@ -210,7 +210,6 @@ mypy-extensions = ">=0.4.3" pathspec = ">=0.9.0" platformdirs = ">=2" tomli = {version = ">=1.1.0", markers = "python_version < \"3.11\""} -typing-extensions = {version = ">=3.10.0.0", markers = "python_version < \"3.10\""} [package.extras] colorama = ["colorama (>=0.4.3)"] @@ -882,14 +881,14 @@ pytest = ">=3" [[package]] name = "python-dotenv" -version = "1.0.0" +version = "0.21.1" description = "Read key-value pairs from a .env file and set them as environment variables" category = "main" optional = false -python-versions = ">=3.8" +python-versions = ">=3.7" files = [ - {file = "python-dotenv-1.0.0.tar.gz", hash = "sha256:a8df96034aae6d2d50a4ebe8216326c61c3eb64836776504fcca410e5937a3ba"}, - {file = "python_dotenv-1.0.0-py3-none-any.whl", hash = "sha256:f5971a9226b701070a4bf2c38c89e5a3f0d64de8debda981d1db98583009122a"}, + {file = "python-dotenv-0.21.1.tar.gz", hash = "sha256:1c93de8f636cde3ce377292818d0e440b6e45a82f215c3744979151fa8151c49"}, + {file = "python_dotenv-0.21.1-py3-none-any.whl", hash = "sha256:41e12e0318bebc859fcc4d97d4db8d20ad21721a6aa5047dd59f090391cb549a"}, ] [package.extras] @@ -964,14 +963,14 @@ telegram = ["requests"] [[package]] name = "typing-extensions" -version = "4.7.0" +version = "4.7.1" description = "Backported and Experimental Type Hints for Python 3.7+" category = "main" optional = false python-versions = ">=3.7" files = [ - {file = "typing_extensions-4.7.0-py3-none-any.whl", hash = "sha256:5d8c9dac95c27d20df12fb1d97b9793ab8b2af8a3a525e68c80e21060c161771"}, - {file = "typing_extensions-4.7.0.tar.gz", hash = "sha256:935ccf31549830cda708b42289d44b6f74084d616a00be651601a4f968e77c82"}, + {file = "typing_extensions-4.7.1-py3-none-any.whl", hash = "sha256:440d5dd3af93b060174bf433bccd69b0babc3b15b1a8dca43789fd7f61514b36"}, + {file = "typing_extensions-4.7.1.tar.gz", hash = "sha256:b75ddc264f0ba5615db7ba217daeb99701ad295353c45f9e95963337ceeeffb2"}, ] [[package]] @@ -1083,4 +1082,4 @@ multidict = ">=4.0" [metadata] lock-version = "2.0" python-versions = "^3.10" -content-hash = "7b5ef821765fd03ae347d42a62be71cb50e97b778544da90a06d35e1808f8ac3" +content-hash = "44b5789494e73f3cb8bcb9d25daa62143e59352a246fd7724fdb3ad58c2560ae" diff --git a/pyproject.toml b/pyproject.toml index b458f44b..7e95969a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -14,7 +14,7 @@ requests = "^2.31.0" openai = "^0.27.8" pydantic = "^1.10.9" pytest-depends = "^1.0.1" -python-dotenv = "^1.0.0" +python-dotenv = "^0.21.0" click = "^8.1.3" [tool.poetry.group.dev.dependencies]