From e56b112aabbd862c97db48dd5d60d09efbedd5b7 Mon Sep 17 00:00:00 2001 From: Silen Naihin Date: Sat, 8 Jul 2023 03:27:31 -0400 Subject: [PATCH] i/o workspace, adding superagi (#60) --- .github/workflows/superagi.yml | 62 ++++++++++++++++++++++++++++++++++ .gitmodules | 4 +++ agbenchmark/agent_interface.py | 2 +- agbenchmark/challenge.py | 10 +++--- agbenchmark/conftest.py | 57 ++++++++++++++++++++----------- agent/SuperAGI | 1 + config.json | 6 ++-- 7 files changed, 114 insertions(+), 28 deletions(-) create mode 100644 .github/workflows/superagi.yml create mode 160000 agent/SuperAGI diff --git a/.github/workflows/superagi.yml b/.github/workflows/superagi.yml new file mode 100644 index 00000000..128c28dd --- /dev/null +++ b/.github/workflows/superagi.yml @@ -0,0 +1,62 @@ +name: SuperAgi Regression Test + +on: + workflow_dispatch: + branches: [master] + push: + branches: [stable, master, ci-test*] + +jobs: + regression-tests: + permissions: + pull-requests: write + contents: write + runs-on: ubuntu-latest + timeout-minutes: 30 + strategy: + matrix: + python-version: ['3.10'] + + steps: + - name: Checkout repository + uses: actions/checkout@v3 + with: + fetch-depth: 0 + ref: ${{ github.event.pull_request.head.ref }} + repository: ${{ github.event.pull_request.head.repo.full_name }} + submodules: true + + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v2 + with: + python-version: ${{ matrix.python-version }} + + - id: get_date + name: Get date + run: echo "date=$(date +'%Y-%m-%d')" >> $GITHUB_OUTPUT + + - name: Install Poetry + run: | + curl -sSL https://install.python-poetry.org | python - + + - name: Set up Poetry cache + uses: actions/cache@v2 + with: + path: | + ~/.cache/pypoetry + .venv + key: ${{ runner.os }}-poetry-${{ hashFiles('**/pyproject.toml') }}-${{ hashFiles('**/poetry.lock') }}-${{ steps.get_date.outputs.date }} + + - name: Set up venv and install Python dependencies + run: | + poetry install --only main + poetry build + + - name: Run regression tests + run: | + cd agent/SuperAgi + cp config_template.yaml config.yaml + sed -i 's/OPENAI_API_KEY:.*/OPENAI_API_KEY: "'"${{ secrets.OPENAI_API_KEY }}"'"/' config.yaml + docker-compose up --build + pip install ../../dist/agbenchmark-0.1.0-py3-none-any.whl + agbenchmark start --reg diff --git a/.gitmodules b/.gitmodules index 5af445f7..f14b5e07 100644 --- a/.gitmodules +++ b/.gitmodules @@ -14,3 +14,7 @@ path = agent/smol-developer url = https://github.com/merwanehamadi/developer.git branch = benchmark-integration +[submodule "agent/SuperAGI"] + path = agent/SuperAGI + url = https://github.com/SilenNaihin/SuperAGI.git + branch = benchmark-integration diff --git a/agbenchmark/agent_interface.py b/agbenchmark/agent_interface.py index 77eb110b..4244fa08 100644 --- a/agbenchmark/agent_interface.py +++ b/agbenchmark/agent_interface.py @@ -15,7 +15,7 @@ MOCK_FLAG = os.getenv("MOCK_TEST") def run_agent( - task: Optional[str], + task: str, mock_func: Optional[str], config: Dict[str, Any], challenge_location: str, diff --git a/agbenchmark/challenge.py b/agbenchmark/challenge.py index 29bc3ff9..d7e1c896 100644 --- a/agbenchmark/challenge.py +++ b/agbenchmark/challenge.py @@ -48,8 +48,8 @@ class Challenge(ABC, metaclass=ChallengeMeta): return self.data.mock.mock_func if self.data.mock else None @property - def task(self) -> Optional[str]: - return ( + def task(self) -> str: + return str( self.data.mock.mock_task if self.data.mock and MOCK_TEST else self.data.task ) @@ -80,13 +80,13 @@ class Challenge(ABC, metaclass=ChallengeMeta): @staticmethod def open_file(workspace: str, filename: str) -> str: - script_dir = os.path.abspath(workspace) + script_dir = workspace workspace_dir = os.path.join(script_dir, filename) with open(workspace_dir, "r") as f: return f.read() def get_artifacts_out(self, workspace: str, file_patterns: list) -> List[str]: - script_dir = os.path.abspath(workspace) + script_dir = workspace files_contents = [] for file_pattern in file_patterns: @@ -115,7 +115,7 @@ class Challenge(ABC, metaclass=ChallengeMeta): @staticmethod def write_to_file(workspace: str, filename: str, content: str) -> None: - script_dir = os.path.abspath(workspace) + script_dir = workspace print("Writing file at", script_dir) workspace_dir = os.path.join(script_dir, filename) diff --git a/agbenchmark/conftest.py b/agbenchmark/conftest.py index 7203ee6b..40457fb6 100644 --- a/agbenchmark/conftest.py +++ b/agbenchmark/conftest.py @@ -10,19 +10,24 @@ from agbenchmark.start_benchmark import CONFIG_PATH, REGRESSION_TESTS_PATH from agbenchmark.tests.regression.RegressionManager import RegressionManager -def get_dynamic_workspace(config: Dict[str, Any]) -> str: - # Extract the string inside ${...} - path_expr = config["workspace"][2:-1] +def resolve_workspace(config: Dict[str, Any]) -> str: + if config.get("workspace", "").startswith("${") and config.get( + "workspace", "" + ).endswith("}"): + # Extract the string inside ${...} + path_expr = config["workspace"][2:-1] - # Check if it starts with "os.path.join" - if path_expr.strip().startswith("os.path.join"): - # Evaluate the path string - path_value = eval(path_expr) + # Check if it starts with "os.path.join" + if path_expr.strip().startswith("os.path.join"): + # Evaluate the path string + path_value = eval(path_expr) - # Replace the original string with the evaluated result - return path_value + # Replace the original string with the evaluated result + return path_value + else: + raise ValueError("Invalid workspace path expression.") else: - raise ValueError("Invalid workspace path expression.") + return os.path.abspath(Path(os.getcwd()) / config["workspace"]) @pytest.fixture(scope="module") @@ -31,22 +36,36 @@ def config(request: Any) -> None: with open(CONFIG_PATH, "r") as f: config = json.load(f) - if config.get("workspace", "").startswith("${") and config.get( - "workspace", "" - ).endswith("}"): - path = get_dynamic_workspace(config) - config["workspace"] = path - else: - config["workspace"] = Path(os.getcwd()) / config["workspace"] + if request.config.getoption("--mock"): + config["workspace"] = "agbenchmark/mocks/workspace" + elif isinstance(config["workspace"], str): + config["workspace"] = resolve_workspace(config) + else: # it's a input output dict + config["workspace"]["input"] = resolve_workspace(config) + config["workspace"]["output"] = resolve_workspace(config) + return config @pytest.fixture(scope="module", autouse=True) def workspace(config: Dict[str, Any]) -> Generator[str, None, None]: + output_path = config["workspace"] + + # checks if its an input output paradigm + if not isinstance(config["workspace"], str): + output_path = config["workspace"]["output"] + if not os.path.exists(config["workspace"]["input"]): + os.makedirs(config["workspace"]["input"], exist_ok=True) + + # create output directory if it doesn't exist + if not os.path.exists(output_path): + os.makedirs(output_path, exist_ok=True) + yield config["workspace"] # teardown after test function completes - for filename in os.listdir(config["workspace"]): - file_path = os.path.join(config["workspace"], filename) + + for filename in os.listdir(output_path): + file_path = os.path.join(output_path, filename) try: if os.path.isfile(file_path) or os.path.islink(file_path): os.unlink(file_path) diff --git a/agent/SuperAGI b/agent/SuperAGI new file mode 160000 index 00000000..12e248e9 --- /dev/null +++ b/agent/SuperAGI @@ -0,0 +1 @@ +Subproject commit 12e248e90112e50ee011f0dcb1b3fa02030661a4 diff --git a/config.json b/config.json index 378e6902..88526a13 100644 --- a/config.json +++ b/config.json @@ -1,6 +1,6 @@ { - "workspace": "${os.path.join(Path.home(), 'miniagi')}", - "entry_path": "benchmarks.py", - "home_path": "agent/mini-agi/", + "workspace": "projects/my-new-project/workspace", + "entry_path": "agent/gpt-engineer/benchmarks.py", + "home_path": "agent/gpt-engineer", "cutoff": 60 }