i/o workspace, adding superagi (#60)

.github/workflows/superagi.yml

@@ -0,0 +1,62 @@
+name: SuperAgi Regression Test
+
+on:
+  workflow_dispatch:
+    branches: [master]
+  push:
+    branches: [stable, master, ci-test*]
+
+jobs:
+  regression-tests:
+    permissions:
+      pull-requests: write
+      contents: write
+    runs-on: ubuntu-latest
+    timeout-minutes: 30
+    strategy:
+      matrix:
+        python-version: ['3.10']
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
+          ref: ${{ github.event.pull_request.head.ref }}
+          repository: ${{ github.event.pull_request.head.repo.full_name }}
+          submodules: true
+
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v2
+        with:
+          python-version: ${{ matrix.python-version }}
+
+      - id: get_date
+        name: Get date
+        run: echo "date=$(date +'%Y-%m-%d')" >> $GITHUB_OUTPUT
+
+      - name: Install Poetry
+        run: |
+          curl -sSL https://install.python-poetry.org | python -
+
+      - name: Set up Poetry cache
+        uses: actions/cache@v2
+        with:
+          path: |
+            ~/.cache/pypoetry
+            .venv
+          key: ${{ runner.os }}-poetry-${{ hashFiles('**/pyproject.toml') }}-${{ hashFiles('**/poetry.lock') }}-${{ steps.get_date.outputs.date }}
+
+      - name: Set up venv and install Python dependencies
+        run: |
+          poetry install --only main
+          poetry build
+
+      - name: Run regression tests
+        run: |
+          cd agent/SuperAgi
+          cp config_template.yaml config.yaml
+          sed -i 's/OPENAI_API_KEY:.*/OPENAI_API_KEY: "'"${{ secrets.OPENAI_API_KEY }}"'"/' config.yaml
+          docker-compose up --build
+          pip install ../../dist/agbenchmark-0.1.0-py3-none-any.whl
+          agbenchmark start --reg

.gitmodules

@@ -14,3 +14,7 @@
 	path = agent/smol-developer
 	url = https://github.com/merwanehamadi/developer.git
 	branch = benchmark-integration
+[submodule "agent/SuperAGI"]
+	path = agent/SuperAGI
+	url = https://github.com/SilenNaihin/SuperAGI.git
+	branch = benchmark-integration

agbenchmark/agent_interface.py

@@ -15,7 +15,7 @@ MOCK_FLAG = os.getenv("MOCK_TEST")
 
 
 def run_agent(
-    task: Optional[str],
+    task: str,
     mock_func: Optional[str],
     config: Dict[str, Any],
     challenge_location: str,

agbenchmark/challenge.py

@@ -48,8 +48,8 @@ class Challenge(ABC, metaclass=ChallengeMeta):
         return self.data.mock.mock_func if self.data.mock else None
 
     @property
-    def task(self) -> Optional[str]:
-        return (
+    def task(self) -> str:
+        return str(
             self.data.mock.mock_task if self.data.mock and MOCK_TEST else self.data.task
         )
 
@@ -80,13 +80,13 @@ class Challenge(ABC, metaclass=ChallengeMeta):
 
     @staticmethod
     def open_file(workspace: str, filename: str) -> str:
-        script_dir = os.path.abspath(workspace)
+        script_dir = workspace
         workspace_dir = os.path.join(script_dir, filename)
         with open(workspace_dir, "r") as f:
             return f.read()
 
     def get_artifacts_out(self, workspace: str, file_patterns: list) -> List[str]:
-        script_dir = os.path.abspath(workspace)
+        script_dir = workspace
         files_contents = []
 
         for file_pattern in file_patterns:
@@ -115,7 +115,7 @@ class Challenge(ABC, metaclass=ChallengeMeta):
 
     @staticmethod
     def write_to_file(workspace: str, filename: str, content: str) -> None:
-        script_dir = os.path.abspath(workspace)
+        script_dir = workspace
         print("Writing file at", script_dir)
         workspace_dir = os.path.join(script_dir, filename)
 

agbenchmark/conftest.py

@@ -10,19 +10,24 @@ from agbenchmark.start_benchmark import CONFIG_PATH, REGRESSION_TESTS_PATH
 from agbenchmark.tests.regression.RegressionManager import RegressionManager
 
 
-def get_dynamic_workspace(config: Dict[str, Any]) -> str:
-    # Extract the string inside ${...}
-    path_expr = config["workspace"][2:-1]
+def resolve_workspace(config: Dict[str, Any]) -> str:
+    if config.get("workspace", "").startswith("${") and config.get(
+        "workspace", ""
+    ).endswith("}"):
+        # Extract the string inside ${...}
+        path_expr = config["workspace"][2:-1]
 
-    # Check if it starts with "os.path.join"
-    if path_expr.strip().startswith("os.path.join"):
-        # Evaluate the path string
-        path_value = eval(path_expr)
+        # Check if it starts with "os.path.join"
+        if path_expr.strip().startswith("os.path.join"):
+            # Evaluate the path string
+            path_value = eval(path_expr)
 
-        # Replace the original string with the evaluated result
-        return path_value
+            # Replace the original string with the evaluated result
+            return path_value
+        else:
+            raise ValueError("Invalid workspace path expression.")
     else:
-        raise ValueError("Invalid workspace path expression.")
+        return os.path.abspath(Path(os.getcwd()) / config["workspace"])
 
 
 @pytest.fixture(scope="module")
@@ -31,22 +36,36 @@ def config(request: Any) -> None:
     with open(CONFIG_PATH, "r") as f:
         config = json.load(f)
 
-    if config.get("workspace", "").startswith("${") and config.get(
-        "workspace", ""
-    ).endswith("}"):
-        path = get_dynamic_workspace(config)
-        config["workspace"] = path
-    else:
-        config["workspace"] = Path(os.getcwd()) / config["workspace"]
+    if request.config.getoption("--mock"):
+        config["workspace"] = "agbenchmark/mocks/workspace"
+    elif isinstance(config["workspace"], str):
+        config["workspace"] = resolve_workspace(config)
+    else:  # it's a input output dict
+        config["workspace"]["input"] = resolve_workspace(config)
+        config["workspace"]["output"] = resolve_workspace(config)
+
     return config
 
 
 @pytest.fixture(scope="module", autouse=True)
 def workspace(config: Dict[str, Any]) -> Generator[str, None, None]:
+    output_path = config["workspace"]
+
+    # checks if its an input output paradigm
+    if not isinstance(config["workspace"], str):
+        output_path = config["workspace"]["output"]
+        if not os.path.exists(config["workspace"]["input"]):
+            os.makedirs(config["workspace"]["input"], exist_ok=True)
+
+    # create output directory if it doesn't exist
+    if not os.path.exists(output_path):
+        os.makedirs(output_path, exist_ok=True)
+
     yield config["workspace"]
     # teardown after test function completes
-    for filename in os.listdir(config["workspace"]):
-        file_path = os.path.join(config["workspace"], filename)
+
+    for filename in os.listdir(output_path):
+        file_path = os.path.join(output_path, filename)
         try:
             if os.path.isfile(file_path) or os.path.islink(file_path):
                 os.unlink(file_path)

config.json

@@ -1,6 +1,6 @@
 {
-  "workspace": "${os.path.join(Path.home(), 'miniagi')}",
-  "entry_path": "benchmarks.py",
-  "home_path": "agent/mini-agi/",
+  "workspace": "projects/my-new-project/workspace",
+  "entry_path": "agent/gpt-engineer/benchmarks.py",
+  "home_path": "agent/gpt-engineer",
   "cutoff": 60
 }