feat(benchmark): Make report output folder configurable

- Make `AgentBenchmarkConfig.reports_folder` directly configurable (through `REPORTS_FOLDER` env variable). The default is still `./agbenchmark_config/reports`.
- Change all mentions of `REPORT_LOCATION` (which fulfilled the same function at some point in the past) to `REPORTS_FOLDER`.

.github/workflows/autogpts-ci.yml

@@ -68,4 +68,4 @@ jobs:
           REQUESTS_CA_BUNDLE: /etc/ssl/certs/ca-certificates.crt
           HELICONE_CACHE_ENABLED: false
           HELICONE_PROPERTY_AGENT: ${{ matrix.agent-name }}
-          REPORT_LOCATION: ${{ format('../../reports/{0}', matrix.agent-name) }}
+          REPORTS_FOLDER: ${{ format('../../reports/{0}', matrix.agent-name) }}

benchmark/.env.example

@@ -1,4 +1,4 @@
 AGENT_NAME=mini-agi
-REPORT_LOCATION="reports/mini-agi"
+REPORTS_FOLDER="reports/mini-agi"
 OPENAI_API_KEY="sk-" # for LLM eval
 BUILD_SKILL_TREE=false # set to true to build the skill tree.

benchmark/agbenchmark/README.md

@@ -30,7 +30,7 @@
 1. Navigate to `auto-gpt-benchmarks/agent/mini-agi`
 2. `pip install -r requirements.txt`
 3. `cp .env_example .env`, set `PROMPT_USER=false` and add your `OPENAI_API_KEY=`. Sset `MODEL="gpt-3.5-turbo"` if you don't have access to `gpt-4` yet. Also make sure you have Python 3.10^ installed
-4. set `AGENT_NAME=mini-agi` in `.env` file and where you want your `REPORT_LOCATION` to be
+4. set `AGENT_NAME=mini-agi` in `.env` file and where you want your `REPORTS_FOLDER` to be
 5. Make sure to follow the commands above, and remove mock flag `agbenchmark`
 
 - To add requirements `poetry add requirement`.

benchmark/agbenchmark/config.py

@@ -4,7 +4,7 @@ from datetime import datetime
 from pathlib import Path
 from typing import Optional
 
-from pydantic import BaseSettings, Field
+from pydantic import BaseSettings, Field, validator
 
 
 def _calculate_info_test_path(base_path: Path, benchmark_start_time: datetime) -> Path:
@@ -66,6 +66,12 @@ class AgentBenchmarkConfig(BaseSettings, extra="allow"):
     host: str
     """Host (scheme://address:port) of the subject agent application."""
 
+    reports_folder: Path = Field(None)
+    """
+    Path to the folder where new reports should be stored.
+    Defaults to {agbenchmark_config_dir}/reports.
+    """
+
     @classmethod
     def load(cls, config_dir: Optional[Path] = None) -> "AgentBenchmarkConfig":
         config_dir = config_dir or cls.find_config_folder()
@@ -95,9 +101,11 @@ class AgentBenchmarkConfig(BaseSettings, extra="allow"):
     def config_file(self) -> Path:
         return self.agbenchmark_config_dir / "config.json"
 
-    @property
+    @validator("reports_folder", pre=True, always=True)
-    def reports_folder(self) -> Path:
+    def set_reports_folder(cls, v, values):
-        return self.agbenchmark_config_dir / "reports"
+        if not v:
+            return values["agbenchmark_config_dir"] / "reports"
+        return v
 
     def get_report_dir(self, benchmark_start_time: datetime) -> Path:
         return _calculate_info_test_path(self.reports_folder, benchmark_start_time)

benchmark/agbenchmark/reports/ReportManager.py

@@ -153,7 +153,7 @@ class SessionReportManager(BaseReportManager):
                 total_cost=self.get_total_costs(),
             ),
             tests=copy.copy(self.tests),
-            config=config.dict(exclude_none=True),
+            config=config.dict(exclude={"reports_folder"}, exclude_none=True),
         )
 
         agent_categories = get_highest_achieved_difficulty_per_category(self.tests)

benchmark/agbenchmark/utils/utils.py

@@ -14,7 +14,6 @@ from agbenchmark.utils.data_types import DIFFICULTY_MAP, DifficultyLevel
 load_dotenv()
 
 AGENT_NAME = os.getenv("AGENT_NAME")
-REPORT_LOCATION = os.getenv("REPORT_LOCATION", None)
 
 logger = logging.getLogger(__name__)