mirror of
https://github.com/aljazceru/Auto-GPT.git
synced 2025-12-17 05:54:26 +01:00
9012ff4db2
Squashed commit of the following: commit 7d6476d3297860f74c276d571da995d958a8cc1a Author: Reinier van der Leer <pwuts@agpt.co> Date: Tue Jan 9 18:10:45 2024 +0100 refactor(benchmark/challenge): Set up structure to support more challenge providers - Move `Challenge`, `ChallengeData`, `load_challenges` to `challenges/builtin.py` and rename to `BuiltinChallenge`, `BuiltinChallengeSpec`, `load_builtin_challenges` - Create `BaseChallenge` to serve as interface and base class for different challenge implementations - Create `ChallengeInfo` model to serve as universal challenge info object - Create `get_challenge_from_source_uri` function in `challenges/__init__.py` - Replace `ChallengeData` by `ChallengeInfo` everywhere except in `BuiltinChallenge` - Add strong typing to `task_informations` store in app.py - Use `call.duration` in `finalize_test_report` and remove `timer` fixture - Update docstring on `challenges/__init__.py:get_unique_categories` - Add docstring to `generate_test.py` commit 5df2aa7939b45d85a2c2b5de9ac0522330d1502a Author: Reinier van der Leer <pwuts@agpt.co> Date: Tue Jan 9 16:58:01 2024 +0100 refactor(benchmark): Refactor & rename functions in agent_interface.py and agent_api_interface.py - `copy_artifacts_into_temp_folder` -> `copy_challenge_artifacts_into_workspace` - `copy_agent_artifacts_into_folder` -> `download_agent_artifacts_into_folder` - Reorder parameters of `run_api_agent`, `copy_challenge_artifacts_into_workspace`; use `Path` instead of `str` commit 6a256fef4c7950b7ee82fb801e70c83afe6b6f8b Author: Reinier van der Leer <pwuts@agpt.co> Date: Tue Jan 9 16:02:25 2024 +0100 refactor(benchmark): Refactor & typefix report generation and handling logic - Rename functions in reports.py and ReportManager.py to better reflect what they do - `get_previous_test_results` -> `get_and_update_success_history` - `generate_single_call_report` -> `initialize_test_report` - `finalize_reports` -> `finalize_test_report` - `ReportManager.end_info_report` -> `SessionReportManager.finalize_session_report` - Modify `pytest_runtest_makereport` hook in conftest.py to finalize the report immediately after the challenge finishes running instead of after teardown - Move result processing logic from `initialize_test_report` to `finalize_test_report` in reports.py - Use `Test` and `Report` types from report_types.py where possible instead of untyped dicts: reports.py, utils.py, ReportManager.py - Differentiate `ReportManager` into `SessionReportManager`, `RegressionTestsTracker`, `SuccessRateTracker` - Move filtering of optional challenge categories from challenge.py (`Challenge.skip_optional_categories`) to conftest.py (`pytest_collection_modifyitems`) - Remove unused `scores` fixture in conftest.py commit 370d6dbf5df75d78e3878877968e8cd309d6d7fb Author: Reinier van der Leer <pwuts@agpt.co> Date: Tue Jan 9 15:16:43 2024 +0100 refactor(benchmark): Simplify models in report_types.py - Removed ForbidOptionalMeta and BaseModelBenchmark classes. - Changed model attributes to optional: `Metrics.difficulty`, `Metrics.success`, `Metrics.success_percentage`, `Metrics.run_time`, and `Test.reached_cutoff`. - Added validator to `Metrics` model to require `success` and `run_time` fields if `attempted=True`. - Added default values to all optional model fields. - Removed duplicate imports. - Added condition in process_report.py to prevent null lookups if `metrics.difficulty` is not set.
105 lines
3.2 KiB
Python
105 lines
3.2 KiB
Python
import logging
|
|
import os
|
|
import time
|
|
from pathlib import Path
|
|
from typing import AsyncIterator, Optional
|
|
|
|
from agent_protocol_client import (
|
|
AgentApi,
|
|
ApiClient,
|
|
Configuration,
|
|
Step,
|
|
TaskRequestBody,
|
|
)
|
|
|
|
from agbenchmark.agent_interface import get_list_of_file_paths
|
|
from agbenchmark.config import AgentBenchmarkConfig
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
|
|
async def run_api_agent(
|
|
task: str,
|
|
config: AgentBenchmarkConfig,
|
|
timeout: int,
|
|
artifacts_location: Optional[Path] = None,
|
|
) -> AsyncIterator[Step]:
|
|
configuration = Configuration(host=config.host)
|
|
async with ApiClient(configuration) as api_client:
|
|
api_instance = AgentApi(api_client)
|
|
task_request_body = TaskRequestBody(input=task)
|
|
|
|
start_time = time.time()
|
|
response = await api_instance.create_agent_task(
|
|
task_request_body=task_request_body
|
|
)
|
|
task_id = response.task_id
|
|
|
|
if artifacts_location:
|
|
await upload_artifacts(
|
|
api_instance, artifacts_location, task_id, "artifacts_in"
|
|
)
|
|
|
|
while True:
|
|
step = await api_instance.execute_agent_task_step(task_id=task_id)
|
|
yield step
|
|
|
|
if time.time() - start_time > timeout:
|
|
raise TimeoutError("Time limit exceeded")
|
|
if not step or step.is_last:
|
|
break
|
|
|
|
if artifacts_location:
|
|
# In "mock" mode, we cheat by giving the correct artifacts to pass the test
|
|
if os.getenv("IS_MOCK"):
|
|
await upload_artifacts(
|
|
api_instance, artifacts_location, task_id, "artifacts_out"
|
|
)
|
|
|
|
await download_agent_artifacts_into_folder(
|
|
api_instance, task_id, config.temp_folder
|
|
)
|
|
|
|
|
|
async def download_agent_artifacts_into_folder(
|
|
api_instance: AgentApi, task_id: str, folder: Path
|
|
):
|
|
artifacts = await api_instance.list_agent_task_artifacts(task_id=task_id)
|
|
|
|
for artifact in artifacts.artifacts:
|
|
# current absolute path of the directory of the file
|
|
if artifact.relative_path:
|
|
path: str = (
|
|
artifact.relative_path
|
|
if not artifact.relative_path.startswith("/")
|
|
else artifact.relative_path[1:]
|
|
)
|
|
folder = (folder / path).parent
|
|
|
|
if not folder.exists():
|
|
folder.mkdir(parents=True)
|
|
|
|
file_path = folder / artifact.file_name
|
|
logger.debug(f"Downloading agent artifact {artifact.file_name} to {folder}")
|
|
with open(file_path, "wb") as f:
|
|
content = await api_instance.download_agent_task_artifact(
|
|
task_id=task_id, artifact_id=artifact.artifact_id
|
|
)
|
|
|
|
f.write(content)
|
|
|
|
|
|
async def upload_artifacts(
|
|
api_instance: AgentApi, artifacts_location: Path, task_id: str, type: str
|
|
) -> None:
|
|
for file_path in get_list_of_file_paths(artifacts_location, type):
|
|
relative_path: Optional[str] = "/".join(
|
|
str(file_path).split(f"{type}/", 1)[-1].split("/")[:-1]
|
|
)
|
|
if not relative_path:
|
|
relative_path = None
|
|
|
|
await api_instance.upload_agent_task_artifacts(
|
|
task_id=task_id, file=str(file_path), relative_path=relative_path
|
|
)
|