diff --git a/autogpt/log_cycle/log_cycle.py b/autogpt/log_cycle/log_cycle.py index 8daed25c..71aecda8 100644 --- a/autogpt/log_cycle/log_cycle.py +++ b/autogpt/log_cycle/log_cycle.py @@ -34,7 +34,7 @@ class LogCycleHandler: if os.environ.get("OVERWRITE_DEBUG") == "1": outer_folder_name = "auto_gpt" else: - ai_name_short = ai_name[:15] if ai_name else DEFAULT_PREFIX + ai_name_short = self.get_agent_short_name(ai_name) outer_folder_name = f"{created_at}_{ai_name_short}" outer_folder_path = os.path.join(log_directory, "DEBUG", outer_folder_name) @@ -42,6 +42,9 @@ class LogCycleHandler: return outer_folder_path + def get_agent_short_name(self, ai_name): + return ai_name[:15] if ai_name else DEFAULT_PREFIX + def create_inner_directory(self, outer_folder_path: str, cycle_count: int) -> str: nested_folder_name = str(cycle_count).zfill(3) nested_folder_path = os.path.join(outer_folder_path, nested_folder_name) diff --git a/tests/challenges/basic_abilities/test_browse_website.py b/tests/challenges/basic_abilities/test_browse_website.py index 6ee80d33..1c4eb27e 100644 --- a/tests/challenges/basic_abilities/test_browse_website.py +++ b/tests/challenges/basic_abilities/test_browse_website.py @@ -13,9 +13,12 @@ def test_browse_website( patched_api_requestor: None, monkeypatch: pytest.MonkeyPatch, level_to_run: int, + challenge_name: str, ) -> None: file_path = browser_agent.workspace.get_path("browse_website.txt") - run_interaction_loop(monkeypatch, browser_agent, CYCLE_COUNT) + run_interaction_loop( + monkeypatch, browser_agent, CYCLE_COUNT, challenge_name, level_to_run + ) # content = read_file(file_path, config) content = open(file_path, encoding="utf-8").read() diff --git a/tests/challenges/basic_abilities/test_write_file.py b/tests/challenges/basic_abilities/test_write_file.py index 9aa837e9..39a45ec6 100644 --- a/tests/challenges/basic_abilities/test_write_file.py +++ b/tests/challenges/basic_abilities/test_write_file.py @@ -20,10 +20,15 @@ def test_write_file( patched_api_requestor: None, monkeypatch: pytest.MonkeyPatch, level_to_run: int, + challenge_name: str, ) -> None: file_system_agent = file_system_agents[level_to_run - 1] run_interaction_loop( - monkeypatch, file_system_agent, CYCLE_COUNT_PER_LEVEL[level_to_run - 1] + monkeypatch, + file_system_agent, + CYCLE_COUNT_PER_LEVEL[level_to_run - 1], + challenge_name, + level_to_run, ) expected_outputs = EXPECTED_OUTPUTS_PER_LEVEL[level_to_run - 1] diff --git a/tests/challenges/challenge_decorator/challenge.py b/tests/challenges/challenge_decorator/challenge.py index fd3b60cb..e875ac99 100644 --- a/tests/challenges/challenge_decorator/challenge.py +++ b/tests/challenges/challenge_decorator/challenge.py @@ -3,6 +3,7 @@ from typing import Optional class Challenge: BEAT_CHALLENGES = False + DEFAULT_CHALLENGE_NAME = "default_challenge_name" def __init__( self, @@ -10,7 +11,7 @@ class Challenge: category: str, max_level: int, is_new_challenge: bool, - max_level_beaten: Optional[int], + max_level_beaten: Optional[int] = None, level_to_run: Optional[int] = None, ) -> None: self.name = name diff --git a/tests/challenges/challenge_decorator/challenge_decorator.py b/tests/challenges/challenge_decorator/challenge_decorator.py index e1fd80c4..52d796c0 100644 --- a/tests/challenges/challenge_decorator/challenge_decorator.py +++ b/tests/challenges/challenge_decorator/challenge_decorator.py @@ -42,6 +42,7 @@ def challenge( ) if challenge.level_to_run is not None: kwargs["level_to_run"] = challenge.level_to_run + kwargs["challenge_name"] = challenge.name try: func(*args, **kwargs) challenge.succeeded = True diff --git a/tests/challenges/conftest.py b/tests/challenges/conftest.py index dff45f11..c0604c31 100644 --- a/tests/challenges/conftest.py +++ b/tests/challenges/conftest.py @@ -51,6 +51,11 @@ def level_to_run(request: FixtureRequest) -> int: return request.config.option.level +@pytest.fixture +def challenge_name() -> str: + return Challenge.DEFAULT_CHALLENGE_NAME + + @pytest.fixture(autouse=True) def check_beat_challenges(request: FixtureRequest) -> None: Challenge.BEAT_CHALLENGES = request.config.getoption("--beat-challenges") diff --git a/tests/challenges/debug_code/test_debug_code_challenge_a.py b/tests/challenges/debug_code/test_debug_code_challenge_a.py index 43f524c7..305c9693 100644 --- a/tests/challenges/debug_code/test_debug_code_challenge_a.py +++ b/tests/challenges/debug_code/test_debug_code_challenge_a.py @@ -25,6 +25,7 @@ def test_debug_code_challenge_a( monkeypatch: pytest.MonkeyPatch, patched_api_requestor: MockerFixture, level_to_run: int, + challenge_name: str, ) -> None: """ Test whether the agent can debug a simple code snippet. @@ -39,7 +40,9 @@ def test_debug_code_challenge_a( copy_file_into_workspace(debug_code_agent, DIRECTORY_PATH, CODE_FILE_PATH) copy_file_into_workspace(debug_code_agent, DIRECTORY_PATH, TEST_FILE_PATH) - run_interaction_loop(monkeypatch, debug_code_agent, CYCLE_COUNT) + run_interaction_loop( + monkeypatch, debug_code_agent, CYCLE_COUNT, challenge_name, level_to_run + ) output = execute_python_file( get_workspace_path(debug_code_agent, TEST_FILE_PATH), debug_code_agent diff --git a/tests/challenges/information_retrieval/test_information_retrieval_challenge_a.py b/tests/challenges/information_retrieval/test_information_retrieval_challenge_a.py index 8e0fdb63..56d5d4ec 100644 --- a/tests/challenges/information_retrieval/test_information_retrieval_challenge_a.py +++ b/tests/challenges/information_retrieval/test_information_retrieval_challenge_a.py @@ -18,6 +18,7 @@ def test_information_retrieval_challenge_a( monkeypatch: pytest.MonkeyPatch, patched_api_requestor: MockerFixture, level_to_run: int, + challenge_name: str, ) -> None: """ Test the challenge_a function in a given agent by mocking user inputs and checking the output file content. @@ -26,7 +27,13 @@ def test_information_retrieval_challenge_a( :param monkeypatch: pytest's monkeypatch utility for modifying builtins. """ information_retrieval_agent = information_retrieval_agents[level_to_run - 1] - run_interaction_loop(monkeypatch, information_retrieval_agent, CYCLE_COUNT) + run_interaction_loop( + monkeypatch, + information_retrieval_agent, + CYCLE_COUNT, + challenge_name, + level_to_run, + ) file_path = get_workspace_path(information_retrieval_agent, OUTPUT_LOCATION) content = read_file(file_path, information_retrieval_agent) diff --git a/tests/challenges/information_retrieval/test_information_retrieval_challenge_b.py b/tests/challenges/information_retrieval/test_information_retrieval_challenge_b.py index 0672b879..580b8b0b 100644 --- a/tests/challenges/information_retrieval/test_information_retrieval_challenge_b.py +++ b/tests/challenges/information_retrieval/test_information_retrieval_challenge_b.py @@ -18,6 +18,7 @@ def test_information_retrieval_challenge_b( monkeypatch: pytest.MonkeyPatch, patched_api_requestor: MockerFixture, level_to_run: int, + challenge_name: str, ) -> None: """ Test the challenge_b function in a given agent by mocking user inputs and checking the output file content. @@ -29,7 +30,13 @@ def test_information_retrieval_challenge_b( """ with contextlib.suppress(SystemExit): - run_interaction_loop(monkeypatch, get_nobel_prize_agent, CYCLE_COUNT) + run_interaction_loop( + monkeypatch, + get_nobel_prize_agent, + CYCLE_COUNT, + challenge_name, + level_to_run, + ) file_path = get_workspace_path(get_nobel_prize_agent, OUTPUT_LOCATION) content = read_file(file_path, get_nobel_prize_agent) diff --git a/tests/challenges/kubernetes/test_kubernetes_template_challenge_a.py b/tests/challenges/kubernetes/test_kubernetes_template_challenge_a.py index c2eaab75..98f239cb 100644 --- a/tests/challenges/kubernetes/test_kubernetes_template_challenge_a.py +++ b/tests/challenges/kubernetes/test_kubernetes_template_challenge_a.py @@ -17,6 +17,7 @@ def test_kubernetes_template_challenge_a( monkeypatch: pytest.MonkeyPatch, patched_api_requestor: MockerFixture, level_to_run: int, + challenge_name: str, ) -> None: """ Test the challenge_a function in a given agent by mocking user inputs @@ -27,7 +28,9 @@ def test_kubernetes_template_challenge_a( monkeypatch (pytest.MonkeyPatch) level_to_run (int) """ - run_interaction_loop(monkeypatch, kubernetes_agent, CYCLE_COUNT) + run_interaction_loop( + monkeypatch, kubernetes_agent, CYCLE_COUNT, challenge_name, level_to_run + ) file_path = get_workspace_path(kubernetes_agent, OUTPUT_LOCATION) content = read_file(file_path, kubernetes_agent) diff --git a/tests/challenges/memory/test_memory_challenge_a.py b/tests/challenges/memory/test_memory_challenge_a.py index 7050696a..41453b25 100644 --- a/tests/challenges/memory/test_memory_challenge_a.py +++ b/tests/challenges/memory/test_memory_challenge_a.py @@ -15,6 +15,7 @@ def test_memory_challenge_a( patched_api_requestor: MockerFixture, monkeypatch: pytest.MonkeyPatch, level_to_run: int, + challenge_name: str, ) -> None: """ The agent reads a file containing a task_id. Then, it reads a series of other files. @@ -28,7 +29,13 @@ def test_memory_challenge_a( task_id = "2314" create_instructions_files(memory_management_agent, level_to_run, task_id) - run_interaction_loop(monkeypatch, memory_management_agent, level_to_run + 2) + run_interaction_loop( + monkeypatch, + memory_management_agent, + level_to_run + 2, + challenge_name, + level_to_run, + ) file_path = get_workspace_path(memory_management_agent, OUTPUT_LOCATION) content = read_file(file_path, memory_management_agent) diff --git a/tests/challenges/memory/test_memory_challenge_b.py b/tests/challenges/memory/test_memory_challenge_b.py index 4abc4188..b381df1b 100644 --- a/tests/challenges/memory/test_memory_challenge_b.py +++ b/tests/challenges/memory/test_memory_challenge_b.py @@ -20,6 +20,7 @@ def test_memory_challenge_b( patched_api_requestor: MockerFixture, monkeypatch: pytest.MonkeyPatch, level_to_run: int, + challenge_name: str, ) -> None: """ The agent reads a series of files, each containing a task_id and noise. After reading 'n' files, @@ -34,7 +35,13 @@ def test_memory_challenge_b( task_ids = [str(i * 1111) for i in range(1, level_to_run + 1)] create_instructions_files(memory_management_agent, level_to_run, task_ids) - run_interaction_loop(monkeypatch, memory_management_agent, level_to_run + 2) + run_interaction_loop( + monkeypatch, + memory_management_agent, + level_to_run + 2, + challenge_name, + level_to_run, + ) file_path = get_workspace_path(memory_management_agent, OUTPUT_LOCATION) content = read_file(file_path, memory_management_agent) diff --git a/tests/challenges/memory/test_memory_challenge_c.py b/tests/challenges/memory/test_memory_challenge_c.py index 2754d6b0..3cfeb2c0 100644 --- a/tests/challenges/memory/test_memory_challenge_c.py +++ b/tests/challenges/memory/test_memory_challenge_c.py @@ -20,6 +20,7 @@ def test_memory_challenge_c( patched_api_requestor: MockerFixture, monkeypatch: pytest.MonkeyPatch, level_to_run: int, + challenge_name: str, ) -> None: """ Instead of reading task Ids from files as with the previous challenges, the agent now must remember @@ -52,7 +53,13 @@ def test_memory_challenge_c( level_silly_phrases, ) - run_interaction_loop(monkeypatch, memory_management_agent, level_to_run + 2) + run_interaction_loop( + monkeypatch, + memory_management_agent, + level_to_run + 2, + challenge_name, + level_to_run, + ) file_path = get_workspace_path(memory_management_agent, OUTPUT_LOCATION) content = read_file(file_path, agent=memory_management_agent) for phrase in level_silly_phrases: diff --git a/tests/challenges/memory/test_memory_challenge_d.py b/tests/challenges/memory/test_memory_challenge_d.py index ff35e183..ae44831b 100644 --- a/tests/challenges/memory/test_memory_challenge_d.py +++ b/tests/challenges/memory/test_memory_challenge_d.py @@ -20,6 +20,7 @@ def test_memory_challenge_d( patched_api_requestor: MockerFixture, monkeypatch: pytest.MonkeyPatch, level_to_run: int, + challenge_name: str, ) -> None: """ The agent is given a series of events and must remember the respective beliefs of the characters. @@ -38,7 +39,13 @@ def test_memory_challenge_d( create_instructions_files( memory_management_agent, level_to_run, level_sally_anne_test_phrases ) - run_interaction_loop(monkeypatch, memory_management_agent, level_to_run + 2) + run_interaction_loop( + monkeypatch, + memory_management_agent, + level_to_run + 2, + challenge_name, + level_to_run, + ) file_path = get_workspace_path(memory_management_agent, OUTPUT_LOCATION) content = read_file(file_path, memory_management_agent) diff --git a/tests/challenges/utils.py b/tests/challenges/utils.py index 9a357d96..0b5f54c5 100644 --- a/tests/challenges/utils.py +++ b/tests/challenges/utils.py @@ -2,11 +2,12 @@ import contextlib import random import shutil from pathlib import Path -from typing import Generator +from typing import Any, Generator import pytest from autogpt.agent import Agent +from autogpt.log_cycle.log_cycle import LogCycleHandler def generate_noise(noise_size: int) -> str: @@ -39,13 +40,30 @@ def setup_mock_input(monkeypatch: pytest.MonkeyPatch, cycle_count: int) -> None: def run_interaction_loop( - monkeypatch: pytest.MonkeyPatch, agent: Agent, cycle_count: int + monkeypatch: pytest.MonkeyPatch, + agent: Agent, + cycle_count: int, + challenge_name: str, + level_to_run: int, ) -> None: setup_mock_input(monkeypatch, cycle_count) + + setup_mock_log_cycle_agent_name(monkeypatch, challenge_name, level_to_run) with contextlib.suppress(SystemExit): agent.start_interaction_loop() +def setup_mock_log_cycle_agent_name( + monkeypatch: pytest.MonkeyPatch, challenge_name: str, level_to_run: int +) -> None: + def mock_get_agent_short_name(*args: Any, **kwargs: Any) -> str: + return f"{challenge_name}_level_{level_to_run}" + + monkeypatch.setattr( + LogCycleHandler, "get_agent_short_name", mock_get_agent_short_name + ) + + def get_workspace_path(agent: Agent, file_name: str) -> str: return str(agent.workspace.get_path(file_name)) diff --git a/tests/integration/test_web_selenium.py b/tests/integration/test_web_selenium.py index 6f594a78..baf3653c 100644 --- a/tests/integration/test_web_selenium.py +++ b/tests/integration/test_web_selenium.py @@ -1,9 +1,13 @@ +import pytest from pytest_mock import MockerFixture from autogpt.agent.agent import Agent from autogpt.commands.web_selenium import browse_website +from tests.utils import requires_api_key +@pytest.mark.vcr +@requires_api_key("OPENAI_API_KEY") def test_browse_website(agent: Agent, patched_api_requestor: MockerFixture): url = "https://barrel-roll.com" question = "How to execute a barrel roll"