Add challenge name and level to pytest logs (#4661)

autogpt/log_cycle/log_cycle.py

@@ -34,7 +34,7 @@ class LogCycleHandler:
         if os.environ.get("OVERWRITE_DEBUG") == "1":
             outer_folder_name = "auto_gpt"
         else:
-            ai_name_short = ai_name[:15] if ai_name else DEFAULT_PREFIX
+            ai_name_short = self.get_agent_short_name(ai_name)
             outer_folder_name = f"{created_at}_{ai_name_short}"
 
         outer_folder_path = os.path.join(log_directory, "DEBUG", outer_folder_name)
@@ -42,6 +42,9 @@ class LogCycleHandler:
 
         return outer_folder_path
 
+    def get_agent_short_name(self, ai_name):
+        return ai_name[:15] if ai_name else DEFAULT_PREFIX
+
     def create_inner_directory(self, outer_folder_path: str, cycle_count: int) -> str:
         nested_folder_name = str(cycle_count).zfill(3)
         nested_folder_path = os.path.join(outer_folder_path, nested_folder_name)

tests/challenges/basic_abilities/test_browse_website.py

@@ -13,9 +13,12 @@ def test_browse_website(
     patched_api_requestor: None,
     monkeypatch: pytest.MonkeyPatch,
     level_to_run: int,
+    challenge_name: str,
 ) -> None:
     file_path = browser_agent.workspace.get_path("browse_website.txt")
-    run_interaction_loop(monkeypatch, browser_agent, CYCLE_COUNT)
+    run_interaction_loop(
+        monkeypatch, browser_agent, CYCLE_COUNT, challenge_name, level_to_run
+    )
 
     # content = read_file(file_path, config)
     content = open(file_path, encoding="utf-8").read()

tests/challenges/basic_abilities/test_write_file.py

@@ -20,10 +20,15 @@ def test_write_file(
     patched_api_requestor: None,
     monkeypatch: pytest.MonkeyPatch,
     level_to_run: int,
+    challenge_name: str,
 ) -> None:
     file_system_agent = file_system_agents[level_to_run - 1]
     run_interaction_loop(
-        monkeypatch, file_system_agent, CYCLE_COUNT_PER_LEVEL[level_to_run - 1]
+        monkeypatch,
+        file_system_agent,
+        CYCLE_COUNT_PER_LEVEL[level_to_run - 1],
+        challenge_name,
+        level_to_run,
     )
 
     expected_outputs = EXPECTED_OUTPUTS_PER_LEVEL[level_to_run - 1]

tests/challenges/challenge_decorator/challenge.py

@@ -3,6 +3,7 @@ from typing import Optional
 
 class Challenge:
     BEAT_CHALLENGES = False
+    DEFAULT_CHALLENGE_NAME = "default_challenge_name"
 
     def __init__(
         self,
@@ -10,7 +11,7 @@ class Challenge:
         category: str,
         max_level: int,
         is_new_challenge: bool,
-        max_level_beaten: Optional[int],
+        max_level_beaten: Optional[int] = None,
         level_to_run: Optional[int] = None,
     ) -> None:
         self.name = name

tests/challenges/challenge_decorator/challenge_decorator.py

@@ -42,6 +42,7 @@ def challenge(
                 )
                 if challenge.level_to_run is not None:
                     kwargs["level_to_run"] = challenge.level_to_run
+                    kwargs["challenge_name"] = challenge.name
                     try:
                         func(*args, **kwargs)
                         challenge.succeeded = True

tests/challenges/conftest.py

@@ -51,6 +51,11 @@ def level_to_run(request: FixtureRequest) -> int:
     return request.config.option.level
 
 
+@pytest.fixture
+def challenge_name() -> str:
+    return Challenge.DEFAULT_CHALLENGE_NAME
+
+
 @pytest.fixture(autouse=True)
 def check_beat_challenges(request: FixtureRequest) -> None:
     Challenge.BEAT_CHALLENGES = request.config.getoption("--beat-challenges")

tests/challenges/debug_code/test_debug_code_challenge_a.py

@@ -25,6 +25,7 @@ def test_debug_code_challenge_a(
     monkeypatch: pytest.MonkeyPatch,
     patched_api_requestor: MockerFixture,
     level_to_run: int,
+    challenge_name: str,
 ) -> None:
     """
     Test whether the agent can debug a simple code snippet.
@@ -39,7 +40,9 @@ def test_debug_code_challenge_a(
     copy_file_into_workspace(debug_code_agent, DIRECTORY_PATH, CODE_FILE_PATH)
     copy_file_into_workspace(debug_code_agent, DIRECTORY_PATH, TEST_FILE_PATH)
 
-    run_interaction_loop(monkeypatch, debug_code_agent, CYCLE_COUNT)
+    run_interaction_loop(
+        monkeypatch, debug_code_agent, CYCLE_COUNT, challenge_name, level_to_run
+    )
 
     output = execute_python_file(
         get_workspace_path(debug_code_agent, TEST_FILE_PATH), debug_code_agent

tests/challenges/information_retrieval/test_information_retrieval_challenge_a.py

@@ -18,6 +18,7 @@ def test_information_retrieval_challenge_a(
     monkeypatch: pytest.MonkeyPatch,
     patched_api_requestor: MockerFixture,
     level_to_run: int,
+    challenge_name: str,
 ) -> None:
     """
     Test the challenge_a function in a given agent by mocking user inputs and checking the output file content.
@@ -26,7 +27,13 @@ def test_information_retrieval_challenge_a(
     :param monkeypatch: pytest's monkeypatch utility for modifying builtins.
     """
     information_retrieval_agent = information_retrieval_agents[level_to_run - 1]
-    run_interaction_loop(monkeypatch, information_retrieval_agent, CYCLE_COUNT)
+    run_interaction_loop(
+        monkeypatch,
+        information_retrieval_agent,
+        CYCLE_COUNT,
+        challenge_name,
+        level_to_run,
+    )
 
     file_path = get_workspace_path(information_retrieval_agent, OUTPUT_LOCATION)
     content = read_file(file_path, information_retrieval_agent)

tests/challenges/information_retrieval/test_information_retrieval_challenge_b.py

@@ -18,6 +18,7 @@ def test_information_retrieval_challenge_b(
     monkeypatch: pytest.MonkeyPatch,
     patched_api_requestor: MockerFixture,
     level_to_run: int,
+    challenge_name: str,
 ) -> None:
     """
     Test the challenge_b function in a given agent by mocking user inputs and checking the output file content.
@@ -29,7 +30,13 @@ def test_information_retrieval_challenge_b(
     """
 
     with contextlib.suppress(SystemExit):
-        run_interaction_loop(monkeypatch, get_nobel_prize_agent, CYCLE_COUNT)
+        run_interaction_loop(
+            monkeypatch,
+            get_nobel_prize_agent,
+            CYCLE_COUNT,
+            challenge_name,
+            level_to_run,
+        )
     file_path = get_workspace_path(get_nobel_prize_agent, OUTPUT_LOCATION)
 
     content = read_file(file_path, get_nobel_prize_agent)

tests/challenges/kubernetes/test_kubernetes_template_challenge_a.py

@@ -17,6 +17,7 @@ def test_kubernetes_template_challenge_a(
     monkeypatch: pytest.MonkeyPatch,
     patched_api_requestor: MockerFixture,
     level_to_run: int,
+    challenge_name: str,
 ) -> None:
     """
     Test the challenge_a function in a given agent by mocking user inputs
@@ -27,7 +28,9 @@ def test_kubernetes_template_challenge_a(
         monkeypatch (pytest.MonkeyPatch)
         level_to_run (int)
     """
-    run_interaction_loop(monkeypatch, kubernetes_agent, CYCLE_COUNT)
+    run_interaction_loop(
+        monkeypatch, kubernetes_agent, CYCLE_COUNT, challenge_name, level_to_run
+    )
 
     file_path = get_workspace_path(kubernetes_agent, OUTPUT_LOCATION)
     content = read_file(file_path, kubernetes_agent)

tests/challenges/memory/test_memory_challenge_a.py

@@ -15,6 +15,7 @@ def test_memory_challenge_a(
     patched_api_requestor: MockerFixture,
     monkeypatch: pytest.MonkeyPatch,
     level_to_run: int,
+    challenge_name: str,
 ) -> None:
     """
     The agent reads a file containing a task_id. Then, it reads a series of other files.
@@ -28,7 +29,13 @@ def test_memory_challenge_a(
     task_id = "2314"
     create_instructions_files(memory_management_agent, level_to_run, task_id)
 
-    run_interaction_loop(monkeypatch, memory_management_agent, level_to_run + 2)
+    run_interaction_loop(
+        monkeypatch,
+        memory_management_agent,
+        level_to_run + 2,
+        challenge_name,
+        level_to_run,
+    )
 
     file_path = get_workspace_path(memory_management_agent, OUTPUT_LOCATION)
     content = read_file(file_path, memory_management_agent)

tests/challenges/memory/test_memory_challenge_b.py

@@ -20,6 +20,7 @@ def test_memory_challenge_b(
     patched_api_requestor: MockerFixture,
     monkeypatch: pytest.MonkeyPatch,
     level_to_run: int,
+    challenge_name: str,
 ) -> None:
     """
     The agent reads a series of files, each containing a task_id and noise. After reading 'n' files,
@@ -34,7 +35,13 @@ def test_memory_challenge_b(
     task_ids = [str(i * 1111) for i in range(1, level_to_run + 1)]
     create_instructions_files(memory_management_agent, level_to_run, task_ids)
 
-    run_interaction_loop(monkeypatch, memory_management_agent, level_to_run + 2)
+    run_interaction_loop(
+        monkeypatch,
+        memory_management_agent,
+        level_to_run + 2,
+        challenge_name,
+        level_to_run,
+    )
 
     file_path = get_workspace_path(memory_management_agent, OUTPUT_LOCATION)
     content = read_file(file_path, memory_management_agent)

tests/challenges/memory/test_memory_challenge_c.py

@@ -20,6 +20,7 @@ def test_memory_challenge_c(
     patched_api_requestor: MockerFixture,
     monkeypatch: pytest.MonkeyPatch,
     level_to_run: int,
+    challenge_name: str,
 ) -> None:
     """
     Instead of reading task Ids from files as with the previous challenges, the agent now must remember
@@ -52,7 +53,13 @@ def test_memory_challenge_c(
         level_silly_phrases,
     )
 
-    run_interaction_loop(monkeypatch, memory_management_agent, level_to_run + 2)
+    run_interaction_loop(
+        monkeypatch,
+        memory_management_agent,
+        level_to_run + 2,
+        challenge_name,
+        level_to_run,
+    )
     file_path = get_workspace_path(memory_management_agent, OUTPUT_LOCATION)
     content = read_file(file_path, agent=memory_management_agent)
     for phrase in level_silly_phrases:

tests/challenges/memory/test_memory_challenge_d.py

@@ -20,6 +20,7 @@ def test_memory_challenge_d(
     patched_api_requestor: MockerFixture,
     monkeypatch: pytest.MonkeyPatch,
     level_to_run: int,
+    challenge_name: str,
 ) -> None:
     """
     The agent is given a series of events and must remember the respective beliefs of the characters.
@@ -38,7 +39,13 @@ def test_memory_challenge_d(
     create_instructions_files(
         memory_management_agent, level_to_run, level_sally_anne_test_phrases
     )
-    run_interaction_loop(monkeypatch, memory_management_agent, level_to_run + 2)
+    run_interaction_loop(
+        monkeypatch,
+        memory_management_agent,
+        level_to_run + 2,
+        challenge_name,
+        level_to_run,
+    )
     file_path = get_workspace_path(memory_management_agent, OUTPUT_LOCATION)
 
     content = read_file(file_path, memory_management_agent)

tests/challenges/utils.py

@@ -2,11 +2,12 @@ import contextlib
 import random
 import shutil
 from pathlib import Path
-from typing import Generator
+from typing import Any, Generator
 
 import pytest
 
 from autogpt.agent import Agent
+from autogpt.log_cycle.log_cycle import LogCycleHandler
 
 
 def generate_noise(noise_size: int) -> str:
@@ -39,13 +40,30 @@ def setup_mock_input(monkeypatch: pytest.MonkeyPatch, cycle_count: int) -> None:
 
 
 def run_interaction_loop(
-    monkeypatch: pytest.MonkeyPatch, agent: Agent, cycle_count: int
+    monkeypatch: pytest.MonkeyPatch,
+    agent: Agent,
+    cycle_count: int,
+    challenge_name: str,
+    level_to_run: int,
 ) -> None:
     setup_mock_input(monkeypatch, cycle_count)
+
+    setup_mock_log_cycle_agent_name(monkeypatch, challenge_name, level_to_run)
     with contextlib.suppress(SystemExit):
         agent.start_interaction_loop()
 
 
+def setup_mock_log_cycle_agent_name(
+    monkeypatch: pytest.MonkeyPatch, challenge_name: str, level_to_run: int
+) -> None:
+    def mock_get_agent_short_name(*args: Any, **kwargs: Any) -> str:
+        return f"{challenge_name}_level_{level_to_run}"
+
+    monkeypatch.setattr(
+        LogCycleHandler, "get_agent_short_name", mock_get_agent_short_name
+    )
+
+
 def get_workspace_path(agent: Agent, file_name: str) -> str:
     return str(agent.workspace.get_path(file_name))
 

tests/integration/test_web_selenium.py

@@ -1,9 +1,13 @@
+import pytest
 from pytest_mock import MockerFixture
 
 from autogpt.agent.agent import Agent
 from autogpt.commands.web_selenium import browse_website
+from tests.utils import requires_api_key
 
 
+@pytest.mark.vcr
+@requires_api_key("OPENAI_API_KEY")
 def test_browse_website(agent: Agent, patched_api_requestor: MockerFixture):
     url = "https://barrel-roll.com"
     question = "How to execute a barrel roll"