aljaz/Auto-GPT
1
0
Fork 0
mirror of https://github.com/aljazceru/Auto-GPT.git synced 2025-12-22 16:34:25 +01:00
Code Issues Packages Projects Releases Wiki Activity

Add challenge name and level to pytest logs (#4661)

Browse Source
This commit is contained in:
merwanehamadi
2023-06-12 08:03:14 -07:00
committed by GitHub
parent a9d177eeeb
commit d5afbbee26
16 changed files with 102 additions and 14 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
autogpt/log_cycle/log_cycle.py
View File

@@ -34,7 +34,7 @@ class LogCycleHandler:
if os.environ.get("OVERWRITE_DEBUG") == "1":
outer_folder_name = "auto_gpt"
else:
ai_name_short = ai_name[:15] if ai_name else DEFAULT_PREFIX
ai_name_short = self.get_agent_short_name(ai_name)
outer_folder_name = f"{created_at}_{ai_name_short}"
outer_folder_path = os.path.join(log_directory, "DEBUG", outer_folder_name)
@@ -42,6 +42,9 @@ class LogCycleHandler:
return outer_folder_path
def get_agent_short_name(self, ai_name):
return ai_name[:15] if ai_name else DEFAULT_PREFIX
def create_inner_directory(self, outer_folder_path: str, cycle_count: int) -> str:
nested_folder_name = str(cycle_count).zfill(3)
nested_folder_path = os.path.join(outer_folder_path, nested_folder_name)

5
tests/challenges/basic_abilities/test_browse_website.py
View File

@@ -13,9 +13,12 @@ def test_browse_website(
patched_api_requestor: None,
monkeypatch: pytest.MonkeyPatch,
level_to_run: int,
challenge_name: str,
) -> None:
file_path = browser_agent.workspace.get_path("browse_website.txt")
run_interaction_loop(monkeypatch, browser_agent, CYCLE_COUNT)
run_interaction_loop(
monkeypatch, browser_agent, CYCLE_COUNT, challenge_name, level_to_run
)
# content = read_file(file_path, config)
content = open(file_path, encoding="utf-8").read()

7
tests/challenges/basic_abilities/test_write_file.py
View File

@@ -20,10 +20,15 @@ def test_write_file(
patched_api_requestor: None,
monkeypatch: pytest.MonkeyPatch,
level_to_run: int,
challenge_name: str,
) -> None:
file_system_agent = file_system_agents[level_to_run - 1]
run_interaction_loop(
monkeypatch, file_system_agent, CYCLE_COUNT_PER_LEVEL[level_to_run - 1]
monkeypatch,
file_system_agent,
CYCLE_COUNT_PER_LEVEL[level_to_run - 1],
challenge_name,
level_to_run,
)
expected_outputs = EXPECTED_OUTPUTS_PER_LEVEL[level_to_run - 1]

3
tests/challenges/challenge_decorator/challenge.py
View File

@@ -3,6 +3,7 @@ from typing import Optional
class Challenge:
BEAT_CHALLENGES = False
DEFAULT_CHALLENGE_NAME = "default_challenge_name"
def __init__(
self,
@@ -10,7 +11,7 @@ class Challenge:
category: str,
max_level: int,
is_new_challenge: bool,
max_level_beaten: Optional[int],
max_level_beaten: Optional[int] = None,
level_to_run: Optional[int] = None,
) -> None:
self.name = name

1
tests/challenges/challenge_decorator/challenge_decorator.py
View File

@@ -42,6 +42,7 @@ def challenge(
)
if challenge.level_to_run is not None:
kwargs["level_to_run"] = challenge.level_to_run
kwargs["challenge_name"] = challenge.name
try:
func(*args, **kwargs)
challenge.succeeded = True

5
tests/challenges/conftest.py
View File

@@ -51,6 +51,11 @@ def level_to_run(request: FixtureRequest) -> int:
return request.config.option.level
@pytest.fixture
def challenge_name() -> str:
return Challenge.DEFAULT_CHALLENGE_NAME
@pytest.fixture(autouse=True)
def check_beat_challenges(request: FixtureRequest) -> None:
Challenge.BEAT_CHALLENGES = request.config.getoption("--beat-challenges")

5
tests/challenges/debug_code/test_debug_code_challenge_a.py
View File

@@ -25,6 +25,7 @@ def test_debug_code_challenge_a(
monkeypatch: pytest.MonkeyPatch,
patched_api_requestor: MockerFixture,
level_to_run: int,
challenge_name: str,
) -> None:
"""
Test whether the agent can debug a simple code snippet.
@@ -39,7 +40,9 @@ def test_debug_code_challenge_a(
copy_file_into_workspace(debug_code_agent, DIRECTORY_PATH, CODE_FILE_PATH)
copy_file_into_workspace(debug_code_agent, DIRECTORY_PATH, TEST_FILE_PATH)
run_interaction_loop(monkeypatch, debug_code_agent, CYCLE_COUNT)
run_interaction_loop(
monkeypatch, debug_code_agent, CYCLE_COUNT, challenge_name, level_to_run
)
output = execute_python_file(
get_workspace_path(debug_code_agent, TEST_FILE_PATH), debug_code_agent

9
tests/challenges/information_retrieval/test_information_retrieval_challenge_a.py
View File

@@ -18,6 +18,7 @@ def test_information_retrieval_challenge_a(
monkeypatch: pytest.MonkeyPatch,
patched_api_requestor: MockerFixture,
level_to_run: int,
challenge_name: str,
) -> None:
"""
Test the challenge_a function in a given agent by mocking user inputs and checking the output file content.
@@ -26,7 +27,13 @@ def test_information_retrieval_challenge_a(
:param monkeypatch: pytest's monkeypatch utility for modifying builtins.
"""
information_retrieval_agent = information_retrieval_agents[level_to_run - 1]
run_interaction_loop(monkeypatch, information_retrieval_agent, CYCLE_COUNT)
run_interaction_loop(
monkeypatch,
information_retrieval_agent,
CYCLE_COUNT,
challenge_name,
level_to_run,
)
file_path = get_workspace_path(information_retrieval_agent, OUTPUT_LOCATION)
content = read_file(file_path, information_retrieval_agent)

9
tests/challenges/information_retrieval/test_information_retrieval_challenge_b.py
View File

@@ -18,6 +18,7 @@ def test_information_retrieval_challenge_b(
monkeypatch: pytest.MonkeyPatch,
patched_api_requestor: MockerFixture,
level_to_run: int,
challenge_name: str,
) -> None:
"""
Test the challenge_b function in a given agent by mocking user inputs and checking the output file content.
@@ -29,7 +30,13 @@ def test_information_retrieval_challenge_b(
"""
with contextlib.suppress(SystemExit):
run_interaction_loop(monkeypatch, get_nobel_prize_agent, CYCLE_COUNT)
run_interaction_loop(
monkeypatch,
get_nobel_prize_agent,
CYCLE_COUNT,
challenge_name,
level_to_run,
)
file_path = get_workspace_path(get_nobel_prize_agent, OUTPUT_LOCATION)
content = read_file(file_path, get_nobel_prize_agent)

5
tests/challenges/kubernetes/test_kubernetes_template_challenge_a.py
View File

@@ -17,6 +17,7 @@ def test_kubernetes_template_challenge_a(
monkeypatch: pytest.MonkeyPatch,
patched_api_requestor: MockerFixture,
level_to_run: int,
challenge_name: str,
) -> None:
"""
Test the challenge_a function in a given agent by mocking user inputs
@@ -27,7 +28,9 @@ def test_kubernetes_template_challenge_a(
monkeypatch (pytest.MonkeyPatch)
level_to_run (int)
"""
run_interaction_loop(monkeypatch, kubernetes_agent, CYCLE_COUNT)
run_interaction_loop(
monkeypatch, kubernetes_agent, CYCLE_COUNT, challenge_name, level_to_run
)
file_path = get_workspace_path(kubernetes_agent, OUTPUT_LOCATION)
content = read_file(file_path, kubernetes_agent)

9
tests/challenges/memory/test_memory_challenge_a.py
View File

@@ -15,6 +15,7 @@ def test_memory_challenge_a(
patched_api_requestor: MockerFixture,
monkeypatch: pytest.MonkeyPatch,
level_to_run: int,
challenge_name: str,
) -> None:
"""
The agent reads a file containing a task_id. Then, it reads a series of other files.
@@ -28,7 +29,13 @@ def test_memory_challenge_a(
task_id = "2314"
create_instructions_files(memory_management_agent, level_to_run, task_id)
run_interaction_loop(monkeypatch, memory_management_agent, level_to_run + 2)
run_interaction_loop(
monkeypatch,
memory_management_agent,
level_to_run + 2,
challenge_name,
level_to_run,
)
file_path = get_workspace_path(memory_management_agent, OUTPUT_LOCATION)
content = read_file(file_path, memory_management_agent)

9
tests/challenges/memory/test_memory_challenge_b.py
View File

@@ -20,6 +20,7 @@ def test_memory_challenge_b(
patched_api_requestor: MockerFixture,
monkeypatch: pytest.MonkeyPatch,
level_to_run: int,
challenge_name: str,
) -> None:
"""
The agent reads a series of files, each containing a task_id and noise. After reading 'n' files,
@@ -34,7 +35,13 @@ def test_memory_challenge_b(
task_ids = [str(i * 1111) for i in range(1, level_to_run + 1)]
create_instructions_files(memory_management_agent, level_to_run, task_ids)
run_interaction_loop(monkeypatch, memory_management_agent, level_to_run + 2)
run_interaction_loop(
monkeypatch,
memory_management_agent,
level_to_run + 2,
challenge_name,
level_to_run,
)
file_path = get_workspace_path(memory_management_agent, OUTPUT_LOCATION)
content = read_file(file_path, memory_management_agent)

9
tests/challenges/memory/test_memory_challenge_c.py
View File

@@ -20,6 +20,7 @@ def test_memory_challenge_c(
patched_api_requestor: MockerFixture,
monkeypatch: pytest.MonkeyPatch,
level_to_run: int,
challenge_name: str,
) -> None:
"""
Instead of reading task Ids from files as with the previous challenges, the agent now must remember
@@ -52,7 +53,13 @@ def test_memory_challenge_c(
level_silly_phrases,
)
run_interaction_loop(monkeypatch, memory_management_agent, level_to_run + 2)
run_interaction_loop(
monkeypatch,
memory_management_agent,
level_to_run + 2,
challenge_name,
level_to_run,
)
file_path = get_workspace_path(memory_management_agent, OUTPUT_LOCATION)
content = read_file(file_path, agent=memory_management_agent)
for phrase in level_silly_phrases:

9
tests/challenges/memory/test_memory_challenge_d.py
View File

@@ -20,6 +20,7 @@ def test_memory_challenge_d(
patched_api_requestor: MockerFixture,
monkeypatch: pytest.MonkeyPatch,
level_to_run: int,
challenge_name: str,
) -> None:
"""
The agent is given a series of events and must remember the respective beliefs of the characters.
@@ -38,7 +39,13 @@ def test_memory_challenge_d(
create_instructions_files(
memory_management_agent, level_to_run, level_sally_anne_test_phrases
)
run_interaction_loop(monkeypatch, memory_management_agent, level_to_run + 2)
run_interaction_loop(
monkeypatch,
memory_management_agent,
level_to_run + 2,
challenge_name,
level_to_run,
)
file_path = get_workspace_path(memory_management_agent, OUTPUT_LOCATION)
content = read_file(file_path, memory_management_agent)

22
tests/challenges/utils.py
View File

@@ -2,11 +2,12 @@ import contextlib
import random
import shutil
from pathlib import Path
from typing import Generator
from typing import Any, Generator
import pytest
from autogpt.agent import Agent
from autogpt.log_cycle.log_cycle import LogCycleHandler
def generate_noise(noise_size: int) -> str:
@@ -39,13 +40,30 @@ def setup_mock_input(monkeypatch: pytest.MonkeyPatch, cycle_count: int) -> None:
def run_interaction_loop(
monkeypatch: pytest.MonkeyPatch, agent: Agent, cycle_count: int
monkeypatch: pytest.MonkeyPatch,
agent: Agent,
cycle_count: int,
challenge_name: str,
level_to_run: int,
) -> None:
setup_mock_input(monkeypatch, cycle_count)
setup_mock_log_cycle_agent_name(monkeypatch, challenge_name, level_to_run)
with contextlib.suppress(SystemExit):
agent.start_interaction_loop()
def setup_mock_log_cycle_agent_name(
monkeypatch: pytest.MonkeyPatch, challenge_name: str, level_to_run: int
) -> None:
def mock_get_agent_short_name(*args: Any, **kwargs: Any) -> str:
return f"{challenge_name}_level_{level_to_run}"
monkeypatch.setattr(
LogCycleHandler, "get_agent_short_name", mock_get_agent_short_name
)
def get_workspace_path(agent: Agent, file_name: str) -> str:
return str(agent.workspace.get_path(file_name))

4
tests/integration/test_web_selenium.py
View File

@@ -1,9 +1,13 @@
import pytest
from pytest_mock import MockerFixture
from autogpt.agent.agent import Agent
from autogpt.commands.web_selenium import browse_website
from tests.utils import requires_api_key
@pytest.mark.vcr
@requires_api_key("OPENAI_API_KEY")
def test_browse_website(agent: Agent, patched_api_requestor: MockerFixture):
url = "https://barrel-roll.com"
question = "How to execute a barrel roll"