Rearrange tests & fix CI (#4596)

* Rearrange tests into unit/integration/challenge categories

* Fix linting + `tests.challenges` imports

* Fix obscured duplicate test in test_url_validation.py

* Move VCR conftest to tests.vcr

* Specify tests to run & their order (unit -> integration -> challenges) in CI

* Fail Docker CI when tests fail

* Fix import & linting errors in tests

* Fix `get_text_summary`

* Fix linting errors

* Clean up pytest args in CI

* Remove bogus tests from GoCodeo

tests/challenges/memory/test_memory_challenge_a.py

@@ -0,0 +1,83 @@
+import pytest
+from pytest_mock import MockerFixture
+
+from autogpt.agent import Agent
+from autogpt.commands.file_operations import read_file, write_to_file
+from autogpt.config import Config
+from tests.challenges.challenge_decorator.challenge_decorator import challenge
+from tests.challenges.utils import run_interaction_loop
+from tests.utils import requires_api_key
+
+
+@pytest.mark.vcr
+@requires_api_key("OPENAI_API_KEY")
+@challenge
+def test_memory_challenge_a(
+    memory_management_agent: Agent,
+    patched_api_requestor: MockerFixture,
+    monkeypatch: pytest.MonkeyPatch,
+    config: Config,
+    level_to_run: int,
+) -> None:
+    """
+    The agent reads a file containing a task_id. Then, it reads a series of other files.
+    After reading 'n' files, the agent must write the task_id into a new file.
+    Args:
+        memory_management_agent (Agent)
+        patched_api_requestor (MockerFixture)
+        monkeypatch (pytest.MonkeyPatch)
+        config (Config)
+        level_to_run (int)
+    """
+
+    task_id = "2314"
+    create_instructions_files(memory_management_agent, level_to_run, task_id, config)
+
+    run_interaction_loop(monkeypatch, memory_management_agent, level_to_run + 2)
+
+    file_path = str(memory_management_agent.workspace.get_path("output.txt"))
+    content = read_file(file_path, config)
+    assert task_id in content, f"Expected the file to contain {task_id}"
+
+
+def create_instructions_files(
+    memory_management_agent: Agent,
+    num_files: int,
+    task_id: str,
+    config: Config,
+    base_filename: str = "instructions_",
+) -> None:
+    """
+    Creates a series of instructions files for the memory challenge.
+    Args:
+        memory_management_agent (Agent)
+        num_files (int)
+        task_id (str)
+        base_filename (str, optional)
+    """
+    for i in range(1, num_files + 1):
+        content = generate_content(i, task_id, base_filename, num_files)
+        file_name = f"{base_filename}{i}.txt"
+        file_path = str(memory_management_agent.workspace.get_path(file_name))
+        write_to_file(file_path, content, config)
+
+
+def generate_content(
+    index: int, task_id: str, base_filename: str, num_files: int
+) -> str:
+    """
+    Args:
+        index: int
+        task_id: str
+        base_filename: str
+        num_files: int
+
+    Returns: str
+    """
+    if index == 1:
+        return (
+            f"This task_id is {task_id}\nRead the file {base_filename}{index + 1}.txt"
+        )
+    if index != num_files:
+        return f"Read the file {base_filename}{index + 1}.txt"
+    return "Write the task_id into the file output.txt\nShutdown"

tests/challenges/memory/test_memory_challenge_b.py

@@ -0,0 +1,84 @@
+import pytest
+from pytest_mock import MockerFixture
+
+from autogpt.agent import Agent
+from autogpt.commands.file_operations import read_file, write_to_file
+from autogpt.config import Config
+from tests.challenges.challenge_decorator.challenge_decorator import challenge
+from tests.challenges.utils import generate_noise, run_interaction_loop
+from tests.utils import requires_api_key
+
+NOISE = 1000
+
+
+@pytest.mark.vcr
+@requires_api_key("OPENAI_API_KEY")
+@challenge
+def test_memory_challenge_b(
+    memory_management_agent: Agent,
+    patched_api_requestor: MockerFixture,
+    monkeypatch: pytest.MonkeyPatch,
+    config: Config,
+    level_to_run: int,
+) -> None:
+    """
+    The agent reads a series of files, each containing a task_id and noise. After reading 'n' files,
+    the agent must write all the task_ids into a new file, filtering out the noise.
+
+    Args:
+        memory_management_agent (Agent)
+        patched_api_requestor (MockerFixture)
+        monkeypatch (pytest.MonkeyPatch)
+        level_to_run (int)
+    """
+    task_ids = [str(i * 1111) for i in range(1, level_to_run + 1)]
+    create_instructions_files(memory_management_agent, level_to_run, task_ids, config)
+
+    run_interaction_loop(monkeypatch, memory_management_agent, level_to_run + 2)
+
+    file_path = str(memory_management_agent.workspace.get_path("output.txt"))
+    content = read_file(file_path, config)
+    for task_id in task_ids:
+        assert task_id in content, f"Expected the file to contain {task_id}"
+
+
+def create_instructions_files(
+    memory_management_agent: Agent,
+    level: int,
+    task_ids: list,
+    config: Config,
+    base_filename: str = "instructions_",
+) -> None:
+    """
+    Creates a series of instructions files for the memory challenge.
+    Args:
+        level:
+        memory_management_agent (Agent)
+        num_files (int)
+        task_ids (list)
+        base_filename (str, optional)
+    """
+    for i in range(1, level + 1):
+        content = generate_content(i, task_ids, base_filename, level)
+        file_name = f"{base_filename}{i}.txt"
+        file_path = str(memory_management_agent.workspace.get_path(file_name))
+        write_to_file(file_path, content, config)
+
+
+def generate_content(index: int, task_ids: list, base_filename: str, level: int) -> str:
+    """
+    Args:
+        index: int
+        task_ids: list
+        base_filename: str
+        num_files: int
+
+    Returns: str
+    """
+    task_id = task_ids[index - 1]
+    noise = generate_noise(NOISE)
+    if index != level:
+        if level == 1:
+            return f"{noise}\nThe current task_id is {task_id}.\n{noise}\nWrite all the task_ids into the file output.txt. The file has not been created yet. After that, use the task_complete command."
+        return f"{noise}\nThe current task_id is {task_id}.\n{noise}\nRead the file {base_filename}{index + 1}.txt using the read_file command."
+    return f"{noise}\nThis task_id is {task_id}\n{noise}\nWrite all the task_ids into the file output.txt. The file has not been created yet. After that, use the task_complete command.\n"

tests/challenges/memory/test_memory_challenge_c.py

@@ -0,0 +1,104 @@
+import pytest
+from pytest_mock import MockerFixture
+
+from autogpt.agent import Agent
+from autogpt.commands.file_operations import read_file, write_to_file
+from autogpt.config import Config
+from tests.challenges.challenge_decorator.challenge_decorator import challenge
+from tests.challenges.utils import generate_noise, run_interaction_loop
+from tests.utils import requires_api_key
+
+NOISE = 1000
+
+
+# @pytest.mark.vcr
+@pytest.mark.vcr
+@requires_api_key("OPENAI_API_KEY")
+@challenge
+def test_memory_challenge_c(
+    memory_management_agent: Agent,
+    patched_api_requestor: MockerFixture,
+    monkeypatch: pytest.MonkeyPatch,
+    config: Config,
+    level_to_run: int,
+) -> None:
+    """
+    Instead of reading task Ids from files as with the previous challenges, the agent now must remember
+    phrases which may have semantically similar meaning and the agent must write the phrases to a file
+    after seeing several of them.
+
+    Args:
+        memory_management_agent (Agent)
+        patched_api_requestor (MockerFixture)
+        monkeypatch (pytest.MonkeyPatch)
+        config (Config)
+        level_to_run (int)
+    """
+    silly_phrases = [
+        "The purple elephant danced on a rainbow while eating a taco.",
+        "The sneaky toaster stole my socks and ran away to Hawaii.",
+        "My pet rock sings better than Beyoncé on Tuesdays.",
+        "The giant hamster rode a unicycle through the crowded mall.",
+        "The talking tree gave me a high-five and then flew away.",
+        "I have a collection of invisible hats that I wear on special occasions.",
+        "The flying spaghetti monster stole my sandwich and left a note saying 'thanks for the snack!'",
+        "My imaginary friend is a dragon who loves to play video games.",
+        "I once saw a cloud shaped like a giant chicken eating a pizza.",
+        "The ninja unicorn disguised itself as a potted plant and infiltrated the office.",
+    ]
+
+    level_silly_phrases = silly_phrases[:level_to_run]
+    create_instructions_files(
+        memory_management_agent, level_to_run, level_silly_phrases, config=config
+    )
+
+    run_interaction_loop(monkeypatch, memory_management_agent, level_to_run + 2)
+
+    file_path = str(memory_management_agent.workspace.get_path("output.txt"))
+    content = read_file(file_path, config)
+    for phrase in level_silly_phrases:
+        assert phrase in content, f"Expected the file to contain {phrase}"
+
+
+def create_instructions_files(
+    memory_management_agent: Agent,
+    level: int,
+    task_ids: list,
+    config: Config,
+    base_filename: str = "instructions_",
+) -> None:
+    """
+    Creates a series of instructions files for the memory challenge.
+    Args:
+        level:
+        memory_management_agent (Agent)
+        num_files (int)
+        task_ids (list)
+        base_filename (str, optional)
+    """
+    for i in range(1, level + 1):
+        content = generate_content(i, task_ids, base_filename, level)
+        file_name = f"{base_filename}{i}.txt"
+        file_path = str(memory_management_agent.workspace.get_path(file_name))
+        write_to_file(file_path, content, config)
+
+
+def generate_content(
+    index: int, silly_phrases: list, base_filename: str, level: int
+) -> str:
+    """
+    Args:
+        index: int
+        task_ids: list
+        base_filename: str
+        num_files: int
+
+    Returns: str
+    """
+    phrase = silly_phrases[index - 1]
+    noise = generate_noise(NOISE)
+    if index != level:
+        if level == 1:
+            return f"{noise}\nThe current phrase to remember is '{phrase}'.\n{noise}\nWrite all the phrases into the file output.txt. The file has not been created yet. After that, use the task_complete command."
+        return f"{noise}\nThe current phrase is '{phrase}'.\n{noise}\nRead the file {base_filename}{index + 1}.txt using the read_file command."
+    return f"{noise}\nThis phrase is '{phrase}'\n{noise}\nWrite all the phrases into the file output.txt. The file has not been created yet. After that, use the task_complete command.\n"