Rearrange tests & fix CI (#4596)

* Rearrange tests into unit/integration/challenge categories

* Fix linting + `tests.challenges` imports

* Fix obscured duplicate test in test_url_validation.py

* Move VCR conftest to tests.vcr

* Specify tests to run & their order (unit -> integration -> challenges) in CI

* Fail Docker CI when tests fail

* Fix import & linting errors in tests

* Fix `get_text_summary`

* Fix linting errors

* Clean up pytest args in CI

* Remove bogus tests from GoCodeo

tests/challenges/basic_abilities/test_browse_website.py

@@ -1,10 +1,8 @@
 import pytest
 
 from autogpt.agent import Agent
-from tests.integration.challenges.challenge_decorator.challenge_decorator import (
-    challenge,
-)
-from tests.integration.challenges.utils import run_interaction_loop
+from tests.challenges.challenge_decorator.challenge_decorator import challenge
+from tests.challenges.utils import run_interaction_loop
 from tests.utils import requires_api_key
 
 CYCLE_COUNT = 2

tests/challenges/basic_abilities/test_write_file.py

@@ -4,10 +4,8 @@ from pytest_mock import MockerFixture
 from autogpt.agent import Agent
 from autogpt.commands.file_operations import read_file
 from autogpt.config import Config
-from tests.integration.challenges.challenge_decorator.challenge_decorator import (
-    challenge,
-)
-from tests.integration.challenges.utils import run_interaction_loop
+from tests.challenges.challenge_decorator.challenge_decorator import challenge
+from tests.challenges.utils import run_interaction_loop
 from tests.utils import requires_api_key
 
 CYCLE_COUNT = 3

tests/challenges/challenge_decorator/challenge_decorator.py

@@ -4,11 +4,9 @@ from typing import Any, Callable, Optional
 
 import pytest
 
-from tests.integration.challenges.challenge_decorator.challenge import Challenge
-from tests.integration.challenges.challenge_decorator.challenge_utils import (
-    create_challenge,
-)
-from tests.integration.challenges.challenge_decorator.score_utils import (
+from tests.challenges.challenge_decorator.challenge import Challenge
+from tests.challenges.challenge_decorator.challenge_utils import create_challenge
+from tests.challenges.challenge_decorator.score_utils import (
     get_scores,
     update_new_score,
 )

tests/challenges/challenge_decorator/challenge_utils.py

@@ -1,7 +1,7 @@
 import os
 from typing import Any, Callable, Dict, Optional, Tuple
 
-from tests.integration.challenges.challenge_decorator.challenge import Challenge
+from tests.challenges.challenge_decorator.challenge import Challenge
 
 CHALLENGE_PREFIX = "test_"
 

tests/challenges/challenge_decorator/score_utils.py

@@ -2,7 +2,7 @@ import json
 import os
 from typing import Any, Dict, Optional, Tuple
 
-from tests.integration.challenges.challenge_decorator.challenge import Challenge
+from tests.challenges.challenge_decorator.challenge import Challenge
 
 CURRENT_SCORE_LOCATION = "../current_score"
 NEW_SCORE_LOCATION = "../new_score"

tests/challenges/conftest.py

@@ -5,9 +5,8 @@ from _pytest.config import Config
 from _pytest.config.argparsing import Parser
 from _pytest.fixtures import FixtureRequest
 
-from tests.integration.challenges.challenge_decorator.challenge import Challenge
-from tests.integration.conftest import BASE_VCR_CONFIG
-from tests.vcr.vcr_filter import before_record_response
+from tests.challenges.challenge_decorator.challenge import Challenge
+from tests.vcr import BASE_VCR_CONFIG, before_record_response
 
 
 def before_record_response_filter_errors(

tests/challenges/debug_code/test_debug_code_challenge_a.py

@@ -7,10 +7,8 @@ from autogpt.agent import Agent
 from autogpt.commands.execute_code import execute_python_file
 from autogpt.commands.file_operations import append_to_file, write_to_file
 from autogpt.config import Config
-from tests.integration.challenges.challenge_decorator.challenge_decorator import (
-    challenge,
-)
-from tests.integration.challenges.utils import run_interaction_loop
+from tests.challenges.challenge_decorator.challenge_decorator import challenge
+from tests.challenges.utils import run_interaction_loop
 from tests.utils import requires_api_key
 
 CYCLE_COUNT = 5

tests/challenges/information_retrieval/test_information_retrieval_challenge_a.py

@@ -3,10 +3,8 @@ from pytest_mock import MockerFixture
 
 from autogpt.commands.file_operations import read_file
 from autogpt.config import Config
-from tests.integration.challenges.challenge_decorator.challenge_decorator import (
-    challenge,
-)
-from tests.integration.challenges.utils import run_interaction_loop
+from tests.challenges.challenge_decorator.challenge_decorator import challenge
+from tests.challenges.utils import run_interaction_loop
 from tests.utils import requires_api_key
 
 CYCLE_COUNT = 3

tests/challenges/information_retrieval/test_information_retrieval_challenge_b.py

@@ -6,10 +6,8 @@ from pytest_mock import MockerFixture
 from autogpt.agent import Agent
 from autogpt.commands.file_operations import read_file
 from autogpt.config import Config
-from tests.integration.challenges.challenge_decorator.challenge_decorator import (
-    challenge,
-)
-from tests.integration.challenges.utils import run_interaction_loop
+from tests.challenges.challenge_decorator.challenge_decorator import challenge
+from tests.challenges.utils import run_interaction_loop
 from tests.utils import requires_api_key
 
 CYCLE_COUNT = 3

tests/challenges/kubernetes/test_kubernetes_template_challenge_a.py

@@ -5,10 +5,8 @@ from pytest_mock import MockerFixture
 from autogpt.agent import Agent
 from autogpt.commands.file_operations import read_file
 from autogpt.config import Config
-from tests.integration.challenges.challenge_decorator.challenge_decorator import (
-    challenge,
-)
-from tests.integration.challenges.utils import run_interaction_loop
+from tests.challenges.challenge_decorator.challenge_decorator import challenge
+from tests.challenges.utils import run_interaction_loop
 from tests.utils import requires_api_key
 
 CYCLE_COUNT = 3

tests/challenges/memory/test_memory_challenge_a.py

@@ -4,10 +4,8 @@ from pytest_mock import MockerFixture
 from autogpt.agent import Agent
 from autogpt.commands.file_operations import read_file, write_to_file
 from autogpt.config import Config
-from tests.integration.challenges.challenge_decorator.challenge_decorator import (
-    challenge,
-)
-from tests.integration.challenges.utils import run_interaction_loop
+from tests.challenges.challenge_decorator.challenge_decorator import challenge
+from tests.challenges.utils import run_interaction_loop
 from tests.utils import requires_api_key
 
 

tests/challenges/memory/test_memory_challenge_b.py

@@ -4,10 +4,8 @@ from pytest_mock import MockerFixture
 from autogpt.agent import Agent
 from autogpt.commands.file_operations import read_file, write_to_file
 from autogpt.config import Config
-from tests.integration.challenges.challenge_decorator.challenge_decorator import (
-    challenge,
-)
-from tests.integration.challenges.utils import generate_noise, run_interaction_loop
+from tests.challenges.challenge_decorator.challenge_decorator import challenge
+from tests.challenges.utils import generate_noise, run_interaction_loop
 from tests.utils import requires_api_key
 
 NOISE = 1000

tests/challenges/memory/test_memory_challenge_c.py

@@ -4,10 +4,8 @@ from pytest_mock import MockerFixture
 from autogpt.agent import Agent
 from autogpt.commands.file_operations import read_file, write_to_file
 from autogpt.config import Config
-from tests.integration.challenges.challenge_decorator.challenge_decorator import (
-    challenge,
-)
-from tests.integration.challenges.utils import generate_noise, run_interaction_loop
+from tests.challenges.challenge_decorator.challenge_decorator import challenge
+from tests.challenges.utils import generate_noise, run_interaction_loop
 from tests.utils import requires_api_key
 
 NOISE = 1000

tests/challenges/utils/build_current_score.py

@@ -26,12 +26,8 @@ def recursive_sort_dict(data: dict) -> dict:
 
 
 cwd = os.getcwd()  # get current working directory
-new_score_filename_pattern = os.path.join(
-    cwd, "tests/integration/challenges/new_score_*.json"
-)
-current_score_filename = os.path.join(
-    cwd, "tests/integration/challenges/current_score.json"
-)
+new_score_filename_pattern = os.path.join(cwd, "tests/challenges/new_score_*.json")
+current_score_filename = os.path.join(cwd, "tests/challenges/current_score.json")
 
 merged_data: Dict[str, Any] = {}
 for filename in glob.glob(new_score_filename_pattern):

tests/conftest.py

@@ -1,4 +1,3 @@
-import os
 from pathlib import Path
 
 import pytest
@@ -8,15 +7,11 @@ from autogpt.config.config import Config
 from autogpt.llm.api_manager import ApiManager
 from autogpt.workspace import Workspace
 
-pytest_plugins = ["tests.integration.agent_factory", "tests.integration.memory.utils"]
-
-PROXY = os.environ.get("PROXY")
-
-
-@pytest.fixture()
-def vcr_cassette_dir(request):
-    test_name = os.path.splitext(request.node.name)[0]
-    return os.path.join("tests/Auto-GPT-test-cassettes", test_name)
+pytest_plugins = [
+    "tests.integration.agent_factory",
+    "tests.integration.memory.utils",
+    "tests.vcr",
+]
 
 
 @pytest.fixture()

tests/integration/conftest.py

@@ -1,56 +0,0 @@
-import os
-
-import openai.api_requestor
-import pytest
-from pytest_mock import MockerFixture
-
-from tests.conftest import PROXY
-from tests.vcr.vcr_filter import before_record_request, before_record_response
-
-BASE_VCR_CONFIG = {
-    "record_mode": "new_episodes",
-    "before_record_request": before_record_request,
-    "before_record_response": before_record_response,
-    "filter_headers": [
-        "Authorization",
-        "X-OpenAI-Client-User-Agent",
-        "User-Agent",
-    ],
-    "match_on": ["method", "body"],
-}
-
-
-@pytest.fixture(scope="session")
-def vcr_config():
-    # this fixture is called by the pytest-recording vcr decorator.
-    return BASE_VCR_CONFIG
-
-
-def patch_api_base(requestor):
-    new_api_base = f"{PROXY}/v1"
-    requestor.api_base = new_api_base
-    return requestor
-
-
-@pytest.fixture
-def patched_api_requestor(mocker: MockerFixture):
-    original_init = openai.api_requestor.APIRequestor.__init__
-    original_validate_headers = openai.api_requestor.APIRequestor._validate_headers
-
-    def patched_init(requestor, *args, **kwargs):
-        original_init(requestor, *args, **kwargs)
-        patch_api_base(requestor)
-
-    def patched_validate_headers(self, supplied_headers):
-        headers = original_validate_headers(self, supplied_headers)
-        headers["AGENT-MODE"] = os.environ.get("AGENT_MODE")
-        headers["AGENT-TYPE"] = os.environ.get("AGENT_TYPE")
-        return headers
-
-    if PROXY:
-        mocker.patch("openai.api_requestor.APIRequestor.__init__", new=patched_init)
-        mocker.patch.object(
-            openai.api_requestor.APIRequestor,
-            "_validate_headers",
-            new=patched_validate_headers,
-        )

tests/integration/test_commands.py

@@ -1,32 +0,0 @@
-"""Unit tests for the commands module"""
-from unittest.mock import MagicMock, patch
-
-import pytest
-
-from autogpt.app import list_agents, start_agent
-from tests.utils import requires_api_key
-
-
-@pytest.mark.vcr
-@pytest.mark.integration_test
-@requires_api_key("OPENAI_API_KEY")
-def test_make_agent(patched_api_requestor, config) -> None:
-    """Test that an agent can be created"""
-    # Use the mock agent manager to avoid creating a real agent
-    with patch("openai.ChatCompletion.create") as mock:
-        response = MagicMock()
-        # del response.error
-        response.choices[0].messages[0].content = "Test message"
-        response.usage.prompt_tokens = 1
-        response.usage.completion_tokens = 1
-        mock.return_value = response
-        start_agent(
-            "Test Agent", "chat", "Hello, how are you?", config, "gpt-3.5-turbo"
-        )
-        agents = list_agents(config)
-        assert "List of agents:\n0: chat" == agents
-        start_agent(
-            "Test Agent 2", "write", "Hello, how are you?", config, "gpt-3.5-turbo"
-        )
-        agents = list_agents(config)
-        assert "List of agents:\n0: chat\n1: write" == agents

tests/test_analyze_code.py

@@ -1,74 +0,0 @@
-# Date: 2023-5-13
-# Author: Generated by GoCodeo.
-import pytest
-
-from autogpt.commands.analyze_code import analyze_code
-from autogpt.config import Config
-
-
-@pytest.fixture
-def mock_call_ai_function(mocker):
-    return mocker.patch("autogpt.commands.analyze_code.call_ai_function")
-
-
-class TestAnalyzeCode:
-    def test_positive_analyze_code(self, mock_call_ai_function):
-        # Positive Test
-        mock_call_ai_function.return_value = ["Suggestion 1", "Suggestion 2"]
-        code = "def example_function():\n    pass"
-        config = Config()
-        result = analyze_code(code, config)
-        assert result == ["Suggestion 1", "Suggestion 2"]
-        mock_call_ai_function.assert_called_once_with(
-            "def analyze_code(code: str) -> list[str]:",
-            [code],
-            "Analyzes the given code and returns a list of suggestions for improvements.",
-            config=config,
-        )
-
-    def test_negative_analyze_code(
-        self,
-        mock_call_ai_function,
-        config: Config,
-    ):
-        # Negative Test
-        mock_call_ai_function.return_value = []
-        code = "def example_function():\n    pass"
-        result = analyze_code(code, config)
-        assert result == []
-        mock_call_ai_function.assert_called_once_with(
-            "def analyze_code(code: str) -> list[str]:",
-            [code],
-            "Analyzes the given code and returns a list of suggestions for improvements.",
-            config=config,
-        )
-
-    def test_error_analyze_code(self, mock_call_ai_function, config: Config):
-        # Error Test
-        mock_call_ai_function.side_effect = Exception("Error occurred")
-        code = "def example_function():\n    pass"
-        with pytest.raises(Exception):
-            result = analyze_code(code, config)
-        mock_call_ai_function.assert_called_once_with(
-            "def analyze_code(code: str) -> list[str]:",
-            [code],
-            "Analyzes the given code and returns a list of suggestions for improvements.",
-            config=config,
-        )
-
-    def test_edge_analyze_code_empty_code(
-        self,
-        mock_call_ai_function,
-        config: Config,
-    ):
-        # Edge Test
-        mock_call_ai_function.return_value = ["Suggestion 1", "Suggestion 2"]
-        code = ""
-        result = analyze_code(code, config)
-        assert result == ["Suggestion 1", "Suggestion 2"]
-        mock_call_ai_function.assert_called_once_with(
-            "def analyze_code(code: str) -> list[str]:",
-            [code],
-            "Analyzes the given code and returns a list of suggestions for improvements.",
-            config=config,
-        )

tests/test_audio_text_read_audio.py

@@ -1,56 +0,0 @@
-# Date: 2023-5-13
-# Author: Generated by GoCodeo.
-import json
-from unittest.mock import MagicMock, patch
-
-import pytest
-
-from autogpt.commands.audio_text import read_audio
-
-
-class TestReadAudio:
-    @patch("requests.post")
-    def test_positive_read_audio(self, mock_post, config):
-        # Positive Test
-        audio_data = b"test_audio_data"
-        mock_response = MagicMock()
-        mock_response.content.decode.return_value = json.dumps(
-            {"text": "Hello, world!"}
-        )
-        mock_post.return_value = mock_response
-
-        config.huggingface_api_token = "testing-token"
-        result = read_audio(audio_data, config)
-        assert result == "The audio says: Hello, world!"
-        mock_post.assert_called_once_with(
-            f"https://api-inference.huggingface.co/models/{config.huggingface_audio_to_text_model}",
-            headers={"Authorization": f"Bearer {config.huggingface_api_token}"},
-            data=audio_data,
-        )
-
-    @patch("requests.post")
-    def test_negative_read_audio(self, mock_post, config):
-        # Negative Test
-        audio_data = b"test_audio_data"
-        mock_response = MagicMock()
-        mock_response.content.decode.return_value = json.dumps({"text": ""})
-        mock_post.return_value = mock_response
-        config.huggingface_api_token = "testing-token"
-        result = read_audio(audio_data, config)
-        assert result == "The audio says: "
-        mock_post.assert_called_once_with(
-            f"https://api-inference.huggingface.co/models/{config.huggingface_audio_to_text_model}",
-            headers={"Authorization": f"Bearer {config.huggingface_api_token}"},
-            data=audio_data,
-        )
-
-    def test_error_read_audio(self, config):
-        # Error Test
-        config.huggingface_api_token = None
-        with pytest.raises(ValueError):
-            read_audio(b"test_audio_data", config)
-
-    def test_edge_read_audio_empty_audio(self, config):
-        # Edge Test
-        with pytest.raises(ValueError):
-            read_audio(b"", config)

tests/test_audio_text_read_audio_from_file.py

@@ -1,55 +0,0 @@
-# Date: 2023-5-13
-# Author: Generated by GoCodeo.
-
-
-from unittest.mock import mock_open, patch
-
-import pytest
-
-from autogpt.commands.audio_text import read_audio_from_file
-from autogpt.config import Config
-
-
-@pytest.fixture
-def mock_read_audio(mocker):
-    return mocker.patch("autogpt.commands.audio_text.read_audio")
-
-
-class TestReadAudioFromFile:
-    def test_positive_read_audio_from_file(self, mock_read_audio):
-        # Positive test
-        mock_read_audio.return_value = "This is a sample text."
-        mock_file_data = b"Audio data"
-        m = mock_open(read_data=mock_file_data)
-
-        with patch("builtins.open", m):
-            result = read_audio_from_file("test_audio.wav", Config())
-            assert result == "This is a sample text."
-            m.assert_called_once_with("test_audio.wav", "rb")
-
-    def test_negative_read_audio_from_file(self, mock_read_audio):
-        # Negative test
-        mock_read_audio.return_value = "This is a sample text."
-        mock_file_data = b"Audio data"
-        m = mock_open(read_data=mock_file_data)
-
-        with patch("builtins.open", m):
-            result = read_audio_from_file("test_audio.wav", Config())
-            assert result != "Incorrect text."
-            m.assert_called_once_with("test_audio.wav", "rb")
-
-    def test_error_read_audio_from_file(self):
-        # Error test
-        with pytest.raises(FileNotFoundError):
-            read_audio_from_file("non_existent_file.wav", Config())
-
-    def test_edge_empty_audio_file(self, mock_read_audio):
-        # Edge test
-        mock_read_audio.return_value = ""
-        mock_file_data = b""
-        m = mock_open(read_data=mock_file_data)
-
-        with patch("builtins.open", m):
-            result = read_audio_from_file("empty_audio.wav", Config())
-            assert result == ""
-            m.assert_called_once_with("empty_audio.wav", "rb")

tests/unit/test_make_agent.py

@@ -0,0 +1,24 @@
+from unittest.mock import MagicMock
+
+from pytest_mock import MockerFixture
+
+from autogpt.app import list_agents, start_agent
+from autogpt.config import Config
+
+
+def test_make_agent(config: Config, mocker: MockerFixture) -> None:
+    """Test that an agent can be created"""
+    mock = mocker.patch("openai.ChatCompletion.create")
+
+    response = MagicMock()
+    # del response.error
+    response.choices[0].messages[0].content = "Test message"
+    response.usage.prompt_tokens = 1
+    response.usage.completion_tokens = 1
+    mock.return_value = response
+    start_agent("Test Agent", "chat", "Hello, how are you?", config, "gpt-3.5-turbo")
+    agents = list_agents(config)
+    assert "List of agents:\n0: chat" == agents
+    start_agent("Test Agent 2", "write", "Hello, how are you?", config, "gpt-3.5-turbo")
+    agents = list_agents(config)
+    assert "List of agents:\n0: chat\n1: write" == agents

tests/unit/test_url_validation.py

@@ -49,25 +49,17 @@ def test_url_validation_succeeds(url):
     assert dummy_method(url) == url
 
 
-bad_protocol_data = (
-    ("htt://example.com"),
-    ("httppp://example.com"),
-    (" https://example.com"),
+@pytest.mark.parametrize(
+    "url,expected_error",
+    [
+        ("htt://example.com", "Invalid URL format"),
+        ("httppp://example.com", "Invalid URL format"),
+        (" https://example.com", "Invalid URL format"),
+        ("http://?query=q", "Missing Scheme or Network location"),
+    ],
 )
-
-
-@pytest.mark.parametrize("url", bad_protocol_data)
-def test_url_validation_fails_bad_protocol(url):
-    with raises(ValueError, match="Invalid URL format"):
-        dummy_method(url)
-
-
-missing_loc = (("http://?query=q"),)
-
-
-@pytest.mark.parametrize("url", missing_loc)
-def test_url_validation_fails_bad_protocol(url):
-    with raises(ValueError, match="Missing Scheme or Network location"):
+def test_url_validation_fails_invalid_url(url, expected_error):
+    with raises(ValueError, match=expected_error):
         dummy_method(url)
 
 

tests/unit/test_utils.py

@@ -1,7 +1,6 @@
 import os
 from unittest.mock import patch
 
-import pytest
 import requests
 
 from autogpt.utils import (
@@ -151,7 +150,3 @@ def test_get_current_git_branch_failure(mock_repo):
     branch_name = get_current_git_branch()
 
     assert branch_name == ""
-
-
-if __name__ == "__main__":
-    pytest.main()

tests/vcr/__init__.py

@@ -0,0 +1,61 @@
+import os
+
+import openai.api_requestor
+import pytest
+from pytest_mock import MockerFixture
+
+from .vcr_filter import PROXY, before_record_request, before_record_response
+
+BASE_VCR_CONFIG = {
+    "record_mode": "new_episodes",
+    "before_record_request": before_record_request,
+    "before_record_response": before_record_response,
+    "filter_headers": [
+        "Authorization",
+        "X-OpenAI-Client-User-Agent",
+        "User-Agent",
+    ],
+    "match_on": ["method", "body"],
+}
+
+
+@pytest.fixture(scope="session")
+def vcr_config():
+    # this fixture is called by the pytest-recording vcr decorator.
+    return BASE_VCR_CONFIG
+
+
+@pytest.fixture()
+def vcr_cassette_dir(request):
+    test_name = os.path.splitext(request.node.name)[0]
+    return os.path.join("tests/Auto-GPT-test-cassettes", test_name)
+
+
+def patch_api_base(requestor):
+    new_api_base = f"{PROXY}/v1"
+    requestor.api_base = new_api_base
+    return requestor
+
+
+@pytest.fixture
+def patched_api_requestor(mocker: MockerFixture):
+    original_init = openai.api_requestor.APIRequestor.__init__
+    original_validate_headers = openai.api_requestor.APIRequestor._validate_headers
+
+    def patched_init(requestor, *args, **kwargs):
+        original_init(requestor, *args, **kwargs)
+        patch_api_base(requestor)
+
+    def patched_validate_headers(self, supplied_headers):
+        headers = original_validate_headers(self, supplied_headers)
+        headers["AGENT-MODE"] = os.environ.get("AGENT_MODE")
+        headers["AGENT-TYPE"] = os.environ.get("AGENT_TYPE")
+        return headers
+
+    if PROXY:
+        mocker.patch("openai.api_requestor.APIRequestor.__init__", new=patched_init)
+        mocker.patch.object(
+            openai.api_requestor.APIRequestor,
+            "_validate_headers",
+            new=patched_validate_headers,
+        )

tests/vcr/vcr_filter.py

@@ -1,8 +1,9 @@
 import json
+import os
 import re
 from typing import Any, Dict, List
 
-from tests.conftest import PROXY
+PROXY = os.environ.get("PROXY")
 
 REPLACEMENTS: List[Dict[str, str]] = [
     {