Retry regression tests (#4648)

requirements.txt

@@ -61,3 +61,4 @@ pytest-mock
 vcrpy @ git+https://github.com/Significant-Gravitas/vcrpy.git@master
 pytest-recording
 pytest-xdist
+flaky

tests/challenges/basic_abilities/test_browse_website.py

@@ -3,14 +3,11 @@ import pytest
 from autogpt.agent import Agent
 from tests.challenges.challenge_decorator.challenge_decorator import challenge
 from tests.challenges.utils import run_interaction_loop
-from tests.utils import requires_api_key
 
 CYCLE_COUNT = 2
 
 
-@requires_api_key("OPENAI_API_KEY")
+@challenge()
-@pytest.mark.vcr
-@challenge
 def test_browse_website(
     browser_agent: Agent,
     patched_api_requestor: None,

tests/challenges/basic_abilities/test_write_file.py

@@ -6,7 +6,6 @@ from autogpt.agent import Agent
 from autogpt.commands.file_operations import read_file
 from tests.challenges.challenge_decorator.challenge_decorator import challenge
 from tests.challenges.utils import get_workspace_path, run_interaction_loop
-from tests.utils import requires_api_key
 
 CYCLE_COUNT_PER_LEVEL = [1, 1]
 EXPECTED_OUTPUTS_PER_LEVEL = [
@@ -15,9 +14,7 @@ EXPECTED_OUTPUTS_PER_LEVEL = [
 ]
 
 
-@requires_api_key("OPENAI_API_KEY")
+@challenge()
-@pytest.mark.vcr
-@challenge
 def test_write_file(
     file_system_agents: List[Agent],
     patched_api_requestor: None,

tests/challenges/challenge_decorator/challenge_decorator.py

@@ -3,6 +3,7 @@ from functools import wraps
 from typing import Any, Callable, Optional
 
 import pytest
+from flaky import flaky  # type: ignore
 
 from tests.challenges.challenge_decorator.challenge import Challenge
 from tests.challenges.challenge_decorator.challenge_utils import create_challenge
@@ -10,6 +11,7 @@ from tests.challenges.challenge_decorator.score_utils import (
     get_scores,
     update_new_score,
 )
+from tests.utils import requires_api_key
 
 MAX_LEVEL_TO_IMPROVE_ON = (
     1  # we will attempt to beat 1 level above the current level for now.
@@ -18,7 +20,13 @@ MAX_LEVEL_TO_IMPROVE_ON = (
 CHALLENGE_FAILED_MESSAGE = "Challenges can sometimes fail randomly, please run this test again and if it fails reach out to us on https://discord.gg/autogpt and reach out to us on the 'challenges' channel to let us know the challenge you're struggling with."
 
 
-def challenge(func: Callable[..., Any]) -> Callable[..., None]:
+def challenge(
+    max_runs: int = 2, min_passes: int = 1, api_key: str = "OPENAI_API_KEY"
+) -> Callable[[Callable[..., Any]], Callable[..., None]]:
+    def decorator(func: Callable[..., Any]) -> Callable[..., None]:
+        @requires_api_key(api_key)
+        @pytest.mark.vcr
+        @flaky(max_runs=max_runs, min_passes=min_passes)
         @wraps(func)
         def wrapper(*args: Any, **kwargs: Any) -> None:
             run_remaining = MAX_LEVEL_TO_IMPROVE_ON if Challenge.BEAT_CHALLENGES else 1
@@ -26,7 +34,9 @@ def challenge(func: Callable[..., Any]) -> Callable[..., None]:
 
             while run_remaining > 0:
                 current_score, new_score, new_score_location = get_scores()
-            level_to_run = kwargs["level_to_run"] if "level_to_run" in kwargs else None
+                level_to_run = (
+                    kwargs["level_to_run"] if "level_to_run" in kwargs else None
+                )
                 challenge = create_challenge(
                     func, current_score, Challenge.BEAT_CHALLENGES, level_to_run
                 )
@@ -57,7 +67,6 @@ def challenge(func: Callable[..., Any]) -> Callable[..., None]:
 
                 if not challenge.succeeded:
                     if Challenge.BEAT_CHALLENGES or challenge.is_new_challenge:
-                    # xfail
                         pytest.xfail(str(original_error))
                     if original_error:
                         raise original_error
@@ -65,6 +74,8 @@ def challenge(func: Callable[..., Any]) -> Callable[..., None]:
 
         return wrapper
 
+    return decorator
+
 
 def get_new_max_level_beaten(
     challenge: Challenge, beat_challenges: bool

tests/challenges/debug_code/test_debug_code_challenge_a.py

@@ -11,7 +11,6 @@ from tests.challenges.utils import (
     get_workspace_path,
     run_interaction_loop,
 )
-from tests.utils import requires_api_key
 
 CYCLE_COUNT = 5
 EXPECTED_VALUES = ["[0, 1]", "[2, 5]", "[0, 3]"]
@@ -20,9 +19,7 @@ CODE_FILE_PATH = "code.py"
 TEST_FILE_PATH = "test.py"
 
 
-@pytest.mark.vcr
+@challenge()
-@requires_api_key("OPENAI_API_KEY")
-@challenge
 def test_debug_code_challenge_a(
     debug_code_agents: Agent,
     monkeypatch: pytest.MonkeyPatch,

tests/challenges/information_retrieval/test_information_retrieval_challenge_a.py

@@ -4,7 +4,6 @@ from pytest_mock import MockerFixture
 from autogpt.commands.file_operations import read_file
 from tests.challenges.challenge_decorator.challenge_decorator import challenge
 from tests.challenges.utils import get_workspace_path, run_interaction_loop
-from tests.utils import requires_api_key
 
 CYCLE_COUNT = 3
 EXPECTED_REVENUES = [["81"], ["81"], ["81", "53", "24", "21", "11", "7", "4", "3", "2"]]
@@ -13,9 +12,7 @@ from autogpt.agent import Agent
 OUTPUT_LOCATION = "output.txt"
 
 
-@pytest.mark.vcr
+@challenge()
-@requires_api_key("OPENAI_API_KEY")
-@challenge
 def test_information_retrieval_challenge_a(
     information_retrieval_agents: Agent,
     monkeypatch: pytest.MonkeyPatch,

tests/challenges/information_retrieval/test_information_retrieval_challenge_b.py

@@ -7,15 +7,12 @@ from autogpt.agent import Agent
 from autogpt.commands.file_operations import read_file
 from tests.challenges.challenge_decorator.challenge_decorator import challenge
 from tests.challenges.utils import get_workspace_path, run_interaction_loop
-from tests.utils import requires_api_key
 
 CYCLE_COUNT = 3
 OUTPUT_LOCATION = "2010_nobel_prize_winners.txt"
 
 
-@pytest.mark.vcr
+@challenge()
-@requires_api_key("OPENAI_API_KEY")
-@challenge
 def test_information_retrieval_challenge_b(
     get_nobel_prize_agent: Agent,
     monkeypatch: pytest.MonkeyPatch,

tests/challenges/kubernetes/test_kubernetes_template_challenge_a.py

@@ -6,15 +6,12 @@ from autogpt.agent import Agent
 from autogpt.commands.file_operations import read_file
 from tests.challenges.challenge_decorator.challenge_decorator import challenge
 from tests.challenges.utils import get_workspace_path, run_interaction_loop
-from tests.utils import requires_api_key
 
 CYCLE_COUNT = 3
 OUTPUT_LOCATION = "kube.yaml"
 
 
-@pytest.mark.vcr
+@challenge()
-@requires_api_key("OPENAI_API_KEY")
-@challenge
 def test_kubernetes_template_challenge_a(
     kubernetes_agent: Agent,
     monkeypatch: pytest.MonkeyPatch,

tests/challenges/memory/test_memory_challenge_a.py

@@ -5,14 +5,11 @@ from autogpt.agent import Agent
 from autogpt.commands.file_operations import read_file, write_to_file
 from tests.challenges.challenge_decorator.challenge_decorator import challenge
 from tests.challenges.utils import get_workspace_path, run_interaction_loop
-from tests.utils import requires_api_key
 
 OUTPUT_LOCATION = "output.txt"
 
 
-@pytest.mark.vcr
+@challenge()
-@requires_api_key("OPENAI_API_KEY")
-@challenge
 def test_memory_challenge_a(
     memory_management_agent: Agent,
     patched_api_requestor: MockerFixture,
@@ -28,7 +25,6 @@ def test_memory_challenge_a(
         monkeypatch (pytest.MonkeyPatch)
         level_to_run (int)
     """
-
     task_id = "2314"
     create_instructions_files(memory_management_agent, level_to_run, task_id)
 

tests/challenges/memory/test_memory_challenge_b.py

@@ -9,15 +9,12 @@ from tests.challenges.utils import (
     get_workspace_path,
     run_interaction_loop,
 )
-from tests.utils import requires_api_key
 
 NOISE = 1000
 OUTPUT_LOCATION = "output.txt"
 
 
-@pytest.mark.vcr
+@challenge()
-@requires_api_key("OPENAI_API_KEY")
-@challenge
 def test_memory_challenge_b(
     memory_management_agent: Agent,
     patched_api_requestor: MockerFixture,

tests/challenges/memory/test_memory_challenge_c.py

@@ -9,16 +9,12 @@ from tests.challenges.utils import (
     get_workspace_path,
     run_interaction_loop,
 )
-from tests.utils import requires_api_key
 
 NOISE = 1000
 OUTPUT_LOCATION = "output.txt"
 
 
-# @pytest.mark.vcr
+@challenge()
-@pytest.mark.vcr
-@requires_api_key("OPENAI_API_KEY")
-@challenge
 def test_memory_challenge_c(
     memory_management_agent: Agent,
     patched_api_requestor: MockerFixture,

tests/challenges/memory/test_memory_challenge_d.py

@@ -8,16 +8,13 @@ from autogpt.agent import Agent
 from autogpt.commands.file_operations import read_file, write_to_file
 from tests.challenges.challenge_decorator.challenge_decorator import challenge
 from tests.challenges.utils import get_workspace_path, run_interaction_loop
-from tests.utils import requires_api_key
 
 LEVEL_CURRENTLY_BEATEN = 1
 MAX_LEVEL = 5
 OUTPUT_LOCATION = "output.txt"
 
 
-@pytest.mark.vcr
+@challenge()
-@requires_api_key("OPENAI_API_KEY")
-@challenge
 def test_memory_challenge_d(
     memory_management_agent: Agent,
     patched_api_requestor: MockerFixture,