From f9792ed7f351147b0994097e38debb485cc458a7 Mon Sep 17 00:00:00 2001 From: Reinier van der Leer Date: Fri, 16 Feb 2024 18:43:37 +0100 Subject: [PATCH] fix(benchmark): Unbreak `-N`/`--attempts` option --- benchmark/agbenchmark/challenges/builtin.py | 2 +- benchmark/agbenchmark/challenges/webarena.py | 2 +- benchmark/agbenchmark/conftest.py | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/benchmark/agbenchmark/challenges/builtin.py b/benchmark/agbenchmark/challenges/builtin.py index 30bd50f8..694d1018 100644 --- a/benchmark/agbenchmark/challenges/builtin.py +++ b/benchmark/agbenchmark/challenges/builtin.py @@ -158,7 +158,7 @@ class BuiltinChallenge(BaseChallenge): self, config: AgentBenchmarkConfig, request: pytest.FixtureRequest, - i_attempt: int = 0, + i_attempt: int, ) -> None: # if os.environ.get("HELICONE_API_KEY"): # from helicone.lock import HeliconeLockManager diff --git a/benchmark/agbenchmark/challenges/webarena.py b/benchmark/agbenchmark/challenges/webarena.py index d3a081e0..2e51ab2b 100644 --- a/benchmark/agbenchmark/challenges/webarena.py +++ b/benchmark/agbenchmark/challenges/webarena.py @@ -377,7 +377,7 @@ class WebArenaChallenge(BaseChallenge): self, config: AgentBenchmarkConfig, request: pytest.FixtureRequest, - i_attempt: int = 0, + i_attempt: int, ) -> None: if not self._spec.available: pytest.skip(self._spec.unavailable_reason) diff --git a/benchmark/agbenchmark/conftest.py b/benchmark/agbenchmark/conftest.py index 7e417a60..cf40493b 100644 --- a/benchmark/agbenchmark/conftest.py +++ b/benchmark/agbenchmark/conftest.py @@ -216,8 +216,8 @@ def pytest_sessionfinish(session: pytest.Session) -> None: def pytest_generate_tests(metafunc: pytest.Metafunc): - if type(n := metafunc.config.getoption("-N")) is str: - metafunc.parametrize("i_attempt", range(int(n))) + n = metafunc.config.getoption("-N") + metafunc.parametrize("i_attempt", range(int(n)) if type(n) is str else [0]) def pytest_collection_modifyitems(