aljaz/Auto-GPT
1
0
Fork 0
mirror of https://github.com/aljazceru/Auto-GPT.git synced 2026-01-31 11:54:30 +01:00
Code Issues Packages Projects Releases Wiki Activity

fix(benchmark/report): Fix and clean up logic in update_challenges_already_beaten

Browse Source 
- `update_challenges_already_beaten` incorrectly marked challenges as beaten if it was present in the report file but set to `false`
This commit is contained in:
Reinier van der Leer
2024-01-19 19:52:09 +01:00
parent fc37ffdfcf
commit 05b018a837
1 changed files with 5 additions and 8 deletions
Download Patch File Download Diff File Expand all files Collapse all files

13
benchmark/agbenchmark/reports/reports.py
View File

@@ -96,7 +96,6 @@ def finalize_test_report(
mock = os.getenv("IS_MOCK") # Check if --mock is in sys.argv
logger.debug(f"Finalizing report with CallInfo: {vars(call)}")
if call.excinfo is None:
info_details.metrics.success = True
else:
@@ -135,17 +134,15 @@ def update_challenges_already_beaten(
current_run_successful = info_details.metrics.success
try:
with open(challenges_already_beaten_file, "r") as f:
challenge_data = json.load(f)
challenges_beaten_before = json.load(f)
except FileNotFoundError:
challenge_data = {}
challenge_beaten_in_the_past = challenge_data.get(test_name)
challenges_beaten_before = {}
challenge_data[test_name] = True
if challenge_beaten_in_the_past is None and not current_run_successful:
challenge_data[test_name] = False
has_ever_been_beaten = challenges_beaten_before.get(test_name)
challenges_beaten_before[test_name] = has_ever_been_beaten or current_run_successful
with open(challenges_already_beaten_file, "w") as f:
json.dump(challenge_data, f, indent=4)
json.dump(challenges_beaten_before, f, indent=4)
def session_finish(agbenchmark_config: AgentBenchmarkConfig) -> None: