From 34814d837abea2e6bf3a56b437e4c92c6deeee2a Mon Sep 17 00:00:00 2001 From: merwanehamadi Date: Fri, 4 Aug 2023 11:28:45 -0700 Subject: [PATCH] Fix "attempted" metric being incorrect (#251) Signed-off-by: Merwane Hamadi --- agbenchmark/reports/reports.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/agbenchmark/reports/reports.py b/agbenchmark/reports/reports.py index 7dddfc01..4891aa3e 100644 --- a/agbenchmark/reports/reports.py +++ b/agbenchmark/reports/reports.py @@ -239,7 +239,13 @@ def finalize_reports(item: Any, challenge_data: dict[str, Any]) -> None: cost = get_data_from_helicone(test_name) else: print("Helicone not setup or mock flag set, not getting cost") + info_details["metrics"]["cost"] = cost + + if info_details["metrics"].get("success", None) is None: + info_details["metrics"]["attempted"] = False + info_details["metrics"]["success"] = False + info_details["metrics"]["run_time"] = f"{str(round(run_time, 3))} seconds" info_details["reached_cutoff"] = float(run_time) > challenge_data["cutoff"] @@ -259,7 +265,6 @@ def generate_separate_suite_reports(suite_reports: dict) -> None: "percentage": 0, "highest_difficulty": "", "run_time": "0 seconds", - "attempted": True, }, "tests": {}, }