From 09c307d6796bc445150005f54dcb339268982061 Mon Sep 17 00:00:00 2001 From: Reinier van der Leer Date: Sat, 17 Feb 2024 13:32:22 +0100 Subject: [PATCH] debug(benchmark): Add log statement to validator on `TestResult` Validation errors don't mention the values causing the error, making it hard to debug. This happened a few times in autogpts-benchmark.yml, so let's put this log statement here until we figure out what makes it crash. --- benchmark/agbenchmark/reports/processing/report_types.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/benchmark/agbenchmark/reports/processing/report_types.py b/benchmark/agbenchmark/reports/processing/report_types.py index 2ed4acf3..0475455a 100644 --- a/benchmark/agbenchmark/reports/processing/report_types.py +++ b/benchmark/agbenchmark/reports/processing/report_types.py @@ -2,11 +2,13 @@ Model definitions used internally and for reports generated during command-line runs. """ +import logging from typing import Any, Dict, List from pydantic import BaseModel, Field, constr, validator datetime_format = r"^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\+00:00$" +logger = logging.getLogger(__name__) class TestResult(BaseModel): @@ -27,6 +29,12 @@ class TestResult(BaseModel): @validator("fail_reason") def success_xor_fail_reason(cls, v: str | None, values: dict[str, Any]): + if bool(v) == bool(values["success"]): + logger.error( + "Error validating `success ^ fail_reason` on TestResult: " + f"success = {repr(values['success'])}; " + f"fail_reason = {repr(v)}" + ) if v: success = values["success"] assert not success, "fail_reason must only be specified if success=False"