debug(benchmark): Add log statement to validator on TestResult

Validation errors don't mention the values causing the error, making it hard to debug. This happened a few times in autogpts-benchmark.yml, so let's put this log statement here until we figure out what makes it crash.
This commit is contained in:
Reinier van der Leer
2024-02-17 13:32:22 +01:00
parent 880c8e804c
commit 09c307d679

View File

@@ -2,11 +2,13 @@
Model definitions used internally and for reports generated during command-line runs.
"""
import logging
from typing import Any, Dict, List
from pydantic import BaseModel, Field, constr, validator
datetime_format = r"^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\+00:00$"
logger = logging.getLogger(__name__)
class TestResult(BaseModel):
@@ -27,6 +29,12 @@ class TestResult(BaseModel):
@validator("fail_reason")
def success_xor_fail_reason(cls, v: str | None, values: dict[str, Any]):
if bool(v) == bool(values["success"]):
logger.error(
"Error validating `success ^ fail_reason` on TestResult: "
f"success = {repr(values['success'])}; "
f"fail_reason = {repr(v)}"
)
if v:
success = values["success"]
assert not success, "fail_reason must only be specified if success=False"