fix: Fixing Benchmarking

- Importing missing metadata field in Test class in report_types.py
- Adding GAIA categories 1, 2, and 3 in data_types.py
This commit is contained in:
SwiftyOS
2023-11-09 10:00:50 +01:00
parent 6c84035c7e
commit fa357dd139
2 changed files with 6 additions and 0 deletions

View File

@@ -51,6 +51,8 @@ class Test(BaseModelBenchmark):
category: List[str]
task: str
reached_cutoff: bool
metadata: Any
class ReportBase(BaseModelBenchmark):
@@ -68,6 +70,7 @@ class Report(ReportBase):
tests: Dict[str, Test]
class ReportV2(Test, ReportBase):
test_name: str
run_id: str | None

View File

@@ -174,6 +174,9 @@ class Category(str, Enum):
GENERALIST = "general"
CODING = "coding"
SCRAPE_SYNTHESIZE = "scrape_synthesize"
GAIA_1 = "GAIA_1"
GAIA_2 = "GAIA_2"
GAIA_3 = "GAIA_3"
class ChallengeData(BaseModel):