Add helicone dynamic headers (#199)

Signed-off-by: Merwane Hamadi <merwanehamadi@gmail.com>

agbenchmark/conftest.py

@@ -5,6 +5,7 @@ import sys
 import time
 from pathlib import Path  # noqa
 from typing import Any, Dict, Generator
+from helicone.lock import HeliconeLockManager
 
 import pytest
 
@@ -132,6 +133,9 @@ suite_reports: dict[str, list] = {}
 
 def pytest_runtest_makereport(item: Any, call: Any) -> None:
     challenge_data = item.funcargs.get("challenge_data", None)
+
+    HeliconeLockManager.write_custom_property("challenge", challenge_data["name"])
+
     if not challenge_data:
         # this will only happen for dummy dependency setup tests
         return

agbenchmark/reports/ReportManager.py

@@ -5,6 +5,7 @@ import time
 from datetime import datetime
 from typing import Any, Dict, Optional
 
+from agbenchmark.start_benchmark import BENCHMARK_START_TIME
 from agbenchmark.utils.utils import get_highest_success_difficulty
 
 
@@ -62,6 +63,7 @@ class ReportManager:
         self.tests = {
             "command": command.split(os.sep)[-1],
             "completion_time": datetime.now().strftime("%Y-%m-%d-%H:%M"),
+            "benchmark_start_time": BENCHMARK_START_TIME,
             "metrics": {
                 "run_time": str(round(time.time() - self.start_time, 2)) + " seconds",
                 "highest_difficulty": get_highest_success_difficulty(self.tests),

agbenchmark/start_benchmark.py

@@ -1,16 +1,22 @@
 import json
 import os
 import sys
+from datetime import datetime
 from pathlib import Path
 from typing import Any
+from helicone.lock import HeliconeLockManager
+from agbenchmark.utils.utils import AGENT_NAME, calculate_dynamic_paths
 
 import click
 import pytest
 
-from agbenchmark.utils.utils import calculate_dynamic_paths
 
 CURRENT_DIRECTORY = Path(__file__).resolve().parent
+BENCHMARK_START_TIME = datetime.now().strftime("%Y-%m-%d-%H:%M")
 
+HeliconeLockManager.write_custom_property("benchmark_start_time", BENCHMARK_START_TIME)
+if AGENT_NAME:
+    HeliconeLockManager.write_custom_property("agent_name", AGENT_NAME)
 (
     HOME_DIRECTORY,
     CONFIG_PATH,