From 8fa67ea4668cedddf23234d915bea24e1deb40a5 Mon Sep 17 00:00:00 2001 From: merwanehamadi Date: Wed, 2 Aug 2023 14:44:14 -0700 Subject: [PATCH] Correct agent and benchmark commit sha (#245) Signed-off-by: Merwane Hamadi --- agbenchmark/reports/ReportManager.py | 6 ++++-- agbenchmark/start_benchmark.py | 3 ++- agbenchmark/utils/utils.py | 14 ++++++++------ send_to_googledrive.py | 3 ++- 4 files changed, 16 insertions(+), 10 deletions(-) diff --git a/agbenchmark/reports/ReportManager.py b/agbenchmark/reports/ReportManager.py index daeb8061..69b61c4d 100644 --- a/agbenchmark/reports/ReportManager.py +++ b/agbenchmark/reports/ReportManager.py @@ -10,8 +10,9 @@ from agbenchmark.reports.processing.graphs import save_single_radar_chart from agbenchmark.reports.processing.process_report import get_agent_category from agbenchmark.reports.processing.report_types import Report from agbenchmark.start_benchmark import ( + AGENT_GIT_COMMIT_SHA, + BENCHMARK_GIT_COMMIT_SHA, BENCHMARK_START_TIME, - GIT_COMMIT_SHA, REPORTS_PATH, ) from agbenchmark.utils.utils import get_highest_success_difficulty @@ -60,7 +61,8 @@ class ReportManager: command = " ".join(sys.argv) self.tests = { "command": command.split(os.sep)[-1], - "git_commit_sha": GIT_COMMIT_SHA, + "benchmark_git_commit_sha": BENCHMARK_GIT_COMMIT_SHA, + "agent_git_commit_sha": AGENT_GIT_COMMIT_SHA, "completion_time": datetime.now().strftime("%Y-%m-%d-%H:%M"), "benchmark_start_time": BENCHMARK_START_TIME, "metrics": { diff --git a/agbenchmark/start_benchmark.py b/agbenchmark/start_benchmark.py index da0e03a0..88b00289 100644 --- a/agbenchmark/start_benchmark.py +++ b/agbenchmark/start_benchmark.py @@ -28,7 +28,8 @@ HeliconeLockManager.write_custom_property("benchmark_start_time", BENCHMARK_STAR SUCCESS_RATE_PATH, CHALLENGES_PATH, ) = calculate_dynamic_paths() -GIT_COMMIT_SHA = get_git_commit_sha() +BENCHMARK_GIT_COMMIT_SHA = get_git_commit_sha(HOME_DIRECTORY / ".." / "..") +AGENT_GIT_COMMIT_SHA = get_git_commit_sha(HOME_DIRECTORY) @click.group() diff --git a/agbenchmark/utils/utils.py b/agbenchmark/utils/utils.py index 42c06a1c..5c65ae63 100644 --- a/agbenchmark/utils/utils.py +++ b/agbenchmark/utils/utils.py @@ -272,14 +272,16 @@ def calculate_dynamic_paths() -> tuple[Path, str, str, str, str, str]: ) -def get_git_commit_sha() -> Optional[str]: - from agbenchmark.start_benchmark import HOME_DIRECTORY - +def get_git_commit_sha(directory) -> Optional[str]: try: - repo = git.Repo(HOME_DIRECTORY) - git_commit_sha = repo.head.commit.hexsha + repo = git.Repo(directory) + remote_url = repo.remotes.origin.url + if remote_url.endswith(".git"): + remote_url = remote_url[:-4] + git_commit_sha = f"{remote_url}/tree/{repo.head.commit.hexsha}" + print(f"GIT_COMMIT_SHA: {git_commit_sha}") return git_commit_sha except Exception: - print(f"{HOME_DIRECTORY} is not a git repository!") + print(f"{directory} is not a git repository!") return None diff --git a/send_to_googledrive.py b/send_to_googledrive.py index 10b20bed..a85dba54 100644 --- a/send_to_googledrive.py +++ b/send_to_googledrive.py @@ -56,7 +56,8 @@ def process_test( "non_mock_success_%", "" ), "Run Time": test_info.get("metrics", {}).get("run_time", ""), - "Benchmark Git Commit Sha": common_data.get("git_commit_sha", None), + "Benchmark Git Commit Sha": common_data.get("benchmark_git_commit_sha", None), + "Agent Git Commit Sha": common_data.get("agent_git_commit_sha", None), "Cost": test_info.get("metrics", {}).get("cost", ""), "Attempted": test_info.get("metrics", {}).get("attempted", ""), }