aljaz/goose
1
0
Fork 0
mirror of https://github.com/aljazceru/goose.git synced 2025-12-18 22:54:24 +01:00
Code Issues Packages Projects Releases Wiki Activity

fix: goosebench selector collection issue (#2129)

Browse Source
This commit is contained in:
Alice Hau
2025-04-10 12:16:31 -04:00
committed by GitHub
parent ac56756df6
commit 2a70707c91
2 changed files with 16 additions and 37 deletions
Download Patch File Download Diff File Expand all files Collapse all files

39
crates/goose-bench/src/runners/model_runner.rs
View File

@@ -2,7 +2,7 @@ use crate::bench_config::{BenchEval, BenchModel, BenchRunConfig};
use crate::eval_suites::EvaluationSuite; use crate::eval_suites::EvaluationSuite;
use crate::reporting::{BenchmarkResults, SuiteResult}; use crate::reporting::{BenchmarkResults, SuiteResult};
use crate::runners::eval_runner::EvalRunner; use crate::runners::eval_runner::EvalRunner;
use crate::utilities::{await_process_exits, parallel_bench_cmd, union_hashmaps}; use crate::utilities::{await_process_exits, parallel_bench_cmd};
use std::collections::HashMap; use std::collections::HashMap;
use std::fs::read_to_string; use std::fs::read_to_string;
use std::io::{self, BufRead}; use std::io::{self, BufRead};
@@ -189,30 +189,23 @@ impl ModelRunner {
Ok(results) Ok(results)
} }
fn collect_evals_for_run(&self) -> HashMap<String, Vec<BenchEval>> { fn collect_evals_for_run(&self) -> HashMap<String, Vec<BenchEval>> {
// convert suites map {suite_name => [eval_selector_str] to map suite_name => [BenchEval] // convert suites map {suite_name => [eval_selector_str] to map suite_name => [BenchEval]
let suites = self let mut result: HashMap<String, Vec<BenchEval>> = HashMap::new();
.config for eval in self.config.evals.iter() {
.evals let selected_suites = EvaluationSuite::select(vec![eval.selector.clone()]);
.iter() for (suite, evals) in selected_suites {
.map(|eval| { let entry: &mut Vec<BenchEval> = result.entry(suite).or_default();
EvaluationSuite::select(vec![eval.clone().selector]) entry.reserve(evals.len());
.iter() for suite_eval in evals {
.map(|(suite, evals)| { let mut updated_eval = eval.clone();
let bench_evals = evals updated_eval.selector = suite_eval.to_string();
.iter() entry.push(updated_eval);
.map(|suite_eval| { }
let mut updated_eval = eval.clone(); }
updated_eval.selector = (*suite_eval).to_string(); }
updated_eval result
})
.collect::<Vec<_>>();
(suite.clone(), bench_evals)
})
.collect()
})
.collect();
union_hashmaps(suites)
} }
fn toolshim_envs(&self) -> Vec<(String, String)> { fn toolshim_envs(&self) -> Vec<(String, String)> {

14
crates/goose-bench/src/utilities.rs
View File

@@ -1,21 +1,7 @@
use std::collections::HashMap;
use std::env; use std::env;
use std::process::{Child, Command}; use std::process::{Child, Command};
use std::thread::JoinHandle; use std::thread::JoinHandle;
pub fn union_hashmaps<K, V>(maps: Vec<HashMap<K, V>>) -> HashMap<K, V>
where
K: Eq + std::hash::Hash,
V: Clone,
{
// We can use the fold method to accumulate all maps into one
maps.into_iter().fold(HashMap::new(), |mut result, map| {
// For each map in the vector, extend the result with its entries
result.extend(map);
result
})
}
pub fn await_process_exits( pub fn await_process_exits(
child_processes: &mut [Child], child_processes: &mut [Child],
handles: Vec<JoinHandle<anyhow::Result<()>>>, handles: Vec<JoinHandle<anyhow::Result<()>>>,