wip-4a3
CI / test (push) Failing after 3s

This commit is contained in:
Omar Sobh
2026-08-07 04:50:29 -07:00
parent 60cf0b7b99
commit 743dffa660
3 changed files with 77 additions and 32 deletions
@@ -384,11 +384,7 @@ impl DatasetProfile {
}
}
fn print_report(
overall: &Metrics,
by_type: &HashMap<String, Metrics>,
profile: &DatasetProfile,
) {
fn print_report(overall: &Metrics, by_type: &HashMap<String, Metrics>, profile: &DatasetProfile) {
println!("=================================================================");
println!(" LongMemEval Benchmark (BM25-only retrieval, zero embeddings)");
println!("=================================================================");
@@ -532,7 +528,10 @@ fn print_report(
" \"evidence_session_density\": {:.4},",
profile.evidence_density
);
println!(" \"mean_sessions_per_question\": {:.2},", profile.mean_sessions);
println!(
" \"mean_sessions_per_question\": {:.2},",
profile.mean_sessions
);
println!(" \"mean_turns_per_question\": {:.1},", profile.mean_turns);
println!(
" \"total_questions\": {},",
@@ -556,10 +555,16 @@ fn print_report(
overall.mrr_turn()
);
println!(" }},");
println!(
" \"abstention_accuracy\": {:.4},",
overall.abstention_pct() / 100.0
);
// `null`, not 0.0 — a corpus with no abstention questions has no abstention
// accuracy, and emitting 0.0 reads as total failure at a task never posed.
if overall.abstention_total > 0 {
println!(
" \"abstention_accuracy\": {:.4},",
overall.abstention_pct() / 100.0
);
} else {
println!(" \"abstention_accuracy\": null,");
}
println!(" \"latency_us\": {{");
println!(
" \"avg\": {:.1}, \"p50\": {:.1}, \"p95\": {:.1}, \"p99\": {:.1}",