feat(topology): LLM-judge scorer + async Scorer trait
Make Scorer async and add JudgeScorer (behind the `provider` feature): asks a cm-llm model to rate a run's output 0-100 vs the task and normalizes to [0,1], giving the comparison harness real quality numbers. Robust integer parsing (handles "Score: 92/100", clamps >100); provider errors score 0.0. 11 tests with --features provider (judge incl. parse + scripted-provider score); core stays 7. Clippy clean. Co-Authored-By: Claude Opus 4.8 <[email protected]>
This commit is contained in:
co-authored by
Claude Opus 4.8
parent
7370adb78e
commit
3725c96e4f
@@ -13,10 +13,12 @@ use cm_topology::{TopologyGraph, TopologyKind};
|
||||
use crate::{execute, OrchestratorError, RunRecord, TurnExecutor};
|
||||
|
||||
/// Scores the quality of a run's final output in `[0,1]`. Real deployments use
|
||||
/// an LLM judge; tests/benchmarks can use a deterministic scorer.
|
||||
/// an LLM judge ([`crate::JudgeScorer`]); tests/benchmarks can use a
|
||||
/// deterministic scorer.
|
||||
#[allow(async_fn_in_trait)]
|
||||
pub trait Scorer {
|
||||
/// Quality of `record` for `task`, in `[0,1]` (higher is better).
|
||||
fn score(&self, task: &str, record: &RunRecord) -> f64;
|
||||
async fn score(&self, task: &str, record: &RunRecord) -> f64;
|
||||
}
|
||||
|
||||
/// One topology's result within a comparison.
|
||||
@@ -72,7 +74,7 @@ pub async fn compare<E: TurnExecutor, S: Scorer>(
|
||||
let mut results: Vec<TopologyResult> = Vec::with_capacity(graphs.len());
|
||||
for g in graphs {
|
||||
let rec = execute(g, task, executor).await?;
|
||||
let quality = scorer.score(task, &rec).clamp(0.0, 1.0);
|
||||
let quality = scorer.score(task, &rec).await.clamp(0.0, 1.0);
|
||||
results.push(TopologyResult {
|
||||
kind: rec.kind,
|
||||
quality,
|
||||
@@ -152,7 +154,7 @@ mod tests {
|
||||
/// Deterministic quality proxy: longer (richer) output scores higher.
|
||||
struct LengthScorer;
|
||||
impl Scorer for LengthScorer {
|
||||
fn score(&self, _task: &str, record: &RunRecord) -> f64 {
|
||||
async fn score(&self, _task: &str, record: &RunRecord) -> f64 {
|
||||
(record.final_output.len() as f64 / 200.0).min(1.0)
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user