feat(decide): cm-decide — typed calibrated decisions; Jev + local NLI backends; skill triage in shadow
A third kind of decision-maker between deterministic code and a full LLM call: Choice / Score / Noul questions answered as probability distributions with a confidence, behind one Decider trait, with the composition patterns (confidence gating, composite scoring, rerank) as code. Two backends: TypeSafe's Jev over HTTP, and a DeBERTa-v3 MNLI cross-encoder run in-process with candle (feature nli; metal/cuda). decide-eval measures a backend on labelled cases the way judge-eval measures the judge. eval/skill-triage.json: 20 mission tasks × 53 skills, 75 positives, hand-labelled. Measured 2026-09-21: lexical overlap AUROC 0.851 [email protected] 0.47 top-k 48/75 ECE 0.095 jev (named wording) AUROC 0.989 [email protected] 0.84 top-k 63/75 ECE 0.064 213 ms jev (plain wording) AUROC 0.970 [email protected] 0.66 top-k 52/75 nli mnli-base AUROC 0.790 [email protected] 0.28 top-k 38/75 ECE 0.263 1.5 s nli zeroshot-v2 AUROC 0.782 [email protected] 0.43 top-k 39/75 ECE 0.054 1.2 s The vendor's calibration claim survives our data; the local cross-encoder ranks below keyword overlap on either checkpoint or wording and is kept as the measured negative, not shipped. A local backend would need the logit-readout route over the fleet's 9B model — a separate spike. Shadow: one Jev call per phase launch (spawned, 10 s cap, silent without TYPESAFE_API_KEY) records a skill.triage event; the Skill-Use report carries triage_p beside each skill's Trigger verdict. It selects nothing. Co-Authored-By: Claude Opus 5 <[email protected]> Claude-Session: https://claude.ai/code/session_01WZb5A2kfVfjpdwSochkuHz
This commit is contained in:
Omar Sobh
co-authored by
Claude Opus 5
Claude Opus 5
parent
650a556029
commit
0a2bd6f868
@@ -0,0 +1,348 @@
|
||||
//! Measure a decision backend against labelled cases.
|
||||
//!
|
||||
//! The judge got an eval before it got trusted (`scripts/judge-eval.sh`);
|
||||
//! a triage model gets the same. Every backend answers the SAME Noul per
|
||||
//! (task, skill) pair, and is scored on ranking (AUROC), on the operating
|
||||
//! point (best-F1 threshold and F1 at 0.5), and on calibration (Brier, ECE)
|
||||
//! — because a calibrated middle band is the whole reason to have this tier,
|
||||
//! and a backend that ranks well but says 0.9 to everything has none.
|
||||
//!
|
||||
//! decide-eval [--backend jev|nli|lexical|all] [--wording named|plain]
|
||||
//! [--set path] [--skills dir] [--dump dir]
|
||||
//!
|
||||
//! `--dump` writes every pair's probability so a temperature can be fitted
|
||||
//! offline without re-running the model.
|
||||
|
||||
use std::collections::BTreeMap;
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::time::Instant;
|
||||
|
||||
use cm_decide::{Answer, DecideError, Decider, Decision, Question};
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
#[derive(Deserialize)]
|
||||
struct EvalSet {
|
||||
cases: Vec<Case>,
|
||||
}
|
||||
|
||||
#[derive(Deserialize, Clone)]
|
||||
struct Case {
|
||||
id: String,
|
||||
task: String,
|
||||
positives: Vec<String>,
|
||||
}
|
||||
|
||||
#[derive(Clone)]
|
||||
struct Skill {
|
||||
name: String,
|
||||
when_to_use: String,
|
||||
}
|
||||
|
||||
/// The question a backend gets. `--wording` picks the template; a backend
|
||||
/// is reported under the wording it was run with, and each backend ships
|
||||
/// with the wording that measured best FOR IT — a prompt is part of the
|
||||
/// backend, and an NLI cross-encoder and a decision model do not want the
|
||||
/// same sentence. Both are run on both so the table says so.
|
||||
fn question_for(skill: &Skill, wording: &str) -> Question {
|
||||
let text = match wording {
|
||||
// The shipped question, shared with the server's shadow path.
|
||||
"named" => return cm_decide::triage::question(&skill.name, &skill.when_to_use),
|
||||
// The when_to_use line alone, second person rewritten to the agent,
|
||||
// as a plain declarative the NLI head was trained on.
|
||||
"plain" => third_person(&skill.when_to_use),
|
||||
other => panic!("unknown wording {other}"),
|
||||
};
|
||||
Question::noul(text)
|
||||
}
|
||||
|
||||
/// "You're the tester on a mobile team" → "The agent is the tester on a
|
||||
/// mobile team". Crude on purpose: it is a rewrite of a dozen fixed
|
||||
/// openings, not a grammar, and it exists so the NLI hypothesis reads like
|
||||
/// an MNLI hypothesis.
|
||||
fn third_person(when: &str) -> String {
|
||||
let w = when.trim();
|
||||
let rules: &[(&str, &str)] = &[
|
||||
("You're the ", "The agent is the "),
|
||||
("You are the ", "The agent is the "),
|
||||
("You're a ", "The agent is a "),
|
||||
("You are a ", "The agent is a "),
|
||||
("You're on a ", "The agent is on a "),
|
||||
("You are on a ", "The agent is on a "),
|
||||
("You're ", "The agent is "),
|
||||
("You are ", "The agent is "),
|
||||
("You need ", "The agent needs "),
|
||||
("You have ", "The agent has "),
|
||||
];
|
||||
for (from, to) in rules {
|
||||
if let Some(rest) = w.strip_prefix(from) {
|
||||
return format!("{to}{rest}");
|
||||
}
|
||||
}
|
||||
format!("In this task, {}", w.trim_end_matches('.').to_string() + ".")
|
||||
}
|
||||
|
||||
fn load_skills(dir: &Path) -> Vec<Skill> {
|
||||
let mut out = Vec::new();
|
||||
fn walk(dir: &Path, out: &mut Vec<Skill>) {
|
||||
let Ok(rd) = std::fs::read_dir(dir) else { return };
|
||||
let mut entries: Vec<_> = rd.flatten().collect();
|
||||
entries.sort_by_key(|e| e.path());
|
||||
for e in entries {
|
||||
let p = e.path();
|
||||
if p.is_dir() {
|
||||
walk(&p, out);
|
||||
} else if p.extension().is_some_and(|x| x == "md") {
|
||||
let Ok(text) = std::fs::read_to_string(&p) else { continue };
|
||||
let field = |k: &str| {
|
||||
text.lines()
|
||||
.find_map(|l| l.strip_prefix(k).map(|v| v.trim().trim_matches('"').to_string()))
|
||||
};
|
||||
if let (Some(name), Some(when)) = (field("name:"), field("when_to_use:")) {
|
||||
out.push(Skill { name, when_to_use: when });
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
walk(dir, &mut out);
|
||||
out
|
||||
}
|
||||
|
||||
/// Keyword overlap between the task and the skill's name + when_to_use:
|
||||
/// the bar a model has to clear to be worth a network call.
|
||||
struct Lexical;
|
||||
|
||||
fn tokens(s: &str) -> std::collections::BTreeSet<String> {
|
||||
s.to_ascii_lowercase()
|
||||
.split(|c: char| !c.is_alphanumeric())
|
||||
.filter(|w| w.len() > 3)
|
||||
.map(str::to_string)
|
||||
.collect()
|
||||
}
|
||||
|
||||
#[async_trait::async_trait]
|
||||
impl Decider for Lexical {
|
||||
fn name(&self) -> &str {
|
||||
"lexical"
|
||||
}
|
||||
async fn decide(
|
||||
&self,
|
||||
state: &str,
|
||||
questions: &BTreeMap<String, Question>,
|
||||
) -> Result<Decision, DecideError> {
|
||||
let started = Instant::now();
|
||||
let st = tokens(state);
|
||||
let answers = questions
|
||||
.iter()
|
||||
.map(|(id, q)| {
|
||||
let Question::Noul { instructions, .. } = q else { unreachable!() };
|
||||
let ht = tokens(instructions);
|
||||
let inter = st.intersection(&ht).count() as f64;
|
||||
let p = if ht.is_empty() { 0.0 } else { (inter / ht.len() as f64 * 4.0).min(1.0) };
|
||||
(id.clone(), Answer::Noul { noul: p })
|
||||
})
|
||||
.collect();
|
||||
Ok(Decision { model: "lexical".into(), answers, usage: None, latency: started.elapsed() })
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Serialize)]
|
||||
struct Pair {
|
||||
case: String,
|
||||
skill: String,
|
||||
label: bool,
|
||||
p: f64,
|
||||
}
|
||||
|
||||
#[derive(Default)]
|
||||
struct Report {
|
||||
pairs: Vec<Pair>,
|
||||
latency_ms: Vec<u128>,
|
||||
input_tokens: u64,
|
||||
per_case_topk_hits: Vec<(usize, usize)>,
|
||||
errors: usize,
|
||||
}
|
||||
|
||||
fn auroc(pairs: &[Pair]) -> f64 {
|
||||
// Mann–Whitney: fraction of (positive, negative) pairs ranked correctly.
|
||||
let pos: Vec<f64> = pairs.iter().filter(|p| p.label).map(|p| p.p).collect();
|
||||
let neg: Vec<f64> = pairs.iter().filter(|p| !p.label).map(|p| p.p).collect();
|
||||
if pos.is_empty() || neg.is_empty() {
|
||||
return f64::NAN;
|
||||
}
|
||||
let mut s = 0.0;
|
||||
for a in &pos {
|
||||
for b in &neg {
|
||||
s += if a > b { 1.0 } else if a == b { 0.5 } else { 0.0 };
|
||||
}
|
||||
}
|
||||
s / (pos.len() * neg.len()) as f64
|
||||
}
|
||||
|
||||
fn f1_at(pairs: &[Pair], t: f64) -> (f64, f64, f64) {
|
||||
let (mut tp, mut fp, mut fn_) = (0.0, 0.0, 0.0);
|
||||
for p in pairs {
|
||||
match (p.p >= t, p.label) {
|
||||
(true, true) => tp += 1.0,
|
||||
(true, false) => fp += 1.0,
|
||||
(false, true) => fn_ += 1.0,
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
let prec = if tp + fp > 0.0 { tp / (tp + fp) } else { 0.0 };
|
||||
let rec = if tp + fn_ > 0.0 { tp / (tp + fn_) } else { 0.0 };
|
||||
let f1 = if prec + rec > 0.0 { 2.0 * prec * rec / (prec + rec) } else { 0.0 };
|
||||
(f1, prec, rec)
|
||||
}
|
||||
|
||||
fn brier(pairs: &[Pair]) -> f64 {
|
||||
pairs.iter().map(|p| (p.p - if p.label { 1.0 } else { 0.0 }).powi(2)).sum::<f64>() / pairs.len() as f64
|
||||
}
|
||||
|
||||
/// Expected calibration error, ten equal-width bins.
|
||||
fn ece(pairs: &[Pair]) -> f64 {
|
||||
let mut bins = vec![(0usize, 0.0f64, 0.0f64); 10];
|
||||
for p in pairs {
|
||||
let b = ((p.p * 10.0).floor() as usize).min(9);
|
||||
bins[b].0 += 1;
|
||||
bins[b].1 += p.p;
|
||||
bins[b].2 += if p.label { 1.0 } else { 0.0 };
|
||||
}
|
||||
let n = pairs.len() as f64;
|
||||
bins.iter()
|
||||
.filter(|(c, _, _)| *c > 0)
|
||||
.map(|(c, sp, sy)| (*c as f64 / n) * ((sp / *c as f64) - (sy / *c as f64)).abs())
|
||||
.sum()
|
||||
}
|
||||
|
||||
async fn run(backend: &dyn Decider, cases: &[Case], skills: &[Skill], wording: &str) -> Report {
|
||||
let mut r = Report::default();
|
||||
let questions: BTreeMap<String, Question> =
|
||||
skills.iter().map(|s| (s.name.clone(), question_for(s, wording))).collect();
|
||||
for case in cases {
|
||||
let d = match backend.decide(&case.task, &questions).await {
|
||||
Ok(d) => d,
|
||||
Err(e) => {
|
||||
eprintln!(" {}: {} failed: {e}", backend.name(), case.id);
|
||||
r.errors += 1;
|
||||
continue;
|
||||
}
|
||||
};
|
||||
r.latency_ms.push(d.latency.as_millis());
|
||||
if let Some(u) = d.usage {
|
||||
r.input_tokens += u.input_tokens;
|
||||
}
|
||||
let mut ranked: Vec<(String, f64)> = Vec::new();
|
||||
for s in skills {
|
||||
let p = match d.answers.get(&s.name) {
|
||||
Some(Answer::Noul { noul }) => *noul,
|
||||
_ => 0.0,
|
||||
};
|
||||
let label = case.positives.iter().any(|x| x == &s.name);
|
||||
r.pairs.push(Pair { case: case.id.clone(), skill: s.name.clone(), label, p });
|
||||
ranked.push((s.name.clone(), p));
|
||||
}
|
||||
ranked.sort_by(|a, b| b.1.partial_cmp(&a.1).unwrap());
|
||||
let k = case.positives.len();
|
||||
let hits = ranked.iter().take(k).filter(|(n, _)| case.positives.contains(n)).count();
|
||||
r.per_case_topk_hits.push((hits, k));
|
||||
}
|
||||
r
|
||||
}
|
||||
|
||||
fn print_report(name: &str, r: &Report) {
|
||||
if r.pairs.is_empty() {
|
||||
println!("{name:<24} no results ({} errors)", r.errors);
|
||||
return;
|
||||
}
|
||||
let best = (5..=95)
|
||||
.map(|i| i as f64 / 100.0)
|
||||
.map(|t| (t, f1_at(&r.pairs, t)))
|
||||
.max_by(|a, b| a.1 .0.partial_cmp(&b.1 .0).unwrap())
|
||||
.unwrap();
|
||||
let (f1_half, p_half, r_half) = f1_at(&r.pairs, 0.5);
|
||||
let topk: (usize, usize) = r.per_case_topk_hits.iter().fold((0, 0), |a, b| (a.0 + b.0, a.1 + b.1));
|
||||
let lat = if r.latency_ms.is_empty() { 0 } else { r.latency_ms.iter().sum::<u128>() / r.latency_ms.len() as u128 };
|
||||
println!(
|
||||
"{name:<24} AUROC {:.3} [email protected] {:.2} (P {:.2} R {:.2}) bestF1 {:.2}@{:.2} top-k {}/{} Brier {:.3} ECE {:.3} {} ms/call {} tok errors {}",
|
||||
auroc(&r.pairs), f1_half, p_half, r_half, best.1 .0, best.0, topk.0, topk.1,
|
||||
brier(&r.pairs), ece(&r.pairs), lat, r.input_tokens, r.errors
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::main]
|
||||
async fn main() {
|
||||
let mut args = std::env::args().skip(1);
|
||||
let (mut backend, mut set, mut skills_dir, mut dump, mut wording) = (
|
||||
"all".to_string(),
|
||||
PathBuf::from("crates/cm-decide/eval/skill-triage.json"),
|
||||
PathBuf::from("skills"),
|
||||
None::<PathBuf>,
|
||||
"named".to_string(),
|
||||
);
|
||||
while let Some(a) = args.next() {
|
||||
match a.as_str() {
|
||||
"--backend" => backend = args.next().unwrap_or_default(),
|
||||
"--wording" => wording = args.next().unwrap_or_default(),
|
||||
"--set" => set = args.next().map(PathBuf::from).unwrap_or(set),
|
||||
"--skills" => skills_dir = args.next().map(PathBuf::from).unwrap_or(skills_dir),
|
||||
"--dump" => dump = args.next().map(PathBuf::from),
|
||||
other => {
|
||||
eprintln!("unknown arg {other}");
|
||||
std::process::exit(2);
|
||||
}
|
||||
}
|
||||
}
|
||||
let text = std::fs::read_to_string(&set).expect("read eval set");
|
||||
let set: EvalSet = serde_json::from_str(&text).expect("parse eval set");
|
||||
let skills = load_skills(&skills_dir);
|
||||
// Every positive must name a real skill, or the label is a typo scored
|
||||
// as a miss against every backend.
|
||||
for c in &set.cases {
|
||||
for p in &c.positives {
|
||||
assert!(skills.iter().any(|s| &s.name == p), "case {}: unknown skill {p:?}", c.id);
|
||||
}
|
||||
}
|
||||
println!(
|
||||
"{} cases × {} skills = {} pairs, {} positive\n",
|
||||
set.cases.len(),
|
||||
skills.len(),
|
||||
set.cases.len() * skills.len(),
|
||||
set.cases.iter().map(|c| c.positives.len()).sum::<usize>()
|
||||
);
|
||||
|
||||
let mut backends: Vec<Box<dyn Decider>> = Vec::new();
|
||||
if backend == "all" || backend == "lexical" {
|
||||
backends.push(Box::new(Lexical));
|
||||
}
|
||||
#[cfg(feature = "jev")]
|
||||
if backend == "all" || backend == "jev" {
|
||||
match cm_decide::jev::Jev::from_env() {
|
||||
Some(j) => backends.push(Box::new(j)),
|
||||
None => eprintln!("jev: TYPESAFE_API_KEY unset — skipped"),
|
||||
}
|
||||
}
|
||||
#[cfg(feature = "nli")]
|
||||
if backend == "all" || backend == "nli" {
|
||||
let t0 = Instant::now();
|
||||
match cm_decide::nli::Nli::load(cm_decide::nli::NliConfig::from_env()) {
|
||||
Ok(n) => {
|
||||
eprintln!("nli: loaded {} in {:?}", n.name(), t0.elapsed());
|
||||
backends.push(Box::new(n));
|
||||
}
|
||||
Err(e) => eprintln!("nli: could not load — {e}"),
|
||||
}
|
||||
}
|
||||
if backends.is_empty() {
|
||||
eprintln!("no backend to run");
|
||||
std::process::exit(2);
|
||||
}
|
||||
for b in &backends {
|
||||
let r = run(b.as_ref(), &set.cases, &skills, &wording).await;
|
||||
print_report(&format!("{} [{wording}]", b.name()), &r);
|
||||
if let Some(dir) = &dump {
|
||||
std::fs::create_dir_all(dir).ok();
|
||||
let path = dir.join(format!("{}-{wording}.json", b.name().replace([':', '/'], "-")));
|
||||
std::fs::write(&path, serde_json::to_string_pretty(&r.pairs).unwrap()).ok();
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user