//! Skill triage, in shadow: which of the visible skills a phase's task calls //! for, by a calibrated decision model, recorded beside what the agent then //! actually read. //! //! SRA-Bench (arXiv 2604.24594) found agents load skills at the same rate //! whether or not one applies — the bottleneck is knowing WHEN, and the //! agent's only signal today is the `when_to_use` line in its own prompt. A //! host-side oracle that answers the same question in 200 ms is the thing //! to measure against that. `cm_decide::jev` scored AUROC 0.989 on the //! labelled set (`crates/cm-decide/eval`); this records its answer per phase //! as a `skill.triage` event and the Skill-Use scorer reads it back next to //! the agent's Trigger. It selects nothing: the files arm still installs //! every visible skill. Promotion to a real selector is a later, measured //! step, once the agreement numbers from real missions say what the //! oracle's misses cost. //! //! One call per phase launch, spawned so the launch never waits on it, and //! silent when `TYPESAFE_API_KEY` is unset. The key never leaves the server. use std::collections::BTreeMap; use cm_decide::{Answer, Decider}; use sqlx::PgPool; use uuid::Uuid; pub const EVENT: &str = "skill.triage"; /// How long a shadow decision may take before it is dropped. Jev measures /// ~200 ms; a backend that takes ten seconds is not the one to shadow. const TIMEOUT: std::time::Duration = std::time::Duration::from_secs(10); /// Fire the triage for one phase and record it. Best-effort throughout: a /// missing key, a failed call, or a timeout leaves no event and one log line. pub fn spawn(pool: PgPool, mission_id: Uuid, phase_id: Uuid, workspace_id: Uuid, task: String) { let Some(jev) = cm_decide::jev::Jev::from_env() else { return; }; tokio::spawn(async move { let skills = match cm_db::repo::skills_catalog::list_visible(&pool, workspace_id).await { Ok(s) => s, Err(e) => { eprintln!("skill_triage: could not list skills for {mission_id}: {e}"); return; } }; let questions: BTreeMap = skills .iter() .map(|s| { ( s.name.clone(), cm_decide::triage::question(&s.name, s.when_to_use.as_deref().unwrap_or(&s.description)), ) }) .collect(); if questions.is_empty() { return; } let decision = match tokio::time::timeout(TIMEOUT, jev.decide(&task, &questions)).await { Ok(Ok(d)) => d, Ok(Err(e)) => { eprintln!("skill_triage: {} failed for phase {phase_id}: {e}", jev.name()); return; } Err(_) => { eprintln!("skill_triage: {} timed out for phase {phase_id}", jev.name()); return; } }; let probabilities: BTreeMap<&str, f64> = decision .answers .iter() .filter_map(|(k, a)| match a { Answer::Noul { noul } => Some((k.as_str(), *noul)), _ => None, }) .collect(); let applies = probabilities .iter() .filter(|(_, p)| **p >= cm_decide::triage::APPLIES_AT) .count(); eprintln!( "skill_triage: phase {phase_id} — {} says {applies} of {} skills apply ({} ms, {} tokens)", decision.model, probabilities.len(), decision.latency.as_millis(), decision.usage.map(|u| u.input_tokens).unwrap_or(0), ); crate::mission_events::record( &pool, crate::mission_events::MissionEvent::new(mission_id, EVENT) .phase(phase_id) .detail(serde_json::json!({ "backend": jev.name(), "model": decision.model, "wording": cm_decide::triage::WORDING, "latency_ms": decision.latency.as_millis() as u64, "input_tokens": decision.usage.map(|u| u.input_tokens), "applies_at": cm_decide::triage::APPLIES_AT, "skills": probabilities, })), ) .await; }); } /// The recorded triage for a mission: skill → highest probability any phase /// gave it. Empty when no event was recorded (no key, or before this existed). pub async fn recorded(pool: &PgPool, mission_id: Uuid) -> BTreeMap { let rows: Vec<(serde_json::Value,)> = sqlx::query_as( "SELECT detail FROM mission_events WHERE mission_id = $1 AND kind = $2 ORDER BY id", ) .bind(mission_id) .bind(EVENT) .fetch_all(pool) .await .unwrap_or_default(); let mut out: BTreeMap = BTreeMap::new(); for (detail,) in rows { if let Some(map) = detail.get("skills").and_then(|s| s.as_object()) { for (name, p) in map { if let Some(p) = p.as_f64() { let e = out.entry(name.clone()).or_insert(0.0); if p > *e { *e = p; } } } } } out }