clawmates/crates/cm-api/src/skill_triage.rs

//! Skill triage, in shadow: which of the visible skills a phase's task calls
//! for, by a calibrated decision model, recorded beside what the agent then
//! actually read.
//!
//! SRA-Bench (arXiv 2604.24594) found agents load skills at the same rate
//! whether or not one applies — the bottleneck is knowing WHEN, and the
//! agent's only signal today is the `when_to_use` line in its own prompt. A
//! host-side oracle that answers the same question in 200 ms is the thing
//! to measure against that. `cm_decide::jev` scored AUROC 0.989 on the
//! labelled set (`crates/cm-decide/eval`); this records its answer per phase
//! as a `skill.triage` event and the Skill-Use scorer reads it back next to
//! the agent's Trigger. It selects nothing: the files arm still installs
//! every visible skill. Promotion to a real selector is a later, measured
//! step, once the agreement numbers from real missions say what the
//! oracle's misses cost.
//!
//! One call per phase launch, spawned so the launch never waits on it, and
//! silent when `TYPESAFE_API_KEY` is unset. The key never leaves the server.

use std::collections::BTreeMap;

use cm_decide::{Answer, Decider};
use sqlx::PgPool;
use uuid::Uuid;

pub const EVENT: &str = "skill.triage";

/// How long a shadow decision may take before it is dropped. Jev measures
/// ~200 ms; a backend that takes ten seconds is not the one to shadow.
const TIMEOUT: std::time::Duration = std::time::Duration::from_secs(10);

/// Fire the triage for one phase and record it. Best-effort throughout: a
/// missing key, a failed call, or a timeout leaves no event and one log line.
pub fn spawn(pool: PgPool, mission_id: Uuid, phase_id: Uuid, workspace_id: Uuid, task: String) {
    let Some(jev) = cm_decide::jev::Jev::from_env() else {
        return;
    };
    tokio::spawn(async move {
        let skills = match cm_db::repo::skills_catalog::list_visible(&pool, workspace_id).await {
            Ok(s) => s,
            Err(e) => {
                eprintln!("skill_triage: could not list skills for {mission_id}: {e}");
                return;
            }
        };
        let questions: BTreeMap<String, cm_decide::Question> = skills
            .iter()
            .map(|s| {
                (
                    s.name.clone(),
                    cm_decide::triage::question(&s.name, s.when_to_use.as_deref().unwrap_or(&s.description)),
                )
            })
            .collect();
        if questions.is_empty() {
            return;
        }
        let decision = match tokio::time::timeout(TIMEOUT, jev.decide(&task, &questions)).await {
            Ok(Ok(d)) => d,
            Ok(Err(e)) => {
                eprintln!("skill_triage: {} failed for phase {phase_id}: {e}", jev.name());
                return;
            }
            Err(_) => {
                eprintln!("skill_triage: {} timed out for phase {phase_id}", jev.name());
                return;
            }
        };
        let probabilities: BTreeMap<&str, f64> = decision
            .answers
            .iter()
            .filter_map(|(k, a)| match a {
                Answer::Noul { noul } => Some((k.as_str(), *noul)),
                _ => None,
            })
            .collect();
        let applies = probabilities
            .iter()
            .filter(|(_, p)| **p >= cm_decide::triage::APPLIES_AT)
            .count();
        eprintln!(
            "skill_triage: phase {phase_id} — {} says {applies} of {} skills apply ({} ms, {} tokens)",
            decision.model,
            probabilities.len(),
            decision.latency.as_millis(),
            decision.usage.map(|u| u.input_tokens).unwrap_or(0),
        );
        crate::mission_events::record(
            &pool,
            crate::mission_events::MissionEvent::new(mission_id, EVENT)
                .phase(phase_id)
                .detail(serde_json::json!({
                    "backend": jev.name(),
                    "model": decision.model,
                    "wording": cm_decide::triage::WORDING,
                    "latency_ms": decision.latency.as_millis() as u64,
                    "input_tokens": decision.usage.map(|u| u.input_tokens),
                    "applies_at": cm_decide::triage::APPLIES_AT,
                    "skills": probabilities,
                })),
        )
        .await;
    });
}

/// The recorded triage for a mission: skill → highest probability any phase
/// gave it. Empty when no event was recorded (no key, or before this existed).
pub async fn recorded(pool: &PgPool, mission_id: Uuid) -> BTreeMap<String, f64> {
    let rows: Vec<(serde_json::Value,)> = sqlx::query_as(
        "SELECT detail FROM mission_events WHERE mission_id = $1 AND kind = $2 ORDER BY id",
    )
    .bind(mission_id)
    .bind(EVENT)
    .fetch_all(pool)
    .await
    .unwrap_or_default();
    let mut out: BTreeMap<String, f64> = BTreeMap::new();
    for (detail,) in rows {
        if let Some(map) = detail.get("skills").and_then(|s| s.as_object()) {
            for (name, p) in map {
                if let Some(p) = p.as_f64() {
                    let e = out.entry(name.clone()).or_insert(0.0);
                    if p > *e {
                        *e = p;
                    }
                }
            }
        }
    }
    out
}