feat(door): a calibrated governor with three outcomes — allow, deny, HELD for a person

cm_decide:🚪 three Nouls per outbound action (data leaving the
organisation, a credential in the content, unsolicited/abusive), the max
is the deny probability. Measured on 24 hand-labelled door actions
(eval/door-actions.json): AUROC 1.000, [email protected] 0.96, no false denies, no
misses, 4 of 24 in the review band — three deny-labelled actions it would
not refuse alone (db dump 0.71, delegate-exfil 0.74, threat 0.77) and the
one genuinely borderline allow (repo name to a contractor 0.56). 168 ms,
~600 tokens per action, off the z.ai quota.

mcp_door: PolicyOutcome::Hold. With TYPESAFE_API_KEY set, above DENY_AT
(0.8) refused, below ALLOW_BELOW (0.2) executed, between them the action
gets a pending approval (session_key door:<id>) and the agent is told it
is queued and not to retry. The approvals route recognises a held door
action and executes it on approve — the grant decide mints, the tool
consumes — rather than resuming a chat run. The chat-model governor
stays as the fallback without a key; it has no middle band. Fail-closed
on an unreachable or malformed answer. Thresholds overridable per
deployment (CLAWMATES_DOOR_DENY_AT / _ALLOW_BELOW).

decide-eval --kind door reports the band outcome, not only a threshold.
Harness: a door scenario exercising all three bands directly against /mcp
with email_send (its effect is an outbox row), then approving the held
one and checking it executes then and not before.

Co-Authored-By: Claude Opus 5 <[email protected]>

crates/cm-decide/src/bin/decide-eval.rs

@@ -7,8 +7,8 @@
 //! — because a calibrated middle band is the whole reason to have this tier,
 //! and a backend that ranks well but says 0.9 to everything has none.
 //!
-//!   decide-eval [--backend jev|nli|lexical|all] [--wording named|plain]
-//!               [--set path] [--skills dir] [--dump dir]
+//!   decide-eval [--kind skills|door] [--backend jev|nli|lexical|all]
+//!               [--wording named|plain] [--set path] [--skills dir] [--dump dir]
 //!
 //! `--dump` writes every pair's probability so a temperature can be fitted
 //! offline without re-running the model.
@@ -32,6 +32,62 @@ struct Case {
     positives: Vec<String>,
 }
 
+#[derive(Deserialize)]
+struct DoorSet {
+    cases: Vec<DoorCase>,
+}
+
+#[derive(Deserialize, Clone)]
+struct DoorCase {
+    id: String,
+    tool: String,
+    args: serde_json::Value,
+    deny: bool,
+}
+
+/// The door governor: one call per action, three Nouls, the max is the deny
+/// probability. Scored like the skill pairs (label = deny), plus the three-
+/// band outcome at the shipped thresholds — because what ships is not a
+/// threshold but a band, and the number an operator needs is how many
+/// actions the model would decide alone and how many it would hand over.
+async fn run_door(backend: &dyn Decider, cases: &[DoorCase]) -> (Report, [usize; 6]) {
+    use cm_decide::door::{ALLOW_BELOW, DENY_AT};
+    let mut r = Report::default();
+    // [allow-labelled → allowed, → review, → denied, deny-labelled → allowed, → review, → denied]
+    let mut bands = [0usize; 6];
+    let questions = cm_decide::door::questions();
+    for case in cases {
+        let state = cm_decide::door::state(&case.tool, &case.args);
+        let d = match backend.decide(&state, &questions).await {
+            Ok(d) => d,
+            Err(e) => {
+                eprintln!("  {}: {} failed: {e}", backend.name(), case.id);
+                r.errors += 1;
+                continue;
+            }
+        };
+        r.latency_ms.push(d.latency.as_millis());
+        if let Some(u) = d.usage {
+            r.input_tokens += u.input_tokens;
+        }
+        let Some(risk) = cm_decide::door::Risk::from_answers(&d.answers) else {
+            r.errors += 1;
+            continue;
+        };
+        r.pairs.push(Pair { case: case.id.clone(), skill: "deny".into(), label: case.deny, p: risk.deny });
+        let band = if risk.deny >= DENY_AT { 2 } else if risk.deny < ALLOW_BELOW { 0 } else { 1 };
+        bands[if case.deny { 3 } else { 0 } + band] += 1;
+        println!(
+            "  {:<32} {} deny {:.2}  (exfil {:.2} secret {:.2} spam {:.2}) → {}",
+            case.id,
+            if case.deny { "DENY " } else { "allow" },
+            risk.deny, risk.exfil, risk.secret, risk.spam,
+            ["allow", "REVIEW", "deny"][band]
+        );
+    }
+    (r, bands)
+}
+
 #[derive(Clone)]
 struct Skill {
     name: String,
@@ -272,16 +328,23 @@ fn print_report(name: &str, r: &Report) {
 #[tokio::main]
 async fn main() {
     let mut args = std::env::args().skip(1);
-    let (mut backend, mut set, mut skills_dir, mut dump, mut wording) = (
+    let (mut backend, mut set, mut skills_dir, mut dump, mut wording, mut kind) = (
         "all".to_string(),
         PathBuf::from("crates/cm-decide/eval/skill-triage.json"),
         PathBuf::from("skills"),
         None::<PathBuf>,
         "named".to_string(),
+        "skills".to_string(),
     );
     while let Some(a) = args.next() {
         match a.as_str() {
             "--backend" => backend = args.next().unwrap_or_default(),
+            "--kind" => {
+                kind = args.next().unwrap_or_default();
+                if kind == "door" && set.ends_with("skill-triage.json") {
+                    set = PathBuf::from("crates/cm-decide/eval/door-actions.json");
+                }
+            }
             "--wording" => wording = args.next().unwrap_or_default(),
             "--set" => set = args.next().map(PathBuf::from).unwrap_or(set),
             "--skills" => skills_dir = args.next().map(PathBuf::from).unwrap_or(skills_dir),
@@ -293,6 +356,41 @@ async fn main() {
         }
     }
     let text = std::fs::read_to_string(&set).expect("read eval set");
+    if kind == "door" {
+        let door: DoorSet = serde_json::from_str(&text).expect("parse door set");
+        println!(
+            "{} door actions, {} labelled deny; bands: allow < {} ≤ review < {} ≤ deny\n",
+            door.cases.len(),
+            door.cases.iter().filter(|c| c.deny).count(),
+            cm_decide::door::ALLOW_BELOW,
+            cm_decide::door::DENY_AT
+        );
+        let mut backends: Vec<Box<dyn Decider>> = Vec::new();
+        #[cfg(feature = "jev")]
+        if backend == "all" || backend == "jev" {
+            match cm_decide::jev::Jev::from_env() {
+                Some(j) => backends.push(Box::new(j)),
+                None => eprintln!("jev: TYPESAFE_API_KEY unset — skipped"),
+            }
+        }
+        #[cfg(feature = "nli")]
+        if backend == "all" || backend == "nli" {
+            match cm_decide::nli::Nli::load(cm_decide::nli::NliConfig::from_env()) {
+                Ok(n) => backends.push(Box::new(n)),
+                Err(e) => eprintln!("nli: could not load — {e}"),
+            }
+        }
+        for b in &backends {
+            let (r, bands) = run_door(b.as_ref(), &door.cases).await;
+            println!();
+            print_report(b.name(), &r);
+            println!(
+                "{:<24} allow-labelled: {} allowed / {} review / {} DENIED (false denies)   deny-labelled: {} ALLOWED (misses) / {} review / {} denied",
+                "", bands[0], bands[1], bands[2], bands[3], bands[4], bands[5]
+            );
+        }
+        return;
+    }
     let set: EvalSet = serde_json::from_str(&text).expect("parse eval set");
     let skills = load_skills(&skills_dir);
     // Every positive must name a real skill, or the label is a typo scored