feat(judge): Kimi as a fallback independent judge when GLM cannot answer

GLM's plan limit ran out for the second time in a month, and with one judge
every conditioned phase on every mission fails until it resets. When the
primary independent judge errors, CLAWMATES_VALIDATOR_FALLBACK_MODEL (prod:
kimi:kimi-for-coding) judges instead, through the same independence checks
(one shared function) plus one more: never the primary's own family.

Measured first: judge-eval 15 cases x 3, kimi-for-coding 44/45 vs glm-5.3
43/45; goodhart — the false positive that once ruled Kimi out — 3/3.

When both fail, the primary's error leads so the phase runner still reads the
z.ai plan-limit code and does not spend the pass. Prod's kimi provider moved
to the anthropic format the eval used (host config, backed up).

Co-Authored-By: Claude Opus 5.5 (1M context) <[email protected]>

crates/cm-api/src/evaluator.rs

@@ -494,7 +494,55 @@ async fn cross_provider_judge(
         per_mission.as_deref(),
         std::env::var("CLAWMATES_VALIDATOR_MODEL").ok().as_deref(),
     )?;
-    let spec = spec.as_str();
+    independent_judge(runtime, &spec, implementer)
+}
+
+/// The second independent judge, used only when the first could not answer.
+///
+/// `CLAWMATES_VALIDATOR_FALLBACK_MODEL`, e.g. `kimi:kimi-for-coding`. Added
+/// 2026-09-23 after GLM's plan limit ran out for the second time in a month:
+/// with one judge, every conditioned phase on every mission fails on the judge
+/// until the quota resets (days). Held to every check the primary is, plus one
+/// more: a different family from the primary as well, or it is the same outage
+/// twice.
+///
+/// Measured before it was wired: `scripts/judge-eval.sh`, 15 cases x 3 draws,
+/// kimi-for-coding 44/45 against glm-5.3's 43/45. `goodhart`, the false
+/// positive that once ruled Kimi out, was 3/3. The miss was one draw of
+/// `should-panic-hack`, the shape GLM also misses.
+async fn fallback_judge(
+    runtime: &cm_runtime::Runtime,
+    implementer: &str,
+    primary_model_family: &str,
+) -> Option<(std::sync::Arc<dyn cm_llm::LlmProvider>, String)> {
+    let spec = fallback_spec(
+        std::env::var("CLAWMATES_VALIDATOR_FALLBACK_MODEL").ok().as_deref(),
+        primary_model_family,
+    )?;
+    independent_judge(runtime, &spec, implementer)
+}
+
+/// The fallback spec, if one is set and it is not the primary's own family.
+fn fallback_spec(env: Option<&str>, primary_family: &str) -> Option<String> {
+    let spec = env.map(str::trim).filter(|s| !s.is_empty())?;
+    if provider_family(spec) == primary_family {
+        eprintln!(
+            "evaluator: CLAWMATES_VALIDATOR_FALLBACK_MODEL={spec} is the primary judge's own \
+             family ({primary_family}) — a quota or outage takes both down; ignoring it"
+        );
+        return None;
+    }
+    Some(spec.to_string())
+}
+
+/// Resolve `spec` to a judge that is genuinely independent of `implementer`,
+/// or `None` with the reason logged. Shared by the primary and the fallback so
+/// neither can be held to a weaker standard than the other.
+fn independent_judge(
+    runtime: &cm_runtime::Runtime,
+    spec: &str,
+    implementer: &str,
+) -> Option<(std::sync::Arc<dyn cm_llm::LlmProvider>, String)> {
     let family = provider_family(spec);
     if family == implementer {
         eprintln!(
@@ -670,8 +718,67 @@ pub async fn evaluate(
                 // not have. The phase stays unmet this pass and says why; the next
                 // sweep retries.
                 Err(e) => {
+                    // A SECOND independent family, when one is configured. Still
+                    // never the agent's own: `fallback_judge` applies the same
+                    // checks as the primary.
+                    let primary_family = provider_family(&model);
+                    let primary_family = if primary_family == "unknown" {
+                        std::env::var("CLAWMATES_VALIDATOR_MODEL")
+                            .map(|s| provider_family(&s))
+                            .unwrap_or(primary_family)
+                    } else {
+                        primary_family
+                    };
+                    if let Some((fb, fb_model)) =
+                        fallback_judge(runtime, implementer, &primary_family).await
+                    {
                         eprintln!(
-                        "evaluator: the independent judge ({model}) failed — NOT falling back to                      the agent's own provider: {e}"
+                            "evaluator: the independent judge ({model}) failed ({}) — falling \
+                             back to {fb_model}, also independent of the agent",
+                            e.chars().take(160).collect::<String>()
+                        );
+                        let mut fb_usage = Usage::default();
+                        let fb_expectation =
+                            commit_expectation(fb.as_ref(), condition, &fb_model, &mut fb_usage).await;
+                        let fb_user = judge_user(condition, evidence, fb_expectation.as_deref());
+                        match judge_with_tools(
+                            fb.as_ref(),
+                            &system,
+                            &fb_user,
+                            &fb_model,
+                            sandbox.as_ref(),
+                            &mut fb_usage,
+                        )
+                        .await
+                        {
+                            Ok((text, checks)) => {
+                                let mut v = parse_verdict(&fb_model, &text);
+                                v.guidance = sanitize_guidance(condition, evidence, &v.guidance);
+                                v.checks = checks;
+                                v.independent = true;
+                                v.usage = fb_usage;
+                                v.expectation = fb_expectation;
+                                return v;
+                            }
+                            // Both down. The PRIMARY's error leads: the phase
+                            // runner reads it to tell a plan limit (do not
+                            // spend the pass) from a transient failure.
+                            Err(e2) => {
+                                eprintln!("evaluator: the fallback judge ({fb_model}) failed too: {e2}");
+                                let mut v = Verdict::not_met(
+                                    &model,
+                                    "neither independent validator could be reached this pass",
+                                    Some(format!("{e} | fallback {fb_model}: {e2}")),
+                                );
+                                v.usage = usage;
+                                v.expectation = expectation;
+                                return v;
+                            }
+                        }
+                    }
+                    eprintln!(
+                        "evaluator: the independent judge ({model}) failed — NOT falling back to \
+                         the agent's own provider: {e}"
                     );
                     let mut v = Verdict::not_met(
                         &model,
@@ -1652,3 +1759,51 @@ mod tests {
         assert!(head("abc", 200).ends_with('c'));
     }
 }
+
+#[cfg(test)]
+mod fallback_judge_tests {
+    use super::*;
+
+    #[test]
+    fn a_fallback_from_another_family_is_taken() {
+        assert_eq!(
+            fallback_spec(Some("kimi:kimi-for-coding"), "glm").as_deref(),
+            Some("kimi:kimi-for-coding")
+        );
+    }
+
+    /// The same family is the same outage: GLM's plan limit covers every GLM
+    /// model, so `glm:glm-4.7` behind `glm:glm-5.3` would fail in the same breath.
+    #[test]
+    fn a_fallback_from_the_primarys_family_is_refused() {
+        assert_eq!(fallback_spec(Some("glm:glm-4.7"), "glm"), None);
+    }
+
+    #[test]
+    fn no_fallback_configured_means_none() {
+        assert_eq!(fallback_spec(None, "glm"), None);
+        assert_eq!(fallback_spec(Some("  "), "glm"), None);
+    }
+
+    /// The fallback goes through the SAME independence checks as the primary —
+    /// one function, so a Kimi fallback on a Kimi-implemented mission is refused
+    /// exactly as a Kimi primary would be.
+    #[test]
+    fn both_judges_share_one_independence_check() {
+        let src = include_str!("evaluator.rs");
+        let body = src.split("async fn fallback_judge(").nth(1).unwrap();
+        let body = &body[..body.find("\n}\n").unwrap()];
+        assert!(body.contains("independent_judge(runtime, &spec, implementer)"), "{body}");
+        let primary = src.split("async fn cross_provider_judge(").nth(1).unwrap();
+        let primary = &primary[..primary.find("\n}\n").unwrap()];
+        assert!(primary.contains("independent_judge(runtime, &spec, implementer)"));
+    }
+
+    /// When both fail, the primary's error leads — the phase runner reads it
+    /// for z.ai's plan-limit code to avoid spending the pass.
+    #[test]
+    fn when_both_fail_the_primary_error_leads() {
+        let src = include_str!("evaluator.rs");
+        assert!(src.contains(r#"Some(format!("{e} | fallback {fb_model}: {e2}"))"#));
+    }
+}

deploy/compose/docker-compose.override.yml

@@ -123,6 +123,11 @@ services:
       # so the shape is compatible — but the headroom is why the evaluator's
       # max_tokens was raised alongside this.
       CLAWMATES_VALIDATOR_MODEL: glm:glm-5.3
+      # Second independent judge, used only when GLM cannot answer (its plan
+      # limit ran out twice in a month). Needs the `kimi` provider in
+      # anthropic format (base_url https://api.kimi.com/coding). judge-eval:
+      # 44/45 vs glm-5.3's 43/45, goodhart 3/3.
+      CLAWMATES_VALIDATOR_FALLBACK_MODEL: kimi:kimi-for-coding
       CLAWMATES_JUDGE_MODEL: glm:glm-5.3
       # The §15 door is closed by default since 2026-09-20: with no governor
       # and no CLAWMATES_DOOR_POLICY=allow every outbound action is denied.

scripts/judge-eval.sh

@@ -47,6 +47,11 @@
 # a verification plan before reading the evidence (evaluator.rs), neither of
 # which this script exercises.
 #
+# MEASURED 2026-09-23, kimi-for-coding, three draws of all fifteen: 44/45.
+# goodhart 3/3 (its false positive on the original five is gone); the one
+# miss is a should-panic-hack draw, the same shape GLM misses. Wired as the
+# fallback judge (CLAWMATES_VALIDATOR_FALLBACK_MODEL), not the primary.
+#
 # Usage:
 #
 #   scripts/judge-eval.sh                      # the configured validator