diff --git a/crates/cm-api/src/evaluator.rs b/crates/cm-api/src/evaluator.rs index f74c48c..ebdfd77 100644 --- a/crates/cm-api/src/evaluator.rs +++ b/crates/cm-api/src/evaluator.rs @@ -494,7 +494,55 @@ async fn cross_provider_judge( per_mission.as_deref(), std::env::var("CLAWMATES_VALIDATOR_MODEL").ok().as_deref(), )?; - let spec = spec.as_str(); + independent_judge(runtime, &spec, implementer) +} + +/// The second independent judge, used only when the first could not answer. +/// +/// `CLAWMATES_VALIDATOR_FALLBACK_MODEL`, e.g. `kimi:kimi-for-coding`. Added +/// 2026-09-23 after GLM's plan limit ran out for the second time in a month: +/// with one judge, every conditioned phase on every mission fails on the judge +/// until the quota resets (days). Held to every check the primary is, plus one +/// more: a different family from the primary as well, or it is the same outage +/// twice. +/// +/// Measured before it was wired: `scripts/judge-eval.sh`, 15 cases x 3 draws, +/// kimi-for-coding 44/45 against glm-5.3's 43/45. `goodhart`, the false +/// positive that once ruled Kimi out, was 3/3. The miss was one draw of +/// `should-panic-hack`, the shape GLM also misses. +async fn fallback_judge( + runtime: &cm_runtime::Runtime, + implementer: &str, + primary_model_family: &str, +) -> Option<(std::sync::Arc, String)> { + let spec = fallback_spec( + std::env::var("CLAWMATES_VALIDATOR_FALLBACK_MODEL").ok().as_deref(), + primary_model_family, + )?; + independent_judge(runtime, &spec, implementer) +} + +/// The fallback spec, if one is set and it is not the primary's own family. +fn fallback_spec(env: Option<&str>, primary_family: &str) -> Option { + let spec = env.map(str::trim).filter(|s| !s.is_empty())?; + if provider_family(spec) == primary_family { + eprintln!( + "evaluator: CLAWMATES_VALIDATOR_FALLBACK_MODEL={spec} is the primary judge's own \ + family ({primary_family}) — a quota or outage takes both down; ignoring it" + ); + return None; + } + Some(spec.to_string()) +} + +/// Resolve `spec` to a judge that is genuinely independent of `implementer`, +/// or `None` with the reason logged. Shared by the primary and the fallback so +/// neither can be held to a weaker standard than the other. +fn independent_judge( + runtime: &cm_runtime::Runtime, + spec: &str, + implementer: &str, +) -> Option<(std::sync::Arc, String)> { let family = provider_family(spec); if family == implementer { eprintln!( @@ -670,8 +718,67 @@ pub async fn evaluate( // not have. The phase stays unmet this pass and says why; the next // sweep retries. Err(e) => { + // A SECOND independent family, when one is configured. Still + // never the agent's own: `fallback_judge` applies the same + // checks as the primary. + let primary_family = provider_family(&model); + let primary_family = if primary_family == "unknown" { + std::env::var("CLAWMATES_VALIDATOR_MODEL") + .map(|s| provider_family(&s)) + .unwrap_or(primary_family) + } else { + primary_family + }; + if let Some((fb, fb_model)) = + fallback_judge(runtime, implementer, &primary_family).await + { + eprintln!( + "evaluator: the independent judge ({model}) failed ({}) — falling \ + back to {fb_model}, also independent of the agent", + e.chars().take(160).collect::() + ); + let mut fb_usage = Usage::default(); + let fb_expectation = + commit_expectation(fb.as_ref(), condition, &fb_model, &mut fb_usage).await; + let fb_user = judge_user(condition, evidence, fb_expectation.as_deref()); + match judge_with_tools( + fb.as_ref(), + &system, + &fb_user, + &fb_model, + sandbox.as_ref(), + &mut fb_usage, + ) + .await + { + Ok((text, checks)) => { + let mut v = parse_verdict(&fb_model, &text); + v.guidance = sanitize_guidance(condition, evidence, &v.guidance); + v.checks = checks; + v.independent = true; + v.usage = fb_usage; + v.expectation = fb_expectation; + return v; + } + // Both down. The PRIMARY's error leads: the phase + // runner reads it to tell a plan limit (do not + // spend the pass) from a transient failure. + Err(e2) => { + eprintln!("evaluator: the fallback judge ({fb_model}) failed too: {e2}"); + let mut v = Verdict::not_met( + &model, + "neither independent validator could be reached this pass", + Some(format!("{e} | fallback {fb_model}: {e2}")), + ); + v.usage = usage; + v.expectation = expectation; + return v; + } + } + } eprintln!( - "evaluator: the independent judge ({model}) failed — NOT falling back to the agent's own provider: {e}" + "evaluator: the independent judge ({model}) failed — NOT falling back to \ + the agent's own provider: {e}" ); let mut v = Verdict::not_met( &model, @@ -1652,3 +1759,51 @@ mod tests { assert!(head("abc", 200).ends_with('c')); } } + +#[cfg(test)] +mod fallback_judge_tests { + use super::*; + + #[test] + fn a_fallback_from_another_family_is_taken() { + assert_eq!( + fallback_spec(Some("kimi:kimi-for-coding"), "glm").as_deref(), + Some("kimi:kimi-for-coding") + ); + } + + /// The same family is the same outage: GLM's plan limit covers every GLM + /// model, so `glm:glm-4.7` behind `glm:glm-5.3` would fail in the same breath. + #[test] + fn a_fallback_from_the_primarys_family_is_refused() { + assert_eq!(fallback_spec(Some("glm:glm-4.7"), "glm"), None); + } + + #[test] + fn no_fallback_configured_means_none() { + assert_eq!(fallback_spec(None, "glm"), None); + assert_eq!(fallback_spec(Some(" "), "glm"), None); + } + + /// The fallback goes through the SAME independence checks as the primary — + /// one function, so a Kimi fallback on a Kimi-implemented mission is refused + /// exactly as a Kimi primary would be. + #[test] + fn both_judges_share_one_independence_check() { + let src = include_str!("evaluator.rs"); + let body = src.split("async fn fallback_judge(").nth(1).unwrap(); + let body = &body[..body.find("\n}\n").unwrap()]; + assert!(body.contains("independent_judge(runtime, &spec, implementer)"), "{body}"); + let primary = src.split("async fn cross_provider_judge(").nth(1).unwrap(); + let primary = &primary[..primary.find("\n}\n").unwrap()]; + assert!(primary.contains("independent_judge(runtime, &spec, implementer)")); + } + + /// When both fail, the primary's error leads — the phase runner reads it + /// for z.ai's plan-limit code to avoid spending the pass. + #[test] + fn when_both_fail_the_primary_error_leads() { + let src = include_str!("evaluator.rs"); + assert!(src.contains(r#"Some(format!("{e} | fallback {fb_model}: {e2}"))"#)); + } +} diff --git a/deploy/compose/docker-compose.override.yml b/deploy/compose/docker-compose.override.yml index 9288bc6..801572f 100644 --- a/deploy/compose/docker-compose.override.yml +++ b/deploy/compose/docker-compose.override.yml @@ -123,6 +123,11 @@ services: # so the shape is compatible — but the headroom is why the evaluator's # max_tokens was raised alongside this. CLAWMATES_VALIDATOR_MODEL: glm:glm-5.3 + # Second independent judge, used only when GLM cannot answer (its plan + # limit ran out twice in a month). Needs the `kimi` provider in + # anthropic format (base_url https://api.kimi.com/coding). judge-eval: + # 44/45 vs glm-5.3's 43/45, goodhart 3/3. + CLAWMATES_VALIDATOR_FALLBACK_MODEL: kimi:kimi-for-coding CLAWMATES_JUDGE_MODEL: glm:glm-5.3 # The §15 door is closed by default since 2026-09-20: with no governor # and no CLAWMATES_DOOR_POLICY=allow every outbound action is denied. diff --git a/scripts/judge-eval.sh b/scripts/judge-eval.sh index 3526ee7..621da14 100755 --- a/scripts/judge-eval.sh +++ b/scripts/judge-eval.sh @@ -47,6 +47,11 @@ # a verification plan before reading the evidence (evaluator.rs), neither of # which this script exercises. # +# MEASURED 2026-09-23, kimi-for-coding, three draws of all fifteen: 44/45. +# goodhart 3/3 (its false positive on the original five is gone); the one +# miss is a should-panic-hack draw, the same shape GLM misses. Wired as the +# fallback judge (CLAWMATES_VALIDATOR_FALLBACK_MODEL), not the primary. +# # Usage: # # scripts/judge-eval.sh # the configured validator