//! Can the independent validator actually be reached? //! //! The sibling of [`crate::runtime_preflight`], for the same class of failure: //! the code is right and the machine is not, and nothing says so until a mission //! pays for it. //! //! `evaluator::cross_provider_judge` deliberately refuses to fall back to the //! agent's own provider — a verdict from the same family is not an independent //! check, and quietly producing one would claim a property the verdict does not //! have. That refusal is correct, and its cost is that a dead validator makes //! every `done_when` phase UNMEETABLE. The mission still boots a VM, still runs //! an agent turn, still collects and delivers, and only then records //! "the independent validator could not be reached this pass" on one evaluation //! row. //! //! That happened: the z.ai credential expired mid-session and the first symptom //! was a two-phase mission failing after both VMs had run. The information //! existed the whole time; nobody was told until it was expensive. //! //! A report, not a gate — the same stance `runtime_preflight` takes. The server //! must still boot with a broken validator, because refusing to start would turn //! a degraded deployment into a dead one, and because a mission that opts out //! (`validator_model = ''`) is unaffected. What this buys is that the degradation //! is visible at startup instead of inferred from a failed mission. /// The smallest question that proves a credential works end to end. /// /// A real completion rather than a models-list or a HEAD: an expired key, a /// revoked key and a key with no quota can all pass a cheaper check and fail the /// call that matters. Two tokens of output. const PROBE_PROMPT: &str = "Reply with exactly: OK"; /// What the probe found. #[derive(Debug, PartialEq, Eq)] pub enum Verdict { /// No independent validator is configured; phases are judged by the house /// model. Not a fault — a deployment may choose this. NotConfigured, /// Configured, resolved, and it answered. Reachable { spec: String }, /// Configured but the registry has no such provider, so /// `cross_provider_judge` will refuse it rather than judge with the default. Unregistered { spec: String }, /// Configured and resolved, and the call failed. Unreachable { spec: String, error: String }, } impl Verdict { /// Is every `done_when` phase currently unmeetable because of this? pub fn breaks_gated_phases(&self) -> bool { matches!( self, Verdict::Unregistered { .. } | Verdict::Unreachable { .. } ) } } /// Ask the configured independent validator to answer one trivial question. pub async fn probe(runtime: &cm_runtime::Runtime) -> Verdict { let Some(spec) = std::env::var("CLAWMATES_VALIDATOR_MODEL") .ok() .map(|s| s.trim().to_string()) .filter(|s| !s.is_empty()) else { return Verdict::NotConfigured; }; let (_provider, model) = runtime.resolve_provider(&spec); // `resolve_provider` falls back to the DEFAULT provider for an unknown name, // and the fallback is detectable because the returned model still carries the // `name:` prefix. Checked here for the same reason the evaluator checks it: // a validator that is silently the house model is worse than none. if model.contains(':') { return Verdict::Unregistered { spec }; } // Through `Runtime::complete`, which is the same resolve-then-stream path // the evaluator's judge takes. A probe that dialled the provider its own way // could pass while the real call fails. match runtime.complete("", PROBE_PROMPT, &spec, 16, false).await { Ok(_) => Verdict::Reachable { spec }, Err(e) => Verdict::Unreachable { spec, error: e.chars().take(160).collect(), }, } } /// Probe at boot and say plainly what it means for missions. pub fn report_at_boot(runtime: cm_runtime::Runtime) { tokio::spawn(async move { match probe(&runtime).await { Verdict::NotConfigured => eprintln!( "validator_preflight: no CLAWMATES_VALIDATOR_MODEL — phase verdicts are judged \ by the house model, which is NOT an independent check" ), Verdict::Reachable { spec } => { eprintln!("validator_preflight: independent validator {spec} answered") } Verdict::Unregistered { spec } => eprintln!( "validator_preflight: CLAWMATES_VALIDATOR_MODEL={spec} has no registered \ provider — the evaluator will refuse it rather than judge with the default, \ so EVERY phase with a done_when condition will fail as unmet. Register the \ provider, or set the mission's validator_model to '' to opt out." ), Verdict::Unreachable { spec, error } => eprintln!( "validator_preflight: independent validator {spec} is UNREACHABLE ({error}) — \ EVERY phase with a done_when condition will fail as unmet, after running its \ agent. Fix the credential, or set validator_model to '' per mission to judge \ with the house model." ), } }); } #[cfg(test)] mod tests { use super::*; /// The two states that make gated phases unmeetable, and the two that do /// not. This is the distinction the whole module exists to draw: "no /// validator configured" is a choice, "configured and broken" is a fault /// that silently fails every conditioned mission. #[test] fn only_a_configured_but_broken_validator_breaks_gated_phases() { assert!(!Verdict::NotConfigured.breaks_gated_phases()); assert!(!Verdict::Reachable { spec: "glm:glm-4.7".into() } .breaks_gated_phases()); assert!(Verdict::Unregistered { spec: "glm:glm-4.7".into() } .breaks_gated_phases()); assert!(Verdict::Unreachable { spec: "glm:glm-4.7".into(), error: "401".into() } .breaks_gated_phases()); } /// An unregistered provider is NOT reported as unreachable, and the /// difference is actionable: one is fixed by registering a provider, the /// other by fixing a credential. Collapsing them sends an operator to the /// wrong place. #[test] fn the_two_faults_are_distinguishable() { let a = Verdict::Unregistered { spec: "glm:glm-4.7".into(), }; let b = Verdict::Unreachable { spec: "glm:glm-4.7".into(), error: "401 Authentication Failed".into(), }; assert_ne!(a, b); } }