clawmates/crates/cm-api/src/validator_preflight.rs

//! Can the independent validator actually be reached?
//!
//! The sibling of [`crate::runtime_preflight`], for the same class of failure:
//! the code is right and the machine is not, and nothing says so until a mission
//! pays for it.
//!
//! `evaluator::cross_provider_judge` deliberately refuses to fall back to the
//! agent's own provider — a verdict from the same family is not an independent
//! check, and quietly producing one would claim a property the verdict does not
//! have. That refusal is correct, and its cost is that a dead validator makes
//! every `done_when` phase UNMEETABLE. The mission still boots a VM, still runs
//! an agent turn, still collects and delivers, and only then records
//! "the independent validator could not be reached this pass" on one evaluation
//! row.
//!
//! That happened: the z.ai credential expired mid-session and the first symptom
//! was a two-phase mission failing after both VMs had run. The information
//! existed the whole time; nobody was told until it was expensive.
//!
//! A report, not a gate — the same stance `runtime_preflight` takes. The server
//! must still boot with a broken validator, because refusing to start would turn
//! a degraded deployment into a dead one, and because a mission that opts out
//! (`validator_model = ''`) is unaffected. What this buys is that the degradation
//! is visible at startup instead of inferred from a failed mission.

/// The smallest question that proves a credential works end to end.
///
/// A real completion rather than a models-list or a HEAD: an expired key, a
/// revoked key and a key with no quota can all pass a cheaper check and fail the
/// call that matters. Two tokens of output.
const PROBE_PROMPT: &str = "Reply with exactly: OK";

/// What the probe found.
#[derive(Debug, PartialEq, Eq)]
pub enum Verdict {
    /// No independent validator is configured; phases are judged by the house
    /// model. Not a fault — a deployment may choose this.
    NotConfigured,
    /// Configured, resolved, and it answered.
    Reachable { spec: String },
    /// Configured but the registry has no such provider, so
    /// `cross_provider_judge` will refuse it rather than judge with the default.
    Unregistered { spec: String },
    /// Configured and resolved, and the call failed.
    Unreachable { spec: String, error: String },
}

impl Verdict {
    /// Is every `done_when` phase currently unmeetable because of this?
    pub fn breaks_gated_phases(&self) -> bool {
        matches!(
            self,
            Verdict::Unregistered { .. } | Verdict::Unreachable { .. }
        )
    }
}

/// Ask the configured independent validator to answer one trivial question.
pub async fn probe(runtime: &cm_runtime::Runtime) -> Verdict {
    let Some(spec) = std::env::var("CLAWMATES_VALIDATOR_MODEL")
        .ok()
        .map(|s| s.trim().to_string())
        .filter(|s| !s.is_empty())
    else {
        return Verdict::NotConfigured;
    };

    let (_provider, model) = runtime.resolve_provider(&spec);
    // `resolve_provider` falls back to the DEFAULT provider for an unknown name,
    // and the fallback is detectable because the returned model still carries the
    // `name:` prefix. Checked here for the same reason the evaluator checks it:
    // a validator that is silently the house model is worse than none.
    if model.contains(':') {
        return Verdict::Unregistered { spec };
    }

    // Through `Runtime::complete`, which is the same resolve-then-stream path
    // the evaluator's judge takes. A probe that dialled the provider its own way
    // could pass while the real call fails.
    match runtime.complete("", PROBE_PROMPT, &spec, 16, false).await {
        Ok(_) => Verdict::Reachable { spec },
        Err(e) => Verdict::Unreachable {
            spec,
            error: e.chars().take(160).collect(),
        },
    }
}

/// Probe at boot and say plainly what it means for missions.
pub fn report_at_boot(runtime: cm_runtime::Runtime) {
    tokio::spawn(async move {
        match probe(&runtime).await {
            Verdict::NotConfigured => eprintln!(
                "validator_preflight: no CLAWMATES_VALIDATOR_MODEL — phase verdicts are judged \
                 by the house model, which is NOT an independent check"
            ),
            Verdict::Reachable { spec } => {
                eprintln!("validator_preflight: independent validator {spec} answered")
            }
            Verdict::Unregistered { spec } => eprintln!(
                "validator_preflight: CLAWMATES_VALIDATOR_MODEL={spec} has no registered \
                 provider — the evaluator will refuse it rather than judge with the default, \
                 so EVERY phase with a done_when condition will fail as unmet. Register the \
                 provider, or set the mission's validator_model to '' to opt out."
            ),
            Verdict::Unreachable { spec, error } => eprintln!(
                "validator_preflight: independent validator {spec} is UNREACHABLE ({error}) — \
                 EVERY phase with a done_when condition will fail as unmet, after running its \
                 agent. Fix the credential, or set validator_model to '' per mission to judge \
                 with the house model."
            ),
        }
    });
}

#[cfg(test)]
mod tests {
    use super::*;

    /// The two states that make gated phases unmeetable, and the two that do
    /// not. This is the distinction the whole module exists to draw: "no
    /// validator configured" is a choice, "configured and broken" is a fault
    /// that silently fails every conditioned mission.
    #[test]
    fn only_a_configured_but_broken_validator_breaks_gated_phases() {
        assert!(!Verdict::NotConfigured.breaks_gated_phases());
        assert!(!Verdict::Reachable {
            spec: "glm:glm-4.7".into()
        }
        .breaks_gated_phases());

        assert!(Verdict::Unregistered {
            spec: "glm:glm-4.7".into()
        }
        .breaks_gated_phases());
        assert!(Verdict::Unreachable {
            spec: "glm:glm-4.7".into(),
            error: "401".into()
        }
        .breaks_gated_phases());
    }

    /// An unregistered provider is NOT reported as unreachable, and the
    /// difference is actionable: one is fixed by registering a provider, the
    /// other by fixing a credential. Collapsing them sends an operator to the
    /// wrong place.
    #[test]
    fn the_two_faults_are_distinguishable() {
        let a = Verdict::Unregistered {
            spec: "glm:glm-4.7".into(),
        };
        let b = Verdict::Unreachable {
            spec: "glm:glm-4.7".into(),
            error: "401 Authentication Failed".into(),
        };
        assert_ne!(a, b);
    }
}