feat(missions): choose the independent validator per mission (#53)

`CLAWMATES_VALIDATOR_MODEL` is deployment-wide, so proving Slice 2 put a second
provider on the critical path of EVERY phase verdict. `cross_provider_judge`
deliberately does not fall back when the independent judge fails — a verdict
quietly produced by a same-family model would claim a property it does not have —
so a z.ai outage makes phases unmeetable rather than merely unverified. That is a
per-mission trade, not a per-deployment one.

`missions.validator_model` (0068), settable at create, with three distinct states
because an empty string and NULL mean opposite things in a nullable text column:

  NULL          use the deployment default
  ''            explicitly NO independent validator — judge with the house model.
                The default must not quietly reinstate independence a mission was
                told to skip.
  'glm:glm-4.7' this spec, subject to the same three refusals as before:
                same-family rejected, unregistered provider rejected, and a failed
                independent judge does not fall back.

Whitespace counts as empty: a column hand-set to " " meant to say nothing.

478 tests pass, clippy clean. Behaviour is unchanged for existing missions — they
have NULL and so keep following the deployment default.
This commit is contained in:
Omar Sobh
2026-08-05 23:07:38 -07:00
parent b17e18aa67
commit c840688adb
4 changed files with 108 additions and 9 deletions
+77 -7
View File
@@ -247,6 +247,29 @@ pub fn provider_family(spec: &str) -> String {
/// parameter that only ever has one value. /// parameter that only ever has one value.
const IMPLEMENTER_FAMILY: &str = "anthropic"; const IMPLEMENTER_FAMILY: &str = "anthropic";
/// Which validator spec applies, given the mission's own setting and the
/// deployment default.
///
/// The three cases are distinct on purpose, and an empty string is not the same
/// as unset:
/// - `Some("")` on the mission — an explicit opt OUT. This mission wants the house
/// judge, and the deployment default must not quietly reinstate independence it
/// was told to skip.
/// - `Some(spec)` — this mission's choice, which wins.
/// - `None` — nothing said, so the deployment default applies.
///
/// Whitespace counts as empty: a column set to `" "` by hand meant to say nothing.
fn resolve_validator_spec(mission: Option<&str>, deployment: Option<&str>) -> Option<String> {
match mission {
Some(s) if s.trim().is_empty() => None,
Some(s) => Some(s.trim().to_string()),
None => deployment
.map(str::trim)
.filter(|s| !s.is_empty())
.map(str::to_string),
}
}
/// A judge from a different provider family, if one is configured and REGISTERED. /// A judge from a different provider family, if one is configured and REGISTERED.
/// ///
/// `CLAWMATES_VALIDATOR_MODEL` holds a registry spec such as `glm:glm-4.7`. /// `CLAWMATES_VALIDATOR_MODEL` holds a registry spec such as `glm:glm-4.7`.
@@ -258,14 +281,24 @@ const IMPLEMENTER_FAMILY: &str = "anthropic";
/// back Claude while the caller believed it had asked for GLM. The fallback is /// back Claude while the caller believed it had asked for GLM. The fallback is
/// detectable because the returned model still carries the `name:` prefix, and it /// detectable because the returned model still carries the `name:` prefix, and it
/// is checked here rather than trusted. /// is checked here rather than trusted.
fn cross_provider_judge( async fn cross_provider_judge(
runtime: &cm_runtime::Runtime, runtime: &cm_runtime::Runtime,
mission_id: Uuid,
) -> Option<(std::sync::Arc<dyn cm_llm::LlmProvider>, String)> { ) -> Option<(std::sync::Arc<dyn cm_llm::LlmProvider>, String)> {
let spec = std::env::var("CLAWMATES_VALIDATOR_MODEL").ok()?; // Read per mission rather than widening `Mission` for one caller. One extra
let spec = spec.trim(); // query per evaluation, against a path that is about to make a model call.
if spec.is_empty() { let per_mission: Option<String> =
return None; sqlx::query_scalar("SELECT validator_model FROM missions WHERE id = $1")
} .bind(mission_id)
.fetch_optional(runtime.pool())
.await
.unwrap_or(None)
.flatten();
let spec = resolve_validator_spec(
per_mission.as_deref(),
std::env::var("CLAWMATES_VALIDATOR_MODEL").ok().as_deref(),
)?;
let spec = spec.as_str();
let family = provider_family(spec); let family = provider_family(spec);
if family == IMPLEMENTER_FAMILY { if family == IMPLEMENTER_FAMILY {
eprintln!( eprintln!(
@@ -354,7 +387,7 @@ pub async fn evaluate(
// failure — the model that talked itself into a shortcut is the one disposed // failure — the model that talked itself into a shortcut is the one disposed
// to accept it — and the tool loop is what makes the check evidence rather // to accept it — and the tool loop is what makes the check evidence rather
// than opinion, so an independent judge must have it too. // than opinion, so an independent judge must have it too.
if let Some((provider, model)) = cross_provider_judge(runtime) { if let Some((provider, model)) = cross_provider_judge(runtime, mission_id).await {
let system = match &sandbox { let system = match &sandbox {
Some(_) => format!("{EVAL_SYSTEM_VERIFYING}\n\n{VERDICT_CONTRACT}"), Some(_) => format!("{EVAL_SYSTEM_VERIFYING}\n\n{VERDICT_CONTRACT}"),
None => format!("{EVAL_SYSTEM_EVIDENCE_ONLY}\n\n{VERDICT_CONTRACT}"), None => format!("{EVAL_SYSTEM_EVIDENCE_ONLY}\n\n{VERDICT_CONTRACT}"),
@@ -784,6 +817,43 @@ mod cross_provider_tests {
} }
} }
/// A mission's own choice wins over the deployment default.
#[test]
fn a_mission_can_choose_its_validator() {
assert_eq!(
resolve_validator_spec(Some("kimi:kimi-k2"), Some("glm:glm-4.7")).as_deref(),
Some("kimi:kimi-k2")
);
assert_eq!(
resolve_validator_spec(None, Some("glm:glm-4.7")).as_deref(),
Some("glm:glm-4.7"),
"nothing said on the mission means the deployment default applies"
);
}
/// An EMPTY value on the mission is an explicit opt-out, not "unset". The
/// deployment default must not quietly reinstate independence a mission was
/// told to skip — the two cases look the same in a nullable text column and
/// mean opposite things.
#[test]
fn an_empty_mission_setting_opts_out_rather_than_falling_back() {
for spelling in [Some(""), Some(" ")] {
assert_eq!(
resolve_validator_spec(spelling, Some("glm:glm-4.7")),
None,
"{spelling:?} asked for no independent validator"
);
}
}
/// And with neither set, there is no independent judge — which is the state
/// every deployment starts in.
#[test]
fn no_setting_anywhere_means_no_independent_judge() {
assert_eq!(resolve_validator_spec(None, None), None);
assert_eq!(resolve_validator_spec(None, Some(" ")), None);
}
/// A phase that ran out of passes without meeting its condition did NOT /// A phase that ran out of passes without meeting its condition did NOT
/// succeed. It used to be recorded `completed` alongside a verdict saying /// succeed. It used to be recorded `completed` alongside a verdict saying
/// `met=false`, so mission status reported a goal that was never reached as a /// `met=false`, so mission status reported a goal that was never reached as a
+5
View File
@@ -41,6 +41,10 @@ pub struct CreateMissionRequest {
/// Which per-CLI rootfs a `microvm` mission boots (`missions.backend`), e.g. /// Which per-CLI rootfs a `microvm` mission boots (`missions.backend`), e.g.
/// "claude". NULL boots the node's default image. /// "claude". NULL boots the node's default image.
pub backend: Option<String>, pub backend: Option<String>,
/// Model that independently validates this mission's phase verdicts, e.g.
/// `glm:glm-4.7`. Omit to use the deployment default; send `""` to opt out of
/// independent validation and judge with the house model.
pub validator_model: Option<String>,
} }
fn default_schedule() -> Value { fn default_schedule() -> Value {
@@ -285,6 +289,7 @@ pub async fn create(
runtime_kind: Some(runtime_kind), runtime_kind: Some(runtime_kind),
target_node_id: body.target_node_id, target_node_id: body.target_node_id,
backend: body.backend.as_deref(), backend: body.backend.as_deref(),
validator_model: body.validator_model.as_deref(),
phases: phases_for_create( phases: phases_for_create(
crate::workflow_registry::get(body.template_kind.trim()), crate::workflow_registry::get(body.template_kind.trim()),
body.phases, body.phases,
+6 -2
View File
@@ -137,6 +137,9 @@ pub struct NewMission<'a> {
/// rootfs. Deliberately unconstrained in the schema: which images exist is a /// rootfs. Deliberately unconstrained in the schema: which images exist is a
/// property of the NODES, not of the database. /// property of the NODES, not of the database.
pub backend: Option<&'a str>, pub backend: Option<&'a str>,
/// Independent validator for this mission's verdicts. `None` = deployment
/// default; `Some("")` = explicitly none. See migration 0068.
pub validator_model: Option<&'a str>,
pub phases: Vec<NewMissionPhase>, pub phases: Vec<NewMissionPhase>,
} }
@@ -165,9 +168,9 @@ pub async fn insert(pool: &PgPool, m: NewMission<'_>) -> Result<Uuid, DbError> {
"INSERT INTO missions "INSERT INTO missions
(id, workspace_id, title, template_kind, team_id, (id, workspace_id, title, template_kind, team_id,
team_template_id, repo_id, schedule, status, description, config, team_template_id, repo_id, schedule, status, description, config,
runtime_kind, target_node_id, backend) runtime_kind, target_node_id, backend, validator_model)
VALUES ($1,$2,$3,$4,$5,$6,$7,$8,'draft',$9,$10, VALUES ($1,$2,$3,$4,$5,$6,$7,$8,'draft',$9,$10,
COALESCE($11,'zeroclaw'),$12,$13)", COALESCE($11,'zeroclaw'),$12,$13,$14)",
) )
.bind(mission_id) .bind(mission_id)
.bind(m.workspace_id) .bind(m.workspace_id)
@@ -182,6 +185,7 @@ pub async fn insert(pool: &PgPool, m: NewMission<'_>) -> Result<Uuid, DbError> {
.bind(m.runtime_kind) .bind(m.runtime_kind)
.bind(m.target_node_id) .bind(m.target_node_id)
.bind(m.backend) .bind(m.backend)
.bind(m.validator_model)
.execute(&mut *tx) .execute(&mut *tx)
.await?; .await?;
+20
View File
@@ -0,0 +1,20 @@
-- Which model independently validates a mission's phase verdicts.
--
-- `CLAWMATES_VALIDATOR_MODEL` is deployment-wide, so proving cross-provider
-- validation meant putting a second provider on the critical path of EVERY phase
-- verdict in the deployment. `cross_provider_judge` deliberately does not fall
-- back to the agent's own provider when the independent judge fails — a verdict
-- silently produced by a same-family model would claim a property it does not
-- have — so a z.ai outage makes phases unmeetable rather than merely unverified.
-- That is the right trade to make per mission, not per deployment.
--
-- NULL = use the deployment default (`CLAWMATES_VALIDATOR_MODEL`).
-- '' = explicitly no independent validator; judge with the house model.
-- 'glm:glm-4.7' = this registry spec, subject to the same refusals as the env
-- default (same-family rejected, unregistered provider rejected).
--
-- No CHECK constraint: which providers a deployment registered is a property of
-- its configuration, not of the schema — the same reason `missions.backend` has
-- none.
ALTER TABLE missions
ADD COLUMN IF NOT EXISTS validator_model text;