feat(missions): choose the independent validator per mission (#53)
`CLAWMATES_VALIDATOR_MODEL` is deployment-wide, so proving Slice 2 put a second
provider on the critical path of EVERY phase verdict. `cross_provider_judge`
deliberately does not fall back when the independent judge fails — a verdict
quietly produced by a same-family model would claim a property it does not have —
so a z.ai outage makes phases unmeetable rather than merely unverified. That is a
per-mission trade, not a per-deployment one.
`missions.validator_model` (0068), settable at create, with three distinct states
because an empty string and NULL mean opposite things in a nullable text column:
NULL use the deployment default
'' explicitly NO independent validator — judge with the house model.
The default must not quietly reinstate independence a mission was
told to skip.
'glm:glm-4.7' this spec, subject to the same three refusals as before:
same-family rejected, unregistered provider rejected, and a failed
independent judge does not fall back.
Whitespace counts as empty: a column hand-set to " " meant to say nothing.
478 tests pass, clippy clean. Behaviour is unchanged for existing missions — they
have NULL and so keep following the deployment default.
This commit is contained in:
@@ -247,6 +247,29 @@ pub fn provider_family(spec: &str) -> String {
|
|||||||
/// parameter that only ever has one value.
|
/// parameter that only ever has one value.
|
||||||
const IMPLEMENTER_FAMILY: &str = "anthropic";
|
const IMPLEMENTER_FAMILY: &str = "anthropic";
|
||||||
|
|
||||||
|
/// Which validator spec applies, given the mission's own setting and the
|
||||||
|
/// deployment default.
|
||||||
|
///
|
||||||
|
/// The three cases are distinct on purpose, and an empty string is not the same
|
||||||
|
/// as unset:
|
||||||
|
/// - `Some("")` on the mission — an explicit opt OUT. This mission wants the house
|
||||||
|
/// judge, and the deployment default must not quietly reinstate independence it
|
||||||
|
/// was told to skip.
|
||||||
|
/// - `Some(spec)` — this mission's choice, which wins.
|
||||||
|
/// - `None` — nothing said, so the deployment default applies.
|
||||||
|
///
|
||||||
|
/// Whitespace counts as empty: a column set to `" "` by hand meant to say nothing.
|
||||||
|
fn resolve_validator_spec(mission: Option<&str>, deployment: Option<&str>) -> Option<String> {
|
||||||
|
match mission {
|
||||||
|
Some(s) if s.trim().is_empty() => None,
|
||||||
|
Some(s) => Some(s.trim().to_string()),
|
||||||
|
None => deployment
|
||||||
|
.map(str::trim)
|
||||||
|
.filter(|s| !s.is_empty())
|
||||||
|
.map(str::to_string),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
/// A judge from a different provider family, if one is configured and REGISTERED.
|
/// A judge from a different provider family, if one is configured and REGISTERED.
|
||||||
///
|
///
|
||||||
/// `CLAWMATES_VALIDATOR_MODEL` holds a registry spec such as `glm:glm-4.7`.
|
/// `CLAWMATES_VALIDATOR_MODEL` holds a registry spec such as `glm:glm-4.7`.
|
||||||
@@ -258,14 +281,24 @@ const IMPLEMENTER_FAMILY: &str = "anthropic";
|
|||||||
/// back Claude while the caller believed it had asked for GLM. The fallback is
|
/// back Claude while the caller believed it had asked for GLM. The fallback is
|
||||||
/// detectable because the returned model still carries the `name:` prefix, and it
|
/// detectable because the returned model still carries the `name:` prefix, and it
|
||||||
/// is checked here rather than trusted.
|
/// is checked here rather than trusted.
|
||||||
fn cross_provider_judge(
|
async fn cross_provider_judge(
|
||||||
runtime: &cm_runtime::Runtime,
|
runtime: &cm_runtime::Runtime,
|
||||||
|
mission_id: Uuid,
|
||||||
) -> Option<(std::sync::Arc<dyn cm_llm::LlmProvider>, String)> {
|
) -> Option<(std::sync::Arc<dyn cm_llm::LlmProvider>, String)> {
|
||||||
let spec = std::env::var("CLAWMATES_VALIDATOR_MODEL").ok()?;
|
// Read per mission rather than widening `Mission` for one caller. One extra
|
||||||
let spec = spec.trim();
|
// query per evaluation, against a path that is about to make a model call.
|
||||||
if spec.is_empty() {
|
let per_mission: Option<String> =
|
||||||
return None;
|
sqlx::query_scalar("SELECT validator_model FROM missions WHERE id = $1")
|
||||||
}
|
.bind(mission_id)
|
||||||
|
.fetch_optional(runtime.pool())
|
||||||
|
.await
|
||||||
|
.unwrap_or(None)
|
||||||
|
.flatten();
|
||||||
|
let spec = resolve_validator_spec(
|
||||||
|
per_mission.as_deref(),
|
||||||
|
std::env::var("CLAWMATES_VALIDATOR_MODEL").ok().as_deref(),
|
||||||
|
)?;
|
||||||
|
let spec = spec.as_str();
|
||||||
let family = provider_family(spec);
|
let family = provider_family(spec);
|
||||||
if family == IMPLEMENTER_FAMILY {
|
if family == IMPLEMENTER_FAMILY {
|
||||||
eprintln!(
|
eprintln!(
|
||||||
@@ -354,7 +387,7 @@ pub async fn evaluate(
|
|||||||
// failure — the model that talked itself into a shortcut is the one disposed
|
// failure — the model that talked itself into a shortcut is the one disposed
|
||||||
// to accept it — and the tool loop is what makes the check evidence rather
|
// to accept it — and the tool loop is what makes the check evidence rather
|
||||||
// than opinion, so an independent judge must have it too.
|
// than opinion, so an independent judge must have it too.
|
||||||
if let Some((provider, model)) = cross_provider_judge(runtime) {
|
if let Some((provider, model)) = cross_provider_judge(runtime, mission_id).await {
|
||||||
let system = match &sandbox {
|
let system = match &sandbox {
|
||||||
Some(_) => format!("{EVAL_SYSTEM_VERIFYING}\n\n{VERDICT_CONTRACT}"),
|
Some(_) => format!("{EVAL_SYSTEM_VERIFYING}\n\n{VERDICT_CONTRACT}"),
|
||||||
None => format!("{EVAL_SYSTEM_EVIDENCE_ONLY}\n\n{VERDICT_CONTRACT}"),
|
None => format!("{EVAL_SYSTEM_EVIDENCE_ONLY}\n\n{VERDICT_CONTRACT}"),
|
||||||
@@ -784,6 +817,43 @@ mod cross_provider_tests {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// A mission's own choice wins over the deployment default.
|
||||||
|
#[test]
|
||||||
|
fn a_mission_can_choose_its_validator() {
|
||||||
|
assert_eq!(
|
||||||
|
resolve_validator_spec(Some("kimi:kimi-k2"), Some("glm:glm-4.7")).as_deref(),
|
||||||
|
Some("kimi:kimi-k2")
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
resolve_validator_spec(None, Some("glm:glm-4.7")).as_deref(),
|
||||||
|
Some("glm:glm-4.7"),
|
||||||
|
"nothing said on the mission means the deployment default applies"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
/// An EMPTY value on the mission is an explicit opt-out, not "unset". The
|
||||||
|
/// deployment default must not quietly reinstate independence a mission was
|
||||||
|
/// told to skip — the two cases look the same in a nullable text column and
|
||||||
|
/// mean opposite things.
|
||||||
|
#[test]
|
||||||
|
fn an_empty_mission_setting_opts_out_rather_than_falling_back() {
|
||||||
|
for spelling in [Some(""), Some(" ")] {
|
||||||
|
assert_eq!(
|
||||||
|
resolve_validator_spec(spelling, Some("glm:glm-4.7")),
|
||||||
|
None,
|
||||||
|
"{spelling:?} asked for no independent validator"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// And with neither set, there is no independent judge — which is the state
|
||||||
|
/// every deployment starts in.
|
||||||
|
#[test]
|
||||||
|
fn no_setting_anywhere_means_no_independent_judge() {
|
||||||
|
assert_eq!(resolve_validator_spec(None, None), None);
|
||||||
|
assert_eq!(resolve_validator_spec(None, Some(" ")), None);
|
||||||
|
}
|
||||||
|
|
||||||
/// A phase that ran out of passes without meeting its condition did NOT
|
/// A phase that ran out of passes without meeting its condition did NOT
|
||||||
/// succeed. It used to be recorded `completed` alongside a verdict saying
|
/// succeed. It used to be recorded `completed` alongside a verdict saying
|
||||||
/// `met=false`, so mission status reported a goal that was never reached as a
|
/// `met=false`, so mission status reported a goal that was never reached as a
|
||||||
|
|||||||
@@ -41,6 +41,10 @@ pub struct CreateMissionRequest {
|
|||||||
/// Which per-CLI rootfs a `microvm` mission boots (`missions.backend`), e.g.
|
/// Which per-CLI rootfs a `microvm` mission boots (`missions.backend`), e.g.
|
||||||
/// "claude". NULL boots the node's default image.
|
/// "claude". NULL boots the node's default image.
|
||||||
pub backend: Option<String>,
|
pub backend: Option<String>,
|
||||||
|
/// Model that independently validates this mission's phase verdicts, e.g.
|
||||||
|
/// `glm:glm-4.7`. Omit to use the deployment default; send `""` to opt out of
|
||||||
|
/// independent validation and judge with the house model.
|
||||||
|
pub validator_model: Option<String>,
|
||||||
}
|
}
|
||||||
|
|
||||||
fn default_schedule() -> Value {
|
fn default_schedule() -> Value {
|
||||||
@@ -285,6 +289,7 @@ pub async fn create(
|
|||||||
runtime_kind: Some(runtime_kind),
|
runtime_kind: Some(runtime_kind),
|
||||||
target_node_id: body.target_node_id,
|
target_node_id: body.target_node_id,
|
||||||
backend: body.backend.as_deref(),
|
backend: body.backend.as_deref(),
|
||||||
|
validator_model: body.validator_model.as_deref(),
|
||||||
phases: phases_for_create(
|
phases: phases_for_create(
|
||||||
crate::workflow_registry::get(body.template_kind.trim()),
|
crate::workflow_registry::get(body.template_kind.trim()),
|
||||||
body.phases,
|
body.phases,
|
||||||
|
|||||||
@@ -137,6 +137,9 @@ pub struct NewMission<'a> {
|
|||||||
/// rootfs. Deliberately unconstrained in the schema: which images exist is a
|
/// rootfs. Deliberately unconstrained in the schema: which images exist is a
|
||||||
/// property of the NODES, not of the database.
|
/// property of the NODES, not of the database.
|
||||||
pub backend: Option<&'a str>,
|
pub backend: Option<&'a str>,
|
||||||
|
/// Independent validator for this mission's verdicts. `None` = deployment
|
||||||
|
/// default; `Some("")` = explicitly none. See migration 0068.
|
||||||
|
pub validator_model: Option<&'a str>,
|
||||||
pub phases: Vec<NewMissionPhase>,
|
pub phases: Vec<NewMissionPhase>,
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -165,9 +168,9 @@ pub async fn insert(pool: &PgPool, m: NewMission<'_>) -> Result<Uuid, DbError> {
|
|||||||
"INSERT INTO missions
|
"INSERT INTO missions
|
||||||
(id, workspace_id, title, template_kind, team_id,
|
(id, workspace_id, title, template_kind, team_id,
|
||||||
team_template_id, repo_id, schedule, status, description, config,
|
team_template_id, repo_id, schedule, status, description, config,
|
||||||
runtime_kind, target_node_id, backend)
|
runtime_kind, target_node_id, backend, validator_model)
|
||||||
VALUES ($1,$2,$3,$4,$5,$6,$7,$8,'draft',$9,$10,
|
VALUES ($1,$2,$3,$4,$5,$6,$7,$8,'draft',$9,$10,
|
||||||
COALESCE($11,'zeroclaw'),$12,$13)",
|
COALESCE($11,'zeroclaw'),$12,$13,$14)",
|
||||||
)
|
)
|
||||||
.bind(mission_id)
|
.bind(mission_id)
|
||||||
.bind(m.workspace_id)
|
.bind(m.workspace_id)
|
||||||
@@ -182,6 +185,7 @@ pub async fn insert(pool: &PgPool, m: NewMission<'_>) -> Result<Uuid, DbError> {
|
|||||||
.bind(m.runtime_kind)
|
.bind(m.runtime_kind)
|
||||||
.bind(m.target_node_id)
|
.bind(m.target_node_id)
|
||||||
.bind(m.backend)
|
.bind(m.backend)
|
||||||
|
.bind(m.validator_model)
|
||||||
.execute(&mut *tx)
|
.execute(&mut *tx)
|
||||||
.await?;
|
.await?;
|
||||||
|
|
||||||
|
|||||||
@@ -0,0 +1,20 @@
|
|||||||
|
-- Which model independently validates a mission's phase verdicts.
|
||||||
|
--
|
||||||
|
-- `CLAWMATES_VALIDATOR_MODEL` is deployment-wide, so proving cross-provider
|
||||||
|
-- validation meant putting a second provider on the critical path of EVERY phase
|
||||||
|
-- verdict in the deployment. `cross_provider_judge` deliberately does not fall
|
||||||
|
-- back to the agent's own provider when the independent judge fails — a verdict
|
||||||
|
-- silently produced by a same-family model would claim a property it does not
|
||||||
|
-- have — so a z.ai outage makes phases unmeetable rather than merely unverified.
|
||||||
|
-- That is the right trade to make per mission, not per deployment.
|
||||||
|
--
|
||||||
|
-- NULL = use the deployment default (`CLAWMATES_VALIDATOR_MODEL`).
|
||||||
|
-- '' = explicitly no independent validator; judge with the house model.
|
||||||
|
-- 'glm:glm-4.7' = this registry spec, subject to the same refusals as the env
|
||||||
|
-- default (same-family rejected, unregistered provider rejected).
|
||||||
|
--
|
||||||
|
-- No CHECK constraint: which providers a deployment registered is a property of
|
||||||
|
-- its configuration, not of the schema — the same reason `missions.backend` has
|
||||||
|
-- none.
|
||||||
|
ALTER TABLE missions
|
||||||
|
ADD COLUMN IF NOT EXISTS validator_model text;
|
||||||
Reference in New Issue
Block a user