diff --git a/crates/cm-api/src/evaluator.rs b/crates/cm-api/src/evaluator.rs index 2f05fc6..16c7fc4 100644 --- a/crates/cm-api/src/evaluator.rs +++ b/crates/cm-api/src/evaluator.rs @@ -291,13 +291,27 @@ fn names_a_provider(spec: &str) -> bool { spec.contains(':') } -/// The provider family the mission's agent ran on. +/// The provider family the mission's agent ran on, from `missions.backend`. /// -/// Today every mission backend is Claude Code (`agent-claude`), including the -/// microVM path. When `agent-glm` / `agent-kimi` images exist this should read -/// `missions.backend`; until then, hardcoding the truth is better than plumbing a -/// parameter that only ever has one value. -const IMPLEMENTER_FAMILY: &str = "anthropic"; +/// Mirrors `mission_runtime::microvm_credential_for`: the backend decides which +/// credential the guest gets and which host its egress proxy allows, so it is +/// the one honest source for "who answered the agent's turns". This was a +/// hardcoded `"anthropic"` while every backend was Claude Code on Anthropic; +/// once `glm` and `kimi` rootfs existed that constant made a glm-backend +/// mission judged by `glm:glm-5.3` read as `independent = true`, which is the +/// one claim this path exists to make honestly. +/// +/// `unknown` for anything unrecognised, for the same reason `provider_family` +/// says it: a guess in either direction misstates independence. +pub fn implementer_family(backend: Option<&str>) -> &'static str { + match backend.map(str::trim) { + None | Some("") | Some("default") | Some("claude") | Some("canary-claude") => "anthropic", + Some("glm") => "glm", + Some("kimi") => "kimi", + Some("local-ornith") => "local", + Some(_) => "unknown", + } +} /// Which validator spec applies, given the mission's own setting and the /// deployment default. @@ -333,9 +347,23 @@ fn resolve_validator_spec(mission: Option<&str>, deployment: Option<&str>) -> Op /// back Claude while the caller believed it had asked for GLM. The fallback is /// detectable because the returned model still carries the `name:` prefix, and it /// is checked here rather than trusted. +/// The mission's implementer family, read from its row. `anthropic` when the +/// row cannot be read — the pre-2026-09-18 behaviour, and the family every +/// backend actually had until then. +async fn mission_implementer_family(runtime: &cm_runtime::Runtime, mission_id: Uuid) -> &'static str { + let backend: Option = sqlx::query_scalar("SELECT backend FROM missions WHERE id = $1") + .bind(mission_id) + .fetch_optional(runtime.pool()) + .await + .unwrap_or(None) + .flatten(); + implementer_family(backend.as_deref()) +} + async fn cross_provider_judge( runtime: &cm_runtime::Runtime, mission_id: Uuid, + implementer: &str, ) -> Option<(std::sync::Arc, String)> { // Read per mission rather than widening `Mission` for one caller. One extra // query per evaluation, against a path that is about to make a model call. @@ -352,10 +380,10 @@ async fn cross_provider_judge( )?; let spec = spec.as_str(); let family = provider_family(spec); - if family == IMPLEMENTER_FAMILY { + if family == implementer { eprintln!( "evaluator: CLAWMATES_VALIDATOR_MODEL={spec} is the same provider family as the \ - agent ({IMPLEMENTER_FAMILY}) — that is not an independent check, ignoring it" + agent ({implementer}) — that is not an independent check, ignoring it" ); return None; } @@ -474,7 +502,8 @@ pub async fn evaluate( // failure — the model that talked itself into a shortcut is the one disposed // to accept it — and the tool loop is what makes the check evidence rather // than opinion, so an independent judge must have it too. - if let Some((provider, model)) = cross_provider_judge(runtime, mission_id).await { + let implementer = mission_implementer_family(runtime, mission_id).await; + if let Some((provider, model)) = cross_provider_judge(runtime, mission_id, implementer).await { let system = match &sandbox { Some(_) => format!("{EVAL_SYSTEM_VERIFYING}\n\n{VERDICT_CONTRACT}"), None => format!("{EVAL_SYSTEM_EVIDENCE_ONLY}\n\n{VERDICT_CONTRACT}"), @@ -535,7 +564,9 @@ pub async fn evaluate( let outcome = judge_with_tools(&provider, &system, &user, &model, sandbox.as_ref(), &mut usage) .await; - // Same family as the agent; `independent` stays false below. + // Independent exactly when the agent did NOT run on Anthropic: a + // glm- or kimi-backend mission judged by the Anthropic subscription + // is a cross-provider check, and a claude-backend one is not. let mut v = match outcome { Err(e) => Verdict::not_met( &model, @@ -550,6 +581,7 @@ pub async fn evaluate( } }; v.usage = usage; + v.independent = implementer != "anthropic"; return v; } @@ -1088,7 +1120,23 @@ mod cross_provider_tests { #[test] fn an_unrecognised_model_is_not_assumed_to_be_ours() { assert_eq!(provider_family("some-new-model-v9"), "unknown"); - assert_ne!(provider_family("some-new-model-v9"), IMPLEMENTER_FAMILY); + assert_ne!(provider_family("some-new-model-v9"), implementer_family(None)); + } + + /// The implementer family comes from the mission's backend, and the two + /// readers have to agree on the spelling of a family or a glm mission + /// judged by glm reads as independent — which it did, while this was a + /// constant. + #[test] + fn the_implementer_family_follows_the_backend() { + assert_eq!(implementer_family(None), "anthropic"); + for b in ["", "default", "claude", "canary-claude"] { + assert_eq!(implementer_family(Some(b)), "anthropic", "{b}"); + } + assert_eq!(implementer_family(Some("glm")), provider_family("glm:glm-5.3")); + assert_eq!(implementer_family(Some("kimi")), provider_family("kimi:kimi-k2")); + assert_ne!(implementer_family(Some("claude")), provider_family("glm:glm-5.3")); + assert_eq!(implementer_family(Some("something-else")), "unknown"); } /// The whole point: a judge in the implementer's own family is not @@ -1098,13 +1146,15 @@ mod cross_provider_tests { for spec in ["claude-opus-4-8", "runtime:claw_x", "sonnet"] { assert_eq!( provider_family(spec), - IMPLEMENTER_FAMILY, - "{spec} would have to be rejected as a validator" + implementer_family(Some("claude")), + "{spec} would have to be rejected as a validator of a claude mission" ); } for spec in ["glm:glm-4.7", "kimi:kimi-k2"] { - assert_ne!(provider_family(spec), IMPLEMENTER_FAMILY, "{spec}"); + assert_ne!(provider_family(spec), implementer_family(Some("claude")), "{spec}"); } + // And the other way round: glm judging a glm mission is the same trap. + assert_eq!(provider_family("glm:glm-5.3"), implementer_family(Some("glm"))); } /// A mission's own choice wins over the deployment default. diff --git a/crates/cm-api/src/microvm_executor.rs b/crates/cm-api/src/microvm_executor.rs index 3042718..610b9f7 100644 --- a/crates/cm-api/src/microvm_executor.rs +++ b/crates/cm-api/src/microvm_executor.rs @@ -306,6 +306,13 @@ const TEAMMATE_PROBE: &str = "cat /root/.claude/teams/*/config.json 2>/dev/null /// says so, which is the difference between losing a check and losing the work. const SETTINGS_PROBE: &str = "claude --help 2>&1 | grep -q -- '--settings' && echo SETTINGS-OK"; +/// What the guest's `claude` reports itself as. Recorded beside the rootfs the +/// node said it booted, so "which CLI did this mission run on" is a query +/// against `topology_runs`, not an archaeology of image mtimes. Found necessary +/// on 2026-09-18: every rootfs on the fleet had been on 2.1.223–2.1.226 for a +/// month while the container tier moved to 2.1.276, and nothing recorded either. +const CLI_VERSION_PROBE: &str = "claude --version 2>/dev/null | head -c 80"; + /// How many times the stop gate refused to let the agent finish. const BLOCKS_PROBE: &str = "cat /root/gate/blocks 2>/dev/null || echo 0"; @@ -400,6 +407,12 @@ pub struct VmOutcome { /// field cannot tell them apart — the unmatched-frame log and the install /// error are what separate them. pub tools: Vec, + /// The rootfs the node reported booting (`vm_create` reply), e.g. + /// `/opt/clawmates-fc/rootfs-glm.ext4`. `None` if the reply carried none. + pub rootfs: Option, + /// The guest's own `claude --version`, e.g. `2.1.276 (Claude Code)`. + /// `None` if the probe failed — which is a fact worth seeing, not a zero. + pub cli_version: Option, } /// Boot a VM, run the phase in it, collect the result, and destroy it. @@ -618,6 +631,16 @@ async fn run_inside( .await .map(|p| p.stdout.contains("SETTINGS-OK")) .unwrap_or(false); + let cli_version = vm + .exec(CLI_VERSION_PROBE, None, 60, &[]) + .await + .ok() + .map(|p| p.stdout.trim().to_string()) + .filter(|v| !v.is_empty()); + let rootfs = created + .get("rootfs") + .and_then(serde_json::Value::as_str) + .map(str::to_string); let gate_dir = match gate { None => None, Some(g) => { @@ -954,6 +977,8 @@ async fn run_inside( stop_blocks, released_at_cap, tools, + rootfs, + cli_version, }) } diff --git a/crates/cm-api/src/microvm_turn_executor.rs b/crates/cm-api/src/microvm_turn_executor.rs index 11ecf02..aec6096 100644 --- a/crates/cm-api/src/microvm_turn_executor.rs +++ b/crates/cm-api/src/microvm_turn_executor.rs @@ -429,6 +429,8 @@ mod tests { stop_blocks: None, released_at_cap: None, tools: Vec::new(), + rootfs: None, + cli_version: None, }) } } @@ -624,6 +626,8 @@ mod tests { stop_blocks: None, released_at_cap: None, tools: Vec::new(), + rootfs: None, + cli_version: None, }) } } @@ -656,6 +660,8 @@ mod tests { stop_blocks: Some(crate::vm_stop_gate::MAX_BLOCKS), released_at_cap: Some(true), tools: Vec::new(), + rootfs: None, + cli_version: None, }) } } @@ -683,6 +689,8 @@ mod tests { stop_blocks: Some(crate::vm_stop_gate::MAX_BLOCKS), released_at_cap: Some(false), tools: Vec::new(), + rootfs: None, + cli_version: None, }) } } diff --git a/crates/cm-api/src/phase_runner.rs b/crates/cm-api/src/phase_runner.rs index e4801d9..c9738ad 100644 --- a/crates/cm-api/src/phase_runner.rs +++ b/crates/cm-api/src/phase_runner.rs @@ -1781,6 +1781,18 @@ async fn launch_microvm_phase( if let Ok(o) = &outcome { record_vm_tools(&pool2, mission_id, phase_id, run_id, &o.tools, &[]).await; } + // What actually ran: the rootfs the node booted and the CLI the guest + // reported. Persisted on the run so "which image and version did this + // mission use" is a query, not an inference from file mtimes — on + // 2026-09-18 every fleet rootfs had sat on 2.1.223–2.1.226 for a month + // while the container tier moved on, and nothing had recorded either. + let vm = serde_json::json!({ + "vm_id": crate::microvm_executor::vm_id_for(phase_id, iteration, None), + "node_id": target_node_id, + "backend": backend, + "rootfs": outcome.as_ref().ok().and_then(|o| o.rootfs.clone()), + "cli_version": outcome.as_ref().ok().and_then(|o| o.cli_version.clone()), + }); let (status, note) = match outcome { // The gate gave up. It is the ONLY thing that runs a // `done_when_check`, so a release at the cap means the phase's own @@ -1813,7 +1825,9 @@ async fn launch_microvm_phase( eprintln!( "phase_runner: microvm phase {phase_id} of mission {mission_id} → {status} \ (subagents: {subagents}, teammates: {teammates}, stop-gate blocks: \ - {blocked}) — {}", + {blocked}; rootfs: {}, cli: {}) — {}", + vm["rootfs"].as_str().unwrap_or("?"), + vm["cli_version"].as_str().unwrap_or("?"), note.chars().take(300).collect::() ); // Never overwrite a cancellation. The operator asking to stop is a decision; @@ -1843,7 +1857,9 @@ async fn launch_microvm_phase( "output": note, "tokens": 0, "gated": [], - }] + }], + // Beside `records`, not inside: the two readers parse only `records`. + "vm": vm, }); if let Err(e) = sqlx::query( "UPDATE topology_runs diff --git a/images/agent-claude/Dockerfile b/images/agent-claude/Dockerfile index 17b30be..75f8f81 100644 --- a/images/agent-claude/Dockerfile +++ b/images/agent-claude/Dockerfile @@ -27,7 +27,14 @@ FROM clawmates/agent-toolchain:dev # could undo. 2.1.221 also fixes `--mcp-config` servers not connecting before the # first turn in print mode, which is exactly the mode we run and will matter when # the MCP door reaches a VM. -ARG CLAUDE_CODE_VERSION=2.1.226 +# 2.1.276, 2026-09-18. Between 2.1.226 and here, 2.1.265 and 2.1.275 each broke +# every turn on ANTHROPIC_BASE_URL endpoints (HTTP 400) — the path the glm and +# kimi images use — and 2.1.276 is the first version after both that is fixed. +# All four agent-* images pin the SAME version; scripts/fc-build-rootfs.sh +# refuses to build if they drift. Bump them together, on purpose, and run a +# mission on each backend before promoting (see deploy/clawmates-runtime/Dockerfile +# for the same rule on the container tier). +ARG CLAUDE_CODE_VERSION=2.1.276 RUN npm install -g "@anthropic-ai/claude-code@${CLAUDE_CODE_VERSION}" \ && npm cache clean --force \ && rm -rf /root/.npm \ diff --git a/images/agent-glm/Dockerfile b/images/agent-glm/Dockerfile index db4bd90..9ee14f7 100644 --- a/images/agent-glm/Dockerfile +++ b/images/agent-glm/Dockerfile @@ -21,7 +21,14 @@ FROM clawmates/agent-toolchain:dev # composed run's verifier node should differ by provider and by nothing else; two # CLI versions in one graph would make "the verifier disagreed" ambiguous between # the model and the harness. -ARG CLAUDE_CODE_VERSION=2.1.223 +# 2.1.276, 2026-09-18. Between 2.1.226 and here, 2.1.265 and 2.1.275 each broke +# every turn on ANTHROPIC_BASE_URL endpoints (HTTP 400) — the path the glm and +# kimi images use — and 2.1.276 is the first version after both that is fixed. +# All four agent-* images pin the SAME version; scripts/fc-build-rootfs.sh +# refuses to build if they drift. Bump them together, on purpose, and run a +# mission on each backend before promoting (see deploy/clawmates-runtime/Dockerfile +# for the same rule on the container tier). +ARG CLAUDE_CODE_VERSION=2.1.276 RUN npm install -g "@anthropic-ai/claude-code@${CLAUDE_CODE_VERSION}" \ && npm cache clean --force \ && rm -rf /root/.npm \ diff --git a/images/agent-kimi/Dockerfile b/images/agent-kimi/Dockerfile index 53b66d8..b4fb4db 100644 --- a/images/agent-kimi/Dockerfile +++ b/images/agent-kimi/Dockerfile @@ -34,7 +34,14 @@ # own model exactly as z.ai does. No ANTHROPIC_MODEL override is needed. FROM clawmates/agent-toolchain:dev -ARG CLAUDE_CODE_VERSION=2.1.223 +# 2.1.276, 2026-09-18. Between 2.1.226 and here, 2.1.265 and 2.1.275 each broke +# every turn on ANTHROPIC_BASE_URL endpoints (HTTP 400) — the path the glm and +# kimi images use — and 2.1.276 is the first version after both that is fixed. +# All four agent-* images pin the SAME version; scripts/fc-build-rootfs.sh +# refuses to build if they drift. Bump them together, on purpose, and run a +# mission on each backend before promoting (see deploy/clawmates-runtime/Dockerfile +# for the same rule on the container tier). +ARG CLAUDE_CODE_VERSION=2.1.276 RUN npm install -g "@anthropic-ai/claude-code@${CLAUDE_CODE_VERSION}" \ && npm cache clean --force \ && rm -rf /root/.npm \ diff --git a/images/agent-ornith/Dockerfile b/images/agent-ornith/Dockerfile index 09391df..4907a7a 100644 --- a/images/agent-ornith/Dockerfile +++ b/images/agent-ornith/Dockerfile @@ -31,7 +31,14 @@ FROM clawmates/agent-toolchain:dev # Pinned to the SAME version as agent-claude and agent-glm. A run that differs # by provider should differ by nothing else, or "the local backend behaved # differently" is ambiguous between the model and the harness. -ARG CLAUDE_CODE_VERSION=2.1.223 +# 2.1.276, 2026-09-18. Between 2.1.226 and here, 2.1.265 and 2.1.275 each broke +# every turn on ANTHROPIC_BASE_URL endpoints (HTTP 400) — the path the glm and +# kimi images use — and 2.1.276 is the first version after both that is fixed. +# All four agent-* images pin the SAME version; scripts/fc-build-rootfs.sh +# refuses to build if they drift. Bump them together, on purpose, and run a +# mission on each backend before promoting (see deploy/clawmates-runtime/Dockerfile +# for the same rule on the container tier). +ARG CLAUDE_CODE_VERSION=2.1.276 RUN npm install -g "@anthropic-ai/claude-code@${CLAUDE_CODE_VERSION}" \ && npm cache clean --force \ && rm -rf /root/.npm \ diff --git a/scripts/fc-build-rootfs.sh b/scripts/fc-build-rootfs.sh index 3fbece0..e916994 100755 --- a/scripts/fc-build-rootfs.sh +++ b/scripts/fc-build-rootfs.sh @@ -23,6 +23,19 @@ set -uo pipefail +# Drift guard. The four agent-* images must pin ONE Claude Code version: a solo +# run and a composed run's verifier should differ by provider and by nothing +# else, and on 2026-09-18 they had already drifted (claude 2.1.226, the rest +# 2.1.223) under comments saying "same version on purpose". Refuse to build a +# rootfs from a set that disagrees, and say which. +repo_root="$(cd "$(dirname "$0")/.." && pwd)" +pins=$(grep -h '^ARG CLAUDE_CODE_VERSION=' "$repo_root"/images/agent-{claude,glm,kimi,ornith}/Dockerfile 2>/dev/null | sort -u) +if [ "$(printf '%s\n' "$pins" | grep -c .)" -ne 1 ]; then + echo "fc-build-rootfs: the agent-* images pin different Claude Code versions — fix that first:" >&2 + grep -H '^ARG CLAUDE_CODE_VERSION=' "$repo_root"/images/agent-{claude,glm,kimi,ornith}/Dockerfile >&2 + exit 2 +fi + WORK="${FC_WORK:-/opt/clawmates-fc}" FAILURES=0 pass() { printf 'PASS %s\n' "$*"; } diff --git a/scripts/verify-mission-delivery.sh b/scripts/verify-mission-delivery.sh index 9c748cb..5c02190 100755 --- a/scripts/verify-mission-delivery.sh +++ b/scripts/verify-mission-delivery.sh @@ -31,6 +31,8 @@ # scripts/verify-mission-delivery.sh multirole # 3 roles + real tests # scripts/verify-mission-delivery.sh noop # empty phase must FAIL # scripts/verify-mission-delivery.sh microvm # runs in a guest kernel + fans out +# scripts/verify-mission-delivery.sh glm # microvm on the z.ai backend; provider proven by the node's egress log +# scripts/verify-mission-delivery.sh kimi # microvm on the Kimi backend; same proof # scripts/verify-mission-delivery.sh capacity # a burst > the fleet must QUEUE # scripts/verify-mission-delivery.sh drain-midmission # a drained node hands the mission on # scripts/verify-mission-delivery.sh all # everything @@ -57,7 +59,11 @@ MISSIONS_ROOT="${CLAWMATES_MISSIONS_ROOT:-/var/lib/clawmates-missions}" # vmlinux version that an upgrade would invalidate. GW_KERNEL=$(ssh "$HOST" 'uname -r' 2>/dev/null | tr -d '[:space:]') NODE_KERNEL=$(ssh "${FLEET_NODE:-osobh@tank}" 'uname -r' 2>/dev/null | tr -d '[:space:]') -REPO_ID="${CLAWMATES_REPO_ID:-f8bbe4d7-2878-40c8-b657-7a7f6031def1}" +# The scratch repo is re-registered whenever the Gitea connection re-syncs, and +# gets a new id each time (f8bbe4d7-… died in the 2026-09-14 wipe). Override +# with CLAWMATES_REPO_ID, or find it: select id from repos where name = +# 'clawmates-delivery-scratch'. +REPO_ID="${CLAWMATES_REPO_ID:-01a0052f-dc7f-7b73-8afd-2984a91338bb}" TEAM_TEMPLATE="${CLAWMATES_TEAM_TEMPLATE:-7e453826-41c4-4425-bab5-8f11fd0a14d7}" MISSION_TIMEOUT="${MISSION_TIMEOUT:-1800}" @@ -1111,6 +1117,79 @@ assert_local() { # fi } +# ── Provider proof for a microVM mission ───────────────────────── +# +# Which provider served a VM's turns is answered by the NODE's egress proxy +# log and by nothing the agent wrote: a z.ai-served agent called itself Claude +# Opus 5 (memory: glm-microvm-backend). The proxy allow-lists one provider host +# per backend and logs every dial and every denial, so "dialled the right host, +# never denied it, dialled no other" is the whole proof — and it is what the +# 2.1.276 rollout had to establish for glm and kimi, whose turns ride +# ANTHROPIC_BASE_URL through a CLI that broke that path twice between 2.1.226 +# and 2.1.276. + +placed_node() { # → ssh target of the node that ran it, or 1 + local node + node=$(ssh "$HOST" "docker exec clawmates_postgres_1 psql -U postgres -d clawmates -tAc \ + \"select coalesce(n.name,'') from missions m left join nodes n on n.id = m.target_node_id where m.id='$1';\"" \ + | head -1 | tr -d '[:space:]') + case "$node" in '') return 1 ;; tank) echo osobh@tank ;; *) echo "$node" ;; esac +} + +vm_journal() { # → this mission's VM lines only + # The VM id is m-- + # (microvm_executor::vm_id_for), so a grep on that prefix cannot pick up a + # neighbour's VM the way a time window can. + local p12 + p12=$(ssh "$HOST" "docker exec clawmates_postgres_1 psql -U postgres -d clawmates -tAc \ + \"select substr(replace(id::text,'-',''),1,12) from mission_phases where mission_id='$2' order by order_idx limit 1;\"" \ + | head -1 | tr -d '[:space:]') + [ -n "$p12" ] || return 1 + ssh "$1" "journalctl -u clawmates-node --since '-3 hours' --no-pager 2>/dev/null | grep -F 'microvm m-$p12-'" +} + +assert_provider_egress() { #