fix(missions): capture a failed phase's work; harness gains a microvm scenario (#51)

A REGRESSION I INTRODUCED ONE COMMIT AGO. `capture_finished_coding_phases`
selects on `mp.status = 'completed'`, so the moment an unmet phase correctly began
reporting `failed`, its diff stopped being captured, committed or pushed — the work
was silently discarded. Found by the new harness scenario, whose phase legitimately
missed its condition and then had no artifact at all.

What was produced, and whether the goal was met, are different facts. The artifact
records the first; `mp.status` records the second. Capture now covers terminal
phases (`completed`, `failed`), so a phase that did real work and missed its goal
still delivers a reviewable diff — which is exactly what the next pass needs.

`scripts/verify-mission-delivery.sh microvm` — the regression net this session was
missing. Everything the microVM track proved by hand was guarded by nothing:

  - THE KERNEL LINE is the assertion that cannot pass by accident. Every other
    check would also pass if the phase had quietly run in a container on the
    gateway; only the kernel says WHERE it ran. Compared against the real gateway
    and node kernels read at start-up rather than pinned to a version, so
    upgrading vmlinux does not manufacture a failure.
  - subagent count > 0, from the server's own count of Claude Code's per-subagent
    transcripts. Before `Agent` was in the allowlist this was structurally
    impossible and nothing said so. A probe that could not run reports "?" and
    FAILS the check rather than reading as zero.
  - the verdict's judge and whether it was independent.
  - negative control, observed passing: a mission whose backend no node can run is
    refused at launch and stays draft. Without it the positive scenario would pass
    just as well against a scheduler that ignored `backend` entirely — which is
    what it did until the first real microvm mission landed on a node with no such
    rootfs.

Also fixed in the harness: `api` now sends the JSON body on STDIN (`curl -d @-`)
instead of interpolating it into a single-quoted argument inside a double-quoted
ssh command. A task description containing "the crate's test suite" ended the
quoting and killed the remote shell; two attempts to escape it were themselves
wrong, because the backslashes must survive bash AND sed AND sh. Removing the
interpolation removes the class, and the next author does not need to know that
apostrophes were forbidden.

475 tests pass, clippy clean.

crates/cm-api/src/phase_runner.rs

@@ -101,7 +101,15 @@ async fn capture_finished_coding_phases(pool: &PgPool) -> Result<(), String> {
         "SELECT mp.id, mp.mission_id, mp.kind, mp.config, m.runtime_kind
            FROM mission_phases mp
            JOIN missions m ON m.id = mp.mission_id
-          WHERE mp.status = 'completed'
+          -- Terminal, not successful. A phase that did the work and missed its
+          -- goal condition still produced a diff, and that diff is what an
+          -- operator needs in order to see WHY it missed and what the next pass
+          -- can build on. Capturing only completed phases meant that the moment
+          -- an unmet phase began reporting failed — correctly — its work was
+          -- silently discarded. What was produced, and whether the goal was met,
+          -- are different facts: the artifact records the first and mp.status
+          -- the second.
+          WHERE mp.status IN ('completed', 'failed')
             AND m.repo_id IS NOT NULL
             AND NOT EXISTS (
                 SELECT 1 FROM mission_artifacts a

scripts/verify-mission-delivery.sh

@@ -30,6 +30,7 @@
 #   scripts/verify-mission-delivery.sh chain           # phase continuity
 #   scripts/verify-mission-delivery.sh multirole       # 3 roles + real tests
 #   scripts/verify-mission-delivery.sh noop            # empty phase must FAIL
+#   scripts/verify-mission-delivery.sh microvm         # runs in a guest kernel + fans out
 #   scripts/verify-mission-delivery.sh all             # everything
 #
 # Environment:
@@ -46,6 +47,11 @@ set -uo pipefail
 HOST="${CLAWMATES_HOST:-gw-04}"
 OWNER="${CLAWMATES_OWNER_EMAIL:-om[email protected]}"
 MISSIONS_ROOT="${CLAWMATES_MISSIONS_ROOT:-/var/lib/clawmates-missions}"
+# Real host kernels, read at start-up. The microvm scenario asserts the agent's
+# kernel differs from BOTH, which proves it ran in a guest without pinning a
+# vmlinux version that an upgrade would invalidate.
+GW_KERNEL=$(ssh "$HOST" 'uname -r' 2>/dev/null | tr -d '[:space:]')
+NODE_KERNEL=$(ssh "${FLEET_NODE:-osobh@tank}" 'uname -r' 2>/dev/null | tr -d '[:space:]')
 REPO_ID="${CLAWMATES_REPO_ID:-f8bbe4d7-2878-40c8-b657-7a7f6031def1}"
 TEAM_TEMPLATE="${CLAWMATES_TEAM_TEMPLATE:-7e453826-41c4-4425-bab5-8f11fd0a14d7}"
 MISSION_TIMEOUT="${MISSION_TIMEOUT:-1800}"
@@ -89,9 +95,19 @@ mint_session() {
 
 api() { # api <token> <METHOD> <path> [json]
   local t="$1" m="$2" p="$3" b="${4:-}"
+  # The JSON body travels on STDIN (`curl -d @-`), not embedded in the command.
+  #
+  # It used to be interpolated into a single-quoted `-d '...'` inside a
+  # double-quoted ssh command, which works only for bodies containing neither
+  # apostrophes nor anything else the two shells rewrite. A task description
+  # saying "the crate's test suite" ended the quoting and the remote shell died
+  # with "unexpected EOF"; two attempts at escaping it were themselves wrong,
+  # because the backslashes have to survive bash AND sed AND sh. Removing the
+  # interpolation removes the whole class.
   if [ -n "$b" ]; then
-    ssh "$HOST" "docker run --rm --network clawmates_core curlimages/curl:latest -s -X $m \
+    printf '%s' "$b" | ssh "$HOST" "docker run --rm -i --network clawmates_core \
-      -H 'Authorization: Bearer $t' -H 'Content-Type: application/json' -d '$b' \
+      curlimages/curl:latest -s -X $m \
+      -H 'Authorization: Bearer $t' -H 'Content-Type: application/json' -d @- \
       http://clawmates_server_1:8080$p"
   else
     ssh "$HOST" "docker run --rm --network clawmates_core curlimages/curl:latest -s -X $m \
@@ -99,6 +115,7 @@ api() { # api <token> <METHOD> <path> [json]
   fi
 }
 
+
 # ── The uid probe ────────────────────────────────────────────────
 #
 # The structural claim copy mode makes is that the host checkout has exactly
@@ -230,6 +247,114 @@ for a in d.get("artifacts") or []:
     'https://git.redclaw.dev/api/v1/repos/$repo/raw/$3?ref=$enc'"
 }
 
+# ── Scenario: a phase runs inside a microVM, and fans out ─────────
+#
+# Guards everything the microVM track proved by hand: that the agent ran in a
+# GUEST kernel rather than on a host, that Claude Code could actually delegate,
+# and that the work came back and landed.
+#
+# The kernel line is the assertion that cannot pass by accident. Every other
+# check here would also pass if the phase had quietly run in a container on the
+# gateway; only the kernel says WHERE it ran. It is compared against the real
+# host kernels rather than pinned to a version, so upgrading `vmlinux` does not
+# turn this into a false failure.
+
+MICROVM_BODY=$(cat <<JSON
+{"title":"verify: a phase runs inside a microVM",
+ "template_kind":"research_and_code",
+ "repo_id":"$REPO_ID",
+ "runtime_kind":"microvm",
+ "backend":"claude",
+ "description":"Prove a coding phase executes in a Firecracker microVM and can delegate.",
+ "phases":[
+  {"kind":"coding","order_idx":0,"config":{"commit_policy":"always","max_iterations":1,
+    "done_when":"A file named MICROVM.md exists at the repository root and contains at least two lines, the second of which is a Linux kernel release string.",
+    "task":"1. Use the verifier subagent to run the COMPLETE test suite of this crate and report what it found. Do not run it yourself and report that instead — the point is an independent check.\n2. Write MICROVM.md at the repository root with exactly two lines: the first is the test-result summary line the verifier reported, the second is the kernel release from running uname -r.\nCreate no other files."}}
+ ]}
+JSON
+)
+
+# How many subagents the phase actually spawned, per the server's own count of
+# Claude Code's per-subagent transcripts. Read from the log because that is
+# where `phase_runner` reports it; "?" means the probe could not run, which is
+# not the same as zero.
+subagent_count() { # subagent_count <mission>
+  ssh "$HOST" "docker logs --since 60m clawmates_server_1 2>&1 \
+    | grep -F 'microvm phase' | grep -F '$1' | tail -1" \
+    | sed -n 's/.*subagents: \([0-9?]*\).*/\1/p'
+}
+
+assert_microvm() { # <token> <mission> <report>
+  local token="$1" mission="$2" report="$3" delivered kernel subs indep
+  while read -r idx status files pushed _branch cerr perr; do
+    [ "$status" = "completed" ] || fail "microvm: phase $idx status=$status (commit_error=$cerr push_error=$perr)"
+    [ "$pushed" = "True" ] || fail "microvm: phase $idx not pushed (commit_error=$cerr push_error=$perr)"
+    case "$files" in 0|-) fail "microvm: phase $idx delivered no files" ;; esac
+  done <<<"$report"
+
+  delivered=$(fetch_delivered "$token" "$mission" MICROVM.md) \
+    || { fail "microvm: could not read MICROVM.md from the pushed branch"; return 1; }
+
+  # The decisive check: the second line is the kernel the agent ran on.
+  kernel=$(printf '%s\n' "$delivered" | sed -n '2p' | tr -d '[:space:]')
+  if [ -z "$kernel" ]; then
+    fail "microvm: MICROVM.md has no kernel line: $(printf '%s' "$delivered" | tr '\n' '|')"
+  elif [ "$kernel" = "$GW_KERNEL" ]; then
+    fail "microvm: the agent ran on the GATEWAY kernel ($kernel) — not in a VM at all"
+  elif [ "$kernel" = "$NODE_KERNEL" ]; then
+    fail "microvm: the agent ran on the fleet NODE's kernel ($kernel) — not in a VM at all"
+  else
+    pass "microvm: the agent ran under a guest kernel ($kernel), not the gateway's ($GW_KERNEL) or the node's ($NODE_KERNEL)"
+  fi
+
+  # Fan-out. Before the `Agent` tool was added to the allowlist this was
+  # structurally impossible, and nothing said so.
+  subs=$(subagent_count "$mission")
+  case "$subs" in
+    ''|'?') fail "microvm: could not count subagents (probe did not run) — delegation UNPROVEN" ;;
+    0)      fail "microvm: the phase spawned no subagents, so the verifier never ran" ;;
+    *)      pass "microvm: the lead delegated to $subs subagent(s)" ;;
+  esac
+
+  # And the verdict: judged, and by whom. `independent` is only true when the
+  # judge came from a different provider family than the agent.
+  indep=$(ssh "$HOST" "docker exec clawmates_postgres_1 psql -U postgres -d clawmates -tAc \
+    \"select met || ' ' || independent || ' ' || model from mission_phase_evaluations \
+      where phase_id in (select id from mission_phases where mission_id='$mission') \
+      order by created_at desc limit 1;\"" | head -1 | tr -d '\r')
+  case "$indep" in
+    "t t "*) pass "microvm: condition met, judged independently (${indep##* })" ;;
+    "t f "*) info "microvm: condition met but judged by the agent's own provider family (${indep##* }) — set CLAWMATES_VALIDATOR_MODEL" ;;
+    "f "*)   fail "microvm: the judge says the condition was NOT met: $indep" ;;
+    *)       fail "microvm: no verdict recorded for the phase (done_when was set)" ;;
+  esac
+}
+
+# ── Negative control: a backend no node can run ──────────────────
+#
+# Placement requires the mission's backend image to exist ON a node, not merely
+# that the node has KVM. Without this check the positive scenario above would
+# pass just as well against a scheduler that ignored `backend` entirely — which
+# is what it did until the first real microvm mission landed on a node that had
+# no such rootfs.
+scenario_microvm_unavailable_backend() {
+  local token mission body status
+  token=$(mint_session) || { norun "microvm-negctl: could not mint a session"; return 1; }
+  body=$(printf '%s' "$MICROVM_BODY" | sed 's/"backend":"claude"/"backend":"definitely-not-built"/')
+  mission=$(create_mission "$token" "$(echo "$body" | tr -d '\n')") \
+    || { norun "microvm-negctl: mission create failed"; return 1; }
+  info "microvm-negctl: mission=$mission"
+  api "$token" PATCH "/api/missions/$mission/status" '{"status":"running"}' >/dev/null 2>&1
+  sleep 5
+  status=$(ssh "$HOST" "docker exec clawmates_postgres_1 psql -U postgres -d clawmates -tAc \
+    \"select status from missions where id='$mission';\"" | head -1 | tr -d '[:space:]')
+  if [ "$status" = "draft" ]; then
+    pass "microvm-negctl: a backend no node can run is refused at launch (mission stayed draft)"
+  else
+    fail "microvm-negctl: a mission asking for an unbuilt image reached status=$status"
+  fi
+}
+
 run_scenario() { # run_scenario <label> <json> <assert-fn>
   local label="$1" body="$2" assert_fn="$3" token mission status
   # Every one of these MUST go through fail()/norun(). The first version of
@@ -412,15 +537,21 @@ case "${1:-all}" in
   noop)
     run_scenario noop "$NOOP_BODY" assert_noop
     ;;
+  microvm)
+    run_scenario microvm "$(echo "$MICROVM_BODY" | tr -d '\n')" assert_microvm
+    scenario_microvm_unavailable_backend
+    ;;
   all)
     selftest_uid_probe
     run_scenario chain "$CHAIN_BODY" assert_chain
     body=${MULTIROLE_BODY//__TEAM__/$TEAM_TEMPLATE}
     run_scenario multirole "${body//__REPO__/$REPO_ID}" assert_multirole
     run_scenario noop "$NOOP_BODY" assert_noop
+    run_scenario microvm "$(echo "$MICROVM_BODY" | tr -d '\n')" assert_microvm
+    scenario_microvm_unavailable_backend
     ;;
   *)
-    die "unknown scenario: $1 (selftest|uids|chain|multirole|noop|all)"
+    die "unknown scenario: $1 (selftest|uids|chain|multirole|noop|microvm|all)"
     ;;
 esac