Merge: stop three launch failures from passing as success

Verified against the deployed stack: scripts/verify-mission-delivery.sh all
→ 7/7, chain phase 0 now files=1 pushed=true (was 0 files, no error).

Co-Authored-By: Claude Opus 5 <[email protected]>

crates/cm-api/src/mission_delivery.rs

@@ -227,13 +227,41 @@ pub async fn capture_phase_diff_at(
     // A repo with nothing to add is fine; keep going and let the diff be empty.
     let _ = git(&repo, &add).await;
 
+    // A failed `git diff` and a phase that changed nothing both yield an empty
+    // string, and `unwrap_or_default` used to erase the difference: a corrupt
+    // index or an unreadable base would land `empty: true, files_changed: 0` —
+    // byte-identical to an honest no-op, and just as quiet. Whatever went
+    // wrong is recorded so the artifact can say which of the two it was.
+    let mut diff_error: Option<String> = None;
+    let mut note_diff_failure = |what: &str, e: String| {
+        eprintln!(
+            "mission_delivery: mission {mission_id} phase {phase_id} could not compute \
+             {what} against {base_sha}: {e}"
+        );
+        if diff_error.is_none() {
+            diff_error = Some(format!("{what}: {}", e.chars().take(300).collect::<String>()));
+        }
+    };
+
     let mut diff_args = vec!["diff", base_sha.as_str(), "--"];
     diff_args.extend(excludes.iter().map(String::as_str));
-    let patch = git(&repo, &diff_args).await.unwrap_or_default();
+    let patch = match git(&repo, &diff_args).await {
+        Ok(p) => p,
+        Err(e) => {
+            note_diff_failure("patch", e);
+            String::new()
+        }
+    };
 
     let mut stat_args = vec!["diff", base_sha.as_str(), "--stat", "--"];
     stat_args.extend(excludes.iter().map(String::as_str));
-    let diffstat = git(&repo, &stat_args).await.unwrap_or_default();
+    let diffstat = match git(&repo, &stat_args).await {
+        Ok(s) => s,
+        Err(e) => {
+            note_diff_failure("diffstat", e);
+            String::new()
+        }
+    };
 
     // Put the index back. `--intent-to-add` is a mutation of the agent's
     // workspace, and capture must not change what a later commit would see.
@@ -293,9 +321,14 @@ pub async fn capture_phase_diff_at(
     // Gate, then publish. Both are best-effort on top of an artifact that has
     // already landed: a phase whose tests fail, or whose push is rejected,
     // still has its patch on disk and its work on a local branch.
+    //
+    // `empty` suppresses publishing, so a diff we could not COMPUTE would
+    // otherwise skip the push and leave `push_error: null` — the phase looking
+    // exactly like one that correctly had nothing to publish. See
+    // [`untrusted_empty_reason`].
     let mut outcome: Option<TestOutcome> = None;
     let mut published: Option<Publish> = None;
-    let mut publish_error: Option<String> = None;
+    let mut publish_error: Option<String> = untrusted_empty_reason(empty, diff_error.as_deref());
     if let Some(c) = committed.as_ref() {
         if !empty {
             if gate == Gate::OnGreenTests {
@@ -379,6 +412,10 @@ pub async fn capture_phase_diff_at(
         "insertions": insertions,
         "deletions": deletions,
         "empty": empty,
+        // Non-null means `empty`/`files_changed` describe a failed read, not
+        // an unchanged tree. Readers that treat `empty: true` as "the phase
+        // did nothing" must check this first.
+        "diff_error": diff_error,
         "truncated": truncated,
         "excluded_paths": EXCLUDED_PATHS,
     });
@@ -958,6 +995,24 @@ pub async fn record_uncapturable(
     .map_err(|e| format!("register uncapturable marker: {e}"))
 }
 
+/// Why an empty patch must not be believed, if it must not be believed.
+///
+/// An empty patch has two causes that produce identical bytes: the tree really
+/// did not change, or `git diff` failed and we have no idea what the tree
+/// looks like. The first is an ordinary outcome; the second is a platform
+/// fault. Returning `Some` for the second is what stops the fault from being
+/// filed under the ordinary outcome — the recurring shape where a failure and
+/// a legitimate negative share one representation.
+fn untrusted_empty_reason(empty: bool, diff_error: Option<&str>) -> Option<String> {
+    match (empty, diff_error) {
+        (true, Some(why)) => Some(format!(
+            "not published: the diff could not be computed, so an empty patch \
+             cannot be trusted to mean an unchanged tree ({why})"
+        )),
+        _ => None,
+    }
+}
+
 #[cfg(test)]
 mod tests {
     use super::*;
@@ -1060,6 +1115,28 @@ mod tests {
 
     /// An empty stat means an empty phase, not a parse failure. This is the
     /// case that must still produce an artifact.
+    /// The whole point: a tree that genuinely did not change stays silent, and
+    /// a diff that could not be computed does not get to borrow that silence.
+    #[test]
+    fn an_uncomputable_diff_is_not_an_unchanged_tree() {
+        assert_eq!(
+            untrusted_empty_reason(true, None),
+            None,
+            "a genuinely unchanged tree must not report an error"
+        );
+        let reason = untrusted_empty_reason(true, Some("patch: fatal: bad object"))
+            .expect("an empty patch from a FAILED diff must be reported, not accepted");
+        assert!(
+            reason.contains("bad object"),
+            "the reason must name what went wrong, got: {reason}"
+        );
+        assert_eq!(
+            untrusted_empty_reason(false, Some("diffstat: fatal: bad object")),
+            None,
+            "a non-empty patch stands on its own even if the diffstat failed"
+        );
+    }
+
     #[test]
     fn an_empty_diffstat_is_all_zeroes() {
         assert_eq!(parse_diffstat(""), (0, 0, 0));

crates/cm-api/src/mission_fs.rs

@@ -90,6 +90,60 @@ pub async fn copy_in(
         .map_err(|e| format!("copy into {container}:{CONTAINER_MISSION_DIR}: {e}"))
 }
 
+/// Build a one-entry tar. Split out from [`put_file`] so the size-independence
+/// that is the whole point can be tested without Docker.
+fn single_file_archive(name: &str, contents: &[u8]) -> Result<Vec<u8>, String> {
+    let mut header = tar::Header::new_gnu();
+    header
+        .set_path(name)
+        .map_err(|e| format!("tar path {name}: {e}"))?;
+    header.set_size(contents.len() as u64);
+    header.set_mode(0o600);
+    header.set_entry_type(tar::EntryType::Regular);
+    header.set_cksum();
+
+    let mut builder = tar::Builder::new(Vec::new());
+    builder
+        .append(&header, contents)
+        .map_err(|e| format!("tar {name}: {e}"))?;
+    builder
+        .into_inner()
+        .map_err(|e| format!("finish archive for {name}: {e}"))
+}
+
+/// Write one file into a container, at any size.
+///
+/// The obvious way to do this is `sh -c "printf … > file"`, and it works right
+/// up until the payload approaches `ARG_MAX`, at which point exec fails with
+/// `argument list too long`. That is a size-dependent failure in a code path
+/// whose payload grows with use, which makes it a bug that ships green and
+/// surfaces in production — as it did, silently unpinning every agent in
+/// mission `019fcf62`. Tar has no argv limit.
+///
+/// The write is not atomic. Callers that need it can upload beside the target
+/// and rename; the config writer does not, because the daemon reads its config
+/// once at boot and is restarted afterwards.
+pub async fn put_file(
+    docker: &Docker,
+    container: &str,
+    path: &str,
+    contents: &[u8],
+) -> Result<(), String> {
+    let (dir, file) = path
+        .rsplit_once('/')
+        .ok_or_else(|| format!("{path} is not an absolute path"))?;
+    let dir = if dir.is_empty() { "/" } else { dir };
+
+    let archive = single_file_archive(file, contents)?;
+    let opts = bollard::query_parameters::UploadToContainerOptionsBuilder::default()
+        .path(dir)
+        .build();
+    docker
+        .upload_to_container(container, Some(opts), bollard::body_full(archive.into()))
+        .await
+        .map_err(|e| format!("upload {path} to {container}: {e}"))
+}
+
 /// Copy a directory back out of a container onto the host.
 pub async fn copy_out(
     docker: &Docker,
@@ -260,6 +314,39 @@ mod tests {
         }
     }
 
+    /// The regression this exists for: a config large enough to blow `ARG_MAX`
+    /// via `sh -c` must round-trip untouched. 2 MB is well past the ~128 KB
+    /// limit that unpinned every agent in mission `019fcf62`.
+    #[test]
+    fn a_file_far_past_arg_max_round_trips() {
+        let big = "workspace_path = \"/mission/repo\"\n".repeat(64 * 1024);
+        assert!(big.len() > 2_000_000, "the fixture must exceed ARG_MAX");
+
+        let archive = single_file_archive("config.toml", big.as_bytes()).unwrap();
+        let tmp = tempfile::tempdir().unwrap();
+        unpack_into(&archive, tmp.path()).unwrap();
+
+        assert_eq!(
+            std::fs::read_to_string(tmp.path().join("config.toml")).unwrap(),
+            big,
+            "a large config must survive byte-for-byte"
+        );
+    }
+
+    /// TOML holding quotes, newlines and backslashes went through a shell
+    /// before; nothing may depend on quoting now.
+    #[test]
+    fn shell_metacharacters_survive_the_archive() {
+        let nasty = "path = \"/a'b\\\"c\"\n$(rm -rf /) `id` \\\\ \n";
+        let archive = single_file_archive("config.toml", nasty.as_bytes()).unwrap();
+        let tmp = tempfile::tempdir().unwrap();
+        unpack_into(&archive, tmp.path()).unwrap();
+        assert_eq!(
+            std::fs::read_to_string(tmp.path().join("config.toml")).unwrap(),
+            nasty
+        );
+    }
+
     #[test]
     fn an_empty_directory_packs_without_error() {
         let tmp = tempfile::tempdir().unwrap();

crates/cm-api/src/mission_orchestrator.rs

@@ -228,31 +228,35 @@ pub async fn on_launch(
     // is a PathBuf the prop-schema won't expose — see provision_claw), so
     // we patch the shared config file directly on the per-mission runtime
     // container. The daemon picks it up on the same reload that surfaces
-    // the freshly-provisioned claws for the run. Non-fatal: without the
+    // the freshly-provisioned claws for the run.
-    // pin, agents still write (to the sandbox) but the committer can't
+    //
-    // find the changes in /mission/repo.
+    // FATAL, deliberately. This was "non-fatal: agents still write (to the
+    // sandbox) but the committer can't find the changes in /mission/repo" —
+    // which is to say, the mission runs to completion and delivers nothing.
+    // Mission `019fcf62` did exactly that: the pin failed with `argument list
+    // too long`, one line of stderr scrolled past, and phase 0 reported
+    // `completed` with zero files, no commit error and no push error. A launch
+    // that cannot bind its agents to the repo has no path to delivering work,
+    // so it must fail at launch where someone is still looking.
     if !provisioned_claws.is_empty() && mission_gateway.is_some() {
         if let Some(mp) = crate::mission_runtime::MissionRuntimeProvisioner::from_env() {
-            match mp
+            mp.pin_agent_workspaces(mission_id, &provisioned_claws, "/mission/repo")
-                .pin_agent_workspaces(mission_id, &provisioned_claws, "/mission/repo")
                 .await
-            {
+                .map_err(|e| {
-                Ok(()) => {
+                    format!(
-                    // The daemon reads config ONCE at boot and never re-reads
+                        "could not pin agent workspaces to /mission/repo ({e}) — the mission \
-                    // the file, so the pin is invisible until it restarts. Its
+                         would run with its agents writing to their sandboxes, delivering nothing"
-                    // agents were created through its own config API, so they
+                    )
-                    // are already persisted to the file and survive the
+                })?;
-                    // restart; the pairing code is re-minted on every launch.
+            // The daemon reads config ONCE at boot and never re-reads the
-                    if let Err(e) = mp.restart_container(mission_id).await {
+            // file, so the pin is invisible until it restarts. Its agents were
-                        eprintln!(
+            // created through its own config API, so they are already
-                            "mission_orchestrator: restart runtime for {mission_id} failed (continuing, workspace pin will not apply): {e}"
+            // persisted to the file and survive the restart; the pairing code
-                        );
+            // is re-minted on every launch. Equally fatal: an unrestarted
-                    }
+            // daemon is an unpinned daemon.
-                }
+            mp.restart_container(mission_id).await.map_err(|e| {
-                Err(e) => eprintln!(
+                format!("could not restart the runtime to apply the workspace pin: {e}")
-                    "mission_orchestrator: pin workspaces for mission {mission_id} failed (continuing): {e}"
+            })?;
-                ),
-            }
         }
     }
 

crates/cm-api/src/mission_runtime.rs

@@ -24,7 +24,6 @@
 //! carry the current binding (both null when torn down or never
 //! provisioned).
 
-use base64::Engine;
 use bollard::exec::{CreateExecOptions, StartExecResults};
 use bollard::models::{
     ContainerCreateBody, EndpointSettings, HostConfig, Mount, MountTypeEnum, NetworkConnectRequest,
@@ -660,18 +659,16 @@ impl MissionRuntimeProvisioner {
             return Ok(());
         }
 
-        let b64 = base64::engine::general_purpose::STANDARD.encode(edited.as_bytes());
+        // Upload rather than exec. This used to base64 the whole config into a
-        // Decode to a sibling temp then atomically move over the live file,
+        // single `sh -c` argument, which works until the file grows — config
-        // so a partial write can never leave the daemon with truncated TOML.
+        // gains a block per provisioned claw — and then fails with
-        let script = format!(
+        // `argument list too long`. Mission `019fcf62` hit exactly that: the
-            "printf %s '{b64}' | base64 -d > {CONFIG_PATH}.tmp && mv {CONFIG_PATH}.tmp {CONFIG_PATH}"
+        // pin never applied, its agents never saw `/mission/repo`, and phase 0
-        );
+        // completed having written nothing. The tar API has no argv limit, so
-        let out = self
+        // the failure mode is gone rather than merely further away.
-            .exec_capture(&name, vec!["sh".into(), "-c".into(), script])
+        crate::mission_fs::put_file(&self.docker, &name, CONFIG_PATH, edited.as_bytes())
-            .await?;
+            .await
-        if !out.trim().is_empty() {
+            .map_err(|e| format!("write runtime config.toml: {e}"))?;
-            return Err(format!("write runtime config.toml: {out}"));
-        }
         eprintln!(
             "mission_runtime: pinned {pinned} workspace(s) → {workspace_path} for mission {mission_id}"
         );

scripts/verify-mission-delivery.sh

@@ -0,0 +1,394 @@
+#!/usr/bin/env bash
+# Verify mission delivery end-to-end against the deployed stack.
+#
+# Every verification of the delivery chain so far has been a throwaway bash
+# script, written fresh per run and discarded. One of them printed
+#
+#     --- host .git owner (should be one uid) ---
+#     UNKNOWN
+#
+# and that `UNKNOWN` is the whole reason this file exists: the probe could not
+# read its subject, and said so in a way that looked like output rather than
+# like failure. It would have printed `UNKNOWN` just as happily if the uid
+# split had come back. That is seam 4 — absence encoded as a legitimate value —
+# reappearing inside the tool built to detect seam 1.
+#
+# So the rules here are structural, not stylistic:
+#
+#   1. A probe returns a value or exits non-zero. There is no third outcome,
+#      no placeholder, no empty string that a caller might read as "fine".
+#   2. The uid probe is self-tested against a mission KNOWN to have the split,
+#      before any result from it is believed. A probe that cannot see the
+#      known-bad case has not verified the good one — it has only failed to
+#      look. `--selftest-only` runs that check alone.
+#   3. A scenario that never ran is FAIL-NORUN, never PASS. An absent branch
+#      is indistinguishable from a dead container, and once scored PASS.
+#
+# Usage:
+#   scripts/verify-mission-delivery.sh selftest        # probe self-test alone
+#   scripts/verify-mission-delivery.sh uids <mission>  # uid probe, one mission
+#   scripts/verify-mission-delivery.sh chain           # phase continuity
+#   scripts/verify-mission-delivery.sh multirole       # 3 roles + real tests
+#   scripts/verify-mission-delivery.sh all             # everything
+#
+# Environment:
+#   CLAWMATES_HOST          ssh host running the stack   (default gw-04)
+#   CLAWMATES_OWNER_EMAIL   account to mint a session for (default om.sobh@…)
+#   CLAWMATES_UID_CONTROL   mission id/prefix known to have a uid split;
+#                           auto-discovered when unset
+#   CLAWMATES_REPO_ID       scratch repo for the delivery scenarios
+#   CLAWMATES_TEAM_TEMPLATE team template for the delivery scenarios
+#   MISSION_TIMEOUT         seconds to wait for a mission (default 1800)
+
+set -uo pipefail
+
+HOST="${CLAWMATES_HOST:-gw-04}"
+OWNER="${CLAWMATES_OWNER_EMAIL:-om[email protected]}"
+MISSIONS_ROOT="${CLAWMATES_MISSIONS_ROOT:-/var/lib/clawmates-missions}"
+REPO_ID="${CLAWMATES_REPO_ID:-f8bbe4d7-2878-40c8-b657-7a7f6031def1}"
+TEAM_TEMPLATE="${CLAWMATES_TEAM_TEMPLATE:-7e453826-41c4-4425-bab5-8f11fd0a14d7}"
+MISSION_TIMEOUT="${MISSION_TIMEOUT:-1800}"
+
+FAILURES=0
+CHECKS=0
+
+# ── Reporting ────────────────────────────────────────────────────
+#
+# Deliberately only three verdicts, and NORUN is one of them. "The scenario
+# did not execute" must not be able to borrow PASS's vocabulary.
+
+pass()  { printf 'PASS       %s\n' "$*"; CHECKS=$((CHECKS + 1)); }
+fail()  { printf 'FAIL       %s\n' "$*"; CHECKS=$((CHECKS + 1)); FAILURES=$((FAILURES + 1)); }
+norun() { printf 'FAIL-NORUN %s\n' "$*"; CHECKS=$((CHECKS + 1)); FAILURES=$((FAILURES + 1)); }
+info()  { printf '           %s\n' "$*"; }
+die()   { printf 'ABORT      %s\n' "$*" >&2; exit 2; }
+
+# ── Session + API ────────────────────────────────────────────────
+
+mint_session() {
+  local secret hash
+  secret="verify-$(openssl rand -hex 16)"
+  hash=$(printf '%s' "$secret" | openssl dgst -sha256 -binary \
+    | openssl base64 -A | tr '+/' '-_' | tr -d '=')
+  local rows
+  rows=$(ssh "$HOST" "docker exec clawmates_postgres_1 psql -U postgres -d clawmates -tAc \
+    \"insert into auth_sessions (user_id, token_hash, expires_at) \
+      select id, '$hash', now() + interval '90 minutes' \
+      from users where email='$OWNER' limit 1 returning 1;\"" 2>/dev/null \
+    | head -1 | tr -d '[:space:]')
+  # `head -1`: psql emits the returned row AND its `INSERT 0 1` command tag,
+  # and collapsing both gave `1INSERT01`, which failed this check on a mint
+  # that had in fact worked.
+  #
+  # No row inserted means no such user — without this check the script would
+  # carry on and report every API call as a delivery failure.
+  [ "$rows" = "1" ] || { echo "could not mint a session for $OWNER (no such user?)" >&2; return 1; }
+  printf '%s' "$secret"
+}
+
+api() { # api <token> <METHOD> <path> [json]
+  local t="$1" m="$2" p="$3" b="${4:-}"
+  if [ -n "$b" ]; then
+    ssh "$HOST" "docker run --rm --network clawmates_core curlimages/curl:latest -s -X $m \
+      -H 'Authorization: Bearer $t' -H 'Content-Type: application/json' -d '$b' \
+      http://clawmates_server_1:8080$p"
+  else
+    ssh "$HOST" "docker run --rm --network clawmates_core curlimages/curl:latest -s -X $m \
+      -H 'Authorization: Bearer $t' http://clawmates_server_1:8080$p"
+  fi
+}
+
+# ── The uid probe ────────────────────────────────────────────────
+#
+# The structural claim copy mode makes is that the host checkout has exactly
+# one writer. This is what tests it. It prints a sorted, comma-separated uid
+# list on stdout and exits 0, or prints nothing and exits 1.
+
+probe_uids() { # probe_uids <mission-id-or-prefix>
+  local mission="$1" dir uids
+  dir=$(ssh "$HOST" "ls -d $MISSIONS_ROOT/$mission* 2>/dev/null | head -1" | tr -d '\r')
+  [ -n "$dir" ] || { echo "no mission dir under $MISSIONS_ROOT for $mission" >&2; return 1; }
+  ssh "$HOST" "sudo test -d '$dir/repo'" 2>/dev/null \
+    || { echo "$dir/repo is not a directory" >&2; return 1; }
+  # -printf '%U' over stat: one process for the whole tree, and it reports the
+  # numeric uid even when the host has no passwd entry for it.
+  uids=$(ssh "$HOST" "sudo find '$dir/repo' -xdev -printf '%U\n' 2>/dev/null | sort -un | paste -sd, -" | tr -d '\r')
+  # An empty result is not "no uids", it is a failed read. A checkout always
+  # contains files; if find returned nothing, find did not work.
+  [ -n "$uids" ] || { echo "uid read produced no output for $dir/repo" >&2; return 1; }
+  printf '%s' "$uids"
+}
+
+# Find a mission whose checkout still shows the pre-copy-mode split. Used as
+# the probe's negative control.
+discover_split_control() {
+  ssh "$HOST" "for d in $MISSIONS_ROOT/*/repo; do
+      [ -d \"\$d\" ] || continue
+      n=\$(sudo find \"\$d\" -xdev -printf '%U\n' 2>/dev/null | sort -un | wc -l)
+      if [ \"\$n\" -gt 1 ]; then basename \$(dirname \"\$d\"); break; fi
+    done" 2>/dev/null | tr -d '\r' | head -1
+}
+
+# ── Probe self-test ──────────────────────────────────────────────
+#
+# Run BEFORE trusting any uid result. A green uid report from a probe that
+# cannot detect the split is not evidence of anything.
+
+selftest_uid_probe() {
+  local control uids
+  control="${CLAWMATES_UID_CONTROL:-$(discover_split_control)}"
+  if [ -z "$control" ]; then
+    # Not a pass. Every checkout on the host is single-uid, which is the
+    # desired end state but leaves the probe unexercised — so say exactly
+    # that rather than implying the probe was validated.
+    info "selftest: no split-uid mission remains on $HOST to use as a control"
+    info "selftest: uid results below are UNVALIDATED (set CLAWMATES_UID_CONTROL)"
+    return 0
+  fi
+  uids=$(probe_uids "$control") || { fail "selftest: probe failed on control $control"; return 1; }
+  case "$uids" in
+    *,*) pass "selftest: probe reports the split on control $control (uids=$uids)" ;;
+    *)   fail "selftest: control $control reports a single uid ($uids) — the probe cannot detect the split it exists to find" ;;
+  esac
+}
+
+check_single_uid() { # check_single_uid <mission> <label>
+  local mission="$1" label="$2" uids
+  uids=$(probe_uids "$mission") || { fail "$label: uid probe could not read the checkout"; return 1; }
+  case "$uids" in
+    *,*) fail "$label: checkout has multiple writers (uids=$uids)" ;;
+    *)   pass "$label: checkout has exactly one writer (uid=$uids)" ;;
+  esac
+}
+
+# ── Mission lifecycle ────────────────────────────────────────────
+
+create_mission() { # create_mission <token> <json> -> mission id
+  local out id
+  out=$(api "$1" POST /api/missions "$(echo "$2" | tr -d '\n')")
+  id=$(printf '%s' "$out" | python3 -c 'import json,sys
+try: print(json.load(sys.stdin).get("id",""))
+except Exception: pass' 2>/dev/null)
+  [ -n "$id" ] || { echo "create failed: $out" >&2; return 1; }
+  printf '%s' "$id"
+}
+
+await_mission() { # await_mission <token> <mission> -> final status
+  local t="$1" m="$2" waited=0 status
+  while [ "$waited" -lt "$MISSION_TIMEOUT" ]; do
+    status=$(api "$t" GET "/api/missions/$m" | python3 -c 'import json,sys
+try: print(json.load(sys.stdin).get("status",""))
+except Exception: pass' 2>/dev/null)
+    case "$status" in
+      completed|failed|cancelled) printf '%s' "$status"; return 0 ;;
+    esac
+    sleep 20
+    waited=$((waited + 20))
+  done
+  printf 'timeout'
+}
+
+# Emit one `order_idx status files pushed commit_error push_error` line per
+# phase by joining phases to their code_diff artifacts.
+phase_report() { # phase_report <token> <mission>
+  api "$1" GET "/api/missions/$2" | python3 -c '
+import json, sys
+d = json.load(sys.stdin)
+art = {}
+for a in d.get("artifacts") or []:
+    if a.get("kind") == "code_diff":
+        art[a.get("phase_id")] = a.get("metadata") or {}
+for p in sorted(d.get("phases") or [], key=lambda p: p.get("order_idx", 0)):
+    m = art.get(p["id"], {})
+    print(p.get("order_idx"), p.get("status"),
+          m.get("files_changed", m.get("files", "-")),
+          m.get("pushed", "-"), m.get("branch", "-"),
+          json.dumps(m.get("commit_error")), json.dumps(m.get("push_error")))
+'
+}
+
+# Read a file back from the branch the mission actually pushed, so the
+# assertion is against the forge rather than against the host staging dir.
+fetch_delivered() { # fetch_delivered <token> <mission> <path>
+  local branch repo token
+  branch=$(api "$1" GET "/api/missions/$2" | python3 -c '
+import json, sys
+d = json.load(sys.stdin)
+for a in d.get("artifacts") or []:
+    b = (a.get("metadata") or {}).get("branch")
+    if b: print(b); break
+' 2>/dev/null)
+  [ -n "$branch" ] || return 1
+  token=$(ssh "$HOST" 'docker exec clawmates_server_1 printenv GITEA_TOKEN' | tr -d '\r')
+  repo=$(ssh "$HOST" "docker exec clawmates_postgres_1 psql -U postgres -d clawmates -tAc \
+    \"select owner || '/' || name from repos where id='$REPO_ID';\"" \
+    | head -1 | tr -d '[:space:]')
+  [ -n "$token" ] && [ -n "$repo" ] || return 1
+  local enc; enc=$(printf '%s' "$branch" | sed 's|/|%2F|g')
+  ssh "$HOST" "curl -sf -H 'Authorization: token $token' \
+    'https://git.redclaw.dev/api/v1/repos/$repo/raw/$3?ref=$enc'"
+}
+
+run_scenario() { # run_scenario <label> <json> <assert-fn>
+  local label="$1" body="$2" assert_fn="$3" token mission status
+  # Every one of these MUST go through fail()/norun(). The first version of
+  # this function called a `die` that lived inside `$(...)` — which exits the
+  # command substitution's subshell, not the script — so a run where the
+  # session could not be minted printed two ABORT lines, incremented nothing,
+  # and ended with "all checks passed" and exit 0. The harness written to
+  # catch silent success produced silent success on its first real run.
+  token=$(mint_session) || { norun "$label: could not mint a session"; return 1; }
+  mission=$(create_mission "$token" "$body") || { norun "$label: mission create failed"; return 1; }
+  info "$label: mission=$mission"
+  api "$token" PATCH "/api/missions/$mission/status" '{"status":"running"}' >/dev/null
+  status=$(await_mission "$token" "$mission")
+  if [ "$status" = "timeout" ]; then
+    norun "$label: mission did not reach a terminal status in ${MISSION_TIMEOUT}s"
+    return 1
+  fi
+  local report; report=$(phase_report "$token" "$mission")
+  if [ -z "$report" ]; then
+    norun "$label: no phases reported — nothing executed"
+    return 1
+  fi
+  printf '%s\n' "$report" | sed 's/^/           phase /'
+  "$assert_fn" "$token" "$mission" "$report"
+  check_single_uid "$mission" "$label"
+}
+
+# ── Scenario: phase continuity ───────────────────────────────────
+#
+# Phase 1 must READ what phase 0 wrote. Independent phases cannot tell a
+# preserved checkout from a wiped one — which is exactly how a `reset --hard`
+# survived three green-looking runs.
+
+CHAIN_BODY=$(cat <<JSON
+{"title":"verify: phase 2 builds on phase 1",
+ "template_kind":"research_and_code",
+ "team_template_id":"$TEAM_TEMPLATE",
+ "repo_id":"$REPO_ID",
+ "description":"Build up a file CHAIN.md across two phases.",
+ "phases":[
+  {"kind":"coding","order_idx":0,"config":{"commit_policy":"always","max_iterations":1,
+    "task":"Create a file named CHAIN.md at the repository root containing exactly one line: STEP-ONE-OK. Do not create or modify any other file."}},
+  {"kind":"coding","order_idx":1,"config":{"commit_policy":"always","max_iterations":1,
+    "task":"Read the existing file CHAIN.md at the repository root. It was written by the previous phase and must already contain the line STEP-ONE-OK. Append a second line reading STEP-TWO-SAW-STEP-ONE, keeping the first line intact. If CHAIN.md does not exist, instead create a file named CHAIN_MISSING.md containing the single line PRIOR-PHASE-WORK-WAS-LOST, and do not create CHAIN.md."}}
+ ]}
+JSON
+)
+
+assert_chain() { # <token> <mission> <report>
+  local token="$1" mission="$2" report="$3" delivered
+  local phases; phases=$(printf '%s\n' "$report" | wc -l | tr -d ' ')
+  [ "$phases" = "2" ] || fail "chain: expected 2 phases, got $phases"
+  # Here-string, not a pipe: a `while` on the right of a pipe runs in a
+  # subshell, so every fail() inside it would increment a FAILURES that dies
+  # with the subshell and the script would exit 0 having reported failures.
+  while read -r idx status files pushed _branch cerr perr; do
+    [ "$status" = "completed" ] || fail "chain: phase $idx status=$status"
+    [ "$pushed" = "True" ] || fail "chain: phase $idx not pushed (commit_error=$cerr push_error=$perr)"
+    case "$files" in 0|-) fail "chain: phase $idx delivered no files" ;; esac
+  done <<<"$report"
+  delivered=$(fetch_delivered "$token" "$mission" CHAIN.md) \
+    || { fail "chain: could not read CHAIN.md from the pushed branch"; return 1; }
+  case "$delivered" in
+    *STEP-ONE-OK*STEP-TWO-SAW-STEP-ONE*)
+      pass "chain: phase 1 read phase 0's work and appended to it" ;;
+    *PRIOR-PHASE-WORK-WAS-LOST*)
+      fail "chain: phase 1 reported the prior phase's work was lost" ;;
+    *)
+      fail "chain: CHAIN.md does not show both lines: $(printf '%s' "$delivered" | tr '\n' '|')" ;;
+  esac
+}
+
+# ── Scenario: multi-role with a real test suite ──────────────────
+#
+# The workload that failed with `COMMIT_EDITMSG: Permission denied` under the
+# bind mount. REVIEW.md must carry cargo's own summary line, so the reviewer
+# had to run the suite rather than assert that it passed.
+
+MULTIROLE_BODY=$(cat <<'JSON'
+{"title":"verify: implement, test, review",
+ "template_kind":"research_and_code",
+ "team_template_id":"__TEAM__",
+ "repo_id":"__REPO__",
+ "description":"Extend the scratch Rust crate with a reviewed, tested function.",
+ "phases":[
+  {"kind":"coding","order_idx":0,"config":{"commit_policy":"always","max_iterations":1,
+    "task":"This task needs THREE distinct roles. Use a separate agent for each role; do not do all three yourself.\n1. IMPLEMENTER: in src/lib.rs add `pub fn divide(a: i64, b: i64) -> Option<i64>` returning None when b == 0, else Some(a / b).\n2. TESTER: add unit tests in the existing tests module named `divide_works` and `divide_by_zero_is_none`, covering both branches. Run `cargo test` and make it pass.\n3. REVIEWER: read the final src/lib.rs and write REVIEW.md at the repository root containing exactly three lines: `ROLES: 3`, `TESTS: <the test result summary line from cargo test>`, and `VERDICT: <one sentence>`.\nAll three files (src/lib.rs, REVIEW.md) must be left in the working tree."}}
+ ]}
+JSON
+)
+
+assert_multirole() { # <token> <mission> <report>
+  local token="$1" mission="$2" report="$3" review lib
+  while read -r idx status files pushed _branch cerr perr; do
+    [ "$status" = "completed" ] || fail "multirole: phase $idx status=$status"
+    [ "$pushed" = "True" ] || fail "multirole: phase $idx not pushed (commit_error=$cerr push_error=$perr)"
+    case "$files" in 0|-) fail "multirole: phase $idx delivered no files" ;; esac
+  done <<<"$report"
+  lib=$(fetch_delivered "$token" "$mission" src/lib.rs) \
+    || { fail "multirole: could not read src/lib.rs from the pushed branch"; return 1; }
+  case "$lib" in
+    *"fn divide"*) pass "multirole: divide() was delivered" ;;
+    *)             fail "multirole: src/lib.rs has no divide()" ;;
+  esac
+  case "$lib" in
+    *divide_by_zero_is_none*) pass "multirole: the zero-divisor test was delivered" ;;
+    *)                        fail "multirole: no divide_by_zero_is_none test" ;;
+  esac
+  review=$(fetch_delivered "$token" "$mission" REVIEW.md) \
+    || { fail "multirole: no REVIEW.md on the pushed branch"; return 1; }
+  # cargo's own words. A reviewer who merely claimed the tests passed cannot
+  # produce this line, which is the point of asserting on it.
+  case "$review" in
+    *"test result: ok."*) pass "multirole: REVIEW.md carries cargo's own test summary" ;;
+    *)                    fail "multirole: REVIEW.md has no 'test result: ok.' line" ;;
+  esac
+}
+
+# ── Entry point ──────────────────────────────────────────────────
+
+ssh -o BatchMode=yes -o ConnectTimeout=10 "$HOST" true 2>/dev/null \
+  || die "cannot ssh to $HOST"
+
+case "${1:-all}" in
+  selftest)
+    selftest_uid_probe
+    ;;
+  uids)
+    [ $# -ge 2 ] || die "usage: $0 uids <mission-id>"
+    selftest_uid_probe
+    check_single_uid "$2" "uids($2)"
+    ;;
+  chain)
+    selftest_uid_probe
+    run_scenario chain "$CHAIN_BODY" assert_chain
+    ;;
+  multirole)
+    selftest_uid_probe
+    body=${MULTIROLE_BODY//__TEAM__/$TEAM_TEMPLATE}
+    run_scenario multirole "${body//__REPO__/$REPO_ID}" assert_multirole
+    ;;
+  all)
+    selftest_uid_probe
+    run_scenario chain "$CHAIN_BODY" assert_chain
+    body=${MULTIROLE_BODY//__TEAM__/$TEAM_TEMPLATE}
+    run_scenario multirole "${body//__REPO__/$REPO_ID}" assert_multirole
+    ;;
+  *)
+    die "unknown scenario: $1 (selftest|uids|chain|multirole|all)"
+    ;;
+esac
+
+if [ "$FAILURES" -gt 0 ]; then
+  printf '\n%d of %d check(s) failed\n' "$FAILURES" "$CHECKS"
+  exit 1
+fi
+# Zero checks is not success. A run that asserted nothing must not be able to
+# print the same closing line as a run that asserted everything.
+if [ "$CHECKS" -eq 0 ]; then
+  printf '\nno checks ran — nothing was verified\n'
+  exit 1
+fi
+printf '\nall %d check(s) passed\n' "$CHECKS"