Omar Sobh
b17e18aa67
`select met || ' ' || independent` casts the booleans to `true`/`false`, but the
pattern matched `t`/`f` — psql's *column display* form. So the check reported "no
verdict recorded for the phase" while the row sat in the table saying met=true,
independent=true, glm-4.7.
A check that fails for a reason unrelated to what it checks is worse than no check:
it trains you to ignore the output. The booleans are cast explicitly now so the
shape cannot drift again, and the failure message prints what it actually got.
`verify-mission-delivery.sh microvm` now passes 5/5 against production:
- the agent ran under guest kernel 6.1.128, not the gateway's 6.8.0-124 or the
node's 7.0.0-28 — the one assertion that cannot pass by accident
- the lead delegated to 1 subagent
- the condition was met and judged INDEPENDENTLY by glm-4.7
- the checkout has exactly one writer (uid 65532)
- negative control: a backend no node can run is refused at launch
574 lines
28 KiB
Bash
Executable File
574 lines
28 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
# Verify mission delivery end-to-end against the deployed stack.
|
|
#
|
|
# Every verification of the delivery chain so far has been a throwaway bash
|
|
# script, written fresh per run and discarded. One of them printed
|
|
#
|
|
# --- host .git owner (should be one uid) ---
|
|
# UNKNOWN
|
|
#
|
|
# and that `UNKNOWN` is the whole reason this file exists: the probe could not
|
|
# read its subject, and said so in a way that looked like output rather than
|
|
# like failure. It would have printed `UNKNOWN` just as happily if the uid
|
|
# split had come back. That is seam 4 — absence encoded as a legitimate value —
|
|
# reappearing inside the tool built to detect seam 1.
|
|
#
|
|
# So the rules here are structural, not stylistic:
|
|
#
|
|
# 1. A probe returns a value or exits non-zero. There is no third outcome,
|
|
# no placeholder, no empty string that a caller might read as "fine".
|
|
# 2. The uid probe is self-tested against a mission KNOWN to have the split,
|
|
# before any result from it is believed. A probe that cannot see the
|
|
# known-bad case has not verified the good one — it has only failed to
|
|
# look. `--selftest-only` runs that check alone.
|
|
# 3. A scenario that never ran is FAIL-NORUN, never PASS. An absent branch
|
|
# is indistinguishable from a dead container, and once scored PASS.
|
|
#
|
|
# Usage:
|
|
# scripts/verify-mission-delivery.sh selftest # probe self-test alone
|
|
# scripts/verify-mission-delivery.sh uids <mission> # uid probe, one mission
|
|
# scripts/verify-mission-delivery.sh chain # phase continuity
|
|
# scripts/verify-mission-delivery.sh multirole # 3 roles + real tests
|
|
# scripts/verify-mission-delivery.sh noop # empty phase must FAIL
|
|
# scripts/verify-mission-delivery.sh microvm # runs in a guest kernel + fans out
|
|
# scripts/verify-mission-delivery.sh all # everything
|
|
#
|
|
# Environment:
|
|
# CLAWMATES_HOST ssh host running the stack (default gw-04)
|
|
# CLAWMATES_OWNER_EMAIL account to mint a session for (default om.sobh@…)
|
|
# CLAWMATES_UID_CONTROL mission id/prefix known to have a uid split;
|
|
# auto-discovered when unset
|
|
# CLAWMATES_REPO_ID scratch repo for the delivery scenarios
|
|
# CLAWMATES_TEAM_TEMPLATE team template for the delivery scenarios
|
|
# MISSION_TIMEOUT seconds to wait for a mission (default 1800)
|
|
|
|
set -uo pipefail
|
|
|
|
HOST="${CLAWMATES_HOST:-gw-04}"
|
|
OWNER="${CLAWMATES_OWNER_EMAIL:-om[email protected]}"
|
|
MISSIONS_ROOT="${CLAWMATES_MISSIONS_ROOT:-/var/lib/clawmates-missions}"
|
|
# Real host kernels, read at start-up. The microvm scenario asserts the agent's
|
|
# kernel differs from BOTH, which proves it ran in a guest without pinning a
|
|
# vmlinux version that an upgrade would invalidate.
|
|
GW_KERNEL=$(ssh "$HOST" 'uname -r' 2>/dev/null | tr -d '[:space:]')
|
|
NODE_KERNEL=$(ssh "${FLEET_NODE:-osobh@tank}" 'uname -r' 2>/dev/null | tr -d '[:space:]')
|
|
REPO_ID="${CLAWMATES_REPO_ID:-f8bbe4d7-2878-40c8-b657-7a7f6031def1}"
|
|
TEAM_TEMPLATE="${CLAWMATES_TEAM_TEMPLATE:-7e453826-41c4-4425-bab5-8f11fd0a14d7}"
|
|
MISSION_TIMEOUT="${MISSION_TIMEOUT:-1800}"
|
|
|
|
FAILURES=0
|
|
CHECKS=0
|
|
|
|
# ── Reporting ────────────────────────────────────────────────────
|
|
#
|
|
# Deliberately only three verdicts, and NORUN is one of them. "The scenario
|
|
# did not execute" must not be able to borrow PASS's vocabulary.
|
|
|
|
pass() { printf 'PASS %s\n' "$*"; CHECKS=$((CHECKS + 1)); }
|
|
fail() { printf 'FAIL %s\n' "$*"; CHECKS=$((CHECKS + 1)); FAILURES=$((FAILURES + 1)); }
|
|
norun() { printf 'FAIL-NORUN %s\n' "$*"; CHECKS=$((CHECKS + 1)); FAILURES=$((FAILURES + 1)); }
|
|
info() { printf ' %s\n' "$*"; }
|
|
die() { printf 'ABORT %s\n' "$*" >&2; exit 2; }
|
|
|
|
# ── Session + API ────────────────────────────────────────────────
|
|
|
|
mint_session() {
|
|
local secret hash
|
|
secret="verify-$(openssl rand -hex 16)"
|
|
hash=$(printf '%s' "$secret" | openssl dgst -sha256 -binary \
|
|
| openssl base64 -A | tr '+/' '-_' | tr -d '=')
|
|
local rows
|
|
rows=$(ssh "$HOST" "docker exec clawmates_postgres_1 psql -U postgres -d clawmates -tAc \
|
|
\"insert into auth_sessions (user_id, token_hash, expires_at) \
|
|
select id, '$hash', now() + interval '90 minutes' \
|
|
from users where email='$OWNER' limit 1 returning 1;\"" 2>/dev/null \
|
|
| head -1 | tr -d '[:space:]')
|
|
# `head -1`: psql emits the returned row AND its `INSERT 0 1` command tag,
|
|
# and collapsing both gave `1INSERT01`, which failed this check on a mint
|
|
# that had in fact worked.
|
|
#
|
|
# No row inserted means no such user — without this check the script would
|
|
# carry on and report every API call as a delivery failure.
|
|
[ "$rows" = "1" ] || { echo "could not mint a session for $OWNER (no such user?)" >&2; return 1; }
|
|
printf '%s' "$secret"
|
|
}
|
|
|
|
api() { # api <token> <METHOD> <path> [json]
|
|
local t="$1" m="$2" p="$3" b="${4:-}"
|
|
# The JSON body travels on STDIN (`curl -d @-`), not embedded in the command.
|
|
#
|
|
# It used to be interpolated into a single-quoted `-d '...'` inside a
|
|
# double-quoted ssh command, which works only for bodies containing neither
|
|
# apostrophes nor anything else the two shells rewrite. A task description
|
|
# saying "the crate's test suite" ended the quoting and the remote shell died
|
|
# with "unexpected EOF"; two attempts at escaping it were themselves wrong,
|
|
# because the backslashes have to survive bash AND sed AND sh. Removing the
|
|
# interpolation removes the whole class.
|
|
if [ -n "$b" ]; then
|
|
printf '%s' "$b" | ssh "$HOST" "docker run --rm -i --network clawmates_core \
|
|
curlimages/curl:latest -s -X $m \
|
|
-H 'Authorization: Bearer $t' -H 'Content-Type: application/json' -d @- \
|
|
http://clawmates_server_1:8080$p"
|
|
else
|
|
ssh "$HOST" "docker run --rm --network clawmates_core curlimages/curl:latest -s -X $m \
|
|
-H 'Authorization: Bearer $t' http://clawmates_server_1:8080$p"
|
|
fi
|
|
}
|
|
|
|
|
|
# ── The uid probe ────────────────────────────────────────────────
|
|
#
|
|
# The structural claim copy mode makes is that the host checkout has exactly
|
|
# one writer. This is what tests it. It prints a sorted, comma-separated uid
|
|
# list on stdout and exits 0, or prints nothing and exits 1.
|
|
|
|
probe_uids() { # probe_uids <mission-id-or-prefix>
|
|
local mission="$1" dir uids
|
|
dir=$(ssh "$HOST" "ls -d $MISSIONS_ROOT/$mission* 2>/dev/null | head -1" | tr -d '\r')
|
|
[ -n "$dir" ] || { echo "no mission dir under $MISSIONS_ROOT for $mission" >&2; return 1; }
|
|
ssh "$HOST" "sudo test -d '$dir/repo'" 2>/dev/null \
|
|
|| { echo "$dir/repo is not a directory" >&2; return 1; }
|
|
# -printf '%U' over stat: one process for the whole tree, and it reports the
|
|
# numeric uid even when the host has no passwd entry for it.
|
|
uids=$(ssh "$HOST" "sudo find '$dir/repo' -xdev -printf '%U\n' 2>/dev/null | sort -un | paste -sd, -" | tr -d '\r')
|
|
# An empty result is not "no uids", it is a failed read. A checkout always
|
|
# contains files; if find returned nothing, find did not work.
|
|
[ -n "$uids" ] || { echo "uid read produced no output for $dir/repo" >&2; return 1; }
|
|
printf '%s' "$uids"
|
|
}
|
|
|
|
# Find a mission whose checkout still shows the pre-copy-mode split. Used as
|
|
# the probe's negative control.
|
|
discover_split_control() {
|
|
ssh "$HOST" "for d in $MISSIONS_ROOT/*/repo; do
|
|
[ -d \"\$d\" ] || continue
|
|
n=\$(sudo find \"\$d\" -xdev -printf '%U\n' 2>/dev/null | sort -un | wc -l)
|
|
if [ \"\$n\" -gt 1 ]; then basename \$(dirname \"\$d\"); break; fi
|
|
done" 2>/dev/null | tr -d '\r' | head -1
|
|
}
|
|
|
|
# ── Probe self-test ──────────────────────────────────────────────
|
|
#
|
|
# Run BEFORE trusting any uid result. A green uid report from a probe that
|
|
# cannot detect the split is not evidence of anything.
|
|
|
|
selftest_uid_probe() {
|
|
local control uids
|
|
control="${CLAWMATES_UID_CONTROL:-$(discover_split_control)}"
|
|
if [ -z "$control" ]; then
|
|
# Not a pass. Every checkout on the host is single-uid, which is the
|
|
# desired end state but leaves the probe unexercised — so say exactly
|
|
# that rather than implying the probe was validated.
|
|
info "selftest: no split-uid mission remains on $HOST to use as a control"
|
|
info "selftest: uid results below are UNVALIDATED (set CLAWMATES_UID_CONTROL)"
|
|
return 0
|
|
fi
|
|
uids=$(probe_uids "$control") || { fail "selftest: probe failed on control $control"; return 1; }
|
|
case "$uids" in
|
|
*,*) pass "selftest: probe reports the split on control $control (uids=$uids)" ;;
|
|
*) fail "selftest: control $control reports a single uid ($uids) — the probe cannot detect the split it exists to find" ;;
|
|
esac
|
|
}
|
|
|
|
check_single_uid() { # check_single_uid <mission> <label>
|
|
local mission="$1" label="$2" uids
|
|
uids=$(probe_uids "$mission") || { fail "$label: uid probe could not read the checkout"; return 1; }
|
|
case "$uids" in
|
|
*,*) fail "$label: checkout has multiple writers (uids=$uids)" ;;
|
|
*) pass "$label: checkout has exactly one writer (uid=$uids)" ;;
|
|
esac
|
|
}
|
|
|
|
# ── Mission lifecycle ────────────────────────────────────────────
|
|
|
|
create_mission() { # create_mission <token> <json> -> mission id
|
|
local out id
|
|
out=$(api "$1" POST /api/missions "$(echo "$2" | tr -d '\n')")
|
|
id=$(printf '%s' "$out" | python3 -c 'import json,sys
|
|
try: print(json.load(sys.stdin).get("id",""))
|
|
except Exception: pass' 2>/dev/null)
|
|
[ -n "$id" ] || { echo "create failed: $out" >&2; return 1; }
|
|
printf '%s' "$id"
|
|
}
|
|
|
|
await_mission() { # await_mission <token> <mission> -> final status
|
|
local t="$1" m="$2" waited=0 status
|
|
while [ "$waited" -lt "$MISSION_TIMEOUT" ]; do
|
|
status=$(api "$t" GET "/api/missions/$m" | python3 -c 'import json,sys
|
|
try: print(json.load(sys.stdin).get("status",""))
|
|
except Exception: pass' 2>/dev/null)
|
|
case "$status" in
|
|
completed|failed|cancelled) printf '%s' "$status"; return 0 ;;
|
|
esac
|
|
sleep 20
|
|
waited=$((waited + 20))
|
|
done
|
|
printf 'timeout'
|
|
}
|
|
|
|
# Emit one `order_idx status files pushed commit_error push_error` line per
|
|
# phase by joining phases to their code_diff artifacts.
|
|
phase_report() { # phase_report <token> <mission>
|
|
api "$1" GET "/api/missions/$2" | python3 -c '
|
|
import json, sys
|
|
d = json.load(sys.stdin)
|
|
art = {}
|
|
for a in d.get("artifacts") or []:
|
|
if a.get("kind") == "code_diff":
|
|
art[a.get("phase_id")] = a.get("metadata") or {}
|
|
for p in sorted(d.get("phases") or [], key=lambda p: p.get("order_idx", 0)):
|
|
m = art.get(p["id"], {})
|
|
print(p.get("order_idx"), p.get("status"),
|
|
m.get("files_changed", m.get("files", "-")),
|
|
m.get("pushed", "-"), m.get("branch", "-"),
|
|
json.dumps(m.get("commit_error")), json.dumps(m.get("push_error")))
|
|
'
|
|
}
|
|
|
|
# Read a file back from the branch the mission actually pushed, so the
|
|
# assertion is against the forge rather than against the host staging dir.
|
|
fetch_delivered() { # fetch_delivered <token> <mission> <path>
|
|
local branch repo token
|
|
branch=$(api "$1" GET "/api/missions/$2" | python3 -c '
|
|
import json, sys
|
|
d = json.load(sys.stdin)
|
|
for a in d.get("artifacts") or []:
|
|
b = (a.get("metadata") or {}).get("branch")
|
|
if b: print(b); break
|
|
' 2>/dev/null)
|
|
[ -n "$branch" ] || return 1
|
|
token=$(ssh "$HOST" 'docker exec clawmates_server_1 printenv GITEA_TOKEN' | tr -d '\r')
|
|
repo=$(ssh "$HOST" "docker exec clawmates_postgres_1 psql -U postgres -d clawmates -tAc \
|
|
\"select owner || '/' || name from repos where id='$REPO_ID';\"" \
|
|
| head -1 | tr -d '[:space:]')
|
|
[ -n "$token" ] && [ -n "$repo" ] || return 1
|
|
local enc; enc=$(printf '%s' "$branch" | sed 's|/|%2F|g')
|
|
ssh "$HOST" "curl -sf -H 'Authorization: token $token' \
|
|
'https://git.redclaw.dev/api/v1/repos/$repo/raw/$3?ref=$enc'"
|
|
}
|
|
|
|
# ── Scenario: a phase runs inside a microVM, and fans out ─────────
|
|
#
|
|
# Guards everything the microVM track proved by hand: that the agent ran in a
|
|
# GUEST kernel rather than on a host, that Claude Code could actually delegate,
|
|
# and that the work came back and landed.
|
|
#
|
|
# The kernel line is the assertion that cannot pass by accident. Every other
|
|
# check here would also pass if the phase had quietly run in a container on the
|
|
# gateway; only the kernel says WHERE it ran. It is compared against the real
|
|
# host kernels rather than pinned to a version, so upgrading `vmlinux` does not
|
|
# turn this into a false failure.
|
|
|
|
MICROVM_BODY=$(cat <<JSON
|
|
{"title":"verify: a phase runs inside a microVM",
|
|
"template_kind":"research_and_code",
|
|
"repo_id":"$REPO_ID",
|
|
"runtime_kind":"microvm",
|
|
"backend":"claude",
|
|
"description":"Prove a coding phase executes in a Firecracker microVM and can delegate.",
|
|
"phases":[
|
|
{"kind":"coding","order_idx":0,"config":{"commit_policy":"always","max_iterations":1,
|
|
"done_when":"A file named MICROVM.md exists at the repository root and contains at least two lines, the second of which is a Linux kernel release string.",
|
|
"task":"1. Use the verifier subagent to run the COMPLETE test suite of this crate and report what it found. Do not run it yourself and report that instead — the point is an independent check.\n2. Write MICROVM.md at the repository root with exactly two lines: the first is the test-result summary line the verifier reported, the second is the kernel release from running uname -r.\nCreate no other files."}}
|
|
]}
|
|
JSON
|
|
)
|
|
|
|
# How many subagents the phase actually spawned, per the server's own count of
|
|
# Claude Code's per-subagent transcripts. Read from the log because that is
|
|
# where `phase_runner` reports it; "?" means the probe could not run, which is
|
|
# not the same as zero.
|
|
subagent_count() { # subagent_count <mission>
|
|
ssh "$HOST" "docker logs --since 60m clawmates_server_1 2>&1 \
|
|
| grep -F 'microvm phase' | grep -F '$1' | tail -1" \
|
|
| sed -n 's/.*subagents: \([0-9?]*\).*/\1/p'
|
|
}
|
|
|
|
assert_microvm() { # <token> <mission> <report>
|
|
local token="$1" mission="$2" report="$3" delivered kernel subs indep
|
|
while read -r idx status files pushed _branch cerr perr; do
|
|
[ "$status" = "completed" ] || fail "microvm: phase $idx status=$status (commit_error=$cerr push_error=$perr)"
|
|
[ "$pushed" = "True" ] || fail "microvm: phase $idx not pushed (commit_error=$cerr push_error=$perr)"
|
|
case "$files" in 0|-) fail "microvm: phase $idx delivered no files" ;; esac
|
|
done <<<"$report"
|
|
|
|
delivered=$(fetch_delivered "$token" "$mission" MICROVM.md) \
|
|
|| { fail "microvm: could not read MICROVM.md from the pushed branch"; return 1; }
|
|
|
|
# The decisive check: the second line is the kernel the agent ran on.
|
|
kernel=$(printf '%s\n' "$delivered" | sed -n '2p' | tr -d '[:space:]')
|
|
if [ -z "$kernel" ]; then
|
|
fail "microvm: MICROVM.md has no kernel line: $(printf '%s' "$delivered" | tr '\n' '|')"
|
|
elif [ "$kernel" = "$GW_KERNEL" ]; then
|
|
fail "microvm: the agent ran on the GATEWAY kernel ($kernel) — not in a VM at all"
|
|
elif [ "$kernel" = "$NODE_KERNEL" ]; then
|
|
fail "microvm: the agent ran on the fleet NODE's kernel ($kernel) — not in a VM at all"
|
|
else
|
|
pass "microvm: the agent ran under a guest kernel ($kernel), not the gateway's ($GW_KERNEL) or the node's ($NODE_KERNEL)"
|
|
fi
|
|
|
|
# Fan-out. Before the `Agent` tool was added to the allowlist this was
|
|
# structurally impossible, and nothing said so.
|
|
subs=$(subagent_count "$mission")
|
|
case "$subs" in
|
|
''|'?') fail "microvm: could not count subagents (probe did not run) — delegation UNPROVEN" ;;
|
|
0) fail "microvm: the phase spawned no subagents, so the verifier never ran" ;;
|
|
*) pass "microvm: the lead delegated to $subs subagent(s)" ;;
|
|
esac
|
|
|
|
# And the verdict: judged, and by whom. `independent` is only true when the
|
|
# judge came from a different provider family than the agent.
|
|
indep=$(ssh "$HOST" "docker exec clawmates_postgres_1 psql -U postgres -d clawmates -tAc \
|
|
\"select met::text || ' ' || independent::text || ' ' || model from mission_phase_evaluations \
|
|
where phase_id in (select id from mission_phases where mission_id='$mission') \
|
|
order by created_at desc limit 1;\"" | head -1 | tr -d '\r')
|
|
# `true`/`false`, not psql's display `t`/`f`: concatenating a boolean with text
|
|
# casts it to its full spelling. The first version matched "t t " and therefore
|
|
# never matched a real verdict, reporting "no verdict recorded" while the row was
|
|
# sitting in the table — a check that failed for a reason that had nothing to do
|
|
# with what it was checking.
|
|
case "$indep" in
|
|
"true true "*) pass "microvm: condition met, judged independently (${indep##* })" ;;
|
|
"true false "*) info "microvm: condition met but judged by the agent's own provider family (${indep##* }) — set CLAWMATES_VALIDATOR_MODEL for an independent check" ;;
|
|
"false "*) fail "microvm: the judge says the condition was NOT met: $indep" ;;
|
|
*) fail "microvm: no verdict recorded for the phase (done_when was set), got: ${indep:-<empty>}" ;;
|
|
esac
|
|
}
|
|
|
|
# ── Negative control: a backend no node can run ──────────────────
|
|
#
|
|
# Placement requires the mission's backend image to exist ON a node, not merely
|
|
# that the node has KVM. Without this check the positive scenario above would
|
|
# pass just as well against a scheduler that ignored `backend` entirely — which
|
|
# is what it did until the first real microvm mission landed on a node that had
|
|
# no such rootfs.
|
|
scenario_microvm_unavailable_backend() {
|
|
local token mission body status
|
|
token=$(mint_session) || { norun "microvm-negctl: could not mint a session"; return 1; }
|
|
body=$(printf '%s' "$MICROVM_BODY" | sed 's/"backend":"claude"/"backend":"definitely-not-built"/')
|
|
mission=$(create_mission "$token" "$(echo "$body" | tr -d '\n')") \
|
|
|| { norun "microvm-negctl: mission create failed"; return 1; }
|
|
info "microvm-negctl: mission=$mission"
|
|
api "$token" PATCH "/api/missions/$mission/status" '{"status":"running"}' >/dev/null 2>&1
|
|
sleep 5
|
|
status=$(ssh "$HOST" "docker exec clawmates_postgres_1 psql -U postgres -d clawmates -tAc \
|
|
\"select status from missions where id='$mission';\"" | head -1 | tr -d '[:space:]')
|
|
if [ "$status" = "draft" ]; then
|
|
pass "microvm-negctl: a backend no node can run is refused at launch (mission stayed draft)"
|
|
else
|
|
fail "microvm-negctl: a mission asking for an unbuilt image reached status=$status"
|
|
fi
|
|
}
|
|
|
|
run_scenario() { # run_scenario <label> <json> <assert-fn>
|
|
local label="$1" body="$2" assert_fn="$3" token mission status
|
|
# Every one of these MUST go through fail()/norun(). The first version of
|
|
# this function called a `die` that lived inside `$(...)` — which exits the
|
|
# command substitution's subshell, not the script — so a run where the
|
|
# session could not be minted printed two ABORT lines, incremented nothing,
|
|
# and ended with "all checks passed" and exit 0. The harness written to
|
|
# catch silent success produced silent success on its first real run.
|
|
token=$(mint_session) || { norun "$label: could not mint a session"; return 1; }
|
|
mission=$(create_mission "$token" "$body") || { norun "$label: mission create failed"; return 1; }
|
|
info "$label: mission=$mission"
|
|
api "$token" PATCH "/api/missions/$mission/status" '{"status":"running"}' >/dev/null
|
|
status=$(await_mission "$token" "$mission")
|
|
if [ "$status" = "timeout" ]; then
|
|
norun "$label: mission did not reach a terminal status in ${MISSION_TIMEOUT}s"
|
|
return 1
|
|
fi
|
|
local report; report=$(phase_report "$token" "$mission")
|
|
if [ -z "$report" ]; then
|
|
norun "$label: no phases reported — nothing executed"
|
|
return 1
|
|
fi
|
|
printf '%s\n' "$report" | sed 's/^/ phase /'
|
|
"$assert_fn" "$token" "$mission" "$report"
|
|
check_single_uid "$mission" "$label"
|
|
}
|
|
|
|
# ── Scenario: phase continuity ───────────────────────────────────
|
|
#
|
|
# Phase 1 must READ what phase 0 wrote. Independent phases cannot tell a
|
|
# preserved checkout from a wiped one — which is exactly how a `reset --hard`
|
|
# survived three green-looking runs.
|
|
|
|
CHAIN_BODY=$(cat <<JSON
|
|
{"title":"verify: phase 2 builds on phase 1",
|
|
"template_kind":"research_and_code",
|
|
"team_template_id":"$TEAM_TEMPLATE",
|
|
"repo_id":"$REPO_ID",
|
|
"description":"Build up a file CHAIN.md across two phases.",
|
|
"phases":[
|
|
{"kind":"coding","order_idx":0,"config":{"commit_policy":"always","max_iterations":1,
|
|
"task":"Create a file named CHAIN.md at the repository root containing exactly one line: STEP-ONE-OK. Do not create or modify any other file."}},
|
|
{"kind":"coding","order_idx":1,"config":{"commit_policy":"always","max_iterations":1,
|
|
"task":"Read the existing file CHAIN.md at the repository root. It was written by the previous phase and must already contain the line STEP-ONE-OK. Append a second line reading STEP-TWO-SAW-STEP-ONE, keeping the first line intact. If CHAIN.md does not exist, instead create a file named CHAIN_MISSING.md containing the single line PRIOR-PHASE-WORK-WAS-LOST, and do not create CHAIN.md."}}
|
|
]}
|
|
JSON
|
|
)
|
|
|
|
assert_chain() { # <token> <mission> <report>
|
|
local token="$1" mission="$2" report="$3" delivered
|
|
local phases; phases=$(printf '%s\n' "$report" | wc -l | tr -d ' ')
|
|
[ "$phases" = "2" ] || fail "chain: expected 2 phases, got $phases"
|
|
# Here-string, not a pipe: a `while` on the right of a pipe runs in a
|
|
# subshell, so every fail() inside it would increment a FAILURES that dies
|
|
# with the subshell and the script would exit 0 having reported failures.
|
|
while read -r idx status files pushed _branch cerr perr; do
|
|
[ "$status" = "completed" ] || fail "chain: phase $idx status=$status"
|
|
[ "$pushed" = "True" ] || fail "chain: phase $idx not pushed (commit_error=$cerr push_error=$perr)"
|
|
case "$files" in 0|-) fail "chain: phase $idx delivered no files" ;; esac
|
|
done <<<"$report"
|
|
delivered=$(fetch_delivered "$token" "$mission" CHAIN.md) \
|
|
|| { fail "chain: could not read CHAIN.md from the pushed branch"; return 1; }
|
|
case "$delivered" in
|
|
*STEP-ONE-OK*STEP-TWO-SAW-STEP-ONE*)
|
|
pass "chain: phase 1 read phase 0's work and appended to it" ;;
|
|
*PRIOR-PHASE-WORK-WAS-LOST*)
|
|
fail "chain: phase 1 reported the prior phase's work was lost" ;;
|
|
*)
|
|
fail "chain: CHAIN.md does not show both lines: $(printf '%s' "$delivered" | tr '\n' '|')" ;;
|
|
esac
|
|
}
|
|
|
|
# ── Scenario: multi-role with a real test suite ──────────────────
|
|
#
|
|
# The workload that failed with `COMMIT_EDITMSG: Permission denied` under the
|
|
# bind mount. REVIEW.md must carry cargo's own summary line, so the reviewer
|
|
# had to run the suite rather than assert that it passed.
|
|
|
|
MULTIROLE_BODY=$(cat <<'JSON'
|
|
{"title":"verify: implement, test, review",
|
|
"template_kind":"research_and_code",
|
|
"team_template_id":"__TEAM__",
|
|
"repo_id":"__REPO__",
|
|
"description":"Extend the scratch Rust crate with a reviewed, tested function.",
|
|
"phases":[
|
|
{"kind":"coding","order_idx":0,"config":{"commit_policy":"always","max_iterations":1,
|
|
"task":"This task needs THREE distinct roles. Use a separate agent for each role; do not do all three yourself.\n1. IMPLEMENTER: in src/lib.rs add `pub fn divide(a: i64, b: i64) -> Option<i64>` returning None when b == 0, else Some(a / b).\n2. TESTER: add unit tests in the existing tests module named `divide_works` and `divide_by_zero_is_none`, covering both branches. Run `cargo test` and make it pass.\n3. REVIEWER: read the final src/lib.rs and write REVIEW.md at the repository root containing exactly three lines: `ROLES: 3`, `TESTS: <the test result summary line from cargo test>`, and `VERDICT: <one sentence>`.\nAll three files (src/lib.rs, REVIEW.md) must be left in the working tree."}}
|
|
]}
|
|
JSON
|
|
)
|
|
|
|
assert_multirole() { # <token> <mission> <report>
|
|
local token="$1" mission="$2" report="$3" review lib
|
|
while read -r idx status files pushed _branch cerr perr; do
|
|
[ "$status" = "completed" ] || fail "multirole: phase $idx status=$status"
|
|
[ "$pushed" = "True" ] || fail "multirole: phase $idx not pushed (commit_error=$cerr push_error=$perr)"
|
|
case "$files" in 0|-) fail "multirole: phase $idx delivered no files" ;; esac
|
|
done <<<"$report"
|
|
lib=$(fetch_delivered "$token" "$mission" src/lib.rs) \
|
|
|| { fail "multirole: could not read src/lib.rs from the pushed branch"; return 1; }
|
|
case "$lib" in
|
|
*"fn divide"*) pass "multirole: divide() was delivered" ;;
|
|
*) fail "multirole: src/lib.rs has no divide()" ;;
|
|
esac
|
|
case "$lib" in
|
|
*divide_by_zero_is_none*) pass "multirole: the zero-divisor test was delivered" ;;
|
|
*) fail "multirole: no divide_by_zero_is_none test" ;;
|
|
esac
|
|
review=$(fetch_delivered "$token" "$mission" REVIEW.md) \
|
|
|| { fail "multirole: no REVIEW.md on the pushed branch"; return 1; }
|
|
# cargo's own words. A reviewer who merely claimed the tests passed cannot
|
|
# produce this line, which is the point of asserting on it.
|
|
case "$review" in
|
|
*"test result: ok."*) pass "multirole: REVIEW.md carries cargo's own test summary" ;;
|
|
*) fail "multirole: REVIEW.md has no 'test result: ok.' line" ;;
|
|
esac
|
|
}
|
|
|
|
# ── Scenario: a phase that delivers nothing must FAIL ────────────
|
|
#
|
|
# The negative control for the delivery guard, and the same discipline as the
|
|
# uid self-test: a check that has never been seen to fire has not been shown to
|
|
# work. This phase is told to change nothing, so `empty_delivery_is_a_failure`
|
|
# must catch it — and the scenario PASSES when the phase comes back `failed`.
|
|
|
|
NOOP_BODY=$(cat <<JSON
|
|
{"title":"verify: a phase that delivers nothing must fail",
|
|
"template_kind":"research_and_code",
|
|
"team_template_id":"$TEAM_TEMPLATE",
|
|
"repo_id":"$REPO_ID",
|
|
"description":"Negative control for the empty-delivery guard.",
|
|
"phases":[
|
|
{"kind":"coding","order_idx":0,"config":{"commit_policy":"always","max_iterations":1,
|
|
"task":"Do NOT create, modify or delete any file. Read README.md and reply with a one-sentence summary of it as your final answer. Leave the working tree exactly as you found it."}}
|
|
]}
|
|
JSON
|
|
)
|
|
|
|
assert_noop() { # <token> <mission> <report>
|
|
local report="$3" saw_failed=0
|
|
while read -r idx status files _pushed _branch _cerr _perr; do
|
|
case "$files" in
|
|
0|-) ;;
|
|
*) fail "noop: phase $idx changed $files file(s) — the scenario did not \
|
|
exercise the guard (agent ignored the instruction; re-run)"; continue ;;
|
|
esac
|
|
if [ "$status" = "failed" ]; then
|
|
saw_failed=1
|
|
else
|
|
fail "noop: phase $idx delivered nothing but reports status=$status"
|
|
fi
|
|
done <<<"$report"
|
|
[ "$saw_failed" = "1" ] && pass "noop: an empty coding phase was failed, not completed"
|
|
return 0
|
|
}
|
|
|
|
# ── Entry point ──────────────────────────────────────────────────
|
|
|
|
ssh -o BatchMode=yes -o ConnectTimeout=10 "$HOST" true 2>/dev/null \
|
|
|| die "cannot ssh to $HOST"
|
|
|
|
case "${1:-all}" in
|
|
selftest)
|
|
selftest_uid_probe
|
|
;;
|
|
uids)
|
|
[ $# -ge 2 ] || die "usage: $0 uids <mission-id>"
|
|
selftest_uid_probe
|
|
check_single_uid "$2" "uids($2)"
|
|
;;
|
|
chain)
|
|
selftest_uid_probe
|
|
run_scenario chain "$CHAIN_BODY" assert_chain
|
|
;;
|
|
multirole)
|
|
selftest_uid_probe
|
|
body=${MULTIROLE_BODY//__TEAM__/$TEAM_TEMPLATE}
|
|
run_scenario multirole "${body//__REPO__/$REPO_ID}" assert_multirole
|
|
;;
|
|
noop)
|
|
run_scenario noop "$NOOP_BODY" assert_noop
|
|
;;
|
|
microvm)
|
|
run_scenario microvm "$(echo "$MICROVM_BODY" | tr -d '\n')" assert_microvm
|
|
scenario_microvm_unavailable_backend
|
|
;;
|
|
all)
|
|
selftest_uid_probe
|
|
run_scenario chain "$CHAIN_BODY" assert_chain
|
|
body=${MULTIROLE_BODY//__TEAM__/$TEAM_TEMPLATE}
|
|
run_scenario multirole "${body//__REPO__/$REPO_ID}" assert_multirole
|
|
run_scenario noop "$NOOP_BODY" assert_noop
|
|
run_scenario microvm "$(echo "$MICROVM_BODY" | tr -d '\n')" assert_microvm
|
|
scenario_microvm_unavailable_backend
|
|
;;
|
|
*)
|
|
die "unknown scenario: $1 (selftest|uids|chain|multirole|noop|microvm|all)"
|
|
;;
|
|
esac
|
|
|
|
if [ "$FAILURES" -gt 0 ]; then
|
|
printf '\n%d of %d check(s) failed\n' "$FAILURES" "$CHECKS"
|
|
exit 1
|
|
fi
|
|
# Zero checks is not success. A run that asserted nothing must not be able to
|
|
# print the same closing line as a run that asserted everything.
|
|
if [ "$CHECKS" -eq 0 ]; then
|
|
printf '\nno checks ran — nothing was verified\n'
|
|
exit 1
|
|
fi
|
|
printf '\nall %d check(s) passed\n' "$CHECKS"
|