test(harness): a composed scenario, and the stop gate asserted in a real VM
`verify-mission-delivery.sh composed` runs a `team_engine=composed` mission and checks the one property that cannot be checked any other way: a VM is inject → run → collect → destroy, so unless the tree is carried node to node, node 2 boots from the original checkout, sees nothing of node 1's work, and still reports success. The task makes each node append ONE line to STAGES.md, so the delivered file IS the evidence — a run that lost the handoff delivers one line, and no amount of agent confidence can fabricate the missing ones. It also asserts the run's tier is `microvm_graph`. A composed mission that quietly fell back to the solo path would deliver a one-line file and look exactly like a graph that ran one node. `assert_stop_gate` reads the count `phase_runner` reports and distinguishes three outcomes that matter: a number (installed, fired that often), `0` (installed, never needed), and `-` (could NOT be installed — usually a CLI in the image with no `--settings`). Wired into the microvm scenario rather than its own, because it applies to every coding phase on that path. Both ran against the deployed stack: composed — 4/4. STAGES.md carried 5 stage lines through 5 separate VMs (planner → coder → tester → reviewer → committer), each stamped with the guest kernel 6.1.128 rather than the gateway's 6.8.0 or the node's 7.0.0. The run checkpointed 5 steps on the worker, and `updated_at` stayed ~2s old mid-turn, which is the keepalive doing its job — without it `requeue_stale` flips a live run at 180 seconds. microvm — 6/6, including the gate installed in a real VM (`blocks: 0`), one subagent, the GLM judge, and the unavailable-backend negative control. Co-Authored-By: Claude Opus 5 <[email protected]>
This commit is contained in:
co-authored by
Claude Opus 5
parent
6991e21f94
commit
abb97e6f03
@@ -292,6 +292,12 @@ assert_microvm() { # <token> <mission> <report>
|
|||||||
case "$files" in 0|-) fail "microvm: phase $idx delivered no files" ;; esac
|
case "$files" in 0|-) fail "microvm: phase $idx delivered no files" ;; esac
|
||||||
done <<<"$report"
|
done <<<"$report"
|
||||||
|
|
||||||
|
# The completion gate, in a real VM. Checked here rather than in its own
|
||||||
|
# scenario because it applies to every coding phase on this path — a gate that
|
||||||
|
# stopped being installed would otherwise cost nothing visible until the next
|
||||||
|
# phase that needed it.
|
||||||
|
assert_stop_gate "$mission" microvm
|
||||||
|
|
||||||
delivered=$(fetch_delivered "$token" "$mission" MICROVM.md) \
|
delivered=$(fetch_delivered "$token" "$mission" MICROVM.md) \
|
||||||
|| { fail "microvm: could not read MICROVM.md from the pushed branch"; return 1; }
|
|| { fail "microvm: could not read MICROVM.md from the pushed branch"; return 1; }
|
||||||
|
|
||||||
@@ -360,6 +366,108 @@ scenario_microvm_unavailable_backend() {
|
|||||||
fi
|
fi
|
||||||
}
|
}
|
||||||
|
|
||||||
|
# ── Scenario: the two engines composed ───────────────────────────
|
||||||
|
#
|
||||||
|
# A `team_engine=composed` mission is a durable ZeroClaw graph whose every node
|
||||||
|
# is a whole Claude-Code-in-a-microVM session. The property that cannot be
|
||||||
|
# checked any other way is the FILE HANDOFF: a VM is inject → run → collect →
|
||||||
|
# destroy, so unless the tree is carried node to node, node 2 boots from the
|
||||||
|
# original checkout, sees none of node 1's work, and still reports success.
|
||||||
|
#
|
||||||
|
# The task is written so the delivered file IS the evidence. Each node appends
|
||||||
|
# one line; a run that lost the handoff delivers a file with one line, and no
|
||||||
|
# amount of agent confidence can fake the missing ones.
|
||||||
|
|
||||||
|
COMPOSED_BODY=$(cat <<JSON
|
||||||
|
{"title":"verify: the two engines composed",
|
||||||
|
"template_kind":"research_and_code",
|
||||||
|
"team_template_id":"$TEAM_TEMPLATE",
|
||||||
|
"repo_id":"$REPO_ID",
|
||||||
|
"runtime_kind":"microvm",
|
||||||
|
"backend":"claude",
|
||||||
|
"team_engine":"composed",
|
||||||
|
"description":"Prove a ZeroClaw graph of microVM nodes carries file work between its nodes.",
|
||||||
|
"phases":[
|
||||||
|
{"kind":"coding","order_idx":0,"config":{"commit_policy":"always","max_iterations":1,
|
||||||
|
"done_when":"STAGES.md exists at the repository root and its last line is a Linux kernel release string.",
|
||||||
|
"task":"Append EXACTLY ONE line to STAGES.md at the repository root, creating the file if it does not exist. The line is: your stage name, a space, and the output of \`uname -r\`.\n\nDo NOT rewrite, reorder or remove any line already in the file — earlier stages wrote those, and they are the record of this run. If the file already has lines, yours goes after them. Change no other file."}}
|
||||||
|
]}
|
||||||
|
JSON
|
||||||
|
)
|
||||||
|
|
||||||
|
# The graph nodes that actually ran, from the composed executor's own log line.
|
||||||
|
composed_steps() { # composed_steps <run-id-or-mission>
|
||||||
|
ssh "$HOST" "docker logs --since 90m clawmates_server_1 2>&1 \
|
||||||
|
| grep -F 'microvm_turn_executor:' | grep -c 'ok — subagents'" | tr -d '[:space:]'
|
||||||
|
}
|
||||||
|
|
||||||
|
assert_composed() { # <token> <mission> <report>
|
||||||
|
local token="$1" mission="$2" report="$3" delivered lines tier steps
|
||||||
|
while read -r idx status files pushed _branch cerr perr; do
|
||||||
|
[ "$status" = "completed" ] || fail "composed: phase $idx status=$status (commit_error=$cerr push_error=$perr)"
|
||||||
|
[ "$pushed" = "True" ] || fail "composed: phase $idx not pushed (push_error=$perr)"
|
||||||
|
case "$files" in 0|-) fail "composed: phase $idx delivered no files" ;; esac
|
||||||
|
done <<<"$report"
|
||||||
|
|
||||||
|
# The run must have gone through the WORKER, on the composed tier. A composed
|
||||||
|
# mission that quietly fell back to the solo path would deliver a one-line
|
||||||
|
# file and look like a graph that ran one node.
|
||||||
|
tier=$(ssh "$HOST" "docker exec clawmates_postgres_1 psql -U postgres -d clawmates -tAc \
|
||||||
|
\"select tier from topology_runs where mission_id='$mission' order by created_at desc limit 1;\"" \
|
||||||
|
| head -1 | tr -d '[:space:]')
|
||||||
|
if [ "$tier" = "microvm_graph" ]; then
|
||||||
|
pass "composed: the run was claimed by the worker on the composed tier"
|
||||||
|
else
|
||||||
|
fail "composed: run tier is '$tier', not microvm_graph — it did not compose"
|
||||||
|
fi
|
||||||
|
|
||||||
|
delivered=$(fetch_delivered "$token" "$mission" STAGES.md) \
|
||||||
|
|| { fail "composed: could not read STAGES.md from the pushed branch"; return 1; }
|
||||||
|
lines=$(printf '%s\n' "$delivered" | grep -c '[^[:space:]]')
|
||||||
|
steps=$(composed_steps "$mission")
|
||||||
|
|
||||||
|
# THE assertion. One line per node means every node saw what the last one
|
||||||
|
# left; fewer means the tree did not survive a node boundary, which is the
|
||||||
|
# silent-success shape this whole slice exists to prevent.
|
||||||
|
if [ "${lines:-0}" -ge 2 ]; then
|
||||||
|
pass "composed: STAGES.md carries $lines stage lines — the tree survived the node boundary"
|
||||||
|
else
|
||||||
|
fail "composed: STAGES.md has $lines line(s), so a later node did not see the earlier node's file: $(printf '%s' "$delivered" | tr '\n' '|')"
|
||||||
|
fi
|
||||||
|
info "composed: executor logged $steps node turn(s); delivered:$(printf '%s' "$delivered" | tr '\n' '|')"
|
||||||
|
|
||||||
|
# The kernel line again — a composed node must still be a VM, not a container
|
||||||
|
# on the gateway.
|
||||||
|
local kernel
|
||||||
|
kernel=$(printf '%s\n' "$delivered" | tail -1 | awk '{print $NF}')
|
||||||
|
case "$kernel" in
|
||||||
|
"$GW_KERNEL") fail "composed: the last node ran on the GATEWAY kernel ($kernel)" ;;
|
||||||
|
"$NODE_KERNEL") fail "composed: the last node ran on the fleet NODE's kernel ($kernel)" ;;
|
||||||
|
"") fail "composed: no kernel on the last line" ;;
|
||||||
|
*) pass "composed: the last node ran under a guest kernel ($kernel)" ;;
|
||||||
|
esac
|
||||||
|
}
|
||||||
|
|
||||||
|
# The stop gate, inside a real VM.
|
||||||
|
#
|
||||||
|
# Reported by `phase_runner` for every microVM phase. Three distinguishable
|
||||||
|
# outcomes, and the difference matters: a number means the gate was installed
|
||||||
|
# and this is how often it sent the agent back; `-` means it could not be
|
||||||
|
# installed at all (usually: the CLI in the image has no `--settings`), which is
|
||||||
|
# a degradation the log states rather than a silent absence.
|
||||||
|
assert_stop_gate() { # <mission> <label>
|
||||||
|
local blocks
|
||||||
|
blocks=$(ssh "$HOST" "docker logs --since 90m clawmates_server_1 2>&1 \
|
||||||
|
| grep -F 'microvm phase' | grep -F '$1' | tail -1" \
|
||||||
|
| sed -n 's/.*stop-gate blocks: \([0-9-]*\).*/\1/p')
|
||||||
|
case "$blocks" in
|
||||||
|
'') fail "$2-gate: the phase reported no stop-gate field at all — old server image?" ;;
|
||||||
|
'-') fail "$2-gate: the gate was NOT installed in the VM (check the server log for the --settings probe)" ;;
|
||||||
|
0) pass "$2-gate: the gate was installed and never fired — the agent finished the work first time" ;;
|
||||||
|
*) pass "$2-gate: the gate sent the agent back $blocks time(s) inside its own turn" ;;
|
||||||
|
esac
|
||||||
|
}
|
||||||
|
|
||||||
run_scenario() { # run_scenario <label> <json> <assert-fn>
|
run_scenario() { # run_scenario <label> <json> <assert-fn>
|
||||||
local label="$1" body="$2" assert_fn="$3" token mission status
|
local label="$1" body="$2" assert_fn="$3" token mission status
|
||||||
# Every one of these MUST go through fail()/norun(). The first version of
|
# Every one of these MUST go through fail()/norun(). The first version of
|
||||||
@@ -546,6 +654,9 @@ case "${1:-all}" in
|
|||||||
run_scenario microvm "$(echo "$MICROVM_BODY" | tr -d '\n')" assert_microvm
|
run_scenario microvm "$(echo "$MICROVM_BODY" | tr -d '\n')" assert_microvm
|
||||||
scenario_microvm_unavailable_backend
|
scenario_microvm_unavailable_backend
|
||||||
;;
|
;;
|
||||||
|
composed)
|
||||||
|
run_scenario composed "$(echo "$COMPOSED_BODY" | tr -d '\n')" assert_composed
|
||||||
|
;;
|
||||||
all)
|
all)
|
||||||
selftest_uid_probe
|
selftest_uid_probe
|
||||||
run_scenario chain "$CHAIN_BODY" assert_chain
|
run_scenario chain "$CHAIN_BODY" assert_chain
|
||||||
@@ -554,9 +665,10 @@ case "${1:-all}" in
|
|||||||
run_scenario noop "$NOOP_BODY" assert_noop
|
run_scenario noop "$NOOP_BODY" assert_noop
|
||||||
run_scenario microvm "$(echo "$MICROVM_BODY" | tr -d '\n')" assert_microvm
|
run_scenario microvm "$(echo "$MICROVM_BODY" | tr -d '\n')" assert_microvm
|
||||||
scenario_microvm_unavailable_backend
|
scenario_microvm_unavailable_backend
|
||||||
|
run_scenario composed "$(echo "$COMPOSED_BODY" | tr -d '\n')" assert_composed
|
||||||
;;
|
;;
|
||||||
*)
|
*)
|
||||||
die "unknown scenario: $1 (selftest|uids|chain|multirole|noop|microvm|all)"
|
die "unknown scenario: $1 (selftest|uids|chain|multirole|noop|microvm|composed|all)"
|
||||||
;;
|
;;
|
||||||
esac
|
esac
|
||||||
|
|
||||||
|
|||||||
Reference in New Issue
Block a user