test(harness): a gate that gives up, proven against a real VM

The unit tests prove the plumbing GIVEN `released_at_cap: Some(true)`. They
cannot prove the guest writes the marker, that the probe reads it back across
the vsock, or that the phase lands `failed` for the right reason — and every
one of those is where this class of bug has actually lived.

The check is `exit 1`: impossible by construction, so the run exercises the
release path rather than hoping to catch it.

`blocks` reaching the cap is deliberately NOT the assertion. A healthy agent
blocked three times and succeeding on the fourth reports the same 3. The phase
STATUS is the assertion; the block count and the failure reason are corroborating
checks, so a phase that failed for some unrelated reason cannot pass this.

Measured on gw-04 against b36ae00, all 4 checks green:
  phase 0 failed
  the gate spent all 3 blocks before giving up
  the failure names the cap release as the reason
Before b36ae00 that same mission completed green.
This commit is contained in:
Omar Sobh
2026-08-07 09:58:41 -07:00
parent b36ae00ea5
commit bcf4866abc
+65 -1
View File
@@ -366,6 +366,66 @@ scenario_microvm_unavailable_backend() {
fi fi
} }
# ── Negative control: a gate that gives up must fail the phase ───
#
# `done_when_check` is run in exactly ONE place — the Stop hook inside the
# guest. The hook is capped so a stuck agent cannot wedge the turn, and at that
# cap it lets the agent stop. Until the `capped` marker existed, that release
# was invisible: the process exits 0 and the work IS collected, so both signals
# the run status was decided from said "fine" and the phase completed GREEN with
# its check still failing.
#
# The check here is `exit 1`, which no agent can satisfy. That is the point: it
# guarantees the cap is reached, so this exercises the release path itself
# rather than hoping to catch it. `blocks` reaching the cap is NOT the
# assertion — a healthy agent blocked three times and succeeding on the fourth
# reports the same number. The phase STATUS is the assertion.
GATECAP_BODY=$(cat <<JSON
{"title":"verify: a gate that gives up fails the phase",
"template_kind":"research_and_code",
"repo_id":"$REPO_ID",
"runtime_kind":"microvm",
"backend":"claude",
"description":"Negative control for the stop gate's cap release.",
"phases":[
{"kind":"coding","order_idx":0,"config":{"commit_policy":"always","max_iterations":1,
"done_when_check":"exit 1",
"task":"Create a file NOTES.md at the repository root containing the single word: hello. Create no other files."}}
]}
JSON
)
assert_gate_cap() { # <token> <mission> <report>
local mission="$2" report="$3" blocks note
while read -r idx status _files _pushed _branch _cerr _perr; do
if [ "$status" = "completed" ]; then
fail "gatecap: phase $idx COMPLETED with a check that cannot pass — the cap release is invisible again"
else
pass "gatecap: phase $idx did not complete (status=$status)"
fi
done <<<"$report"
# The gate must actually have run out of blocks; a phase that failed for some
# OTHER reason would satisfy the status check above while proving nothing.
blocks=$(ssh "$HOST" "docker logs --since 60m clawmates_server_1 2>&1 \
| grep -F 'microvm phase' | grep -F '$mission' | tail -1" \
| sed -n 's/.*stop-gate blocks: \([0-9-]*\).*/\1/p')
case "$blocks" in
3) pass "gatecap: the gate spent all 3 blocks before giving up" ;;
'') fail "gatecap: no stop-gate field in the log — was a gate installed at all?" ;;
*) fail "gatecap: expected 3 blocks before the cap, got $blocks" ;;
esac
note=$(ssh "$HOST" "docker logs --since 60m clawmates_server_1 2>&1 \
| grep -F 'microvm phase' | grep -F '$mission' | tail -1")
case "$note" in
*"completion gate released"*)
pass "gatecap: the failure names the cap release as the reason" ;;
*)
fail "gatecap: the phase failed but not for the gate's reason: $(printf '%s' "$note" | tail -c 200)" ;;
esac
}
# ── Scenario: the two engines composed ─────────────────────────── # ── Scenario: the two engines composed ───────────────────────────
# #
# A `team_engine=composed` mission is a durable ZeroClaw graph whose every node # A `team_engine=composed` mission is a durable ZeroClaw graph whose every node
@@ -741,6 +801,9 @@ case "${1:-all}" in
run_scenario microvm "$(echo "$MICROVM_BODY" | tr -d '\n')" assert_microvm run_scenario microvm "$(echo "$MICROVM_BODY" | tr -d '\n')" assert_microvm
scenario_microvm_unavailable_backend scenario_microvm_unavailable_backend
;; ;;
gatecap)
run_scenario gatecap "$(echo "$GATECAP_BODY" | tr -d '\n')" assert_gate_cap
;;
composed) composed)
run_scenario composed "$(echo "$COMPOSED_BODY" | tr -d '\n')" assert_composed run_scenario composed "$(echo "$COMPOSED_BODY" | tr -d '\n')" assert_composed
;; ;;
@@ -755,11 +818,12 @@ case "${1:-all}" in
run_scenario noop "$NOOP_BODY" assert_noop run_scenario noop "$NOOP_BODY" assert_noop
run_scenario microvm "$(echo "$MICROVM_BODY" | tr -d '\n')" assert_microvm run_scenario microvm "$(echo "$MICROVM_BODY" | tr -d '\n')" assert_microvm
scenario_microvm_unavailable_backend scenario_microvm_unavailable_backend
run_scenario gatecap "$(echo "$GATECAP_BODY" | tr -d '\n')" assert_gate_cap
run_scenario composed "$(echo "$COMPOSED_BODY" | tr -d '\n')" assert_composed run_scenario composed "$(echo "$COMPOSED_BODY" | tr -d '\n')" assert_composed
scenario_roster scenario_roster
;; ;;
*) *)
die "unknown scenario: $1 (selftest|uids|chain|multirole|noop|microvm|composed|roster|all)" die "unknown scenario: $1 (selftest|uids|chain|multirole|noop|microvm|gatecap|composed|roster|all)"
;; ;;
esac esac