test(harness): a gate that gives up, proven against a real VM
The unit tests prove the plumbing GIVEN `released_at_cap: Some(true)`. They cannot prove the guest writes the marker, that the probe reads it back across the vsock, or that the phase lands `failed` for the right reason — and every one of those is where this class of bug has actually lived. The check is `exit 1`: impossible by construction, so the run exercises the release path rather than hoping to catch it. `blocks` reaching the cap is deliberately NOT the assertion. A healthy agent blocked three times and succeeding on the fourth reports the same 3. The phase STATUS is the assertion; the block count and the failure reason are corroborating checks, so a phase that failed for some unrelated reason cannot pass this. Measured on gw-04 againstb36ae00, all 4 checks green: phase 0 failed the gate spent all 3 blocks before giving up the failure names the cap release as the reason Beforeb36ae00that same mission completed green.
This commit is contained in:
@@ -366,6 +366,66 @@ scenario_microvm_unavailable_backend() {
|
|||||||
fi
|
fi
|
||||||
}
|
}
|
||||||
|
|
||||||
|
# ── Negative control: a gate that gives up must fail the phase ───
|
||||||
|
#
|
||||||
|
# `done_when_check` is run in exactly ONE place — the Stop hook inside the
|
||||||
|
# guest. The hook is capped so a stuck agent cannot wedge the turn, and at that
|
||||||
|
# cap it lets the agent stop. Until the `capped` marker existed, that release
|
||||||
|
# was invisible: the process exits 0 and the work IS collected, so both signals
|
||||||
|
# the run status was decided from said "fine" and the phase completed GREEN with
|
||||||
|
# its check still failing.
|
||||||
|
#
|
||||||
|
# The check here is `exit 1`, which no agent can satisfy. That is the point: it
|
||||||
|
# guarantees the cap is reached, so this exercises the release path itself
|
||||||
|
# rather than hoping to catch it. `blocks` reaching the cap is NOT the
|
||||||
|
# assertion — a healthy agent blocked three times and succeeding on the fourth
|
||||||
|
# reports the same number. The phase STATUS is the assertion.
|
||||||
|
GATECAP_BODY=$(cat <<JSON
|
||||||
|
{"title":"verify: a gate that gives up fails the phase",
|
||||||
|
"template_kind":"research_and_code",
|
||||||
|
"repo_id":"$REPO_ID",
|
||||||
|
"runtime_kind":"microvm",
|
||||||
|
"backend":"claude",
|
||||||
|
"description":"Negative control for the stop gate's cap release.",
|
||||||
|
"phases":[
|
||||||
|
{"kind":"coding","order_idx":0,"config":{"commit_policy":"always","max_iterations":1,
|
||||||
|
"done_when_check":"exit 1",
|
||||||
|
"task":"Create a file NOTES.md at the repository root containing the single word: hello. Create no other files."}}
|
||||||
|
]}
|
||||||
|
JSON
|
||||||
|
)
|
||||||
|
|
||||||
|
assert_gate_cap() { # <token> <mission> <report>
|
||||||
|
local mission="$2" report="$3" blocks note
|
||||||
|
while read -r idx status _files _pushed _branch _cerr _perr; do
|
||||||
|
if [ "$status" = "completed" ]; then
|
||||||
|
fail "gatecap: phase $idx COMPLETED with a check that cannot pass — the cap release is invisible again"
|
||||||
|
else
|
||||||
|
pass "gatecap: phase $idx did not complete (status=$status)"
|
||||||
|
fi
|
||||||
|
done <<<"$report"
|
||||||
|
|
||||||
|
# The gate must actually have run out of blocks; a phase that failed for some
|
||||||
|
# OTHER reason would satisfy the status check above while proving nothing.
|
||||||
|
blocks=$(ssh "$HOST" "docker logs --since 60m clawmates_server_1 2>&1 \
|
||||||
|
| grep -F 'microvm phase' | grep -F '$mission' | tail -1" \
|
||||||
|
| sed -n 's/.*stop-gate blocks: \([0-9-]*\).*/\1/p')
|
||||||
|
case "$blocks" in
|
||||||
|
3) pass "gatecap: the gate spent all 3 blocks before giving up" ;;
|
||||||
|
'') fail "gatecap: no stop-gate field in the log — was a gate installed at all?" ;;
|
||||||
|
*) fail "gatecap: expected 3 blocks before the cap, got $blocks" ;;
|
||||||
|
esac
|
||||||
|
|
||||||
|
note=$(ssh "$HOST" "docker logs --since 60m clawmates_server_1 2>&1 \
|
||||||
|
| grep -F 'microvm phase' | grep -F '$mission' | tail -1")
|
||||||
|
case "$note" in
|
||||||
|
*"completion gate released"*)
|
||||||
|
pass "gatecap: the failure names the cap release as the reason" ;;
|
||||||
|
*)
|
||||||
|
fail "gatecap: the phase failed but not for the gate's reason: $(printf '%s' "$note" | tail -c 200)" ;;
|
||||||
|
esac
|
||||||
|
}
|
||||||
|
|
||||||
# ── Scenario: the two engines composed ───────────────────────────
|
# ── Scenario: the two engines composed ───────────────────────────
|
||||||
#
|
#
|
||||||
# A `team_engine=composed` mission is a durable ZeroClaw graph whose every node
|
# A `team_engine=composed` mission is a durable ZeroClaw graph whose every node
|
||||||
@@ -741,6 +801,9 @@ case "${1:-all}" in
|
|||||||
run_scenario microvm "$(echo "$MICROVM_BODY" | tr -d '\n')" assert_microvm
|
run_scenario microvm "$(echo "$MICROVM_BODY" | tr -d '\n')" assert_microvm
|
||||||
scenario_microvm_unavailable_backend
|
scenario_microvm_unavailable_backend
|
||||||
;;
|
;;
|
||||||
|
gatecap)
|
||||||
|
run_scenario gatecap "$(echo "$GATECAP_BODY" | tr -d '\n')" assert_gate_cap
|
||||||
|
;;
|
||||||
composed)
|
composed)
|
||||||
run_scenario composed "$(echo "$COMPOSED_BODY" | tr -d '\n')" assert_composed
|
run_scenario composed "$(echo "$COMPOSED_BODY" | tr -d '\n')" assert_composed
|
||||||
;;
|
;;
|
||||||
@@ -755,11 +818,12 @@ case "${1:-all}" in
|
|||||||
run_scenario noop "$NOOP_BODY" assert_noop
|
run_scenario noop "$NOOP_BODY" assert_noop
|
||||||
run_scenario microvm "$(echo "$MICROVM_BODY" | tr -d '\n')" assert_microvm
|
run_scenario microvm "$(echo "$MICROVM_BODY" | tr -d '\n')" assert_microvm
|
||||||
scenario_microvm_unavailable_backend
|
scenario_microvm_unavailable_backend
|
||||||
|
run_scenario gatecap "$(echo "$GATECAP_BODY" | tr -d '\n')" assert_gate_cap
|
||||||
run_scenario composed "$(echo "$COMPOSED_BODY" | tr -d '\n')" assert_composed
|
run_scenario composed "$(echo "$COMPOSED_BODY" | tr -d '\n')" assert_composed
|
||||||
scenario_roster
|
scenario_roster
|
||||||
;;
|
;;
|
||||||
*)
|
*)
|
||||||
die "unknown scenario: $1 (selftest|uids|chain|multirole|noop|microvm|composed|roster|all)"
|
die "unknown scenario: $1 (selftest|uids|chain|multirole|noop|microvm|gatecap|composed|roster|all)"
|
||||||
;;
|
;;
|
||||||
esac
|
esac
|
||||||
|
|
||||||
|
|||||||
Reference in New Issue
Block a user