fix(harness): the capacity burst verifies its own launches
The re-run reported FAIL-NORUN "the burst did not finish in 1800s". The fleet was fine — 3 of the 16 missions were still in `draft`. Each PATCH-to-running is an ssh plus a `docker run curl`, and 16 at once does not reliably land; the response was going to /dev/null, so a launch that never happened spent the full timeout looking like a platform stall. That is precisely the swallowed-error shape this file was written to catch, committed inside the file itself. Launches are now verified against the mission rows, retried once for the stragglers, and reported as "the burst never happened" rather than as a timeout — a scenario that did not run must not be able to describe itself as a slow one. Co-Authored-By: Claude Opus 5 <[email protected]>
This commit is contained in:
co-authored by
Claude Opus 5
parent
4fedfcec30
commit
e5f097c291
@@ -1060,10 +1060,45 @@ JSON
|
|||||||
# Launch them as close to simultaneously as the API allows. Serialising the
|
# Launch them as close to simultaneously as the API allows. Serialising the
|
||||||
# launches would let each placement see the previous VM already counted,
|
# launches would let each placement see the previous VM already counted,
|
||||||
# which is the easy case; the race is the point.
|
# which is the easy case; the race is the point.
|
||||||
for mission in "${ids[@]}"; do
|
#
|
||||||
api "$token" PATCH "/api/missions/$mission/status" '{"status":"running"}' >/dev/null 2>&1 &
|
# Each PATCH is an ssh + `docker run curl`, and 16 of those at once does not
|
||||||
|
# always land: a first run of this scenario left 3 missions in `draft` and
|
||||||
|
# then spent the full 1800s waiting for them, reporting a platform timeout
|
||||||
|
# for a launch that never happened. Discarding the launch response is the
|
||||||
|
# same swallowed-error shape this harness exists to catch, so the launches
|
||||||
|
# are now VERIFIED against the mission rows rather than assumed.
|
||||||
|
launch_all() { # launch_all <mission…>
|
||||||
|
local m
|
||||||
|
for m in "$@"; do
|
||||||
|
api "$token" PATCH "/api/missions/$m/status" '{"status":"running"}' >/dev/null 2>&1 &
|
||||||
|
done
|
||||||
|
wait
|
||||||
|
}
|
||||||
|
still_draft() {
|
||||||
|
ssh "$HOST" "docker exec clawmates_postgres_1 psql -U postgres -d clawmates -tAc \
|
||||||
|
\"select id from missions where id in ($idlist) and status = 'draft';\"" \
|
||||||
|
2>/dev/null | tr -d '\r' | grep -v '^[[:space:]]*$'
|
||||||
|
}
|
||||||
|
|
||||||
|
launch_all "${ids[@]}"
|
||||||
|
local straggler_wait=0 stragglers
|
||||||
|
while [ "$straggler_wait" -lt 60 ]; do
|
||||||
|
stragglers=$(still_draft)
|
||||||
|
[ -z "$stragglers" ] && break
|
||||||
|
sleep 10
|
||||||
|
straggler_wait=$((straggler_wait + 10))
|
||||||
done
|
done
|
||||||
wait
|
if [ -n "${stragglers:-}" ]; then
|
||||||
|
info "capacity: $(printf '%s\n' "$stragglers" | wc -l | tr -d ' ') mission(s) never left draft — retrying the launch"
|
||||||
|
# shellcheck disable=SC2086
|
||||||
|
launch_all $stragglers
|
||||||
|
sleep 15
|
||||||
|
stragglers=$(still_draft)
|
||||||
|
fi
|
||||||
|
if [ -n "${stragglers:-}" ]; then
|
||||||
|
norun "capacity: $(printf '%s\n' "$stragglers" | wc -l | tr -d ' ') of $burst mission(s) could not be started — the burst never happened"
|
||||||
|
return 1
|
||||||
|
fi
|
||||||
|
|
||||||
# ── Sample while it runs ───────────────────────────────────────
|
# ── Sample while it runs ───────────────────────────────────────
|
||||||
#
|
#
|
||||||
|
|||||||
Reference in New Issue
Block a user