test(fleet): prove the queue and the spread under a real burst
Phase 1 shipped placement-at-phase-launch and a queue made of `start_pending_phases` leaving a phase `pending`, both deployed unproven under load — the exact condition this project keeps getting burned by: the code is right, the system is wrong, and nothing errors. `capacity` launches `slots + 2` microVM missions simultaneously and asserts two things. That no node ever exceeds the slots `vm_placement` gave it — overcommit does not fail loudly, it swaps, and every mission on that node gets slow rather than dead. And that the excess QUEUES: a burst that drops the extras and one that wedges them both look identical to any check that only reads the end state. `capacity_blocked_since` is cleared the instant a phase is placed, so the evidence only exists mid-flight; the scenario samples while it runs. Capacity comes from `/api/fleet/capacity`, never recomputed here — a bash copy of the slot arithmetic would drift from the scheduler and then agree with itself. A burst that does not exceed capacity is reported NORUN, per rule 3. `drain-midmission` drains the node phase 0 ran on, before phase 1 is placed, and asserts phase 1 lands elsewhere AND still reads phase 0's file. That is the test of the affinity decision: mission state lives on the gateway, so re-placement is free — if it were not, this would either strand the mission or silently lose the earlier work, and "silently lose" is what a status-only check calls success. The node is restored before any assertion runs, so a failure cannot leave the fleet permanently one node smaller. Smoke-checked at CAPACITY_BURST=2: sampling, spread and completion all report, and the queue check correctly returned NORUN rather than a green tick. Co-Authored-By: Claude Opus 5 <[email protected]>
This commit is contained in:
Omar Sobh
co-authored by
Claude Opus 5
Claude Opus 5
parent
dc0443de34
commit
2056bb1d9e
@@ -31,6 +31,8 @@
|
|||||||
# scripts/verify-mission-delivery.sh multirole # 3 roles + real tests
|
# scripts/verify-mission-delivery.sh multirole # 3 roles + real tests
|
||||||
# scripts/verify-mission-delivery.sh noop # empty phase must FAIL
|
# scripts/verify-mission-delivery.sh noop # empty phase must FAIL
|
||||||
# scripts/verify-mission-delivery.sh microvm # runs in a guest kernel + fans out
|
# scripts/verify-mission-delivery.sh microvm # runs in a guest kernel + fans out
|
||||||
|
# scripts/verify-mission-delivery.sh capacity # a burst > the fleet must QUEUE
|
||||||
|
# scripts/verify-mission-delivery.sh drain-midmission # a drained node hands the mission on
|
||||||
# scripts/verify-mission-delivery.sh all # everything
|
# scripts/verify-mission-delivery.sh all # everything
|
||||||
#
|
#
|
||||||
# Environment:
|
# Environment:
|
||||||
@@ -41,6 +43,9 @@
|
|||||||
# CLAWMATES_REPO_ID scratch repo for the delivery scenarios
|
# CLAWMATES_REPO_ID scratch repo for the delivery scenarios
|
||||||
# CLAWMATES_TEAM_TEMPLATE team template for the delivery scenarios
|
# CLAWMATES_TEAM_TEMPLATE team template for the delivery scenarios
|
||||||
# MISSION_TIMEOUT seconds to wait for a mission (default 1800)
|
# MISSION_TIMEOUT seconds to wait for a mission (default 1800)
|
||||||
|
# CAPACITY_BURST missions to launch in the capacity burst
|
||||||
|
# (default: fleet slots + 2 — smaller is reported
|
||||||
|
# NORUN, since it never exercises the queue)
|
||||||
|
|
||||||
set -uo pipefail
|
set -uo pipefail
|
||||||
|
|
||||||
@@ -986,6 +991,294 @@ exercise the guard (agent ignored the instruction; re-run)"; continue ;;
|
|||||||
return 0
|
return 0
|
||||||
}
|
}
|
||||||
|
|
||||||
|
# ── Scenario: a burst larger than the fleet QUEUES, and spreads ───
|
||||||
|
#
|
||||||
|
# Phase 1 of the fleet-intelligence plan shipped placement-at-phase-launch and
|
||||||
|
# a queue built out of `start_pending_phases` leaving a phase `pending`. Both
|
||||||
|
# were deployed unproven under load, which is the condition this project keeps
|
||||||
|
# getting burned by: the code is right, the system is wrong, and nothing errors.
|
||||||
|
#
|
||||||
|
# Two invariants, and the second is the one that costs money to get wrong:
|
||||||
|
#
|
||||||
|
# 1. No node ever runs more VMs than `vm_placement` said it had room for.
|
||||||
|
# Overcommit does not fail loudly — it swaps, and every mission on that
|
||||||
|
# node gets slow instead of dead.
|
||||||
|
# 2. The excess QUEUES. A burst that silently drops the extras, or wedges
|
||||||
|
# them forever, both "pass" any check that only looks at the end state.
|
||||||
|
#
|
||||||
|
# Capacity comes from `/api/fleet/capacity`, which returns `vm_placement`'s own
|
||||||
|
# survey. Recomputing the slot arithmetic here would let this test drift from
|
||||||
|
# the scheduler and then agree with itself.
|
||||||
|
|
||||||
|
fleet_capacity() { # fleet_capacity <token> -> json
|
||||||
|
api "$1" GET "/api/fleet/capacity?backend=claude"
|
||||||
|
}
|
||||||
|
|
||||||
|
CAPACITY_TASK='Create a file named BURST.md at the repository root containing exactly one line: BURST-OK. Do not create or modify any other file.'
|
||||||
|
|
||||||
|
scenario_capacity() {
|
||||||
|
local token cap slots burst i mission ids=() idlist
|
||||||
|
token=$(mint_session) || { norun "capacity: could not mint a session"; return 1; }
|
||||||
|
|
||||||
|
cap=$(fleet_capacity "$token")
|
||||||
|
slots=$(printf '%s' "$cap" | python3 -c 'import json,sys
|
||||||
|
try: print(json.load(sys.stdin)["slots"])
|
||||||
|
except Exception: pass' 2>/dev/null)
|
||||||
|
# A probe returns a value or fails. An unreadable capacity survey must not
|
||||||
|
# become a burst of size 2 that passes because it never saturated anything.
|
||||||
|
case "${slots:-}" in
|
||||||
|
''|*[!0-9]*) norun "capacity: could not read /api/fleet/capacity: $(printf '%s' "$cap" | head -c 200)"; return 1 ;;
|
||||||
|
esac
|
||||||
|
local fitnodes; fitnodes=$(printf '%s' "$cap" | python3 -c 'import json,sys; print(len(json.load(sys.stdin)["nodes"]))')
|
||||||
|
info "capacity: fleet has $slots free slot(s) across $fitnodes node(s)"
|
||||||
|
printf '%s' "$cap" | python3 -c 'import json,sys
|
||||||
|
for n in json.load(sys.stdin)["nodes"]: print(" capacity: %-12s %d slot(s), %d committed" % (n["name"], n["slots"], n["committedVms"]))'
|
||||||
|
[ "$slots" -gt 0 ] || { norun "capacity: fleet reports 0 slots — nothing to saturate"; return 1; }
|
||||||
|
|
||||||
|
# Two more than the fleet can hold, so the queue is EXERCISED rather than
|
||||||
|
# merely available. Overridable for a cheaper smoke run, but a burst that
|
||||||
|
# does not exceed capacity is reported as NORUN below, never as PASS.
|
||||||
|
burst="${CAPACITY_BURST:-$((slots + 2))}"
|
||||||
|
info "capacity: launching a burst of $burst mission(s)"
|
||||||
|
|
||||||
|
for i in $(seq 1 "$burst"); do
|
||||||
|
mission=$(create_mission "$token" "$(cat <<JSON | tr -d '\n'
|
||||||
|
{"title":"verify: burst $i of $burst",
|
||||||
|
"template_kind":"research_and_code",
|
||||||
|
"repo_id":"$REPO_ID",
|
||||||
|
"runtime_kind":"microvm",
|
||||||
|
"backend":"claude",
|
||||||
|
"description":"Placement load test — one trivial file.",
|
||||||
|
"phases":[{"kind":"coding","order_idx":0,"config":{"commit_policy":"always","max_iterations":1,
|
||||||
|
"task":"$CAPACITY_TASK"}}]}
|
||||||
|
JSON
|
||||||
|
)") || { norun "capacity: mission $i of $burst failed to create"; return 1; }
|
||||||
|
ids+=("$mission")
|
||||||
|
done
|
||||||
|
idlist=$(printf "'%s'," "${ids[@]}"); idlist="${idlist%,}"
|
||||||
|
|
||||||
|
# Launch them as close to simultaneously as the API allows. Serialising the
|
||||||
|
# launches would let each placement see the previous VM already counted,
|
||||||
|
# which is the easy case; the race is the point.
|
||||||
|
for mission in "${ids[@]}"; do
|
||||||
|
api "$token" PATCH "/api/missions/$mission/status" '{"status":"running"}' >/dev/null 2>&1 &
|
||||||
|
done
|
||||||
|
wait
|
||||||
|
|
||||||
|
# ── Sample while it runs ───────────────────────────────────────
|
||||||
|
#
|
||||||
|
# `capacity_blocked_since` is CLEARED the moment a phase is placed, so a
|
||||||
|
# post-hoc query cannot prove a queue ever formed. The evidence only exists
|
||||||
|
# while the burst is in flight.
|
||||||
|
local waited=0 peak_file queued=0 overcommit=""
|
||||||
|
peak_file=$(mktemp)
|
||||||
|
while [ "$waited" -lt "$MISSION_TIMEOUT" ]; do
|
||||||
|
local done_count
|
||||||
|
done_count=$(ssh "$HOST" "docker exec clawmates_postgres_1 psql -U postgres -d clawmates -tAc \
|
||||||
|
\"select count(*) from missions where id in ($idlist) and status in ('completed','failed','cancelled');\"" \
|
||||||
|
| head -1 | tr -d '[:space:]')
|
||||||
|
|
||||||
|
# Per-node concurrent phase VMs, right now.
|
||||||
|
ssh "$HOST" "docker exec clawmates_postgres_1 psql -U postgres -d clawmates -tAc \
|
||||||
|
\"select coalesce(n.name,'unpinned'), count(*) from mission_phases p
|
||||||
|
join missions m on m.id = p.mission_id
|
||||||
|
left join nodes n on n.id = m.target_node_id
|
||||||
|
where m.id in ($idlist) and p.status = 'running'
|
||||||
|
group by 1;\"" 2>/dev/null | tr -d '\r' >> "$peak_file"
|
||||||
|
|
||||||
|
local q
|
||||||
|
q=$(ssh "$HOST" "docker exec clawmates_postgres_1 psql -U postgres -d clawmates -tAc \
|
||||||
|
\"select count(*) from mission_phases p join missions m on m.id = p.mission_id
|
||||||
|
where m.id in ($idlist) and p.capacity_blocked_since is not null;\"" \
|
||||||
|
| head -1 | tr -d '[:space:]')
|
||||||
|
[ "${q:-0}" -gt 0 ] 2>/dev/null && queued=1
|
||||||
|
|
||||||
|
[ "${done_count:-0}" = "$burst" ] && break
|
||||||
|
sleep 15
|
||||||
|
waited=$((waited + 15))
|
||||||
|
done
|
||||||
|
|
||||||
|
if [ "$waited" -ge "$MISSION_TIMEOUT" ]; then
|
||||||
|
norun "capacity: the burst did not finish in ${MISSION_TIMEOUT}s — $(cat "$peak_file" | tail -3)"
|
||||||
|
rm -f "$peak_file"
|
||||||
|
return 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
# ── Invariant 1: nobody exceeded their slot count ──────────────
|
||||||
|
local peaks
|
||||||
|
peaks=$(sort -t'|' -k1,1 "$peak_file" | grep -v '^\s*$' | python3 -c '
|
||||||
|
import sys, collections
|
||||||
|
peak = collections.defaultdict(int)
|
||||||
|
for line in sys.stdin:
|
||||||
|
line = line.strip()
|
||||||
|
if not line or "|" not in line: continue
|
||||||
|
name, count = line.rsplit("|", 1)
|
||||||
|
try: peak[name.strip()] = max(peak[name.strip()], int(count))
|
||||||
|
except ValueError: pass
|
||||||
|
for k, v in sorted(peak.items()): print(k, v)
|
||||||
|
')
|
||||||
|
rm -f "$peak_file"
|
||||||
|
if [ -z "$peaks" ]; then
|
||||||
|
norun "capacity: never observed a running phase — the burst did not execute"
|
||||||
|
return 1
|
||||||
|
fi
|
||||||
|
while read -r name peak; do
|
||||||
|
[ -n "$name" ] || continue
|
||||||
|
local allowed
|
||||||
|
allowed=$(printf '%s' "$cap" | python3 -c "import json,sys
|
||||||
|
n = {x['name']: x['slots'] for x in json.load(sys.stdin)['nodes']}
|
||||||
|
print(n.get('$name', -1))")
|
||||||
|
if [ "$allowed" = "-1" ]; then
|
||||||
|
fail "capacity: work ran on '$name', which the survey did not list as fit"
|
||||||
|
elif [ "$peak" -gt "$allowed" ]; then
|
||||||
|
overcommit="yes"
|
||||||
|
fail "capacity: '$name' peaked at $peak concurrent VM(s) with only $allowed slot(s)"
|
||||||
|
else
|
||||||
|
info "capacity: $name peaked at $peak of $allowed slot(s)"
|
||||||
|
fi
|
||||||
|
done <<<"$peaks"
|
||||||
|
[ -n "$overcommit" ] || pass "capacity: no node exceeded the slots the scheduler gave it"
|
||||||
|
|
||||||
|
# ── Invariant 2: the excess queued rather than vanishing ───────
|
||||||
|
if [ "$burst" -le "$slots" ]; then
|
||||||
|
norun "capacity: burst ($burst) did not exceed capacity ($slots) — the queue was never exercised"
|
||||||
|
elif [ "$queued" = "1" ]; then
|
||||||
|
pass "capacity: the over-capacity missions QUEUED (capacity_blocked_since was set)"
|
||||||
|
else
|
||||||
|
fail "capacity: $burst missions on $slots slots and nothing ever queued — placement is not counting commitments"
|
||||||
|
fi
|
||||||
|
|
||||||
|
# ── Every mission still delivered ──────────────────────────────
|
||||||
|
local completed
|
||||||
|
completed=$(ssh "$HOST" "docker exec clawmates_postgres_1 psql -U postgres -d clawmates -tAc \
|
||||||
|
\"select count(*) from missions where id in ($idlist) and status = 'completed';\"" \
|
||||||
|
| head -1 | tr -d '[:space:]')
|
||||||
|
if [ "${completed:-0}" = "$burst" ]; then
|
||||||
|
pass "capacity: all $burst queued/placed missions completed"
|
||||||
|
else
|
||||||
|
fail "capacity: only ${completed:-0} of $burst missions completed — a queued mission must still run"
|
||||||
|
fi
|
||||||
|
|
||||||
|
# ── Spread, not stack ─────────────────────────────────────────
|
||||||
|
local used
|
||||||
|
used=$(ssh "$HOST" "docker exec clawmates_postgres_1 psql -U postgres -d clawmates -tAc \
|
||||||
|
\"select count(distinct target_node_id) from missions where id in ($idlist) and target_node_id is not null;\"" \
|
||||||
|
| head -1 | tr -d '[:space:]')
|
||||||
|
if [ "$fitnodes" -lt 2 ]; then
|
||||||
|
info "capacity: only one fit node — spread is not assertable"
|
||||||
|
elif [ "${used:-0}" -ge 2 ]; then
|
||||||
|
pass "capacity: the burst spread across ${used} node(s)"
|
||||||
|
else
|
||||||
|
fail "capacity: $burst missions all landed on ${used:-0} node(s) — ranking is stacking, not spreading"
|
||||||
|
fi
|
||||||
|
}
|
||||||
|
|
||||||
|
# ── Scenario: draining a node mid-mission re-places the next phase ─
|
||||||
|
#
|
||||||
|
# The affinity decision made this possible and this test is what proves it was
|
||||||
|
# taken for real: mission state lives on the GATEWAY, every phase is
|
||||||
|
# inject-tar → run → collect-tar → destroy, so phase 2 owes nothing to the node
|
||||||
|
# phase 1 ran on. If that were false, draining here would either strand the
|
||||||
|
# mission or silently lose phase 1's work — and "silently lose" is the outcome
|
||||||
|
# a status-only check reports as success.
|
||||||
|
|
||||||
|
DRAIN_BODY=$(cat <<JSON
|
||||||
|
{"title":"verify: a drained node hands the mission on",
|
||||||
|
"template_kind":"research_and_code",
|
||||||
|
"repo_id":"$REPO_ID",
|
||||||
|
"runtime_kind":"microvm",
|
||||||
|
"backend":"claude",
|
||||||
|
"description":"Build up DRAIN.md across two phases while the fleet changes under it.",
|
||||||
|
"phases":[
|
||||||
|
{"kind":"coding","order_idx":0,"config":{"commit_policy":"always","max_iterations":1,
|
||||||
|
"task":"Create a file named DRAIN.md at the repository root containing exactly one line: PHASE-ONE-OK. Do not create or modify any other file."}},
|
||||||
|
{"kind":"coding","order_idx":1,"config":{"commit_policy":"always","max_iterations":1,
|
||||||
|
"task":"Read the existing file DRAIN.md at the repository root. It was written by the previous phase and must already contain the line PHASE-ONE-OK. Append a second line reading PHASE-TWO-OK, keeping the first line intact. If DRAIN.md does not exist, instead create a file named DRAIN_MISSING.md containing the single line PRIOR-PHASE-WORK-WAS-LOST, and do not create DRAIN.md."}}
|
||||||
|
]}
|
||||||
|
JSON
|
||||||
|
)
|
||||||
|
|
||||||
|
undrain() { # undrain <node-uuid>
|
||||||
|
[ -n "${1:-}" ] || return 0
|
||||||
|
ssh "$HOST" "docker exec clawmates_postgres_1 psql -U postgres -d clawmates -tAc \
|
||||||
|
\"update nodes set status='online' where id='$1' and status='draining';\"" >/dev/null 2>&1
|
||||||
|
}
|
||||||
|
|
||||||
|
scenario_drain_midmission() {
|
||||||
|
local token mission first_node second_node waited=0 status delivered
|
||||||
|
token=$(mint_session) || { norun "drain-midmission: could not mint a session"; return 1; }
|
||||||
|
local fit
|
||||||
|
fit=$(fleet_capacity "$token" | python3 -c 'import json,sys
|
||||||
|
try: print(len(json.load(sys.stdin)["nodes"]))
|
||||||
|
except Exception: print(0)')
|
||||||
|
if [ "${fit:-0}" -lt 2 ]; then
|
||||||
|
norun "drain-midmission: needs 2+ fit nodes, fleet has ${fit:-0}"
|
||||||
|
return 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
mission=$(create_mission "$token" "$(echo "$DRAIN_BODY" | tr -d '\n')") \
|
||||||
|
|| { norun "drain-midmission: mission create failed"; return 1; }
|
||||||
|
info "drain-midmission: mission=$mission"
|
||||||
|
api "$token" PATCH "/api/missions/$mission/status" '{"status":"running"}' >/dev/null
|
||||||
|
|
||||||
|
# Wait for phase 0 to finish, then drain the node it used — before phase 1
|
||||||
|
# is placed. Polling is on the PHASE, not the mission: by the time the
|
||||||
|
# mission is terminal there is nothing left to re-place.
|
||||||
|
while [ "$waited" -lt "$MISSION_TIMEOUT" ]; do
|
||||||
|
first_node=$(ssh "$HOST" "docker exec clawmates_postgres_1 psql -U postgres -d clawmates -tAc \
|
||||||
|
\"select m.target_node_id from mission_phases p join missions m on m.id = p.mission_id
|
||||||
|
where p.mission_id = '$mission' and p.order_idx = 0 and p.status = 'completed';\"" \
|
||||||
|
| head -1 | tr -d '[:space:]')
|
||||||
|
[ -n "$first_node" ] && break
|
||||||
|
# A mission that died before phase 0 completed has nothing to test.
|
||||||
|
status=$(api "$token" GET "/api/missions/$mission" | python3 -c 'import json,sys
|
||||||
|
try: print(json.load(sys.stdin).get("status",""))
|
||||||
|
except Exception: pass' 2>/dev/null)
|
||||||
|
case "$status" in failed|cancelled)
|
||||||
|
norun "drain-midmission: mission ended $status before phase 0 completed"; return 1 ;;
|
||||||
|
esac
|
||||||
|
sleep 10
|
||||||
|
waited=$((waited + 10))
|
||||||
|
done
|
||||||
|
if [ -z "$first_node" ]; then
|
||||||
|
norun "drain-midmission: phase 0 never completed in ${MISSION_TIMEOUT}s"
|
||||||
|
return 1
|
||||||
|
fi
|
||||||
|
info "drain-midmission: phase 0 ran on $first_node — draining it"
|
||||||
|
ssh "$HOST" "docker exec clawmates_postgres_1 psql -U postgres -d clawmates -tAc \
|
||||||
|
\"update nodes set status='draining' where id='$first_node';\"" >/dev/null
|
||||||
|
|
||||||
|
status=$(await_mission "$token" "$mission")
|
||||||
|
# Restore the node BEFORE asserting, so a failed assertion cannot leave the
|
||||||
|
# fleet one node smaller for every later run.
|
||||||
|
undrain "$first_node"
|
||||||
|
[ "$status" != "timeout" ] || { norun "drain-midmission: mission did not finish in ${MISSION_TIMEOUT}s"; return 1; }
|
||||||
|
printf '%s\n' "$(phase_report "$token" "$mission")" | sed 's/^/ phase /'
|
||||||
|
|
||||||
|
second_node=$(ssh "$HOST" "docker exec clawmates_postgres_1 psql -U postgres -d clawmates -tAc \
|
||||||
|
\"select target_node_id from missions where id = '$mission';\"" \
|
||||||
|
| head -1 | tr -d '[:space:]')
|
||||||
|
if [ -z "$second_node" ]; then
|
||||||
|
fail "drain-midmission: the mission has no target node after phase 1"
|
||||||
|
elif [ "$second_node" = "$first_node" ]; then
|
||||||
|
fail "drain-midmission: phase 1 ran on the DRAINED node $first_node"
|
||||||
|
else
|
||||||
|
pass "drain-midmission: phase 1 re-placed onto $second_node"
|
||||||
|
fi
|
||||||
|
|
||||||
|
delivered=$(fetch_delivered "$token" "$mission" DRAIN.md) \
|
||||||
|
|| { fail "drain-midmission: could not read DRAIN.md from the pushed branch"; return 1; }
|
||||||
|
case "$delivered" in
|
||||||
|
*PHASE-ONE-OK*PHASE-TWO-OK*)
|
||||||
|
pass "drain-midmission: phase 1 read phase 0's work from a DIFFERENT node and appended" ;;
|
||||||
|
*PRIOR-PHASE-WORK-WAS-LOST*)
|
||||||
|
fail "drain-midmission: re-placement lost the previous phase's work" ;;
|
||||||
|
*)
|
||||||
|
fail "drain-midmission: DRAIN.md is neither outcome: $(printf '%s' "$delivered" | tr '\n' '|')" ;;
|
||||||
|
esac
|
||||||
|
check_single_uid "$mission" drain-midmission
|
||||||
|
}
|
||||||
|
|
||||||
# ── Entry point ──────────────────────────────────────────────────
|
# ── Entry point ──────────────────────────────────────────────────
|
||||||
|
|
||||||
ssh -o BatchMode=yes -o ConnectTimeout=10 "$HOST" true 2>/dev/null \
|
ssh -o BatchMode=yes -o ConnectTimeout=10 "$HOST" true 2>/dev/null \
|
||||||
@@ -1046,6 +1339,12 @@ case "${1:-all}" in
|
|||||||
roster)
|
roster)
|
||||||
scenario_roster
|
scenario_roster
|
||||||
;;
|
;;
|
||||||
|
capacity)
|
||||||
|
scenario_capacity
|
||||||
|
;;
|
||||||
|
drain-midmission)
|
||||||
|
scenario_drain_midmission
|
||||||
|
;;
|
||||||
all)
|
all)
|
||||||
selftest_uid_probe
|
selftest_uid_probe
|
||||||
run_scenario chain "$CHAIN_BODY" assert_chain
|
run_scenario chain "$CHAIN_BODY" assert_chain
|
||||||
@@ -1061,9 +1360,11 @@ case "${1:-all}" in
|
|||||||
run_scenario refactor "$(echo "$REFACTOR_BODY" | tr -d '\n')" assert_refactor
|
run_scenario refactor "$(echo "$REFACTOR_BODY" | tr -d '\n')" assert_refactor
|
||||||
run_scenario composed "$(echo "$COMPOSED_BODY" | tr -d '\n')" assert_composed
|
run_scenario composed "$(echo "$COMPOSED_BODY" | tr -d '\n')" assert_composed
|
||||||
scenario_roster
|
scenario_roster
|
||||||
|
scenario_capacity
|
||||||
|
scenario_drain_midmission
|
||||||
;;
|
;;
|
||||||
*)
|
*)
|
||||||
die "unknown scenario: $1 (selftest|uids|chain|multirole|noop|microvm|canary|gatecap|research-only|benchmark|security|refactor|composed|roster|all)"
|
die "unknown scenario: $1 (selftest|uids|chain|multirole|noop|microvm|canary|gatecap|research-only|benchmark|security|refactor|composed|roster|capacity|drain-midmission|all)"
|
||||||
;;
|
;;
|
||||||
esac
|
esac
|
||||||
|
|
||||||
|
|||||||
Reference in New Issue
Block a user