feat(ops): fleet-reset — delete every mission and PROVE the disk came back
For a clean slate before a UI session, and for the thing that keeps being true here: deleting a row has never deleted a directory. A full harness run leaves ~35 missions, each with a repo checkout and a runtime-data tree, on the smallest disk in the fleet. There are 125 rows and 117 directories right now. Deletes through the API, never with SQL. `missions::delete` tears down the per-mission runtime container, hard-purges the FK graph in order, and removes the workspace directory — falling back to a root purge for the files the per-mission daemon leaves as root. A `DELETE FROM missions` skips all three and orphans every one of them, which is how the orphans got there. Then it checks, because rows gone is not bytes back and every incarnation of this cleanup has managed the first while silently failing the second: it names each directory left without a row, and counts root-owned residue separately because that is the specific way it fails. Refuses outright while any mission is RUNNING. Yanking a live mission's checkout leaves a VM writing into a directory that no longer exists, and the symptom is a phase that hangs rather than one that fails. Verified: it stopped exactly there against the in-flight harness. Dry by default; `--yes` to act; `KEEP=<substring>` to spare some. Co-Authored-By: Claude Opus 5 <[email protected]>
This commit is contained in:
Omar Sobh
co-authored by
Claude Opus 5
Claude Opus 5
parent
e7b412d578
commit
171f901bcd
Executable
+124
@@ -0,0 +1,124 @@
|
|||||||
|
#!/usr/bin/env bash
|
||||||
|
# Delete every mission and prove the disk came back with them.
|
||||||
|
#
|
||||||
|
# For getting to a clean slate before a UI session, and for the thing that keeps
|
||||||
|
# being true here: deleting a row has never deleted a directory. The harness
|
||||||
|
# leaves ~35 missions per full run and each carries a repo checkout plus a
|
||||||
|
# runtime-data tree, on the gateway — the smallest disk in the fleet.
|
||||||
|
#
|
||||||
|
# Deletes through the API, never with SQL. `missions::delete` tears down the
|
||||||
|
# per-mission runtime container, hard-purges the FK graph in order, and removes
|
||||||
|
# the workspace directory (falling back to a root purge for the files the
|
||||||
|
# per-mission daemon leaves as root). A `DELETE FROM missions` would skip all
|
||||||
|
# three and orphan every one of those.
|
||||||
|
#
|
||||||
|
# Then it CHECKS, because the whole point is that the cleanup path has been
|
||||||
|
# quietly incomplete before: rows gone is not the same as bytes back.
|
||||||
|
#
|
||||||
|
# Usage:
|
||||||
|
# scripts/fleet-reset.sh # dry run — says what it would delete
|
||||||
|
# scripts/fleet-reset.sh --yes # do it
|
||||||
|
# KEEP='verify: a model sizes' scripts/fleet-reset.sh --yes # keep matches
|
||||||
|
set -uo pipefail
|
||||||
|
|
||||||
|
HOST="${CLAWMATES_HOST:-gw-04}"
|
||||||
|
OWNER="${CLAWMATES_OWNER_EMAIL:-om[email protected]}"
|
||||||
|
MISSIONS_ROOT="${CLAWMATES_MISSIONS_ROOT:-/var/lib/clawmates-missions}"
|
||||||
|
KEEP="${KEEP:-}"
|
||||||
|
GO=0
|
||||||
|
[ "${1:-}" = "--yes" ] && GO=1
|
||||||
|
|
||||||
|
die() { printf 'ABORT %s\n' "$*" >&2; exit 2; }
|
||||||
|
|
||||||
|
psql_() { ssh "$HOST" "docker exec clawmates_postgres_1 psql -U postgres -d clawmates -tAc \"$1\""; }
|
||||||
|
|
||||||
|
mint_session() {
|
||||||
|
local secret hash rows
|
||||||
|
secret="reset-$(openssl rand -hex 16)"
|
||||||
|
hash=$(printf '%s' "$secret" | openssl dgst -sha256 -binary | openssl base64 -A | tr '+/' '-_' | tr -d '=')
|
||||||
|
rows=$(psql_ "insert into auth_sessions (user_id, token_hash, expires_at) \
|
||||||
|
select id, '$hash', now() + interval '30 minutes' from users where email='$OWNER' limit 1 returning 1;" \
|
||||||
|
2>/dev/null | head -1 | tr -d '[:space:]')
|
||||||
|
[ "$rows" = "1" ] || return 1
|
||||||
|
printf '%s' "$secret"
|
||||||
|
}
|
||||||
|
|
||||||
|
api() { # api <token> <METHOD> <path>
|
||||||
|
ssh "$HOST" "docker run --rm --network clawmates_core curlimages/curl:latest -s -o /dev/null -w '%{http_code}' \
|
||||||
|
-X $2 -H 'Authorization: Bearer $1' http://clawmates_server_1:8080$3"
|
||||||
|
}
|
||||||
|
|
||||||
|
ssh -o BatchMode=yes -o ConnectTimeout=10 "$HOST" true 2>/dev/null || die "cannot ssh to $HOST"
|
||||||
|
|
||||||
|
# ── Before ───────────────────────────────────────────────────────
|
||||||
|
before_rows=$(psql_ "select count(*) from missions;" | tr -d '[:space:]')
|
||||||
|
before_dirs=$(ssh "$HOST" "ls $MISSIONS_ROOT 2>/dev/null | grep -c '^[0-9a-f-]\{36\}$'" | tr -d '[:space:]')
|
||||||
|
before_kb=$(ssh "$HOST" "du -sk $MISSIONS_ROOT 2>/dev/null | cut -f1" | tr -d '[:space:]')
|
||||||
|
printf 'before: %s mission row(s), %s director(y|ies), %s MiB\n' \
|
||||||
|
"$before_rows" "$before_dirs" "$((${before_kb:-0} / 1024))"
|
||||||
|
|
||||||
|
# Never delete a mission that is still working. A reset that yanks a running
|
||||||
|
# mission's checkout leaves a VM writing into a directory that no longer exists,
|
||||||
|
# and the symptom is a phase that hangs rather than fails.
|
||||||
|
busy=$(psql_ "select count(*) from missions where status = 'running';" | tr -d '[:space:]')
|
||||||
|
if [ "${busy:-0}" != "0" ]; then
|
||||||
|
die "$busy mission(s) are still RUNNING — wait for them or cancel them first"
|
||||||
|
fi
|
||||||
|
|
||||||
|
filter=""
|
||||||
|
[ -n "$KEEP" ] && filter=" and title not like '%${KEEP}%'"
|
||||||
|
ids=$(psql_ "select id from missions where true$filter order by created_at;" | tr -d '\r' | grep -v '^[[:space:]]*$')
|
||||||
|
count=$(printf '%s\n' "$ids" | grep -c '[^[:space:]]')
|
||||||
|
[ "${count:-0}" -gt 0 ] || { echo "nothing to delete"; exit 0; }
|
||||||
|
|
||||||
|
if [ "$GO" != "1" ]; then
|
||||||
|
printf '\nwould delete %s mission(s)%s. Re-run with --yes.\n' "$count" \
|
||||||
|
"${KEEP:+ (keeping titles matching '$KEEP')}"
|
||||||
|
exit 0
|
||||||
|
fi
|
||||||
|
|
||||||
|
token=$(mint_session) || die "could not mint a session for $OWNER"
|
||||||
|
|
||||||
|
ok=0; bad=0
|
||||||
|
for id in $ids; do
|
||||||
|
code=$(api "$token" DELETE "/api/missions/$id")
|
||||||
|
case "$code" in
|
||||||
|
2*) ok=$((ok + 1)) ;;
|
||||||
|
*) bad=$((bad + 1)); printf 'FAIL %s -> HTTP %s\n' "$id" "$code" ;;
|
||||||
|
esac
|
||||||
|
done
|
||||||
|
printf 'deleted %s, failed %s\n' "$ok" "$bad"
|
||||||
|
|
||||||
|
# ── After, and the part that matters ─────────────────────────────
|
||||||
|
#
|
||||||
|
# Rows gone is not the same as bytes back. Every incarnation of this cleanup has
|
||||||
|
# been able to do the first while silently failing the second.
|
||||||
|
sleep 5
|
||||||
|
after_rows=$(psql_ "select count(*) from missions;" | tr -d '[:space:]')
|
||||||
|
after_dirs=$(ssh "$HOST" "ls $MISSIONS_ROOT 2>/dev/null | grep -c '^[0-9a-f-]\{36\}$'" | tr -d '[:space:]')
|
||||||
|
after_kb=$(ssh "$HOST" "du -sk $MISSIONS_ROOT 2>/dev/null | cut -f1" | tr -d '[:space:]')
|
||||||
|
printf 'after: %s mission row(s), %s director(y|ies), %s MiB\n' \
|
||||||
|
"$after_rows" "$after_dirs" "$((${after_kb:-0} / 1024))"
|
||||||
|
|
||||||
|
# Any directory left without a row is an orphan the reaper missed. Named
|
||||||
|
# individually — a count alone is something to shrug at.
|
||||||
|
orphans=$(ssh "$HOST" "cd $MISSIONS_ROOT 2>/dev/null && ls -d */ 2>/dev/null | tr -d '/' \
|
||||||
|
| grep -E '^[0-9a-f-]{36}$'" | tr -d '\r' | grep -v '^[[:space:]]*$' || true)
|
||||||
|
left=0
|
||||||
|
for d in $orphans; do
|
||||||
|
row=$(psql_ "select count(*) from missions where id = '$d';" | tr -d '[:space:]')
|
||||||
|
[ "${row:-0}" = "0" ] && { left=$((left + 1)); printf 'ORPHAN %s (no row, still on disk)\n' "$d"; }
|
||||||
|
done
|
||||||
|
|
||||||
|
# Root-owned residue is the specific way this fails: the server runs as 65532
|
||||||
|
# and cannot delete what the per-mission daemon wrote as root.
|
||||||
|
rootfiles=$(ssh "$HOST" "find $MISSIONS_ROOT ! -uid 65532 2>/dev/null | wc -l" | tr -d '[:space:]')
|
||||||
|
|
||||||
|
echo
|
||||||
|
if [ "$bad" = "0" ] && [ "$left" = "0" ]; then
|
||||||
|
printf 'clean: %s mission(s) deleted, no orphaned directories\n' "$ok"
|
||||||
|
[ "${rootfiles:-0}" = "0" ] || printf 'note: %s root-owned file(s) remain under %s\n' "$rootfiles" "$MISSIONS_ROOT"
|
||||||
|
exit 0
|
||||||
|
fi
|
||||||
|
printf 'NOT clean: %s delete(s) failed, %s orphaned director(y|ies) left\n' "$bad" "$left"
|
||||||
|
exit 1
|
||||||
Reference in New Issue
Block a user