fix(rehearsal): never adopt the production compose project
INCIDENT: the release rehearsal destroyed production data on gw-04. deploy/compose/docker-compose.yml declares `name: clawmates` at the top level, and that beats --project-directory. So `compose up` from a temp directory did not create an isolated stack — it ADOPTED the running production stack of the same name, recreated its containers, and then the cleanup trap's `down -v` deleted its volumes, including clawmates_pgdata. Prod came back with an empty database: 177 repos, all missions and all agents gone. There were no backups. The fix is `-p rehearse-$$` on every invocation, plus an assertion that refuses to run under the production project name. Isolation here was implicit and therefore not isolation at all. Co-Authored-By: Claude Opus 5 <[email protected]>
This commit is contained in:
co-authored by
Claude Opus 5
parent
41854c70e1
commit
e7d2fc9696
@@ -32,9 +32,23 @@ if [ -z "${COMPOSE:-}" ]; then
|
|||||||
fi
|
fi
|
||||||
fi
|
fi
|
||||||
WORK="$(mktemp -d)"
|
WORK="$(mktemp -d)"
|
||||||
|
|
||||||
|
# A UNIQUE compose project, and never the production one.
|
||||||
|
#
|
||||||
|
# deploy/compose/docker-compose.yml declares `name: clawmates` at the top level,
|
||||||
|
# and that wins over --project-directory. On a shared host this rehearsal
|
||||||
|
# therefore ADOPTS any running stack called `clawmates` — and its cleanup trap
|
||||||
|
# runs `down -v`, which deletes that stack's volumes. That is exactly what
|
||||||
|
# happened on gw-04: the release rehearsal recreated the live containers and
|
||||||
|
# destroyed the production pgdata volume. -p is the fix; the assertion below is
|
||||||
|
# the seatbelt.
|
||||||
|
PROJECT="rehearse-$$"
|
||||||
|
case "$PROJECT" in
|
||||||
|
clawmates|clawmates-*) echo "refusing to use production project name" >&2; exit 1 ;;
|
||||||
|
esac
|
||||||
export CLAWMATES_HOME="$WORK/opt"
|
export CLAWMATES_HOME="$WORK/opt"
|
||||||
cleanup() {
|
cleanup() {
|
||||||
$COMPOSE --project-directory "$CLAWMATES_HOME" down -v >/dev/null 2>&1 || true
|
$COMPOSE -p "$PROJECT" --project-directory "$CLAWMATES_HOME" down -v >/dev/null 2>&1 || true
|
||||||
rm -rf "$WORK"
|
rm -rf "$WORK"
|
||||||
}
|
}
|
||||||
trap cleanup EXIT
|
trap cleanup EXIT
|
||||||
@@ -82,7 +96,7 @@ echo "==> Customer install: verify -> load -> stage"
|
|||||||
|
|
||||||
echo "==> First boot"
|
echo "==> First boot"
|
||||||
echo "POSTGRES_PASSWORD=rehearse-$$" > "$CLAWMATES_HOME/.env"
|
echo "POSTGRES_PASSWORD=rehearse-$$" > "$CLAWMATES_HOME/.env"
|
||||||
$COMPOSE --project-directory "$CLAWMATES_HOME" up -d --no-build
|
$COMPOSE -p "$PROJECT" --project-directory "$CLAWMATES_HOME" up -d --no-build
|
||||||
|
|
||||||
echo "==> Waiting for the platform"
|
echo "==> Waiting for the platform"
|
||||||
for _ in $(seq 1 60); do
|
for _ in $(seq 1 60); do
|
||||||
@@ -98,5 +112,5 @@ for _ in $(seq 1 60); do
|
|||||||
done
|
done
|
||||||
|
|
||||||
echo "REHEARSAL FAILED: platform never became healthy"
|
echo "REHEARSAL FAILED: platform never became healthy"
|
||||||
$COMPOSE --project-directory "$CLAWMATES_HOME" logs --tail 30
|
$COMPOSE -p "$PROJECT" --project-directory "$CLAWMATES_HOME" logs --tail 30
|
||||||
exit 1
|
exit 1
|
||||||
|
|||||||
Reference in New Issue
Block a user