Omar SobhandClaude Opus 5
8470534e33
morpheus is packed for a move. It is drained in the `nodes` table — heartbeats preserve `draining`, so it stays out of placement when it comes back — and removed from the agent-image loop here. An unreachable host in NODES does not merely skip it. The image loop fails the whole script BEFORE its verify stage, so four deploys in a row rolled the server and frontend correctly and then reported nothing at all; every one had to be confirmed by hand. Keep this list to hosts that answer. The name is left in a comment rather than deleted: putting it back is one word, and the next person will want to know where it went. Co-Authored-By: Claude Opus 5 <[email protected]>
206 lines
11 KiB
Bash
Executable File
206 lines
11 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
# Deploy ClawMates.
|
|
#
|
|
# The server + frontend run on the gateway from images pulled from the web-01
|
|
# registry (100.94.185.103:5000) — /opt/clawmates/docker-compose.yml references
|
|
# `$REGISTRY/clawmates/{server,frontend}:latest`, and `docker-compose up` PULLS
|
|
# them. So the ONLY reliable way to ship server/frontend is: build on the build
|
|
# host → push to the registry → pull + recreate on the gateway. A `docker save |
|
|
# docker load` + local retag does NOT work here — the next `up` pulls `:latest`
|
|
# from the registry and silently reverts to whatever was last pushed. (That is
|
|
# exactly the footgun this rewrite removes; the old save|load path is gone.)
|
|
#
|
|
# The agent runtime images (agent-base / agent-browser / agent-terminal) are
|
|
# locally-built dev images NOT in any registry, so those still ship via
|
|
# `docker save | docker load` to the gateway AND every fleet node (a `docker
|
|
# pull` can't fetch them; the load prevents "No such image" 404s).
|
|
#
|
|
# Usage: scripts/deploy.sh # full deploy
|
|
# IMAGES_ONLY=1 scripts/deploy.sh # just (re)build + load the agent images
|
|
#
|
|
# Override via env: BUILD_HOST, GW, GW_DIR, NODES, TAG, REGISTRY.
|
|
set -euo pipefail
|
|
|
|
BUILD_HOST=${BUILD_HOST:-osobh@tank} # builds + pushes the server/frontend images
|
|
GW=${GW:-gw-04} # the gateway (server + frontend + postgres)
|
|
GW_DIR=${GW_DIR:-/opt/clawmates} # the compose project dir ON the gateway
|
|
REGISTRY=${REGISTRY:-100.94.185.103:5000} # web-01 registry the compose pulls from
|
|
# Fleet nodes that provision agent containers.
|
|
#
|
|
# morpheus was removed 2026-08-11: it is drained in the `nodes` table (which
|
|
# heartbeats preserve, so it stays out of placement when it comes back) and
|
|
# packed for a move. It is listed here as a comment rather than deleted because
|
|
# putting it back is one word, and the next person will want to know where.
|
|
# was: "morpheus osobh@tank architect"
|
|
# An unreachable host here does not just skip — the image loop fails the whole
|
|
# script BEFORE its verify stage, so four deploys in a row rolled correctly and
|
|
# reported nothing. Keep this list to hosts that answer.
|
|
NODES=${NODES:-"osobh@tank architect"}
|
|
TAG=${TAG:-latest}
|
|
# Immutable per-deploy tag so a deploy is traceable and rollback is a repoint,
|
|
# not a rebuild. Falls back to a timestamp-free literal when not in a git tree.
|
|
SHA=$(git rev-parse --short HEAD 2>/dev/null || echo manual)
|
|
AGENT_IMAGES=(agent-base agent-browser agent-terminal)
|
|
|
|
# Stream an image between two remote hosts. The tar crosses two SSH
|
|
# connections spliced through this workstation, so a stall on either side
|
|
# truncates it — that surfaces as `unexpected EOF` from `docker load`, which
|
|
# is a genuine failure and was previously indistinguishable from success
|
|
# because nothing checked afterwards. Compress (these images are mostly
|
|
# filesystem, and less bytes is less exposure to a stall) and set pipefail so
|
|
# a failed `save` cannot be masked by a `load` that exits 0 on a short stream.
|
|
load() {
|
|
( set -o pipefail
|
|
ssh "$BUILD_HOST" "docker save $1 | gzip -1" | ssh "$2" "gunzip | docker load" )
|
|
}
|
|
|
|
# Load only if the target lacks the exact image (skips re-transferring unchanged
|
|
# multi-hundred-MB agent images to every node on each code deploy).
|
|
#
|
|
# Verifies by image ID afterwards rather than trusting the exit status: a
|
|
# truncated stream can still leave a partially-populated image, and shipping a
|
|
# corrupt agent image to every fleet node is worse than failing the deploy.
|
|
# One retry, because the observed failure is a transient stream stall.
|
|
#
|
|
# Identity is the image's `Created` stamp, NOT its `Id`. A BuildKit image on
|
|
# the build host carries attestation manifests that `docker save | docker load`
|
|
# does not reproduce, so the same build legitimately arrives with a different
|
|
# Id and a different reported Size — comparing Ids fails every transfer of a
|
|
# correctly-shipped image. `Created` comes from the config blob, survives the
|
|
# round trip, and is what actually answers "is the new build here".
|
|
load_if_changed() {
|
|
local lts rts attempt
|
|
lts=$(ssh "$BUILD_HOST" "docker image inspect -f '{{.Created}}' $1 2>/dev/null" || true)
|
|
rts=$(ssh "$2" "docker image inspect -f '{{.Created}}' $1 2>/dev/null" || true)
|
|
if [ -n "$lts" ] && [ "$lts" = "$rts" ]; then
|
|
echo " (unchanged — skip)"
|
|
return 0
|
|
fi
|
|
for attempt in 1 2; do
|
|
load "$1" "$2" || echo " (transfer attempt $attempt failed)"
|
|
rts=$(ssh "$2" "docker image inspect -f '{{.Created}}' $1 2>/dev/null" || true)
|
|
if [ -n "$lts" ] && [ "$lts" = "$rts" ]; then
|
|
[ "$attempt" -gt 1 ] && echo " (recovered on attempt $attempt)"
|
|
return 0
|
|
fi
|
|
echo " (build stamp mismatch after attempt $attempt — retrying)" >&2
|
|
done
|
|
echo " ✗ $1 did not land on $2 (wanted $lts, got ${rts:-nothing})" >&2
|
|
return 1
|
|
}
|
|
|
|
echo "→ sync to $BUILD_HOST"
|
|
rsync -az --delete --exclude target/ --exclude node_modules/ --exclude .git/ \
|
|
--exclude '.next/' --exclude 'frontend/public/dl/' --exclude '**/.DS_Store' \
|
|
./ "$BUILD_HOST":~/clawmates/
|
|
|
|
if [ -z "${IMAGES_ONLY:-}" ]; then
|
|
echo "→ build + push server + frontend on $BUILD_HOST (registry $REGISTRY, sha $SHA)"
|
|
# Build, tag both :latest and the immutable :main-<sha>, and push to the
|
|
# registry the gateway pulls from. Pushing (not save|load) is what makes the
|
|
# deploy stick — see the header note.
|
|
ssh "$BUILD_HOST" 'set -e; cd ~/clawmates
|
|
export PATH=$HOME/.cargo/bin:$PATH CARGO_NET_GIT_FETCH_WITH_CLI=true SQLX_OFFLINE=true
|
|
cargo build --release -p clawmates-node
|
|
# rsync excludes frontend/public/dl/ (build artifacts), so the dir may not
|
|
# exist on the build host — create it before staging the daemon binary.
|
|
mkdir -p frontend/public/dl
|
|
cp target/release/clawmates-node frontend/public/dl/clawmates-node-linux-amd64
|
|
for svc in server frontend; do
|
|
docker build -f images/$svc.Dockerfile -t '"$REGISTRY"'/clawmates/$svc:'"$TAG"' \
|
|
-t '"$REGISTRY"'/clawmates/$svc:main-'"$SHA"' .
|
|
docker push '"$REGISTRY"'/clawmates/$svc:main-'"$SHA"'
|
|
docker push '"$REGISTRY"'/clawmates/$svc:'"$TAG"'
|
|
done'
|
|
fi
|
|
|
|
echo "→ build agent runtime images on $BUILD_HOST"
|
|
ssh "$BUILD_HOST" 'set -e; cd ~/clawmates
|
|
for i in '"${AGENT_IMAGES[*]}"'; do
|
|
docker build -f images/$i/Dockerfile -t clawmates/$i:dev images/$i/
|
|
done'
|
|
|
|
if [ -z "${IMAGES_ONLY:-}" ]; then
|
|
echo "→ repoint registry :latest → main-$SHA"
|
|
# gw-04 does NOT deploy from this script's push alone. A systemd timer
|
|
# (clawmates-deploy.timer, every 60s, /usr/local/bin/clawmates-deploy.sh)
|
|
# pulls `$REGISTRY/clawmates/<svc>:latest` and rolls the stack onto it
|
|
# whenever the running image differs. So ANY local `docker tag`/recreate on
|
|
# the gateway is reverted within a minute — the registry's `:latest` is the
|
|
# single source of truth for what prod runs.
|
|
#
|
|
# And `docker push …:latest` does NOT reliably move that tag here: when the
|
|
# manifest already exists in the registry under another tag (which it does,
|
|
# we just pushed main-$SHA), the push reports a digest but `:latest` keeps
|
|
# resolving to the old image. Writing the manifest to the tag directly over
|
|
# the HTTP API is what actually moves it. Verified: PUT → 201, and the
|
|
# timer then rolls prod on its own.
|
|
ssh "$BUILD_HOST" "set -e
|
|
for svc in server frontend; do
|
|
ct=\$(curl -s -o /tmp/cm-manifest.json -D- \
|
|
-H 'Accept: application/vnd.oci.image.index.v1+json,application/vnd.docker.distribution.manifest.list.v2+json,application/vnd.docker.distribution.manifest.v2+json,application/vnd.oci.image.manifest.v1+json' \
|
|
http://$REGISTRY/v2/clawmates/\$svc/manifests/main-$SHA \
|
|
| awk -F': ' '/^[Cc]ontent-[Tt]ype/{print \$2}' | tr -d '\r')
|
|
code=\$(curl -s -o /dev/null -w '%{http_code}' -X PUT \
|
|
-H \"Content-Type: \$ct\" --data-binary @/tmp/cm-manifest.json \
|
|
http://$REGISTRY/v2/clawmates/\$svc/manifests/latest)
|
|
echo \" \$svc :latest → main-$SHA (HTTP \$code)\"
|
|
case \"\$code\" in 20*) ;; *) echo \" ✗ tag write failed\"; exit 1 ;; esac
|
|
done"
|
|
|
|
echo "→ roll $GW onto main-$SHA"
|
|
# Roll immediately rather than waiting up to 60s for the timer. Snapshot the
|
|
# outgoing image as :rollback first so a revert is a repoint, not a rebuild.
|
|
#
|
|
# NOT fatal on its own, and that is the point. The 60s rolling timer
|
|
# (clawmates-deploy.timer) rolls the same stack onto the same `:latest`, so the
|
|
# two race — and the loser reports a container-name conflict ("already in use",
|
|
# "Renaming a container with the same name") for a container the winner has
|
|
# already recreated CORRECTLY. That happened twice in one afternoon, and a
|
|
# deploy signal an operator has to second-guess is exactly what this script
|
|
# exists to prevent: the original failure mode was a green edge on a stale
|
|
# image, and crying wolf trains people to ignore the alarm.
|
|
#
|
|
# So the roll is best-effort and the VERIFY below is the authority — it
|
|
# compares the RUNNING image id against the resolved `:latest`, which is the
|
|
# only question that matters and is unaffected by which process did the roll.
|
|
if ! ssh "$GW" "set -e
|
|
for svc in server frontend; do
|
|
docker tag $REGISTRY/clawmates/\$svc:$TAG $REGISTRY/clawmates/\$svc:rollback 2>/dev/null || true
|
|
docker pull -q $REGISTRY/clawmates/\$svc:$TAG >/dev/null
|
|
done
|
|
cd $GW_DIR
|
|
docker-compose -p clawmates up -d --no-deps server frontend"; then
|
|
echo " (the recreate reported an error — most often the 60s rolling timer"
|
|
echo " got there first. Verifying the running image rather than assuming"
|
|
echo " either outcome.)"
|
|
fi
|
|
fi
|
|
|
|
echo "→ load agent runtime images onto $GW + every fleet node"
|
|
for img in "${AGENT_IMAGES[@]}"; do
|
|
for host in "$GW" $NODES; do
|
|
printf ' %-26s → %s\n' "clawmates/$img:dev" "$host"
|
|
load_if_changed "clawmates/$img:dev" "$host"
|
|
done
|
|
done
|
|
|
|
if [ -z "${IMAGES_ONLY:-}" ]; then
|
|
echo "→ verify"
|
|
# Verify the RUNNING image is the one we just published — not just that the
|
|
# edge is up. A green edge on the OLD image is the silent-revert failure
|
|
# mode this check exists to catch. Compare against the resolved :latest,
|
|
# which is what both compose and the rolling timer deploy from.
|
|
want=$(ssh "$GW" "docker image inspect -f '{{.Id}}' $REGISTRY/clawmates/server:$TAG 2>/dev/null" || true)
|
|
got=$(ssh "$GW" "docker inspect -f '{{.Image}}' clawmates_server_1 2>/dev/null" || true)
|
|
if [ -n "$want" ] && [ "$want" = "$got" ]; then
|
|
echo " server running expected image ($SHA): ${got:7:12}"
|
|
else
|
|
echo " ✗ server image MISMATCH — running ${got:7:12}, expected ${want:7:12}"
|
|
echo " check that :latest was repointed and the roll succeeded on $GW"
|
|
exit 1
|
|
fi
|
|
ssh "$GW" 'curl -s -o /dev/null -w " edge HTTP %{http_code}\n" -m 10 https://clawmates.work/ || true'
|
|
fi
|
|
echo "✓ deploy complete"
|