fix(deploy): ship server/frontend via registry push, not save|load

The gateway compose pulls server/frontend from the web-01 registry
(100.94.185.103:5000, tags main-<sha> + :latest), so `docker save | docker
load` + a local retag does NOT stick — the next `docker-compose up` pulls
`:latest` and silently reverts to the last-pushed image (a green edge on the
old image hid this). Rewrite the server/frontend path to: build on the build
host → tag :latest + :main-<sha> → push to the registry → `compose pull +
up --force-recreate` in /opt/clawmates (the real project dir, not the stale
/root/clawmates) → verify the RUNNING image id equals the pushed one (fail
loudly on mismatch instead of trusting HTTP 200). Agent :dev images stay on
the save|load path (not in any registry).

Co-Authored-By: Claude Opus 4.8 <[email protected]>

scripts/deploy.sh

@@ -1,21 +1,35 @@
 #!/usr/bin/env bash
-# Deploy ClawMates: build the server, frontend, node daemon, AND the agent runtime
+# Deploy ClawMates.
-# images on the build host (tank), then load + recreate on the gateway. The agent
+#
-# images (agent-base / agent-browser / agent-terminal) are locally-built dev images
+# The server + frontend run on the gateway from images pulled from the web-01
-# — not in any registry — so a `docker pull` can't fetch them. This script loads
+# registry (100.94.185.103:5000) — /opt/clawmates/docker-compose.yml references
-# them onto the gateway AND every fleet node, so agent/terminal containers never
+# `$REGISTRY/clawmates/{server,frontend}:latest`, and `docker-compose up` PULLS
-# 404 with "No such image" (the failure mode this script exists to prevent).
+# them. So the ONLY reliable way to ship server/frontend is: build on the build
+# host → push to the registry → pull + recreate on the gateway. A `docker save |
+# docker load` + local retag does NOT work here — the next `up` pulls `:latest`
+# from the registry and silently reverts to whatever was last pushed. (That is
+# exactly the footgun this rewrite removes; the old save|load path is gone.)
+#
+# The agent runtime images (agent-base / agent-browser / agent-terminal) are
+# locally-built dev images NOT in any registry, so those still ship via
+# `docker save | docker load` to the gateway AND every fleet node (a `docker
+# pull` can't fetch them; the load prevents "No such image" 404s).
 #
 # Usage: scripts/deploy.sh                 # full deploy
 #        IMAGES_ONLY=1 scripts/deploy.sh   # just (re)build + load the agent images
 #
-# Override hosts via env: BUILD_HOST, GW, NODES, TAG.
+# Override via env: BUILD_HOST, GW, GW_DIR, NODES, TAG, REGISTRY.
 set -euo pipefail
 
-BUILD_HOST=${BUILD_HOST:-osobh@tank}     # builds the images + the linux daemon
+BUILD_HOST=${BUILD_HOST:-osobh@tank}     # builds + pushes the server/frontend images
 GW=${GW:-gw-04}                          # the gateway (server + frontend + postgres)
+GW_DIR=${GW_DIR:-/opt/clawmates}         # the compose project dir ON the gateway
+REGISTRY=${REGISTRY:-100.94.185.103:5000}   # web-01 registry the compose pulls from
 NODES=${NODES:-"morpheus osobh@tank architect"}    # fleet nodes that provision agent containers
 TAG=${TAG:-latest}
+# Immutable per-deploy tag so a deploy is traceable and rollback is a repoint,
+# not a rebuild. Falls back to a timestamp-free literal when not in a git tree.
+SHA=$(git rev-parse --short HEAD 2>/dev/null || echo manual)
 AGENT_IMAGES=(agent-base agent-browser agent-terminal)
 
 load() { ssh "$BUILD_HOST" "docker save $1" | ssh "$2" "docker load"; }
@@ -39,13 +53,20 @@ rsync -az --delete --exclude target/ --exclude node_modules/ --exclude .git/ \
   ./ "$BUILD_HOST":~/clawmates/
 
 if [ -z "${IMAGES_ONLY:-}" ]; then
-  echo "→ build server + frontend + daemon on $BUILD_HOST"
+  echo "→ build + push server + frontend on $BUILD_HOST (registry $REGISTRY, sha $SHA)"
+  # Build, tag both :latest and the immutable :main-<sha>, and push to the
+  # registry the gateway pulls from. Pushing (not save|load) is what makes the
+  # deploy stick — see the header note.
   ssh "$BUILD_HOST" 'set -e; cd ~/clawmates
     export PATH=$HOME/.cargo/bin:$PATH CARGO_NET_GIT_FETCH_WITH_CLI=true SQLX_OFFLINE=true
     cargo build --release -p clawmates-node
     cp target/release/clawmates-node frontend/public/dl/clawmates-node-linux-amd64
-    docker build -f images/server.Dockerfile   -t clawmates/server:'"$TAG"'   .
+    for svc in server frontend; do
-    docker build -f images/frontend.Dockerfile -t clawmates/frontend:'"$TAG"' .'
+      docker build -f images/$svc.Dockerfile -t '"$REGISTRY"'/clawmates/$svc:'"$TAG"' \
+                                              -t '"$REGISTRY"'/clawmates/$svc:main-'"$SHA"' .
+      docker push '"$REGISTRY"'/clawmates/$svc:main-'"$SHA"'
+      docker push '"$REGISTRY"'/clawmates/$svc:'"$TAG"'
+    done'
 fi
 
 echo "→ build agent runtime images on $BUILD_HOST"
@@ -55,11 +76,17 @@ ssh "$BUILD_HOST" 'set -e; cd ~/clawmates
   done'
 
 if [ -z "${IMAGES_ONLY:-}" ]; then
-  echo "→ load + recreate server + frontend on $GW"
+  echo "→ pull + recreate server + frontend on $GW ($GW_DIR)"
-  ssh "$GW" "docker tag clawmates/server:$TAG clawmates/server:rollback 2>/dev/null || true
+  # Snapshot the currently-deployed images as :rollback (a repoint, cheap) so a
-             docker tag clawmates/frontend:$TAG clawmates/frontend:rollback 2>/dev/null || true"
+  # bad deploy can be reverted without a rebuild, then pull the freshly-pushed
-  load "clawmates/server:$TAG"   "$GW"
+  # images and recreate.
-  load "clawmates/frontend:$TAG" "$GW"
+  ssh "$GW" "set -e
+    for svc in server frontend; do
+      docker tag $REGISTRY/clawmates/\$svc:$TAG $REGISTRY/clawmates/\$svc:rollback 2>/dev/null || true
+    done
+    cd $GW_DIR
+    docker-compose -p clawmates pull server frontend
+    docker-compose -p clawmates up -d --force-recreate server frontend"
 fi
 
 echo "→ load agent runtime images onto $GW + every fleet node"
@@ -71,9 +98,19 @@ for img in "${AGENT_IMAGES[@]}"; do
 done
 
 if [ -z "${IMAGES_ONLY:-}" ]; then
-  echo "→ recreate server + frontend"
+  echo "→ verify"
-  ssh "$GW" "cd /root/clawmates && docker-compose -p clawmates up -d --force-recreate server frontend"
+  # Verify the RUNNING image matches what we just pushed — not just that the
-  sleep 6
+  # edge is up. A green edge on the OLD image is the silent-revert failure mode
-  curl -s -o /dev/null -w "edge HTTP %{http_code}\n" -m 10 https://clawmates.work/ || true
+  # this check exists to catch.
+  want=$(ssh "$GW" "docker image inspect -f '{{.Id}}' $REGISTRY/clawmates/server:main-$SHA 2>/dev/null" || true)
+  got=$(ssh "$GW" "docker inspect -f '{{.Image}}' clawmates_server_1 2>/dev/null" || true)
+  if [ -n "$want" ] && [ "$want" = "$got" ]; then
+    echo "   server running expected image ($SHA): ${got:7:12}"
+  else
+    echo "   ✗ server image MISMATCH — running ${got:7:12}, expected main-$SHA (${want:7:12})"
+    echo "     the deploy did NOT take effect; check the registry pull on $GW"
+    exit 1
+  fi
+  ssh "$GW" 'curl -s -o /dev/null -w "   edge HTTP %{http_code}\n" -m 10 https://clawmates.work/ || true'
 fi
 echo "✓ deploy complete"