fix(deploy): verify agent images landed instead of trusting the pipe
`docker save | docker load` across two SSH connections spliced through a workstation truncates when either side stalls — observed as `unexpected EOF` mid-deploy. Nothing checked afterwards, and `docker load` can exit 0 on a short stream, so a partially-populated image could ship to every fleet node and look like a success. Now compressed, pipefail-guarded, and verified by comparing image IDs on the target after the transfer, with one retry for the transient stall. A failed transfer fails the deploy rather than passing quietly. The runtime image no longer travels this path at all: it is registry- hosted now (100.94.185.103:5000/clawmates-runtime:v083-toolchain), built from deploy/clawmates-runtime/Dockerfile on tank. Only agent-base / agent-browser / agent-terminal still need save|load, because they exist in no registry. Co-Authored-By: Claude Opus 5 <[email protected]>
This commit is contained in:
co-authored by
Claude Opus 5
parent
f7e336ff5f
commit
bb34ef1b7e
+28
-3
@@ -32,19 +32,44 @@ TAG=${TAG:-latest}
|
|||||||
SHA=$(git rev-parse --short HEAD 2>/dev/null || echo manual)
|
SHA=$(git rev-parse --short HEAD 2>/dev/null || echo manual)
|
||||||
AGENT_IMAGES=(agent-base agent-browser agent-terminal)
|
AGENT_IMAGES=(agent-base agent-browser agent-terminal)
|
||||||
|
|
||||||
load() { ssh "$BUILD_HOST" "docker save $1" | ssh "$2" "docker load"; }
|
# Stream an image between two remote hosts. The tar crosses two SSH
|
||||||
|
# connections spliced through this workstation, so a stall on either side
|
||||||
|
# truncates it — that surfaces as `unexpected EOF` from `docker load`, which
|
||||||
|
# is a genuine failure and was previously indistinguishable from success
|
||||||
|
# because nothing checked afterwards. Compress (these images are mostly
|
||||||
|
# filesystem, and less bytes is less exposure to a stall) and set pipefail so
|
||||||
|
# a failed `save` cannot be masked by a `load` that exits 0 on a short stream.
|
||||||
|
load() {
|
||||||
|
( set -o pipefail
|
||||||
|
ssh "$BUILD_HOST" "docker save $1 | gzip -1" | ssh "$2" "gunzip | docker load" )
|
||||||
|
}
|
||||||
|
|
||||||
# Load only if the target lacks the exact image (skips re-transferring unchanged
|
# Load only if the target lacks the exact image (skips re-transferring unchanged
|
||||||
# multi-hundred-MB agent images to every node on each code deploy).
|
# multi-hundred-MB agent images to every node on each code deploy).
|
||||||
|
#
|
||||||
|
# Verifies by image ID afterwards rather than trusting the exit status: a
|
||||||
|
# truncated stream can still leave a partially-populated image, and shipping a
|
||||||
|
# corrupt agent image to every fleet node is worse than failing the deploy.
|
||||||
|
# One retry, because the observed failure is a transient stream stall.
|
||||||
load_if_changed() {
|
load_if_changed() {
|
||||||
local lid rid
|
local lid rid attempt
|
||||||
lid=$(ssh "$BUILD_HOST" "docker image inspect -f '{{.Id}}' $1 2>/dev/null" || true)
|
lid=$(ssh "$BUILD_HOST" "docker image inspect -f '{{.Id}}' $1 2>/dev/null" || true)
|
||||||
rid=$(ssh "$2" "docker image inspect -f '{{.Id}}' $1 2>/dev/null" || true)
|
rid=$(ssh "$2" "docker image inspect -f '{{.Id}}' $1 2>/dev/null" || true)
|
||||||
if [ -n "$lid" ] && [ "$lid" = "$rid" ]; then
|
if [ -n "$lid" ] && [ "$lid" = "$rid" ]; then
|
||||||
echo " (unchanged — skip)"
|
echo " (unchanged — skip)"
|
||||||
return 0
|
return 0
|
||||||
fi
|
fi
|
||||||
load "$1" "$2"
|
for attempt in 1 2; do
|
||||||
|
load "$1" "$2" || echo " (transfer attempt $attempt failed)"
|
||||||
|
rid=$(ssh "$2" "docker image inspect -f '{{.Id}}' $1 2>/dev/null" || true)
|
||||||
|
if [ -n "$lid" ] && [ "$lid" = "$rid" ]; then
|
||||||
|
[ "$attempt" -gt 1 ] && echo " (recovered on attempt $attempt)"
|
||||||
|
return 0
|
||||||
|
fi
|
||||||
|
echo " (image ID mismatch after attempt $attempt — retrying)" >&2
|
||||||
|
done
|
||||||
|
echo " ✗ $1 did not land on $2 (wanted ${lid:0:19}, got ${rid:0:19})" >&2
|
||||||
|
return 1
|
||||||
}
|
}
|
||||||
|
|
||||||
echo "→ sync to $BUILD_HOST"
|
echo "→ sync to $BUILD_HOST"
|
||||||
|
|||||||
Reference in New Issue
Block a user