#!/usr/bin/env bash
# Deploy ClawMates.
#
# The server + frontend run on the gateway from images pulled from the web-01
# registry (100.94.185.103:5000) — /opt/clawmates/docker-compose.yml references
# `$REGISTRY/clawmates/{server,frontend}:latest`, and `docker-compose up` PULLS
# them. So the ONLY reliable way to ship server/frontend is: build on the build
# host → push to the registry → pull + recreate on the gateway. A `docker save |
# docker load` + local retag does NOT work here — the next `up` pulls `:latest`
# from the registry and silently reverts to whatever was last pushed. (That is
# exactly the footgun this rewrite removes; the old save|load path is gone.)
#
# The agent runtime images (agent-base / agent-browser / agent-terminal) are
# locally-built dev images NOT in any registry, so those still ship via
# `docker save | docker load` to the gateway AND every fleet node (a `docker
# pull` can't fetch them; the load prevents "No such image" 404s).
#
# Usage: scripts/deploy.sh                 # full deploy
#        IMAGES_ONLY=1 scripts/deploy.sh   # just (re)build + load the agent images
#
# Override via env: BUILD_HOST, GW, GW_DIR, NODES, TAG, REGISTRY.
set -euo pipefail

BUILD_HOST=${BUILD_HOST:-osobh@tank}     # builds + pushes the server/frontend images
GW=${GW:-gw-04}                          # the gateway (server + frontend + postgres)
GW_DIR=${GW_DIR:-/opt/clawmates}         # the compose project dir ON the gateway
REGISTRY=${REGISTRY:-100.94.185.103:5000}   # web-01 registry the compose pulls from
# Fleet nodes that provision agent containers.
#
# morpheus was removed 2026-08-11: it is drained in the `nodes` table (which
# heartbeats preserve, so it stays out of placement when it comes back) and
# packed for a move. It is listed here as a comment rather than deleted because
# putting it back is one word, and the next person will want to know where.
#   was: "morpheus osobh@tank architect"
# An unreachable host here does not just skip — the image loop fails the whole
# script BEFORE its verify stage, so four deploys in a row rolled correctly and
# reported nothing. Keep this list to hosts that answer.
NODES=${NODES:-"osobh@tank architect"}
TAG=${TAG:-latest}
# Immutable per-deploy tag so a deploy is traceable and rollback is a repoint,
# not a rebuild. Falls back to a timestamp-free literal when not in a git tree.
SHA=$(git rev-parse --short HEAD 2>/dev/null || echo manual)
AGENT_IMAGES=(agent-base agent-browser agent-terminal)

# Stream an image between two remote hosts. The tar crosses two SSH
# connections spliced through this workstation, so a stall on either side
# truncates it — that surfaces as `unexpected EOF` from `docker load`, which
# is a genuine failure and was previously indistinguishable from success
# because nothing checked afterwards. Compress (these images are mostly
# filesystem, and less bytes is less exposure to a stall) and set pipefail so
# a failed `save` cannot be masked by a `load` that exits 0 on a short stream.
load() {
  ( set -o pipefail
    ssh "$BUILD_HOST" "docker save $1 | gzip -1" | ssh "$2" "gunzip | docker load" )
}

# Load only if the target lacks the exact image (skips re-transferring unchanged
# multi-hundred-MB agent images to every node on each code deploy).
#
# Verifies by image ID afterwards rather than trusting the exit status: a
# truncated stream can still leave a partially-populated image, and shipping a
# corrupt agent image to every fleet node is worse than failing the deploy.
# One retry, because the observed failure is a transient stream stall.
#
# Identity is the image's `Created` stamp, NOT its `Id`. A BuildKit image on
# the build host carries attestation manifests that `docker save | docker load`
# does not reproduce, so the same build legitimately arrives with a different
# Id and a different reported Size — comparing Ids fails every transfer of a
# correctly-shipped image. `Created` comes from the config blob, survives the
# round trip, and is what actually answers "is the new build here".
load_if_changed() {
  local lts rts attempt
  lts=$(ssh "$BUILD_HOST" "docker image inspect -f '{{.Created}}' $1 2>/dev/null" || true)
  rts=$(ssh "$2" "docker image inspect -f '{{.Created}}' $1 2>/dev/null" || true)
  if [ -n "$lts" ] && [ "$lts" = "$rts" ]; then
    echo "      (unchanged — skip)"
    return 0
  fi
  for attempt in 1 2; do
    load "$1" "$2" || echo "      (transfer attempt $attempt failed)"
    rts=$(ssh "$2" "docker image inspect -f '{{.Created}}' $1 2>/dev/null" || true)
    if [ -n "$lts" ] && [ "$lts" = "$rts" ]; then
      [ "$attempt" -gt 1 ] && echo "      (recovered on attempt $attempt)"
      return 0
    fi
    echo "      (build stamp mismatch after attempt $attempt — retrying)" >&2
  done
  echo "   ✗ $1 did not land on $2 (wanted $lts, got ${rts:-nothing})" >&2
  return 1
}

echo "→ sync to $BUILD_HOST"
rsync -az --delete --exclude target/ --exclude node_modules/ --exclude .git/ \
  --exclude '.next/' --exclude 'frontend/public/dl/' --exclude '**/.DS_Store' \
  ./ "$BUILD_HOST":~/clawmates/

if [ -z "${IMAGES_ONLY:-}" ]; then
  echo "→ build + push server + frontend on $BUILD_HOST (registry $REGISTRY, sha $SHA)"
  # Build, tag both :latest and the immutable :main-<sha>, and push to the
  # registry the gateway pulls from. Pushing (not save|load) is what makes the
  # deploy stick — see the header note.
  ssh "$BUILD_HOST" 'set -e; cd ~/clawmates
    export PATH=$HOME/.cargo/bin:$PATH CARGO_NET_GIT_FETCH_WITH_CLI=true SQLX_OFFLINE=true
    cargo build --release -p clawmates-node
    # rsync excludes frontend/public/dl/ (build artifacts), so the dir may not
    # exist on the build host — create it before staging the daemon binary.
    mkdir -p frontend/public/dl
    cp target/release/clawmates-node frontend/public/dl/clawmates-node-linux-amd64
    for svc in server frontend; do
      docker build -f images/$svc.Dockerfile -t '"$REGISTRY"'/clawmates/$svc:'"$TAG"' \
                                              -t '"$REGISTRY"'/clawmates/$svc:main-'"$SHA"' .
      docker push '"$REGISTRY"'/clawmates/$svc:main-'"$SHA"'
      docker push '"$REGISTRY"'/clawmates/$svc:'"$TAG"'
    done'
fi

echo "→ build agent runtime images on $BUILD_HOST"
ssh "$BUILD_HOST" 'set -e; cd ~/clawmates
  for i in '"${AGENT_IMAGES[*]}"'; do
    docker build -f images/$i/Dockerfile -t clawmates/$i:dev images/$i/
  done'

if [ -z "${IMAGES_ONLY:-}" ]; then
  echo "→ repoint registry :latest → main-$SHA"
  # gw-04 does NOT deploy from this script's push alone. A systemd timer
  # (clawmates-deploy.timer, every 60s, /usr/local/bin/clawmates-deploy.sh)
  # pulls `$REGISTRY/clawmates/<svc>:latest` and rolls the stack onto it
  # whenever the running image differs. So ANY local `docker tag`/recreate on
  # the gateway is reverted within a minute — the registry's `:latest` is the
  # single source of truth for what prod runs.
  #
  # And `docker push …:latest` does NOT reliably move that tag here: when the
  # manifest already exists in the registry under another tag (which it does,
  # we just pushed main-$SHA), the push reports a digest but `:latest` keeps
  # resolving to the old image. Writing the manifest to the tag directly over
  # the HTTP API is what actually moves it. Verified: PUT → 201, and the
  # timer then rolls prod on its own.
  ssh "$BUILD_HOST" "set -e
    for svc in server frontend; do
      ct=\$(curl -s -o /tmp/cm-manifest.json -D- \
        -H 'Accept: application/vnd.oci.image.index.v1+json,application/vnd.docker.distribution.manifest.list.v2+json,application/vnd.docker.distribution.manifest.v2+json,application/vnd.oci.image.manifest.v1+json' \
        http://$REGISTRY/v2/clawmates/\$svc/manifests/main-$SHA \
        | awk -F': ' '/^[Cc]ontent-[Tt]ype/{print \$2}' | tr -d '\r')
      code=\$(curl -s -o /dev/null -w '%{http_code}' -X PUT \
        -H \"Content-Type: \$ct\" --data-binary @/tmp/cm-manifest.json \
        http://$REGISTRY/v2/clawmates/\$svc/manifests/latest)
      echo \"   \$svc :latest → main-$SHA (HTTP \$code)\"
      case \"\$code\" in 20*) ;; *) echo \"   ✗ tag write failed\"; exit 1 ;; esac
    done"

  echo "→ roll $GW onto main-$SHA"
  # Roll immediately rather than waiting up to 60s for the timer. Snapshot the
  # outgoing image as :rollback first so a revert is a repoint, not a rebuild.
  #
  # NOT fatal on its own, and that is the point. The 60s rolling timer
  # (clawmates-deploy.timer) rolls the same stack onto the same `:latest`, so the
  # two race — and the loser reports a container-name conflict ("already in use",
  # "Renaming a container with the same name") for a container the winner has
  # already recreated CORRECTLY. That happened twice in one afternoon, and a
  # deploy signal an operator has to second-guess is exactly what this script
  # exists to prevent: the original failure mode was a green edge on a stale
  # image, and crying wolf trains people to ignore the alarm.
  #
  # So the roll is best-effort and the VERIFY below is the authority — it
  # compares the RUNNING image id against the resolved `:latest`, which is the
  # only question that matters and is unaffected by which process did the roll.
  if ! ssh "$GW" "set -e
    for svc in server frontend; do
      docker tag $REGISTRY/clawmates/\$svc:$TAG $REGISTRY/clawmates/\$svc:rollback 2>/dev/null || true
      docker pull -q $REGISTRY/clawmates/\$svc:$TAG >/dev/null
    done
    cd $GW_DIR
    docker-compose -p clawmates up -d --no-deps server frontend"; then
    echo "   (the recreate reported an error — most often the 60s rolling timer"
    echo "    got there first. Verifying the running image rather than assuming"
    echo "    either outcome.)"
  fi
fi

echo "→ load agent runtime images onto $GW + every fleet node"
for img in "${AGENT_IMAGES[@]}"; do
  for host in "$GW" $NODES; do
    printf '   %-26s → %s\n' "clawmates/$img:dev" "$host"
    load_if_changed "clawmates/$img:dev" "$host"
  done
done

if [ -z "${IMAGES_ONLY:-}" ]; then
  echo "→ verify"
  # Verify the RUNNING image is the one we just published — not just that the
  # edge is up. A green edge on the OLD image is the silent-revert failure
  # mode this check exists to catch. Compare against the resolved :latest,
  # which is what both compose and the rolling timer deploy from.
  want=$(ssh "$GW" "docker image inspect -f '{{.Id}}' $REGISTRY/clawmates/server:$TAG 2>/dev/null" || true)
  got=$(ssh "$GW" "docker inspect -f '{{.Image}}' clawmates_server_1 2>/dev/null" || true)
  if [ -n "$want" ] && [ "$want" = "$got" ]; then
    echo "   server running expected image ($SHA): ${got:7:12}"
  else
    echo "   ✗ server image MISMATCH — running ${got:7:12}, expected ${want:7:12}"
    echo "     check that :latest was repointed and the roll succeeded on $GW"
    exit 1
  fi
  ssh "$GW" 'curl -s -o /dev/null -w "   edge HTTP %{http_code}\n" -m 10 https://clawmates.work/ || true'
fi
echo "✓ deploy complete"