# Local → production pipeline.
#
#   push to main → test → build amd64 images → push to the fleet registry
#   → move :latest → gw-04's existing 60s rolling timer picks it up.
#
# The last hop is NOT in this file and does not need to be: gw-04 already runs
# `clawmates-deploy.timer` every minute, which pulls
# `$REGISTRY/clawmates/<svc>:latest`, compares it to the running image id, and
# recreates on drift. This workflow's job is to make `:latest` mean the newest
# green commit. See deploy/gw-04/clawmates-deploy.sh.
#
# Runs on the `gw04` runner (host executor, systemd unit act-runner). gw-04 is
# the only reachable x86_64 host — web-01 is aarch64 and the fleet build boxes
# are packed — and prod images must be linux/amd64, so builds are native here
# rather than emulated.
name: deploy

on:
  push:
    branches: [main]
  # Lets you re-run a deploy without an empty commit.
  workflow_dispatch:

# Two pushes close together used to STOMP each other. Runs 490 and 491 started
# 16 minutes apart, a full suite takes longer than that, and the first thing a
# run does is `docker rm -fv` the shared test Postgres — so the newer run
# deleted the older run's database mid-suite and both failed. Nothing in the
# code was wrong; the logs blamed the tests.
#
# `cancel-in-progress` because a superseded run is testing a commit that is no
# longer the tip: finishing it costs 20 minutes to learn something that no
# longer matters.
concurrency:
  group: deploy-${{ gitea.ref }}
  cancel-in-progress: true

env:
  REGISTRY: 100.94.185.103:5000
  NAMESPACE: clawmates

jobs:
  test:
    runs-on: gw04
    env:
      # Shared by the start and stop steps.
      PG: cm-ci-pg-${{ gitea.run_id }}
    steps:
      - uses: actions/checkout@v4

      # A throwaway Postgres so the integration tests actually run. Without
      # CM_TEST_DATABASE_URL, cm-testkit tries a default admin URL and the
      # approvals_api tests die on PoolTimedOut — which looks like a failure but
      # only means "no database here".
      - name: Start test Postgres
        run: |
          # Where every step leaves its full output, on the HOST, so a failed
          # run can be read afterwards without the actions-log API.
          #
          # STEP is a breadcrumb: each step overwrites it on entry, so the last
          # value names the step that died. Two steps used to create this
          # directory, which made "the directory exists" ambiguous about how far
          # the job got — and that ambiguity cost a whole debugging cycle.
          mkdir -p /tmp/ci-logs && rm -f /tmp/ci-logs/*.log /tmp/ci-logs/STEP
          echo "1-start-postgres" > /tmp/ci-logs/STEP
          set -x
          # Run-scoped name. `cm-ci-pg` was shared by every run, so a second
          # run removed the first one's database while it was still being used.
          # The concurrency group above should prevent overlap; this makes the
          # failure impossible rather than merely unlikely.
          docker rm -fv "$PG" 2>/dev/null || true
          # --shm-size: Docker defaults /dev/shm to 64MB. cm-testkit creates a
          # database per test and the suite runs many at once, so Postgres
          # exhausts its parallel-query segments mid-run. It surfaces as
          # `could not resize shared memory segment ... No space left on device`
          # during MIGRATIONS, which reads like a schema fault and is not one.
          # Hit locally on 2026-08-19; scripts/test-server.sh carries the same
          # flag for the same reason.
          docker run -d --name "$PG" \
            --shm-size=1g \
            -e POSTGRES_PASSWORD=postgres -e POSTGRES_USER=postgres \
            -p 127.0.0.1:55432:5432 postgres:16-alpine
          for i in $(seq 1 30); do
            docker exec "$PG" pg_isready -U postgres >/dev/null 2>&1 && break
            sleep 2
          done
          docker exec "$PG" pg_isready -U postgres

      # Rust lives in a container because gw-04 has no cargo. The named volumes
      # are the whole reason this is not painfully slow: without them every run
      # recompiles the world.
      # Docker socket AND the host's docker binary are mounted:
      #   - cm-files' s3_store test uses testcontainers (socket only).
      #   - cm-runtime/cm-sandbox tests (browser_tool, shell_exec, warm_pool,
      #     security, socket_proxy) shell out to `docker` via std::process, so
      #     they need the CLI on PATH too. Mounting the host binary beats
      #     apt-installing docker.io on every run — that is ~100 MB of download
      #     per job, and the container is fresh each time so nothing caches it.
      # These tests do NOT skip when the capability is missing; they fail in a
      # way that reads like broken code (SocketNotFoundError / NotFound), which
      # is why they are worth wiring up rather than excluding.
      #
      # They also need clawmates/agent-{base,browser,terminal}:dev, which are
      # locally-built images present on gw-04 but in no registry. If this job
      # ever moves hosts, those images must move with it.
      #
      # `cargo test --workspace` builds cm-brain, which pulls clawhdf5 from
      # git.redclaw.dev — a PRIVATE repo. Two things are needed and neither is
      # optional:
      #   CARGO_NET_GIT_FETCH_WITH_CLI — libgit2 fails against Gitea's smart-HTTP
      #     with "invalid packet line" (the server Dockerfile sets it for the
      #     same reason). Note it is _GIT_FETCH_WITH_CLI, not _NET_FETCH_.
      #   the insteadOf rewrite — supplies the credential to that CLI fetch.
      # The token is a repo secret, so it is masked in logs and never in git.
      - name: Rust tests
        run: |
          echo "2-rust" > /tmp/ci-logs/STEP
          # The DOCKER RUN's own output, on the host. cargo's log only exists
          # if cargo runs; run 494 died in this step with no rust.log at all,
          # which means apt-get, git config or docker itself failed and the
          # message went only to the job log we cannot read.
          set +e
          docker run --rm --network host \
            -v "$PWD":/w -w /w \
            -v cm-ci-cargo-registry:/usr/local/cargo/registry \
            -v cm-ci-cargo-git:/usr/local/cargo/git \
            -v cm-ci-target:/w/target \
            -v /var/run/docker.sock:/var/run/docker.sock \
            -v /usr/bin/docker:/usr/bin/docker:ro \
            -e SQLX_OFFLINE=true \
            -e CARGO_NET_GIT_FETCH_WITH_CLI=true \
            -e FORGE_TOKEN='${{ secrets.FORGE_TOKEN }}' \
            -e CM_TEST_DATABASE_URL=postgres://postgres:postgres@127.0.0.1:55432/postgres \
            -v /tmp/ci-logs:/cilog \
            rust:1.96-slim \
            sh -c 'set -e
              # NO APOSTROPHES BELOW THIS LINE. Everything here is inside a
              # single-quoted sh -c, so one apostrophe in a COMMENT closes the
              # quote and the step dies with "unexpected EOF while looking for
              # matching quote" — before running anything, which is why no log
              # ever appeared. Runs 491 through 496 failed on the word
              # "cm-api" followed by an apostrophe-s.
              apt-get update -qq
              # nodejs: the vm_tool_gate shell tests in cm-api EXECUTE the generated
              # PreToolUse hook, which parses its JSON payload with node (no jq
              # in the runtime image; node is guaranteed there because Claude
              # Code is a node program). Without it the hook takes its
              # allow-and-record-inert path and the two "blocks" tests fail —
              # which is how this was found, on the first push that carried them.
              apt-get install -y -qq pkg-config libssl-dev cmake git nodejs >/dev/null
              git config --global url."https://oauth2:$FORGE_TOKEN@git.redclaw.dev/".insteadOf "https://git.redclaw.dev/"
              # Full output to a host-mounted file, then the tail, then exit
              # with the cargo status. Piping cargo into `tail` would report
              # the exit code of tail — a green job over a red suite. The log
              # survives the container so a failure is diagnosable at all:
              # the Gitea actions-log API returns 403 for our token, and three
              # failed runs were debugged blind before this existed.
              set +e
              cargo test --workspace > /cilog/rust.log 2>&1
              rc=$?
              set -e
              grep -nE "test result: FAILED|^error(\[|:)|panicked at" /cilog/rust.log | head -40 || true
              tail -40 /cilog/rust.log
              exit $rc' > /tmp/ci-logs/rust-step.log 2>&1
          rc=$?
          set -e
          tail -60 /tmp/ci-logs/rust-step.log
          exit $rc

      # -v, not just -f. The postgres image declares a VOLUME, so removing the
      # container without it orphans an anonymous data directory EVERY run.
      # cm-testkit creates a database per test, so those grew to 2.8 GB each —
      # 38 GB of leaked volumes before anyone noticed.
      - name: Stop test Postgres
        if: always()
        run: |
          echo "3-stop-postgres" >> /tmp/ci-logs/STEP
          docker rm -fv "$PG" 2>/dev/null || true

      # node 22 is on the host, so these run directly.
      - name: Frontend checks
        working-directory: frontend
        run: |
          echo "4-frontend" >> /tmp/ci-logs/STEP
          set +e
          npm ci --no-audit --no-fund > /tmp/ci-logs/npm-ci.log 2>&1;   ci=$?
          npm run typecheck                > /tmp/ci-logs/typecheck.log 2>&1; tc=$?
          npm run test                     > /tmp/ci-logs/vitest.log 2>&1;    vt=$?
          set -e
          for f in npm-ci typecheck vitest; do
            printf '=== %s ===\n' "$f"; tail -25 "/tmp/ci-logs/$f.log" || true
          done
          [ "$ci" -eq 0 ] && [ "$tc" -eq 0 ] && [ "$vt" -eq 0 ]
          # Lint is advisory: the repo currently has pre-existing max-lines and
          # set-state-in-effect errors that predate this pipeline. Failing the
          # deploy on them would mean nothing could ship until they are cleared.
          npm run lint || echo "::warning::lint reported problems (advisory)"

  build:
    runs-on: gw04
    needs: test
    steps:
      - uses: actions/checkout@v4

      - name: Build + push images
        run: |
          # Same host-log treatment as the test job. The build job failed four
          # runs in a row with nothing readable: the actions-log API returns
          # 403 for our token, so "failure" was the entire message. It turned
          # out to be transient disk pressure — a runtime image being built on
          # this same host at the same time — and a docs-only commit was the
          # first casualty, which made it look like a code regression.
          mkdir -p /tmp/ci-logs
          echo "5-build" > /tmp/ci-logs/STEP
          df -h / > /tmp/ci-logs/build-disk.log 2>&1
          set -eu
          SHA=$(git rev-parse --short HEAD)
          echo "SHA=$SHA" >> "$GITHUB_ENV"
          # The daemon binary the frontend serves at /dl. images/frontend.Dockerfile
          # expects it staged; rsync-based deploys create it out of band, so build
          # it here or the image ships without the node installer.
          mkdir -p frontend/public/dl
          docker run --rm \
            -v "$PWD":/w -w /w \
            -v cm-ci-cargo-registry:/usr/local/cargo/registry \
            -v cm-ci-cargo-git:/usr/local/cargo/git \
            -v cm-ci-target:/w/target \
            -e SQLX_OFFLINE=true -e CARGO_NET_GIT_FETCH_WITH_CLI=true \
            -e FORGE_TOKEN='${{ secrets.FORGE_TOKEN }}' \
            rust:1.96-slim \
            sh -c 'set -e
              apt-get update -qq
              apt-get install -y -qq pkg-config libssl-dev cmake git >/dev/null
              git config --global url."https://oauth2:$FORGE_TOKEN@git.redclaw.dev/".insteadOf "https://git.redclaw.dev/"
              cargo build --release -p clawmates-node
              cp target/release/clawmates-node frontend/public/dl/clawmates-node-linux-amd64'

          for svc in server frontend broker; do
            docker build -f "images/$svc.Dockerfile" \
              -t "$REGISTRY/$NAMESPACE/$svc:main-$SHA" \
              -t "$REGISTRY/$NAMESPACE/$svc:latest" .
            docker push "$REGISTRY/$NAMESPACE/$svc:main-$SHA"
            docker push "$REGISTRY/$NAMESPACE/$svc:latest"
            echo "$svc built+pushed" >> /tmp/ci-logs/build-progress.log
          done

      # `docker push :latest` does NOT reliably move the tag on this registry:
      # when the manifest already exists under another tag (it does — we just
      # pushed main-$SHA), the push reports a digest but `:latest` keeps
      # resolving to the OLD image. Writing the manifest to the tag over the
      # HTTP API is what actually moves it. This is the same trick
      # scripts/deploy.sh uses, and the reason a "successful" deploy could
      # previously leave prod on a stale image.
      - name: Repoint :latest
        run: |
          echo "6-repoint" > /tmp/ci-logs/STEP
          set -eu
          for svc in server frontend broker; do
            ct=$(curl -s -o /tmp/m.json -D- \
              -H 'Accept: application/vnd.oci.image.index.v1+json,application/vnd.docker.distribution.manifest.list.v2+json,application/vnd.docker.distribution.manifest.v2+json,application/vnd.oci.image.manifest.v1+json' \
              "http://$REGISTRY/v2/$NAMESPACE/$svc/manifests/main-$SHA" \
              | awk -F': ' '/^[Cc]ontent-[Tt]ype/{print $2}' | tr -d '\r')
            code=$(curl -s -o /dev/null -w '%{http_code}' -X PUT \
              -H "Content-Type: $ct" --data-binary @/tmp/m.json \
              "http://$REGISTRY/v2/$NAMESPACE/$svc/manifests/latest")
            echo "$svc :latest → main-$SHA (HTTP $code)"
            case "$code" in 20*) ;; *) echo "tag write failed"; exit 1 ;; esac
          done

      # Verify the thing that actually matters: what prod is RUNNING, not what
      # we pushed. A green edge on a stale image is the failure mode this whole
      # pipeline exists to prevent.
      - name: Wait for the rolling deploy
        run: |
          echo "7-wait-deploy" > /tmp/ci-logs/STEP
          set -eu
          want=$(docker image inspect -f '{{.Id}}' "$REGISTRY/$NAMESPACE/server:latest")
          for i in $(seq 1 30); do
            got=$(docker inspect -f '{{.Image}}' clawmates_server_1 2>/dev/null || echo none)
            if [ "$got" = "$want" ]; then
              echo "prod is running main-$SHA"
              curl -s -o /dev/null -w "edge HTTP %{http_code}\n" -m 10 https://clawmates.work/ || true
              exit 0
            fi
            sleep 10
          done
          echo "prod did not roll onto main-$SHA within 5m — check clawmates-deploy.timer"
          exit 1