fix(harness): a blind sampler must not report an idle fleet

The burst re-run printed "architect peaked at 1 of 6" and "nothing ever
queued" for a run I could watch sitting at architect=6 tank=6 morpheus=2 with
2 phases queued. The fleet was right; the sampler was blind.

Three separate ssh+psql calls per 15s tick, each with stderr to /dev/null, and
under the load of 16 concurrent missions most came back empty. Empty was then
read as "nothing running" — absence encoded as a legitimate value, which is the
exact seam the header of this file was written about, reproduced in a scenario
added to catch it.

One query per tick now, returning done/blocked/per-node in a single row, and
unreadable samples are COUNTED rather than silently treated as zeroes. Fewer
than ten usable samples is NORUN: a sampler that barely looked must not be able
to describe itself as a fleet that was idle.

Co-Authored-By: Claude Opus 5 <[email protected]>

scripts/verify-mission-delivery.sh

@@ -1105,36 +1105,57 @@ JSON
   # `capacity_blocked_since` is CLEARED the moment a phase is placed, so a
   # post-hoc query cannot prove a queue ever formed. The evidence only exists
   # while the burst is in flight.
-  local waited=0 peak_file queued=0 overcommit=""
+  #
+  # ONE ssh per tick, not three. The first version issued three, each with its
+  # errors sent to /dev/null, and under the load of 16 concurrent missions most
+  # of them came back empty: the run recorded peaks of 1 and "nothing ever
+  # queued" for a burst I could watch sitting at 6/6/2 with 2 queued. A sampler
+  # that cannot see is not the same as a fleet that is idle, and it must not be
+  # able to report the second when it means the first — so failed samples are
+  # COUNTED, and too many of them is NORUN.
+  local waited=0 peak_file queued=0 overcommit="" samples=0 blind=0
   peak_file=$(mktemp)
+  local sample_sql="SELECT
+      (SELECT count(*) FROM missions WHERE id IN ($idlist)
+         AND status IN ('completed','failed','cancelled')),
+      (SELECT count(*) FROM mission_phases p JOIN missions m ON m.id = p.mission_id
+        WHERE m.id IN ($idlist) AND p.capacity_blocked_since IS NOT NULL),
+      COALESCE((SELECT string_agg(nm || '=' || c, ' ') FROM (
+          SELECT COALESCE(n.name,'unpinned') AS nm, count(*) AS c
+            FROM mission_phases p JOIN missions m ON m.id = p.mission_id
+            LEFT JOIN nodes n ON n.id = m.target_node_id
+           WHERE m.id IN ($idlist) AND p.status = 'running'
+           GROUP BY 1) s), '')"
   while [ "$waited" -lt "$MISSION_TIMEOUT" ]; do
-    local done_count
+    local sample done_count blocked running
-    done_count=$(ssh "$HOST" "docker exec clawmates_postgres_1 psql -U postgres -d clawmates -tAc \
+    sample=$(ssh "$HOST" "docker exec clawmates_postgres_1 psql -U postgres -d clawmates -tAc \
-      \"select count(*) from missions where id in ($idlist) and status in ('completed','failed','cancelled');\"" \
+      \"$(printf '%s' "$sample_sql" | tr '\n' ' ')\"" 2>&1 | head -1 | tr -d '\r')
-      | head -1 | tr -d '[:space:]')
+    IFS='|' read -r done_count blocked running <<<"$sample"
-
+    case "${done_count:-}" in
-    # Per-node concurrent phase VMs, right now.
+      ''|*[!0-9]*) blind=$((blind + 1)) ;;
-    ssh "$HOST" "docker exec clawmates_postgres_1 psql -U postgres -d clawmates -tAc \
+      *)
-      \"select coalesce(n.name,'unpinned'), count(*) from mission_phases p
+        samples=$((samples + 1))
-          join missions m on m.id = p.mission_id
+        [ "${blocked:-0}" -gt 0 ] 2>/dev/null && queued=1
-          left join nodes n on n.id = m.target_node_id
+        # One `name count` line per node, so the peak reducer below is unchanged.
-         where m.id in ($idlist) and p.status = 'running'
+        for pair in $running; do printf '%s|%s\n' "${pair%%=*}" "${pair##*=}" >> "$peak_file"; done
-         group by 1;\"" 2>/dev/null | tr -d '\r' >> "$peak_file"
+        [ "$done_count" = "$burst" ] && break
-
+        ;;
-    local q
+    esac
-    q=$(ssh "$HOST" "docker exec clawmates_postgres_1 psql -U postgres -d clawmates -tAc \
-      \"select count(*) from mission_phases p join missions m on m.id = p.mission_id
-         where m.id in ($idlist) and p.capacity_blocked_since is not null;\"" \
-      | head -1 | tr -d '[:space:]')
-    [ "${q:-0}" -gt 0 ] 2>/dev/null && queued=1
-
-    [ "${done_count:-0}" = "$burst" ] && break
     sleep 15
     waited=$((waited + 15))
   done
 
   if [ "$waited" -ge "$MISSION_TIMEOUT" ]; then
-    norun "capacity: the burst did not finish in ${MISSION_TIMEOUT}s — $(cat "$peak_file" | tail -3)"
+    norun "capacity: the burst did not finish in ${MISSION_TIMEOUT}s ($samples good samples, $blind blind)"
+    rm -f "$peak_file"
+    return 1
+  fi
+  info "capacity: took $samples sample(s) during the burst ($blind unreadable)"
+  # A handful of good samples cannot establish either invariant. Ten ticks is
+  # 150s — less than one mission — so anything under that means the sampler,
+  # not the fleet, decided the answer.
+  if [ "$samples" -lt 10 ]; then
+    norun "capacity: only $samples usable sample(s) ($blind unreadable) — too blind to judge the burst"
     rm -f "$peak_file"
     return 1
   fi