#!/usr/bin/env bash # recover-uno-q.sh — recover an APESS Uno Q node after a sudden USB/adb # disconnect. Run this from the host (Mac) once the board is physically back. # # A disconnect drops the adb tunnels, kills held-shell services, and can leave # llama wedged — none of it self-heals (see memory: unoq-disconnect-recovery). # This re-establishes every hop and verifies it by ENDPOINT, not by pgrep. # # ./recover-uno-q.sh [cloud-shim-port] # # Steps: reconnect adb → re-tunnel (8080 gateway out, shim port back) → ensure # the on-board supervisor is running (it restarts llama + daemon) → health-check # gateway, llama, and board→shim reachability. set -uo pipefail SERIAL="${1:?usage: recover-uno-q.sh [cloud-shim-port]}" SHIM_PORT="${2:-8090}" SUPERVISOR="/home/arduino/zeroclaw-supervisor.sh" a() { adb -s "$SERIAL" "$@"; } echo "==> [1/5] reconnect adb" adb kill-server >/dev/null 2>&1 || true adb start-server >/dev/null 2>&1 || true for _ in $(seq 1 15); do [ "$(a get-state 2>/dev/null || true)" = "device" ] && break sleep 1 done [ "$(a get-state 2>/dev/null || true)" = "device" ] \ || { echo " board $SERIAL not found — a physical re-plug is required"; exit 1; } echo " device up" echo "==> [2/5] re-establish tunnels (out :8080 gateway, back :$SHIM_PORT shim)" a forward tcp:8080 tcp:8080 >/dev/null a reverse "tcp:$SHIM_PORT" "tcp:$SHIM_PORT" >/dev/null echo " forward :8080 · reverse :$SHIM_PORT" echo "==> [3/5] ensure on-board supervisor is running" if a shell "test -x $SUPERVISOR" 2>/dev/null; then # Bracket the pattern so the pgrep's own shell (whose args contain the # supervisor path) doesn't self-match and skip the launch. a shell "pgrep -f '[z]eroclaw-supervisor.sh' >/dev/null 2>&1 \ || (setsid nohup $SUPERVISOR >/dev/null 2>&1 < /dev/null &)" >/dev/null 2>&1 echo " supervisor ensured (restarts llama + daemon on death/wedge)" else echo " WARN: $SUPERVISOR not on board — falling back to a one-shot daemon start" a shell 'pkill -f "zeroclaw daemon" 2>/dev/null; sleep 1; \ cd /home/arduino && TMPDIR=/tmp setsid nohup ./zeroclaw daemon >> zc-daemon.log 2>&1 < /dev/null &' >/dev/null 2>&1 fi echo "==> [4/5] wait for board services (endpoint, not pgrep)" for _ in $(seq 1 20); do a shell 'curl -sf --max-time 3 http://127.0.0.1:8080/health >/dev/null' 2>/dev/null \ && { echo " gateway :8080 healthy"; break; } sleep 3 done a shell 'curl -sf --max-time 3 http://127.0.0.1:8083/health >/dev/null' 2>/dev/null \ && echo " llama :8083 healthy" \ || echo " llama :8083 still warming (cold GGUF load is 3–5 min)" echo "==> [5/5] verify cloud shim reachable from the board" a shell "curl -sf --max-time 4 http://127.0.0.1:$SHIM_PORT/v1/models >/dev/null" 2>/dev/null \ && echo " board can reach cloud shim :$SHIM_PORT" \ || echo " WARN: shim unreachable from board — is it running on the host?" echo "==> recovery complete for $SERIAL"