deploy/uno-q/README.md

@@ -71,6 +71,43 @@ allow_public_bind = true
 Then pair + register once (steps 4–5 of the script) so APESS has the board's
 `{ url, token }`.
 
+## Resilience — surviving a disconnect (no-root boards)
+
+A sudden USB/adb drop breaks things that don't self-heal: the adb tunnels
+vanish (board loses the cloud shim), held-shell services die, llama can wedge
+(process alive but `:8083` dead), and flashes silently stop landing while the
+tool still reports success. The MCU keeps its last sketch; the paired token
+survives.
+
+The systemd units above are the clean answer **when you have root**. Some dev
+boards don't — an expired account blocks `sudo` and there's no user session bus,
+so neither system nor user units can run. For those, use the no-root pieces:
+
+- **`zeroclaw-supervisor.sh`** (runs on the board) — a watchdog that polls the
+  `/health` **endpoints** (a wedged process passes `pgrep` but fails here) and
+  restarts llama / the daemon when they die or wedge. Children are launched with
+  `setsid … exec` so they survive the shell that started them — the property
+  plain `nohup … &` inside `adb shell` does **not** give you. Install + persist:
+
+  ```sh
+  adb -s <serial> push zeroclaw-supervisor.sh /home/arduino/ && \
+    adb -s <serial> shell 'chmod +x /home/arduino/zeroclaw-supervisor.sh; \
+      setsid nohup /home/arduino/zeroclaw-supervisor.sh >/dev/null 2>&1 </dev/null &'
+  # boot persistence (no root; cron must be running):
+  adb -s <serial> shell '(crontab -l 2>/dev/null | grep -v zeroclaw-supervisor.sh; \
+    echo "@reboot /home/arduino/zeroclaw-supervisor.sh") | crontab -'
+  ```
+
+- **`recover-uno-q.sh`** (runs on the host) — after the board is physically back,
+  re-does adb + both tunnels, ensures the supervisor is up, and health-checks
+  every hop by endpoint:
+
+  ```sh
+  ./recover-uno-q.sh <serial> [cloud-shim-port]   # default shim port 8090
+  ```
+
+See the `unoq-disconnect-recovery` note for the full failure-mode list.
+
 ## Verify
 
 ```sh
@@ -89,4 +126,6 @@ cloud-with-fallback. In APESS, the team's provider/fallback toggle picks the ali
 - `config.template.toml` — the node config (secrets stripped; `__CLOUD_URI__` /
   `__CLOUD_MODEL__` substituted at provision time).
 - `provision-uno-q.sh` — one-shot provisioner (adb-driven).
-- `systemd/*.service` — production units.
+- `systemd/*.service` — production units (need root).
+- `zeroclaw-supervisor.sh` — on-board no-root watchdog (endpoint health + restart).
+- `recover-uno-q.sh` — host-side post-disconnect recovery (re-tunnel + health-check).

deploy/uno-q/recover-uno-q.sh

@@ -0,0 +1,63 @@
+#!/usr/bin/env bash
+# recover-uno-q.sh — recover an APESS Uno Q node after a sudden USB/adb
+# disconnect. Run this from the host (Mac) once the board is physically back.
+#
+# A disconnect drops the adb tunnels, kills held-shell services, and can leave
+# llama wedged — none of it self-heals (see memory: unoq-disconnect-recovery).
+# This re-establishes every hop and verifies it by ENDPOINT, not by pgrep.
+#
+#   ./recover-uno-q.sh <adb-serial> [cloud-shim-port]
+#
+# Steps: reconnect adb → re-tunnel (8080 gateway out, shim port back) → ensure
+# the on-board supervisor is running (it restarts llama + daemon) → health-check
+# gateway, llama, and board→shim reachability.
+set -uo pipefail
+
+SERIAL="${1:?usage: recover-uno-q.sh <adb-serial> [cloud-shim-port]}"
+SHIM_PORT="${2:-8090}"
+SUPERVISOR="/home/arduino/zeroclaw-supervisor.sh"
+a() { adb -s "$SERIAL" "$@"; }
+
+echo "==> [1/5] reconnect adb"
+adb kill-server  >/dev/null 2>&1 || true
+adb start-server >/dev/null 2>&1 || true
+for _ in $(seq 1 15); do
+  [ "$(a get-state 2>/dev/null || true)" = "device" ] && break
+  sleep 1
+done
+[ "$(a get-state 2>/dev/null || true)" = "device" ] \
+  || { echo "    board $SERIAL not found — a physical re-plug is required"; exit 1; }
+echo "    device up"
+
+echo "==> [2/5] re-establish tunnels (out :8080 gateway, back :$SHIM_PORT shim)"
+a forward tcp:8080 tcp:8080 >/dev/null
+a reverse "tcp:$SHIM_PORT" "tcp:$SHIM_PORT" >/dev/null
+echo "    forward :8080 · reverse :$SHIM_PORT"
+
+echo "==> [3/5] ensure on-board supervisor is running"
+if a shell "test -x $SUPERVISOR" 2>/dev/null; then
+  a shell "pgrep -f zeroclaw-supervisor >/dev/null 2>&1 \
+    || (setsid nohup $SUPERVISOR >/dev/null 2>&1 < /dev/null &)" >/dev/null 2>&1
+  echo "    supervisor ensured (restarts llama + daemon on death/wedge)"
+else
+  echo "    WARN: $SUPERVISOR not on board — falling back to a one-shot daemon start"
+  a shell 'pkill -f "zeroclaw daemon" 2>/dev/null; sleep 1; \
+    cd /home/arduino && TMPDIR=/tmp setsid nohup ./zeroclaw daemon >> zc-daemon.log 2>&1 < /dev/null &' >/dev/null 2>&1
+fi
+
+echo "==> [4/5] wait for board services (endpoint, not pgrep)"
+for _ in $(seq 1 20); do
+  a shell 'curl -sf --max-time 3 http://127.0.0.1:8080/health >/dev/null' 2>/dev/null \
+    && { echo "    gateway :8080 healthy"; break; }
+  sleep 3
+done
+a shell 'curl -sf --max-time 3 http://127.0.0.1:8083/health >/dev/null' 2>/dev/null \
+  && echo "    llama :8083 healthy" \
+  || echo "    llama :8083 still warming (cold GGUF load is 3–5 min)"
+
+echo "==> [5/5] verify cloud shim reachable from the board"
+a shell "curl -sf --max-time 4 http://127.0.0.1:$SHIM_PORT/v1/models >/dev/null" 2>/dev/null \
+  && echo "    board can reach cloud shim :$SHIM_PORT" \
+  || echo "    WARN: shim unreachable from board — is it running on the host?"
+
+echo "==> recovery complete for $SERIAL"

deploy/uno-q/zeroclaw-supervisor.sh

@@ -0,0 +1,77 @@
+#!/usr/bin/env bash
+# zeroclaw-supervisor.sh — no-root service watchdog for an APESS Uno Q node.
+#
+# Keeps llama-server (:8083) and the zeroclaw daemon (:8080) alive by polling
+# their /health ENDPOINTS — a wedged process passes `pgrep` but fails here — and
+# restarting whatever is down. This is the fallback for boards where systemd
+# isn't usable (expired account / no root / no user session bus, which is the
+# case on the workshop dev board). On a properly-imaged board with root, prefer
+# the systemd units in ./systemd/ instead.
+#
+# Children are started with `setsid` so they survive the shell that launched the
+# supervisor closing — that is exactly the property plain `nohup … &` inside an
+# adb shell does NOT give you, and why services died on the last disconnect.
+#
+# Launch once:  setsid nohup /home/arduino/zeroclaw-supervisor.sh >/dev/null 2>&1 < /dev/null &
+# Boot persist: crontab -l | { cat; echo '@reboot /home/arduino/zeroclaw-supervisor.sh'; } | crontab -
+set -u
+
+HOME_DIR=/home/arduino
+LLAMA_DIR="$HOME_DIR/llama"
+MODEL="$HOME_DIR/models/qwen.gguf"
+ZC="$HOME_DIR/zeroclaw"
+LOG="$HOME_DIR/zc-supervisor.log"
+LOCK="$HOME_DIR/.zc-supervisor.lock"
+INTERVAL="${INTERVAL:-15}"
+LLAMA_WARMUP="${LLAMA_WARMUP:-300}"  # cold GGUF load is 3–5 min; don't reap it mid-load
+DAEMON_WARMUP="${DAEMON_WARMUP:-20}"
+
+log() { echo "[$(date '+%F %T')] $*" >> "$LOG"; }
+now() { date +%s; }
+healthy() { curl -sf --max-time 4 "http://127.0.0.1:$1/health" >/dev/null 2>&1; }
+
+# single-instance guard (no flock dependency)
+if [ -f "$LOCK" ] && kill -0 "$(cat "$LOCK" 2>/dev/null)" 2>/dev/null; then
+  log "supervisor already running (pid $(cat "$LOCK")) — exiting"
+  exit 0
+fi
+echo $$ > "$LOCK"
+trap 'rm -f "$LOCK"' EXIT
+
+llama_ok_after=0
+daemon_ok_after=0
+
+# `setsid sh -c '… exec …'` detaches into a new session AND replaces the wrapper
+# shell with the target — so no stray bash lingers per restart (a plain
+# `( … & )` wrapper leaks one shell each time).
+start_llama() {
+  log "starting llama-server :8083"
+  setsid sh -c "cd '$LLAMA_DIR' && LD_LIBRARY_PATH='$LLAMA_DIR' exec ./llama-server \
+    -m '$MODEL' --host 127.0.0.1 --port 8083 -np 1 -c 16384 --jinja --mlock" \
+    >> "$HOME_DIR/llama8083.log" 2>&1 < /dev/null &
+  llama_ok_after=$(( $(now) + LLAMA_WARMUP ))
+}
+
+start_daemon() {
+  log "starting zeroclaw daemon :8080"
+  setsid sh -c "cd '$HOME_DIR' && TMPDIR=/tmp exec '$ZC' daemon" \
+    >> "$HOME_DIR/zc-daemon.log" 2>&1 < /dev/null &
+  daemon_ok_after=$(( $(now) + DAEMON_WARMUP ))
+}
+
+log "supervisor up (pid $$, interval ${INTERVAL}s)"
+while true; do
+  if ! healthy 8083; then
+    if [ "$(now)" -ge "$llama_ok_after" ]; then
+      log "llama :8083 unhealthy past grace — restarting"
+      pkill -f llama-server 2>/dev/null; sleep 1; start_llama
+    fi
+  fi
+  if ! healthy 8080; then
+    if [ "$(now)" -ge "$daemon_ok_after" ]; then
+      log "daemon :8080 unhealthy past grace — restarting"
+      pkill -f "zeroclaw daemon" 2>/dev/null; sleep 1; start_daemon
+    fi
+  fi
+  sleep "$INTERVAL"
+done