feat(uno-q): disconnect resilience — on-board supervisor + host recovery #4
+40
-1
@@ -71,6 +71,43 @@ allow_public_bind = true
|
||||
Then pair + register once (steps 4–5 of the script) so APESS has the board's
|
||||
`{ url, token }`.
|
||||
|
||||
## Resilience — surviving a disconnect (no-root boards)
|
||||
|
||||
A sudden USB/adb drop breaks things that don't self-heal: the adb tunnels
|
||||
vanish (board loses the cloud shim), held-shell services die, llama can wedge
|
||||
(process alive but `:8083` dead), and flashes silently stop landing while the
|
||||
tool still reports success. The MCU keeps its last sketch; the paired token
|
||||
survives.
|
||||
|
||||
The systemd units above are the clean answer **when you have root**. Some dev
|
||||
boards don't — an expired account blocks `sudo` and there's no user session bus,
|
||||
so neither system nor user units can run. For those, use the no-root pieces:
|
||||
|
||||
- **`zeroclaw-supervisor.sh`** (runs on the board) — a watchdog that polls the
|
||||
`/health` **endpoints** (a wedged process passes `pgrep` but fails here) and
|
||||
restarts llama / the daemon when they die or wedge. Children are launched with
|
||||
`setsid … exec` so they survive the shell that started them — the property
|
||||
plain `nohup … &` inside `adb shell` does **not** give you. Install + persist:
|
||||
|
||||
```sh
|
||||
adb -s <serial> push zeroclaw-supervisor.sh /home/arduino/ && \
|
||||
adb -s <serial> shell 'chmod +x /home/arduino/zeroclaw-supervisor.sh; \
|
||||
setsid nohup /home/arduino/zeroclaw-supervisor.sh >/dev/null 2>&1 </dev/null &'
|
||||
# boot persistence (no root; cron must be running):
|
||||
adb -s <serial> shell '(crontab -l 2>/dev/null | grep -v zeroclaw-supervisor.sh; \
|
||||
echo "@reboot /home/arduino/zeroclaw-supervisor.sh") | crontab -'
|
||||
```
|
||||
|
||||
- **`recover-uno-q.sh`** (runs on the host) — after the board is physically back,
|
||||
re-does adb + both tunnels, ensures the supervisor is up, and health-checks
|
||||
every hop by endpoint:
|
||||
|
||||
```sh
|
||||
./recover-uno-q.sh <serial> [cloud-shim-port] # default shim port 8090
|
||||
```
|
||||
|
||||
See the `unoq-disconnect-recovery` note for the full failure-mode list.
|
||||
|
||||
## Verify
|
||||
|
||||
```sh
|
||||
@@ -89,4 +126,6 @@ cloud-with-fallback. In APESS, the team's provider/fallback toggle picks the ali
|
||||
- `config.template.toml` — the node config (secrets stripped; `__CLOUD_URI__` /
|
||||
`__CLOUD_MODEL__` substituted at provision time).
|
||||
- `provision-uno-q.sh` — one-shot provisioner (adb-driven).
|
||||
- `systemd/*.service` — production units.
|
||||
- `systemd/*.service` — production units (need root).
|
||||
- `zeroclaw-supervisor.sh` — on-board no-root watchdog (endpoint health + restart).
|
||||
- `recover-uno-q.sh` — host-side post-disconnect recovery (re-tunnel + health-check).
|
||||
|
||||
Executable
+63
@@ -0,0 +1,63 @@
|
||||
#!/usr/bin/env bash
|
||||
# recover-uno-q.sh — recover an APESS Uno Q node after a sudden USB/adb
|
||||
# disconnect. Run this from the host (Mac) once the board is physically back.
|
||||
#
|
||||
# A disconnect drops the adb tunnels, kills held-shell services, and can leave
|
||||
# llama wedged — none of it self-heals (see memory: unoq-disconnect-recovery).
|
||||
# This re-establishes every hop and verifies it by ENDPOINT, not by pgrep.
|
||||
#
|
||||
# ./recover-uno-q.sh <adb-serial> [cloud-shim-port]
|
||||
#
|
||||
# Steps: reconnect adb → re-tunnel (8080 gateway out, shim port back) → ensure
|
||||
# the on-board supervisor is running (it restarts llama + daemon) → health-check
|
||||
# gateway, llama, and board→shim reachability.
|
||||
set -uo pipefail
|
||||
|
||||
SERIAL="${1:?usage: recover-uno-q.sh <adb-serial> [cloud-shim-port]}"
|
||||
SHIM_PORT="${2:-8090}"
|
||||
SUPERVISOR="/home/arduino/zeroclaw-supervisor.sh"
|
||||
a() { adb -s "$SERIAL" "$@"; }
|
||||
|
||||
echo "==> [1/5] reconnect adb"
|
||||
adb kill-server >/dev/null 2>&1 || true
|
||||
adb start-server >/dev/null 2>&1 || true
|
||||
for _ in $(seq 1 15); do
|
||||
[ "$(a get-state 2>/dev/null || true)" = "device" ] && break
|
||||
sleep 1
|
||||
done
|
||||
[ "$(a get-state 2>/dev/null || true)" = "device" ] \
|
||||
|| { echo " board $SERIAL not found — a physical re-plug is required"; exit 1; }
|
||||
echo " device up"
|
||||
|
||||
echo "==> [2/5] re-establish tunnels (out :8080 gateway, back :$SHIM_PORT shim)"
|
||||
a forward tcp:8080 tcp:8080 >/dev/null
|
||||
a reverse "tcp:$SHIM_PORT" "tcp:$SHIM_PORT" >/dev/null
|
||||
echo " forward :8080 · reverse :$SHIM_PORT"
|
||||
|
||||
echo "==> [3/5] ensure on-board supervisor is running"
|
||||
if a shell "test -x $SUPERVISOR" 2>/dev/null; then
|
||||
a shell "pgrep -f zeroclaw-supervisor >/dev/null 2>&1 \
|
||||
|| (setsid nohup $SUPERVISOR >/dev/null 2>&1 < /dev/null &)" >/dev/null 2>&1
|
||||
echo " supervisor ensured (restarts llama + daemon on death/wedge)"
|
||||
else
|
||||
echo " WARN: $SUPERVISOR not on board — falling back to a one-shot daemon start"
|
||||
a shell 'pkill -f "zeroclaw daemon" 2>/dev/null; sleep 1; \
|
||||
cd /home/arduino && TMPDIR=/tmp setsid nohup ./zeroclaw daemon >> zc-daemon.log 2>&1 < /dev/null &' >/dev/null 2>&1
|
||||
fi
|
||||
|
||||
echo "==> [4/5] wait for board services (endpoint, not pgrep)"
|
||||
for _ in $(seq 1 20); do
|
||||
a shell 'curl -sf --max-time 3 http://127.0.0.1:8080/health >/dev/null' 2>/dev/null \
|
||||
&& { echo " gateway :8080 healthy"; break; }
|
||||
sleep 3
|
||||
done
|
||||
a shell 'curl -sf --max-time 3 http://127.0.0.1:8083/health >/dev/null' 2>/dev/null \
|
||||
&& echo " llama :8083 healthy" \
|
||||
|| echo " llama :8083 still warming (cold GGUF load is 3–5 min)"
|
||||
|
||||
echo "==> [5/5] verify cloud shim reachable from the board"
|
||||
a shell "curl -sf --max-time 4 http://127.0.0.1:$SHIM_PORT/v1/models >/dev/null" 2>/dev/null \
|
||||
&& echo " board can reach cloud shim :$SHIM_PORT" \
|
||||
|| echo " WARN: shim unreachable from board — is it running on the host?"
|
||||
|
||||
echo "==> recovery complete for $SERIAL"
|
||||
Executable
+77
@@ -0,0 +1,77 @@
|
||||
#!/usr/bin/env bash
|
||||
# zeroclaw-supervisor.sh — no-root service watchdog for an APESS Uno Q node.
|
||||
#
|
||||
# Keeps llama-server (:8083) and the zeroclaw daemon (:8080) alive by polling
|
||||
# their /health ENDPOINTS — a wedged process passes `pgrep` but fails here — and
|
||||
# restarting whatever is down. This is the fallback for boards where systemd
|
||||
# isn't usable (expired account / no root / no user session bus, which is the
|
||||
# case on the workshop dev board). On a properly-imaged board with root, prefer
|
||||
# the systemd units in ./systemd/ instead.
|
||||
#
|
||||
# Children are started with `setsid` so they survive the shell that launched the
|
||||
# supervisor closing — that is exactly the property plain `nohup … &` inside an
|
||||
# adb shell does NOT give you, and why services died on the last disconnect.
|
||||
#
|
||||
# Launch once: setsid nohup /home/arduino/zeroclaw-supervisor.sh >/dev/null 2>&1 < /dev/null &
|
||||
# Boot persist: crontab -l | { cat; echo '@reboot /home/arduino/zeroclaw-supervisor.sh'; } | crontab -
|
||||
set -u
|
||||
|
||||
HOME_DIR=/home/arduino
|
||||
LLAMA_DIR="$HOME_DIR/llama"
|
||||
MODEL="$HOME_DIR/models/qwen.gguf"
|
||||
ZC="$HOME_DIR/zeroclaw"
|
||||
LOG="$HOME_DIR/zc-supervisor.log"
|
||||
LOCK="$HOME_DIR/.zc-supervisor.lock"
|
||||
INTERVAL="${INTERVAL:-15}"
|
||||
LLAMA_WARMUP="${LLAMA_WARMUP:-300}" # cold GGUF load is 3–5 min; don't reap it mid-load
|
||||
DAEMON_WARMUP="${DAEMON_WARMUP:-20}"
|
||||
|
||||
log() { echo "[$(date '+%F %T')] $*" >> "$LOG"; }
|
||||
now() { date +%s; }
|
||||
healthy() { curl -sf --max-time 4 "http://127.0.0.1:$1/health" >/dev/null 2>&1; }
|
||||
|
||||
# single-instance guard (no flock dependency)
|
||||
if [ -f "$LOCK" ] && kill -0 "$(cat "$LOCK" 2>/dev/null)" 2>/dev/null; then
|
||||
log "supervisor already running (pid $(cat "$LOCK")) — exiting"
|
||||
exit 0
|
||||
fi
|
||||
echo $$ > "$LOCK"
|
||||
trap 'rm -f "$LOCK"' EXIT
|
||||
|
||||
llama_ok_after=0
|
||||
daemon_ok_after=0
|
||||
|
||||
# `setsid sh -c '… exec …'` detaches into a new session AND replaces the wrapper
|
||||
# shell with the target — so no stray bash lingers per restart (a plain
|
||||
# `( … & )` wrapper leaks one shell each time).
|
||||
start_llama() {
|
||||
log "starting llama-server :8083"
|
||||
setsid sh -c "cd '$LLAMA_DIR' && LD_LIBRARY_PATH='$LLAMA_DIR' exec ./llama-server \
|
||||
-m '$MODEL' --host 127.0.0.1 --port 8083 -np 1 -c 16384 --jinja --mlock" \
|
||||
>> "$HOME_DIR/llama8083.log" 2>&1 < /dev/null &
|
||||
llama_ok_after=$(( $(now) + LLAMA_WARMUP ))
|
||||
}
|
||||
|
||||
start_daemon() {
|
||||
log "starting zeroclaw daemon :8080"
|
||||
setsid sh -c "cd '$HOME_DIR' && TMPDIR=/tmp exec '$ZC' daemon" \
|
||||
>> "$HOME_DIR/zc-daemon.log" 2>&1 < /dev/null &
|
||||
daemon_ok_after=$(( $(now) + DAEMON_WARMUP ))
|
||||
}
|
||||
|
||||
log "supervisor up (pid $$, interval ${INTERVAL}s)"
|
||||
while true; do
|
||||
if ! healthy 8083; then
|
||||
if [ "$(now)" -ge "$llama_ok_after" ]; then
|
||||
log "llama :8083 unhealthy past grace — restarting"
|
||||
pkill -f llama-server 2>/dev/null; sleep 1; start_llama
|
||||
fi
|
||||
fi
|
||||
if ! healthy 8080; then
|
||||
if [ "$(now)" -ge "$daemon_ok_after" ]; then
|
||||
log "daemon :8080 unhealthy past grace — restarting"
|
||||
pkill -f "zeroclaw daemon" 2>/dev/null; sleep 1; start_daemon
|
||||
fi
|
||||
fi
|
||||
sleep "$INTERVAL"
|
||||
done
|
||||
Reference in New Issue
Block a user