Omar SobhandClaude Opus 4.8
8130b0e30a
A sudden USB/adb drop breaks the Uno Q node in ways that don't self-heal: adb tunnels vanish, held-shell services die, llama wedges (alive but not serving), and flashes silently stop landing. systemd is the clean fix with root — but the dev board's account is expired (no sudo) and has no user session bus, so neither system nor user units run. These are the no-root equivalents, both validated on hardware: - zeroclaw-supervisor.sh — on-board watchdog. Polls the /health ENDPOINTS (a wedged process passes pgrep but fails here) and restarts llama / the daemon on death or wedge. Launches children with `setsid … exec` so they survive the launching shell — the property `nohup … &` in adb shell lacks. Startup grace avoids reaping llama mid-cold-load; single-instance lock; no per-restart shell leak. Proven: kill -9 the daemon → auto-restarted. Boot-persisted via `@reboot` crontab (no root). - recover-uno-q.sh — host-side. After the board is back, re-does adb + both tunnels (out :8080, back :8090 shim), ensures the supervisor is running, and health-checks every hop by endpoint. Proven end-to-end after a simulated tunnel drop. README: new Resilience section documenting the no-root reality + both tools. Co-Authored-By: Claude Opus 4.8 (1M context) <[email protected]>
78 lines
3.0 KiB
Bash
Executable File
78 lines
3.0 KiB
Bash
Executable File
#!/usr/bin/env bash
|
||
# zeroclaw-supervisor.sh — no-root service watchdog for an APESS Uno Q node.
|
||
#
|
||
# Keeps llama-server (:8083) and the zeroclaw daemon (:8080) alive by polling
|
||
# their /health ENDPOINTS — a wedged process passes `pgrep` but fails here — and
|
||
# restarting whatever is down. This is the fallback for boards where systemd
|
||
# isn't usable (expired account / no root / no user session bus, which is the
|
||
# case on the workshop dev board). On a properly-imaged board with root, prefer
|
||
# the systemd units in ./systemd/ instead.
|
||
#
|
||
# Children are started with `setsid` so they survive the shell that launched the
|
||
# supervisor closing — that is exactly the property plain `nohup … &` inside an
|
||
# adb shell does NOT give you, and why services died on the last disconnect.
|
||
#
|
||
# Launch once: setsid nohup /home/arduino/zeroclaw-supervisor.sh >/dev/null 2>&1 < /dev/null &
|
||
# Boot persist: crontab -l | { cat; echo '@reboot /home/arduino/zeroclaw-supervisor.sh'; } | crontab -
|
||
set -u
|
||
|
||
HOME_DIR=/home/arduino
|
||
LLAMA_DIR="$HOME_DIR/llama"
|
||
MODEL="$HOME_DIR/models/qwen.gguf"
|
||
ZC="$HOME_DIR/zeroclaw"
|
||
LOG="$HOME_DIR/zc-supervisor.log"
|
||
LOCK="$HOME_DIR/.zc-supervisor.lock"
|
||
INTERVAL="${INTERVAL:-15}"
|
||
LLAMA_WARMUP="${LLAMA_WARMUP:-300}" # cold GGUF load is 3–5 min; don't reap it mid-load
|
||
DAEMON_WARMUP="${DAEMON_WARMUP:-20}"
|
||
|
||
log() { echo "[$(date '+%F %T')] $*" >> "$LOG"; }
|
||
now() { date +%s; }
|
||
healthy() { curl -sf --max-time 4 "http://127.0.0.1:$1/health" >/dev/null 2>&1; }
|
||
|
||
# single-instance guard (no flock dependency)
|
||
if [ -f "$LOCK" ] && kill -0 "$(cat "$LOCK" 2>/dev/null)" 2>/dev/null; then
|
||
log "supervisor already running (pid $(cat "$LOCK")) — exiting"
|
||
exit 0
|
||
fi
|
||
echo $$ > "$LOCK"
|
||
trap 'rm -f "$LOCK"' EXIT
|
||
|
||
llama_ok_after=0
|
||
daemon_ok_after=0
|
||
|
||
# `setsid sh -c '… exec …'` detaches into a new session AND replaces the wrapper
|
||
# shell with the target — so no stray bash lingers per restart (a plain
|
||
# `( … & )` wrapper leaks one shell each time).
|
||
start_llama() {
|
||
log "starting llama-server :8083"
|
||
setsid sh -c "cd '$LLAMA_DIR' && LD_LIBRARY_PATH='$LLAMA_DIR' exec ./llama-server \
|
||
-m '$MODEL' --host 127.0.0.1 --port 8083 -np 1 -c 16384 --jinja --mlock" \
|
||
>> "$HOME_DIR/llama8083.log" 2>&1 < /dev/null &
|
||
llama_ok_after=$(( $(now) + LLAMA_WARMUP ))
|
||
}
|
||
|
||
start_daemon() {
|
||
log "starting zeroclaw daemon :8080"
|
||
setsid sh -c "cd '$HOME_DIR' && TMPDIR=/tmp exec '$ZC' daemon" \
|
||
>> "$HOME_DIR/zc-daemon.log" 2>&1 < /dev/null &
|
||
daemon_ok_after=$(( $(now) + DAEMON_WARMUP ))
|
||
}
|
||
|
||
log "supervisor up (pid $$, interval ${INTERVAL}s)"
|
||
while true; do
|
||
if ! healthy 8083; then
|
||
if [ "$(now)" -ge "$llama_ok_after" ]; then
|
||
log "llama :8083 unhealthy past grace — restarting"
|
||
pkill -f llama-server 2>/dev/null; sleep 1; start_llama
|
||
fi
|
||
fi
|
||
if ! healthy 8080; then
|
||
if [ "$(now)" -ge "$daemon_ok_after" ]; then
|
||
log "daemon :8080 unhealthy past grace — restarting"
|
||
pkill -f "zeroclaw daemon" 2>/dev/null; sleep 1; start_daemon
|
||
fi
|
||
fi
|
||
sleep "$INTERVAL"
|
||
done
|