Omar SobhandClaude Opus 4.8
21fc93556e
A board power-cycle exposed two bugs that left the node dead after boot: - recover-uno-q.sh step 3 ran `pgrep -f zeroclaw-supervisor`, which matches the pgrep's OWN shell (its args contain the supervisor path) — so it always concluded "already running" and never launched the supervisor. Bracket the pattern (`[z]eroclaw-supervisor.sh`) so only the real process matches. - The supervisor's single-instance lock only checked that the locked pid was alive. After a reboot the stale pid can be recycled by an unrelated process, falsely blocking startup. Now require the live pid's /proc/<pid>/cmdline to actually be a supervisor before deferring — otherwise treat the lock as stale. - Also export a full PATH in the supervisor for cron's minimal @reboot env. Validated on hardware: discriminator matches a real supervisor and rejects init's pid; fixed recover detects the running supervisor without spawning a duplicate; board comes back healthy (gateway + llama up). Co-Authored-By: Claude Opus 4.8 (1M context) <[email protected]>
87 lines
3.4 KiB
Bash
Executable File
87 lines
3.4 KiB
Bash
Executable File
#!/usr/bin/env bash
|
||
# zeroclaw-supervisor.sh — no-root service watchdog for an APESS Uno Q node.
|
||
#
|
||
# Keeps llama-server (:8083) and the zeroclaw daemon (:8080) alive by polling
|
||
# their /health ENDPOINTS — a wedged process passes `pgrep` but fails here — and
|
||
# restarting whatever is down. This is the fallback for boards where systemd
|
||
# isn't usable (expired account / no root / no user session bus, which is the
|
||
# case on the workshop dev board). On a properly-imaged board with root, prefer
|
||
# the systemd units in ./systemd/ instead.
|
||
#
|
||
# Children are started with `setsid` so they survive the shell that launched the
|
||
# supervisor closing — that is exactly the property plain `nohup … &` inside an
|
||
# adb shell does NOT give you, and why services died on the last disconnect.
|
||
#
|
||
# Launch once: setsid nohup /home/arduino/zeroclaw-supervisor.sh >/dev/null 2>&1 < /dev/null &
|
||
# Boot persist: crontab -l | { cat; echo '@reboot /home/arduino/zeroclaw-supervisor.sh'; } | crontab -
|
||
set -u
|
||
|
||
# cron's @reboot env is minimal — guarantee the tools we shell out to are found.
|
||
export PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:${PATH:-}
|
||
|
||
HOME_DIR=/home/arduino
|
||
LLAMA_DIR="$HOME_DIR/llama"
|
||
MODEL="$HOME_DIR/models/qwen.gguf"
|
||
ZC="$HOME_DIR/zeroclaw"
|
||
LOG="$HOME_DIR/zc-supervisor.log"
|
||
LOCK="$HOME_DIR/.zc-supervisor.lock"
|
||
INTERVAL="${INTERVAL:-15}"
|
||
LLAMA_WARMUP="${LLAMA_WARMUP:-300}" # cold GGUF load is 3–5 min; don't reap it mid-load
|
||
DAEMON_WARMUP="${DAEMON_WARMUP:-20}"
|
||
|
||
log() { echo "[$(date '+%F %T')] $*" >> "$LOG"; }
|
||
now() { date +%s; }
|
||
healthy() { curl -sf --max-time 4 "http://127.0.0.1:$1/health" >/dev/null 2>&1; }
|
||
|
||
# single-instance guard (no flock dependency). Pid-reuse-safe: a stale lock
|
||
# whose pid was recycled by an unrelated process (common right after a reboot)
|
||
# must NOT block startup — so require the live pid to actually be a supervisor.
|
||
if [ -f "$LOCK" ]; then
|
||
OLDPID=$(cat "$LOCK" 2>/dev/null)
|
||
if [ -n "$OLDPID" ] && kill -0 "$OLDPID" 2>/dev/null \
|
||
&& grep -qa zeroclaw-supervisor "/proc/$OLDPID/cmdline" 2>/dev/null; then
|
||
log "supervisor already running (pid $OLDPID) — exiting"
|
||
exit 0
|
||
fi
|
||
fi
|
||
echo $$ > "$LOCK"
|
||
trap 'rm -f "$LOCK"' EXIT
|
||
|
||
llama_ok_after=0
|
||
daemon_ok_after=0
|
||
|
||
# `setsid sh -c '… exec …'` detaches into a new session AND replaces the wrapper
|
||
# shell with the target — so no stray bash lingers per restart (a plain
|
||
# `( … & )` wrapper leaks one shell each time).
|
||
start_llama() {
|
||
log "starting llama-server :8083"
|
||
setsid sh -c "cd '$LLAMA_DIR' && LD_LIBRARY_PATH='$LLAMA_DIR' exec ./llama-server \
|
||
-m '$MODEL' --host 127.0.0.1 --port 8083 -np 1 -c 16384 --jinja --mlock" \
|
||
>> "$HOME_DIR/llama8083.log" 2>&1 < /dev/null &
|
||
llama_ok_after=$(( $(now) + LLAMA_WARMUP ))
|
||
}
|
||
|
||
start_daemon() {
|
||
log "starting zeroclaw daemon :8080"
|
||
setsid sh -c "cd '$HOME_DIR' && TMPDIR=/tmp exec '$ZC' daemon" \
|
||
>> "$HOME_DIR/zc-daemon.log" 2>&1 < /dev/null &
|
||
daemon_ok_after=$(( $(now) + DAEMON_WARMUP ))
|
||
}
|
||
|
||
log "supervisor up (pid $$, interval ${INTERVAL}s)"
|
||
while true; do
|
||
if ! healthy 8083; then
|
||
if [ "$(now)" -ge "$llama_ok_after" ]; then
|
||
log "llama :8083 unhealthy past grace — restarting"
|
||
pkill -f llama-server 2>/dev/null; sleep 1; start_llama
|
||
fi
|
||
fi
|
||
if ! healthy 8080; then
|
||
if [ "$(now)" -ge "$daemon_ok_after" ]; then
|
||
log "daemon :8080 unhealthy past grace — restarting"
|
||
pkill -f "zeroclaw daemon" 2>/dev/null; sleep 1; start_daemon
|
||
fi
|
||
fi
|
||
sleep "$INTERVAL"
|
||
done
|