#!/usr/bin/env bash
# zeroclaw-supervisor.sh — no-root service watchdog for an APESS Uno Q node.
#
# Keeps llama-server (:8083) and the zeroclaw daemon (:8080) alive by polling
# their /health ENDPOINTS — a wedged process passes `pgrep` but fails here — and
# restarting whatever is down. This is the fallback for boards where systemd
# isn't usable (expired account / no root / no user session bus, which is the
# case on the workshop dev board). On a properly-imaged board with root, prefer
# the systemd units in ./systemd/ instead.
#
# Children are started with `setsid` so they survive the shell that launched the
# supervisor closing — that is exactly the property plain `nohup … &` inside an
# adb shell does NOT give you, and why services died on the last disconnect.
#
# Launch once:  setsid nohup /home/arduino/zeroclaw-supervisor.sh >/dev/null 2>&1 < /dev/null &
# Boot persist: crontab -l | { cat; echo '@reboot /home/arduino/zeroclaw-supervisor.sh'; } | crontab -
set -u

# cron's @reboot env is minimal — guarantee the tools we shell out to are found.
export PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:${PATH:-}

HOME_DIR=/home/arduino
LLAMA_DIR="$HOME_DIR/llama"
MODEL="$HOME_DIR/models/qwen.gguf"
ZC="$HOME_DIR/zeroclaw"
LOG="$HOME_DIR/zc-supervisor.log"
LOCK="$HOME_DIR/.zc-supervisor.lock"
INTERVAL="${INTERVAL:-15}"
LLAMA_WARMUP="${LLAMA_WARMUP:-300}"  # cold GGUF load is 3–5 min; don't reap it mid-load
DAEMON_WARMUP="${DAEMON_WARMUP:-20}"

log() { echo "[$(date '+%F %T')] $*" >> "$LOG"; }
now() { date +%s; }
healthy() { curl -sf --max-time 4 "http://127.0.0.1:$1/health" >/dev/null 2>&1; }

# single-instance guard (no flock dependency). Pid-reuse-safe: a stale lock
# whose pid was recycled by an unrelated process (common right after a reboot)
# must NOT block startup — so require the live pid to actually be a supervisor.
if [ -f "$LOCK" ]; then
  OLDPID=$(cat "$LOCK" 2>/dev/null)
  if [ -n "$OLDPID" ] && kill -0 "$OLDPID" 2>/dev/null \
     && grep -qa zeroclaw-supervisor "/proc/$OLDPID/cmdline" 2>/dev/null; then
    log "supervisor already running (pid $OLDPID) — exiting"
    exit 0
  fi
fi
echo $$ > "$LOCK"
trap 'rm -f "$LOCK"' EXIT

llama_ok_after=0
daemon_ok_after=0

# `setsid sh -c '… exec …'` detaches into a new session AND replaces the wrapper
# shell with the target — so no stray bash lingers per restart (a plain
# `( … & )` wrapper leaks one shell each time).
start_llama() {
  log "starting llama-server :8083"
  setsid sh -c "cd '$LLAMA_DIR' && LD_LIBRARY_PATH='$LLAMA_DIR' exec ./llama-server \
    -m '$MODEL' --host 127.0.0.1 --port 8083 -np 1 -c 8192 --jinja --mlock" \
    >> "$HOME_DIR/llama8083.log" 2>&1 < /dev/null &
  llama_ok_after=$(( $(now) + LLAMA_WARMUP ))
}

start_daemon() {
  log "starting zeroclaw daemon :8080"
  setsid sh -c "cd '$HOME_DIR' && TMPDIR=/tmp exec '$ZC' daemon" \
    >> "$HOME_DIR/zc-daemon.log" 2>&1 < /dev/null &
  daemon_ok_after=$(( $(now) + DAEMON_WARMUP ))
}

log "supervisor up (pid $$, interval ${INTERVAL}s)"
while true; do
  if ! healthy 8083; then
    if [ "$(now)" -ge "$llama_ok_after" ]; then
      log "llama :8083 unhealthy past grace — restarting"
      pkill -f llama-server 2>/dev/null; sleep 1; start_llama
    fi
  fi
  if ! healthy 8080; then
    if [ "$(now)" -ge "$daemon_ok_after" ]; then
      log "daemon :8080 unhealthy past grace — restarting"
      pkill -f "zeroclaw daemon" 2>/dev/null; sleep 1; start_daemon
    fi
  fi
  sleep "$INTERVAL"
done