Merge feat/uno-q-cross-build-deploy: resilience theater UI + cross-build helper

Folds the last unmerged workshop work into main:
- ResiliencePanel — the failure-injection 'theater' UI (cloud outage → on-board
  Qwen live demo) that visualizes the [agents.chaos] (custom.dead → llamacpp)
  failover, plus Module2 integration + api/nodes support.
- build-deploy.sh — one-command aarch64-musl cross-build + adb deploy helper,
  REFRESHED for 0.8.3: adds the web SPA prebuild step and the full feature set
  (hardware, peripheral-rpi, embedded-web, gateway-voice-duplex) + the
  embedded-web/web_dist_dir gotcha.

Config conflict resolved as a union: the branch's custom.dead + [agents.chaos]
demo blocks alongside this session's channels/voice/skills/risk-profile edits.
Verified: tsc clean, web 229 + api 58 tests green.

Co-Authored-By: Claude Opus 4.8 <[email protected]>

api/src/nodes.test.ts

@@ -64,6 +64,20 @@ describe('mapNodeEvent — ZeroClaw /api/events → WsEvent', () => {
     expect(ev).toMatchObject({ type: 'node:activity', kind: 'error' })
   })
 
+  it('maps a cloud-exhausted / failover log to a fallback activity (not an error)', () => {
+    const ev = mapNodeEvent('t1', {
+      severity_text: 'ERROR',
+      message: 'Exhausted retries, trying next model_provider/model',
+    })
+    expect(ev).toMatchObject({ type: 'node:activity', kind: 'fallback' })
+    expect((ev as { label: string }).label).toMatch(/on-board Qwen/i)
+  })
+
+  it('maps a mid-retry ModelProvider failure to a fallback activity', () => {
+    const ev = mapNodeEvent('t1', { message: 'ModelProvider call failed, retrying' })
+    expect(ev).toMatchObject({ type: 'node:activity', kind: 'fallback' })
+  })
+
   it('maps agent_end to a response activity', () => {
     const ev = mapNodeEvent('t1', { type: 'agent_end', timestamp: 'T' })
     expect(ev).toMatchObject({ type: 'node:activity', kind: 'response' })

api/src/nodes.ts

@@ -83,6 +83,13 @@ export function mapNodeEvent(teamId: string, raw: unknown): WsEvent | null {
       const addr = message.match(/0x[0-9A-Fa-f]+/)?.[0]
       return activity('flash', addr ? `Flashed to ${addr}` : 'Flashed to the MCU')
     }
+    // Graceful degradation: the cloud provider failed and the agent is failing
+    // over to the on-board Qwen. This is the resilience story — surface it as a
+    // first-class `fallback`, NOT swallowed by the generic error branch below.
+    if (/model[_ ]?provider call failed|exhausted retries|trying next model|falling back/i.test(message)) {
+      const decisive = /exhausted retries|trying next model|falling back/i.test(message)
+      return activity('fallback', decisive ? 'Cloud unreachable — falling back to on-board Qwen' : 'Cloud call failed — retrying')
+    }
     const isError = str(e.severity_text).toUpperCase() === 'ERROR' || /\b(error|failed|failure)\b/i.test(message)
     if (isError) return activity('error', message.slice(0, 200))
   }

api/src/types.ts

@@ -63,8 +63,12 @@ export interface LeaderboardRow {
   scoreCount: number
 }
 
-/** Normalized on-device agent activity, distilled from a node's raw event stream. */
+/**
-export type NodeActivityKind = 'thinking' | 'tool' | 'flash' | 'error' | 'response'
+ * Normalized on-device agent activity, distilled from a node's raw event stream.
+ * `fallback` = graceful degradation: the cloud provider failed and the agent is
+ * failing over to the on-board Qwen (a resilience signal, not an error).
+ */
+export type NodeActivityKind = 'thinking' | 'tool' | 'flash' | 'error' | 'response' | 'fallback'
 
 export type WsEvent =
   | { type: 'snapshot'; teams: TeamSnapshot[]; submissions: SubmissionSummary[]; unclaimed?: string[] }

deploy/uno-q/build-deploy.sh

@@ -0,0 +1,103 @@
+#!/usr/bin/env bash
+# build-deploy.sh — cross-build ZeroClaw for the Arduino Uno Q (aarch64) on this
+# Mac and deploy it to the board over adb, as a proper drop-in replacement for
+# /home/arduino/zeroclaw. One command for the "make a fix → get it on the board"
+# loop.
+#
+# WHY THE FLAGS (learned the hard way — see memory apess-node-modalities-and-083):
+#   hardware              MANDATORY. Without it the build silently drops the 10
+#                         Uno Q peripheral tools (GPIO bridge, FLASH, sysfs_led,
+#                         camera, network, i2cdetect). Symptom: agent load log
+#                         shows "before:48" tools instead of "before:58".
+#   peripheral-rpi        the rppal-backed Linux peripheral path (sysfs LED, etc.).
+#   embedded-web          bakes the web dashboard SPA (web/dist) into the binary,
+#                         so the board serves the ZeroClaw dashboard at :8080/.
+#                         REQUIRES web/dist to be built FIRST (step 1) — and built
+#                         with base="/_app/" (the production vite build), else the
+#                         gateway serves index.html for /assets/* and the SPA won't
+#                         boot. If the embed ever goes stale, set
+#                         `[gateway] web_dist_dir = "/home/arduino/web-dist"` and
+#                         push web/dist to the board as a runtime fallback.
+#   zeroclaw-gateway/gateway-voice-duplex
+#                         browser-mic streaming voice over /ws/chat (namespaced —
+#                         no root-crate alias).
+#   musl target           the deployed binary is static (glibc-independent).
+#   cargo-zigbuild        cross-links from macOS via zig (no cross toolchain).
+#                         (Alt without zig: cargo build with
+#                          CARGO_TARGET_AARCH64_UNKNOWN_LINUX_MUSL_LINKER=aarch64-linux-musl-gcc.)
+#
+# PREREQS (already set up on this Mac): rustup targets aarch64-unknown-linux-musl,
+# cargo-zigbuild + zig on PATH, node/npm for the web build, adb, the board on USB.
+#
+# Usage:
+#   deploy/uno-q/build-deploy.sh                 # build + deploy + verify
+#   ZC_REPO=/path/to/zeroclaw deploy/uno-q/build-deploy.sh
+#   SERIAL=65301572 deploy/uno-q/build-deploy.sh
+#   deploy/uno-q/build-deploy.sh --build-only    # build + verify, no deploy
+#   deploy/uno-q/build-deploy.sh --rollback      # restore the previous binary from backup
+set -euo pipefail
+
+ZC_REPO="${ZC_REPO:-$HOME/projects/zeroclaw}"
+SERIAL="${SERIAL:-65301572}"
+TARGET="aarch64-unknown-linux-musl"
+FEATURES="hardware,peripheral-rpi,embedded-web,zeroclaw-gateway/gateway-voice-duplex"
+BIN="$ZC_REPO/target/$TARGET/release/zeroclaw"
+REMOTE="/home/arduino/zeroclaw"
+SUPERVISOR="/home/arduino/zeroclaw-supervisor.sh"
+adbsh() { adb -s "$SERIAL" shell "$1" </dev/null; }   # </dev/null so adb always returns (avoids the setsid-hang)
+
+require_device() {
+  [ "$(adb -s "$SERIAL" get-state 2>/dev/null)" = "device" ] || { echo "✗ board $SERIAL not connected (adb get-state != device)"; exit 1; }
+}
+
+rollback() {
+  require_device
+  echo "→ rolling back to $REMOTE.prev.bak"
+  adbsh "test -f $REMOTE.prev.bak" || { echo "✗ no backup at $REMOTE.prev.bak"; exit 1; }
+  adbsh "pkill -f '[z]eroclaw-supervisor'; sleep 1"
+  adbsh "pkill -9 -f 'zeroclaw daemon'; sleep 1"
+  adbsh "mv -f $REMOTE $REMOTE.rolledback.bak; mv -f $REMOTE.prev.bak $REMOTE; chmod +x $REMOTE"
+  adbsh "setsid sh -c 'exec $SUPERVISOR' >/home/arduino/zeroclaw-supervisor.log 2>&1 </dev/null &"
+  echo "→ rolled back; supervisor relaunched. sha:$(adbsh "sha256sum $REMOTE | cut -c1-16")"
+  exit 0
+}
+
+[ "${1:-}" = "--rollback" ] && rollback
+
+echo "=== 1/5 prebuild web dashboard SPA (embedded-web bakes web/dist into the binary) ==="
+# gen the API TS bindings, then the production vite build (base=/_app/). embedded-web's
+# include_dir! reads web/dist at compile time, so this MUST run before the cargo build.
+( cd "$ZC_REPO" && cargo run -q -p xtask --bin web -- gen-api && cd web && npm run build )
+
+echo "=== 2/5 cross-build (aarch64 musl; hardware + peripherals + embedded web + voice) ==="
+( cd "$ZC_REPO" && cargo zigbuild --target "$TARGET" --release --locked --features "$FEATURES" --bin zeroclaw )
+
+echo "=== 3/5 verify artifact ==="
+# grep the binary directly (grep -a) rather than `strings | grep -q`: under
+# `set -o pipefail`, grep -q closes the pipe on first match and `strings` dies
+# with SIGPIPE, making the pipeline exit non-zero even on a MATCH.
+file "$BIN" | grep -q "aarch64" || { echo "✗ not an aarch64 binary"; exit 1; }
+grep -qa "Uno Q tools added" "$BIN" || { echo "✗ peripheral tools MISSING — did the hardware feature build?"; exit 1; }
+grep -qa "uno_q_flash" "$BIN" || { echo "✗ flash tool missing"; exit 1; }
+echo "✓ $(ls -la "$BIN" | awk '{print $5}') bytes, peripheral+flash tools present, sha:$(shasum -a256 "$BIN" | cut -c1-16)"
+
+[ "${1:-}" = "--build-only" ] && { echo "build-only: done (artifact at $BIN)"; exit 0; }
+
+require_device
+echo "=== 4/5 push + swap (backup kept at $REMOTE.prev.bak) ==="
+adb -s "$SERIAL" push "$BIN" "$REMOTE.new"
+L=$(shasum -a256 "$BIN" | cut -d' ' -f1); R=$(adbsh "sha256sum $REMOTE.new | cut -d' ' -f1")
+[ "$L" = "$R" ] || { echo "✗ sha mismatch after push ($L != $R)"; exit 1; }
+adbsh "pkill -f '[z]eroclaw-supervisor'; sleep 1"
+adbsh "pkill -9 -f 'zeroclaw daemon'; sleep 1"
+adbsh "cp -f $REMOTE $REMOTE.prev.bak; mv -f $REMOTE.new $REMOTE; chmod +x $REMOTE"
+adbsh "setsid sh -c 'exec $SUPERVISOR' >/home/arduino/zeroclaw-supervisor.log 2>&1 </dev/null &"
+
+echo "=== 5/5 wait for daemon health (:8080 via host forward) ==="
+adb -s "$SERIAL" forward tcp:8080 tcp:8080 >/dev/null 2>&1 || true
+for i in $(seq 1 30); do
+  [ "$(curl -s -m 3 -o /dev/null -w '%{http_code}' http://127.0.0.1:8080/health 2>/dev/null)" = "200" ] && { echo "✓ daemon healthy on new binary (sha:$(adbsh "sha256sum $REMOTE | cut -c1-16"))"; exit 0; }
+  sleep 3
+done
+echo "✗ daemon did not report healthy in time — check the board; roll back with: $0 --rollback"
+exit 1

deploy/uno-q/config.template.toml

@@ -21,6 +21,16 @@ model = "__CLOUD_MODEL__"
 native_tools = false
 fallback = ["llamacpp.local"]
 
+# A deliberately-dead cloud endpoint (nothing listens on :9099) that fails over
+# to the on-board Qwen. Used by the `chaos` agent to DEMONSTRATE resilience: a
+# prompt routed here always finds the cloud unreachable and answers locally —
+# the "simulate cloud outage" button in APESS (Module 2, failure modes / L4).
+[providers.models.custom.dead]
+uri = "http://127.0.0.1:9099/v1"
+model = "__CLOUD_MODEL__"
+native_tools = false
+fallback = ["llamacpp.local"]
+
 [providers.models.llamacpp]
 
 # On-board Qwen via llama-server (see zeroclaw-llama.service).
@@ -128,3 +138,8 @@ mention_only = false
 # Enable once the board runs a voice-capable binary:
 # [channels.voice_duplex.default]
 # enabled = true
+[agents.chaos]             # simulated cloud outage → falls back to on-board Qwen
+enabled = true
+model_provider = "custom.dead"
+risk_profile = "default"
+runtime_profile = "unoq"

src/components/BoardActivity.tsx

@@ -8,6 +8,7 @@ const KIND_DOT: Record<NodeActivityKind, string> = {
   flash: 'bg-primary',
   error: 'bg-destructive',
   response: 'bg-teal',
+  fallback: 'bg-rose',
 }
 
 export interface BoardActivityProps {
@@ -36,6 +37,7 @@ export function BoardActivity({ activity, nameFor }: BoardActivityProps) {
               'truncate',
               e.kind === 'error' && 'text-destructive',
               e.kind === 'flash' && 'text-foreground font-medium',
+              e.kind === 'fallback' && 'text-rose font-medium',
             )}
           >
             {e.label}

src/components/BuildFlash.tsx

@@ -28,6 +28,7 @@ const KIND_DOT: Record<NodeActivityKind, string> = {
   flash: 'bg-primary',
   error: 'bg-destructive',
   response: 'bg-teal',
+  fallback: 'bg-rose',
 }
 
 const isTerminal = (k: NodeActivityKind) => k === 'flash' || k === 'response' || k === 'error'

src/components/ResiliencePanel.test.tsx

@@ -0,0 +1,77 @@
+import { describe, it, expect, beforeEach, afterEach, vi } from 'vitest'
+import { render, screen, fireEvent, act } from '@testing-library/react'
+import userEvent from '@testing-library/user-event'
+import { ResiliencePanel } from './ResiliencePanel'
+import { useSession } from '@/store/session'
+import type { WsEvent } from '@/types'
+
+const sendPrompt = vi.fn()
+let liveOnEvent: ((e: WsEvent) => void) | null = null
+const closeSpy = vi.fn()
+
+vi.mock('@/lib/api', () => ({
+  sendPrompt: (...args: unknown[]) => sendPrompt(...args),
+  openTeamActivity: (_teamId: string, onEvent: (e: WsEvent) => void) => {
+    liveOnEvent = onEvent
+    return closeSpy
+  },
+}))
+
+describe('ResiliencePanel', () => {
+  beforeEach(() => {
+    useSession.getState().reset()
+    sessionStorage.clear()
+    sendPrompt.mockReset()
+    closeSpy.mockReset()
+    liveOnEvent = null
+  })
+
+  describe('simulation mode', () => {
+    beforeEach(() => vi.useFakeTimers())
+    afterEach(() => vi.useRealTimers())
+
+    it('plays the cloud-outage → on-board-Qwen failover without hardware or the api', async () => {
+      render(<ResiliencePanel />)
+      fireEvent.click(screen.getByRole('button', { name: /simulate cloud outage/i }))
+
+      await act(async () => {
+        await vi.advanceTimersByTimeAsync(1700)
+      })
+
+      const log = screen.getByTestId('resilience-log')
+      expect(log).toHaveTextContent(/falling back to on-board qwen/i)
+      expect(log).toHaveTextContent(/on-board qwen answered/i)
+      // the "survived" banner appears once a fallback is followed by a response
+      expect(screen.getByTestId('resilience-recovered')).toBeInTheDocument()
+      expect(sendPrompt).not.toHaveBeenCalled()
+      expect(screen.getByRole('button', { name: /simulate cloud outage/i })).toBeEnabled()
+    })
+  })
+
+  describe('live mode', () => {
+    beforeEach(() => useSession.getState().setMode('live'))
+
+    it('routes the outage through the board’s `chaos` agent', async () => {
+      const user = userEvent.setup()
+      render(<ResiliencePanel />)
+      await user.click(screen.getByRole('button', { name: /simulate cloud outage/i }))
+      expect(sendPrompt).toHaveBeenCalledWith(useSession.getState().teamId, expect.any(String), 'chaos')
+    })
+
+    it('renders a streamed fallback event and marks recovery on the response', () => {
+      render(<ResiliencePanel />)
+      act(() => {
+        liveOnEvent?.({ type: 'node:activity', teamId: 't', kind: 'fallback', label: 'Cloud unreachable — falling back to on-board Qwen', ts: 'T' })
+        liveOnEvent?.({ type: 'node:activity', teamId: 't', kind: 'response', label: 'On-board Qwen answered', ts: 'T' })
+      })
+      expect(screen.getByTestId('resilience-log')).toHaveTextContent(/falling back to on-board qwen/i)
+      expect(screen.getByTestId('resilience-recovered')).toBeInTheDocument()
+    })
+
+    it('closes the activity stream on unmount', () => {
+      const { unmount } = render(<ResiliencePanel />)
+      unmount()
+      expect(closeSpy).toHaveBeenCalled()
+    })
+  })
+})

src/components/ResiliencePanel.tsx

@@ -0,0 +1,144 @@
+import { useEffect, useRef, useState } from 'react'
+import { Card, CardContent, CardHeader, CardTitle } from '@/components/ui/card'
+import { Button } from '@/components/ui/button'
+import { Badge } from '@/components/ui/badge'
+import { cn } from '@/lib/utils'
+import { useSession } from '@/store/session'
+import { sendPrompt, openTeamActivity } from '@/lib/api'
+import type { NodeActivityKind, WsEvent } from '@/types'
+
+interface Entry {
+  kind: NodeActivityKind
+  label: string
+}
+
+/** A short reasoning turn — enough to exercise the provider so the failover shows. */
+const CHAOS_PROMPT = 'In one short sentence, report the current structural status.'
+
+/** Deterministic demo of graceful degradation for simulation mode (no hardware). */
+const SIM_FALLBACK: { kind: NodeActivityKind; label: string; delay: number }[] = [
+  { kind: 'thinking', label: 'Agent started — routing to cloud', delay: 200 },
+  { kind: 'fallback', label: 'Cloud unreachable — falling back to on-board Qwen', delay: 900 },
+  { kind: 'response', label: 'On-board Qwen answered — structure nominal', delay: 1600 },
+]
+
+const KIND_DOT: Record<NodeActivityKind, string> = {
+  thinking: 'bg-muted-foreground',
+  tool: 'bg-amber',
+  flash: 'bg-primary',
+  error: 'bg-destructive',
+  response: 'bg-teal',
+  fallback: 'bg-rose',
+}
+
+const isTerminal = (k: NodeActivityKind) => k === 'response' || k === 'error' || k === 'flash'
+
+/**
+ * Failure-injection theater — the Uno Q's unique story made visible. Routes a
+ * prompt through the board's `chaos` agent (a dead cloud endpoint) so the agent
+ * genuinely fails over to the on-board Qwen, live. This is the concrete evidence
+ * for ADD Layer 4 (failure modes) and Layer 5 (edge-vs-cloud).
+ */
+export function ResiliencePanel() {
+  const mode = useSession((s) => s.mode)
+  const teamId = useSession((s) => s.teamId)
+  const [busy, setBusy] = useState(false)
+  const [entries, setEntries] = useState<Entry[]>([])
+  const timers = useRef<ReturnType<typeof setTimeout>[]>([])
+
+  const append = (e: Entry) => setEntries((prev) => [...prev, e])
+  const recovered =
+    entries.some((e) => e.kind === 'fallback') && entries.some((e) => e.kind === 'response')
+
+  // Live mode: stream this team's own board activity.
+  useEffect(() => {
+    if (mode !== 'live') return
+    return openTeamActivity(teamId, (ev: WsEvent) => {
+      if (ev.type !== 'node:activity') return
+      append({ kind: ev.kind, label: ev.label })
+      if (isTerminal(ev.kind)) setBusy(false)
+    })
+  }, [mode, teamId])
+
+  useEffect(() => () => timers.current.forEach(clearTimeout), [])
+
+  const inject = async () => {
+    if (busy) return
+    setEntries([])
+    setBusy(true)
+
+    if (mode === 'sim') {
+      timers.current = SIM_FALLBACK.map((s) =>
+        setTimeout(() => {
+          append({ kind: s.kind, label: s.label })
+          if (isTerminal(s.kind)) setBusy(false)
+        }, s.delay),
+      )
+      return
+    }
+
+    try {
+      await sendPrompt(teamId, CHAOS_PROMPT, 'chaos')
+    } catch {
+      append({ kind: 'error', label: 'Could not reach your board — is it registered and online?' })
+      setBusy(false)
+    }
+  }
+
+  return (
+    <Card>
+      <CardHeader className="flex-row items-center justify-between space-y-0">
+        <CardTitle className="text-base">Resilience — simulate a cloud outage</CardTitle>
+        <Badge
+          variant={mode === 'live' ? 'default' : 'secondary'}
+          className="font-mono text-[10px] uppercase tracking-widest"
+        >
+          {mode === 'live' ? 'Live board' : 'Simulation'}
+        </Badge>
+      </CardHeader>
+      <CardContent className="space-y-4">
+        <p className="text-sm text-muted-foreground max-w-xl">
+          Your agent is cloud-first with an on-board Qwen fallback. Cut the cloud and watch it keep
+          reasoning on the model running on your own board — this is your evidence for ADD Layer 4
+          (failure modes) and Layer 5 (edge vs cloud).
+        </p>
+
+        <Button variant="destructive" onClick={() => void inject()} disabled={busy}>
+          {busy ? 'Injecting outage…' : 'Simulate cloud outage'}
+        </Button>
+
+        {recovered && (
+          <div
+            data-testid="resilience-recovered"
+            className="font-mono text-[11px] text-rose font-medium"
+          >
+            ✓ Survived — the cloud went dark and your on-board Qwen took over.
+          </div>
+        )}
+
+        <ul data-testid="resilience-log" className="space-y-1.5 min-h-[3rem]">
+          {entries.length === 0 ? (
+            <li className="font-mono text-[11px] text-muted-foreground">
+              Inject a cloud outage and watch the agent degrade gracefully to on-device inference.
+            </li>
+          ) : (
+            entries.map((e, i) => (
+              <li key={i} className="flex items-center gap-2 font-mono text-[11px]">
+                <span className={cn('w-1.5 h-1.5 rounded-full shrink-0', KIND_DOT[e.kind])} />
+                <span
+                  className={cn(
+                    e.kind === 'error' && 'text-destructive',
+                    e.kind === 'fallback' && 'text-rose font-medium',
+                    e.kind === 'response' && 'text-foreground',
+                  )}
+                >
+                  {e.label}
+                </span>
+              </li>
+            ))
+          )}
+        </ul>
+      </CardContent>
+    </Card>
+  )
+}

src/pages/Module2.tsx

@@ -11,6 +11,7 @@ import { LiveFeed } from '@/components/LiveFeed'
 import { LiveBoardFeed } from '@/components/LiveBoardFeed'
 import { StatsTally } from '@/components/StatsTally'
 import { BuildFlash } from '@/components/BuildFlash'
+import { ResiliencePanel } from '@/components/ResiliencePanel'
 import { AddLayerForm } from '@/components/AddLayerForm'
 import { useSerial } from '@/lib/useSerial'
 import { useNodeFeed } from '@/lib/useNodeFeed'
@@ -101,6 +102,8 @@ export function Module2() {
 
         <BuildFlash />
 
+        <ResiliencePanel />
+
         <div className="grid lg:grid-cols-2 gap-6">
           <AddLayerForm
             layer="L2"

src/types.ts

@@ -51,8 +51,12 @@ export interface LeaderboardRow {
 }
 
 /** WebSocket events broadcast by the hub to admin/judge clients. */
-/** Normalized on-device agent activity, distilled from a node's raw event stream. */
+/**
-export type NodeActivityKind = 'thinking' | 'tool' | 'flash' | 'error' | 'response'
+ * Normalized on-device agent activity, distilled from a node's raw event stream.
+ * `fallback` = graceful degradation: cloud failed, agent is failing over to the
+ * on-board Qwen (a resilience signal, not an error).
+ */
+export type NodeActivityKind = 'thinking' | 'tool' | 'flash' | 'error' | 'response' | 'fallback'
 
 export type WsEvent =
   | { type: 'snapshot'; teams: TeamSnapshot[]; submissions: SubmissionSummary[]; unclaimed?: string[] }