feat: failure-injection theater — cloud outage → on-board Qwen, live

Prototype #1 of the APESS innovation set: make the Uno Q's unique story —
a real LLM running on the board — visible and demonstrable. A team can
inject a cloud outage and watch their agent keep reasoning on the on-board
Qwen, which is exactly ADD Layer 4 (failure modes) / Layer 5 (edge vs cloud)
made concrete instead of merely documented.

- New `fallback` NodeActivityKind (api + client, kept in sync). `mapNodeEvent`
  now recognizes ZeroClaw's failover log lines ("ModelProvider call failed",
  "Exhausted retries, trying next model") and surfaces them as a first-class
  resilience signal — NOT swallowed by the generic error branch. Rendered in
  a distinct rose in the board-activity feeds.
- ResiliencePanel (Module 2): a "Simulate cloud outage" button that routes a
  prompt through the board's new `chaos` agent; streams the live failover and
  shows a "survived" banner when a fallback is followed by a response. Sim mode
  plays a deterministic failover so it demos with zero hardware.
- Board config: a `chaos` agent backed by a deliberately-dead cloud endpoint
  (:9099) with `fallback = ["llamacpp.local"]`, so the outage is deterministic
  and workshop-safe (no tunnel-hacking, no real cloud to kill).

Tests: api 34, front-end 191 (+ResiliencePanel), typecheck clean, build passes.
Live-on-board validation pending (board USB link down at commit time).

Co-Authored-By: Claude Opus 4.8 (1M context) <[email protected]>

api/src/nodes.test.ts

@@ -64,6 +64,20 @@ describe('mapNodeEvent — ZeroClaw /api/events → WsEvent', () => {
     expect(ev).toMatchObject({ type: 'node:activity', kind: 'error' })
   })
 
+  it('maps a cloud-exhausted / failover log to a fallback activity (not an error)', () => {
+    const ev = mapNodeEvent('t1', {
+      severity_text: 'ERROR',
+      message: 'Exhausted retries, trying next model_provider/model',
+    })
+    expect(ev).toMatchObject({ type: 'node:activity', kind: 'fallback' })
+    expect((ev as { label: string }).label).toMatch(/on-board Qwen/i)
+  })
+
+  it('maps a mid-retry ModelProvider failure to a fallback activity', () => {
+    const ev = mapNodeEvent('t1', { message: 'ModelProvider call failed, retrying' })
+    expect(ev).toMatchObject({ type: 'node:activity', kind: 'fallback' })
+  })
+
   it('maps agent_end to a response activity', () => {
     const ev = mapNodeEvent('t1', { type: 'agent_end', timestamp: 'T' })
     expect(ev).toMatchObject({ type: 'node:activity', kind: 'response' })

api/src/nodes.ts

@@ -83,6 +83,13 @@ export function mapNodeEvent(teamId: string, raw: unknown): WsEvent | null {
       const addr = message.match(/0x[0-9A-Fa-f]+/)?.[0]
       return activity('flash', addr ? `Flashed to ${addr}` : 'Flashed to the MCU')
     }
+    // Graceful degradation: the cloud provider failed and the agent is failing
+    // over to the on-board Qwen. This is the resilience story — surface it as a
+    // first-class `fallback`, NOT swallowed by the generic error branch below.
+    if (/model[_ ]?provider call failed|exhausted retries|trying next model|falling back/i.test(message)) {
+      const decisive = /exhausted retries|trying next model|falling back/i.test(message)
+      return activity('fallback', decisive ? 'Cloud unreachable — falling back to on-board Qwen' : 'Cloud call failed — retrying')
+    }
     const isError = str(e.severity_text).toUpperCase() === 'ERROR' || /\b(error|failed|failure)\b/i.test(message)
     if (isError) return activity('error', message.slice(0, 200))
   }

api/src/types.ts

@@ -63,8 +63,12 @@ export interface LeaderboardRow {
   scoreCount: number
 }
 
-/** Normalized on-device agent activity, distilled from a node's raw event stream. */
+/**
-export type NodeActivityKind = 'thinking' | 'tool' | 'flash' | 'error' | 'response'
+ * Normalized on-device agent activity, distilled from a node's raw event stream.
+ * `fallback` = graceful degradation: the cloud provider failed and the agent is
+ * failing over to the on-board Qwen (a resilience signal, not an error).
+ */
+export type NodeActivityKind = 'thinking' | 'tool' | 'flash' | 'error' | 'response' | 'fallback'
 
 export type WsEvent =
   | { type: 'snapshot'; teams: TeamSnapshot[]; submissions: SubmissionSummary[] }

deploy/uno-q/config.template.toml

@@ -21,6 +21,16 @@ model = "__CLOUD_MODEL__"
 native_tools = false
 fallback = ["llamacpp.local"]
 
+# A deliberately-dead cloud endpoint (nothing listens on :9099) that fails over
+# to the on-board Qwen. Used by the `chaos` agent to DEMONSTRATE resilience: a
+# prompt routed here always finds the cloud unreachable and answers locally —
+# the "simulate cloud outage" button in APESS (Module 2, failure modes / L4).
+[providers.models.custom.dead]
+uri = "http://127.0.0.1:9099/v1"
+model = "__CLOUD_MODEL__"
+native_tools = false
+fallback = ["llamacpp.local"]
+
 [providers.models.llamacpp]
 
 # On-board Qwen via llama-server (see zeroclaw-llama.service).
@@ -91,3 +101,9 @@ enabled = true
 model_provider = "llamacpp.local"
 risk_profile = "default"
 runtime_profile = "unoq"
+
+[agents.chaos]             # simulated cloud outage → falls back to on-board Qwen
+enabled = true
+model_provider = "custom.dead"
+risk_profile = "default"
+runtime_profile = "unoq"

src/components/BoardActivity.tsx

@@ -8,6 +8,7 @@ const KIND_DOT: Record<NodeActivityKind, string> = {
   flash: 'bg-primary',
   error: 'bg-destructive',
   response: 'bg-teal',
+  fallback: 'bg-rose',
 }
 
 export interface BoardActivityProps {
@@ -36,6 +37,7 @@ export function BoardActivity({ activity, nameFor }: BoardActivityProps) {
               'truncate',
               e.kind === 'error' && 'text-destructive',
               e.kind === 'flash' && 'text-foreground font-medium',
+              e.kind === 'fallback' && 'text-rose font-medium',
             )}
           >
             {e.label}

src/components/BuildFlash.tsx

@@ -28,6 +28,7 @@ const KIND_DOT: Record<NodeActivityKind, string> = {
   flash: 'bg-primary',
   error: 'bg-destructive',
   response: 'bg-teal',
+  fallback: 'bg-rose',
 }
 
 const isTerminal = (k: NodeActivityKind) => k === 'flash' || k === 'response' || k === 'error'

src/components/ResiliencePanel.test.tsx

@@ -0,0 +1,77 @@
+import { describe, it, expect, beforeEach, afterEach, vi } from 'vitest'
+import { render, screen, fireEvent, act } from '@testing-library/react'
+import userEvent from '@testing-library/user-event'
+import { ResiliencePanel } from './ResiliencePanel'
+import { useSession } from '@/store/session'
+import type { WsEvent } from '@/types'
+
+const sendPrompt = vi.fn()
+let liveOnEvent: ((e: WsEvent) => void) | null = null
+const closeSpy = vi.fn()
+
+vi.mock('@/lib/api', () => ({
+  sendPrompt: (...args: unknown[]) => sendPrompt(...args),
+  openTeamActivity: (_teamId: string, onEvent: (e: WsEvent) => void) => {
+    liveOnEvent = onEvent
+    return closeSpy
+  },
+}))
+
+describe('ResiliencePanel', () => {
+  beforeEach(() => {
+    useSession.getState().reset()
+    sessionStorage.clear()
+    sendPrompt.mockReset()
+    closeSpy.mockReset()
+    liveOnEvent = null
+  })
+
+  describe('simulation mode', () => {
+    beforeEach(() => vi.useFakeTimers())
+    afterEach(() => vi.useRealTimers())
+
+    it('plays the cloud-outage → on-board-Qwen failover without hardware or the api', async () => {
+      render(<ResiliencePanel />)
+      fireEvent.click(screen.getByRole('button', { name: /simulate cloud outage/i }))
+
+      await act(async () => {
+        await vi.advanceTimersByTimeAsync(1700)
+      })
+
+      const log = screen.getByTestId('resilience-log')
+      expect(log).toHaveTextContent(/falling back to on-board qwen/i)
+      expect(log).toHaveTextContent(/on-board qwen answered/i)
+      // the "survived" banner appears once a fallback is followed by a response
+      expect(screen.getByTestId('resilience-recovered')).toBeInTheDocument()
+      expect(sendPrompt).not.toHaveBeenCalled()
+      expect(screen.getByRole('button', { name: /simulate cloud outage/i })).toBeEnabled()
+    })
+  })
+
+  describe('live mode', () => {
+    beforeEach(() => useSession.getState().setMode('live'))
+
+    it('routes the outage through the board’s `chaos` agent', async () => {
+      const user = userEvent.setup()
+      render(<ResiliencePanel />)
+      await user.click(screen.getByRole('button', { name: /simulate cloud outage/i }))
+      expect(sendPrompt).toHaveBeenCalledWith(useSession.getState().teamId, expect.any(String), 'chaos')
+    })
+
+    it('renders a streamed fallback event and marks recovery on the response', () => {
+      render(<ResiliencePanel />)
+      act(() => {
+        liveOnEvent?.({ type: 'node:activity', teamId: 't', kind: 'fallback', label: 'Cloud unreachable — falling back to on-board Qwen', ts: 'T' })
+        liveOnEvent?.({ type: 'node:activity', teamId: 't', kind: 'response', label: 'On-board Qwen answered', ts: 'T' })
+      })
+      expect(screen.getByTestId('resilience-log')).toHaveTextContent(/falling back to on-board qwen/i)
+      expect(screen.getByTestId('resilience-recovered')).toBeInTheDocument()
+    })
+
+    it('closes the activity stream on unmount', () => {
+      const { unmount } = render(<ResiliencePanel />)
+      unmount()
+      expect(closeSpy).toHaveBeenCalled()
+    })
+  })
+})

src/components/ResiliencePanel.tsx

@@ -0,0 +1,144 @@
+import { useEffect, useRef, useState } from 'react'
+import { Card, CardContent, CardHeader, CardTitle } from '@/components/ui/card'
+import { Button } from '@/components/ui/button'
+import { Badge } from '@/components/ui/badge'
+import { cn } from '@/lib/utils'
+import { useSession } from '@/store/session'
+import { sendPrompt, openTeamActivity } from '@/lib/api'
+import type { NodeActivityKind, WsEvent } from '@/types'
+
+interface Entry {
+  kind: NodeActivityKind
+  label: string
+}
+
+/** A short reasoning turn — enough to exercise the provider so the failover shows. */
+const CHAOS_PROMPT = 'In one short sentence, report the current structural status.'
+
+/** Deterministic demo of graceful degradation for simulation mode (no hardware). */
+const SIM_FALLBACK: { kind: NodeActivityKind; label: string; delay: number }[] = [
+  { kind: 'thinking', label: 'Agent started — routing to cloud', delay: 200 },
+  { kind: 'fallback', label: 'Cloud unreachable — falling back to on-board Qwen', delay: 900 },
+  { kind: 'response', label: 'On-board Qwen answered — structure nominal', delay: 1600 },
+]
+
+const KIND_DOT: Record<NodeActivityKind, string> = {
+  thinking: 'bg-muted-foreground',
+  tool: 'bg-amber',
+  flash: 'bg-primary',
+  error: 'bg-destructive',
+  response: 'bg-teal',
+  fallback: 'bg-rose',
+}
+
+const isTerminal = (k: NodeActivityKind) => k === 'response' || k === 'error' || k === 'flash'
+
+/**
+ * Failure-injection theater — the Uno Q's unique story made visible. Routes a
+ * prompt through the board's `chaos` agent (a dead cloud endpoint) so the agent
+ * genuinely fails over to the on-board Qwen, live. This is the concrete evidence
+ * for ADD Layer 4 (failure modes) and Layer 5 (edge-vs-cloud).
+ */
+export function ResiliencePanel() {
+  const mode = useSession((s) => s.mode)
+  const teamId = useSession((s) => s.teamId)
+  const [busy, setBusy] = useState(false)
+  const [entries, setEntries] = useState<Entry[]>([])
+  const timers = useRef<ReturnType<typeof setTimeout>[]>([])
+
+  const append = (e: Entry) => setEntries((prev) => [...prev, e])
+  const recovered =
+    entries.some((e) => e.kind === 'fallback') && entries.some((e) => e.kind === 'response')
+
+  // Live mode: stream this team's own board activity.
+  useEffect(() => {
+    if (mode !== 'live') return
+    return openTeamActivity(teamId, (ev: WsEvent) => {
+      if (ev.type !== 'node:activity') return
+      append({ kind: ev.kind, label: ev.label })
+      if (isTerminal(ev.kind)) setBusy(false)
+    })
+  }, [mode, teamId])
+
+  useEffect(() => () => timers.current.forEach(clearTimeout), [])
+
+  const inject = async () => {
+    if (busy) return
+    setEntries([])
+    setBusy(true)
+
+    if (mode === 'sim') {
+      timers.current = SIM_FALLBACK.map((s) =>
+        setTimeout(() => {
+          append({ kind: s.kind, label: s.label })
+          if (isTerminal(s.kind)) setBusy(false)
+        }, s.delay),
+      )
+      return
+    }
+
+    try {
+      await sendPrompt(teamId, CHAOS_PROMPT, 'chaos')
+    } catch {
+      append({ kind: 'error', label: 'Could not reach your board — is it registered and online?' })
+      setBusy(false)
+    }
+  }
+
+  return (
+    <Card>
+      <CardHeader className="flex-row items-center justify-between space-y-0">
+        <CardTitle className="text-base">Resilience — simulate a cloud outage</CardTitle>
+        <Badge
+          variant={mode === 'live' ? 'default' : 'secondary'}
+          className="font-mono text-[10px] uppercase tracking-widest"
+        >
+          {mode === 'live' ? 'Live board' : 'Simulation'}
+        </Badge>
+      </CardHeader>
+      <CardContent className="space-y-4">
+        <p className="text-sm text-muted-foreground max-w-xl">
+          Your agent is cloud-first with an on-board Qwen fallback. Cut the cloud and watch it keep
+          reasoning on the model running on your own board — this is your evidence for ADD Layer 4
+          (failure modes) and Layer 5 (edge vs cloud).
+        </p>
+
+        <Button variant="destructive" onClick={() => void inject()} disabled={busy}>
+          {busy ? 'Injecting outage…' : 'Simulate cloud outage'}
+        </Button>
+
+        {recovered && (
+          <div
+            data-testid="resilience-recovered"
+            className="font-mono text-[11px] text-rose font-medium"
+          >
+            ✓ Survived — the cloud went dark and your on-board Qwen took over.
+          </div>
+        )}
+
+        <ul data-testid="resilience-log" className="space-y-1.5 min-h-[3rem]">
+          {entries.length === 0 ? (
+            <li className="font-mono text-[11px] text-muted-foreground">
+              Inject a cloud outage and watch the agent degrade gracefully to on-device inference.
+            </li>
+          ) : (
+            entries.map((e, i) => (
+              <li key={i} className="flex items-center gap-2 font-mono text-[11px]">
+                <span className={cn('w-1.5 h-1.5 rounded-full shrink-0', KIND_DOT[e.kind])} />
+                <span
+                  className={cn(
+                    e.kind === 'error' && 'text-destructive',
+                    e.kind === 'fallback' && 'text-rose font-medium',
+                    e.kind === 'response' && 'text-foreground',
+                  )}
+                >
+                  {e.label}
+                </span>
+              </li>
+            ))
+          )}
+        </ul>
+      </CardContent>
+    </Card>
+  )
+}

src/pages/Module2.tsx

@@ -10,6 +10,7 @@ import { TriggerButtons } from '@/components/TriggerButtons'
 import { LiveFeed } from '@/components/LiveFeed'
 import { StatsTally } from '@/components/StatsTally'
 import { BuildFlash } from '@/components/BuildFlash'
+import { ResiliencePanel } from '@/components/ResiliencePanel'
 import { AddLayerForm } from '@/components/AddLayerForm'
 import { useSerial } from '@/lib/useSerial'
 import { useSession } from '@/store/session'
@@ -87,6 +88,8 @@ export function Module2() {
 
         <BuildFlash />
 
+        <ResiliencePanel />
+
         <div className="grid lg:grid-cols-2 gap-6">
           <AddLayerForm
             layer="L2"

src/types.ts

@@ -51,8 +51,12 @@ export interface LeaderboardRow {
 }
 
 /** WebSocket events broadcast by the hub to admin/judge clients. */
-/** Normalized on-device agent activity, distilled from a node's raw event stream. */
+/**
-export type NodeActivityKind = 'thinking' | 'tool' | 'flash' | 'error' | 'response'
+ * Normalized on-device agent activity, distilled from a node's raw event stream.
+ * `fallback` = graceful degradation: cloud failed, agent is failing over to the
+ * on-board Qwen (a resilience signal, not an error).
+ */
+export type NodeActivityKind = 'thinking' | 'tool' | 'flash' | 'error' | 'response' | 'fallback'
 
 export type WsEvent =
   | { type: 'snapshot'; teams: TeamSnapshot[]; submissions: SubmissionSummary[] }