feat(voice): browser voice client for talking to the on-board agent
Single-file client: browser-side STT + TTS (Web Speech API, no keys), drives the lean demo agent over /ws/chat (?agent=demo, no auth on the workshop board) to change the LED-matrix animation by voice. Push-to-talk, live transcript, spoken reply, tool activity display. README covers the localhost-serve secure-context requirement and the ElevenLabs/voice-duplex upgrade path. Co-Authored-By: Claude Opus 4.8 <[email protected]>
This commit is contained in:
Omar Sobh
co-authored by
Claude Opus 4.8
Claude Opus 4.8
parent
62e4a3b688
commit
1c762e51ba
@@ -0,0 +1,39 @@
|
|||||||
|
# APESS Voice → Node client
|
||||||
|
|
||||||
|
A self-contained, single-file voice client that lets you **talk to the on-board agent
|
||||||
|
and change the LED-matrix animation by voice** — same agent, same `matrix_pattern` tool
|
||||||
|
as web chat and Telegram.
|
||||||
|
|
||||||
|
It does **STT and TTS entirely in the browser** (Web Speech API), so it needs **no keys,
|
||||||
|
no ElevenLabs, and no server-side TTS**. It talks to the node's `/ws/chat` WebSocket
|
||||||
|
(`?agent=demo`), which requires no auth on the workshop board (`require_pairing = false`).
|
||||||
|
|
||||||
|
## Run it (needs a secure context for mic access)
|
||||||
|
|
||||||
|
`getUserMedia`/`SpeechRecognition` only work on `https://` or `http://localhost`. Serve the
|
||||||
|
folder from localhost on the demo laptop:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
cd deploy/voice-client
|
||||||
|
python3 -m http.server 8090
|
||||||
|
# open http://localhost:8090 in Chrome
|
||||||
|
```
|
||||||
|
|
||||||
|
Then in the page:
|
||||||
|
1. Set the board address (e.g. `192.168.x.x:8080`) and agent (`demo`), click **Connect**
|
||||||
|
(the dot goes green).
|
||||||
|
2. **Hold** the circle, say *"show the wave animation"*, release.
|
||||||
|
3. The node runs the lean on-board agent → `matrix_pattern` → the matrix changes, and the
|
||||||
|
reply is spoken back.
|
||||||
|
|
||||||
|
The `ws://…` connection works fine from an `http://localhost` page (only `https` pages
|
||||||
|
block insecure `ws`). Chrome is required (Web Speech API).
|
||||||
|
|
||||||
|
## Upgrading to ElevenLabs (later)
|
||||||
|
|
||||||
|
Browser TTS is functional but robotic. For ElevenLabs-quality speech, switch to the
|
||||||
|
gateway's **voice-duplex** path (`gateway-voice-duplex` binary already built): configure
|
||||||
|
`[channels.voice_duplex.default]` + an ElevenLabs `tts_provider`, set the demo agent's
|
||||||
|
`tts_provider`, and have the client emit `speech_end {transcript}` and play the server's
|
||||||
|
`tts_chunk` frames instead of using `speechSynthesis`. The STT stays browser-side.
|
||||||
|
</content>
|
||||||
@@ -0,0 +1,161 @@
|
|||||||
|
<!doctype html>
|
||||||
|
<html lang="en">
|
||||||
|
<head>
|
||||||
|
<meta charset="utf-8" />
|
||||||
|
<meta name="viewport" content="width=device-width, initial-scale=1" />
|
||||||
|
<title>APESS · Voice → Node</title>
|
||||||
|
<style>
|
||||||
|
:root { --bg:#0d0d0f; --fg:#eae6df; --dim:#8a857c; --accent:#e8543f; --line:#26242a; }
|
||||||
|
* { box-sizing:border-box; }
|
||||||
|
body { margin:0; background:var(--bg); color:var(--fg);
|
||||||
|
font-family:"JetBrains Mono",ui-monospace,SFMono-Regular,Menlo,monospace;
|
||||||
|
min-height:100vh; display:flex; flex-direction:column; align-items:center; }
|
||||||
|
header { width:100%; border-bottom:1px solid var(--line); padding:14px 18px;
|
||||||
|
text-transform:uppercase; letter-spacing:.18em; font-size:12px; color:var(--dim);
|
||||||
|
display:flex; justify-content:space-between; align-items:center; gap:10px; flex-wrap:wrap; }
|
||||||
|
.cfg { display:flex; gap:8px; align-items:center; flex-wrap:wrap; }
|
||||||
|
.cfg input { background:#141317; color:var(--fg); border:1px solid var(--line);
|
||||||
|
padding:6px 8px; font:inherit; font-size:12px; border-radius:4px; }
|
||||||
|
.cfg input#ip { width:150px; }
|
||||||
|
.cfg input#agent { width:80px; }
|
||||||
|
button { font:inherit; cursor:pointer; }
|
||||||
|
.conn { padding:6px 10px; border:1px solid var(--line); border-radius:4px;
|
||||||
|
background:transparent; color:var(--fg); font-size:12px; text-transform:uppercase; letter-spacing:.1em; }
|
||||||
|
.dot { display:inline-block; width:8px; height:8px; border-radius:50%; background:#5a5750; margin-right:6px; vertical-align:middle; }
|
||||||
|
.dot.on { background:#4caf72; } .dot.err { background:var(--accent); }
|
||||||
|
main { flex:1; width:100%; max-width:720px; padding:24px 18px; display:flex; flex-direction:column; gap:18px; }
|
||||||
|
.mic { align-self:center; width:150px; height:150px; border-radius:50%; border:2px solid var(--line);
|
||||||
|
background:#141317; color:var(--fg); font-size:13px; text-transform:uppercase; letter-spacing:.12em;
|
||||||
|
display:flex; align-items:center; justify-content:center; transition:all .15s; user-select:none; }
|
||||||
|
.mic:hover { border-color:var(--dim); }
|
||||||
|
.mic.live { border-color:var(--accent); background:#2a1512; box-shadow:0 0 0 6px rgba(232,84,63,.12); }
|
||||||
|
.mic:disabled { opacity:.4; cursor:not-allowed; }
|
||||||
|
.hint { text-align:center; color:var(--dim); font-size:12px; margin-top:-8px; }
|
||||||
|
.log { display:flex; flex-direction:column; gap:12px; }
|
||||||
|
.row { border:1px solid var(--line); border-radius:6px; padding:12px 14px; }
|
||||||
|
.row .who { font-size:10px; text-transform:uppercase; letter-spacing:.16em; color:var(--dim); margin-bottom:6px; }
|
||||||
|
.row.you { border-color:#2f3a44; } .row.node { border-color:#3a2f2c; }
|
||||||
|
.row.tool { border-style:dashed; color:var(--dim); font-size:12px; }
|
||||||
|
.txt { font-size:15px; line-height:1.5; white-space:pre-wrap; }
|
||||||
|
.interim { color:var(--dim); font-style:italic; }
|
||||||
|
</style>
|
||||||
|
</head>
|
||||||
|
<body>
|
||||||
|
<header>
|
||||||
|
<span>APESS · VOICE → NODE</span>
|
||||||
|
<div class="cfg">
|
||||||
|
<input id="ip" placeholder="board-ip:8080" />
|
||||||
|
<input id="agent" value="demo" />
|
||||||
|
<button class="conn" id="connect"><span class="dot" id="dot"></span><span id="connlabel">Connect</span></button>
|
||||||
|
</div>
|
||||||
|
</header>
|
||||||
|
<main>
|
||||||
|
<button class="mic" id="mic" disabled>Hold to talk</button>
|
||||||
|
<div class="hint" id="hint">Connect to your node, then hold the circle and speak.</div>
|
||||||
|
<div class="log" id="log"></div>
|
||||||
|
</main>
|
||||||
|
<script>
|
||||||
|
(() => {
|
||||||
|
const $ = id => document.getElementById(id);
|
||||||
|
const ipEl=$('ip'), agentEl=$('agent'), micEl=$('mic'), logEl=$('log'),
|
||||||
|
dot=$('dot'), connLabel=$('connlabel'), hint=$('hint'), connectBtn=$('connect');
|
||||||
|
|
||||||
|
// Persist the board address between sessions.
|
||||||
|
ipEl.value = localStorage.getItem('apess_ip') || location.hostname + ':8080';
|
||||||
|
|
||||||
|
let ws=null, connected=false, nodeRow=null, nodeText='';
|
||||||
|
|
||||||
|
const SR = window.SpeechRecognition || window.webkitSpeechRecognition;
|
||||||
|
if (!SR) { hint.textContent = 'This browser has no Web Speech API — use Chrome.'; }
|
||||||
|
|
||||||
|
function setConn(state){ // 'on' | 'err' | ''
|
||||||
|
dot.className = 'dot' + (state ? ' '+state : '');
|
||||||
|
connected = state==='on';
|
||||||
|
connLabel.textContent = connected ? 'Connected' : (state==='err'?'Retry':'Connect');
|
||||||
|
micEl.disabled = !connected || !SR;
|
||||||
|
if (connected) hint.textContent = 'Hold the circle, speak, release. The node acts and talks back.';
|
||||||
|
}
|
||||||
|
|
||||||
|
function addRow(cls, who, text){
|
||||||
|
const r=document.createElement('div'); r.className='row '+cls;
|
||||||
|
r.innerHTML=`<div class="who">${who}</div><div class="txt"></div>`;
|
||||||
|
r.querySelector('.txt').textContent=text; logEl.appendChild(r);
|
||||||
|
r.scrollIntoView({behavior:'smooth',block:'end'}); return r;
|
||||||
|
}
|
||||||
|
|
||||||
|
function connect(){
|
||||||
|
const ip = ipEl.value.trim(); const agent = agentEl.value.trim() || 'demo';
|
||||||
|
localStorage.setItem('apess_ip', ip);
|
||||||
|
try { ws && ws.close(); } catch(e){}
|
||||||
|
setConn('');
|
||||||
|
ws = new WebSocket(`ws://${ip}/ws/chat?agent=${encodeURIComponent(agent)}`);
|
||||||
|
ws.onopen = () => setConn('on');
|
||||||
|
ws.onclose = () => setConn('');
|
||||||
|
ws.onerror = () => setConn('err');
|
||||||
|
ws.onmessage = ev => {
|
||||||
|
let m; try { m = JSON.parse(ev.data); } catch(e){ return; }
|
||||||
|
switch (m.type) {
|
||||||
|
case 'chunk':
|
||||||
|
if (!nodeRow){ nodeRow=addRow('node','Node',''); nodeText=''; }
|
||||||
|
nodeText += (m.content||''); nodeRow.querySelector('.txt').textContent = nodeText; break;
|
||||||
|
case 'tool_call':
|
||||||
|
addRow('tool','tool', '→ '+(m.tool||m.name||'tool')+ (m.args?(' '+JSON.stringify(m.args)):'')); break;
|
||||||
|
case 'agent_end':
|
||||||
|
case 'done': {
|
||||||
|
const reply = (m.full_response!=null? m.full_response : nodeText).trim();
|
||||||
|
if (reply && !nodeRow) addRow('node','Node',reply);
|
||||||
|
else if (reply && nodeRow) nodeRow.querySelector('.txt').textContent = reply;
|
||||||
|
speak(reply); nodeRow=null; nodeText=''; break;
|
||||||
|
}
|
||||||
|
case 'error':
|
||||||
|
addRow('tool','error', m.message||m.error||'error'); nodeRow=null; break;
|
||||||
|
}
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
function speak(text){
|
||||||
|
if (!text || !window.speechSynthesis) return;
|
||||||
|
window.speechSynthesis.cancel();
|
||||||
|
const u = new SpeechSynthesisUtterance(text);
|
||||||
|
u.rate = 1.02; u.pitch = 1.0; window.speechSynthesis.speak(u);
|
||||||
|
}
|
||||||
|
|
||||||
|
function send(text){
|
||||||
|
if (!text) return;
|
||||||
|
addRow('you','You',text);
|
||||||
|
if (ws && ws.readyState===1) ws.send(JSON.stringify({type:'message', content:text}));
|
||||||
|
else addRow('tool','error','not connected');
|
||||||
|
}
|
||||||
|
|
||||||
|
// --- push-to-talk: hold the circle, speak, release ---
|
||||||
|
let rec=null, listening=false, finalText='';
|
||||||
|
function startListen(){
|
||||||
|
if (!SR || !connected || listening) return;
|
||||||
|
finalText=''; listening=true; micEl.classList.add('live'); micEl.textContent='Listening…';
|
||||||
|
rec = new SR(); rec.lang='en-US'; rec.interimResults=true; rec.continuous=false;
|
||||||
|
rec.onresult = e => {
|
||||||
|
let interim='';
|
||||||
|
for (let i=e.resultIndex;i<e.results.length;i++){
|
||||||
|
const t=e.results[i][0].transcript;
|
||||||
|
if (e.results[i].isFinal) finalText+=t; else interim+=t;
|
||||||
|
}
|
||||||
|
hint.innerHTML = '<span class="interim">'+(finalText+interim||'…')+'</span>';
|
||||||
|
};
|
||||||
|
rec.onerror = () => {};
|
||||||
|
rec.onend = () => { listening=false; micEl.classList.remove('live'); micEl.textContent='Hold to talk';
|
||||||
|
const t=finalText.trim(); hint.textContent='Hold the circle and speak.'; if (t) send(t); };
|
||||||
|
try { rec.start(); } catch(e){ listening=false; }
|
||||||
|
}
|
||||||
|
function stopListen(){ if (rec && listening) { try{ rec.stop(); }catch(e){} } }
|
||||||
|
|
||||||
|
micEl.addEventListener('mousedown', startListen);
|
||||||
|
micEl.addEventListener('mouseup', stopListen);
|
||||||
|
micEl.addEventListener('mouseleave', stopListen);
|
||||||
|
micEl.addEventListener('touchstart', e=>{e.preventDefault();startListen();},{passive:false});
|
||||||
|
micEl.addEventListener('touchend', e=>{e.preventDefault();stopListen();},{passive:false});
|
||||||
|
connectBtn.addEventListener('click', connect);
|
||||||
|
setConn('');
|
||||||
|
})();
|
||||||
|
</script>
|
||||||
|
</body>
|
||||||
|
</html>
|
||||||
Reference in New Issue
Block a user