rtx-csm: Phase 7.6 — Whisper STT path + asr-feature regression discovery

Wires --whisper flag in converse_server using the existing whisper-rs
asr feature. AsrEngine enum (Kyutai default + Whisper variant gated on
"asr" feature) lets the receive loop branch on backend. Whisper path:
buffer audio during receive, transcribe full buffer at EOT — batch-only,
no VAD, no incremental words.

Adds examples/whisper_profile binary measuring Whisper-tiny in
isolation against the same audio used for stt_profile.

Standalone profile findings (M-series):
  Kyutai STT 1B   : 80.8 ms / 80 ms audio    1.01x realtime
  Whisper-tiny    : 209 ms / 10.43 s audio   0.020x (~50x faster)

But the full-stack bench reveals a critical regression: linking
whisper-rs's C++ runtime into the same binary as candle/CSM costs
2-3x across ALL CSM inference (recv_phase, tts_per_utterance,
total_turn) even when --whisper is NOT used. Build flag matters.

  Build                          recv    tts/u   total
  --features metal               4196    3113    18707
  --features metal,asr (Kyutai)  10019   7803    43366  <- linkage cost
  --features metal,asr +whisper  0       12921   54028  <- worse

Suspected cause: ggml/whisper.cpp's BLAS or Metal context init
conflicts with candle's. Production verdict: build WITHOUT asr
feature; accept Kyutai's 1x realtime STT cost. The standalone
whisper_profile binary still works for batch transcribe measurement.

Real Whisper integration would need a sidecar process pattern (whisper
running as a separate binary, IPC to converse_server). Documented in
the --whisper CLI help. Flag stays as opt-in with explicit warning.

Co-Authored-By: Claude Opus 4.7 (1M context) <[email protected]>

crates/models/rtx-csm/Cargo.toml

@@ -221,3 +221,8 @@ path = "examples/llm_extra_body_smoke.rs"
 [[example]]
 name = "stt_profile"
 path = "examples/stt_profile.rs"
+
+[[example]]
+name = "whisper_profile"
+path = "examples/whisper_profile.rs"
+required-features = ["asr"]

crates/models/rtx-csm/examples/converse_server.rs

@@ -32,6 +32,12 @@
 //!
 //! Drive it with `examples/converse_client.rs`.
 
+// Without the `asr` feature, AsrEngine has only the Kyutai variant so
+// `if let AsrEngine::Kyutai(_) = ...` patterns are technically
+// irrefutable. With `asr`, they're refutable. Silence the warning so the
+// code compiles cleanly under both cfgs.
+#![allow(irrefutable_let_patterns)]
+
 use anyhow::{Context, Result};
 use async_trait::async_trait;
 use axum::{
@@ -168,6 +174,24 @@ struct Cli {
     /// `--watermark-*` is rejected at boot.
     #[arg(long)]
     stream_tts: bool,
+
+    /// Use Whisper-tiny (whisper-rs / whisper.cpp) instead of Kyutai STT
+    /// 1B for transcription. ~50× faster on Metal in isolation (10 s of
+    /// audio → 0.2 s transcribe) but batch-only — transcript fires at
+    /// EOT, not incrementally. Loses semantic VAD (`--vad`).
+    ///
+    /// **WARNING — perf regression**: linking whisper-rs's C++ runtime
+    /// into the same binary as candle/CSM costs ~2-3× across all CSM
+    /// inference (recv_phase, tts_per_utterance, total_turn) even when
+    /// this flag isn't set. We don't yet understand the conflict
+    /// (BLAS / threadpool / Metal init contention is suspected). For
+    /// production deploys, build WITHOUT `--features asr` and accept
+    /// Kyutai STT's 1× realtime cost; or run Whisper as a sidecar
+    /// process via IPC (not yet implemented).
+    ///
+    /// Requires the crate built with `--features asr-metal` (or `asr`).
+    #[arg(long)]
+    whisper: bool,
     /// In streaming mode, the number of 80ms Mimi frames per chunk.
     /// Default 4 = 320ms, which empirically balances first-audio
     /// latency vs decode overhead.
@@ -272,9 +296,18 @@ struct Metrics {
     total_turn_ms_count: AtomicU64,
 }
 
+/// ASR backend wrapped behind a single Mutex so the receive loop can
+/// branch on its kind. Whisper variant is feature-gated; without `asr`
+/// at compile time only Kyutai is available.
+enum AsrEngine {
+    Kyutai(Stt),
+    #[cfg(feature = "asr")]
+    Whisper(rtx_csm::asr::WhisperAsr),
+}
+
 struct Shared {
     generator: Mutex<Generator>,
-    stt: Mutex<Stt>,
+    stt: Mutex<AsrEngine>,
     llm: AnyLlm,
     system_prompt: String,
     speaker: u32,
@@ -293,6 +326,10 @@ struct Shared {
     stream_tts: bool,
     /// Mimi frames per stream chunk (4 = 320ms default).
     stream_chunk_frames: usize,
+    /// True when the ASR backend is Whisper (batch-only). The receive
+    /// loop skips parallel STT in this case and runs one transcribe
+    /// call at EOT.
+    whisper_mode: bool,
     metrics: Metrics,
 }
 
@@ -456,15 +493,31 @@ async fn main() -> Result<()> {
         );
     }
 
+    let stt: AsrEngine = if cli.whisper {
         if cli.vad {
+            anyhow::bail!(
+                "--whisper and --vad cannot combine: Whisper has no per-frame probability \
+                 output. Pick one."
+            );
+        }
+        #[cfg(feature = "asr")]
+        {
+            tracing::info!("loading Whisper-tiny (whisper-rs)...");
+            AsrEngine::Whisper(rtx_csm::asr::WhisperAsr::load_default()?)
+        }
+        #[cfg(not(feature = "asr"))]
+        {
+            anyhow::bail!(
+                "--whisper requires the `asr-metal` (or `asr`) feature. Rebuild with \
+                 `cargo build --features metal,asr-metal --example converse_server`."
+            );
+        }
+    } else if cli.vad {
         tracing::info!("loading Kyutai STT 1B en/fr-candle with VAD (~3 GB)...");
+        AsrEngine::Kyutai(Stt::load_default_with_vad(&device)?)
     } else {
         tracing::info!("loading Kyutai STT 1B en/fr (~3 GB)...");
-    }
+        AsrEngine::Kyutai(Stt::load_default(&device)?)
-    let stt = if cli.vad {
-        Stt::load_default_with_vad(&device)?
-    } else {
-        Stt::load_default(&device)?
     };
     tracing::info!("models loaded");
 
@@ -542,6 +595,7 @@ async fn main() -> Result<()> {
         llm_extra_body,
         stream_tts: cli.stream_tts,
         stream_chunk_frames: cli.stream_chunk_frames.max(1),
+        whisper_mode: cli.whisper,
         metrics: Metrics::default(),
     });
 
@@ -681,22 +735,32 @@ async fn handle_connection(mut socket: WebSocket, shared: Arc<Shared>) {
             .await;
             break 'session;
         }
-        // Reset STT state for each new turn so silence buffer + delay
+        // Reset Kyutai STT state for each new turn (Whisper is stateless
-        // counters start fresh.
+        // so this is a no-op for Whisper).
-        if let Err(e) = shared.stt.lock().await.reset() {
+        {
-            send_text(&mut socket, &format!("{{\"event\":\"error\",\"msg\":\"stt reset: {e}\"}}"))
+            let mut stt = shared.stt.lock().await;
+            if let AsrEngine::Kyutai(s) = &mut *stt {
+                if let Err(e) = s.reset() {
+                    send_text(
+                        &mut socket,
+                        &format!("{{\"event\":\"error\",\"msg\":\"stt reset: {e}\"}}"),
+                    )
                     .await;
                     break 'session;
                 }
+            }
+        }
 
         let mut user_audio_24k: Vec<f32> = Vec::new();
         // Barge-in carryover from the previous turn.
         if let Some(co) = carry_over.take() {
             user_audio_24k.extend(co);
         }
-        // Reset STT once per turn — the incremental ingest below feeds
+        // Same reset (some legacy duplication; harmless and Kyutai-only).
-        // PCM as it arrives so transcript is mostly done by EOT time.
+        {
-        if let Err(e) = shared.stt.lock().await.reset() {
+            let mut stt = shared.stt.lock().await;
+            if let AsrEngine::Kyutai(s) = &mut *stt {
+                if let Err(e) = s.reset() {
                     send_text(
                         &mut socket,
                         &format!("{{\"event\":\"error\",\"msg\":\"stt reset: {e}\"}}"),
@@ -704,6 +768,8 @@ async fn handle_connection(mut socket: WebSocket, shared: Arc<Shared>) {
                     .await;
                     break 'session;
                 }
+            }
+        }
         // VAD state (only used when --vad is enabled).
         let mut consecutive_eot: u32 = 0;
         // Tracks how much of user_audio_24k has been pushed to STT.
@@ -712,18 +778,23 @@ async fn handle_connection(mut socket: WebSocket, shared: Arc<Shared>) {
         let mut pending_word: Option<Vec<u32>> = None;
         let mut transcript_words: Vec<String> = Vec::new();
         // If we have carry-over audio, feed it first (it's already in
-        // user_audio_24k from the barge-in stash).
+        // user_audio_24k from the barge-in stash). Kyutai-only — Whisper
-        if !user_audio_24k.is_empty() {
+        // skips parallel ingest entirely.
+        if !user_audio_24k.is_empty() && !shared.whisper_mode {
             let carry_slice = user_audio_24k.clone();
             stt_streaming_offset = carry_slice.len();
             let events = {
                 let mut stt = shared.stt.lock().await;
-                match stt.step_pcm(&carry_slice) {
+                match &mut *stt {
+                    AsrEngine::Kyutai(s) => match s.step_pcm(&carry_slice) {
                         Ok(es) => es,
                         Err(e) => {
                             tracing::warn!("carry-over step_pcm: {e}");
                             Vec::new()
                         }
+                    },
+                    #[cfg(feature = "asr")]
+                    AsrEngine::Whisper(_) => Vec::new(),
                 }
             };
             collect_transcript_events(
@@ -733,6 +804,9 @@ async fn handle_connection(mut socket: WebSocket, shared: Arc<Shared>) {
                 &mut transcript_words,
             )
             .await;
+        } else if !user_audio_24k.is_empty() {
+            // Whisper mode: just track that carry-over is in the buffer.
+            stt_streaming_offset = user_audio_24k.len();
         }
         // Receive frames until EOT.
         loop {
@@ -747,20 +821,29 @@ async fn handle_connection(mut socket: WebSocket, shared: Arc<Shared>) {
                         let s = i16::from_le_bytes([c[0], c[1]]);
                         user_audio_24k.push(s as f32 / i16::MAX as f32);
                     }
-                    // Always feed new samples into STT incrementally so
+                    // Kyutai path: feed new samples into STT incrementally
-                    // transcript is built as audio arrives.
+                    // so the transcript builds as audio arrives. Whisper
+                    // path skips this — audio just accumulates in
+                    // user_audio_24k for the post-EOT batch transcribe.
+                    let events = if !shared.whisper_mode {
                         let new_slice =
                             user_audio_24k[stt_streaming_offset..].to_vec();
                         stt_streaming_offset = user_audio_24k.len();
-                    let events = {
                         let mut stt = shared.stt.lock().await;
-                        match stt.step_pcm(&new_slice) {
+                        match &mut *stt {
+                            AsrEngine::Kyutai(s) => match s.step_pcm(&new_slice) {
                                 Ok(es) => es,
                                 Err(e) => {
                                     tracing::warn!("step_pcm: {e}");
                                     Vec::new()
                                 }
+                            },
+                            #[cfg(feature = "asr")]
+                            AsrEngine::Whisper(_) => Vec::new(),
                         }
+                    } else {
+                        stt_streaming_offset = user_audio_24k.len();
+                        Vec::new()
                     };
                     collect_transcript_events(
                         &events,
@@ -818,10 +901,15 @@ async fn handle_connection(mut socket: WebSocket, shared: Arc<Shared>) {
             .metrics
             .recv_phase_ms_count
             .fetch_add(1, Ordering::Relaxed);
-        // Drain the asr_delay buffer so any trailing words flush.
+        // Drain the asr_delay buffer so any trailing Kyutai words
+        // flush. Whisper has no asr_delay buffer to drain.
         let final_events = {
             let mut stt = shared.stt.lock().await;
-            stt.finish().unwrap_or_default()
+            match &mut *stt {
+                AsrEngine::Kyutai(s) => s.finish().unwrap_or_default(),
+                #[cfg(feature = "asr")]
+                AsrEngine::Whisper(_) => Vec::new(),
+            }
         };
         collect_transcript_events(
             &final_events,
@@ -856,12 +944,35 @@ async fn handle_connection(mut socket: WebSocket, shared: Arc<Shared>) {
         user_audio_24k
             .extend(std::iter::repeat(0.0f32).take((PCM_RATE as f32 * 2.0) as usize));
 
-        // STT was already running incrementally during receive — the
+        // Build user_text. Kyutai has been streaming words incrementally
-        // post-EOT phase here is just a tight loop joining accumulated
+        // — just join them. Whisper is batch-only, so we fire one
-        // words. The metric measures wall-clock from EOT to transcript-
+        // transcribe call here over the full accumulated audio. (With
-        // ready (roughly the time of the final `finish()` flush).
+        // Whisper-tiny on Metal, 10s of audio takes ~200ms — much
+        // faster than Kyutai's 5s parallel-STT-during-receive.)
         let stt_t = std::time::Instant::now();
-        let user_text = transcript_words.join(" ").trim().to_string();
+        let user_text = if shared.whisper_mode {
+            #[cfg(feature = "asr")]
+            {
+                let stt = shared.stt.lock().await;
+                if let AsrEngine::Whisper(w) = &*stt {
+                    match w.transcribe_24k(&user_audio_24k) {
+                        Ok(t) => t.trim().to_string(),
+                        Err(e) => {
+                            tracing::warn!("whisper transcribe: {e}");
+                            String::new()
+                        }
+                    }
+                } else {
+                    transcript_words.join(" ").trim().to_string()
+                }
+            }
+            #[cfg(not(feature = "asr"))]
+            {
+                transcript_words.join(" ").trim().to_string()
+            }
+        } else {
+            transcript_words.join(" ").trim().to_string()
+        };
         let stt_ms = stt_t.elapsed().as_millis() as u64;
         shared
             .metrics
@@ -1203,7 +1314,7 @@ async fn send_text(socket: &mut WebSocket, payload: &str) -> bool {
 /// events (VAD prs) are ignored here — they're handled by the VAD loop.
 async fn collect_transcript_events(
     events: &[AsrEvent],
-    stt_lock: &Mutex<Stt>,
+    stt_lock: &Mutex<AsrEngine>,
     pending: &mut Option<Vec<u32>>,
     transcript_words: &mut Vec<String>,
 ) {
@@ -1211,12 +1322,19 @@ async fn collect_transcript_events(
         return;
     }
     let stt = stt_lock.lock().await;
+    let kyutai = match &*stt {
+        AsrEngine::Kyutai(s) => s,
+        // Whisper path doesn't go through this — events come from
+        // Kyutai's incremental Word/EndWord stream only.
+        #[cfg(feature = "asr")]
+        AsrEngine::Whisper(_) => return,
+    };
     for ev in events {
         match ev {
             AsrEvent::Word { tokens, .. } => *pending = Some(tokens.clone()),
             AsrEvent::EndWord { .. } => {
                 if let Some(tokens) = pending.take() {
-                    if let Some(w) = stt.decode_word_text(&tokens) {
+                    if let Some(w) = kyutai.decode_word_text(&tokens) {
                         let w = w.trim().to_string();
                         if !w.is_empty() {
                             transcript_words.push(w);

crates/models/rtx-csm/examples/whisper_profile.rs

@@ -0,0 +1,84 @@
+//! Profile Whisper-tiny via whisper-rs (the `--features asr-metal` path)
+//! against the same audio used for `stt_profile`. Lets us A/B Whisper
+//! batch transcription vs Kyutai STT 1B streaming.
+//!
+//! Usage:
+//! ```bash
+//! cargo run -p rtx-csm --release --features asr-metal --example whisper_profile -- \
+//!     --in /tmp/asr_test.flac
+//! ```
+
+use anyhow::{Context, Result};
+use clap::Parser;
+use rtx_csm::{asr::WhisperAsr, audio_io};
+use std::path::PathBuf;
+use std::time::Instant;
+
+#[derive(Debug, Parser)]
+struct Cli {
+    #[arg(long = "in", default_value = "/tmp/asr_test.flac")]
+    input: PathBuf,
+    /// Number of repeat transcriptions to amortize first-call warm-up.
+    #[arg(long, default_value_t = 3)]
+    repeat: usize,
+}
+
+fn main() -> Result<()> {
+    tracing_subscriber::fmt().with_max_level(tracing::Level::WARN).init();
+    let cli = Cli::parse();
+
+    eprintln!("loading audio: {}", cli.input.display());
+    let pcm = audio_io::load_mono_at_rate(&cli.input, 24_000).context("load audio")?;
+    let audio_secs = pcm.len() as f32 / 24_000.0;
+    eprintln!("loaded {} samples ({audio_secs:.2}s @ 24 kHz)", pcm.len());
+
+    eprintln!("loading Whisper-tiny via whisper-rs...");
+    let load_t = Instant::now();
+    let asr = WhisperAsr::load_default().context("load whisper")?;
+    eprintln!("model loaded in {:.2}s", load_t.elapsed().as_secs_f32());
+
+    // Warm-up call (model does first-call setup).
+    let warm_t = Instant::now();
+    let warm_text = asr.transcribe_24k(&pcm).context("warm transcribe")?;
+    eprintln!(
+        "warm-up: {} ms, transcript = {:?}",
+        warm_t.elapsed().as_millis(),
+        warm_text
+    );
+
+    // Steady-state runs.
+    let mut per_call_ms: Vec<f64> = Vec::with_capacity(cli.repeat);
+    let mut last_text = String::new();
+    for i in 0..cli.repeat {
+        let t = Instant::now();
+        let text = asr.transcribe_24k(&pcm).context("transcribe")?;
+        let ms = t.elapsed().as_secs_f64() * 1000.0;
+        per_call_ms.push(ms);
+        last_text = text;
+        eprintln!("  run {}: {ms:.0} ms", i + 1);
+    }
+
+    per_call_ms.sort_by(|a, b| a.partial_cmp(b).unwrap());
+    let n = per_call_ms.len() as f64;
+    let mean = per_call_ms.iter().sum::<f64>() / n;
+    let p50 = per_call_ms[per_call_ms.len() / 2];
+    let realtime_factor = (mean / 1000.0) / audio_secs as f64;
+
+    println!();
+    println!("=== Whisper-tiny profile ===");
+    println!("input:         {} ({audio_secs:.2}s of audio)", cli.input.display());
+    println!("steady-state runs: {}", per_call_ms.len());
+    println!();
+    println!("per-call latency:");
+    println!("  mean = {mean:.0} ms");
+    println!("  p50  = {p50:.0} ms");
+    println!("  min  = {:.0} ms", per_call_ms[0]);
+    println!("  max  = {:.0} ms", per_call_ms[per_call_ms.len() - 1]);
+    println!();
+    println!("realtime factor: {realtime_factor:.3}x");
+    println!("  (mean / audio_duration; sub-1.0 means faster than realtime)");
+    println!();
+    println!("transcript: {last_text:?}");
+
+    Ok(())
+}