rtx-csm: Phase 13.8 — emotion2vec port, slice 4 (EmotionDetector + integration)

Port complete. The Phase 13.3 prosody-rule placeholder is now
retire-able by setting one CLI flag — the real candle-ported
emotion2vec_plus_base classifier slots in behind the same
EmotionDetector trait the placeholder used.

impl EmotionDetector for Emotion2Vec — builds (1, 1, T) tensor on
stored device, runs forward, argmaxes the 9 logits, maps to the
5-bucket label via Classifier::tag_for_class. Empty input
short-circuits to Neutral.

Emotion2Vec struct gained a `device` field so the trait impl can
build tensors without an out-of-band handle. new() / load_from_pickle()
threaded through; existing tests + smoke binary updated.

audio_to_manifest --use-emotion2vec — pairs with --auto-emotion-tag
to swap ProsodyDetector for Emotion2Vec, boxed as
Box<dyn EmotionDetector> so the call site is unchanged.

converse_server --use-emotion2vec — same pattern; built once at boot
and stored in Shared as Box<dyn EmotionDetector + Send + Sync>.
~150 ms/turn forward cost vs <1 ms for prosody, but actually runs
SOTA SER. Removed redundant reactive_emotion: bool field — the
Option<Box<dyn>> already encodes the same state.

Verified end-to-end on Metal:
  - audio_to_manifest --use-emotion2vec on 2-speaker concat → both
    tagged [excited] (prosody had said [neutral] on same input)
  - converse_server --quantized-gguf … --lora … --reactive-emotion
    --use-emotion2vec boots, 1 bench turn 0 errors, /metrics shows
    reactive_emotion_total{label="excited"} 1 — same tag
    audio_to_manifest produced. Cross-consumer consistency.

Phase 13.8 complete (slices 1+2+3+4 shipped). The emotional-voice
stack now has a real, trained, candle-ported SER classifier with
no Python sidecar, no ort, no whisper.cpp.

Lib suite 120/120.

Co-Authored-By: Claude Opus 4.7 (1M context) <[email protected]>

crates/models/rtx-csm/examples/audio_to_manifest.rs

@@ -61,13 +61,21 @@ struct Cli {
     #[arg(long)]
     emotion_tag: Option<String>,
 
-    /// Auto-tag every row with a per-segment emotion label inferred via the
+    /// Auto-tag every row with a per-segment emotion label. Default
-    /// Phase 13.3 prosody-rule classifier (RMS + F0 + voicing → 5 buckets:
+    /// classifier is the Phase 13.3 prosody-rule placeholder (RMS + F0
-    /// [neutral]/[calm]/[sad]/[angry]/[excited]). Crude — best as a starting
+    /// + voicing → 5 buckets); pair with `--use-emotion2vec` for the
-    /// point you refine post-hoc, not as ground truth.
+    /// real emotion2vec_plus_base classifier (Phase 13.8).
     #[arg(long, default_value_t = false)]
     auto_emotion_tag: bool,
 
+    /// When set together with `--auto-emotion-tag`, swap the prosody-rule
+    /// classifier for the emotion2vec_plus_base candle port. ~93 M
+    /// params, downloaded once from HF Hub (~1.1 GB). Per-segment cost
+    /// is ~150 ms on Metal vs <1 ms for the prosody rule, but accuracy
+    /// is comparable to a real SOTA SER model on conversational audio.
+    #[arg(long, default_value_t = false)]
+    use_emotion2vec: bool,
+
     /// Static curriculum stage label added to every manifest row
     /// (e.g. `audiobook` for the Phase 12.3 recipe).
     #[arg(long)]
@@ -181,9 +189,22 @@ fn main() -> Result<()> {
         .map_err(|e| anyhow::anyhow!("tokenizer: {e}"))?;
     println!("Moonshine loaded");
 
-    // Optional: prosody-rule SER for per-segment auto-labeling.
+    // Optional: per-segment auto-labeling. Prosody-rule by default; the
-    let ser = if cli.auto_emotion_tag {
+    // emotion2vec_plus_base candle port is opt-in via --use-emotion2vec.
-        Some(rtx_csm::ser::ProsodyDetector::default())
+    // Boxed as `dyn EmotionDetector` so both paths share the call site.
+    let ser: Option<Box<dyn rtx_csm::ser::EmotionDetector>> = if cli.auto_emotion_tag {
+        if cli.use_emotion2vec {
+            let api = hf_hub::api::sync::Api::new()?;
+            let path = api
+                .model("emotion2vec/emotion2vec_plus_base".into())
+                .get("model.pt")
+                .context("download emotion2vec_plus_base/model.pt")?;
+            let model = rtx_csm::emotion2vec::Emotion2Vec::load_from_pickle(&path, &device)?;
+            tracing::info!("loaded emotion2vec_plus_base from {}", path.display());
+            Some(Box::new(model))
+        } else {
+            Some(Box::new(rtx_csm::ser::ProsodyDetector::default()))
+        }
     } else {
         None
     };
@@ -246,8 +267,9 @@ fn main() -> Result<()> {
         let auto_tag_owned: Option<String> = ser
             .as_ref()
             .map(|d| {
-                let label = rtx_csm::ser::EmotionDetector::classify(d, slice)
+                // d: &Box<dyn EmotionDetector>; method-style call
-                    .unwrap_or(rtx_csm::ser::EmotionLabel::Neutral);
+                // auto-derefs through the box to the trait impl.
+                let label = d.classify(slice).unwrap_or(rtx_csm::ser::EmotionLabel::Neutral);
                 label.as_tag().to_string()
             });
         let resolved_tag = auto_tag_owned

crates/models/rtx-csm/examples/converse_server.rs

@@ -97,15 +97,22 @@ struct Cli {
     #[arg(long)]
     emotion_hint: Option<String>,
 
-    /// Detect the user's emotion from each incoming audio buffer (Phase
+    /// Detect the user's emotion from each incoming audio buffer and use
-    /// 13.3 prosody-rule classifier) and use the resulting tag as the
+    /// the resulting tag as the per-turn emotion_hint. Default classifier
-    /// per-turn emotion_hint. Crude — the prosody-rule detector is a
+    /// is the Phase 13.3 prosody-rule placeholder; pair with
-    /// placeholder; expect a real signal once emotion2vec_plus_base is
+    /// `--use-emotion2vec` for the real emotion2vec_plus_base classifier
-    /// ported to candle. Static `--emotion-hint` is the fallback when
+    /// (Phase 13.8). Static `--emotion-hint` is the fallback when
     /// detection returns Neutral.
     #[arg(long, default_value_t = false)]
     reactive_emotion: bool,
 
+    /// With `--reactive-emotion`, swap the prosody-rule classifier for
+    /// the emotion2vec_plus_base candle port. Adds a one-time ~1.1 GB
+    /// download on first run and ~150 ms per turn vs <1 ms for prosody
+    /// rules, in exchange for SOTA-comparable accuracy.
+    #[arg(long, default_value_t = false)]
+    use_emotion2vec: bool,
+
     /// When `--reactive-emotion` produces a non-Neutral label, ALSO append
     /// a per-turn signal to the LLM-facing user message so the LLM's
     /// response text adapts (not just TTS prosody). Per turn only — the
@@ -484,11 +491,11 @@ struct Shared {
     /// useful when the loaded LoRA was fine-tuned with the same tag (Phase 12.2
     /// plumbing); on the un-adapted base it is a no-op cosmetic prefix.
     emotion_hint: Option<String>,
-    /// When true, classify the user's incoming audio per turn (Phase 13.3
+    /// Pre-built reactive-emotion classifier shared across turns. Boxed
-    /// ProsodyDetector) and use the detected tag as `emotion_hint` —
+    /// `dyn` so the same call site handles both ProsodyDetector (cheap,
-    /// reactive-emotion conditioning. `emotion_hint` above is the fallback
+    /// zero-init) and Emotion2Vec (loaded once on boot, ~150 ms/turn).
-    /// when detection yields Neutral.
+    /// `Some` ⇔ `--reactive-emotion` was set; `None` disables the path.
-    reactive_emotion: bool,
+    emotion_detector: Option<Box<dyn rtx_csm::ser::EmotionDetector + Send + Sync>>,
     /// When true (and `reactive_emotion` is also true), append a per-turn
     /// emotion-signal annotation to the LLM-facing user message so the
     /// response text adapts in addition to the TTS prosody. Per-turn,
@@ -825,6 +832,27 @@ async fn main() -> Result<()> {
         );
     }
 
+    // Build the reactive-emotion classifier once at boot. Cheap for
+    // ProsodyDetector; pays the ~1.1 GB download + Metal allocation
+    // upfront for Emotion2Vec so per-turn cost is just ~150 ms forward.
+    let emotion_detector: Option<Box<dyn rtx_csm::ser::EmotionDetector + Send + Sync>> =
+        if cli.reactive_emotion {
+            if cli.use_emotion2vec {
+                let api = hf_hub::api::sync::Api::new()?;
+                let path = api
+                    .model("emotion2vec/emotion2vec_plus_base".into())
+                    .get("model.pt")
+                    .context("download emotion2vec_plus_base/model.pt")?;
+                let model = rtx_csm::emotion2vec::Emotion2Vec::load_from_pickle(&path, &device)?;
+                tracing::info!("loaded emotion2vec_plus_base from {}", path.display());
+                Some(Box::new(model))
+            } else {
+                Some(Box::new(rtx_csm::ser::ProsodyDetector::default()))
+            }
+        } else {
+            None
+        };
+
     let shared = Arc::new(Shared {
         generator: Mutex::new(generator),
         stt: Mutex::new(stt),
@@ -853,7 +881,7 @@ async fn main() -> Result<()> {
             None
         },
         emotion_hint: cli.emotion_hint.clone(),
-        reactive_emotion: cli.reactive_emotion,
+        emotion_detector,
         emotion_aware_llm: cli.emotion_aware_llm,
         metrics: Metrics::default(),
     });
@@ -1315,7 +1343,7 @@ async fn handle_connection(mut socket: WebSocket, shared: Arc<Shared>) {
         // Reuses the audio buffer STT just consumed. ProsodyDetector
         // operates at 16 kHz so we resample once on the fly; cheap for a
         // ~10 s buffer and only paid when --reactive-emotion is on.
-        let reactive_tag: Option<String> = if shared.reactive_emotion {
+        let reactive_tag: Option<String> = if let Some(det) = shared.emotion_detector.as_ref() {
             // Trim the 2 s silence pad we appended for STT before
             // classifying — silence drags the voiced ratio down.
             let trim_to = user_audio_24k
@@ -1328,8 +1356,8 @@ async fn handle_connection(mut socket: WebSocket, shared: Arc<Shared>) {
                 .fetch_add(1, Ordering::Relaxed);
             match rtx_csm::audio_io::resample(speech, PCM_RATE, 16_000) {
                 Ok(speech_16k) => {
-                    let det = rtx_csm::ser::ProsodyDetector::default();
+                    let label = det
-                    let label = rtx_csm::ser::EmotionDetector::classify(&det, &speech_16k)
+                        .classify(&speech_16k)
                         .unwrap_or(rtx_csm::ser::EmotionLabel::Neutral);
                     // Per-label counters.
                     let bucket = match label {

crates/models/rtx-csm/src/emotion2vec.rs

@@ -539,6 +539,11 @@ impl Module for RelativePositionalEncoder {
     }
 }
 
+/// Emotion2Vec stores its inference device alongside the modules so
+/// [`crate::ser::EmotionDetector::classify`] (which only takes `&[f32]`)
+/// can build input tensors without an out-of-band device handle.
+fn _device_struct_doc() {}
+
 /// Top-level emotion2vec_plus_base model. Wires:
 ///   1. [`LocalEncoder`] (raw 16 kHz audio → 50 Hz, 512-d features)
 ///   2. [`ProjectFeatures`] (LN + 512→768 Linear)
@@ -559,6 +564,7 @@ pub struct Emotion2Vec {
     context_encoder: ContextEncoder,
     main_encoder: MainEncoder,
     classifier: Classifier,
+    device: candle_core::Device,
 }
 
 impl Emotion2Vec {
@@ -568,7 +574,11 @@ impl Emotion2Vec {
     ///
     /// `kernel` and `groups` for the relative positional encoder default
     /// to the `_base` config (kernel 19, groups 16).
-    pub fn new(cfg: Emotion2VecConfig, vb: VarBuilder) -> CsmResult<Self> {
+    pub fn new(
+        cfg: Emotion2VecConfig,
+        vb: VarBuilder,
+        device: candle_core::Device,
+    ) -> CsmResult<Self> {
         // d2v_model.* sub-builder
         let vb_d = vb.pp("d2v_model");
         let vb_mod = vb_d.pp("modality_encoders").pp("AUDIO");
@@ -595,6 +605,7 @@ impl Emotion2Vec {
             context_encoder,
             main_encoder,
             classifier,
+            device,
         })
     }
 
@@ -612,7 +623,7 @@ impl Emotion2Vec {
             device,
         )
         .map_err(|e| crate::CsmError::Config(format!("emotion2vec pth load: {e}")))?;
-        Self::new(Emotion2VecConfig::plus_base(), vb)
+        Self::new(Emotion2VecConfig::plus_base(), vb, device.clone())
     }
 
     /// Forward pass: raw audio → 9-class logits.
@@ -638,6 +649,41 @@ impl Emotion2Vec {
     pub fn config(&self) -> &Emotion2VecConfig {
         &self.cfg
     }
+
+    pub fn device(&self) -> &candle_core::Device {
+        &self.device
+    }
+}
+
+/// Implementation of the [`crate::ser::EmotionDetector`] trait so
+/// `Emotion2Vec` is a drop-in replacement for the Phase 13.3
+/// [`crate::ser::ProsodyDetector`] placeholder. Used by
+/// `audio_to_manifest --auto-emotion-tag` and
+/// `converse_server --reactive-emotion` once the user opts in via
+/// `--use-emotion2vec`.
+impl crate::ser::EmotionDetector for Emotion2Vec {
+    fn classify(&self, samples_16k: &[f32]) -> CsmResult<crate::ser::EmotionLabel> {
+        if samples_16k.is_empty() {
+            return Ok(crate::ser::EmotionLabel::Neutral);
+        }
+        let n = samples_16k.len();
+        let audio = Tensor::from_slice(samples_16k, (1, 1, n), &self.device)
+            .map_err(|e| crate::CsmError::Config(format!("emotion2vec audio tensor: {e}")))?;
+        let logits = self
+            .forward(&audio)
+            .map_err(|e| crate::CsmError::Config(format!("emotion2vec forward: {e}")))?;
+        let logits = logits
+            .flatten_all()
+            .and_then(|t| t.to_vec1::<f32>())
+            .map_err(|e| crate::CsmError::Config(format!("emotion2vec logits: {e}")))?;
+        let argmax = logits
+            .iter()
+            .enumerate()
+            .max_by(|(_, a), (_, b)| a.partial_cmp(b).unwrap_or(std::cmp::Ordering::Equal))
+            .map(|(i, _)| i)
+            .unwrap_or(0);
+        Ok(Classifier::tag_for_class(argmax as u32))
+    }
 }
 
 #[cfg(test)]
@@ -752,7 +798,7 @@ mod tests {
         let cfg = Emotion2VecConfig::plus_base();
         let vm = VarMap::new();
         let vb = VarBuilder::from_varmap(&vm, DType::F32, &dev);
-        let model = Emotion2Vec::new(cfg.clone(), vb).expect("build");
+        let model = Emotion2Vec::new(cfg.clone(), vb, dev.clone()).expect("build");
         // 1 second of synthetic audio. Random init won't give meaningful
         // emotion predictions but shape + finiteness should hold.
         let audio = Tensor::randn(0f32, 0.1, (1, 1, 16_000), &dev).unwrap();