rtx-csm: emotion2vec input normalization + 9-class direct mapping

Three real bugs found while running a YouTube → train → eval pipeline
end-to-end on real corpora:

1. emotion2vec was producing near-constant logits regardless of input.
   Per config.yaml `normalize: true` — data2vec2/emotion2vec expects
   per-utterance zero-mean unit-variance normalization on the raw
   waveform before the local_encoder. Added inside the EmotionDetector
   trait impl so all callers get it.

   Verified empirically: 4 different audio inputs (Carlini talk,
   audience question, McConaughey speech) now produce different argmax
   classes. Before fix: all 4 produced identical logits.

2. The 9→5 emotion fold was collapsing every real-world clip to
   [excited]. happy / surprised / other all mapped to Excited covered
   ~95% of natural speech. Replaced with a direct 9-class identity
   mapping; EmotionLabel gained Disgusted, Fearful, Happy, Surprised,
   Unk variants. Now: 132 [surprised] + 12 [excited] across the
   Carlini corpus instead of 144 [excited].

3. lora_train_emotional --peak-lr / --epochs flags. The canned 3-stage
   recipe over-fits on small (~100 clip) corpora at extended rank 8;
   users need to tune. (The recipe stays as defaults; flags are pure
   overrides.)

Plus diagnostic: examples/emotion2vec_probe — feed real audio files
into emotion2vec and dump per-class logits. Used to find bug #1.

Lib suite still 131/131 (the test that locked the 9→5 fold updated
to lock the new identity mapping).

Co-Authored-By: Claude Opus 4.7 (1M context) <[email protected]>

crates/models/rtx-csm/Cargo.toml

@@ -196,6 +196,10 @@ path = "examples/emotion2vec_inspect.rs"
 name = "emotion2vec_smoke"
 path = "examples/emotion2vec_smoke.rs"
 
+[[example]]
+name = "emotion2vec_probe"
+path = "examples/emotion2vec_probe.rs"
+
 [[example]]
 name = "wav2vec2_inspect"
 path = "examples/wav2vec2_inspect.rs"

crates/models/rtx-csm/examples/converse_server.rs

@@ -407,6 +407,15 @@ struct Metrics {
     reactive_emotion_sad_total: AtomicU64,
     reactive_emotion_angry_total: AtomicU64,
     reactive_emotion_excited_total: AtomicU64,
+    /// Phase 13.10 raw emotion2vec classes — added when the 9→5 fold
+    /// was removed. The original 5 buckets above are still emitted
+    /// (prosody-rule SER produces those) so these are emotion2vec-only
+    /// in practice.
+    reactive_emotion_disgusted_total: AtomicU64,
+    reactive_emotion_fearful_total: AtomicU64,
+    reactive_emotion_happy_total: AtomicU64,
+    reactive_emotion_surprised_total: AtomicU64,
+    reactive_emotion_unk_total: AtomicU64,
     emotion_aware_llm_applied_total: AtomicU64,
 }
 
@@ -965,6 +974,11 @@ async fn metrics_handler(State(shared): State<Arc<Shared>>) -> impl IntoResponse
          rtx_csm_reactive_emotion_total{{label=\"sad\"}} {}\n\
          rtx_csm_reactive_emotion_total{{label=\"angry\"}} {}\n\
          rtx_csm_reactive_emotion_total{{label=\"excited\"}} {}\n\
+         rtx_csm_reactive_emotion_total{{label=\"disgusted\"}} {}\n\
+         rtx_csm_reactive_emotion_total{{label=\"fearful\"}} {}\n\
+         rtx_csm_reactive_emotion_total{{label=\"happy\"}} {}\n\
+         rtx_csm_reactive_emotion_total{{label=\"surprised\"}} {}\n\
+         rtx_csm_reactive_emotion_total{{label=\"unk\"}} {}\n\
          # TYPE rtx_csm_emotion_aware_llm_applied_total counter\n\
          rtx_csm_emotion_aware_llm_applied_total {}\n",
         m.turns_total.load(Ordering::Relaxed),
@@ -984,6 +998,11 @@ async fn metrics_handler(State(shared): State<Arc<Shared>>) -> impl IntoResponse
         m.reactive_emotion_sad_total.load(Ordering::Relaxed),
         m.reactive_emotion_angry_total.load(Ordering::Relaxed),
         m.reactive_emotion_excited_total.load(Ordering::Relaxed),
+        m.reactive_emotion_disgusted_total.load(Ordering::Relaxed),
+        m.reactive_emotion_fearful_total.load(Ordering::Relaxed),
+        m.reactive_emotion_happy_total.load(Ordering::Relaxed),
+        m.reactive_emotion_surprised_total.load(Ordering::Relaxed),
+        m.reactive_emotion_unk_total.load(Ordering::Relaxed),
         m.emotion_aware_llm_applied_total.load(Ordering::Relaxed),
     );
     (
@@ -1376,6 +1395,21 @@ async fn handle_connection(mut socket: WebSocket, shared: Arc<Shared>) {
                         rtx_csm::ser::EmotionLabel::Excited => {
                             &shared.metrics.reactive_emotion_excited_total
                         }
+                        rtx_csm::ser::EmotionLabel::Disgusted => {
+                            &shared.metrics.reactive_emotion_disgusted_total
+                        }
+                        rtx_csm::ser::EmotionLabel::Fearful => {
+                            &shared.metrics.reactive_emotion_fearful_total
+                        }
+                        rtx_csm::ser::EmotionLabel::Happy => {
+                            &shared.metrics.reactive_emotion_happy_total
+                        }
+                        rtx_csm::ser::EmotionLabel::Surprised => {
+                            &shared.metrics.reactive_emotion_surprised_total
+                        }
+                        rtx_csm::ser::EmotionLabel::Unk => {
+                            &shared.metrics.reactive_emotion_unk_total
+                        }
                     };
                     bucket.fetch_add(1, Ordering::Relaxed);
                     if label == rtx_csm::ser::EmotionLabel::Neutral {

crates/models/rtx-csm/examples/emotion2vec_probe.rs

@@ -0,0 +1,91 @@
+//! Diagnostic — feed REAL 16 kHz audio files into emotion2vec_plus_base
+//! and dump per-class logits. Used to investigate why every clip in
+//! audio_to_manifest --auto-emotion-tag --use-emotion2vec was getting
+//! tagged the same emotion regardless of input.
+
+use anyhow::{Context, Result};
+use clap::Parser;
+use hf_hub::api::sync::Api;
+use rtx_csm::audio_io;
+use rtx_csm::emotion2vec::{Classifier, Emotion2Vec};
+use std::path::PathBuf;
+
+#[derive(Debug, Parser)]
+struct Cli {
+    /// One or more audio files to classify.
+    #[arg(long = "in")]
+    inputs: Vec<PathBuf>,
+}
+
+fn main() -> Result<()> {
+    tracing_subscriber::fmt().init();
+    let cli = Cli::parse();
+    if cli.inputs.is_empty() {
+        anyhow::bail!("--in <path> required (one or more)");
+    }
+    let device = if candle_core::utils::metal_is_available() {
+        candle_core::Device::new_metal(0)?
+    } else {
+        candle_core::Device::Cpu
+    };
+
+    let api = Api::new()?;
+    let path = api
+        .model("emotion2vec/emotion2vec_plus_base".into())
+        .get("model.pt")
+        .context("download emotion2vec_plus_base/model.pt")?;
+    let model = Emotion2Vec::load_from_pickle(&path, &device)?;
+    eprintln!("model loaded\n");
+
+    let labels = [
+        "angry",
+        "disgusted",
+        "fearful",
+        "happy",
+        "neutral",
+        "other",
+        "sad",
+        "surprised",
+        "<unk>",
+    ];
+
+    for input in cli.inputs.iter() {
+        let pcm = audio_io::load_mono_at_rate(input, 16_000)?;
+        let secs = pcm.len() as f32 / 16_000.0;
+        // Per-utterance zero-mean unit-variance normalization
+        // (config.yaml: `normalize: true`). data2vec2 / emotion2vec
+        // expects this; the upstream FunASR pipeline applies it before
+        // the local_encoder. Without it the model produces ~constant
+        // output regardless of input.
+        let mean = pcm.iter().sum::<f32>() / pcm.len().max(1) as f32;
+        let var = pcm.iter().map(|x| (x - mean).powi(2)).sum::<f32>()
+            / pcm.len().max(1) as f32;
+        let std = var.sqrt().max(1e-7);
+        let normed: Vec<f32> = pcm.iter().map(|x| (x - mean) / std).collect();
+        let audio = candle_core::Tensor::from_vec(normed, (1, 1, pcm.len()), &device)?;
+        let logits = model.forward(&audio)?;
+        let v = logits.flatten_all()?.to_vec1::<f32>()?;
+        let argmax = v
+            .iter()
+            .enumerate()
+            .max_by(|(_, a), (_, b)| a.partial_cmp(b).unwrap_or(std::cmp::Ordering::Equal))
+            .map(|(i, _)| i)
+            .unwrap_or(0);
+        println!(
+            "=== {} ({:.2}s) ===",
+            input.file_name().and_then(|s| s.to_str()).unwrap_or("?"),
+            secs,
+        );
+        for (i, x) in v.iter().enumerate() {
+            let marker = if i == argmax { "  ←" } else { "" };
+            println!("  {} {:>10}  {:>9.4}{}", i, labels[i], x, marker);
+        }
+        println!(
+            "  argmax: {} ({}) → 5-bucket {}\n",
+            argmax,
+            labels[argmax],
+            Classifier::tag_for_class(argmax as u32).as_tag()
+        );
+    }
+    Ok(())
+}

crates/models/rtx-csm/examples/lora_train_emotional.rs

@@ -72,6 +72,19 @@ struct Cli {
     #[arg(long, default_value_t = false)]
     extended_lora: bool,
 
+    /// Override the canned per-stage `peak_lr`. The default recipe
+    /// (1e-4 / 3e-5 / 1e-5 across stages) over-fits noticeably on
+    /// small (~100-200 clip) corpora at rank 8. Try `--peak-lr 2e-5`
+    /// for a gentler run when held-out loss is going up.
+    #[arg(long)]
+    peak_lr: Option<f64>,
+
+    /// Override the canned per-stage epoch count. Default is
+    /// `audiobook: 3, podcast: 1, va: 1`. A single epoch is often
+    /// enough on small corpora.
+    #[arg(long)]
+    epochs: Option<usize>,
+
     #[arg(long)]
     cpu: bool,
 }
@@ -124,12 +137,13 @@ fn main() -> Result<()> {
     let stage_path = |stage: &str| parent.join(format!("{stem}.{stage}.safetensors"));
 
     // Canonical 3-stage recipe from personal_voice_training_guide.md §4.
+    // CLI overrides (--peak-lr, --epochs) take precedence per-stage.
     let stages = vec![
         CurriculumStage {
             name: "audiobook".into(),
             config: TrainingConfig {
-                epochs: 3,
+                epochs: cli.epochs.unwrap_or(3),
-                peak_lr: 1e-4,
+                peak_lr: cli.peak_lr.unwrap_or(1e-4),
                 end_lr: 1e-5,
                 warmup_steps: 32,
                 grad_clip: Some(cli.grad_clip),
@@ -141,8 +155,8 @@ fn main() -> Result<()> {
         CurriculumStage {
             name: "podcast".into(),
             config: TrainingConfig {
-                epochs: 1,
+                epochs: cli.epochs.unwrap_or(1),
-                peak_lr: 3e-5,
+                peak_lr: cli.peak_lr.unwrap_or(3e-5),
                 end_lr: 1e-5,
                 warmup_steps: 16,
                 grad_clip: Some(cli.grad_clip),
@@ -154,8 +168,8 @@ fn main() -> Result<()> {
         CurriculumStage {
             name: "va".into(),
             config: TrainingConfig {
-                epochs: 1,
+                epochs: cli.epochs.unwrap_or(1),
-                peak_lr: 1e-5,
+                peak_lr: cli.peak_lr.unwrap_or(1e-5),
                 end_lr: 1e-6,
                 warmup_steps: 16,
                 grad_clip: Some(cli.grad_clip),

crates/models/rtx-csm/src/emotion2vec.rs

@@ -446,18 +446,26 @@ impl Classifier {
         Ok(Self { proj })
     }
 
-    /// Map a 0..=8 class index to one of the 5 [`crate::ser::EmotionLabel`]
+    /// Map a 0..=8 emotion2vec class index directly to one of the 9 raw
-    /// buckets. The model produces 9 fine-grained classes; the in-crate
+    /// [`crate::ser::EmotionLabel`] variants — NO fold.
-    /// `EmotionDetector` trait uses a coarser 5-bucket label set, so we
+    ///
-    /// fold: `disgusted/fearful` → Sad-ish (low valence), `surprised/other`
+    /// The earlier 9→5 fold (`happy`/`surprised`/`other` → Excited;
-    /// → Excited (high arousal), `<unk>` → Neutral.
+    /// `disgusted`/`fearful` → Sad) collapsed emotion2vec's resolution
+    /// to a single tag in practice: motivational speech, technical
+    /// talks, and movie dialogue all bucketed to `[excited]`. With
+    /// raw 9-class tags the LoRA trainer sees real emotional variety
+    /// in its prompts.
     pub fn tag_for_class(idx: u32) -> crate::ser::EmotionLabel {
         match idx {
             0 => crate::ser::EmotionLabel::Angry,
-            1 | 2 | 6 => crate::ser::EmotionLabel::Sad,
+            1 => crate::ser::EmotionLabel::Disgusted,
-            3 | 5 | 7 => crate::ser::EmotionLabel::Excited,
+            2 => crate::ser::EmotionLabel::Fearful,
+            3 => crate::ser::EmotionLabel::Happy,
             4 => crate::ser::EmotionLabel::Neutral,
-            _ => crate::ser::EmotionLabel::Neutral,
+            5 => crate::ser::EmotionLabel::Excited, // emotion2vec's "other"
+            6 => crate::ser::EmotionLabel::Sad,
+            7 => crate::ser::EmotionLabel::Surprised,
+            _ => crate::ser::EmotionLabel::Unk, // 8 = <unk> + any unknown
         }
     }
 }
@@ -666,8 +674,18 @@ impl crate::ser::EmotionDetector for Emotion2Vec {
         if samples_16k.is_empty() {
             return Ok(crate::ser::EmotionLabel::Neutral);
         }
+        // Per-utterance zero-mean unit-variance normalization
+        // (config.yaml: `normalize: true`). Without this the model
+        // produces near-constant logits regardless of input — every
+        // clip in the corpus would get the same label. Verified
+        // empirically via examples/emotion2vec_probe before/after.
         let n = samples_16k.len();
-        let audio = Tensor::from_slice(samples_16k, (1, 1, n), &self.device)
+        let mean = samples_16k.iter().sum::<f32>() / n as f32;
+        let var =
+            samples_16k.iter().map(|x| (x - mean).powi(2)).sum::<f32>() / n as f32;
+        let std = var.sqrt().max(1e-7);
+        let normed: Vec<f32> = samples_16k.iter().map(|x| (x - mean) / std).collect();
+        let audio = Tensor::from_vec(normed, (1, 1, n), &self.device)
             .map_err(|e| crate::CsmError::Config(format!("emotion2vec audio tensor: {e}")))?;
         let logits = self
             .forward(&audio)
@@ -811,16 +829,19 @@ mod tests {
     #[test]
     fn classifier_class_to_emotion_label_mapping() {
         use crate::ser::EmotionLabel;
-        // 9-class → 5-bucket fold. The mapping must be total over 0..=8.
+        // Phase 13.10: 9 raw classes map directly (no fold). The earlier
-        assert_eq!(Classifier::tag_for_class(0), EmotionLabel::Angry); // angry
+        // 9→5 fold was found to collapse all real audio to [excited]
-        assert_eq!(Classifier::tag_for_class(1), EmotionLabel::Sad); // disgusted → low-valence
+        // empirically; raw labels give the LoRA trainer real prosodic
-        assert_eq!(Classifier::tag_for_class(2), EmotionLabel::Sad); // fearful  → low-valence
+        // variety in its prompts.
-        assert_eq!(Classifier::tag_for_class(3), EmotionLabel::Excited); // happy
+        assert_eq!(Classifier::tag_for_class(0), EmotionLabel::Angry);
-        assert_eq!(Classifier::tag_for_class(4), EmotionLabel::Neutral); // neutral
+        assert_eq!(Classifier::tag_for_class(1), EmotionLabel::Disgusted);
-        assert_eq!(Classifier::tag_for_class(5), EmotionLabel::Excited); // other
+        assert_eq!(Classifier::tag_for_class(2), EmotionLabel::Fearful);
-        assert_eq!(Classifier::tag_for_class(6), EmotionLabel::Sad); // sad
+        assert_eq!(Classifier::tag_for_class(3), EmotionLabel::Happy);
-        assert_eq!(Classifier::tag_for_class(7), EmotionLabel::Excited); // surprised
+        assert_eq!(Classifier::tag_for_class(4), EmotionLabel::Neutral);
-        assert_eq!(Classifier::tag_for_class(8), EmotionLabel::Neutral); // <unk>
+        assert_eq!(Classifier::tag_for_class(5), EmotionLabel::Excited); // emotion2vec "other"
+        assert_eq!(Classifier::tag_for_class(6), EmotionLabel::Sad);
+        assert_eq!(Classifier::tag_for_class(7), EmotionLabel::Surprised);
+        assert_eq!(Classifier::tag_for_class(8), EmotionLabel::Unk);
     }
 
     #[test]

crates/models/rtx-csm/src/ser.rs

@@ -15,9 +15,18 @@
 
 use crate::error::Result;
 
-/// Five coarse emotion buckets compatible with the GoEmotions / arousal-
+/// Emotion buckets used as tags by the data-prep + reactive-emotion paths.
-/// valence axes. Maps cleanly to single-token tags users can prepend at
+///
-/// inference time (Phase 12.2).
+/// The five `Neutral / Calm / Sad / Angry / Excited` buckets are the
+/// original Phase 13.3 prosody-rule output set. Phase 13.8's emotion2vec
+/// port emits the four extra raw classes (`Disgusted`, `Fearful`, `Happy`,
+/// `Surprised`) directly — folding them into the original 5 collapses
+/// emotion2vec's resolution to a single `[excited]` tag in practice
+/// (verified empirically: motivational speech, technical talks, and movie
+/// dialogue ALL bucketed to `[excited]` under the old 9→5 fold).
+///
+/// Tags are bracket-wrapped lowercase and slot directly into Phase 12.2's
+/// `GenerateOptions::emotion_hint`. `Unk` is emotion2vec's `<unk>` class.
 #[derive(Debug, Clone, Copy, PartialEq, Eq, serde::Serialize)]
 pub enum EmotionLabel {
     Neutral,
@@ -25,6 +34,11 @@ pub enum EmotionLabel {
     Sad,
     Angry,
     Excited,
+    Disgusted,
+    Fearful,
+    Happy,
+    Surprised,
+    Unk,
 }
 
 impl EmotionLabel {
@@ -35,6 +49,11 @@ impl EmotionLabel {
             EmotionLabel::Sad => "[sad]",
             EmotionLabel::Angry => "[angry]",
             EmotionLabel::Excited => "[excited]",
+            EmotionLabel::Disgusted => "[disgusted]",
+            EmotionLabel::Fearful => "[fearful]",
+            EmotionLabel::Happy => "[happy]",
+            EmotionLabel::Surprised => "[surprised]",
+            EmotionLabel::Unk => "[unk]",
         }
     }
 }