rtx-csm: Phase 12.7 — lora_eval held-out quality harness

Closes the train→generate→evaluate cycle. Users can now produce a hard
quality number for any adapter without listening manually.

evaluate_held_out runs teacher-forced forward_loss over a JSONL manifest
(same format as 12.3 trainer). Uses apply_emotion_hint so eval prompts
match training prompts. Frame sampling is seed-controlled — identical
seeds across runs score the same frames in the same order, which is what
makes a base-vs-LoRA A/B fair.

EvalRow + EvalSummary types, both serde-Serialize for JSON output.

examples/lora_eval.rs wraps it: --eval-manifest --report [--lora].
Documented usage: run twice with the same seed, diff the summary blocks.

Verified end-to-end on the existing 3-row curriculum manifest with
seed=42, frames-per-example=4: LoRA shifted mean/median/p90 loss
directionally in its favor (-0.011/-0.018/-0.007). Tiny because the
test adapter only saw ~10 training steps, but the eval signal is real
and the A/B path is wired.

Lib suite 99/99.

Co-Authored-By: Claude Opus 4.7 (1M context) <[email protected]>

crates/models/rtx-csm/Cargo.toml

@@ -176,6 +176,10 @@ path = "examples/lora_train.rs"
 name = "lora_train_emotional"
 path = "examples/lora_train_emotional.rs"
 
+[[example]]
+name = "lora_eval"
+path = "examples/lora_eval.rs"
+
 [[example]]
 name = "audioseal_inspect"
 path = "examples/audioseal_inspect.rs"

crates/models/rtx-csm/examples/lora_eval.rs

@@ -0,0 +1,151 @@
+//! Held-out cross-entropy evaluator for LoRA adapters.
+//!
+//! Runs teacher-forced `forward_loss` over a held-out manifest (same JSONL
+//! format as `lora_train_emotional`) and writes a JSON report. Run it twice
+//! for an A/B:
+//!
+//! ```bash
+//! # baseline (no adapter)
+//! lora_eval --eval-manifest test.jsonl --report /tmp/base.json
+//!
+//! # with adapter
+//! lora_eval --eval-manifest test.jsonl --lora /tmp/voice.safetensors \
+//!     --report /tmp/lora.json
+//!
+//! # compare summaries
+//! diff <(jq .summary /tmp/base.json) <(jq .summary /tmp/lora.json)
+//! ```
+//!
+//! Lower mean / median / p90 loss = adapter is fitting the held-out audio
+//! better. If LoRA loss is higher than base, your adapter has overfit the
+//! training set.
+
+use anyhow::Result;
+use clap::Parser;
+use rtx_csm::training::{evaluate_held_out, EvalSummary, TrainingDataset};
+use rtx_csm::Generator;
+use std::path::PathBuf;
+
+#[derive(Debug, Parser)]
+struct Cli {
+    /// JSONL manifest of held-out (wav, transcript, [emotion_tag]) rows.
+    /// Same format as `lora_train_emotional`.
+    #[arg(long)]
+    eval_manifest: PathBuf,
+
+    /// Output JSON report path.
+    #[arg(long)]
+    report: PathBuf,
+
+    /// Optional LoRA adapter to apply before scoring. When omitted, the
+    /// base CSM-1B is scored — that's the report you compare against.
+    #[arg(long)]
+    lora: Option<PathBuf>,
+
+    /// Force LoRA hyperparameters (overrides metadata). See `apply_lora_adapter`.
+    #[arg(long)]
+    lora_rank: Option<usize>,
+    #[arg(long)]
+    lora_alpha: Option<f32>,
+    #[arg(long, default_value_t = false)]
+    extended_lora: bool,
+
+    /// Optional quantized GGUF (Q8/Q4_K_M). Combines with `--lora` per Phase 12.6.
+    #[arg(long)]
+    quantized_gguf: Option<PathBuf>,
+
+    /// Default speaker id when a manifest row doesn't specify one.
+    #[arg(long, default_value_t = 0)]
+    speaker: u32,
+
+    /// Frames sampled per example. 0 = score every frame (slow).
+    #[arg(long, default_value_t = 8)]
+    frames_per_example: usize,
+
+    /// RNG seed for frame sampling. Two runs with the same seed score the
+    /// same frames in the same order — required for a fair A/B.
+    #[arg(long, default_value_t = 42)]
+    seed: u64,
+
+    #[arg(long)]
+    cpu: bool,
+}
+
+#[derive(Debug, serde::Serialize)]
+struct Report {
+    /// What was evaluated: base or path-to-adapter (or path with quantized base).
+    label: String,
+    n_examples: usize,
+    frames_per_example_cap: usize,
+    seed: u64,
+    summary: EvalSummary,
+    rows: Vec<rtx_csm::training::EvalRow>,
+}
+
+fn main() -> Result<()> {
+    tracing_subscriber::fmt().init();
+    let cli = Cli::parse();
+
+    let device = if cli.cpu {
+        candle_core::Device::Cpu
+    } else {
+        Generator::default_device()?
+    };
+    println!("device: {device:?}");
+
+    let mut generator = if let Some(gguf) = cli.quantized_gguf.as_ref() {
+        Generator::load_csm_1b_quantized(gguf, &device, /* enable_cfg */ false)?
+    } else {
+        Generator::load_csm_1b(&device)?
+    };
+    println!("model loaded");
+
+    let label = if let Some(lora_path) = cli.lora.as_ref() {
+        let extended_override = if cli.extended_lora { Some(true) } else { None };
+        rtx_csm::training::apply_lora_adapter(
+            &mut generator,
+            lora_path,
+            cli.lora_rank,
+            cli.lora_alpha,
+            extended_override,
+            &device,
+        )?;
+        format!("lora:{}", lora_path.display())
+    } else {
+        "base".to_string()
+    };
+
+    println!("loading eval manifest from {}", cli.eval_manifest.display());
+    let dataset = TrainingDataset::load_from_manifest(
+        &cli.eval_manifest,
+        cli.speaker,
+        &mut generator,
+    )?;
+    println!("dataset: {} examples", dataset.len());
+
+    let rows = evaluate_held_out(&mut generator, &dataset, cli.frames_per_example, cli.seed)?;
+    let summary = EvalSummary::from_rows(&rows);
+    println!(
+        "summary: n={} frames={} mean={:.4} median={:.4} p90={:.4} min={:.4} max={:.4}",
+        summary.n_examples,
+        summary.n_frames_total,
+        summary.mean_loss,
+        summary.median_loss,
+        summary.p90_loss,
+        summary.min_loss,
+        summary.max_loss,
+    );
+
+    let report = Report {
+        label,
+        n_examples: rows.len(),
+        frames_per_example_cap: cli.frames_per_example,
+        seed: cli.seed,
+        summary,
+        rows,
+    };
+    let json = serde_json::to_string_pretty(&report)?;
+    std::fs::write(&cli.report, json)?;
+    println!("\n✓ wrote report to {}", cli.report.display());
+    Ok(())
+}

crates/models/rtx-csm/src/training.rs

@@ -514,6 +514,140 @@ fn clip_grads(
     Ok(())
 }
 
+/// One-example metric from the held-out evaluator.
+#[derive(Debug, Clone, serde::Serialize)]
+pub struct EvalRow {
+    /// Position in the input dataset.
+    pub idx: usize,
+    /// Source wav (when known) — round-trips through the manifest loader.
+    pub source: Option<PathBuf>,
+    /// Number of frames in the example's Mimi-encoded audio.
+    pub frames: usize,
+    /// Mean teacher-forced cross-entropy across all 32 codebooks, averaged
+    /// over `frames_sampled` frames drawn at random from the example.
+    pub mean_loss: f32,
+    /// Number of frames the loss was averaged over (capped at the example's
+    /// available frames).
+    pub frames_sampled: usize,
+}
+
+/// Aggregate summary of an `evaluate_held_out` run. Useful for
+/// machine-readable A/B comparison between base and LoRA-applied runs.
+#[derive(Debug, Clone, serde::Serialize)]
+pub struct EvalSummary {
+    pub n_examples: usize,
+    pub n_frames_total: usize,
+    pub mean_loss: f32,
+    pub median_loss: f32,
+    pub p90_loss: f32,
+    pub min_loss: f32,
+    pub max_loss: f32,
+}
+
+impl EvalSummary {
+    pub fn from_rows(rows: &[EvalRow]) -> Self {
+        if rows.is_empty() {
+            return Self {
+                n_examples: 0,
+                n_frames_total: 0,
+                mean_loss: 0.0,
+                median_loss: 0.0,
+                p90_loss: 0.0,
+                min_loss: 0.0,
+                max_loss: 0.0,
+            };
+        }
+        let mut losses: Vec<f32> = rows.iter().map(|r| r.mean_loss).collect();
+        losses.sort_by(|a, b| a.partial_cmp(b).unwrap_or(std::cmp::Ordering::Equal));
+        let n_frames_total: usize = rows.iter().map(|r| r.frames_sampled).sum();
+        let mean = losses.iter().sum::<f32>() / losses.len() as f32;
+        let median = losses[losses.len() / 2];
+        let p90_idx = ((losses.len() as f64) * 0.9).ceil() as usize;
+        let p90 = losses[p90_idx.saturating_sub(1).min(losses.len() - 1)];
+        Self {
+            n_examples: rows.len(),
+            n_frames_total,
+            mean_loss: mean,
+            median_loss: median,
+            p90_loss: p90,
+            min_loss: *losses.first().unwrap(),
+            max_loss: *losses.last().unwrap(),
+        }
+    }
+}
+
+/// Run the teacher-forced `forward_loss` over a held-out dataset to get a
+/// scalar quality signal. Reuses the same prompt-build path the trainer uses
+/// (with `apply_emotion_hint` so eval and inference apply tags identically).
+///
+/// `frames_per_example` caps how many frames are scored per example;
+/// uniformly sampled with the given `seed` so two runs (base vs LoRA) over
+/// the same dataset score the same frames in the same order. Set to 0 to
+/// score every frame (slow on large examples).
+pub fn evaluate_held_out(
+    generator: &mut Generator,
+    dataset: &TrainingDataset,
+    frames_per_example: usize,
+    seed: u64,
+) -> Result<Vec<EvalRow>> {
+    use rand::rngs::StdRng;
+    use rand::{Rng, SeedableRng};
+
+    let mut rng = StdRng::seed_from_u64(seed);
+    let mut rows = Vec::with_capacity(dataset.len());
+    for (idx, ex) in dataset.examples.iter().enumerate() {
+        if ex.frame_codes.is_empty() {
+            continue;
+        }
+        let prompt_text = crate::generator::apply_emotion_hint(
+            ex.text.clone(),
+            ex.emotion_tag.as_deref(),
+        );
+        let current = Segment::new_text(ex.speaker, prompt_text);
+        let prompt = build_prompt(
+            &[],
+            &current,
+            &generator.model,
+            &mut generator.mimi,
+            &generator.tokenizer,
+        )?;
+        let total_frames = ex.frame_codes.len();
+        let n_sample = if frames_per_example == 0 {
+            total_frames
+        } else {
+            frames_per_example.min(total_frames)
+        };
+        let mut chosen = if frames_per_example == 0 {
+            (0..total_frames).collect::<Vec<_>>()
+        } else {
+            (0..n_sample).map(|_| rng.gen_range(0..total_frames)).collect()
+        };
+        chosen.sort();
+        let mut sum = 0.0f32;
+        for frame_idx in chosen.iter().copied() {
+            generator.model.clear_kv_cache();
+            let target = &ex.frame_codes[frame_idx];
+            let loss = generator
+                .model
+                .inner
+                .forward_loss(&prompt.tokens, &prompt.mask, 0, target)?;
+            sum += loss.to_scalar::<f32>()?;
+        }
+        let mean = if n_sample > 0 { sum / n_sample as f32 } else { 0.0 };
+        rows.push(EvalRow {
+            idx,
+            source: ex.source.clone(),
+            frames: total_frames,
+            mean_loss: mean,
+            frames_sampled: n_sample,
+        });
+        if (idx + 1) % 10 == 0 {
+            tracing::info!("eval progress: {}/{}", idx + 1, dataset.len());
+        }
+    }
+    Ok(rows)
+}
+
 /// Self-describing metadata embedded in safetensors at save time so the
 /// adapter file knows its own LoRA hyperparameters. Lets `apply_lora_adapter`
 /// auto-configure at inference without the user having to remember matching