rtx-csm: Phase 12.7 — lora_eval held-out quality harness

Closes the train→generate→evaluate cycle. Users can now produce a hard quality number for any adapter without listening manually. evaluate_held_out runs teacher-forced forward_loss over a JSONL manifest (same format as 12.3 trainer). Uses apply_emotion_hint so eval prompts match training prompts. Frame sampling is seed-controlled — identical seeds across runs score the same frames in the same order, which is what makes a base-vs-LoRA A/B fair. EvalRow + EvalSummary types, both serde-Serialize for JSON output. examples/lora_eval.rs wraps it: --eval-manifest --report [--lora]. Documented usage: run twice with the same seed, diff the summary blocks. Verified end-to-end on the existing 3-row curriculum manifest with seed=42, frames-per-example=4: LoRA shifted mean/median/p90 loss directionally in its favor (-0.011/-0.018/-0.007). Tiny because the test adapter only saw ~10 training steps, but the eval signal is real and the A/B path is wired. Lib suite 99/99. Co-Authored-By: Claude Opus 4.7 (1M context) <[email protected]>
2026-04-27 17:22:14 -07:00
parent 48df43810a
commit f7b6deac1a
3 changed files with 289 additions and 0 deletions
@@ -514,6 +514,140 @@ fn clip_grads(
    Ok(())
 }

+/// One-example metric from the held-out evaluator.
+#[derive(Debug, Clone, serde::Serialize)]
+pub struct EvalRow {
+    /// Position in the input dataset.
+    pub idx: usize,
+    /// Source wav (when known) — round-trips through the manifest loader.
+    pub source: Option<PathBuf>,
+    /// Number of frames in the example's Mimi-encoded audio.
+    pub frames: usize,
+    /// Mean teacher-forced cross-entropy across all 32 codebooks, averaged
+    /// over `frames_sampled` frames drawn at random from the example.
+    pub mean_loss: f32,
+    /// Number of frames the loss was averaged over (capped at the example's
+    /// available frames).
+    pub frames_sampled: usize,
+}
+
+/// Aggregate summary of an `evaluate_held_out` run. Useful for
+/// machine-readable A/B comparison between base and LoRA-applied runs.
+#[derive(Debug, Clone, serde::Serialize)]
+pub struct EvalSummary {
+    pub n_examples: usize,
+    pub n_frames_total: usize,
+    pub mean_loss: f32,
+    pub median_loss: f32,
+    pub p90_loss: f32,
+    pub min_loss: f32,
+    pub max_loss: f32,
+}
+
+impl EvalSummary {
+    pub fn from_rows(rows: &[EvalRow]) -> Self {
+        if rows.is_empty() {
+            return Self {
+                n_examples: 0,
+                n_frames_total: 0,
+                mean_loss: 0.0,
+                median_loss: 0.0,
+                p90_loss: 0.0,
+                min_loss: 0.0,
+                max_loss: 0.0,
+            };
+        }
+        let mut losses: Vec<f32> = rows.iter().map(|r| r.mean_loss).collect();
+        losses.sort_by(|a, b| a.partial_cmp(b).unwrap_or(std::cmp::Ordering::Equal));
+        let n_frames_total: usize = rows.iter().map(|r| r.frames_sampled).sum();
+        let mean = losses.iter().sum::<f32>() / losses.len() as f32;
+        let median = losses[losses.len() / 2];
+        let p90_idx = ((losses.len() as f64) * 0.9).ceil() as usize;
+        let p90 = losses[p90_idx.saturating_sub(1).min(losses.len() - 1)];
+        Self {
+            n_examples: rows.len(),
+            n_frames_total,
+            mean_loss: mean,
+            median_loss: median,
+            p90_loss: p90,
+            min_loss: *losses.first().unwrap(),
+            max_loss: *losses.last().unwrap(),
+        }
+    }
+}
+
+/// Run the teacher-forced `forward_loss` over a held-out dataset to get a
+/// scalar quality signal. Reuses the same prompt-build path the trainer uses
+/// (with `apply_emotion_hint` so eval and inference apply tags identically).
+///
+/// `frames_per_example` caps how many frames are scored per example;
+/// uniformly sampled with the given `seed` so two runs (base vs LoRA) over
+/// the same dataset score the same frames in the same order. Set to 0 to
+/// score every frame (slow on large examples).
+pub fn evaluate_held_out(
+    generator: &mut Generator,
+    dataset: &TrainingDataset,
+    frames_per_example: usize,
+    seed: u64,
+) -> Result<Vec<EvalRow>> {
+    use rand::rngs::StdRng;
+    use rand::{Rng, SeedableRng};
+
+    let mut rng = StdRng::seed_from_u64(seed);
+    let mut rows = Vec::with_capacity(dataset.len());
+    for (idx, ex) in dataset.examples.iter().enumerate() {
+        if ex.frame_codes.is_empty() {
+            continue;
+        }
+        let prompt_text = crate::generator::apply_emotion_hint(
+            ex.text.clone(),
+            ex.emotion_tag.as_deref(),
+        );
+        let current = Segment::new_text(ex.speaker, prompt_text);
+        let prompt = build_prompt(
+            &[],
+            &current,
+            &generator.model,
+            &mut generator.mimi,
+            &generator.tokenizer,
+        )?;
+        let total_frames = ex.frame_codes.len();
+        let n_sample = if frames_per_example == 0 {
+            total_frames
+        } else {
+            frames_per_example.min(total_frames)
+        };
+        let mut chosen = if frames_per_example == 0 {
+            (0..total_frames).collect::<Vec<_>>()
+        } else {
+            (0..n_sample).map(|_| rng.gen_range(0..total_frames)).collect()
+        };
+        chosen.sort();
+        let mut sum = 0.0f32;
+        for frame_idx in chosen.iter().copied() {
+            generator.model.clear_kv_cache();
+            let target = &ex.frame_codes[frame_idx];
+            let loss = generator
+                .model
+                .inner
+                .forward_loss(&prompt.tokens, &prompt.mask, 0, target)?;
+            sum += loss.to_scalar::<f32>()?;
+        }
+        let mean = if n_sample > 0 { sum / n_sample as f32 } else { 0.0 };
+        rows.push(EvalRow {
+            idx,
+            source: ex.source.clone(),
+            frames: total_frames,
+            mean_loss: mean,
+            frames_sampled: n_sample,
+        });
+        if (idx + 1) % 10 == 0 {
+            tracing::info!("eval progress: {}/{}", idx + 1, dataset.len());
+        }
+    }
+    Ok(rows)
+}
+
 /// Self-describing metadata embedded in safetensors at save time so the
 /// adapter file knows its own LoRA hyperparameters. Lets `apply_lora_adapter`
 /// auto-configure at inference without the user having to remember matching