rtx-csm: Sprint 1 eval foundation — quality_eval + I2D loop

Sprint 1 of the post-research roadmap. TTSDS2 (arXiv 2506.19441) was
the original target but its install is broken on Python 3.12 + modern
torchaudio (deprecated `torchaudio.sox_effects`, `pyannote.audio` 3.1
calls removed `set_audio_backend`, `openai-whisper==20240927` needs
`pkg_resources`). Pivoted to a Rust-native foundation we already own
end-to-end: WavLM-SV + Moonshine + amplitude.

`examples/quality_eval` consumes a JSONL of `(ref_wav, gen_wav,
ref_text)` rows and emits per-row metrics:
  - speaker_cosine via WavLM-SV (microsoft/wavlm-base-plus-sv)
  - wer via Moonshine v2 transcript vs ref_text (Levenshtein on
    lowercased / punctuation-stripped tokens)
  - gen_peak_db, gen_rms_db (full-band amplitude of gen_wav)

`scripts/i2d_loop.sh` implements I2D (arXiv 2603.24430): synth N
times feeding each output back as the next iteration's context, score
all iterations with quality_eval, emit a TSV degradation curve.

Smoke-tested:
  - quality_eval on the picker A/B set independently confirms the
    picker — bottom-context (score 0) → WER 0.55, top-context
    (score 2.0) → WER 0.18 (3× worse without picker filter).
  - i2d_loop with 3 iterations on Amini context shows clean
    collapse: cos 0.84 → 0.58, WER 0.5 → 1.0 by iter 1.

Foundation for Sprint 2 emotion-steering A/B comparisons.

Co-Authored-By: Claude Opus 4.7 (1M context) <[email protected]>

.gitignore

@@ -9,6 +9,8 @@ bin/
 # Python virtual environments
 .venv/
 **/.venv/
+.venv-*/
+**/.venv-*/
 venv/
 **/venv/
 __pycache__/

crates/models/rtx-csm/examples/quality_eval.rs

@@ -0,0 +1,270 @@
+//! Per-pair TTS quality eval: speaker similarity + WER + amplitude.
+//!
+//! Reads a JSONL where each row has at minimum `ref_wav` and `gen_wav`,
+//! optionally `ref_text` (used as the reference transcript for WER). The
+//! reference text comes from the manifest because the reference audio
+//! might not have a clean public transcript (e.g. context-conditioning
+//! reference clips).
+//!
+//! Designed as the metric foundation for Sprint 1 of the Phase 8 roadmap.
+//! Originally TTSDS2 (arXiv 2506.19441) was on the menu — its install on
+//! Python 3.12 + torchaudio 2.x is broken (`torchaudio.sox_effects` was
+//! removed; `pyannote.audio` 3.1 calls `set_audio_backend`; openai-whisper
+//! pinned dep needs `pkg_resources` which requires `setuptools<81`). For
+//! now we score with what we already own end-to-end in candle:
+//!
+//!   - **speaker_cosine**: WavLM-SV (microsoft/wavlm-base-plus-sv)
+//!   - **wer**: Moonshine v2 transcribe vs `ref_text`
+//!   - **peak_db / rms_db**: full-band amplitude of `gen_wav`
+//!
+//! Output: one JSONL line per input, with the input's fields plus a
+//! `metrics` object. Stable column order = appendable to a CSV/jq pipeline.
+//!
+//! ```bash
+//! target/release/examples/quality_eval \
+//!     --in /tmp/i2d/eval_pairs.jsonl \
+//!     --wavlm-sv /tmp/wavlm_sv.safetensors \
+//!     --out /tmp/i2d/scores.jsonl
+//! ```
+
+use anyhow::{Context, Result};
+use candle_core::{Device, Tensor};
+use clap::Parser;
+use hf_hub::api::sync::Api;
+use rtx_csm::speaker_sim::{SpeakerSimilarity, WavLmSimilarity};
+use rtx_csm::{audio_io, moonshine};
+use serde::{Deserialize, Serialize};
+use std::io::{BufRead, BufWriter, Write};
+use std::path::PathBuf;
+
+#[derive(Debug, Parser)]
+struct Cli {
+    /// Input JSONL with rows: `{"ref_wav": "...", "gen_wav": "...", "ref_text": "..."}`.
+    /// `ref_text` is optional; without it WER is `null`.
+    #[arg(long = "in")]
+    input: PathBuf,
+    /// Output JSONL. One line per input row, with the input's fields plus
+    /// a `metrics` object.
+    #[arg(long)]
+    out: PathBuf,
+    /// Converted WavLM-SV safetensors. See `examples/wavlm_sv_convert`.
+    #[arg(long)]
+    wavlm_sv: PathBuf,
+    /// Max decoded tokens for Moonshine (capped at 194 by config).
+    #[arg(long, default_value_t = 120)]
+    max_tokens: usize,
+    /// Force CPU device.
+    #[arg(long)]
+    cpu: bool,
+}
+
+#[derive(Debug, Deserialize, Serialize)]
+struct PairRow {
+    ref_wav: PathBuf,
+    gen_wav: PathBuf,
+    #[serde(skip_serializing_if = "Option::is_none", default)]
+    ref_text: Option<String>,
+    /// Free-form per-row metadata (iteration index, model id, etc).
+    /// Round-tripped unchanged into the output row.
+    #[serde(flatten, default)]
+    extra: serde_json::Map<String, serde_json::Value>,
+}
+
+#[derive(Debug, Serialize)]
+struct ScoredRow {
+    ref_wav: PathBuf,
+    gen_wav: PathBuf,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    ref_text: Option<String>,
+    #[serde(flatten)]
+    extra: serde_json::Map<String, serde_json::Value>,
+    metrics: Metrics,
+}
+
+#[derive(Debug, Serialize)]
+struct Metrics {
+    /// Cosine in [-1, 1]. Higher = more similar speaker identity.
+    speaker_cosine: f32,
+    /// Word error rate in [0, 1+]. None if no `ref_text`.
+    wer: Option<f32>,
+    /// Moonshine's transcript of `gen_wav` (always emitted; useful for
+    /// debugging).
+    transcript: String,
+    /// Generated audio characteristics.
+    gen_peak_db: f32,
+    gen_rms_db: f32,
+}
+
+fn main() -> Result<()> {
+    tracing_subscriber::fmt().with_target(false).init();
+    let cli = Cli::parse();
+
+    let device = if cli.cpu {
+        Device::Cpu
+    } else if candle_core::utils::metal_is_available() {
+        Device::new_metal(0)?
+    } else {
+        Device::Cpu
+    };
+    tracing::info!("device: {device:?}");
+
+    // Speaker similarity model.
+    let sv = WavLmSimilarity::load(&cli.wavlm_sv, &device)?;
+    tracing::info!("loaded WavLM-SV from {}", cli.wavlm_sv.display());
+
+    // Moonshine encoder + decoder + tokenizer (all from HF cache).
+    let api = Api::new()?;
+    let repo = api.model("UsefulSensors/moonshine-tiny".to_string());
+    let m_weights = repo.get("model.safetensors")?;
+    let m_tok = repo.get("tokenizer.json")?;
+    let m_cfg = moonshine::MoonshineConfig::tiny();
+    let (m_enc, m_dec) = moonshine::load_full(&m_weights, &device, &m_cfg)?;
+    let m_tokenizer =
+        moonshine::load_tokenizer(&m_tok).map_err(|e| anyhow::anyhow!("tokenizer: {e}"))?;
+    tracing::info!("loaded Moonshine-tiny");
+
+    // Stream the input JSONL line by line so big batches don't blow memory.
+    let f = std::fs::File::open(&cli.input)
+        .with_context(|| format!("open input {}", cli.input.display()))?;
+    let reader = std::io::BufReader::new(f);
+
+    if let Some(parent) = cli.out.parent() {
+        std::fs::create_dir_all(parent).ok();
+    }
+    let out_file = std::fs::File::create(&cli.out)
+        .with_context(|| format!("create out {}", cli.out.display()))?;
+    let mut out_w = BufWriter::new(out_file);
+
+    let mut n_rows = 0usize;
+    let mut sum_cos = 0.0f64;
+    let mut sum_wer = 0.0f64;
+    let mut n_wer = 0usize;
+    for (idx, line) in reader.lines().enumerate() {
+        let line = line?;
+        if line.trim().is_empty() {
+            continue;
+        }
+        let row: PairRow = serde_json::from_str(&line)
+            .with_context(|| format!("parse row {idx}: {line}"))?;
+
+        // Speaker similarity at 16 kHz.
+        let ref_16k = audio_io::load_mono_at_rate(&row.ref_wav, 16_000)?;
+        let gen_16k = audio_io::load_mono_at_rate(&row.gen_wav, 16_000)?;
+        let speaker_cosine = sv.score(&ref_16k, &gen_16k)? as f32;
+
+        // Transcribe gen_wav via Moonshine (16 kHz).
+        let pcm_t = Tensor::from_vec(gen_16k.clone(), (1, 1, gen_16k.len()), &device)?;
+        let enc_out = m_enc.forward(&pcm_t)?;
+        let token_ids = m_dec.generate_cached(&enc_out, &m_cfg, cli.max_tokens)?;
+        let transcript = m_tokenizer
+            .decode(&token_ids, /* skip_special */ true)
+            .map_err(|e| anyhow::anyhow!("detok: {e}"))?
+            .trim()
+            .to_string();
+
+        let wer = row.ref_text.as_ref().map(|t| word_error_rate(t, &transcript));
+
+        // Amplitude of gen_wav at native rate.
+        let gen_native = audio_io::load_mono_24k(&row.gen_wav)?;
+        let (peak_db, rms_db) = peak_rms_db(&gen_native);
+
+        let scored = ScoredRow {
+            ref_wav: row.ref_wav,
+            gen_wav: row.gen_wav,
+            ref_text: row.ref_text,
+            extra: row.extra,
+            metrics: Metrics {
+                speaker_cosine,
+                wer,
+                transcript: transcript.clone(),
+                gen_peak_db: peak_db,
+                gen_rms_db: rms_db,
+            },
+        };
+        writeln!(out_w, "{}", serde_json::to_string(&scored)?)?;
+        out_w.flush()?;
+
+        n_rows += 1;
+        sum_cos += speaker_cosine as f64;
+        if let Some(w) = wer {
+            sum_wer += w as f64;
+            n_wer += 1;
+        }
+        tracing::info!(
+            "row {idx}: cos={speaker_cosine:.3} wer={} peak={peak_db:.2} rms={rms_db:.2}",
+            wer.map(|w| format!("{w:.3}")).unwrap_or_else(|| "-".into())
+        );
+    }
+
+    if n_rows == 0 {
+        anyhow::bail!("no rows processed (empty input?)");
+    }
+    eprintln!("--- summary ---");
+    eprintln!("rows           {n_rows}");
+    eprintln!("speaker_cosine {:.3} (mean)", sum_cos / n_rows as f64);
+    if n_wer > 0 {
+        eprintln!("wer            {:.3} (mean over {n_wer})", sum_wer / n_wer as f64);
+    } else {
+        eprintln!("wer            (no ref_text in any row)");
+    }
+    eprintln!("wrote          {}", cli.out.display());
+    Ok(())
+}
+
+/// Word error rate: Levenshtein on whitespace-split, lowercased,
+/// punctuation-stripped tokens. Returns errors / max(ref_len, 1).
+fn word_error_rate(reference: &str, hypothesis: &str) -> f32 {
+    let r = tokenize(reference);
+    let h = tokenize(hypothesis);
+    if r.is_empty() {
+        return if h.is_empty() { 0.0 } else { 1.0 };
+    }
+    let n = r.len();
+    let m = h.len();
+    let mut prev: Vec<usize> = (0..=m).collect();
+    let mut curr = vec![0usize; m + 1];
+    for i in 1..=n {
+        curr[0] = i;
+        for j in 1..=m {
+            let cost = if r[i - 1] == h[j - 1] { 0 } else { 1 };
+            curr[j] = (prev[j] + 1)
+                .min(curr[j - 1] + 1)
+                .min(prev[j - 1] + cost);
+        }
+        std::mem::swap(&mut prev, &mut curr);
+    }
+    prev[m] as f32 / n as f32
+}
+
+fn tokenize(s: &str) -> Vec<String> {
+    let mut buf = String::with_capacity(s.len());
+    for c in s.chars() {
+        if c.is_alphanumeric() || c.is_whitespace() {
+            for lc in c.to_lowercase() {
+                buf.push(lc);
+            }
+        } else {
+            buf.push(' ');
+        }
+    }
+    buf.split_whitespace().map(String::from).collect()
+}
+
+fn peak_rms_db(pcm: &[f32]) -> (f32, f32) {
+    if pcm.is_empty() {
+        return (-100.0, -100.0);
+    }
+    let mut peak = 0.0f32;
+    let mut sum_sq = 0.0f64;
+    for &s in pcm {
+        let a = s.abs();
+        if a > peak {
+            peak = a;
+        }
+        sum_sq += (s as f64) * (s as f64);
+    }
+    let rms = (sum_sq / pcm.len() as f64).sqrt() as f32;
+    let peak_db = if peak < 1e-6 { -100.0 } else { 20.0 * peak.log10() };
+    let rms_db = if rms < 1e-6 { -100.0 } else { 20.0 * rms.log10() };
+    (peak_db, rms_db)
+}

crates/models/rtx-csm/scripts/i2d_loop.sh

@@ -0,0 +1,132 @@
+#!/usr/bin/env bash
+# I2D (Iterate-to-Differentiate, arXiv 2603.24430): synth N times feeding
+# each output back as the next iteration's context. Higher-quality models
+# degrade more slowly; the curve over iterations amplifies inter-model
+# quality deltas that a single-shot eval would miss.
+#
+# Outputs:
+#   <workdir>/iter_<N>.wav           — synthesized audio per iteration
+#   <workdir>/eval_pairs.jsonl       — input to quality_eval (one row per iter)
+#   <workdir>/scores.jsonl           — per-iter (speaker_cosine, wer, peak/rms)
+#   <workdir>/curve.tsv              — pretty-printed degradation curve
+#
+# Reference for speaker similarity stays the original ref_wav; reference
+# for WER stays the original SYNTH_TEXT. Both are constant across iters,
+# so any drift you see is the model degrading, not the target moving.
+#
+# Usage:
+#   scripts/i2d_loop.sh \
+#       --text "What you want each iteration to say." \
+#       --ref-wav /tmp/voice/picked.wav \
+#       --ref-text "Transcript of the reference clip." \
+#       --iterations 5 \
+#       --workdir /tmp/i2d_run \
+#       --wavlm-sv /tmp/wavlm_sv.safetensors
+
+set -euo pipefail
+
+TEXT=""
+REF_WAV=""
+REF_TEXT=""
+ITERATIONS=5
+WORKDIR=""
+WAVLM_SV="/tmp/wavlm_sv.safetensors"
+SPEAKER=0
+SEED=42
+
+while [[ $# -gt 0 ]]; do
+    case "$1" in
+        --text) TEXT="$2"; shift 2 ;;
+        --ref-wav) REF_WAV="$2"; shift 2 ;;
+        --ref-text) REF_TEXT="$2"; shift 2 ;;
+        --iterations) ITERATIONS="$2"; shift 2 ;;
+        --workdir) WORKDIR="$2"; shift 2 ;;
+        --wavlm-sv) WAVLM_SV="$2"; shift 2 ;;
+        --speaker) SPEAKER="$2"; shift 2 ;;
+        --seed) SEED="$2"; shift 2 ;;
+        -h|--help)
+            sed -n '2,28p' "$0"
+            exit 0
+            ;;
+        *) echo "unknown arg: $1" >&2; exit 1 ;;
+    esac
+done
+
+if [[ -z "$TEXT" || -z "$REF_WAV" || -z "$REF_TEXT" || -z "$WORKDIR" ]]; then
+    echo "usage: $0 --text <t> --ref-wav <w> --ref-text <r> --workdir <d> [--iterations N] [--wavlm-sv <f>]" >&2
+    exit 1
+fi
+if [[ ! -f "$REF_WAV" ]]; then
+    echo "ref-wav not found: $REF_WAV" >&2; exit 1
+fi
+if [[ ! -f "$WAVLM_SV" ]]; then
+    echo "wavlm-sv not found: $WAVLM_SV" >&2; exit 1
+fi
+
+SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
+WORKSPACE_DIR="$(cd "$SCRIPT_DIR/../../../.." && pwd)"
+GEN_BIN="$WORKSPACE_DIR/target/release/examples/generate"
+EVAL_BIN="$WORKSPACE_DIR/target/release/examples/quality_eval"
+for bin in "$GEN_BIN" "$EVAL_BIN"; do
+    if [[ ! -x "$bin" ]]; then
+        echo "binary missing: $bin" >&2
+        echo "build with: cargo build -p rtx-csm --release --example generate --example quality_eval" >&2
+        exit 1
+    fi
+done
+
+mkdir -p "$WORKDIR"
+PAIRS="$WORKDIR/eval_pairs.jsonl"
+: > "$PAIRS"
+
+# Iter 0 uses the original ref_wav as context.
+# Iter N>0 uses iter_(N-1).wav as context, with TEXT as the context text
+# (since the previous iteration synthesized TEXT).
+PREV_WAV="$REF_WAV"
+PREV_TEXT="$REF_TEXT"
+
+for ((i=0; i<ITERATIONS; i++)); do
+    OUT_WAV="$WORKDIR/iter_${i}.wav"
+    echo "→ iter $i: synth → $(basename "$OUT_WAV")  (ctx=$(basename "$PREV_WAV"))"
+    "$GEN_BIN" \
+        --text "$TEXT" \
+        --speaker "$SPEAKER" \
+        --context-wav "$PREV_WAV" \
+        --context-text "$PREV_TEXT" \
+        --context-speaker 1 \
+        --seed "$SEED" \
+        --out "$OUT_WAV" 2>&1 | grep -E "generated" | tail -1 || true
+
+    if [[ ! -f "$OUT_WAV" ]]; then
+        echo "iter $i produced no output — aborting" >&2
+        exit 2
+    fi
+
+    # Append eval row: ref always = the *original* reference so speaker
+    # cosine measures drift from the source voice; WER target = TEXT.
+    jq -nc --arg ref "$REF_WAV" --arg gen "$OUT_WAV" \
+        --arg t "$TEXT" --argjson iter "$i" \
+        '{ref_wav: $ref, gen_wav: $gen, ref_text: $t, iter: $iter}' >> "$PAIRS"
+
+    PREV_WAV="$OUT_WAV"
+    PREV_TEXT="$TEXT"
+done
+
+echo
+echo "→ scoring all $ITERATIONS iterations"
+"$EVAL_BIN" \
+    --in "$PAIRS" \
+    --out "$WORKDIR/scores.jsonl" \
+    --wavlm-sv "$WAVLM_SV" 2>&1 | tail -5
+
+echo
+echo "=== degradation curve ==="
+{
+    echo "iter	speaker_cosine	wer	peak_db	rms_db"
+    jq -r '[.iter, .metrics.speaker_cosine, .metrics.wer, .metrics.gen_peak_db, .metrics.gen_rms_db] | @tsv' \
+        "$WORKDIR/scores.jsonl"
+} | tee "$WORKDIR/curve.tsv" | column -t -s $'\t'
+
+echo
+echo "✓ wrote $WORKDIR/curve.tsv"
+echo "  audio in $WORKDIR/iter_*.wav"