rustytorch/crates/models/rtx-csm/examples/ort_conflict_probe.rs

//! Phase 8.1.2 — runtime-conflict gate.
//!
//! whisper-rs (Phase 7.6) revealed that linking ggml/whisper.cpp into
//! the same binary as candle/CSM causes a 2-3× regression in CSM Metal
//! inference even when whisper isn't actively used. This binary checks
//! whether `voice_activity_detector` (Silero V5 via the `ort` ONNX
//! runtime) has the same problem before we wire VAD gating into
//! converse_server.
//!
//! Test pattern: time N CSM Q8 backbone+depth-decoder forwards, load
//! ort + Silero VAD, run a few VAD predictions, then time N more CSM
//! forwards. If the ratio T_after / T_before > 1.05 we treat ort as a
//! regression risk and defer Silero-VAD gating.
//!
//! Usage:
//! ```bash
//! cargo run -p rtx-csm --release --features metal,vad --example ort_conflict_probe
//! ```

use anyhow::{Context, Result};
use rtx_csm::{GenerateOptions, Generator, stt::Stt};
use std::time::Instant;

const N_FORWARDS: usize = 10;

fn time_csm_forwards(generator: &mut Generator, label: &str) -> Result<f64> {
    // Each "forward" here is one short generate() call — exercises the
    // backbone + depth decoder + Mimi decode path. The same path that
    // shows the whisper-rs regression.
    let opts = GenerateOptions {
        max_audio_ms: 200,
        seed: 0,
        ..GenerateOptions::default()
    };
    // Warm-up (not counted) so kernel compilation doesn't pollute the
    // before/after comparison.
    generator.generate("Hi.", 0, &[], opts.clone())?;

    let mut total_ms = 0.0;
    for i in 0..N_FORWARDS {
        let t = Instant::now();
        generator.generate("Hi.", 0, &[], opts.clone())?;
        let ms = t.elapsed().as_secs_f64() * 1000.0;
        total_ms += ms;
        eprintln!("  {label} forward {}: {ms:.1} ms", i + 1);
    }
    Ok(total_ms / N_FORWARDS as f64)
}

#[cfg(feature = "vad")]
fn run_ort_a_few_times() -> Result<()> {
    use voice_activity_detector::VoiceActivityDetector;
    let mut vad = VoiceActivityDetector::builder()
        .sample_rate(16_000_i64)
        .chunk_size(512_usize)
        .build()
        .context("VoiceActivityDetector::builder")?;
    // Five 30 ms predictions. Enough to fully load the runtime + run
    // the model so any global state it touches is initialized.
    for _ in 0..5 {
        let chunk = vec![0.0f32; 512];
        let _ = vad.predict(chunk);
    }
    Ok(())
}

#[cfg(not(feature = "vad"))]
fn run_ort_a_few_times() -> Result<()> {
    anyhow::bail!(
        "this binary requires `--features vad` to load the ort runtime; \
         build with `--features metal,vad`."
    );
}

fn main() -> Result<()> {
    tracing_subscriber::fmt()
        .with_max_level(tracing::Level::WARN)
        .init();

    let device = if candle_core::utils::metal_is_available() {
        candle_core::Device::new_metal(0)?
    } else {
        candle_core::Device::Cpu
    };
    eprintln!("device: {device:?}");

    // Load Q8 GGUF (canonical production config).
    let q8_path = std::path::Path::new("/tmp/csm_q8.gguf");
    if !q8_path.exists() {
        anyhow::bail!(
            "expected /tmp/csm_q8.gguf — produce it with \
             `cargo run --example quantize --release --features metal -- --policy q8`"
        );
    }
    eprintln!("loading CSM-1B Q8 from {}", q8_path.display());
    let mut generator =
        Generator::load_csm_1b_quantized(q8_path, &device, /* enable_cfg */ false)?;
    eprintln!("model loaded");

    // Critical: also load Kyutai STT (which links sentencepiece-sys).
    // Phase 8.1.3 discovered ort + sentencepiece have a hard protobuf
    // version conflict at process startup (sentencepiece bundles 3.14,
    // ort bundles 3.21). The minimal probe above didn't catch it
    // because it skipped Kyutai. Load STT here so the probe matches the
    // real converse_server linkage. If the binary even launches past
    // this point, sentencepiece + ort co-exist (i.e. version conflict
    // resolved in some future bump).
    eprintln!("loading Kyutai STT (verifies ort/sentencepiece linkage)...");
    let _stt = Stt::load_default(&device).context("load Kyutai STT")?;
    eprintln!("Kyutai STT loaded — sentencepiece/ort coexist OK");

    // Phase 1: time CSM forwards before ort is loaded.
    let before_ms = time_csm_forwards(&mut generator, "before")?;
    eprintln!();
    eprintln!("loading ort + Silero VAD...");
    run_ort_a_few_times()?;
    eprintln!("ort loaded; running CSM forwards again...");
    eprintln!();

    // Phase 2: time CSM forwards after ort is loaded + has run.
    let after_ms = time_csm_forwards(&mut generator, "after")?;

    let ratio = after_ms / before_ms;
    let pct = (ratio - 1.0) * 100.0;
    println!();
    println!("=== ort conflict probe ===");
    println!("CSM forwards (n={N_FORWARDS} each):");
    println!("  before ort load: mean = {before_ms:.1} ms");
    println!("  after  ort load: mean = {after_ms:.1} ms");
    println!("  ratio: {ratio:.3}  ({pct:+.1}%)");
    println!();
    if ratio > 1.05 {
        println!("*** REGRESSION DETECTED ***");
        println!("ort load slowed CSM forwards by {pct:.1}% — same risk class as whisper-rs.");
        println!("Defer Silero-VAD gating; pursue VAD only via sidecar process.");
        std::process::exit(2);
    } else {
        println!("PASS: ort coexists with candle Metal at {pct:+.1}% delta (within ±5%).");
        println!("Safe to proceed with Phase 8.1.3 (Silero-VAD gating).");
    }
    Ok(())
}