osobhandClaude Opus 4.7
a5cedfb46a
8-gen bench (4 emotions × 2 corpora) at seed=42 against firdhokk Whisper-LV3: target RAVDESS CREMA-D happy happy (0.999) ✓ happy (0.999) ✓ angry neutral (0.92) sad (0.99) fearful happy (0.998) fearful (0.984) ✓ sad angry (0.99) fearful (0.99) CREMA-D 2/4 vs RAVDESS 1/4. Larger / more naturalistic corpus produces more class-pure fearful direction. Neither corpus solves angry or sad — recipe shifts into 'vague expressivity' rather than class-specific corners. Practical: prefer CREMA-D when available; A/B both per emotion if class precision matters. Co-Authored-By: Claude Opus 4.7 (1M context) <[email protected]>
90 lines
3.0 KiB
Rust
90 lines
3.0 KiB
Rust
//! Profile Whisper-tiny via whisper-rs (the `--features asr-metal` path)
|
|
//! against the same audio used for `stt_profile`. Lets us A/B Whisper
|
|
//! batch transcription vs Kyutai STT 1B streaming.
|
|
//!
|
|
//! Usage:
|
|
//! ```bash
|
|
//! cargo run -p rtx-csm --release --features asr-metal --example whisper_profile -- \
|
|
//! --in /tmp/asr_test.flac
|
|
//! ```
|
|
|
|
use anyhow::{Context, Result};
|
|
use clap::Parser;
|
|
use rtx_csm::{asr::WhisperAsr, audio_io};
|
|
use std::path::PathBuf;
|
|
use std::time::Instant;
|
|
|
|
#[derive(Debug, Parser)]
|
|
struct Cli {
|
|
#[arg(long = "in", default_value = "/tmp/asr_test.flac")]
|
|
input: PathBuf,
|
|
/// Number of repeat transcriptions to amortize first-call warm-up.
|
|
#[arg(long, default_value_t = 3)]
|
|
repeat: usize,
|
|
}
|
|
|
|
fn main() -> Result<()> {
|
|
tracing_subscriber::fmt()
|
|
.with_max_level(tracing::Level::WARN)
|
|
.init();
|
|
let cli = Cli::parse();
|
|
|
|
eprintln!("loading audio: {}", cli.input.display());
|
|
let pcm = audio_io::load_mono_at_rate(&cli.input, 24_000).context("load audio")?;
|
|
let audio_secs = pcm.len() as f32 / 24_000.0;
|
|
eprintln!("loaded {} samples ({audio_secs:.2}s @ 24 kHz)", pcm.len());
|
|
|
|
eprintln!("loading Whisper-tiny via whisper-rs...");
|
|
let load_t = Instant::now();
|
|
let asr = WhisperAsr::load_default().context("load whisper")?;
|
|
eprintln!("model loaded in {:.2}s", load_t.elapsed().as_secs_f32());
|
|
|
|
// Warm-up call (model does first-call setup).
|
|
let warm_t = Instant::now();
|
|
let warm_text = asr.transcribe_24k(&pcm).context("warm transcribe")?;
|
|
eprintln!(
|
|
"warm-up: {} ms, transcript = {:?}",
|
|
warm_t.elapsed().as_millis(),
|
|
warm_text
|
|
);
|
|
|
|
// Steady-state runs.
|
|
let mut per_call_ms: Vec<f64> = Vec::with_capacity(cli.repeat);
|
|
let mut last_text = String::new();
|
|
for i in 0..cli.repeat {
|
|
let t = Instant::now();
|
|
let text = asr.transcribe_24k(&pcm).context("transcribe")?;
|
|
let ms = t.elapsed().as_secs_f64() * 1000.0;
|
|
per_call_ms.push(ms);
|
|
last_text = text;
|
|
eprintln!(" run {}: {ms:.0} ms", i + 1);
|
|
}
|
|
|
|
per_call_ms.sort_by(|a, b| a.partial_cmp(b).unwrap());
|
|
let n = per_call_ms.len() as f64;
|
|
let mean = per_call_ms.iter().sum::<f64>() / n;
|
|
let p50 = per_call_ms[per_call_ms.len() / 2];
|
|
let realtime_factor = (mean / 1000.0) / audio_secs as f64;
|
|
|
|
println!();
|
|
println!("=== Whisper-tiny profile ===");
|
|
println!(
|
|
"input: {} ({audio_secs:.2}s of audio)",
|
|
cli.input.display()
|
|
);
|
|
println!("steady-state runs: {}", per_call_ms.len());
|
|
println!();
|
|
println!("per-call latency:");
|
|
println!(" mean = {mean:.0} ms");
|
|
println!(" p50 = {p50:.0} ms");
|
|
println!(" min = {:.0} ms", per_call_ms[0]);
|
|
println!(" max = {:.0} ms", per_call_ms[per_call_ms.len() - 1]);
|
|
println!();
|
|
println!("realtime factor: {realtime_factor:.3}x");
|
|
println!(" (mean / audio_duration; sub-1.0 means faster than realtime)");
|
|
println!();
|
|
println!("transcript: {last_text:?}");
|
|
|
|
Ok(())
|
|
}
|