rustyverse/rustytorch
2
0
Fork 0
Code Issues Pull Requests Actions 46 Packages Projects Releases Wiki Activity

rtx-csm: emotional_speech_guide — CREMA-D vs RAVDESS firdhokk verdict

Browse Source 
8-gen bench (4 emotions × 2 corpora) at seed=42 against firdhokk
Whisper-LV3:

  target    RAVDESS              CREMA-D
  happy     happy (0.999) ✓      happy (0.999) ✓
  angry     neutral (0.92)       sad (0.99)
  fearful   happy (0.998)        fearful (0.984) ✓
  sad       angry (0.99)         fearful (0.99)

CREMA-D 2/4 vs RAVDESS 1/4. Larger / more naturalistic corpus
produces more class-pure fearful direction. Neither corpus solves
angry or sad — recipe shifts into 'vague expressivity' rather than
class-specific corners.

Practical: prefer CREMA-D when available; A/B both per emotion if
class precision matters.

Co-Authored-By: Claude Opus 4.7 (1M context) <[email protected]>
This commit is contained in:
osobh
2026-04-30 00:01:02 -07:00
co-authored by Claude Opus 4.7
parent f4d8268381
commit a5cedfb46a
69 changed files with 1026 additions and 734 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/models/rtx-csm/src/wavlm_sv.rs
+43 -26
View File
@@ -61,10 +61,10 @@
//! a paired-utterance test set)
use crate::error::{CsmError, Result};
use candle_core::{DType, Device, IndexOp, Module, Tensor, D};
use candle_core::{D, DType, Device, IndexOp, Module, Tensor};
use candle_nn::{
conv1d, group_norm, layer_norm, linear, ops, Conv1d, Conv1dConfig, GroupNorm,
LayerNorm, LayerNormConfig, Linear, VarBuilder,
Conv1d, Conv1dConfig, GroupNorm, LayerNorm, LayerNormConfig, Linear, VarBuilder, conv1d,
group_norm, layer_norm, linear, ops,
};
use std::path::Path;
@@ -133,10 +133,21 @@ impl FeatureExtractor {
stride: *s,
..Default::default()
};
let conv = candle_nn::conv1d_no_bias(in_ch, *out_ch, *k, cfg, vb_layers.pp(i.to_string()).pp("conv"))?;
let conv = candle_nn::conv1d_no_bias(
in_ch,
*out_ch,
*k,
cfg,
vb_layers.pp(i.to_string()).pp("conv"),
)?;
let norm = if i == 0 {
// WavLM/wav2vec2 GroupNorm layer 0: num_groups = num_channels.
Some(group_norm(*out_ch, *out_ch, 1e-5, vb_layers.pp(i.to_string()).pp("layer_norm"))?)
Some(group_norm(
*out_ch,
*out_ch,
1e-5,
vb_layers.pp(i.to_string()).pp("layer_norm"),
)?)
} else {
None
};
@@ -218,7 +229,13 @@ impl PosConv {
groups: 16,
..Default::default()
};
let conv = conv1d(HIDDEN_DIM, HIDDEN_DIM, 128, cfg, vb.pp("pos_conv_embed.conv"))?;
let conv = conv1d(
HIDDEN_DIM,
HIDDEN_DIM,
128,
cfg,
vb.pp("pos_conv_embed.conv"),
)?;
Ok(Self { conv, kernel: 128 })
}
}
@@ -339,7 +356,11 @@ impl WavLmEncoderLayer {
for q in 0..t {
for k in 0..t {
let rel = k as i64 - q as i64;
buckets.push(relative_position_bucket(rel, REL_NUM_BUCKETS, REL_MAX_DISTANCE));
buckets.push(relative_position_bucket(
rel,
REL_NUM_BUCKETS,
REL_MAX_DISTANCE,
));
}
}
let device = embed.embeddings().device();
@@ -587,7 +608,13 @@ impl XVectorHead {
let mut tdnn = Vec::with_capacity(5);
let vb_tdnn = vb.pp("tdnn");
for (i, (in_dim, out_dim, k, d)) in tdnn_specs.iter().enumerate() {
tdnn.push(Tdnn::new(*in_dim, *out_dim, *k, *d, vb_tdnn.pp(i.to_string()))?);
tdnn.push(Tdnn::new(
*in_dim,
*out_dim,
*k,
*d,
vb_tdnn.pp(i.to_string()),
)?);
}
let embedding_proj = linear(STAT_POOL_DIM, EMBEDDING_DIM, vb.pp("feature_extractor"))?;
Ok(Self {
@@ -703,14 +730,8 @@ impl WavLmSv {
/// Cosine similarity between two embeddings, both expected as `(D,)`
/// or `(B, D)` tensors of compatible shape.
pub fn cosine_similarity(a: &Tensor, b: &Tensor) -> candle_core::Result<Tensor> {
let a_norm = a.broadcast_div(
&(a.sqr()?.sum_keepdim(D::Minus1)? + 1e-9)?
.sqrt()?,
)?;
let b_norm = b.broadcast_div(
&(b.sqr()?.sum_keepdim(D::Minus1)? + 1e-9)?
.sqrt()?,
)?;
let a_norm = a.broadcast_div(&(a.sqr()?.sum_keepdim(D::Minus1)? + 1e-9)?.sqrt()?)?;
let b_norm = b.broadcast_div(&(b.sqr()?.sum_keepdim(D::Minus1)? + 1e-9)?.sqrt()?)?;
(a_norm * b_norm)?.sum(D::Minus1)
}
}
@@ -718,16 +739,9 @@ impl WavLmSv {
/// Stub loader. Phase 5c will implement `pytorch_model.bin` → safetensors
/// conversion (weight_norm merge for pos_conv + TDNN kernel reshape) and
/// HF Hub asset resolution under `microsoft/wavlm-base-plus-sv`.
pub fn load_from_safetensors<P: AsRef<Path>>(
safetensors: P,
device: &Device,
) -> Result<WavLmSv> {
pub fn load_from_safetensors<P: AsRef<Path>>(safetensors: P, device: &Device) -> Result<WavLmSv> {
let vb = unsafe {
candle_nn::VarBuilder::from_mmaped_safetensors(
&[safetensors.as_ref()],
DType::F32,
device,
)
candle_nn::VarBuilder::from_mmaped_safetensors(&[safetensors.as_ref()], DType::F32, device)
}
.map_err(|e| CsmError::Config(format!("opening WavLM safetensors: {e}")))?;
WavLmSv::new(vb).map_err(|e| CsmError::Config(format!("WavLmSv::new: {e}")))
@@ -888,6 +902,9 @@ mod tests {
let mean: f32 = s.iter().sum::<f32>() / s.len() as f32;
assert!(mean.abs() < 1e-5);
let var = s.iter().map(|x| x * x).sum::<f32>() / s.len() as f32;
assert!((var - 1.0).abs() < 1e-3, "expected unit variance, got {var}");
assert!(
(var - 1.0).abs() < 1e-3,
"expected unit variance, got {var}"
);
}
}