rtx-csm: emotional_speech_guide — CREMA-D vs RAVDESS firdhokk verdict
8-gen bench (4 emotions × 2 corpora) at seed=42 against firdhokk Whisper-LV3: target RAVDESS CREMA-D happy happy (0.999) ✓ happy (0.999) ✓ angry neutral (0.92) sad (0.99) fearful happy (0.998) fearful (0.984) ✓ sad angry (0.99) fearful (0.99) CREMA-D 2/4 vs RAVDESS 1/4. Larger / more naturalistic corpus produces more class-pure fearful direction. Neither corpus solves angry or sad — recipe shifts into 'vague expressivity' rather than class-specific corners. Practical: prefer CREMA-D when available; A/B both per emotion if class precision matters. Co-Authored-By: Claude Opus 4.7 (1M context) <[email protected]>
This commit is contained in:
osobh
@@ -14,7 +14,7 @@
|
||||
use crate::audio_io;
|
||||
use crate::error::{CsmError, Result};
|
||||
use crate::generator::Generator;
|
||||
use crate::prompt::{build_prompt, Segment};
|
||||
use crate::prompt::{Segment, build_prompt};
|
||||
use candle_core::Tensor;
|
||||
use candle_nn::{AdamW, Optimizer, ParamsAdamW, VarMap};
|
||||
use std::path::{Path, PathBuf};
|
||||
@@ -121,10 +121,7 @@ impl TrainingDataset {
|
||||
}
|
||||
let txt_path = path.with_extension("txt");
|
||||
if !txt_path.exists() {
|
||||
tracing::warn!(
|
||||
"skipping {}: no matching .txt transcript",
|
||||
path.display()
|
||||
);
|
||||
tracing::warn!("skipping {}: no matching .txt transcript", path.display());
|
||||
continue;
|
||||
}
|
||||
let text = std::fs::read_to_string(&txt_path)?.trim().to_string();
|
||||
@@ -169,9 +166,8 @@ impl TrainingDataset {
|
||||
if line.is_empty() || line.starts_with('#') {
|
||||
continue;
|
||||
}
|
||||
let row: ManifestRow = serde_json::from_str(line).map_err(|e| {
|
||||
CsmError::Config(format!("manifest line {}: {e}", lineno + 1))
|
||||
})?;
|
||||
let row: ManifestRow = serde_json::from_str(line)
|
||||
.map_err(|e| CsmError::Config(format!("manifest line {}: {e}", lineno + 1)))?;
|
||||
let text = row.transcript.trim().to_string();
|
||||
if text.is_empty() {
|
||||
tracing::warn!("manifest line {}: empty transcript, skipping", lineno + 1);
|
||||
@@ -208,7 +204,9 @@ impl TrainingDataset {
|
||||
ex.stage = row.stage;
|
||||
tracing::debug!(
|
||||
"manifest example: tag={:?} stage={:?} frames={}",
|
||||
ex.emotion_tag, ex.stage, ex.frame_codes.len()
|
||||
ex.emotion_tag,
|
||||
ex.stage,
|
||||
ex.frame_codes.len()
|
||||
);
|
||||
examples.push(ex);
|
||||
}
|
||||
@@ -442,7 +440,12 @@ impl<'a> CurriculumTrainer<'a> {
|
||||
dataset: &'a TrainingDataset,
|
||||
stages: Vec<CurriculumStage>,
|
||||
) -> Self {
|
||||
Self { generator, vm, dataset, stages }
|
||||
Self {
|
||||
generator,
|
||||
vm,
|
||||
dataset,
|
||||
stages,
|
||||
}
|
||||
}
|
||||
|
||||
/// Run all stages sequentially. Returns one loss-trace `Vec<f32>` per
|
||||
@@ -479,7 +482,9 @@ impl<'a> CurriculumTrainer<'a> {
|
||||
stage.config.epochs,
|
||||
stage.config.peak_lr,
|
||||
);
|
||||
let stage_dataset = TrainingDataset { examples: stage_examples };
|
||||
let stage_dataset = TrainingDataset {
|
||||
examples: stage_examples,
|
||||
};
|
||||
let mut trainer = Trainer::new(
|
||||
self.generator,
|
||||
self.vm,
|
||||
@@ -619,10 +624,8 @@ pub fn evaluate_held_out(
|
||||
if ex.frame_codes.is_empty() {
|
||||
continue;
|
||||
}
|
||||
let prompt_text = crate::generator::apply_emotion_hint(
|
||||
ex.text.clone(),
|
||||
ex.emotion_tag.as_deref(),
|
||||
);
|
||||
let prompt_text =
|
||||
crate::generator::apply_emotion_hint(ex.text.clone(), ex.emotion_tag.as_deref());
|
||||
let current = Segment::new_text(ex.speaker, prompt_text);
|
||||
let prompt = build_prompt(
|
||||
&[],
|
||||
@@ -640,20 +643,27 @@ pub fn evaluate_held_out(
|
||||
let mut chosen = if frames_per_example == 0 {
|
||||
(0..total_frames).collect::<Vec<_>>()
|
||||
} else {
|
||||
(0..n_sample).map(|_| rng.gen_range(0..total_frames)).collect()
|
||||
(0..n_sample)
|
||||
.map(|_| rng.gen_range(0..total_frames))
|
||||
.collect()
|
||||
};
|
||||
chosen.sort();
|
||||
let mut sum = 0.0f32;
|
||||
for frame_idx in chosen.iter().copied() {
|
||||
generator.model.clear_kv_cache();
|
||||
let target = &ex.frame_codes[frame_idx];
|
||||
let loss = generator
|
||||
.model
|
||||
.inner
|
||||
.forward_loss(&prompt.tokens, &prompt.mask, 0, target)?;
|
||||
let loss =
|
||||
generator
|
||||
.model
|
||||
.inner
|
||||
.forward_loss(&prompt.tokens, &prompt.mask, 0, target)?;
|
||||
sum += loss.to_scalar::<f32>()?;
|
||||
}
|
||||
let mean = if n_sample > 0 { sum / n_sample as f32 } else { 0.0 };
|
||||
let mean = if n_sample > 0 {
|
||||
sum / n_sample as f32
|
||||
} else {
|
||||
0.0
|
||||
};
|
||||
rows.push(EvalRow {
|
||||
idx,
|
||||
source: ex.source.clone(),
|
||||
@@ -714,9 +724,7 @@ impl LoraAdapterMetadata {
|
||||
/// Read just the LoRA self-description from a safetensors header, without
|
||||
/// loading any tensor data. Returns `None` when the file has no
|
||||
/// `rtx_csm_lora` metadata key (older adapters trained before Phase 12.5).
|
||||
pub fn read_lora_adapter_metadata<P: AsRef<Path>>(
|
||||
path: P,
|
||||
) -> Result<Option<LoraAdapterMetadata>> {
|
||||
pub fn read_lora_adapter_metadata<P: AsRef<Path>>(path: P) -> Result<Option<LoraAdapterMetadata>> {
|
||||
let data = std::fs::read(path.as_ref())?;
|
||||
let (_n, meta) = safetensors::SafeTensors::read_metadata(&data)
|
||||
.map_err(|e| CsmError::Other(anyhow::anyhow!("read safetensors header: {e}")))?;
|
||||
@@ -728,9 +736,8 @@ pub fn read_lora_adapter_metadata<P: AsRef<Path>>(
|
||||
Some(s) => s,
|
||||
None => return Ok(None),
|
||||
};
|
||||
let parsed: LoraAdapterMetadata = serde_json::from_str(json).map_err(|e| {
|
||||
CsmError::Other(anyhow::anyhow!("parse {LORA_METADATA_KEY}: {e}"))
|
||||
})?;
|
||||
let parsed: LoraAdapterMetadata = serde_json::from_str(json)
|
||||
.map_err(|e| CsmError::Other(anyhow::anyhow!("parse {LORA_METADATA_KEY}: {e}")))?;
|
||||
Ok(Some(parsed))
|
||||
}
|
||||
|
||||
@@ -769,9 +776,7 @@ pub fn apply_lora_adapter<P: AsRef<Path>>(
|
||||
);
|
||||
}
|
||||
|
||||
let resolved_rank = rank
|
||||
.or_else(|| meta.as_ref().map(|m| m.rank))
|
||||
.unwrap_or(8);
|
||||
let resolved_rank = rank.or_else(|| meta.as_ref().map(|m| m.rank)).unwrap_or(8);
|
||||
let resolved_alpha = alpha
|
||||
.or_else(|| meta.as_ref().map(|m| m.alpha))
|
||||
.unwrap_or(16.0);
|
||||
@@ -861,7 +866,11 @@ fn save_lora_adapter_inner<P: AsRef<Path>>(
|
||||
tracing::info!(
|
||||
"saved {n_tensors} LoRA tensors → {}{}",
|
||||
out.as_ref().display(),
|
||||
if header_meta.is_some() { " (with metadata)" } else { "" }
|
||||
if header_meta.is_some() {
|
||||
" (with metadata)"
|
||||
} else {
|
||||
""
|
||||
}
|
||||
);
|
||||
Ok(())
|
||||
}
|
||||
@@ -885,7 +894,10 @@ pub fn load_lora_adapter<P: AsRef<Path>>(
|
||||
tracing::warn!("safetensors has tensor `{name}` but VarMap has no matching Var");
|
||||
}
|
||||
}
|
||||
tracing::info!("loaded {count} LoRA tensors from {}", path.as_ref().display());
|
||||
tracing::info!(
|
||||
"loaded {count} LoRA tensors from {}",
|
||||
path.as_ref().display()
|
||||
);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user