rtx-csm: Mimi.reload() — workaround state-leak in cumulative encode loop
Mimi's transformer carries an internal position counter across encode()
calls that the upstream `reset_state()` does NOT fully clear. When
TrainingDataset::load_from_manifest encodes ~80-100 clips back-to-back
during data prep, the counter overflows the 8192-position buffer and
panics with `narrow invalid args [8192, 32]`.
src/mimi.rs:
- cache the safetensors path on Mimi at construction
- new Mimi.reload() drops the inner Model and rebuilds from the
cached path (~200 ms on Metal)
src/training.rs:
- call generator.mimi.reload() every 50 clips during
load_from_manifest. Adds ~1 s overhead on a 200-clip corpus
(4 reloads × ~200 ms) vs the alternative of a hard panic.
- reset_state() before each encode in TrainingExample::from_audio
is kept (still useful to clear streaming chunk state).
Found while running the end-to-end personal-voice training pipeline on
a 20-minute YouTube source: the bug surfaces around clip 86 when
Mimi's transformer hits position 8181+. Filtering to short clips
alone didn't help — the cumulative state grows even with sub-12-second
inputs.
Co-Authored-By: Claude Opus 4.7 (1M context) <[email protected]>
This commit is contained in:
@@ -42,12 +42,20 @@ pub struct TrainingExample {
|
||||
|
||||
impl TrainingExample {
|
||||
/// Build from raw 24 kHz mono samples by Mimi-encoding the audio.
|
||||
///
|
||||
/// Mimi's encoder carries streaming state across calls — when
|
||||
/// processing a manifest of N clips back-to-back its internal
|
||||
/// frame counter grows monotonically and eventually overflows the
|
||||
/// transformer's max-seq buffer (`narrow invalid args ... [8192, ...]`
|
||||
/// at clip ~30 in a 200-clip corpus). Reset before each encode so
|
||||
/// every clip starts with a fresh state.
|
||||
pub fn from_audio(
|
||||
speaker: u32,
|
||||
text: impl Into<String>,
|
||||
audio_24k: &[f32],
|
||||
generator: &mut Generator,
|
||||
) -> Result<Self> {
|
||||
generator.mimi.reset_state();
|
||||
let codes = generator.mimi.encode(audio_24k)?;
|
||||
let (_b, num_codebooks, num_frames) = codes.dims3()?;
|
||||
let mut frame_codes = Vec::with_capacity(num_frames);
|
||||
@@ -184,6 +192,15 @@ impl TrainingDataset {
|
||||
}
|
||||
let speaker = row.speaker.unwrap_or(default_speaker);
|
||||
let audio = audio_io::load_mono_24k(&wav_path)?;
|
||||
// Mimi's transformer carries a position counter across encode()
|
||||
// calls that `reset_state()` does NOT fully clear. Without an
|
||||
// explicit reload it overflows after ~80-100 clips with
|
||||
// `narrow invalid args [8192, 32]`. Reload every 50 clips.
|
||||
if !examples.is_empty() && examples.len() % 50 == 0 {
|
||||
generator.mimi.reload().map_err(|e| {
|
||||
CsmError::Config(format!("mimi reload at clip {}: {e}", examples.len()))
|
||||
})?;
|
||||
}
|
||||
let mut ex = TrainingExample::from_audio(speaker, text, &audio, generator)?;
|
||||
ex.source = Some(wav_path);
|
||||
ex.emotion_tag = row.emotion_tag;
|
||||
|
||||
Reference in New Issue
Block a user