osobhandClaude Opus 4.7
d42aba0f1b
Three real bugs found while running a YouTube → train → eval pipeline end-to-end on real corpora: 1. emotion2vec was producing near-constant logits regardless of input. Per config.yaml `normalize: true` — data2vec2/emotion2vec expects per-utterance zero-mean unit-variance normalization on the raw waveform before the local_encoder. Added inside the EmotionDetector trait impl so all callers get it. Verified empirically: 4 different audio inputs (Carlini talk, audience question, McConaughey speech) now produce different argmax classes. Before fix: all 4 produced identical logits. 2. The 9→5 emotion fold was collapsing every real-world clip to [excited]. happy / surprised / other all mapped to Excited covered ~95% of natural speech. Replaced with a direct 9-class identity mapping; EmotionLabel gained Disgusted, Fearful, Happy, Surprised, Unk variants. Now: 132 [surprised] + 12 [excited] across the Carlini corpus instead of 144 [excited]. 3. lora_train_emotional --peak-lr / --epochs flags. The canned 3-stage recipe over-fits on small (~100 clip) corpora at extended rank 8; users need to tune. (The recipe stays as defaults; flags are pure overrides.) Plus diagnostic: examples/emotion2vec_probe — feed real audio files into emotion2vec and dump per-class logits. Used to find bug #1. Lib suite still 131/131 (the test that locked the 9→5 fold updated to lock the new identity mapping). Co-Authored-By: Claude Opus 4.7 (1M context) <[email protected]>
328 lines
9.1 KiB
TOML
328 lines
9.1 KiB
TOML
[package]
|
|
name = "rtx-csm"
|
|
version.workspace = true
|
|
edition.workspace = true
|
|
authors.workspace = true
|
|
license.workspace = true
|
|
repository.workspace = true
|
|
description = "Rust-native port of Sesame CSM-1B (Conversational Speech Model) on candle + moshi"
|
|
|
|
# NOTE: candle-transformers 0.8 (workspace pin) does NOT contain the `csm`
|
|
# model module — it was added in 0.9.0. We deliberately pull candle 0.9 + moshi
|
|
# 0.6 directly here, NOT via workspace deps. Cargo will compile candle 0.8 (for
|
|
# the rest of rustytorch) and candle 0.9 (for rtx-csm) side-by-side. No Tensor
|
|
# types are shared across that boundary today.
|
|
|
|
[dependencies]
|
|
# Candle 0.9 — required for the csm model module
|
|
candle-core = { version = "0.9.1", default-features = false }
|
|
candle-nn = { version = "0.9.1", default-features = false }
|
|
candle-transformers = { version = "0.9.1", default-features = false }
|
|
|
|
# Kyutai's moshi crate: provides streaming STT (asr.rs + lm.rs) on top of
|
|
# candle 0.9.1. We use moshi::{asr, lm, mimi} for STT integration. Note:
|
|
# moshi::mimi uses a different weight-key naming than HF's kyutai/mimi
|
|
# (older Kyutai split-format with weight_g/weight_v); we keep our existing
|
|
# Mimi loader on candle_transformers::models::mimi for the HF format. The
|
|
# STT path uses Kyutai's pytorch_mimi file which IS in moshi's expected
|
|
# naming, so they coexist cleanly in different model instances.
|
|
moshi = { version = "0.6.4", default-features = false }
|
|
# SentencePiece tokenizer for Kyutai STT detokenization (token IDs → text).
|
|
sentencepiece = "0.13"
|
|
|
|
# Mimi neural audio codec: we use the HF-compatible `candle-transformers::models::mimi`
|
|
# (not the `moshi` crate, which expects different weight-key naming).
|
|
|
|
# Tokenizer (Llama-3.2 BPE)
|
|
tokenizers = { version = "0.20", default-features = false, features = ["onig"] }
|
|
|
|
# HF Hub asset resolution (synchronous via ureq + rustls)
|
|
hf-hub = { version = "0.5", default-features = false, features = ["ureq", "rustls-tls"] }
|
|
# Direct ureq access for fetching upstream raw assets (e.g. Silero V5
|
|
# safetensors from github raw). Re-exposes the same transport hf-hub
|
|
# already pulls in, no new linkage cost.
|
|
ureq = { version = "3", default-features = false, features = ["rustls"] }
|
|
|
|
# Audio I/O
|
|
hound = "3.5"
|
|
symphonia = { version = "0.5", features = ["all"] }
|
|
rubato = "0.15"
|
|
# FFT primitives for the SilentCipher STFT (Phase 10). Pure Rust,
|
|
# zero C linkage. Ships in the default build because it's <100 KB
|
|
# of compiled code.
|
|
rustfft = "6.2"
|
|
# Loudness normalization (EBU R128 / ITU-R BS.1770-4)
|
|
ebur128 = "0.1"
|
|
|
|
# In-process ASR via whisper.cpp bindings. Optional via the `asr` feature
|
|
# because it pulls a C++ build (cmake + clang). Provides Metal acceleration.
|
|
whisper-rs = { version = "0.16", default-features = false, optional = true }
|
|
|
|
# Silero V5 VAD via ONNX Runtime (`ort` crate). Optional via the `vad`
|
|
# feature because it introduces a second ML inference runtime alongside
|
|
# candle — Phase 8.1.2's `ort_conflict_probe` binary verifies it doesn't
|
|
# regress CSM Metal inference the way whisper-rs (ggml) did.
|
|
voice_activity_detector = { version = "0.2", optional = true }
|
|
|
|
# Text normalization
|
|
unicode-normalization = "0.1"
|
|
regex = "1"
|
|
|
|
# Weight loading. Matches candle 0.9's transitive pin so candle's
|
|
# `Tensor: View` impl applies — letting us call
|
|
# `safetensors::serialize_to_file(&map, &Some(metadata), path)` directly
|
|
# (candle's `save` wrapper doesn't expose the metadata parameter, which we
|
|
# need for Phase 12.5 self-describing LoRA adapters).
|
|
safetensors = "0.7"
|
|
|
|
# Errors / logging / serde
|
|
anyhow.workspace = true
|
|
thiserror.workspace = true
|
|
tracing.workspace = true
|
|
serde.workspace = true
|
|
serde_json.workspace = true
|
|
|
|
# Numerics
|
|
half = "2.3"
|
|
rand = "0.8"
|
|
bytemuck = { version = "1.14", features = ["derive"] }
|
|
|
|
# Async + HTTP for the LlmClient abstraction (Phase 6b). Promoted from
|
|
# dev-dependency to regular dependency so the trait is part of the public
|
|
# library surface.
|
|
tokio = { version = "1", features = ["macros", "rt-multi-thread", "sync"] }
|
|
futures-util = "0.3"
|
|
reqwest = { version = "0.12", default-features = false, features = ["json", "stream", "rustls-tls"] }
|
|
async-trait = "0.1"
|
|
eventsource-stream = "0.2"
|
|
|
|
[dev-dependencies]
|
|
clap = { version = "4.5", features = ["derive"] }
|
|
tempfile = "3.0"
|
|
approx = "0.5"
|
|
tracing-subscriber = "0.3"
|
|
# For the TTS HTTP server + converse_server WebSocket examples.
|
|
axum = { version = "0.7", features = ["multipart", "ws"] }
|
|
# WebSocket client for examples/converse_client.
|
|
tokio-tungstenite = { version = "0.24", default-features = false, features = ["connect", "rustls-tls-webpki-roots"] }
|
|
# tokio with extra features (signal handler) needed by tts_server.
|
|
tokio = { version = "1", features = ["macros", "rt-multi-thread", "signal", "sync"] }
|
|
tower = "0.5"
|
|
tower-http = { version = "0.6", features = ["trace"] }
|
|
# Multipart support added on top of the public reqwest dep for tts_server_bench.
|
|
reqwest = { version = "0.12", default-features = false, features = ["json", "multipart", "rustls-tls"] }
|
|
|
|
[features]
|
|
default = ["cpu"]
|
|
cpu = []
|
|
cuda = ["candle-core/cuda", "candle-nn/cuda", "candle-transformers/cuda"]
|
|
metal = ["candle-core/metal", "candle-nn/metal", "candle-transformers/metal"]
|
|
accelerate = ["candle-core/accelerate", "candle-nn/accelerate"]
|
|
mkl = ["candle-core/mkl", "candle-nn/mkl"]
|
|
# In-process Whisper ASR via whisper.cpp bindings. Brings in C++ build deps.
|
|
asr = ["dep:whisper-rs"]
|
|
asr-metal = ["asr", "whisper-rs/metal"]
|
|
asr-cuda = ["asr", "whisper-rs/cuda"]
|
|
# Silero V5 VAD via ONNX Runtime. Use `--features metal,vad` to enable
|
|
# alongside CSM. Phase 8.1.2 must pass before relying on this in
|
|
# production.
|
|
vad = ["dep:voice_activity_detector"]
|
|
|
|
[[example]]
|
|
name = "generate"
|
|
path = "examples/generate.rs"
|
|
|
|
[[example]]
|
|
name = "bench"
|
|
path = "examples/bench.rs"
|
|
|
|
[[example]]
|
|
name = "quantize"
|
|
path = "examples/quantize.rs"
|
|
|
|
[[example]]
|
|
name = "inspect_gguf"
|
|
path = "examples/inspect_gguf.rs"
|
|
|
|
[[example]]
|
|
name = "qmatmul_repro"
|
|
path = "examples/qmatmul_repro.rs"
|
|
|
|
[[example]]
|
|
name = "qmm_layer_diff"
|
|
path = "examples/qmm_layer_diff.rs"
|
|
|
|
[[example]]
|
|
name = "lora_train_step"
|
|
path = "examples/lora_train_step.rs"
|
|
|
|
[[example]]
|
|
name = "forward_loss_demo"
|
|
path = "examples/forward_loss_demo.rs"
|
|
|
|
[[example]]
|
|
name = "lora_finetune_step"
|
|
path = "examples/lora_finetune_step.rs"
|
|
|
|
[[example]]
|
|
name = "tts_server"
|
|
path = "examples/tts_server.rs"
|
|
|
|
[[example]]
|
|
name = "lora_train"
|
|
path = "examples/lora_train.rs"
|
|
|
|
[[example]]
|
|
name = "lora_train_emotional"
|
|
path = "examples/lora_train_emotional.rs"
|
|
|
|
[[example]]
|
|
name = "lora_eval"
|
|
path = "examples/lora_eval.rs"
|
|
|
|
[[example]]
|
|
name = "diarize"
|
|
path = "examples/diarize.rs"
|
|
|
|
[[example]]
|
|
name = "audio_to_manifest"
|
|
path = "examples/audio_to_manifest.rs"
|
|
|
|
[[example]]
|
|
name = "emotion2vec_inspect"
|
|
path = "examples/emotion2vec_inspect.rs"
|
|
|
|
[[example]]
|
|
name = "emotion2vec_smoke"
|
|
path = "examples/emotion2vec_smoke.rs"
|
|
|
|
[[example]]
|
|
name = "emotion2vec_probe"
|
|
path = "examples/emotion2vec_probe.rs"
|
|
|
|
[[example]]
|
|
name = "wav2vec2_inspect"
|
|
path = "examples/wav2vec2_inspect.rs"
|
|
|
|
[[example]]
|
|
name = "wav2vec2_smoke"
|
|
path = "examples/wav2vec2_smoke.rs"
|
|
|
|
[[example]]
|
|
name = "audioseal_inspect"
|
|
path = "examples/audioseal_inspect.rs"
|
|
|
|
[[example]]
|
|
name = "audioseal_convert"
|
|
path = "examples/audioseal_convert.rs"
|
|
|
|
[[example]]
|
|
name = "audioseal_demo"
|
|
path = "examples/audioseal_demo.rs"
|
|
|
|
[[example]]
|
|
name = "audioseal_apply"
|
|
path = "examples/audioseal_apply.rs"
|
|
|
|
[[example]]
|
|
name = "wavlm_sv_convert"
|
|
path = "examples/wavlm_sv_convert.rs"
|
|
|
|
[[example]]
|
|
name = "wavlm_sv_demo"
|
|
path = "examples/wavlm_sv_demo.rs"
|
|
|
|
[[example]]
|
|
name = "wavlm_sv_inspect"
|
|
path = "examples/wavlm_sv_inspect.rs"
|
|
|
|
[[example]]
|
|
name = "pipeline"
|
|
path = "examples/pipeline.rs"
|
|
|
|
[[example]]
|
|
name = "generate_long"
|
|
path = "examples/generate_long.rs"
|
|
|
|
[[example]]
|
|
name = "tts_server_bench"
|
|
path = "examples/tts_server_bench.rs"
|
|
|
|
[[example]]
|
|
name = "stt_demo"
|
|
path = "examples/stt_demo.rs"
|
|
|
|
[[example]]
|
|
name = "llm_chat"
|
|
path = "examples/llm_chat.rs"
|
|
|
|
[[example]]
|
|
name = "converse"
|
|
path = "examples/converse.rs"
|
|
|
|
[[example]]
|
|
name = "converse_server"
|
|
path = "examples/converse_server.rs"
|
|
|
|
[[example]]
|
|
name = "converse_client"
|
|
path = "examples/converse_client.rs"
|
|
|
|
[[example]]
|
|
name = "converse_server_bench"
|
|
path = "examples/converse_server_bench.rs"
|
|
|
|
[[example]]
|
|
name = "llm_extra_body_smoke"
|
|
path = "examples/llm_extra_body_smoke.rs"
|
|
|
|
[[example]]
|
|
name = "stt_profile"
|
|
path = "examples/stt_profile.rs"
|
|
|
|
[[example]]
|
|
name = "whisper_profile"
|
|
path = "examples/whisper_profile.rs"
|
|
required-features = ["asr"]
|
|
|
|
[[example]]
|
|
name = "ort_conflict_probe"
|
|
path = "examples/ort_conflict_probe.rs"
|
|
required-features = ["vad"]
|
|
|
|
[[example]]
|
|
name = "make_silence_test"
|
|
path = "examples/make_silence_test.rs"
|
|
|
|
[[example]]
|
|
name = "moonshine_inspect"
|
|
path = "examples/moonshine_inspect.rs"
|
|
|
|
[[example]]
|
|
name = "moonshine_smoke"
|
|
path = "examples/moonshine_smoke.rs"
|
|
|
|
[[example]]
|
|
name = "moonshine_transcribe"
|
|
path = "examples/moonshine_transcribe.rs"
|
|
|
|
[[example]]
|
|
name = "moonshine_profile"
|
|
path = "examples/moonshine_profile.rs"
|
|
|
|
[[example]]
|
|
name = "silentcipher_inspect"
|
|
path = "examples/silentcipher_inspect.rs"
|
|
|
|
[[example]]
|
|
name = "silentcipher_smoke"
|
|
path = "examples/silentcipher_smoke.rs"
|
|
|
|
[[example]]
|
|
name = "silentcipher_apply"
|
|
path = "examples/silentcipher_apply.rs"
|
|
|
|
[[example]]
|
|
name = "silero_vad_smoke"
|
|
path = "examples/silero_vad_smoke.rs"
|