osobhandClaude Opus 4.7
e61bd70e03
Two real bugs found via code inspection against HF source:
1. candle's .gelu() is the tanh approximation; PyTorch's default 'gelu'
activation (used in WavLM via ACT2FN['gelu']) is the exact erf-based
version. Switched all 3 sites (feature extractor convs, pos_conv,
FFN) from .gelu() to .gelu_erf() to match the reference.
2. gru_rel_pos_const lookup used vb.pp("name").get(shape, "") which
resolves to "<prefix>.name." (trailing dot) and fails to find the
tensor. The .or_else(|_| zeros) silently swallowed the failure,
leaving all 12 layers' gating constants at zero instead of the
trained values. Fixed to attn.get(shape, "gru_rel_pos_const") which
resolves correctly.
examples/wavlm_sv_inspect.rs: utility for sanity-checking specific
tensors inside converted safetensors (e.g. layer_weights).
Same-content same-speaker cosine: 0.9985 -> 0.9963 (≈unchanged).
Cross-content same-speaker cosine: 0.4882 -> 0.4118 (still drifting).
Phase 5d (Python reference comparison) remains the gate for
identifying the residual numerical drift.
Co-Authored-By: Claude Opus 4.7 (1M context) <[email protected]>
159 lines
4.1 KiB
TOML
159 lines
4.1 KiB
TOML
[package]
|
|
name = "rtx-csm"
|
|
version.workspace = true
|
|
edition.workspace = true
|
|
authors.workspace = true
|
|
license.workspace = true
|
|
repository.workspace = true
|
|
description = "Rust-native port of Sesame CSM-1B (Conversational Speech Model) on candle + moshi"
|
|
|
|
# NOTE: candle-transformers 0.8 (workspace pin) does NOT contain the `csm`
|
|
# model module — it was added in 0.9.0. We deliberately pull candle 0.9 + moshi
|
|
# 0.6 directly here, NOT via workspace deps. Cargo will compile candle 0.8 (for
|
|
# the rest of rustytorch) and candle 0.9 (for rtx-csm) side-by-side. No Tensor
|
|
# types are shared across that boundary today.
|
|
|
|
[dependencies]
|
|
# Candle 0.9 — required for the csm model module
|
|
candle-core = { version = "0.9.1", default-features = false }
|
|
candle-nn = { version = "0.9.1", default-features = false }
|
|
candle-transformers = { version = "0.9.1", default-features = false }
|
|
|
|
# Mimi neural audio codec: we use the HF-compatible `candle-transformers::models::mimi`
|
|
# (not the `moshi` crate, which expects different weight-key naming).
|
|
|
|
# Tokenizer (Llama-3.2 BPE)
|
|
tokenizers = { version = "0.20", default-features = false, features = ["onig"] }
|
|
|
|
# HF Hub asset resolution (synchronous via ureq + rustls)
|
|
hf-hub = { version = "0.5", default-features = false, features = ["ureq", "rustls-tls"] }
|
|
|
|
# Audio I/O
|
|
hound = "3.5"
|
|
symphonia = { version = "0.5", features = ["all"] }
|
|
rubato = "0.15"
|
|
# Loudness normalization (EBU R128 / ITU-R BS.1770-4)
|
|
ebur128 = "0.1"
|
|
|
|
# In-process ASR via whisper.cpp bindings. Optional via the `asr` feature
|
|
# because it pulls a C++ build (cmake + clang). Provides Metal acceleration.
|
|
whisper-rs = { version = "0.16", default-features = false, optional = true }
|
|
|
|
# Text normalization
|
|
unicode-normalization = "0.1"
|
|
regex = "1"
|
|
|
|
# Weight loading
|
|
safetensors = "0.4"
|
|
|
|
# Errors / logging / serde
|
|
anyhow.workspace = true
|
|
thiserror.workspace = true
|
|
tracing.workspace = true
|
|
serde.workspace = true
|
|
serde_json.workspace = true
|
|
|
|
# Numerics
|
|
half = "2.3"
|
|
rand = "0.8"
|
|
bytemuck = { version = "1.14", features = ["derive"] }
|
|
|
|
[dev-dependencies]
|
|
clap = { version = "4.5", features = ["derive"] }
|
|
tempfile = "3.0"
|
|
approx = "0.5"
|
|
tracing-subscriber = "0.3"
|
|
# For the TTS HTTP server example.
|
|
axum = "0.7"
|
|
tokio = { version = "1", features = ["macros", "rt-multi-thread", "signal", "sync"] }
|
|
tower = "0.5"
|
|
tower-http = { version = "0.6", features = ["trace"] }
|
|
|
|
[features]
|
|
default = ["cpu"]
|
|
cpu = []
|
|
cuda = ["candle-core/cuda", "candle-nn/cuda", "candle-transformers/cuda"]
|
|
metal = ["candle-core/metal", "candle-nn/metal", "candle-transformers/metal"]
|
|
accelerate = ["candle-core/accelerate", "candle-nn/accelerate"]
|
|
mkl = ["candle-core/mkl", "candle-nn/mkl"]
|
|
# In-process Whisper ASR via whisper.cpp bindings. Brings in C++ build deps.
|
|
asr = ["dep:whisper-rs"]
|
|
asr-metal = ["asr", "whisper-rs/metal"]
|
|
asr-cuda = ["asr", "whisper-rs/cuda"]
|
|
|
|
[[example]]
|
|
name = "generate"
|
|
path = "examples/generate.rs"
|
|
|
|
[[example]]
|
|
name = "bench"
|
|
path = "examples/bench.rs"
|
|
|
|
[[example]]
|
|
name = "quantize"
|
|
path = "examples/quantize.rs"
|
|
|
|
[[example]]
|
|
name = "inspect_gguf"
|
|
path = "examples/inspect_gguf.rs"
|
|
|
|
[[example]]
|
|
name = "qmatmul_repro"
|
|
path = "examples/qmatmul_repro.rs"
|
|
|
|
[[example]]
|
|
name = "qmm_layer_diff"
|
|
path = "examples/qmm_layer_diff.rs"
|
|
|
|
[[example]]
|
|
name = "lora_train_step"
|
|
path = "examples/lora_train_step.rs"
|
|
|
|
[[example]]
|
|
name = "forward_loss_demo"
|
|
path = "examples/forward_loss_demo.rs"
|
|
|
|
[[example]]
|
|
name = "lora_finetune_step"
|
|
path = "examples/lora_finetune_step.rs"
|
|
|
|
[[example]]
|
|
name = "tts_server"
|
|
path = "examples/tts_server.rs"
|
|
|
|
[[example]]
|
|
name = "lora_train"
|
|
path = "examples/lora_train.rs"
|
|
|
|
[[example]]
|
|
name = "audioseal_inspect"
|
|
path = "examples/audioseal_inspect.rs"
|
|
|
|
[[example]]
|
|
name = "audioseal_convert"
|
|
path = "examples/audioseal_convert.rs"
|
|
|
|
[[example]]
|
|
name = "audioseal_demo"
|
|
path = "examples/audioseal_demo.rs"
|
|
|
|
[[example]]
|
|
name = "audioseal_apply"
|
|
path = "examples/audioseal_apply.rs"
|
|
|
|
[[example]]
|
|
name = "wavlm_sv_convert"
|
|
path = "examples/wavlm_sv_convert.rs"
|
|
|
|
[[example]]
|
|
name = "wavlm_sv_demo"
|
|
path = "examples/wavlm_sv_demo.rs"
|
|
|
|
[[example]]
|
|
name = "wavlm_sv_inspect"
|
|
path = "examples/wavlm_sv_inspect.rs"
|
|
|
|
[[example]]
|
|
name = "pipeline"
|
|
path = "examples/pipeline.rs"
|