rtx-csm: WavLM-SV bug fixes — gelu_erf + gru_rel_pos_const loading
Two real bugs found via code inspection against HF source:
1. candle's .gelu() is the tanh approximation; PyTorch's default 'gelu'
activation (used in WavLM via ACT2FN['gelu']) is the exact erf-based
version. Switched all 3 sites (feature extractor convs, pos_conv,
FFN) from .gelu() to .gelu_erf() to match the reference.
2. gru_rel_pos_const lookup used vb.pp("name").get(shape, "") which
resolves to "<prefix>.name." (trailing dot) and fails to find the
tensor. The .or_else(|_| zeros) silently swallowed the failure,
leaving all 12 layers' gating constants at zero instead of the
trained values. Fixed to attn.get(shape, "gru_rel_pos_const") which
resolves correctly.
examples/wavlm_sv_inspect.rs: utility for sanity-checking specific
tensors inside converted safetensors (e.g. layer_weights).
Same-content same-speaker cosine: 0.9985 -> 0.9963 (≈unchanged).
Cross-content same-speaker cosine: 0.4882 -> 0.4118 (still drifting).
Phase 5d (Python reference comparison) remains the gate for
identifying the residual numerical drift.
Co-Authored-By: Claude Opus 4.7 (1M context) <[email protected]>
This commit is contained in:
@@ -100,7 +100,7 @@ impl ConvLayer {
|
||||
Some(gn) => h.apply(gn)?,
|
||||
None => h,
|
||||
};
|
||||
h.gelu()
|
||||
h.gelu_erf()
|
||||
}
|
||||
}
|
||||
|
||||
@@ -237,7 +237,7 @@ impl Module for PosConv {
|
||||
} else {
|
||||
h
|
||||
};
|
||||
let h = h.gelu()?;
|
||||
let h = h.gelu_erf()?;
|
||||
// Back to (B, T, C).
|
||||
h.transpose(1, 2)?.contiguous()
|
||||
}
|
||||
@@ -298,12 +298,7 @@ impl WavLmEncoderLayer {
|
||||
let fc1 = linear(HIDDEN_DIM, FFN_DIM, ff.pp("intermediate_dense"))?;
|
||||
let fc2 = linear(FFN_DIM, HIDDEN_DIM, ff.pp("output_dense"))?;
|
||||
let final_norm = layer_norm(HIDDEN_DIM, 1e-5, vb.pp("final_layer_norm"))?;
|
||||
let gru_rel_pos_const = attn
|
||||
.pp("gru_rel_pos_const")
|
||||
.get((1, NUM_HEADS, 1, 1), "")
|
||||
.or_else(|_| {
|
||||
Tensor::zeros((1, NUM_HEADS, 1, 1), vb.dtype(), vb.device())
|
||||
})?;
|
||||
let gru_rel_pos_const = attn.get((1, NUM_HEADS, 1, 1), "gru_rel_pos_const")?;
|
||||
let gru_rel_pos_linear = linear(HEAD_DIM, 8, attn.pp("gru_rel_pos_linear"))?;
|
||||
let rel_attn_embed = if layer_idx == 0 {
|
||||
Some(candle_nn::embedding(
|
||||
@@ -440,7 +435,7 @@ impl WavLmEncoderLayer {
|
||||
let h = (xs + attn_out)?;
|
||||
let h = h.apply(&self.attn_norm)?;
|
||||
// FFN: 768 → 3072 → GELU → 768
|
||||
let ffn = h.apply(&self.fc1)?.gelu()?.apply(&self.fc2)?;
|
||||
let ffn = h.apply(&self.fc1)?.gelu_erf()?.apply(&self.fc2)?;
|
||||
let h = (h + ffn)?;
|
||||
let h = h.apply(&self.final_norm)?;
|
||||
Ok((h, bias.clone()))
|
||||
|
||||
Reference in New Issue
Block a user