fix(gaps): G0/G2/G5/G8 — eliminate unimplemented! panics, re-enable rtx-distributed, rtx-tts, fix multimodal forward

G0 (Critical): Replace 45 unimplemented!() panics across three GPU backends
- rtx-backend-cuda: sin/cos/tanh via PTX, relu/sigmoid/leaky_relu/elu via activation.rs,
  pow/clamp/gt_scalar via unary.rs, var/var_dim host-side, conv2d/max_pool2d/avg_pool2d
  CPU fallback in new ops/conv.rs; new PTX kernels in element_wise.cu
- rtx-backend-rocm: all 15 ops via CPU round-trip (to_vec → compute → from_slice)
- rtx-backend-sycl: all 15 ops via CPU round-trip (to_host → compute → from_data)

G2 (High): Re-add rtx-distributed to workspace
- Vendor 4 minimal RNCCL stub crates at crates/vendor/rnccl/*
- Update rtx-distributed RNCCL path deps to point at stubs (../../../../RNCCL/* → ../../vendor/rnccl/*)
- Remove rtx-distributed from workspace exclude list, add to members

G5 (Medium): Re-enable rtx-tts (213 tests restored)
- Fix 15 rtx-nn API drift issues: LayerNorm::new, Conv1d::from_config, Conv1dPadding::Zeros,
  Dropout::new(p, device), tensor methods (relu/tanh/sigmoid/cat/stack), squeeze(Some(n)),
  to_vec() turbofish removal, Tensor::randn with &[...] slices

G8 (Low): Quantum stubs + multimodal forward bug
- rtx-timeseries: remove dead quantum/neuromorphic TODO comment blocks (no module files exist)
- rtx-multimodal/fusion/transformer.rs: wire TransformerBlock loop in forward()
- rtx-multimodal/fusion/strategies.rs: wire bottleneck_layers loop in forward()
- rtx-transformers/architectures/transformer_block.rs: add forward() method (pre-norm residuals;
  full attention+FFN pending when those sub-layers are wired)

Co-Authored-By: Claude Sonnet 4.6 <[email protected]>
This commit is contained in:
Omar Sobh
2026-06-26 13:40:23 +00:00
co-authored by Claude Sonnet 4.6
parent 57e5252caa
commit 228137555f
37 changed files with 1509 additions and 215 deletions
@@ -3,14 +3,18 @@
//! FastSpeech2 is a non-autoregressive TTS model that predicts mel spectrograms
//! from phoneme sequences using variance adaptors for duration, pitch, and energy.
// Legacy rtx_nn layer types are deprecated in favour of Generic* variants.
// rtx-tts uses the concrete-tensor API and does not yet require backend dispatch.
#![allow(deprecated)]
use crate::acoustic::{AcousticModel, MelSpectrogramConfig, VarianceAdaptor, VarianceAdaptorConfig};
use crate::Result;
use rtx_nn::layers::{Module, linear::Linear, attention::{MultiHeadAttention, AttentionConfig}};
use rtx_nn::layers::activation::ReLU;
use rtx_nn::layers::norm::layer_norm::{LayerNorm, LayerNormConfig};
use rtx_nn::layers::norm::layer_norm::LayerNorm;
use rtx_nn::layers::dropout::Dropout;
use rtx_nn::layers::embedding::{Embedding, EmbeddingConfig};
use rtx_nn::layers::conv::{Conv1d, Conv1dConfig};
use rtx_nn::layers::conv::{Conv1d, Conv1dConfig, Conv1dPadding};
use rtx_tensor::{Tensor, Device, DType};
use serde::{Deserialize, Serialize};
@@ -116,8 +120,7 @@ impl FFTBlock {
let self_attn = MultiHeadAttention::new(attn_config, device)
.map_err(|e| crate::TtsError::ModelError(format!("Attention creation failed: {e}")))?;
let attn_norm_config = LayerNormConfig::new(vec![config.hidden_size]);
let attn_norm = LayerNorm::new(attn_norm_config, device)
let attn_norm = LayerNorm::new(vec![config.hidden_size], 1e-5, true, device)
.map_err(|e| crate::TtsError::ModelError(format!("LayerNorm creation failed: {e}")))?;
// Conv layers for feed-forward
@@ -127,6 +130,7 @@ impl FFTBlock {
kernel_size: 3,
stride: 1,
padding: 1,
padding_mode: Conv1dPadding::Zeros(1),
dilation: 1,
groups: 1,
bias: true,
@@ -138,18 +142,18 @@ impl FFTBlock {
kernel_size: 3,
stride: 1,
padding: 1,
padding_mode: Conv1dPadding::Zeros(1),
dilation: 1,
groups: 1,
bias: true,
};
let conv1 = Conv1d::with_config(conv1_config, device)
let conv1 = Conv1d::from_config(conv1_config, device)
.map_err(|e| crate::TtsError::ModelError(format!("Conv1d creation failed: {e}")))?;
let conv2 = Conv1d::with_config(conv2_config, device)
let conv2 = Conv1d::from_config(conv2_config, device)
.map_err(|e| crate::TtsError::ModelError(format!("Conv1d creation failed: {e}")))?;
let conv_norm_config = LayerNormConfig::new(vec![config.hidden_size]);
let conv_norm = LayerNorm::new(conv_norm_config, device)
let conv_norm = LayerNorm::new(vec![config.hidden_size], 1e-5, true, device)
.map_err(|e| crate::TtsError::ModelError(format!("LayerNorm creation failed: {e}")))?;
Ok(Self {
@@ -158,7 +162,7 @@ impl FFTBlock {
conv1,
conv2,
conv_norm,
dropout: Dropout::new(config.dropout),
dropout: Dropout::new(config.dropout, device),
hidden_size: config.hidden_size,
training: true,
})
@@ -190,7 +194,7 @@ impl FFTBlock {
let conv1_out = self.conv1.forward(&ff_input)
.map_err(|e| crate::TtsError::ModelError(format!("Conv1 failed: {e}")))?;
let activated = rtx_tensor::ops::relu(&conv1_out)
let activated = conv1_out.relu()
.map_err(|e| crate::TtsError::TensorError(format!("ReLU failed: {e}")))?;
let conv2_out = self.conv2.forward(&activated)
@@ -366,10 +370,10 @@ impl AcousticModel for FastSpeech2 {
let phoneme_embedded = self.phoneme_emb.forward(phonemes)
.map_err(|e| crate::TtsError::ModelError(format!("Phoneme embedding failed: {e}")))?;
// Create position indices
let positions: Vec<i64> = (0..seq_len as i64).collect();
// Create position indices (cast to f32; Embedding interprets them as integer indices)
let positions: Vec<f32> = (0..seq_len as u32).map(|i| i as f32).collect();
let position_ids = Tensor::from_vec(
positions.repeat(batch_size),
positions.into_iter().cycle().take(batch_size * seq_len).collect(),
&[batch_size, seq_len],
&self.device
).map_err(|e| crate::TtsError::TensorError(format!("Position tensor failed: {e}")))?;
@@ -481,7 +485,7 @@ mod tests {
let batch_size = 2;
let seq_len = 10;
let phonemes = Tensor::randint(0, config.phoneme_vocab_size as i64, &[batch_size, seq_len], &device).unwrap();
let phonemes = Tensor::randint(0, config.phoneme_vocab_size as i32, &[batch_size, seq_len], &device).unwrap();
let encoded = model.encode(&phonemes, None);
assert!(encoded.is_ok());
@@ -501,7 +505,7 @@ mod tests {
let batch_size = 2;
let seq_len = 10;
let phonemes = Tensor::randint(0, config.phoneme_vocab_size as i64, &[batch_size, seq_len], &device).unwrap();
let phonemes = Tensor::randint(0, config.phoneme_vocab_size as i32, &[batch_size, seq_len], &device).unwrap();
let mel = model.forward(&phonemes, None);
assert!(mel.is_ok());
@@ -638,7 +642,7 @@ mod tests {
config.decoder_layers = 2;
let model = FastSpeech2::new(config.clone(), &device).unwrap();
let phonemes = Tensor::randint(0, config.phoneme_vocab_size as i64, &[1, 5], &device).unwrap();
let phonemes = Tensor::randint(0, config.phoneme_vocab_size as i32, &[1, 5], &device).unwrap();
let mel = model.forward(&phonemes, None);
assert!(mel.is_ok());
}
+29 -22
View File
@@ -3,9 +3,13 @@
//! Tacotron 2 is an autoregressive sequence-to-sequence model that generates
//! mel spectrograms from character/phoneme sequences using attention.
// Legacy rtx_nn layer types are deprecated in favour of Generic* variants.
// rtx-tts uses the concrete-tensor API and does not yet require backend dispatch.
#![allow(deprecated)]
use crate::acoustic::{AcousticModel, MelSpectrogramConfig};
use crate::Result;
use rtx_nn::layers::{Module, linear::Linear, conv::{Conv1d, Conv1dConfig}};
use rtx_nn::layers::{Module, linear::Linear, conv::{Conv1d, Conv1dConfig, Conv1dPadding}};
use rtx_nn::layers::activation::{ReLU, Tanh, Sigmoid};
use rtx_nn::layers::dropout::Dropout;
use rtx_nn::layers::embedding::{Embedding, EmbeddingConfig};
@@ -108,7 +112,7 @@ impl PreNet {
Ok(Self {
fc1,
fc2,
dropout: Dropout::new(dropout),
dropout: Dropout::new(dropout, device),
activation: ReLU::new(),
training: true,
})
@@ -165,12 +169,13 @@ impl PostNet {
kernel_size: 5,
stride: 1,
padding: 2,
padding_mode: Conv1dPadding::Zeros(2),
dilation: 1,
groups: 1,
bias: true,
};
conv_layers.push(Conv1d::with_config(conv_config, device)
conv_layers.push(Conv1d::from_config(conv_config, device)
.map_err(|e| crate::TtsError::ModelError(format!("Conv1d creation failed: {e}")))?);
// Simplified batch norm
@@ -181,7 +186,7 @@ impl PostNet {
Ok(Self {
conv_layers,
batch_norm_layers,
dropout: Dropout::new(0.5),
dropout: Dropout::new(0.5, device),
tanh: Tanh::new(),
training: true,
})
@@ -259,12 +264,13 @@ impl LocationAttention {
kernel_size: 31,
stride: 1,
padding: 15,
padding_mode: Conv1dPadding::Zeros(15),
dilation: 1,
groups: 1,
bias: true,
};
let location_conv = Conv1d::with_config(location_conv_config, device)
let location_conv = Conv1d::from_config(location_conv_config, device)
.map_err(|e| crate::TtsError::ModelError(format!("Location conv creation failed: {e}")))?;
let location_layer = Linear::new(32, attention_dim, false, device)
@@ -316,13 +322,13 @@ impl LocationAttention {
let energies = (&energies + &processed_location)
.map_err(|e| crate::TtsError::TensorError(format!("Add failed: {e}")))?;
let energies = rtx_tensor::ops::tanh(&energies)
let energies = energies.tanh()
.map_err(|e| crate::TtsError::TensorError(format!("Tanh failed: {e}")))?;
let alignment = self.v.forward(&energies)
.map_err(|e| crate::TtsError::ModelError(format!("V layer failed: {e}")))?;
let alignment = alignment.squeeze(2)
let alignment = alignment.squeeze(Some(2))
.map_err(|e| crate::TtsError::TensorError(format!("Squeeze failed: {e}")))?;
// Softmax
@@ -355,19 +361,20 @@ impl Encoder {
kernel_size: 5,
stride: 1,
padding: 2,
padding_mode: Conv1dPadding::Zeros(2),
dilation: 1,
groups: 1,
bias: true,
};
conv_layers.push(Conv1d::with_config(conv_config, device)
conv_layers.push(Conv1d::from_config(conv_config, device)
.map_err(|e| crate::TtsError::ModelError(format!("Conv1d creation failed: {e}")))?);
}
Ok(Self {
embedding,
conv_layers,
dropout: Dropout::new(config.dropout),
dropout: Dropout::new(config.dropout, device),
device: device.clone(),
training: true,
})
@@ -385,7 +392,7 @@ impl Encoder {
let conv_out = conv.forward(&hidden)
.map_err(|e| crate::TtsError::ModelError(format!("Conv forward failed: {e}")))?;
hidden = rtx_tensor::ops::relu(&conv_out)
hidden = conv_out.relu()
.map_err(|e| crate::TtsError::TensorError(format!("ReLU failed: {e}")))?;
if self.training {
@@ -501,11 +508,11 @@ impl Tacotron2 {
let attention_context = rtx_tensor::ops::matmul(&attention_weights_expanded, &encoder_outputs_transposed)
.map_err(|e| crate::TtsError::TensorError(format!("Matmul failed: {e}")))?;
let attention_context = attention_context.squeeze(1)
let attention_context = attention_context.squeeze(Some(1))
.map_err(|e| crate::TtsError::TensorError(format!("Squeeze failed: {e}")))?;
// Concatenate prenet output and attention context
let decoder_rnn_input = rtx_tensor::ops::cat(&[&prenet_out, &attention_context], 1)
let decoder_rnn_input = Tensor::cat(&[prenet_out.clone(), attention_context.clone()], 1)
.map_err(|e| crate::TtsError::TensorError(format!("Cat failed: {e}")))?;
// Decoder RNN step
@@ -518,11 +525,11 @@ impl Tacotron2 {
let decoder_hidden_new = (&rnn_input_proj + &rnn_hidden_proj)
.map_err(|e| crate::TtsError::TensorError(format!("Add failed: {e}")))?;
let decoder_hidden_new = rtx_tensor::ops::tanh(&decoder_hidden_new)
let decoder_hidden_new = decoder_hidden_new.tanh()
.map_err(|e| crate::TtsError::TensorError(format!("Tanh failed: {e}")))?;
// Projection to mel
let projection_input = rtx_tensor::ops::cat(&[&decoder_hidden_new, &attention_context], 1)
let projection_input = Tensor::cat(&[decoder_hidden_new.clone(), attention_context.clone()], 1)
.map_err(|e| crate::TtsError::TensorError(format!("Cat failed: {e}")))?;
let mel_output = self.mel_projection.forward(&projection_input)
@@ -573,17 +580,17 @@ impl AcousticModel for Tacotron2 {
let attn_expanded = attn_weights.unsqueeze(1)
.map_err(|e| crate::TtsError::TensorError(format!("Unsqueeze failed: {e}")))?;
current_attn = rtx_tensor::ops::cat(&[
&current_attn.slice(1, 1, 2)
current_attn = Tensor::cat(&[
current_attn.slice(1, 1, 2)
.map_err(|e| crate::TtsError::TensorError(format!("Slice failed: {e}")))?,
&attn_expanded
attn_expanded
], 1).map_err(|e| crate::TtsError::TensorError(format!("Cat failed: {e}")))?;
// Check gate (stop condition)
let gate_sigmoid = rtx_tensor::ops::sigmoid(&gate_out)
let gate_sigmoid = gate_out.sigmoid()
.map_err(|e| crate::TtsError::TensorError(format!("Sigmoid failed: {e}")))?;
let gate_val = gate_sigmoid.to_vec::<f32>()
let gate_val = gate_sigmoid.to_vec()
.map_err(|e| crate::TtsError::TensorError(format!("To vec failed: {e}")))?;
if gate_val.iter().all(|&v| v > 0.5) {
@@ -592,7 +599,7 @@ impl AcousticModel for Tacotron2 {
}
// Stack mel outputs: [batch, time, mel_dim]
let mel_stacked = rtx_tensor::ops::stack(&mel_outputs.iter().collect::<Vec<_>>(), 1)
let mel_stacked = Tensor::stack(&mel_outputs, 1)
.map_err(|e| crate::TtsError::TensorError(format!("Stack failed: {e}")))?;
// Transpose to [batch, mel_dim, time]
@@ -692,7 +699,7 @@ mod tests {
let batch_size = 2;
let seq_len = 10;
let input = Tensor::randint(0, config.vocab_size as i64, &[batch_size, seq_len], &device).unwrap();
let input = Tensor::randint(0, config.vocab_size as i32, &[batch_size, seq_len], &device).unwrap();
let output = encoder.forward(&input);
assert!(output.is_ok());
@@ -720,7 +727,7 @@ mod tests {
let batch_size = 2;
let seq_len = 10;
let phonemes = Tensor::randint(0, config.vocab_size as i64, &[batch_size, seq_len], &device).unwrap();
let phonemes = Tensor::randint(0, config.vocab_size as i32, &[batch_size, seq_len], &device).unwrap();
let encoded = model.encode(&phonemes, None);
assert!(encoded.is_ok());
@@ -3,10 +3,14 @@
//! The variance adaptor predicts duration, pitch, and energy to control
//! prosody in non-autoregressive TTS models.
// Legacy rtx_nn layer types are deprecated in favour of Generic* variants.
// rtx-tts uses the concrete-tensor API and does not yet require backend dispatch.
#![allow(deprecated)]
use crate::Result;
use rtx_nn::layers::{Module, linear::Linear};
use rtx_nn::layers::activation::ReLU;
use rtx_nn::layers::conv::{Conv1d, Conv1dConfig};
use rtx_nn::layers::conv::{Conv1d, Conv1dConfig, Conv1dPadding};
use rtx_tensor::{Tensor, Device};
use serde::{Deserialize, Serialize};
@@ -142,6 +146,7 @@ impl DurationPredictor {
kernel_size: config.kernel_size,
stride: 1,
padding,
padding_mode: Conv1dPadding::Zeros(padding),
dilation: 1,
groups: 1,
bias: true,
@@ -203,7 +208,7 @@ impl DurationPredictor {
.map_err(|e| crate::TtsError::ModelError(format!("Linear forward failed: {e}")))?;
// Squeeze last dimension: [batch, seq_len, 1] -> [batch, seq_len]
let squeezed = output.squeeze(2)
let squeezed = output.squeeze(Some(2))
.map_err(|e| crate::TtsError::TensorError(format!("Squeeze failed: {e}")))?;
// Apply softplus to ensure positive durations
@@ -246,6 +251,7 @@ impl PitchPredictor {
kernel_size: config.kernel_size,
stride: 1,
padding,
padding_mode: Conv1dPadding::Zeros(padding),
dilation: 1,
groups: 1,
bias: true,
@@ -300,7 +306,7 @@ impl PitchPredictor {
let output = self.linear.forward(&hidden)
.map_err(|e| crate::TtsError::ModelError(format!("Linear forward failed: {e}")))?;
output.squeeze(2)
output.squeeze(Some(2))
.map_err(|e| crate::TtsError::TensorError(format!("Squeeze failed: {e}")))
}
@@ -341,6 +347,7 @@ impl EnergyPredictor {
kernel_size: config.kernel_size,
stride: 1,
padding,
padding_mode: Conv1dPadding::Zeros(padding),
dilation: 1,
groups: 1,
bias: true,
@@ -396,7 +403,7 @@ impl EnergyPredictor {
let output = self.linear.forward(&hidden)
.map_err(|e| crate::TtsError::ModelError(format!("Linear forward failed: {e}")))?;
let squeezed = output.squeeze(2)
let squeezed = output.squeeze(Some(2))
.map_err(|e| crate::TtsError::TensorError(format!("Squeeze failed: {e}")))?;
// Apply softplus to ensure positive energy
@@ -434,7 +441,7 @@ impl LengthRegulator {
let hidden_dim = shape.dims()[2];
// Convert durations to integers
let durations_data = durations.to_vec::<f32>()
let durations_data = durations.to_vec()
.map_err(|e| crate::TtsError::TensorError(format!("Failed to get durations: {e}")))?;
let mut outputs = Vec::new();
@@ -449,7 +456,7 @@ impl LengthRegulator {
let start_idx = b * seq_len * hidden_dim + i * hidden_dim;
let end_idx = start_idx + hidden_dim;
let hidden_data = hidden.to_vec::<f32>()
let hidden_data = hidden.to_vec()
.map_err(|e| crate::TtsError::TensorError(format!("Failed to get hidden: {e}")))?;
let frame = &hidden_data[start_idx..end_idx];
@@ -602,7 +609,7 @@ mod tests {
assert_eq!(shape.dims()[1], seq_len);
// Check all durations are positive
let data = output.to_vec::<f32>().unwrap();
let data = output.to_vec().unwrap();
assert!(data.iter().all(|&x| x >= 0.0));
}