Omar SobhandClaude Sonnet 4.6
228137555f
G0 (Critical): Replace 45 unimplemented!() panics across three GPU backends - rtx-backend-cuda: sin/cos/tanh via PTX, relu/sigmoid/leaky_relu/elu via activation.rs, pow/clamp/gt_scalar via unary.rs, var/var_dim host-side, conv2d/max_pool2d/avg_pool2d CPU fallback in new ops/conv.rs; new PTX kernels in element_wise.cu - rtx-backend-rocm: all 15 ops via CPU round-trip (to_vec → compute → from_slice) - rtx-backend-sycl: all 15 ops via CPU round-trip (to_host → compute → from_data) G2 (High): Re-add rtx-distributed to workspace - Vendor 4 minimal RNCCL stub crates at crates/vendor/rnccl/* - Update rtx-distributed RNCCL path deps to point at stubs (../../../../RNCCL/* → ../../vendor/rnccl/*) - Remove rtx-distributed from workspace exclude list, add to members G5 (Medium): Re-enable rtx-tts (213 tests restored) - Fix 15 rtx-nn API drift issues: LayerNorm::new, Conv1d::from_config, Conv1dPadding::Zeros, Dropout::new(p, device), tensor methods (relu/tanh/sigmoid/cat/stack), squeeze(Some(n)), to_vec() turbofish removal, Tensor::randn with &[...] slices G8 (Low): Quantum stubs + multimodal forward bug - rtx-timeseries: remove dead quantum/neuromorphic TODO comment blocks (no module files exist) - rtx-multimodal/fusion/transformer.rs: wire TransformerBlock loop in forward() - rtx-multimodal/fusion/strategies.rs: wire bottleneck_layers loop in forward() - rtx-transformers/architectures/transformer_block.rs: add forward() method (pre-norm residuals; full attention+FFN pending when those sub-layers are wired) Co-Authored-By: Claude Sonnet 4.6 <[email protected]>
66 lines
2.2 KiB
Rust
66 lines
2.2 KiB
Rust
//! Multimodal fusion transformer implementation.
|
|
|
|
use crate::Result;
|
|
use rtx_tensor::{Device, Tensor};
|
|
use rtx_transformers::architectures::TransformerConfig;
|
|
use rtx_transformers::layers::Layer;
|
|
use rtx_transformers::prelude::{LayerNorm, TransformerBlock};
|
|
|
|
use super::config::MultimodalFusionConfig;
|
|
|
|
pub struct MultimodalFusionTransformer {
|
|
blocks: Vec<TransformerBlock>,
|
|
ln_f: LayerNorm,
|
|
config: MultimodalFusionConfig,
|
|
device: Device,
|
|
}
|
|
|
|
impl MultimodalFusionTransformer {
|
|
pub fn new(config: &MultimodalFusionConfig, device: &Device) -> Result<Self> {
|
|
let transformer_config = TransformerConfig {
|
|
vocab_size: 32000, // Default vocab size
|
|
d_model: config.embed_dim,
|
|
num_layers: 1, // We'll create the layers manually
|
|
num_heads: config.num_heads,
|
|
num_key_value_heads: Some(config.num_heads), // Same as num_heads for MHA
|
|
use_mqa: false, // Multi-head attention, not MQA
|
|
d_ff: config.embed_dim * 4, // Standard 4x expansion for FFN
|
|
max_seq_len: 4096, // Default max sequence length
|
|
dropout: config.dropout as f64,
|
|
layer_norm_eps: 1e-5,
|
|
bias: true,
|
|
activation: "gelu".to_string(),
|
|
};
|
|
|
|
let mut blocks = Vec::new();
|
|
for _ in 0..config.num_layers {
|
|
blocks.push(TransformerBlock::new(transformer_config.clone(), device)?);
|
|
}
|
|
|
|
let ln_f = LayerNorm::new(config.embed_dim, 1e-5, true, device)?;
|
|
|
|
Ok(Self {
|
|
blocks,
|
|
ln_f,
|
|
config: config.clone(),
|
|
device: device.clone(),
|
|
})
|
|
}
|
|
|
|
pub fn forward(&self, vision: &Tensor, audio: &Tensor, text: &Tensor) -> Result<Tensor> {
|
|
// Concatenate all modalities along sequence dimension
|
|
let fused = Tensor::cat(&[vision.clone(), audio.clone(), text.clone()], 1)?;
|
|
|
|
// Pass through transformer blocks
|
|
let mut x = fused;
|
|
for block in &self.blocks {
|
|
x = block.forward(&x)?;
|
|
}
|
|
|
|
// Final layer norm
|
|
let x = self.ln_f.forward(&x)?;
|
|
|
|
Ok(x)
|
|
}
|
|
}
|