chore(sweep): delete 43 orphaned source files; document SYCL/demo/duplication status

Deletions (all verified unreferenced by any mod/include/path declaration; git history preserves them): - rtx-transformers: entire orphaned curriculum/ split (mod.rs holds the real inline implementation), non-_simple graph variants, superseded simmim/jepa_integration files, layers/{sliding_window_attention, positional_encoding,ssm_state_cache_original}, lib_full/lib_minimal/ error_full/error_minimal, orphaned MoE impls (moe_layer, moe_integration). - rtx-distributed/parallel_old.rs; rtx-flash-attention/{core_full, lib_full}.rs; rtx-compress legacy_distillation + structured_pruner. - rtx-tensor/tensor_core.rs; rtx-runtime/{cuda_kernel_ops, cuda_backend_mock}.rs; rtx-memory/{gpu_pool_manager,allocator, pool_type}.rs; rtx-losses/{lib_minimal,lib_full}.rs. Docs honesty: - rtx-backend-sycl marked EXPERIMENTAL SKELETON in crate docs and CLAUDE.md backend table (all ops return NotImplemented). - docs/consolidation.md records canonical MoE (layers/mixture_of_experts) and flash-attention (rtx-flash-attention crate) implementations plus remaining duplicates to consolidate. - CLAUDE.md: meta-crate GPU features noted; simulation-only demos named; serving/streaming mock removal noted. Verified: cargo check --workspace clean (rtx-onnx-codegen pre-broken at HEAD, unrelated); lib tests pass for all touched crates (rtx-runtime's 4 failures pre-exist at HEAD). Co-Authored-By: Claude Fable 5 <[email protected]>
2026-07-09 19:32:21 -07:00
parent 1e3c604896
commit 5f32165184
47 changed files with 49 additions and 15391 deletions
@@ -1,234 +0,0 @@
-//! Knowledge distillation for model compression
-
-use crate::error::{CompressionError, Result};
-use rtx_tensor::{Device, Tensor};
-
-/// Distillation loss type
-#[derive(Debug, Clone, Copy)]
-pub enum DistillationLoss {
-    KLDivergence,
-    MSE,
-    CrossEntropy,
-}
-
-/// Configuration for knowledge distillation
-#[derive(Debug, Clone)]
-pub struct DistillationConfig {
-    pub temperature: f32,
-    pub alpha: f32,
-    pub beta: f32,
-    pub loss_type: DistillationLoss,
-    pub use_attention_transfer: bool,
-    pub use_feature_matching: bool,
-}
-
-impl Default for DistillationConfig {
-    fn default() -> Self {
-        Self {
-            temperature: 3.0,
-            alpha: 0.7,
-            beta: 0.3,
-            loss_type: DistillationLoss::KLDivergence,
-            use_attention_transfer: false,
-            use_feature_matching: false,
-        }
-    }
-}
-
-/// Knowledge distiller for teacher-student compression
-#[derive(Debug, Clone)]
-pub struct KnowledgeDistiller {
-    config: DistillationConfig,
-    device: Device,
-    schedule_steps: Option<usize>,
-    initial_temp: f32,
-    final_temp: f32,
-}
-
-impl KnowledgeDistiller {
-    pub fn new(config: DistillationConfig, device: &Device) -> Result<Self> {
-        Ok(Self {
-            config,
-            device: device.clone(),
-            schedule_steps: None,
-            initial_temp: config.temperature,
-            final_temp: config.temperature,
-        })
-    }
-
-    pub fn default(device: &Device) -> Result<Self> {
-        Self::new(DistillationConfig::default(), device)
-    }
-
-    pub fn with_schedule(
-        initial_temp: f32,
-        final_temp: f32,
-        steps: usize,
-        device: &Device,
-    ) -> Result<Self> {
-        let config = DistillationConfig {
-            temperature: initial_temp,
-            ..Default::default()
-        };
-        
-        Ok(Self {
-            config,
-            device: device.clone(),
-            schedule_steps: Some(steps),
-            initial_temp,
-            final_temp,
-        })
-    }
-
-    pub fn compute_loss(&self, teacher_logits: &Tensor, student_logits: &Tensor) -> Result<Tensor> {
-        let teacher_soft = self.apply_temperature(teacher_logits)?;
-        let student_soft = self.apply_temperature(student_logits)?;
-        
-        match self.config.loss_type {
-            DistillationLoss::KLDivergence => self.kl_divergence(&teacher_soft, &student_soft),
-            DistillationLoss::MSE => self.mse_loss(&teacher_soft, &student_soft),
-            DistillationLoss::CrossEntropy => self.cross_entropy(&teacher_soft, &student_soft),
-        }
-    }
-
-    pub fn apply_temperature(&self, logits: &Tensor) -> Result<Tensor> {
-        logits.div_scalar(self.config.temperature)
-    }
-
-    pub fn generate_soft_targets(&self, logits: &Tensor) -> Result<Tensor> {
-        let scaled = self.apply_temperature(logits)?;
-        self.softmax(&scaled)
-    }
-
-    pub fn compute_combined_loss(
-        &self,
-        teacher_logits: &Tensor,
-        student_logits: &Tensor,
-        labels: &Tensor,
-    ) -> Result<Tensor> {
-        let distill_loss = self.compute_loss(teacher_logits, student_logits)?;
-        let student_loss = self.cross_entropy(student_logits, labels)?;
-        
-        let weighted_distill = distill_loss.mul_scalar(self.config.alpha)?;
-        let weighted_student = student_loss.mul_scalar(self.config.beta)?;
-        
-        weighted_distill.add(&weighted_student)
-    }
-
-    pub fn attention_transfer_loss(
-        &self,
-        teacher_attn: &Tensor,
-        student_attn: &Tensor,
-    ) -> Result<Tensor> {
-        // Simplified: MSE between attention maps
-        self.mse_loss(teacher_attn, student_attn)
-    }
-
-    pub fn feature_matching_loss(
-        &self,
-        teacher_features: &[Tensor],
-        student_features: &[Tensor],
-    ) -> Result<Tensor> {
-        if teacher_features.len() != student_features.len() {
-            return Err(CompressionError::CompressionFailed(
-                "Feature lists must have same length".to_string()
-            ));
-        }
-        
-        let mut total_loss = Tensor::zeros(&[1], &self.device)?;
-        for (t, s) in teacher_features.iter().zip(student_features.iter()) {
-            let loss = self.mse_loss(t, s)?;
-            total_loss = total_loss.add(&loss)?;
-        }
-        
-        total_loss.div_scalar(teacher_features.len() as f32)
-    }
-
-    pub fn compute_compression_ratio(&self, teacher_params: usize, student_params: usize) -> f32 {
-        teacher_params as f32 / student_params as f32
-    }
-
-    pub fn get_temperature_at_step(&self, step: usize) -> f32 {
-        if let Some(total_steps) = self.schedule_steps {
-            let progress = (step as f32) / (total_steps as f32).max(1.0);
-            self.initial_temp + (self.final_temp - self.initial_temp) * progress
-        } else {
-            self.config.temperature
-        }
-    }
-
-    pub fn compute_metrics(
-        &self,
-        teacher_preds: &Tensor,
-        student_preds: &Tensor,
-        labels: &Tensor,
-    ) -> Result<DistillationMetrics> {
-        let teacher_correct = self.count_correct(teacher_preds, labels)?;
-        let student_correct = self.count_correct(student_preds, labels)?;
-        let agreement = self.count_agreement(teacher_preds, student_preds)?;
-        let total = labels.numel() as f32;
-        
-        Ok(DistillationMetrics {
-            teacher_accuracy: teacher_correct / total,
-            student_accuracy: student_correct / total,
-            agreement_rate: agreement / total,
-        })
-    }
-
-    // Helper methods
-    fn kl_divergence(&self, p: &Tensor, q: &Tensor) -> Result<Tensor> {
-        // Simplified KL divergence
-        let log_p = p.log()?;
-        let log_q = q.log()?;
-        let diff = log_p.sub(&log_q)?;
-        let kl = p.mul(&diff)?;
-        kl.sum()?.div_scalar(p.numel() as f32)
-    }
-
-    fn mse_loss(&self, pred: &Tensor, target: &Tensor) -> Result<Tensor> {
-        let diff = pred.sub(target)?;
-        let squared = diff.mul(&diff)?;
-        squared.sum()?.div_scalar(pred.numel() as f32)
-    }
-
-    fn cross_entropy(&self, logits: &Tensor, labels: &Tensor) -> Result<Tensor> {
-        // Simplified cross entropy
-        let probs = self.softmax(logits)?;
-        let log_probs = probs.log()?;
-        log_probs.sum()?.mul_scalar(-1.0)?.div_scalar(logits.shape().dims()[0] as f32)
-    }
-
-    fn softmax(&self, logits: &Tensor) -> Result<Tensor> {
-        let exp = logits.exp()?;
-        let sum = exp.sum()?;
-        exp.div(&sum)
-    }
-
-    fn count_correct(&self, preds: &Tensor, labels: &Tensor) -> Result<f32> {
-        let pred_data = preds.to_vec()?;
-        let label_data = labels.to_vec()?;
-        let correct = pred_data.iter()
-            .zip(label_data.iter())
-            .filter(|(p, l)| (p - l).abs() < 1e-5)
-            .count();
-        Ok(correct as f32)
-    }
-
-    fn count_agreement(&self, preds1: &Tensor, preds2: &Tensor) -> Result<f32> {
-        let data1 = preds1.to_vec()?;
-        let data2 = preds2.to_vec()?;
-        let agree = data1.iter()
-            .zip(data2.iter())
-            .filter(|(p1, p2)| (p1 - p2).abs() < 1e-5)
-            .count();
-        Ok(agree as f32)
-    }
-}
-
-/// Metrics for distillation
-#[derive(Debug)]
-pub struct DistillationMetrics {
-    pub teacher_accuracy: f32,
-    pub student_accuracy: f32,
-    pub agreement_rate: f32,
-}