rustytorch/crates/training/rtx-evolution/src/telemetry.rs

//! Telemetry Analysis System
//!
//! Pattern mining, anomaly detection, and performance opportunity identification

use crate::Result;
use statrs::statistics::{Data, Distribution};
use std::collections::HashMap;

/// Telemetry analyzer for pattern mining and anomaly detection
pub struct TelemetryAnalyzer {
    initialized: bool,
    historical_data: Vec<TelemetrySnapshot>,
    pattern_cache: HashMap<String, PerformancePattern>,
    anomaly_threshold: f64,
}

/// Snapshot of system telemetry at a point in time
#[derive(Debug, Clone)]
pub struct TelemetrySnapshot {
    pub timestamp: std::time::SystemTime,
    pub metrics: HashMap<String, f64>,
}

/// Performance pattern detected in telemetry data
#[derive(Debug, Clone)]
pub struct PerformancePattern {
    pub metric: String,
    pub pattern_type: PatternType,
    pub confidence: f64,
    pub trend_direction: TrendDirection,
    pub magnitude: f64,
    pub frequency: Option<f64>, // For periodic patterns
}

/// Type of performance pattern
#[derive(Debug, Clone, PartialEq)]
pub enum PatternType {
    Trend,       // Increasing/decreasing over time
    Periodic,    // Cyclical behavior
    Threshold,   // Near capacity limits
    Correlation, // Related to other metrics
    Anomaly,     // Unusual behavior
}

/// Direction of trend patterns
#[derive(Debug, Clone, PartialEq)]
pub enum TrendDirection {
    Increasing,
    Decreasing,
    Stable,
    Volatile,
}

/// Anomaly detected in system behavior
#[derive(Debug, Clone)]
pub struct Anomaly {
    pub metric: String,
    pub severity: AnomalySeverity,
    pub description: String,
    pub current_value: f64,
    pub expected_range: (f64, f64),
    pub confidence: f64,
}

/// Severity level of anomalies
#[derive(Debug, Clone, PartialEq)]
pub enum AnomalySeverity {
    Low,
    Medium,
    High,
    Critical,
}

impl TelemetryAnalyzer {
    /// Create new telemetry analyzer
    pub fn new() -> Self {
        Self {
            initialized: true,
            historical_data: Vec::new(),
            pattern_cache: HashMap::new(),
            anomaly_threshold: 2.0, // 2 standard deviations
        }
    }

    /// Check if analyzer is properly initialized
    pub fn is_initialized(&self) -> bool {
        self.initialized
    }

    /// Analyze telemetry data to identify optimization patterns
    pub async fn analyze_patterns(
        &self,
        telemetry_data: &[(&str, f64)],
    ) -> Result<Vec<PerformancePattern>> {
        let mut patterns = Vec::new();

        // Convert input data to internal format
        let current_snapshot = TelemetrySnapshot {
            timestamp: std::time::SystemTime::now(),
            metrics: telemetry_data
                .iter()
                .map(|(key, value)| (key.to_string(), *value))
                .collect(),
        };

        // Analyze each metric for patterns
        for (metric_name, current_value) in &current_snapshot.metrics {
            // 1. Trend analysis
            if let Some(trend_pattern) = self.analyze_trend(metric_name, *current_value).await? {
                patterns.push(trend_pattern);
            }

            // 2. Threshold analysis (check if near capacity limits)
            if let Some(threshold_pattern) =
                self.analyze_threshold(metric_name, *current_value).await?
            {
                patterns.push(threshold_pattern);
            }

            // 3. Anomaly detection
            if let Some(anomaly_pattern) = self.detect_anomaly(metric_name, *current_value).await? {
                patterns.push(anomaly_pattern);
            }

            // 4. Correlation analysis with other metrics
            let correlation_patterns = self
                .analyze_correlations(metric_name, &current_snapshot)
                .await?;
            patterns.extend(correlation_patterns);
        }

        // Filter and rank patterns by optimization potential
        patterns = self.rank_by_optimization_potential(patterns);

        Ok(patterns)
    }

    /// Analyze trend patterns in a metric
    async fn analyze_trend(
        &self,
        metric_name: &str,
        current_value: f64,
    ) -> Result<Option<PerformancePattern>> {
        // Get historical data for this metric
        let historical_values: Vec<f64> = self
            .historical_data
            .iter()
            .filter_map(|snapshot| snapshot.metrics.get(metric_name))
            .copied()
            .collect();

        if historical_values.len() < 3 {
            return Ok(None); // Need at least 3 points for trend analysis
        }

        // Calculate trend using linear regression
        let n = historical_values.len() as f64;
        let x_values: Vec<f64> = (0..historical_values.len()).map(|i| i as f64).collect();

        let x_mean = x_values.iter().sum::<f64>() / n;
        let y_mean = historical_values.iter().sum::<f64>() / n;

        let numerator: f64 = x_values
            .iter()
            .zip(&historical_values)
            .map(|(x, y)| (x - x_mean) * (y - y_mean))
            .sum();
        let denominator: f64 = x_values.iter().map(|x| (x - x_mean).powi(2)).sum();

        if denominator == 0.0 {
            return Ok(None);
        }

        let slope = numerator / denominator;
        let r_squared =
            self.calculate_r_squared(&x_values, &historical_values, slope, x_mean, y_mean);

        // Determine trend direction and significance
        let trend_direction = if slope.abs() < 0.001 {
            TrendDirection::Stable
        } else if slope > 0.0 {
            TrendDirection::Increasing
        } else {
            TrendDirection::Decreasing
        };

        // Only return pattern if trend is significant
        if r_squared > 0.5 && slope.abs() > 0.001 {
            Ok(Some(PerformancePattern {
                metric: metric_name.to_string(),
                pattern_type: PatternType::Trend,
                confidence: r_squared,
                trend_direction,
                magnitude: slope.abs(),
                frequency: None,
            }))
        } else {
            Ok(None)
        }
    }

    /// Calculate R-squared for linear regression
    fn calculate_r_squared(
        &self,
        x_values: &[f64],
        y_values: &[f64],
        slope: f64,
        x_mean: f64,
        y_mean: f64,
    ) -> f64 {
        let intercept = y_mean - slope * x_mean;

        let ss_res: f64 = x_values
            .iter()
            .zip(y_values)
            .map(|(x, y)| {
                let predicted = slope * x + intercept;
                (y - predicted).powi(2)
            })
            .sum();

        let ss_tot: f64 = y_values.iter().map(|y| (y - y_mean).powi(2)).sum();

        if ss_tot == 0.0 {
            1.0
        } else {
            1.0 - (ss_res / ss_tot)
        }
    }

    /// Analyze threshold patterns (near capacity limits)
    async fn analyze_threshold(
        &self,
        metric_name: &str,
        current_value: f64,
    ) -> Result<Option<PerformancePattern>> {
        // Define typical capacity thresholds for different metrics
        let threshold = match metric_name {
            "gpu_utilization" => 0.90,       // 90% GPU utilization
            "memory_usage" => 0.85,          // 85% memory usage
            "power_consumption" => 0.95,     // 95% power limit
            "bandwidth_utilization" => 0.80, // 80% bandwidth
            _ => 0.90,                       // Default threshold
        };

        if current_value >= threshold {
            Ok(Some(PerformancePattern {
                metric: metric_name.to_string(),
                pattern_type: PatternType::Threshold,
                confidence: (current_value - threshold) / (1.0 - threshold), // Confidence increases closer to limit
                trend_direction: TrendDirection::Increasing,
                magnitude: current_value - threshold,
                frequency: None,
            }))
        } else {
            Ok(None)
        }
    }

    /// Detect anomalies in metric values
    async fn detect_anomaly(
        &self,
        metric_name: &str,
        current_value: f64,
    ) -> Result<Option<PerformancePattern>> {
        // Get historical data for statistical analysis
        let historical_values: Vec<f64> = self
            .historical_data
            .iter()
            .filter_map(|snapshot| snapshot.metrics.get(metric_name))
            .copied()
            .collect();

        if historical_values.len() < 5 {
            return Ok(None); // Need sufficient historical data
        }

        // Calculate statistical bounds using z-score
        let data = Data::new(historical_values);
        let mean = data.mean().unwrap_or(0.0);
        let std_dev = data.std_dev().unwrap_or(0.0);

        if std_dev == 0.0 {
            return Ok(None); // No variation in historical data
        }

        let z_score = (current_value - mean) / std_dev;

        // Detect anomaly if z-score exceeds threshold
        if z_score.abs() > self.anomaly_threshold {
            Ok(Some(PerformancePattern {
                metric: metric_name.to_string(),
                pattern_type: PatternType::Anomaly,
                confidence: (z_score.abs() - self.anomaly_threshold) / self.anomaly_threshold,
                trend_direction: if z_score > 0.0 {
                    TrendDirection::Increasing
                } else {
                    TrendDirection::Decreasing
                },
                magnitude: z_score.abs(),
                frequency: None,
            }))
        } else {
            Ok(None)
        }
    }

    /// Analyze correlations between metrics
    async fn analyze_correlations(
        &self,
        metric_name: &str,
        current_snapshot: &TelemetrySnapshot,
    ) -> Result<Vec<PerformancePattern>> {
        let mut correlation_patterns = Vec::new();

        // Check for known correlation patterns
        match metric_name {
            "gpu_utilization" => {
                // GPU utilization often correlates with memory usage and power
                if let (Some(&memory_usage), Some(&power_consumption)) = (
                    current_snapshot.metrics.get("memory_usage"),
                    current_snapshot.metrics.get("power_consumption"),
                ) {
                    // If GPU utilization is high but memory usage is low, might indicate memory bottleneck
                    if let Some(&gpu_util) = current_snapshot.metrics.get("gpu_utilization") {
                        if gpu_util > 0.8 && memory_usage < 0.6 {
                            correlation_patterns.push(PerformancePattern {
                                metric: "memory_layout".to_string(),
                                pattern_type: PatternType::Correlation,
                                confidence: 0.7,
                                trend_direction: TrendDirection::Stable,
                                magnitude: gpu_util - memory_usage,
                                frequency: None,
                            });
                        }
                    }
                }
            }
            "kernel_exec_time" => {
                // Long execution times might correlate with memory access patterns
                if let Some(&bandwidth_util) = current_snapshot.metrics.get("bandwidth_utilization")
                {
                    if let Some(&exec_time) = current_snapshot.metrics.get("kernel_exec_time") {
                        if exec_time > 0.05 && bandwidth_util < 0.5 {
                            // >50ms exec time, <50% bandwidth
                            correlation_patterns.push(PerformancePattern {
                                metric: "memory_access_pattern".to_string(),
                                pattern_type: PatternType::Correlation,
                                confidence: 0.6,
                                trend_direction: TrendDirection::Increasing,
                                magnitude: exec_time / bandwidth_util,
                                frequency: None,
                            });
                        }
                    }
                }
            }
            _ => {}
        }

        Ok(correlation_patterns)
    }

    /// Rank patterns by their optimization potential
    fn rank_by_optimization_potential(
        &self,
        mut patterns: Vec<PerformancePattern>,
    ) -> Vec<PerformancePattern> {
        patterns.sort_by(|a, b| {
            // Calculate optimization potential score
            let score_a = self.calculate_optimization_score(a);
            let score_b = self.calculate_optimization_score(b);
            score_b.total_cmp(&score_a)
        });

        patterns
    }

    /// Calculate optimization potential score for a pattern
    fn calculate_optimization_score(&self, pattern: &PerformancePattern) -> f64 {
        let base_score = pattern.confidence * pattern.magnitude;

        // Weight by pattern type (some patterns have higher optimization potential)
        let type_multiplier = match pattern.pattern_type {
            PatternType::Threshold => 2.0,   // High potential - near limits
            PatternType::Correlation => 1.8, // High potential - cross-optimization
            PatternType::Trend => 1.5,       // Medium potential - gradual improvement
            PatternType::Anomaly => 1.2,     // Lower potential - might be temporary
            PatternType::Periodic => 1.0,    // Baseline
        };

        // Weight by metric importance
        let metric_multiplier = match pattern.metric.as_str() {
            "gpu_utilization" => 2.0,
            "memory_usage" => 1.8,
            "kernel_exec_time" => 1.6,
            "bandwidth_utilization" => 1.4,
            _ => 1.0,
        };

        base_score * type_multiplier * metric_multiplier
    }

    /// Add a telemetry snapshot to historical data
    pub fn add_snapshot(&mut self, snapshot: TelemetrySnapshot) {
        self.historical_data.push(snapshot);

        // Keep only recent history (last 1000 snapshots)
        if self.historical_data.len() > 1000 {
            self.historical_data.remove(0);
        }
    }

    /// Detect anomalies across all metrics
    pub async fn detect_anomalies(&self, telemetry_data: &[(&str, f64)]) -> Result<Vec<Anomaly>> {
        let mut anomalies = Vec::new();

        for (metric_name, current_value) in telemetry_data {
            if let Some(anomaly) = self
                .check_metric_anomaly(metric_name, *current_value)
                .await?
            {
                anomalies.push(anomaly);
            }
        }

        Ok(anomalies)
    }

    /// Check if a specific metric value is anomalous
    async fn check_metric_anomaly(
        &self,
        metric_name: &str,
        current_value: f64,
    ) -> Result<Option<Anomaly>> {
        // Get historical data for statistical analysis
        let historical_values: Vec<f64> = self
            .historical_data
            .iter()
            .filter_map(|snapshot| snapshot.metrics.get(metric_name))
            .copied()
            .collect();

        if historical_values.len() < 5 {
            return Ok(None);
        }

        let data = Data::new(historical_values);
        let mean = data.mean().unwrap_or(0.0);
        let std_dev = data.std_dev().unwrap_or(0.0);

        if std_dev == 0.0 {
            return Ok(None);
        }

        let z_score = (current_value - mean) / std_dev;

        if z_score.abs() > self.anomaly_threshold {
            let severity = match z_score.abs() {
                z if z > 4.0 => AnomalySeverity::Critical,
                z if z > 3.0 => AnomalySeverity::High,
                z if z > 2.5 => AnomalySeverity::Medium,
                _ => AnomalySeverity::Low,
            };

            Ok(Some(Anomaly {
                metric: metric_name.to_string(),
                severity,
                description: format!(
                    "Value {:.3} deviates {:.2} standard deviations from mean {:.3}",
                    current_value, z_score, mean
                ),
                current_value,
                expected_range: (mean - 2.0 * std_dev, mean + 2.0 * std_dev),
                confidence: (z_score.abs() - self.anomaly_threshold) / self.anomaly_threshold,
            }))
        } else {
            Ok(None)
        }
    }
}