rustytorch/crates/training/rtx-transformers/src/regression_tests.rs

//! Regression Test Suite for RTX Transformers
//!
//! This module provides continuous validation to detect performance regressions,
//! accuracy degradation, and feature breakage across versions.

use crate::prelude::*;
use crate::validation_framework::*;
use serde::{Deserialize, Serialize};
use std::collections::HashMap;

/// Regression test configuration
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct RegressionTestConfig {
    /// Baseline validation report path
    pub baseline_report_path: String,
    /// Maximum allowed performance regression (0.1 = 10% regression allowed)
    pub max_performance_regression: f64,
    /// Maximum allowed accuracy degradation
    pub max_accuracy_degradation: f64,
    /// Whether to fail on any regression
    pub strict_mode: bool,
    /// Test suite timeout in seconds
    pub timeout_seconds: u64,
}

impl Default for RegressionTestConfig {
    fn default() -> Self {
        Self {
            baseline_report_path: "baseline_validation_report.json".to_string(),
            max_performance_regression: 0.05, // 5% regression threshold
            max_accuracy_degradation: 0.01,   // 1% accuracy degradation
            strict_mode: false,
            timeout_seconds: 300, // 5 minute timeout
        }
    }
}

/// Regression test results
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct RegressionTestResult {
    /// Test timestamp
    pub timestamp: chrono::DateTime<chrono::Utc>,
    /// Whether all tests passed
    pub all_tests_passed: bool,
    /// Performance regression results
    pub performance_regressions: Vec<PerformanceRegression>,
    /// Accuracy regression results
    pub accuracy_regressions: Vec<AccuracyRegression>,
    /// Feature regression results
    pub feature_regressions: Vec<FeatureRegression>,
    /// New issues detected
    pub new_issues: Vec<String>,
    /// Summary of regression analysis
    pub summary: RegressionSummary,
}

/// Performance regression detection result
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct PerformanceRegression {
    /// Test name that regressed
    pub test_name: String,
    /// Baseline performance
    pub baseline_performance: f64,
    /// Current performance
    pub current_performance: f64,
    /// Regression percentage (positive = improvement, negative = regression)
    pub regression_percentage: f64,
    /// Whether this is considered a critical regression
    pub is_critical: bool,
    /// Context and debugging information
    pub context: HashMap<String, String>,
}

/// Accuracy regression detection result
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct AccuracyRegression {
    /// Test name that regressed
    pub test_name: String,
    /// Baseline accuracy metrics
    pub baseline_max_error: f64,
    pub baseline_mean_error: f64,
    /// Current accuracy metrics
    pub current_max_error: f64,
    pub current_mean_error: f64,
    /// Whether accuracy degraded significantly
    pub accuracy_degraded: bool,
    /// Degradation details
    pub degradation_details: String,
}

/// Feature regression detection result
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct FeatureRegression {
    /// Feature name that regressed
    pub feature_name: String,
    /// What aspect regressed (implementation, functionality, integration)
    pub regression_type: String,
    /// Baseline status
    pub baseline_status: FeatureStatus,
    /// Current status
    pub current_status: FeatureStatus,
    /// Issue description
    pub issue_description: String,
}

/// Feature status for regression detection
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct FeatureStatus {
    pub implemented: bool,
    pub functional: bool,
    pub integrated: bool,
    pub performance_score: f64,
}

/// Summary of regression analysis
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct RegressionSummary {
    /// Total regressions found
    pub total_regressions: usize,
    /// Critical regressions
    pub critical_regressions: usize,
    /// Performance regressions
    pub performance_regressions: usize,
    /// Accuracy regressions
    pub accuracy_regressions: usize,
    /// Feature regressions
    pub feature_regressions: usize,
    /// Overall regression risk level
    pub risk_level: RiskLevel,
    /// Recommended actions
    pub recommended_actions: Vec<String>,
}

/// Risk level for regression analysis
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
pub enum RiskLevel {
    Low,
    Medium,
    High,
    Critical,
}

/// Main regression test suite
pub struct RegressionTestSuite {
    config: RegressionTestConfig,
    validation_framework: ValidationFramework,
}

impl RegressionTestSuite {
    /// Create new regression test suite
    pub fn new(config: RegressionTestConfig) -> Self {
        Self {
            validation_framework: ValidationFramework::new(ValidationConfig {
                performance_threshold: 0.8,
                numerical_tolerance: 1e-6,
                benchmark_iterations: 100,
                warmup_iterations: 10,
                verbose: true,
            }),
            config,
        }
    }

    /// Create with default configuration
    pub fn default() -> Self {
        Self::new(RegressionTestConfig::default())
    }

    /// Run complete regression test suite
    pub fn run_regression_tests(&mut self) -> Result<RegressionTestResult> {
        tracing::info!("Starting regression test suite");

        let start_time = std::time::Instant::now();

        // Load baseline report
        let baseline_report = self.load_baseline_report()?;

        // Run current validation
        let current_report = self.validation_framework.run_comprehensive_validation()?;

        // Analyze regressions
        let performance_regressions =
            self.analyze_performance_regressions(&baseline_report, &current_report);
        let accuracy_regressions =
            self.analyze_accuracy_regressions(&baseline_report, &current_report);
        let feature_regressions =
            self.analyze_feature_regressions(&baseline_report, &current_report);

        // Detect new issues
        let new_issues = self.detect_new_issues(&baseline_report, &current_report);

        // Generate summary
        let summary = self.generate_regression_summary(
            &performance_regressions,
            &accuracy_regressions,
            &feature_regressions,
            &new_issues,
        );

        let all_tests_passed = summary.critical_regressions == 0
            && (self.config.strict_mode == false || summary.total_regressions == 0);

        let result = RegressionTestResult {
            timestamp: chrono::Utc::now(),
            all_tests_passed,
            performance_regressions,
            accuracy_regressions,
            feature_regressions,
            new_issues,
            summary,
        };

        let elapsed = start_time.elapsed();
        tracing::info!(
            "Regression test suite completed in {:.2}s: {} total regressions, {} critical",
            elapsed.as_secs_f64(),
            result.summary.total_regressions,
            result.summary.critical_regressions
        );

        Ok(result)
    }

    /// Load baseline validation report
    fn load_baseline_report(&mut self) -> Result<ValidationReport> {
        if std::path::Path::new(&self.config.baseline_report_path).exists() {
            ValidationFramework::load_report(&self.config.baseline_report_path)
        } else {
            // Generate baseline report if none exists
            tracing::warn!("No baseline report found, generating new baseline");
            let report = self.validation_framework.run_comprehensive_validation()?;

            // Save as baseline
            self.validation_framework
                .save_report(&report, &self.config.baseline_report_path)?;
            Ok(report)
        }
    }

    /// Analyze performance regressions
    fn analyze_performance_regressions(
        &self,
        baseline: &ValidationReport,
        current: &ValidationReport,
    ) -> Vec<PerformanceRegression> {
        let mut regressions = Vec::new();

        // Match performance results by test name
        for current_result in &current.performance_results {
            if let Some(baseline_result) = baseline
                .performance_results
                .iter()
                .find(|r| r.test_name == current_result.test_name)
            {
                let regression_percentage = (current_result.actual_performance
                    - baseline_result.actual_performance)
                    / baseline_result.actual_performance;

                // Check if this is a significant regression
                if regression_percentage < -self.config.max_performance_regression {
                    let is_critical = regression_percentage < -0.20; // 20% regression is critical

                    let mut context = HashMap::new();
                    context.insert(
                        "baseline_performance".to_string(),
                        baseline_result.actual_performance.to_string(),
                    );
                    context.insert(
                        "current_performance".to_string(),
                        current_result.actual_performance.to_string(),
                    );
                    context.insert(
                        "expected_performance".to_string(),
                        current_result.expected_performance.to_string(),
                    );

                    regressions.push(PerformanceRegression {
                        test_name: current_result.test_name.clone(),
                        baseline_performance: baseline_result.actual_performance,
                        current_performance: current_result.actual_performance,
                        regression_percentage: regression_percentage * 100.0, // Convert to percentage
                        is_critical,
                        context,
                    });
                }
            }
        }

        regressions
    }

    /// Analyze accuracy regressions
    fn analyze_accuracy_regressions(
        &self,
        baseline: &ValidationReport,
        current: &ValidationReport,
    ) -> Vec<AccuracyRegression> {
        let mut regressions = Vec::new();

        for current_result in &current.accuracy_results {
            if let Some(baseline_result) = baseline
                .accuracy_results
                .iter()
                .find(|r| r.test_name == current_result.test_name)
            {
                // Check if accuracy degraded
                let max_error_increase = current_result.max_error - baseline_result.max_error;
                let mean_error_increase = current_result.mean_error - baseline_result.mean_error;

                let accuracy_degraded = max_error_increase > self.config.max_accuracy_degradation
                    || mean_error_increase > self.config.max_accuracy_degradation;

                if accuracy_degraded {
                    let degradation_details = format!(
                        "Max error: {:.2e} -> {:.2e} (+{:.2e}), Mean error: {:.2e} -> {:.2e} (+{:.2e})",
                        baseline_result.max_error,
                        current_result.max_error,
                        max_error_increase,
                        baseline_result.mean_error,
                        current_result.mean_error,
                        mean_error_increase
                    );

                    regressions.push(AccuracyRegression {
                        test_name: current_result.test_name.clone(),
                        baseline_max_error: baseline_result.max_error,
                        baseline_mean_error: baseline_result.mean_error,
                        current_max_error: current_result.max_error,
                        current_mean_error: current_result.mean_error,
                        accuracy_degraded: true,
                        degradation_details,
                    });
                }
            }
        }

        regressions
    }

    /// Analyze feature regressions
    fn analyze_feature_regressions(
        &self,
        baseline: &ValidationReport,
        current: &ValidationReport,
    ) -> Vec<FeatureRegression> {
        let mut regressions = Vec::new();

        for current_feature in &current.feature_results {
            if let Some(baseline_feature) = baseline
                .feature_results
                .iter()
                .find(|f| f.feature_name == current_feature.feature_name)
            {
                let baseline_status = FeatureStatus {
                    implemented: baseline_feature.implemented,
                    functional: baseline_feature.functional,
                    integrated: baseline_feature.integrated,
                    performance_score: self
                        .calculate_performance_score(&baseline_feature.performance_characteristics),
                };

                let current_status = FeatureStatus {
                    implemented: current_feature.implemented,
                    functional: current_feature.functional,
                    integrated: current_feature.integrated,
                    performance_score: self
                        .calculate_performance_score(&current_feature.performance_characteristics),
                };

                // Check for regressions
                let mut regression_issues = Vec::new();

                if baseline_status.implemented && !current_status.implemented {
                    regression_issues.push("Feature no longer implemented");
                }
                if baseline_status.functional && !current_status.functional {
                    regression_issues.push("Feature no longer functional");
                }
                if baseline_status.integrated && !current_status.integrated {
                    regression_issues.push("Feature no longer integrated");
                }
                if current_status.performance_score < baseline_status.performance_score * 0.9 {
                    regression_issues.push("Performance score degraded significantly");
                }

                if !regression_issues.is_empty() {
                    regressions.push(FeatureRegression {
                        feature_name: current_feature.feature_name.clone(),
                        regression_type: regression_issues.join(", "),
                        baseline_status,
                        current_status,
                        issue_description: format!(
                            "Feature regression detected: {}",
                            regression_issues.join("; ")
                        ),
                    });
                }
            }
        }

        regressions
    }

    /// Calculate performance score from characteristics
    fn calculate_performance_score(&self, characteristics: &HashMap<String, f64>) -> f64 {
        if characteristics.is_empty() {
            return 0.0;
        }

        // Simple average of all performance characteristics
        let sum: f64 = characteristics.values().sum();
        sum / characteristics.len() as f64
    }

    /// Detect new issues not present in baseline
    fn detect_new_issues(
        &self,
        baseline: &ValidationReport,
        current: &ValidationReport,
    ) -> Vec<String> {
        let mut new_issues = Vec::new();

        // Check for new failed tests
        for current_result in &current.performance_results {
            if !current_result.passed {
                if let Some(baseline_result) = baseline
                    .performance_results
                    .iter()
                    .find(|r| r.test_name == current_result.test_name)
                {
                    if baseline_result.passed {
                        new_issues.push(format!(
                            "Performance test '{}' now failing (was passing)",
                            current_result.test_name
                        ));
                    }
                } else {
                    new_issues.push(format!(
                        "New performance test '{}' is failing",
                        current_result.test_name
                    ));
                }
            }
        }

        // Check for new accuracy failures
        for current_result in &current.accuracy_results {
            if !current_result.passed {
                if let Some(baseline_result) = baseline
                    .accuracy_results
                    .iter()
                    .find(|r| r.test_name == current_result.test_name)
                {
                    if baseline_result.passed {
                        new_issues.push(format!(
                            "Accuracy test '{}' now failing (was passing)",
                            current_result.test_name
                        ));
                    }
                } else {
                    new_issues.push(format!(
                        "New accuracy test '{}' is failing",
                        current_result.test_name
                    ));
                }
            }
        }

        // Check for new feature issues
        for current_feature in &current.feature_results {
            if !current_feature.issues.is_empty() {
                if let Some(baseline_feature) = baseline
                    .feature_results
                    .iter()
                    .find(|f| f.feature_name == current_feature.feature_name)
                {
                    let new_feature_issues: Vec<&String> = current_feature
                        .issues
                        .iter()
                        .filter(|issue| !baseline_feature.issues.contains(issue))
                        .collect();

                    for new_issue in new_feature_issues {
                        new_issues.push(format!(
                            "New issue in feature '{}': {}",
                            current_feature.feature_name, new_issue
                        ));
                    }
                } else {
                    // New feature with issues
                    for issue in &current_feature.issues {
                        new_issues.push(format!(
                            "New feature '{}' has issue: {}",
                            current_feature.feature_name, issue
                        ));
                    }
                }
            }
        }

        new_issues
    }

    /// Generate regression summary and recommendations
    fn generate_regression_summary(
        &self,
        performance_regressions: &[PerformanceRegression],
        accuracy_regressions: &[AccuracyRegression],
        feature_regressions: &[FeatureRegression],
        new_issues: &[String],
    ) -> RegressionSummary {
        let total_regressions =
            performance_regressions.len() + accuracy_regressions.len() + feature_regressions.len();

        let critical_regressions = performance_regressions
            .iter()
            .filter(|r| r.is_critical)
            .count();

        // Determine risk level
        let risk_level = if critical_regressions > 0 {
            RiskLevel::Critical
        } else if total_regressions > 5 || !new_issues.is_empty() {
            RiskLevel::High
        } else if total_regressions > 2 {
            RiskLevel::Medium
        } else {
            RiskLevel::Low
        };

        // Generate recommendations
        let mut recommended_actions = Vec::new();

        if critical_regressions > 0 {
            recommended_actions.push(
                "URGENT: Address critical performance regressions before release".to_string(),
            );
        }

        if !performance_regressions.is_empty() {
            recommended_actions
                .push("Review and optimize performance regression areas".to_string());
        }

        if !accuracy_regressions.is_empty() {
            recommended_actions.push("Investigate numerical accuracy degradation".to_string());
        }

        if !feature_regressions.is_empty() {
            recommended_actions.push("Fix revolutionary feature regressions".to_string());
        }

        if !new_issues.is_empty() {
            recommended_actions.push("Address newly introduced issues".to_string());
        }

        if recommended_actions.is_empty() {
            recommended_actions
                .push("All regression tests passed - ready for deployment".to_string());
        }

        RegressionSummary {
            total_regressions,
            critical_regressions,
            performance_regressions: performance_regressions.len(),
            accuracy_regressions: accuracy_regressions.len(),
            feature_regressions: feature_regressions.len(),
            risk_level,
            recommended_actions,
        }
    }

    /// Save regression test results
    pub fn save_results(&self, results: &RegressionTestResult, path: &str) -> Result<()> {
        let json = serde_json::to_string_pretty(results)
            .map_err(|e| TransformerError::SerializationError(e.to_string()))?;

        std::fs::write(path, json).map_err(TransformerError::IoError)?;

        tracing::info!("Regression test results saved to: {}", path);
        Ok(())
    }

    /// Update baseline report with current validation
    pub fn update_baseline(&mut self) -> Result<()> {
        tracing::info!("Updating baseline validation report");

        let current_report = self.validation_framework.run_comprehensive_validation()?;
        self.validation_framework
            .save_report(&current_report, &self.config.baseline_report_path)?;

        tracing::info!("Baseline report updated successfully");
        Ok(())
    }

    /// Run continuous regression monitoring
    pub fn run_continuous_monitoring(&mut self, interval_seconds: u64) -> Result<()> {
        tracing::info!(
            "Starting continuous regression monitoring (interval: {}s)",
            interval_seconds
        );

        loop {
            let results = self.run_regression_tests()?;

            if results.summary.risk_level == RiskLevel::Critical {
                tracing::error!("CRITICAL REGRESSION DETECTED!");
                for action in &results.summary.recommended_actions {
                    tracing::error!("ACTION REQUIRED: {}", action);
                }

                // Save critical results
                let timestamp = chrono::Utc::now().format("%Y%m%d_%H%M%S");
                let critical_path = format!("critical_regression_{}.json", timestamp);
                self.save_results(&results, &critical_path)?;

                return Err(TransformerError::ComputationError(
                    "Critical regression detected - monitoring stopped".to_string(),
                ));
            }

            if results.summary.risk_level == RiskLevel::High {
                tracing::warn!("High risk regression detected:");
                for action in &results.summary.recommended_actions {
                    tracing::warn!("  {}", action);
                }
            }

            tracing::info!(
                "Regression check complete: {} total regressions, risk level: {:?}",
                results.summary.total_regressions,
                results.summary.risk_level
            );

            // Wait for next check
            std::thread::sleep(std::time::Duration::from_secs(interval_seconds));
        }
    }
}

#[cfg(test)]
mod tests {
    use super::*;

    #[test]
    fn test_regression_test_suite_creation() {
        let suite = RegressionTestSuite::default();
        assert_eq!(suite.config.max_performance_regression, 0.05);
        assert_eq!(suite.config.max_accuracy_degradation, 0.01);
    }

    #[tokio::test]
    async fn test_regression_detection() {
        let mut suite = RegressionTestSuite::new(RegressionTestConfig {
            baseline_report_path: "/tmp/test_baseline.json".to_string(),
            ..RegressionTestConfig::default()
        });

        // This should create a baseline and then test against it
        let results = suite.run_regression_tests().unwrap();

        // First run should have no regressions (comparing against itself)
        assert_eq!(results.summary.total_regressions, 0);
        assert_eq!(results.summary.risk_level, RiskLevel::Low);
        assert!(results.all_tests_passed);

        // Clean up
        std::fs::remove_file("/tmp/test_baseline.json").ok();
    }

    #[test]
    fn test_performance_regression_analysis() {
        let config = RegressionTestConfig::default();
        let suite = RegressionTestSuite::new(config);

        // Create mock reports
        let baseline = create_mock_baseline_report();
        let current = create_mock_current_report_with_regression();

        let regressions = suite.analyze_performance_regressions(&baseline, &current);

        assert!(!regressions.is_empty());
        assert!(regressions[0].regression_percentage < 0.0); // Should detect regression
    }

    #[test]
    fn test_risk_level_calculation() {
        let suite = RegressionTestSuite::default();

        // Test critical risk (with critical regressions)
        let perf_regressions = vec![PerformanceRegression {
            test_name: "Critical Test".to_string(),
            baseline_performance: 10.0,
            current_performance: 5.0,
            regression_percentage: -50.0,
            is_critical: true,
            context: HashMap::new(),
        }];

        let summary = suite.generate_regression_summary(&perf_regressions, &[], &[], &[]);
        assert_eq!(summary.risk_level, RiskLevel::Critical);
        assert!(summary.recommended_actions[0].contains("URGENT"));
    }

    fn create_mock_baseline_report() -> ValidationReport {
        ValidationReport {
            timestamp: chrono::Utc::now(),
            performance_results: vec![PerformanceValidationResult {
                test_name: "Flash Attention".to_string(),
                expected_performance: 6.0,
                actual_performance: 6.5, // Good baseline
                passed: true,
                threshold: 0.8,
                metadata: HashMap::new(),
            }],
            accuracy_results: vec![],
            feature_results: vec![],
            overall_pass_rate: 0.9,
            summary: ValidationSummary {
                total_tests: 1,
                tests_passed: 1,
                tests_failed: 0,
                revolutionary_features_count: 4,
                performance_claims_validated: 1,
                key_achievements: vec![],
                improvement_areas: vec![],
            },
        }
    }

    fn create_mock_current_report_with_regression() -> ValidationReport {
        ValidationReport {
            timestamp: chrono::Utc::now(),
            performance_results: vec![PerformanceValidationResult {
                test_name: "Flash Attention".to_string(),
                expected_performance: 6.0,
                actual_performance: 5.0, // Regressed from 6.5 to 5.0
                passed: false,
                threshold: 0.8,
                metadata: HashMap::new(),
            }],
            accuracy_results: vec![],
            feature_results: vec![],
            overall_pass_rate: 0.7, // Worse than baseline
            summary: ValidationSummary {
                total_tests: 1,
                tests_passed: 0,
                tests_failed: 1,
                revolutionary_features_count: 4,
                performance_claims_validated: 0,
                key_achievements: vec![],
                improvement_areas: vec!["Performance regression".to_string()],
            },
        }
    }
}