rustytorch/integration_tests/src/lib.rs

/*!
# RustyTorch++ Integration Test Framework

Comprehensive end-to-end integration testing framework for the entire RustyTorch++
platform using strict TDD methodology. Tests real production workloads and scenarios
without mocked dependencies.

## Test Categories

1. **Complete ML Pipeline Tests**: Data loading → Training → Evaluation → Deployment
2. **Cross-Component Integration**: Component interaction validation
3. **Production Deployment**: Containerized deployment with monitoring
4. **Performance Integration**: SLA validation and scaling tests
5. **Real-world Use Cases**: BERT, vision, GPT, CLIP validation
6. **Multi-tenant System**: Resource isolation and fair scheduling
7. **Disaster Recovery**: Fault tolerance and recovery testing
8. **Security Integration**: Authentication, authorization, audit logging

## TDD Approach

All tests follow strict Test-Driven Development:
1. Write failing integration tests first
2. Full end-to-end execution (no mocking)
3. Production-realistic scenarios
4. Comprehensive error testing
5. Performance regression detection

## Test Environment Requirements

- CUDA-capable GPU (recommended RTX 4090/5090)
- Docker for containerization tests
- Redis for caching tests
- PostgreSQL for metadata storage tests
- At least 32GB RAM for memory-intensive tests
- Network connectivity for distributed tests

## Running Integration Tests

```bash
# Run all integration tests
cargo test --package rustytorch-integration-tests --features full-integration

# Run specific test categories
cargo run --bin ml_pipeline_tests
cargo run --bin production_deployment_tests
cargo run --bin performance_integration_tests

# Run with specific backend
RTX_BACKEND=cuda cargo test
RTX_BACKEND=rocm cargo test

# Performance benchmarks
cargo bench --package rustytorch-integration-tests
```

## Environment Variables

- `RTX_BACKEND`: cuda|rocm|metal|cpu (default: cuda)
- `RTX_DEVICE_COUNT`: Number of GPUs to use
- `RTX_TEST_DATA_PATH`: Path to test datasets
- `RTX_REDIS_URL`: Redis connection string
- `RTX_POSTGRES_URL`: PostgreSQL connection string
- `RTX_TEST_TIMEOUT`: Test timeout in seconds (default: 300)
- `RTX_SKIP_GPU_TESTS`: Skip GPU-dependent tests
- `RTX_PERFORMANCE_MODE`: Enable performance assertions
- `RTX_CONTAINER_RUNTIME`: docker|podman (default: docker)
*/

#[cfg(feature = "integration-tests")]
pub mod common;
pub mod stubs;
#[cfg(feature = "integration-tests")]
pub mod pipeline;
#[cfg(feature = "integration-tests")]
pub mod cross_component;
#[cfg(feature = "integration-tests")]
pub mod production;
#[cfg(feature = "integration-tests")]
pub mod performance;
#[cfg(feature = "integration-tests")]
pub mod real_world;
#[cfg(feature = "integration-tests")]
pub mod multi_tenant;
#[cfg(feature = "integration-tests")]
pub mod disaster_recovery;
#[cfg(feature = "integration-tests")]
pub mod security;
#[cfg(feature = "integration-tests")]
pub mod chaos;

// cuSOLVER integration tests - only available when CUDA feature is enabled
#[cfg(all(feature = "integration-tests", feature = "cuda"))]
pub mod cusolver_integration;

// Re-export stubs for use in test files
pub use stubs::*;

// Re-export common utilities
#[cfg(feature = "integration-tests")]
pub use common::*;

use anyhow::Result;
use std::time::Duration;
use tracing::{info, warn, error};
use sysinfo::SystemExt;

/// Global integration test configuration
#[derive(Debug, Clone)]
pub struct IntegrationTestConfig {
    /// Backend to use for testing
    pub backend: Backend,
    /// Number of devices to use
    pub device_count: usize,
    /// Test timeout duration
    pub timeout: Duration,
    /// Path to test data
    pub test_data_path: std::path::PathBuf,
    /// Redis connection URL
    pub redis_url: Option<String>,
    /// PostgreSQL connection URL
    pub postgres_url: Option<String>,
    /// Skip GPU tests if true
    pub skip_gpu_tests: bool,
    /// Enable performance assertions
    pub performance_mode: bool,
    /// Container runtime (docker/podman)
    pub container_runtime: String,
    /// Maximum memory usage per test (MB)
    pub max_memory_mb: usize,
    /// Enable distributed testing
    pub enable_distributed: bool,
}

impl Default for IntegrationTestConfig {
    fn default() -> Self {
        Self {
            backend: Backend::from_env().unwrap_or(Backend::CUDA),
            device_count: std::env::var("RTX_DEVICE_COUNT")
                .ok()
                .and_then(|s| s.parse().ok())
                .unwrap_or(1),
            timeout: Duration::from_secs(
                std::env::var("RTX_TEST_TIMEOUT")
                    .ok()
                    .and_then(|s| s.parse().ok())
                    .unwrap_or(300)
            ),
            test_data_path: std::env::var("RTX_TEST_DATA_PATH")
                .map(std::path::PathBuf::from)
                .unwrap_or_else(|_| std::path::PathBuf::from("./test_data")),
            redis_url: std::env::var("RTX_REDIS_URL").ok(),
            postgres_url: std::env::var("RTX_POSTGRES_URL").ok(),
            skip_gpu_tests: std::env::var("RTX_SKIP_GPU_TESTS").is_ok(),
            performance_mode: std::env::var("RTX_PERFORMANCE_MODE").is_ok(),
            container_runtime: std::env::var("RTX_CONTAINER_RUNTIME")
                .unwrap_or_else(|_| "docker".to_string()),
            max_memory_mb: std::env::var("RTX_MAX_MEMORY_MB")
                .ok()
                .and_then(|s| s.parse().ok())
                .unwrap_or(16384), // 16GB default
            enable_distributed: std::env::var("RTX_ENABLE_DISTRIBUTED").is_ok(),
        }
    }
}

/// Backend enumeration for testing
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum Backend {
    CUDA,
    ROCm,
    Metal,
    CPU,
}

impl Backend {
    pub fn from_env() -> Result<Self> {
        match std::env::var("RTX_BACKEND").as_deref() {
            Ok("cuda") => Ok(Self::CUDA),
            Ok("rocm") => Ok(Self::ROCm),
            Ok("metal") => Ok(Self::Metal),
            Ok("cpu") => Ok(Self::CPU),
            Ok(other) => anyhow::bail!("Unknown backend: {other}"),
            Err(_) => Ok(Self::CUDA), // Default to CUDA
        }
    }

    pub fn as_str(&self) -> &'static str {
        match self {
            Self::CUDA => "cuda",
            Self::ROCm => "rocm",
            Self::Metal => "metal",
            Self::CPU => "cpu",
        }
    }

    pub fn supports_gpu(&self) -> bool {
        matches!(self, Self::CUDA | Self::ROCm | Self::Metal)
    }
}

impl std::fmt::Display for Backend {
    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
        write!(f, "{}", self.as_str())
    }
}

/// Initialize integration test environment
pub async fn initialize_test_environment() -> Result<IntegrationTestConfig> {
    // Initialize tracing
    tracing_subscriber::fmt()
        .with_env_filter("debug")
        .with_test_writer()
        .init();

    let config = IntegrationTestConfig::default();

    info!("Initializing integration test environment");
    info!("Backend: {:?}", config.backend);
    info!("Device count: {}", config.device_count);
    info!("Timeout: {:?}", config.timeout);
    info!("Test data path: {:?}", config.test_data_path);
    info!("Skip GPU tests: {}", config.skip_gpu_tests);
    info!("Performance mode: {}", config.performance_mode);

    // Create test data directory if it doesn't exist
    if !config.test_data_path.exists() {
        std::fs::create_dir_all(&config.test_data_path)?;
        info!("Created test data directory: {:?}", config.test_data_path);
    }

    // Validate environment
    validate_test_environment(&config).await?;

    Ok(config)
}

/// Validate the test environment setup
async fn validate_test_environment(config: &IntegrationTestConfig) -> Result<()> {
    info!("Validating test environment...");

    // Check GPU availability if not skipping GPU tests
    if !config.skip_gpu_tests && config.backend.supports_gpu() {
        match config.backend {
            Backend::CUDA => {
                #[cfg(feature = "cuda")]
                {
                    if let Ok(nvml) = nvml_wrapper::Nvml::init() {
                        match nvml.device_count() {
                            Ok(device_count) => {
                                info!("CUDA devices available: {}", device_count);
                                if device_count < config.device_count as u32 {
                                    warn!("Requested {} devices but only {} available",
                                        config.device_count, device_count);
                                }
                            }
                            Err(e) => warn!("Could not get CUDA device count: {}", e),
                        }
                    } else {
                        warn!("CUDA not available, falling back to CPU");
                    }
                }
                #[cfg(not(feature = "cuda"))]
                {
                    warn!("CUDA feature not enabled, GPU tests will be limited");
                }
            }
            Backend::ROCm => {
                // ROCm validation would go here
                warn!("ROCm validation not implemented yet");
            }
            Backend::Metal => {
                // Metal validation would go here
                warn!("Metal validation not implemented yet");
            }
            _ => {}
        }
    }

    // Check Docker availability for containerization tests
    if let Err(e) = tokio::process::Command::new(&config.container_runtime)
        .arg("--version")
        .output()
        .await
    {
        warn!("Container runtime '{}' not available: {}", config.container_runtime, e);
    } else {
        info!("Container runtime '{}' available", config.container_runtime);
    }

    // Check database connections if provided
    if let Some(redis_url) = &config.redis_url {
        match redis::Client::open(redis_url.clone()) {
            Ok(_) => info!("Redis connection validated"),
            Err(e) => warn!("Redis connection failed: {}", e),
        }
    }

    if let Some(postgres_url) = &config.postgres_url {
        match sqlx::PgPool::connect(postgres_url).await {
            Ok(_) => info!("PostgreSQL connection validated"),
            Err(e) => warn!("PostgreSQL connection failed: {}", e),
        }
    }

    // Check available system resources
    let sys = sysinfo::System::new_all();
    let total_memory_mb = sys.total_memory() / 1024 / 1024;
    info!("System memory: {} MB", total_memory_mb);

    if total_memory_mb < config.max_memory_mb as u64 {
        warn!("System memory ({} MB) is less than requested max ({} MB)",
            total_memory_mb, config.max_memory_mb);
    }

    info!("Test environment validation completed");
    Ok(())
}

/// Test result tracking and reporting
#[derive(Debug, Clone)]
pub struct TestResults {
    pub passed: usize,
    pub failed: usize,
    pub skipped: usize,
    pub total_duration: Duration,
    pub failures: Vec<TestFailure>,
}

#[derive(Debug, Clone)]
pub struct TestFailure {
    pub test_name: String,
    pub error: String,
    pub duration: Duration,
}

impl TestResults {
    pub fn new() -> Self {
        Self {
            passed: 0,
            failed: 0,
            skipped: 0,
            total_duration: Duration::ZERO,
            failures: Vec::new(),
        }
    }

    pub fn add_pass(&mut self, duration: Duration) {
        self.passed += 1;
        self.total_duration += duration;
    }

    pub fn add_failure(&mut self, test_name: String, error: String, duration: Duration) {
        self.failed += 1;
        self.total_duration += duration;
        self.failures.push(TestFailure { test_name, error, duration });
    }

    pub fn add_skip(&mut self, reason: &str) {
        self.skipped += 1;
        info!("Test skipped: {}", reason);
    }

    pub fn success_rate(&self) -> f64 {
        if self.passed + self.failed == 0 {
            0.0
        } else {
            self.passed as f64 / (self.passed + self.failed) as f64
        }
    }

    pub fn print_summary(&self) {
        info!("=== Integration Test Results ===");
        info!("Passed: {}", self.passed);
        info!("Failed: {}", self.failed);
        info!("Skipped: {}", self.skipped);
        info!("Success Rate: {:.2}%", self.success_rate() * 100.0);
        info!("Total Duration: {:?}", self.total_duration);

        if !self.failures.is_empty() {
            error!("=== Failures ===");
            for failure in &self.failures {
                error!("FAILED: {} ({:?})", failure.test_name, failure.duration);
                error!("  Error: {}", failure.error);
            }
        }
    }
}

impl Default for TestResults {
    fn default() -> Self {
        Self::new()
    }
}

/// Macro for running integration tests with proper error handling and timing
#[macro_export]
macro_rules! integration_test {
    ($name:expr, $test_fn:expr, $results:expr) => {
        {
            use tracing::{info, error};
            use std::time::Instant;

            info!("Running test: {}", $name);
            let start = Instant::now();

            match $test_fn().await {
                Ok(_) => {
                    let duration = start.elapsed();
                    info!("PASSED: {} ({:?})", $name, duration);
                    $results.add_pass(duration);
                }
                Err(e) => {
                    let duration = start.elapsed();
                    error!("FAILED: {} ({:?})", $name, duration);
                    error!("Error: {}", e);
                    $results.add_failure($name.to_string(), e.to_string(), duration);
                }
            }
        }
    };
}

/// Utilities for test data generation and management
pub mod test_data {
    use super::*;

    /// Generate synthetic training data for ML pipeline tests
    pub fn generate_classification_dataset(
        num_samples: usize,
        num_features: usize,
        num_classes: usize,
        seed: u64,
    ) -> Result<(Vec<Vec<f32>>, Vec<usize>)> {
        use rand::{SeedableRng, Rng};
        use rand::rngs::StdRng;

        let mut rng = StdRng::seed_from_u64(seed);

        let mut features = Vec::with_capacity(num_samples);
        let mut labels = Vec::with_capacity(num_samples);

        for _ in 0..num_samples {
            let mut sample = Vec::with_capacity(num_features);
            for _ in 0..num_features {
                sample.push(rng.gen_range(-1.0..1.0));
            }
            features.push(sample);
            labels.push(rng.gen_range(0..num_classes));
        }

        Ok((features, labels))
    }

    /// Generate synthetic text data for NLP tests
    pub fn generate_text_dataset(
        num_samples: usize,
        vocab_size: usize,
        sequence_length: usize,
        seed: u64,
    ) -> Result<Vec<Vec<u32>>> {
        use rand::{SeedableRng, Rng};
        use rand::rngs::StdRng;

        let mut rng = StdRng::seed_from_u64(seed);
        let mut sequences = Vec::with_capacity(num_samples);

        for _ in 0..num_samples {
            let mut sequence = Vec::with_capacity(sequence_length);
            for _ in 0..sequence_length {
                sequence.push(rng.gen_range(0..vocab_size as u32));
            }
            sequences.push(sequence);
        }

        Ok(sequences)
    }

    /// Generate synthetic image data for vision tests
    pub fn generate_image_dataset(
        num_samples: usize,
        height: u32,
        width: u32,
        channels: u32,
        seed: u64,
    ) -> Result<Vec<Vec<f32>>> {
        use rand::{SeedableRng, Rng};
        use rand::rngs::StdRng;

        let mut rng = StdRng::seed_from_u64(seed);
        let pixels_per_image = (height * width * channels) as usize;
        let mut images = Vec::with_capacity(num_samples);

        for _ in 0..num_samples {
            let mut image = Vec::with_capacity(pixels_per_image);
            for _ in 0..pixels_per_image {
                image.push(rng.gen_range(0.0..1.0));
            }
            images.push(image);
        }

        Ok(images)
    }
}