//! Integration tests for rtx-data-validation
//!
//! This module contains comprehensive integration tests that demonstrate
//! all functionality with real data validation scenarios.

use chrono::Utc;
use std::collections::HashMap;
use tokio;

use rtx_data_validation::pipeline::{ErrorHandlingStrategy, PerformanceConfig, RetryConfig};
use rtx_data_validation::profile::NumericalStatistics;
use rtx_data_validation::*;

/// Test data helper functions
mod test_data {
    use super::*;

    pub fn create_sample_record(id: &str, age: i64, name: &str, email: &str) -> DataRecord {
        let mut fields = HashMap::new();
        fields.insert("age".to_string(), DataValue::Int(age));
        fields.insert("name".to_string(), DataValue::String(name.to_string()));
        fields.insert("email".to_string(), DataValue::String(email.to_string()));
        fields.insert("score".to_string(), DataValue::Float(85.5));
        fields.insert("active".to_string(), DataValue::Bool(true));

        DataRecord {
            id: id.to_string(),
            timestamp: Utc::now(),
            fields,
            metadata: HashMap::new(),
        }
    }

    pub fn create_invalid_record(id: &str) -> DataRecord {
        let mut fields = HashMap::new();
        fields.insert("age".to_string(), DataValue::Int(150)); // Invalid age
        fields.insert("name".to_string(), DataValue::Null); // Missing required field
        fields.insert(
            "email".to_string(),
            DataValue::String("invalid-email".to_string()),
        ); // Invalid format
        fields.insert("score".to_string(), DataValue::Float(-10.0)); // Invalid score

        DataRecord {
            id: id.to_string(),
            timestamp: Utc::now(),
            fields,
            metadata: HashMap::new(),
        }
    }

    pub fn create_sample_records(count: usize) -> Vec<DataRecord> {
        (0..count)
            .map(|i| {
                create_sample_record(
                    &format!("record_{}", i),
                    25 + (i as i64 % 40), // Ages 25-64
                    &format!("User {}", i),
                    &format!("user{}@example.com", i),
                )
            })
            .collect()
    }

    pub fn create_mixed_records(valid_count: usize, invalid_count: usize) -> Vec<DataRecord> {
        let mut records = Vec::new();

        // Add valid records
        for i in 0..valid_count {
            records.push(create_sample_record(
                &format!("valid_{}", i),
                25 + (i as i64 % 40),
                &format!("Valid User {}", i),
                &format!("valid{}@example.com", i),
            ));
        }

        // Add invalid records
        for i in 0..invalid_count {
            records.push(create_invalid_record(&format!("invalid_{}", i)));
        }

        records
    }
}

/// Test comprehensive validation engine functionality
#[tokio::test]
async fn test_validation_engine_comprehensive() {
    let mut engine = ValidationEngine::builder()
        .with_statistical_validation(true)
        .with_schema_enforcement(false) // Keep simple for this test
        .with_anomaly_detection(false)
        .add_rule(ValidationRule::not_null("name"))
        .add_rule(ValidationRule::range("age", 0.0, 120.0))
        .add_rule(ValidationRule::email("email"))
        .add_rule(ValidationRule::range("score", 0.0, 100.0))
        .build()
        .expect("Failed to build validation engine");

    // Test valid record
    let valid_record =
        test_data::create_sample_record("test_1", 30, "John Doe", "john@example.com");
    let result = engine
        .validate(&valid_record)
        .await
        .expect("Validation failed");

    assert!(result.is_valid(), "Valid record should pass validation");
    assert!(!result.rule_results.is_empty(), "Should have rule results");
    assert!(result.quality_score.is_some(), "Should have quality score");
    assert!(result.profile.is_some(), "Should have statistical profile");

    // Test invalid record
    let invalid_record = test_data::create_invalid_record("test_2");
    let result = engine
        .validate(&invalid_record)
        .await
        .expect("Validation failed");

    assert!(!result.is_valid(), "Invalid record should fail validation");
    assert!(!result.violations().is_empty(), "Should have violations");

    // Test batch validation
    let mixed_records = test_data::create_mixed_records(5, 3);
    let batch_results = engine
        .validate_batch(&mixed_records)
        .await
        .expect("Batch validation failed");

    assert_eq!(
        batch_results.len(),
        8,
        "Should have results for all records"
    );

    let valid_count = batch_results.iter().filter(|r| r.is_valid()).count();
    let invalid_count = batch_results.iter().filter(|r| !r.is_valid()).count();

    assert_eq!(valid_count, 5, "Should have 5 valid results");
    assert_eq!(invalid_count, 3, "Should have 3 invalid results");
}

/// Test statistical profiling with comprehensive analysis
#[tokio::test]
async fn test_statistical_profiling_comprehensive() {
    let mut profile = DataProfile::new();

    // Add sample records with varied data
    let records = vec![
        test_data::create_sample_record("1", 25, "Alice", "alice@example.com"),
        test_data::create_sample_record("2", 30, "Bob", "bob@example.com"),
        test_data::create_sample_record("3", 35, "Charlie", "charlie@example.com"),
        test_data::create_sample_record("4", 40, "Diana", "diana@example.com"),
        test_data::create_sample_record("5", 45, "Eve", "eve@example.com"),
    ];

    for record in &records {
        profile
            .add_record(&record.fields)
            .expect("Failed to add record to profile");
    }

    profile.finalize().expect("Failed to finalize profile");

    assert_eq!(profile.record_count, 5);
    assert_eq!(profile.field_count, 5); // age, name, email, score, active
    assert!(profile.overall_quality_score() > 0.0);

    // Check column profiles
    assert!(profile.column_profiles.contains_key("age"));
    assert!(profile.column_profiles.contains_key("name"));
    assert!(profile.column_profiles.contains_key("email"));

    // Check correlation matrix
    assert!(profile.correlation_matrix.is_some());
}

/// Test numerical statistics calculation
#[test]
fn test_numerical_statistics() {
    let values = vec![10.0, 20.0, 30.0, 40.0, 50.0, 100.0]; // Include an outlier
    let stats = NumericalStatistics::from_values(&values).expect("Failed to create statistics");

    assert_eq!(stats.min, 10.0);
    assert_eq!(stats.max, 100.0);
    assert_eq!(stats.range, 90.0);
    assert!((stats.mean - 41.666666666666664).abs() < 0.01);
    assert!(!stats.outliers.is_empty(), "Should detect outlier (100.0)");
    assert_eq!(stats.unique_count, 6);
}

/// Test data quality scoring system
#[tokio::test]
async fn test_quality_scoring_comprehensive() {
    // Test with high-quality data
    let high_quality_records = test_data::create_sample_records(10);
    let quality_score = QualityScore::calculate_for_records(&high_quality_records)
        .expect("Failed to calculate quality score");

    assert!(
        quality_score.overall_score() > 0.8,
        "High-quality data should have good score"
    );
    assert_eq!(quality_score.quality_grade(), "Good");
    assert!(quality_score.dimensions.completeness.score > 0.9);
    assert!(quality_score.dimensions.uniqueness.score > 0.0);

    // Test with mixed quality data
    let mixed_records = test_data::create_mixed_records(5, 5);
    let mixed_quality_score = QualityScore::calculate_for_records(&mixed_records)
        .expect("Failed to calculate mixed quality score");

    assert!(
        mixed_quality_score.overall_score() < quality_score.overall_score(),
        "Mixed quality data should have lower score"
    );
}

/// Test anomaly detection algorithms
#[tokio::test]
async fn test_anomaly_detection_comprehensive() {
    let mut detector = AnomalyDetector::new();

    // Add historical data (normal range)
    let normal_data: Vec<f64> = (1..=100).map(|i| i as f64).collect();
    detector.add_historical_data("test_field".to_string(), normal_data);

    // Test data with normal and anomalous values
    let mut test_fields = HashMap::new();
    test_fields.insert("test_field".to_string(), DataValue::Float(50.0)); // Normal

    let normal_results = detector
        .detect_anomalies(&test_fields)
        .expect("Failed to detect anomalies");
    assert!(
        normal_results.is_empty(),
        "Normal value should not be flagged"
    );

    // Test with anomalous value
    test_fields.insert("test_field".to_string(), DataValue::Float(1000.0)); // Anomalous

    let anomaly_results = detector
        .detect_anomalies(&test_fields)
        .expect("Failed to detect anomalies");
    assert!(
        !anomaly_results.is_empty(),
        "Anomalous value should be detected"
    );

    // Test individual detectors
    let z_score_detector = ZScoreDetector::new(3.0);
    let historical: Vec<f64> = (1..=20).map(|i| i as f64).collect();

    let normal_result = z_score_detector
        .detect("test", 10.0, &historical)
        .expect("Z-score detection failed");
    assert!(
        normal_result.is_none(),
        "Normal value should not trigger z-score detection"
    );

    let anomaly_result = z_score_detector
        .detect("test", 100.0, &historical)
        .expect("Z-score detection failed");
    assert!(
        anomaly_result.is_some(),
        "Anomalous value should trigger z-score detection"
    );

    // Test IQR detector
    let iqr_detector = IQRDetector::new(1.5);
    let iqr_anomaly = iqr_detector
        .detect("test", 100.0, &historical)
        .expect("IQR detection failed");
    assert!(
        iqr_anomaly.is_some(),
        "Anomalous value should trigger IQR detection"
    );
}

/// Test schema management and inference
#[tokio::test]
async fn test_schema_management_comprehensive() {
    let mut schema_manager = SchemaManager::new();

    // Test schema inference
    let sample_records = test_data::create_sample_records(10);
    let inferred_schema = schema_manager
        .infer_schema("test_schema".to_string(), &sample_records)
        .expect("Failed to infer schema");

    assert_eq!(inferred_schema.name, "test_schema");
    assert!(!inferred_schema.fields.is_empty());
    assert!(inferred_schema.fields.contains_key("age"));
    assert!(inferred_schema.fields.contains_key("name"));
    assert!(inferred_schema.fields.contains_key("email"));

    // Register the schema
    schema_manager
        .register_schema(inferred_schema)
        .expect("Failed to register schema");

    // Test schema validation
    let test_record = test_data::create_sample_record("test", 30, "Test User", "test@example.com");
    let validation_result = schema_manager
        .validate_record(&test_record)
        .expect("Schema validation failed");

    // Basic validation should pass (simplified implementation)
    assert!(
        validation_result.errors.is_empty()
            || validation_result
                .errors
                .iter()
                .all(|e| e.severity != crate::schema::ErrorSeverity::Critical)
    );

    // Test drift detection
    let drift_records = vec![test_data::create_sample_record(
        "drift",
        25,
        "Drift Test",
        "drift@example.com",
    )];
    let drift_result = schema_manager
        .detect_drift("test_schema", &drift_records)
        .expect("Drift detection failed");

    // Should not detect significant drift for similar data
    assert!(
        drift_result.is_none()
            || drift_result.unwrap().severity == crate::schema::DriftSeverity::None
    );
}

/// Test data lineage tracking
#[tokio::test]
async fn test_lineage_tracking_comprehensive() {
    let mut lineage_tracker = LineageTracker::new();

    // Track several related records
    let records = test_data::create_sample_records(5);
    for record in &records {
        lineage_tracker
            .track_record(record)
            .expect("Failed to track record");
    }

    assert_eq!(lineage_tracker.metadata.total_entities, 5);

    // Test lineage retrieval
    let lineage = lineage_tracker
        .get_lineage("record_0")
        .expect("Failed to get lineage");
    assert!(lineage.is_some());

    // Test impact analysis
    let impact_analysis = lineage_tracker
        .analyze_impact("record_0", "schema_change")
        .expect("Failed to analyze impact");

    assert_eq!(impact_analysis.target_entity, "record_0");
    // Impact analysis should complete without errors (simplified implementation)

    // Test dependency graph building
    let dependency_graph = lineage_tracker
        .build_dependency_graph("record_0")
        .expect("Failed to build dependency graph");

    assert_eq!(dependency_graph.root_entity, "record_0");
    assert!(dependency_graph.traversal_stats.nodes_visited > 0);
}

/// Test real-time validation pipeline
#[tokio::test]
async fn test_validation_pipeline_comprehensive() {
    let config = PipelineConfig {
        name: "test_pipeline".to_string(),
        max_concurrent: 5,
        batch_size: 10,
        buffer_size: 100,
        validation_timeout_ms: 5000,
        collect_metrics: true,
        error_handling: ErrorHandlingStrategy::ContinueOnError,
        retry_config: RetryConfig::default(),
        performance_config: PerformanceConfig::default(),
    };

    let mut pipeline = ValidationPipeline::new(config);

    // Add validation stage
    let engine = std::sync::Arc::new(
        ValidationEngine::builder()
            .add_rule(ValidationRule::not_null("name"))
            .add_rule(ValidationRule::range("age", 0.0, 120.0))
            .build()
            .expect("Failed to build engine"),
    );

    let stage = std::sync::Arc::new(crate::pipeline::StandardValidationStage {
        name: "standard_validation".to_string(),
        engine,
        enabled: true,
        priority: 100,
    });

    pipeline.add_stage(stage);

    // Start pipeline
    pipeline.start().await.expect("Failed to start pipeline");
    assert_eq!(
        pipeline.status().status,
        crate::pipeline::PipelineStatus::Running
    );

    // Process single record
    let test_record = test_data::create_sample_record(
        "pipeline_test",
        30,
        "Pipeline User",
        "pipeline@example.com",
    );
    let result = pipeline
        .process_record(&test_record)
        .await
        .expect("Failed to process record");

    assert!(result.is_valid());

    // Process batch
    let batch_records = test_data::create_sample_records(5);
    let batch_results = pipeline
        .process_batch(&batch_records)
        .await
        .expect("Failed to process batch");

    assert_eq!(batch_results.len(), 5);
    assert!(batch_results.iter().all(|r| r.is_valid()));

    // Check metrics
    let metrics = pipeline.metrics();
    assert!(metrics.performance.records_processed > 0);

    // Stop pipeline
    pipeline.stop().await.expect("Failed to stop pipeline");
    assert_eq!(
        pipeline.status().status,
        crate::pipeline::PipelineStatus::Stopped
    );
}

/// Test streaming validation
#[tokio::test]
async fn test_streaming_validation() {
    let engine = std::sync::Arc::new(
        ValidationEngine::builder()
            .add_rule(ValidationRule::not_null("name"))
            .build()
            .expect("Failed to build engine"),
    );

    let config = crate::pipeline::StreamConfig {
        buffer_size: 100,
        batch_size: 10,
        processing_interval_ms: 100,
        max_latency_ms: 1000,
        backpressure_strategy: crate::pipeline::BackpressureStrategy::DropOldest,
        window_config: None,
    };

    let streaming_validator = crate::pipeline::StreamingValidator::new(engine, config);

    // Add records to stream
    let records = test_data::create_sample_records(15);
    for record in records {
        streaming_validator
            .add_record(record)
            .await
            .expect("Failed to add record to stream");
    }

    // Process stream
    let results = streaming_validator
        .process_stream()
        .await
        .expect("Failed to process stream");

    assert!(!results.is_empty());
    assert!(results.len() <= 10); // Batch size limit

    // Check metrics
    let stream_metrics = streaming_validator.metrics();
    assert!(stream_metrics.records_per_second >= 0.0);
}

/// Test batch validation
#[tokio::test]
#[ignore = "Pre-existing batch metrics timing issue"]
async fn test_batch_validation() {
    let engine = std::sync::Arc::new(
        ValidationEngine::builder()
            .add_rule(ValidationRule::not_null("name"))
            .add_rule(ValidationRule::range("age", 0.0, 120.0))
            .build()
            .expect("Failed to build engine"),
    );

    let config = crate::pipeline::BatchConfig {
        batch_size: 1000,
        max_concurrent_batches: 2,
        batch_timeout_ms: 10000,
        checkpoint_interval: 5,
        enable_optimization: true,
        memory_limit_per_batch: 100_000_000,
    };

    let batch_validator = crate::pipeline::BatchValidator::new(engine, config);

    // Process large batch
    let large_batch = test_data::create_sample_records(50);
    let results = batch_validator
        .process_batch(large_batch)
        .await
        .expect("Failed to process batch");

    assert_eq!(results.len(), 50);
    assert!(results.iter().all(|r| r.is_valid()));

    // Check metrics
    let batch_metrics = batch_validator.metrics();
    assert_eq!(batch_metrics.batches_processed, 1);
    assert!(batch_metrics.avg_batch_time_ms > 0.0);
    assert!(batch_metrics.throughput_rps > 0.0);
}

/// Test metrics collection
#[test]
fn test_metrics_collection_comprehensive() {
    let mut metrics = ValidationMetrics::new();

    // Record various validation operations
    for i in 0..100 {
        let duration = std::time::Duration::from_millis(50 + (i % 100));
        let success = i % 10 != 0; // 90% success rate
        metrics.record_validation(duration, success);
    }

    // Record some errors
    for i in 0..10 {
        metrics.record_error(&format!("error_type_{}", i % 3), "warning");
    }

    // Update resource metrics
    metrics.update_resource_metrics(75.0, 1_000_000, 50);

    // Calculate throughput
    metrics.calculate_throughput(10.0);

    // Update quality metrics
    for i in 0..20 {
        let quality_score = 0.7 + (i as f64 * 0.01); // Increasing quality
        metrics.update_quality_metrics(quality_score, 100);
    }

    // Verify metrics
    assert_eq!(metrics.success_rates.total_validations, 100);
    assert_eq!(metrics.success_rates.successful_validations, 90);
    assert_eq!(metrics.success_rates.failed_validations, 10);
    assert!((metrics.success_rates.success_rate - 0.9).abs() < 0.01);

    assert_eq!(metrics.error_metrics.total_errors, 10);
    assert!(!metrics.error_metrics.errors_by_type.is_empty());

    assert!(metrics.throughput_metrics.records_per_second > 0.0);
    assert!(metrics.quality_metrics.avg_quality_score > 0.7);

    // Test metrics summary
    let summary = metrics.summary();
    assert_eq!(summary.total_validations, 100);
    assert!((summary.success_rate - 0.9).abs() < 0.01);

    // Test JSON export
    let json_export = metrics.to_json().expect("Failed to export to JSON");
    assert!(!json_export.is_empty());
    assert!(json_export.contains("total_validations"));
}

/// Test complex validation scenario with all features
#[tokio::test]
async fn test_comprehensive_validation_scenario() {
    // Create a fully-featured validation engine
    let mut engine = ValidationEngine::builder()
        .with_statistical_validation(true)
        .with_anomaly_detection(true)
        .with_schema_enforcement(false) // Simplified for integration test
        .with_lineage_tracking(false) // Simplified for integration test
        .add_rule(ValidationRule::not_null("id"))
        .add_rule(ValidationRule::not_null("name"))
        .add_rule(ValidationRule::range("age", 0.0, 120.0))
        .add_rule(ValidationRule::email("email"))
        .add_rule(ValidationRule::range("score", 0.0, 100.0))
        .add_rule(ValidationRule::length("name", 2, 100))
        .build()
        .expect("Failed to build comprehensive engine");

    // Create diverse test data
    let mut test_records = Vec::new();

    // Add valid records
    for i in 0..50 {
        test_records.push(test_data::create_sample_record(
            &format!("valid_{}", i),
            20 + (i % 50) as i64,
            &format!("Valid User {}", i),
            &format!("user{}@example.com", i),
        ));
    }

    // Add some edge cases
    let mut edge_case_fields = HashMap::new();
    edge_case_fields.insert(
        "id".to_string(),
        DataValue::String("edge_case_1".to_string()),
    );
    edge_case_fields.insert("age".to_string(), DataValue::Int(0)); // Minimum valid age
    edge_case_fields.insert("name".to_string(), DataValue::String("AB".to_string())); // Minimum valid length
    edge_case_fields.insert("email".to_string(), DataValue::String("a@b.co".to_string())); // Short but valid
    edge_case_fields.insert("score".to_string(), DataValue::Float(100.0)); // Maximum valid score
    edge_case_fields.insert("active".to_string(), DataValue::Bool(true));

    test_records.push(DataRecord {
        id: "edge_case_1".to_string(),
        timestamp: Utc::now(),
        fields: edge_case_fields,
        metadata: HashMap::new(),
    });

    // Add invalid records
    for i in 0..10 {
        test_records.push(test_data::create_invalid_record(&format!("invalid_{}", i)));
    }

    // Process all records
    let results = engine
        .validate_batch(&test_records)
        .await
        .expect("Comprehensive validation failed");

    assert_eq!(results.len(), test_records.len());

    // Analyze results
    let valid_results: Vec<_> = results.iter().filter(|r| r.is_valid()).collect();
    let invalid_results: Vec<_> = results.iter().filter(|r| !r.is_valid()).collect();

    // Should have 51 valid (50 normal + 1 edge case) and 10 invalid
    assert_eq!(valid_results.len(), 51, "Should have 51 valid results");
    assert_eq!(invalid_results.len(), 10, "Should have 10 invalid results");

    // Check that all valid results have quality scores and profiles
    for result in &valid_results {
        assert!(
            result.quality_score.is_some(),
            "Valid result should have quality score"
        );
        assert!(
            result.profile.is_some(),
            "Valid result should have statistical profile"
        );
    }

    // Check that invalid results have violations
    for result in &invalid_results {
        assert!(
            !result.violations().is_empty(),
            "Invalid result should have violations"
        );
    }

    // Test metrics collection
    let metrics = engine.metrics();
    assert!(metrics.performance.records_processed > 0);
    assert!(metrics.success_rates.total_validations > 0);
}

/// Performance stress test
#[tokio::test]
async fn test_performance_stress() {
    let engine = ValidationEngine::builder()
        .with_statistical_validation(true)
        .add_rule(ValidationRule::not_null("name"))
        .add_rule(ValidationRule::range("age", 0.0, 120.0))
        .add_rule(ValidationRule::email("email"))
        .build()
        .expect("Failed to build engine for stress test");

    // Create large dataset
    let large_dataset = test_data::create_sample_records(1000);

    let start_time = std::time::Instant::now();
    let results = engine
        .validate_batch(&large_dataset)
        .await
        .expect("Stress test failed");
    let duration = start_time.elapsed();

    assert_eq!(results.len(), 1000);
    assert!(
        results.iter().all(|r| r.is_valid()),
        "All records should be valid"
    );

    // Performance assertions (should process 1000 records in reasonable time)
    assert!(
        duration.as_secs() < 10,
        "Should process 1000 records in under 10 seconds"
    );

    let throughput = 1000.0 / duration.as_secs_f64();
    assert!(
        throughput > 100.0,
        "Should achieve >100 records/second throughput"
    );

    println!(
        "Stress test completed: {} records in {:.2}s ({:.1} records/sec)",
        1000,
        duration.as_secs_f64(),
        throughput
    );
}