//! Real-time validation pipeline with streaming support
//!
//! This module provides a comprehensive validation pipeline for both batch and
//! streaming data processing with configurable validation stages and metrics collection.

use std::collections::VecDeque;
use std::sync::Arc;
use std::time::{Duration, Instant};

use async_trait::async_trait;
use parking_lot::RwLock;
use serde::{Deserialize, Serialize};
use tokio::sync::oneshot;
use tracing::{debug, error, info, warn};

use crate::{
    DataRecord, Result, ValidationError,
    engine::{ValidationEngine, ValidationResult},
    metrics::ValidationMetrics,
};

/// Main validation pipeline
#[derive(Debug)]
pub struct ValidationPipeline {
    /// Pipeline configuration
    pub config: PipelineConfig,
    /// Validation stages
    pub stages: Vec<Arc<dyn ValidationStage>>,
    /// Pipeline metrics
    pub metrics: Arc<RwLock<ValidationMetrics>>,
    /// Pipeline state
    pub state: Arc<RwLock<PipelineState>>,
}

/// Configuration for validation pipeline
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct PipelineConfig {
    /// Pipeline name
    pub name: String,
    /// Maximum concurrent validations
    pub max_concurrent: usize,
    /// Batch size for batch processing
    pub batch_size: usize,
    /// Buffer size for streaming
    pub buffer_size: usize,
    /// Timeout for individual validations
    pub validation_timeout_ms: u64,
    /// Enable metrics collection
    pub collect_metrics: bool,
    /// Error handling strategy
    pub error_handling: ErrorHandlingStrategy,
    /// Retry configuration
    pub retry_config: RetryConfig,
    /// Performance tuning parameters
    pub performance_config: PerformanceConfig,
}

/// Error handling strategies
#[derive(Debug, Clone, Serialize, Deserialize)]
pub enum ErrorHandlingStrategy {
    /// Fail fast - stop on first error
    FailFast,
    /// Continue processing - collect all errors
    ContinueOnError,
    /// Skip invalid records
    SkipInvalid,
    /// Use fallback validation
    Fallback,
}

/// Retry configuration
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct RetryConfig {
    /// Maximum number of retries
    pub max_retries: u32,
    /// Base delay between retries in milliseconds
    pub base_delay_ms: u64,
    /// Exponential backoff multiplier
    pub backoff_multiplier: f64,
    /// Maximum delay between retries
    pub max_delay_ms: u64,
    /// Enable jitter for retry delays
    pub enable_jitter: bool,
}

/// Performance configuration
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct PerformanceConfig {
    /// Enable parallel processing
    pub parallel_processing: bool,
    /// CPU usage limit (percentage)
    pub cpu_limit_percent: f64,
    /// Memory usage limit (bytes)
    pub memory_limit_bytes: usize,
    /// I/O throttling settings
    pub io_throttling: IoThrottling,
    /// Cache configuration
    pub cache_config: CacheConfig,
}

/// I/O throttling settings
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct IoThrottling {
    /// Enable I/O throttling
    pub enabled: bool,
    /// Maximum I/O operations per second
    pub max_iops: u32,
    /// Maximum bandwidth in bytes per second
    pub max_bandwidth_bps: u64,
}

/// Cache configuration
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct CacheConfig {
    /// Enable validation result caching
    pub enabled: bool,
    /// Cache size limit
    pub size_limit: usize,
    /// Cache TTL in seconds
    pub ttl_seconds: u64,
    /// Cache eviction policy
    pub eviction_policy: CacheEvictionPolicy,
}

/// Cache eviction policies
#[derive(Debug, Clone, Serialize, Deserialize)]
pub enum CacheEvictionPolicy {
    /// Least Recently Used
    LRU,
    /// Least Frequently Used
    LFU,
    /// Time-based expiration
    TTL,
    /// First In First Out
    FIFO,
}

/// Pipeline state
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct PipelineState {
    /// Current pipeline status
    pub status: PipelineStatus,
    /// Number of records processed
    pub records_processed: usize,
    /// Number of validation failures
    pub validation_failures: usize,
    /// Processing start time
    pub started_at: Option<chrono::DateTime<chrono::Utc>>,
    /// Last activity timestamp
    pub last_activity: Option<chrono::DateTime<chrono::Utc>>,
    /// Current throughput (records per second)
    pub current_throughput: f64,
    /// Error details
    pub errors: Vec<PipelineError>,
}

/// Pipeline status
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
pub enum PipelineStatus {
    /// Pipeline is idle
    Idle,
    /// Pipeline is starting up
    Starting,
    /// Pipeline is running
    Running,
    /// Pipeline is paused
    Paused,
    /// Pipeline is stopping
    Stopping,
    /// Pipeline has stopped
    Stopped,
    /// Pipeline encountered an error
    Error,
}

/// Pipeline error
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct PipelineError {
    /// Error timestamp
    pub timestamp: chrono::DateTime<chrono::Utc>,
    /// Error message
    pub message: String,
    /// Error code
    pub code: String,
    /// Record ID that caused the error (if applicable)
    pub record_id: Option<String>,
    /// Stage where error occurred
    pub stage: String,
    /// Error severity
    pub severity: ErrorSeverity,
}

/// Error severity levels
#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize)]
pub enum ErrorSeverity {
    /// Information level
    Info,
    /// Warning level
    Warning,
    /// Error level
    Error,
    /// Critical error level
    Critical,
}

/// Validation stage trait
#[async_trait]
pub trait ValidationStage: Send + Sync + std::fmt::Debug {
    /// Stage name
    fn name(&self) -> &str;

    /// Stage description
    fn description(&self) -> &str;

    /// Execute validation stage
    async fn execute(&self, record: &DataRecord) -> Result<ValidationResult>;

    /// Check if stage is enabled
    fn is_enabled(&self) -> bool {
        true
    }

    /// Stage priority (higher number = higher priority)
    fn priority(&self) -> u32 {
        0
    }
}

/// Streaming validator for real-time processing
#[derive(Debug)]
pub struct StreamingValidator {
    /// Validation engine
    pub engine: Arc<ValidationEngine>,
    /// Stream configuration
    pub config: StreamConfig,
    /// Processing buffer
    pub buffer: Arc<RwLock<VecDeque<DataRecord>>>,
    /// Stream metrics
    pub metrics: Arc<RwLock<StreamMetrics>>,
}

/// Configuration for streaming validation
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct StreamConfig {
    /// Buffer size for incoming records
    pub buffer_size: usize,
    /// Batch size for processing
    pub batch_size: usize,
    /// Processing interval in milliseconds
    pub processing_interval_ms: u64,
    /// Maximum latency before forced processing
    pub max_latency_ms: u64,
    /// Backpressure handling strategy
    pub backpressure_strategy: BackpressureStrategy,
    /// Window configuration for windowed operations
    pub window_config: Option<WindowConfig>,
}

/// Backpressure handling strategies
#[derive(Debug, Clone, Serialize, Deserialize)]
pub enum BackpressureStrategy {
    /// Drop oldest records when buffer is full
    DropOldest,
    /// Drop newest records when buffer is full
    DropNewest,
    /// Block until buffer has space
    Block,
    /// Apply sampling to reduce load
    Sample(f64),
    /// Use spillover to disk
    Spillover,
}

/// Window configuration for streaming operations
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct WindowConfig {
    /// Window type
    pub window_type: WindowType,
    /// Window size
    pub window_size: Duration,
    /// Slide interval (for sliding windows)
    pub slide_interval: Option<Duration>,
    /// Allow late arrivals
    pub allow_late_arrivals: bool,
    /// Maximum lateness allowed
    pub max_lateness: Option<Duration>,
}

/// Types of streaming windows
#[derive(Debug, Clone, Serialize, Deserialize)]
pub enum WindowType {
    /// Tumbling window (non-overlapping)
    Tumbling,
    /// Sliding window (overlapping)
    Sliding,
    /// Session window (gap-based)
    Session,
    /// Global window (all data)
    Global,
}

/// Stream processing metrics
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct StreamMetrics {
    /// Records per second
    pub records_per_second: f64,
    /// Average processing latency
    pub avg_latency_ms: f64,
    /// 99th percentile latency
    pub p99_latency_ms: f64,
    /// Buffer utilization percentage
    pub buffer_utilization: f64,
    /// Backpressure events count
    pub backpressure_events: usize,
    /// Dropped records count
    pub dropped_records: usize,
    /// Processing errors count
    pub processing_errors: usize,
}

/// Batch validator for batch processing
#[derive(Debug)]
pub struct BatchValidator {
    /// Validation engine
    pub engine: Arc<ValidationEngine>,
    /// Batch configuration
    pub config: BatchConfig,
    /// Batch metrics
    pub metrics: Arc<RwLock<BatchMetrics>>,
}

/// Configuration for batch validation
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct BatchConfig {
    /// Batch size
    pub batch_size: usize,
    /// Maximum concurrent batches
    pub max_concurrent_batches: usize,
    /// Batch timeout in milliseconds
    pub batch_timeout_ms: u64,
    /// Checkpoint interval (number of batches)
    pub checkpoint_interval: usize,
    /// Enable batch optimization
    pub enable_optimization: bool,
    /// Memory limit per batch
    pub memory_limit_per_batch: usize,
}

/// Batch processing metrics
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct BatchMetrics {
    /// Total batches processed
    pub batches_processed: usize,
    /// Average batch processing time
    pub avg_batch_time_ms: f64,
    /// Records processed per batch (average)
    pub avg_records_per_batch: f64,
    /// Total processing time
    pub total_processing_time_ms: u64,
    /// Throughput (records per second)
    pub throughput_rps: f64,
    /// Failed batches count
    pub failed_batches: usize,
}

/// Validation command for pipeline control
#[derive(Debug)]
pub enum ValidationCommand {
    /// Start validation pipeline
    Start,
    /// Stop validation pipeline
    Stop,
    /// Pause validation pipeline
    Pause,
    /// Resume validation pipeline
    Resume,
    /// Process single record
    ProcessRecord {
        record: DataRecord,
        response: oneshot::Sender<Result<ValidationResult>>,
    },
    /// Process batch of records
    ProcessBatch {
        records: Vec<DataRecord>,
        response: oneshot::Sender<Result<Vec<ValidationResult>>>,
    },
    /// Get pipeline status
    GetStatus {
        response: oneshot::Sender<PipelineState>,
    },
    /// Get pipeline metrics
    GetMetrics {
        response: oneshot::Sender<ValidationMetrics>,
    },
}

impl Default for PipelineConfig {
    fn default() -> Self {
        Self {
            name: "default_pipeline".to_string(),
            max_concurrent: 10,
            batch_size: 100,
            buffer_size: 1000,
            validation_timeout_ms: 5000,
            collect_metrics: true,
            error_handling: ErrorHandlingStrategy::ContinueOnError,
            retry_config: RetryConfig::default(),
            performance_config: PerformanceConfig::default(),
        }
    }
}

impl Default for RetryConfig {
    fn default() -> Self {
        Self {
            max_retries: 3,
            base_delay_ms: 100,
            backoff_multiplier: 2.0,
            max_delay_ms: 5000,
            enable_jitter: true,
        }
    }
}

impl Default for PerformanceConfig {
    fn default() -> Self {
        Self {
            parallel_processing: true,
            cpu_limit_percent: 80.0,
            memory_limit_bytes: 1_000_000_000, // 1GB
            io_throttling: IoThrottling {
                enabled: false,
                max_iops: 1000,
                max_bandwidth_bps: 100_000_000, // 100MB/s
            },
            cache_config: CacheConfig {
                enabled: true,
                size_limit: 10000,
                ttl_seconds: 300,
                eviction_policy: CacheEvictionPolicy::LRU,
            },
        }
    }
}

impl Default for StreamConfig {
    fn default() -> Self {
        Self {
            buffer_size: 1000,
            batch_size: 100,
            processing_interval_ms: 1000,
            max_latency_ms: 5000,
            backpressure_strategy: BackpressureStrategy::DropOldest,
            window_config: None,
        }
    }
}

impl Default for BatchConfig {
    fn default() -> Self {
        Self {
            batch_size: 1000,
            max_concurrent_batches: 5,
            batch_timeout_ms: 30000,
            checkpoint_interval: 10,
            enable_optimization: true,
            memory_limit_per_batch: 100_000_000, // 100MB
        }
    }
}

impl ValidationPipeline {
    /// Create a new validation pipeline
    pub fn new(config: PipelineConfig) -> Self {
        Self {
            config,
            stages: Vec::new(),
            metrics: Arc::new(RwLock::new(ValidationMetrics::new())),
            state: Arc::new(RwLock::new(PipelineState {
                status: PipelineStatus::Idle,
                records_processed: 0,
                validation_failures: 0,
                started_at: None,
                last_activity: None,
                current_throughput: 0.0,
                errors: Vec::new(),
            })),
        }
    }

    /// Add a validation stage to the pipeline
    pub fn add_stage(&mut self, stage: Arc<dyn ValidationStage>) {
        self.stages.push(stage);
        // Sort stages by priority
        self.stages.sort_by(|a, b| b.priority().cmp(&a.priority()));
    }

    /// Start the validation pipeline
    pub async fn start(&mut self) -> Result<()> {
        info!("Starting validation pipeline: {}", self.config.name);

        {
            let mut state = self.state.write();
            state.status = PipelineStatus::Starting;
            state.started_at = Some(chrono::Utc::now());
        }

        // Perform startup validation
        if self.stages.is_empty() {
            warn!("Pipeline has no validation stages configured");
        }

        {
            let mut state = self.state.write();
            state.status = PipelineStatus::Running;
        }

        info!("Validation pipeline started successfully");
        Ok(())
    }

    /// Stop the validation pipeline
    pub async fn stop(&mut self) -> Result<()> {
        info!("Stopping validation pipeline: {}", self.config.name);

        {
            let mut state = self.state.write();
            state.status = PipelineStatus::Stopping;
        }

        // Wait for active validations to complete
        tokio::time::sleep(Duration::from_millis(100)).await;

        {
            let mut state = self.state.write();
            state.status = PipelineStatus::Stopped;
        }

        info!("Validation pipeline stopped");
        Ok(())
    }

    /// Process a single record through the pipeline
    pub async fn process_record(&self, record: &DataRecord) -> Result<ValidationResult> {
        let start_time = Instant::now();

        // Check pipeline status
        {
            let state = self.state.read();
            if state.status != PipelineStatus::Running {
                return Err(ValidationError::Pipeline(
                    "Pipeline is not running".to_string(),
                ));
            }
        }

        // Execute all validation stages
        let mut final_result = ValidationResult {
            is_valid: true,
            rule_results: Vec::new(),
            quality_score: None,
            profile: None,
            anomaly_results: Vec::new(),
            schema_validation: None,
            performance: crate::metrics::PerformanceMetrics::default(),
            timestamp: chrono::Utc::now(),
        };

        for stage in &self.stages {
            if !stage.is_enabled() {
                continue;
            }

            debug!("Executing stage: {}", stage.name());

            match tokio::time::timeout(
                Duration::from_millis(self.config.validation_timeout_ms),
                stage.execute(record),
            )
            .await
            {
                Ok(Ok(stage_result)) => {
                    // Merge stage result into final result
                    final_result.rule_results.extend(stage_result.rule_results);
                    final_result
                        .anomaly_results
                        .extend(stage_result.anomaly_results);

                    if !stage_result.is_valid {
                        final_result.is_valid = false;
                    }

                    if stage_result.quality_score.is_some() {
                        final_result.quality_score = stage_result.quality_score;
                    }

                    if stage_result.profile.is_some() {
                        final_result.profile = stage_result.profile;
                    }

                    if stage_result.schema_validation.is_some() {
                        final_result.schema_validation = stage_result.schema_validation;
                    }
                }
                Ok(Err(e)) => {
                    error!("Stage {} failed: {}", stage.name(), e);
                    match self.config.error_handling {
                        ErrorHandlingStrategy::FailFast => return Err(e),
                        ErrorHandlingStrategy::ContinueOnError => {
                            final_result.is_valid = false;
                            self.record_error(
                                &format!("Stage {} failed", stage.name()),
                                &e.to_string(),
                                Some(&record.id),
                                stage.name(),
                            );
                        }
                        ErrorHandlingStrategy::SkipInvalid => {
                            return Err(ValidationError::Pipeline(
                                "Record skipped due to stage failure".to_string(),
                            ));
                        }
                        ErrorHandlingStrategy::Fallback => {
                            // Use fallback validation (simplified)
                            final_result.is_valid = false;
                        }
                    }
                }
                Err(_) => {
                    error!("Stage {} timed out", stage.name());
                    match self.config.error_handling {
                        ErrorHandlingStrategy::FailFast => {
                            return Err(ValidationError::Pipeline("Stage timeout".to_string()));
                        }
                        _ => {
                            final_result.is_valid = false;
                            self.record_error(
                                "Stage timeout",
                                "Validation stage exceeded timeout",
                                Some(&record.id),
                                stage.name(),
                            );
                        }
                    }
                }
            }
        }

        // Update metrics and state
        let duration = start_time.elapsed();
        {
            let mut state = self.state.write();
            state.records_processed += 1;
            if !final_result.is_valid {
                state.validation_failures += 1;
            }
            state.last_activity = Some(chrono::Utc::now());

            // Simple throughput calculation
            if let Some(started_at) = state.started_at {
                let elapsed_seconds = (chrono::Utc::now() - started_at).num_seconds() as f64;
                if elapsed_seconds > 0.0 {
                    state.current_throughput = state.records_processed as f64 / elapsed_seconds;
                }
            }
        }

        if self.config.collect_metrics {
            let mut metrics = self.metrics.write();
            metrics.record_validation(duration, final_result.is_valid);
        }

        final_result.performance.validation_duration_ms = duration.as_millis() as u64;
        final_result.performance.rule_count = final_result.rule_results.len();
        final_result.performance.records_processed = 1;

        Ok(final_result)
    }

    /// Process multiple records in batch
    pub async fn process_batch(&self, records: &[DataRecord]) -> Result<Vec<ValidationResult>> {
        let mut results = Vec::with_capacity(records.len());

        // Process records sequentially for now (parallel processing can be added later with proper Arc/Mutex)
        {
            // Sequential processing
            for record in records {
                match self.process_record(record).await {
                    Ok(result) => results.push(result),
                    Err(e) => match self.config.error_handling {
                        ErrorHandlingStrategy::FailFast => return Err(e),
                        ErrorHandlingStrategy::SkipInvalid => continue,
                        _ => {
                            results.push(ValidationResult {
                                is_valid: false,
                                rule_results: Vec::new(),
                                quality_score: None,
                                profile: None,
                                anomaly_results: Vec::new(),
                                schema_validation: None,
                                performance: crate::metrics::PerformanceMetrics::default(),
                                timestamp: chrono::Utc::now(),
                            });
                        }
                    },
                }
            }
        }

        Ok(results)
    }

    /// Get current pipeline status
    pub fn status(&self) -> PipelineState {
        self.state.read().clone()
    }

    /// Get pipeline metrics
    pub fn metrics(&self) -> ValidationMetrics {
        self.metrics.read().clone()
    }

    fn record_error(&self, message: &str, details: &str, record_id: Option<&str>, stage: &str) {
        let error = PipelineError {
            timestamp: chrono::Utc::now(),
            message: message.to_string(),
            code: "STAGE_ERROR".to_string(),
            record_id: record_id.map(|s| s.to_string()),
            stage: stage.to_string(),
            severity: ErrorSeverity::Error,
        };

        let mut state = self.state.write();
        state.errors.push(error);

        // Limit error history
        if state.errors.len() > 1000 {
            state.errors.remove(0);
        }
    }
}

impl StreamingValidator {
    /// Create a new streaming validator
    pub fn new(engine: Arc<ValidationEngine>, config: StreamConfig) -> Self {
        let buffer_size = config.buffer_size;
        Self {
            engine,
            config,
            buffer: Arc::new(RwLock::new(VecDeque::with_capacity(buffer_size))),
            metrics: Arc::new(RwLock::new(StreamMetrics {
                records_per_second: 0.0,
                avg_latency_ms: 0.0,
                p99_latency_ms: 0.0,
                buffer_utilization: 0.0,
                backpressure_events: 0,
                dropped_records: 0,
                processing_errors: 0,
            })),
        }
    }

    /// Add a record to the stream buffer
    pub async fn add_record(&self, record: DataRecord) -> Result<()> {
        let mut buffer = self.buffer.write();

        if buffer.len() >= self.config.buffer_size {
            // Handle backpressure
            match self.config.backpressure_strategy {
                BackpressureStrategy::DropOldest => {
                    buffer.pop_front();
                    let mut metrics = self.metrics.write();
                    metrics.dropped_records += 1;
                    metrics.backpressure_events += 1;
                }
                BackpressureStrategy::DropNewest => {
                    let mut metrics = self.metrics.write();
                    metrics.dropped_records += 1;
                    metrics.backpressure_events += 1;
                    return Ok(());
                }
                BackpressureStrategy::Block => {
                    // Drop lock and wait, then retry
                    drop(buffer);
                    tokio::time::sleep(Duration::from_millis(10)).await;
                    return Box::pin(self.add_record(record)).await;
                }
                BackpressureStrategy::Sample(rate) => {
                    if rand::random::<f64>() > rate {
                        let mut metrics = self.metrics.write();
                        metrics.dropped_records += 1;
                        return Ok(());
                    }
                }
                BackpressureStrategy::Spillover => {
                    // In a real implementation, would spill to disk
                    buffer.pop_front();
                    let mut metrics = self.metrics.write();
                    metrics.dropped_records += 1;
                }
            }
        }

        buffer.push_back(record);
        Ok(())
    }

    /// Process records from the stream buffer
    pub async fn process_stream(&self) -> Result<Vec<ValidationResult>> {
        let mut records_to_process = Vec::new();

        {
            let mut buffer = self.buffer.write();
            let batch_size = self.config.batch_size.min(buffer.len());

            for _ in 0..batch_size {
                if let Some(record) = buffer.pop_front() {
                    records_to_process.push(record);
                }
            }

            // Update buffer utilization metric
            let mut metrics = self.metrics.write();
            metrics.buffer_utilization =
                buffer.len() as f64 / self.config.buffer_size as f64 * 100.0;
        }

        if records_to_process.is_empty() {
            return Ok(Vec::new());
        }

        // Process records
        let start_time = Instant::now();
        let mut results = Vec::new();

        for record in &records_to_process {
            match self.engine.validate(record).await {
                Ok(result) => results.push(result),
                Err(_e) => {
                    let mut metrics = self.metrics.write();
                    metrics.processing_errors += 1;
                }
            }
        }

        // Update metrics
        let duration = start_time.elapsed();
        let mut metrics = self.metrics.write();

        let records_per_ms = records_to_process.len() as f64 / duration.as_millis() as f64;
        metrics.records_per_second = records_per_ms * 1000.0;
        metrics.avg_latency_ms = duration.as_millis() as f64 / records_to_process.len() as f64;

        Ok(results)
    }

    /// Get current stream metrics
    pub fn metrics(&self) -> StreamMetrics {
        self.metrics.read().clone()
    }
}

impl BatchValidator {
    /// Create a new batch validator
    pub fn new(engine: Arc<ValidationEngine>, config: BatchConfig) -> Self {
        Self {
            engine,
            config,
            metrics: Arc::new(RwLock::new(BatchMetrics {
                batches_processed: 0,
                avg_batch_time_ms: 0.0,
                avg_records_per_batch: 0.0,
                total_processing_time_ms: 0,
                throughput_rps: 0.0,
                failed_batches: 0,
            })),
        }
    }

    /// Process a batch of records
    pub async fn process_batch(&self, records: Vec<DataRecord>) -> Result<Vec<ValidationResult>> {
        let start_time = Instant::now();
        let batch_size = records.len();

        // Process batch with timeout
        let validation_future = self.engine.validate_batch(&records);
        let results = match tokio::time::timeout(
            Duration::from_millis(self.config.batch_timeout_ms),
            validation_future,
        )
        .await
        {
            Ok(Ok(results)) => results,
            Ok(Err(e)) => {
                let mut metrics = self.metrics.write();
                metrics.failed_batches += 1;
                return Err(e);
            }
            Err(_) => {
                let mut metrics = self.metrics.write();
                metrics.failed_batches += 1;
                return Err(ValidationError::Pipeline("Batch timeout".to_string()));
            }
        };

        // Update metrics
        let duration = start_time.elapsed();
        let mut metrics = self.metrics.write();

        metrics.batches_processed += 1;
        metrics.total_processing_time_ms += duration.as_millis() as u64;
        metrics.avg_batch_time_ms =
            metrics.total_processing_time_ms as f64 / metrics.batches_processed as f64;

        let total_records = metrics.batches_processed * batch_size;
        metrics.avg_records_per_batch = total_records as f64 / metrics.batches_processed as f64;

        let total_seconds = metrics.total_processing_time_ms as f64 / 1000.0;
        if total_seconds > 0.0 {
            metrics.throughput_rps = total_records as f64 / total_seconds;
        }

        Ok(results)
    }

    /// Get current batch metrics
    pub fn metrics(&self) -> BatchMetrics {
        self.metrics.read().clone()
    }
}

// Standard validation stage implementation
#[derive(Debug)]
pub struct StandardValidationStage {
    pub name: String,
    pub engine: Arc<ValidationEngine>,
    pub enabled: bool,
    pub priority: u32,
}

#[async_trait]
impl ValidationStage for StandardValidationStage {
    fn name(&self) -> &str {
        &self.name
    }

    fn description(&self) -> &str {
        "Standard validation using ValidationEngine"
    }

    async fn execute(&self, record: &DataRecord) -> Result<ValidationResult> {
        self.engine.validate(record).await
    }

    fn is_enabled(&self) -> bool {
        self.enabled
    }

    fn priority(&self) -> u32 {
        self.priority
    }
}

#[cfg(test)]
mod tests {
    use super::*;
    use crate::engine::ValidationEngine;
    use std::collections::HashMap;

    #[tokio::test]
    async fn test_pipeline_creation() {
        let config = PipelineConfig::default();
        let pipeline = ValidationPipeline::new(config);

        assert_eq!(pipeline.stages.len(), 0);
        assert_eq!(pipeline.status().status, PipelineStatus::Idle);
    }

    #[tokio::test]
    async fn test_pipeline_start_stop() {
        let config = PipelineConfig::default();
        let mut pipeline = ValidationPipeline::new(config);

        pipeline.start().await.unwrap();
        assert_eq!(pipeline.status().status, PipelineStatus::Running);

        pipeline.stop().await.unwrap();
        assert_eq!(pipeline.status().status, PipelineStatus::Stopped);
    }

    #[tokio::test]
    async fn test_streaming_validator() {
        let engine = Arc::new(ValidationEngine::new());
        let config = StreamConfig::default();
        let validator = StreamingValidator::new(engine, config);

        let record = crate::DataRecord {
            id: "test".to_string(),
            timestamp: chrono::Utc::now(),
            fields: HashMap::new(),
            metadata: HashMap::new(),
        };

        validator.add_record(record).await.unwrap();
        let results = validator.process_stream().await.unwrap();
        assert!(!results.is_empty());
    }

    #[tokio::test]
    async fn test_batch_validator() {
        let engine = Arc::new(ValidationEngine::new());
        let config = BatchConfig::default();
        let validator = BatchValidator::new(engine, config);

        let records = vec![crate::DataRecord {
            id: "test".to_string(),
            timestamp: chrono::Utc::now(),
            fields: HashMap::new(),
            metadata: HashMap::new(),
        }];

        let results = validator.process_batch(records).await.unwrap();
        assert!(!results.is_empty());
    }

    #[test]
    fn test_error_severity_ordering() {
        assert!(ErrorSeverity::Info < ErrorSeverity::Warning);
        assert!(ErrorSeverity::Warning < ErrorSeverity::Error);
        assert!(ErrorSeverity::Error < ErrorSeverity::Critical);
    }
}