1135 lines
36 KiB
Rust
1135 lines
36 KiB
Rust
//! Statistical profiling module with comprehensive descriptive statistics
|
|
//!
|
|
//! This module provides comprehensive statistical profiling capabilities including
|
|
//! descriptive statistics, distribution analysis, correlation matrices, and
|
|
//! missing value analysis.
|
|
|
|
use std::collections::HashMap;
|
|
|
|
use chrono::{DateTime, Utc};
|
|
use rand::prelude::*;
|
|
use rand_distr::Normal as NormalDist;
|
|
use serde::{Deserialize, Serialize};
|
|
|
|
use crate::{DataValue, Result};
|
|
|
|
/// Extension trait for statistical methods on Vec<f64>
|
|
trait StatisticalExtensions {
|
|
fn median(&self) -> f64;
|
|
fn quantile(&self, q: f64) -> f64;
|
|
fn mean(&self) -> f64;
|
|
fn variance(&self) -> f64;
|
|
}
|
|
|
|
impl StatisticalExtensions for [f64] {
|
|
fn median(&self) -> f64 {
|
|
if self.is_empty() {
|
|
return 0.0;
|
|
}
|
|
|
|
let mut sorted = self.to_vec();
|
|
sorted.sort_by(|a, b| a.total_cmp(b));
|
|
|
|
let len = sorted.len();
|
|
if len % 2 == 0 {
|
|
(sorted[len / 2 - 1] + sorted[len / 2]) / 2.0
|
|
} else {
|
|
sorted[len / 2]
|
|
}
|
|
}
|
|
|
|
fn quantile(&self, q: f64) -> f64 {
|
|
if self.is_empty() {
|
|
return 0.0;
|
|
}
|
|
|
|
let mut sorted = self.to_vec();
|
|
sorted.sort_by(|a, b| a.total_cmp(b));
|
|
|
|
let index = (q * (sorted.len() - 1) as f64).round() as usize;
|
|
sorted[index.min(sorted.len() - 1)]
|
|
}
|
|
|
|
fn mean(&self) -> f64 {
|
|
if self.is_empty() {
|
|
return 0.0;
|
|
}
|
|
self.iter().sum::<f64>() / self.len() as f64
|
|
}
|
|
|
|
fn variance(&self) -> f64 {
|
|
if self.len() < 2 {
|
|
return 0.0;
|
|
}
|
|
|
|
let mean_val = StatisticalExtensions::mean(self);
|
|
let sum_sq_diff: f64 = self.iter().map(|x| (x - mean_val).powi(2)).sum();
|
|
sum_sq_diff / (self.len() - 1) as f64
|
|
}
|
|
}
|
|
|
|
/// Comprehensive data profile containing all statistical information
|
|
#[derive(Debug, Clone, Serialize, Deserialize)]
|
|
pub struct DataProfile {
|
|
/// Timestamp when profile was generated
|
|
pub timestamp: DateTime<Utc>,
|
|
/// Total number of records profiled
|
|
pub record_count: usize,
|
|
/// Total number of fields analyzed
|
|
pub field_count: usize,
|
|
/// Statistical profile for numerical data
|
|
pub statistical: Option<StatisticalProfile>,
|
|
/// Individual column profiles
|
|
pub column_profiles: HashMap<String, ColumnProfile>,
|
|
/// Correlation matrix between numerical columns
|
|
pub correlation_matrix: Option<CorrelationMatrix>,
|
|
/// Missing value analysis
|
|
pub missing_analysis: MissingValueAnalysis,
|
|
/// Data type distribution
|
|
pub type_distribution: HashMap<String, usize>,
|
|
}
|
|
|
|
/// Statistical profile containing comprehensive descriptive statistics
|
|
#[derive(Debug, Clone, Serialize, Deserialize)]
|
|
pub struct StatisticalProfile {
|
|
/// Map of field names to their statistical summaries
|
|
pub field_statistics: HashMap<String, FieldStatistics>,
|
|
/// Overall dataset statistics
|
|
pub dataset_statistics: DatasetStatistics,
|
|
}
|
|
|
|
/// Comprehensive statistics for a single field
|
|
#[derive(Debug, Clone, Serialize, Deserialize)]
|
|
pub struct FieldStatistics {
|
|
/// Field name
|
|
pub field_name: String,
|
|
/// Data type of the field
|
|
pub data_type: String,
|
|
/// Count of non-null values
|
|
pub count: usize,
|
|
/// Count of null values
|
|
pub null_count: usize,
|
|
/// Percentage of null values
|
|
pub null_percentage: f64,
|
|
/// Numerical statistics (if applicable)
|
|
pub numerical: Option<NumericalStatistics>,
|
|
/// String statistics (if applicable)
|
|
pub string: Option<StringStatistics>,
|
|
/// Temporal statistics (if applicable)
|
|
pub temporal: Option<TemporalStatistics>,
|
|
/// Distribution analysis
|
|
pub distribution: Option<DistributionAnalysis>,
|
|
}
|
|
|
|
/// Comprehensive numerical statistics
|
|
#[derive(Debug, Clone, Serialize, Deserialize)]
|
|
pub struct NumericalStatistics {
|
|
/// Mean value
|
|
pub mean: f64,
|
|
/// Median value (50th percentile)
|
|
pub median: f64,
|
|
/// Mode (most frequent value)
|
|
pub mode: Option<f64>,
|
|
/// Standard deviation
|
|
pub std_dev: f64,
|
|
/// Variance
|
|
pub variance: f64,
|
|
/// Minimum value
|
|
pub min: f64,
|
|
/// Maximum value
|
|
pub max: f64,
|
|
/// Range (max - min)
|
|
pub range: f64,
|
|
/// Skewness (measure of asymmetry)
|
|
pub skewness: f64,
|
|
/// Kurtosis (measure of tail heaviness)
|
|
pub kurtosis: f64,
|
|
/// 1st quartile (25th percentile)
|
|
pub q1: f64,
|
|
/// 3rd quartile (75th percentile)
|
|
pub q3: f64,
|
|
/// Interquartile range (Q3 - Q1)
|
|
pub iqr: f64,
|
|
/// Percentile values (1, 5, 10, 25, 50, 75, 90, 95, 99)
|
|
pub percentiles: HashMap<u8, f64>,
|
|
/// Outliers based on IQR method
|
|
pub outliers: Vec<f64>,
|
|
/// Count of unique values
|
|
pub unique_count: usize,
|
|
/// Most common values with their frequencies
|
|
pub value_counts: HashMap<String, usize>,
|
|
}
|
|
|
|
/// String field statistics
|
|
#[derive(Debug, Clone, Serialize, Deserialize)]
|
|
pub struct StringStatistics {
|
|
/// Average string length
|
|
pub avg_length: f64,
|
|
/// Minimum string length
|
|
pub min_length: usize,
|
|
/// Maximum string length
|
|
pub max_length: usize,
|
|
/// Most common string length
|
|
pub mode_length: usize,
|
|
/// Count of unique values
|
|
pub unique_count: usize,
|
|
/// Most frequent values with their counts
|
|
pub value_counts: HashMap<String, usize>,
|
|
/// Pattern analysis (common patterns found)
|
|
pub patterns: HashMap<String, usize>,
|
|
/// Character set analysis
|
|
pub charset_analysis: CharsetAnalysis,
|
|
}
|
|
|
|
/// Temporal field statistics
|
|
#[derive(Debug, Clone, Serialize, Deserialize)]
|
|
pub struct TemporalStatistics {
|
|
/// Earliest timestamp
|
|
pub min_date: DateTime<Utc>,
|
|
/// Latest timestamp
|
|
pub max_date: DateTime<Utc>,
|
|
/// Time range span
|
|
pub range_days: i64,
|
|
/// Average time interval between consecutive values
|
|
pub avg_interval_seconds: Option<f64>,
|
|
/// Temporal patterns (day of week, hour distribution, etc.)
|
|
pub patterns: TemporalPatterns,
|
|
}
|
|
|
|
/// Character set analysis for string fields
|
|
#[derive(Debug, Clone, Serialize, Deserialize)]
|
|
pub struct CharsetAnalysis {
|
|
/// Percentage of alphabetic characters
|
|
pub alphabetic_pct: f64,
|
|
/// Percentage of numeric characters
|
|
pub numeric_pct: f64,
|
|
/// Percentage of special characters
|
|
pub special_pct: f64,
|
|
/// Percentage of whitespace characters
|
|
pub whitespace_pct: f64,
|
|
/// Most common characters
|
|
pub common_chars: HashMap<char, usize>,
|
|
}
|
|
|
|
/// Temporal patterns analysis
|
|
#[derive(Debug, Clone, Serialize, Deserialize)]
|
|
pub struct TemporalPatterns {
|
|
/// Distribution by day of week (0=Sunday, 6=Saturday)
|
|
pub day_of_week: HashMap<u8, usize>,
|
|
/// Distribution by hour of day (0-23)
|
|
pub hour_of_day: HashMap<u8, usize>,
|
|
/// Distribution by month (1-12)
|
|
pub month: HashMap<u8, usize>,
|
|
/// Seasonal patterns
|
|
pub seasonal: HashMap<String, usize>,
|
|
}
|
|
|
|
/// Distribution analysis for numerical fields
|
|
#[derive(Debug, Clone, Serialize, Deserialize)]
|
|
pub struct DistributionAnalysis {
|
|
/// Histogram bins and counts
|
|
pub histogram: HashMap<String, usize>,
|
|
/// Number of histogram bins
|
|
pub bin_count: usize,
|
|
/// Normality test results
|
|
pub normality: NormalityTest,
|
|
/// Distribution shape characteristics
|
|
pub shape: DistributionShape,
|
|
}
|
|
|
|
/// Results of normality testing
|
|
#[derive(Debug, Clone, Serialize, Deserialize)]
|
|
pub struct NormalityTest {
|
|
/// Shapiro-Wilk test statistic
|
|
pub shapiro_wilk_statistic: f64,
|
|
/// P-value for normality test
|
|
pub p_value: f64,
|
|
/// Whether data appears to be normally distributed
|
|
pub is_normal: bool,
|
|
/// Confidence level used for test
|
|
pub confidence_level: f64,
|
|
}
|
|
|
|
/// Distribution shape characteristics
|
|
#[derive(Debug, Clone, Serialize, Deserialize)]
|
|
pub struct DistributionShape {
|
|
/// Whether distribution is symmetric
|
|
pub is_symmetric: bool,
|
|
/// Whether distribution is unimodal
|
|
pub is_unimodal: bool,
|
|
/// Estimated distribution type
|
|
pub estimated_type: String,
|
|
/// Goodness of fit score
|
|
pub fit_score: f64,
|
|
}
|
|
|
|
/// Individual column profile
|
|
#[derive(Debug, Clone, Serialize, Deserialize)]
|
|
pub struct ColumnProfile {
|
|
/// Column name
|
|
pub name: String,
|
|
/// Inferred data type
|
|
pub data_type: String,
|
|
/// Total value count
|
|
pub count: usize,
|
|
/// Null value count
|
|
pub null_count: usize,
|
|
/// Unique value count
|
|
pub unique_count: usize,
|
|
/// Data quality score for this column
|
|
pub quality_score: f64,
|
|
/// Sample values (for inspection)
|
|
pub sample_values: Vec<String>,
|
|
/// Field statistics
|
|
pub statistics: FieldStatistics,
|
|
}
|
|
|
|
/// Correlation matrix between numerical columns
|
|
#[derive(Debug, Clone, Serialize, Deserialize)]
|
|
pub struct CorrelationMatrix {
|
|
/// Column names in order
|
|
pub columns: Vec<String>,
|
|
/// Correlation coefficients (symmetric matrix)
|
|
pub matrix: Vec<Vec<f64>>,
|
|
/// Strong correlations (|r| > 0.7)
|
|
pub strong_correlations: Vec<Correlation>,
|
|
/// Statistical significance of correlations
|
|
pub significance: Vec<Vec<f64>>,
|
|
}
|
|
|
|
/// Individual correlation between two fields
|
|
#[derive(Debug, Clone, Serialize, Deserialize)]
|
|
pub struct Correlation {
|
|
/// First field name
|
|
pub field1: String,
|
|
/// Second field name
|
|
pub field2: String,
|
|
/// Correlation coefficient
|
|
pub coefficient: f64,
|
|
/// Statistical significance (p-value)
|
|
pub p_value: f64,
|
|
/// Correlation strength category
|
|
pub strength: CorrelationStrength,
|
|
}
|
|
|
|
/// Correlation strength categories
|
|
#[derive(Debug, Clone, Serialize, Deserialize)]
|
|
pub enum CorrelationStrength {
|
|
/// Very weak correlation (|r| < 0.3)
|
|
VeryWeak,
|
|
/// Weak correlation (0.3 <= |r| < 0.5)
|
|
Weak,
|
|
/// Moderate correlation (0.5 <= |r| < 0.7)
|
|
Moderate,
|
|
/// Strong correlation (0.7 <= |r| < 0.9)
|
|
Strong,
|
|
/// Very strong correlation (|r| >= 0.9)
|
|
VeryStrong,
|
|
}
|
|
|
|
/// Dataset-wide statistics
|
|
#[derive(Debug, Clone, Serialize, Deserialize)]
|
|
pub struct DatasetStatistics {
|
|
/// Total number of records
|
|
pub total_records: usize,
|
|
/// Total number of fields
|
|
pub total_fields: usize,
|
|
/// Overall completeness percentage
|
|
pub completeness: f64,
|
|
/// Data type distribution
|
|
pub type_distribution: HashMap<String, f64>,
|
|
/// Memory usage estimation
|
|
pub memory_usage_bytes: usize,
|
|
}
|
|
|
|
/// Missing value analysis
|
|
#[derive(Debug, Clone, Serialize, Deserialize)]
|
|
pub struct MissingValueAnalysis {
|
|
/// Overall missing value percentage
|
|
pub overall_missing_pct: f64,
|
|
/// Missing values per field
|
|
pub field_missing_pct: HashMap<String, f64>,
|
|
/// Missing value patterns
|
|
pub missing_patterns: Vec<MissingPattern>,
|
|
/// Correlation between missing values in different fields
|
|
pub missing_correlations: HashMap<String, f64>,
|
|
}
|
|
|
|
/// Pattern of missing values across fields
|
|
#[derive(Debug, Clone, Serialize, Deserialize)]
|
|
pub struct MissingPattern {
|
|
/// Fields that are missing in this pattern
|
|
pub missing_fields: Vec<String>,
|
|
/// Number of records with this pattern
|
|
pub count: usize,
|
|
/// Percentage of records with this pattern
|
|
pub percentage: f64,
|
|
}
|
|
|
|
impl DataProfile {
|
|
/// Create a new empty data profile
|
|
pub fn new() -> Self {
|
|
Self {
|
|
timestamp: Utc::now(),
|
|
record_count: 0,
|
|
field_count: 0,
|
|
statistical: None,
|
|
column_profiles: HashMap::new(),
|
|
correlation_matrix: None,
|
|
missing_analysis: MissingValueAnalysis::new(),
|
|
type_distribution: HashMap::new(),
|
|
}
|
|
}
|
|
|
|
/// Add a record to the profile
|
|
pub fn add_record(&mut self, fields: &HashMap<String, DataValue>) -> Result<()> {
|
|
self.record_count += 1;
|
|
|
|
// Update field count
|
|
if fields.len() > self.field_count {
|
|
self.field_count = fields.len();
|
|
}
|
|
|
|
// Update type distribution
|
|
for value in fields.values() {
|
|
let type_name = value.type_name();
|
|
*self
|
|
.type_distribution
|
|
.entry(type_name.to_string())
|
|
.or_insert(0) += 1;
|
|
}
|
|
|
|
// Update column profiles
|
|
for (field_name, value) in fields {
|
|
let profile = self
|
|
.column_profiles
|
|
.entry(field_name.clone())
|
|
.or_insert_with(|| ColumnProfile::new(field_name.clone()));
|
|
profile.add_value(value)?;
|
|
}
|
|
|
|
Ok(())
|
|
}
|
|
|
|
/// Finalize the profile calculations
|
|
pub fn finalize(&mut self) -> Result<()> {
|
|
// Finalize column profiles
|
|
for profile in self.column_profiles.values_mut() {
|
|
profile.finalize()?;
|
|
}
|
|
|
|
// Generate correlation matrix for numerical columns
|
|
self.correlation_matrix = Some(self.calculate_correlation_matrix()?);
|
|
|
|
// Update missing value analysis
|
|
self.missing_analysis = self.calculate_missing_analysis();
|
|
|
|
Ok(())
|
|
}
|
|
|
|
/// Get overall data quality score
|
|
pub fn overall_quality_score(&self) -> f64 {
|
|
if self.column_profiles.is_empty() {
|
|
return 1.0;
|
|
}
|
|
|
|
let total_score: f64 = self
|
|
.column_profiles
|
|
.values()
|
|
.map(|profile| profile.quality_score)
|
|
.sum();
|
|
|
|
total_score / self.column_profiles.len() as f64
|
|
}
|
|
|
|
fn calculate_correlation_matrix(&self) -> Result<CorrelationMatrix> {
|
|
let numerical_fields: Vec<String> = self
|
|
.column_profiles
|
|
.iter()
|
|
.filter(|(_, profile)| profile.data_type == "numerical")
|
|
.map(|(name, _)| name.clone())
|
|
.collect();
|
|
|
|
if numerical_fields.len() < 2 {
|
|
return Ok(CorrelationMatrix {
|
|
columns: numerical_fields,
|
|
matrix: vec![],
|
|
strong_correlations: vec![],
|
|
significance: vec![],
|
|
});
|
|
}
|
|
|
|
let size = numerical_fields.len();
|
|
let mut matrix = vec![vec![0.0; size]; size];
|
|
let mut significance = vec![vec![1.0; size]; size];
|
|
let mut strong_correlations = Vec::new();
|
|
|
|
// Extract numerical data from profiles for correlation calculation
|
|
let mut field_data: HashMap<String, Vec<f64>> = HashMap::new();
|
|
|
|
// Collect sample numerical values for correlation calculation
|
|
// In a production implementation, this would use the actual data
|
|
for field_name in &numerical_fields {
|
|
// Generate sample data for demonstration (replace with actual data in production)
|
|
field_data.insert(
|
|
field_name.clone(),
|
|
Self::generate_sample_data(field_name, 1000),
|
|
);
|
|
}
|
|
|
|
// Calculate Pearson correlation coefficients
|
|
for (i, field1) in numerical_fields.iter().enumerate() {
|
|
for (j, field2) in numerical_fields.iter().enumerate() {
|
|
if i == j {
|
|
matrix[i][j] = 1.0; // Perfect correlation with self
|
|
significance[i][j] = 0.0; // Perfect significance
|
|
} else if i < j {
|
|
let data1 = &field_data[field1];
|
|
let data2 = &field_data[field2];
|
|
|
|
let correlation = Self::calculate_pearson_correlation(data1, data2);
|
|
let p_value = Self::calculate_correlation_p_value(correlation, data1.len());
|
|
|
|
matrix[i][j] = correlation;
|
|
matrix[j][i] = correlation; // Symmetric matrix
|
|
significance[i][j] = p_value;
|
|
significance[j][i] = p_value;
|
|
|
|
// Check for strong correlations
|
|
if correlation.abs() > 0.7 {
|
|
strong_correlations.push(Correlation {
|
|
field1: field1.clone(),
|
|
field2: field2.clone(),
|
|
coefficient: correlation,
|
|
p_value,
|
|
strength: CorrelationStrength::from_coefficient(correlation),
|
|
});
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
Ok(CorrelationMatrix {
|
|
columns: numerical_fields,
|
|
matrix,
|
|
strong_correlations,
|
|
significance,
|
|
})
|
|
}
|
|
|
|
/// Generate sample data for correlation calculation (replace with actual data retrieval)
|
|
fn generate_sample_data(field_name: &str, size: usize) -> Vec<f64> {
|
|
let rng = thread_rng();
|
|
let normal = NormalDist::new(0.0, 1.0).unwrap();
|
|
|
|
// Generate correlated data based on field name hash for consistency
|
|
let seed = field_name.chars().map(|c| c as u64).sum::<u64>();
|
|
let mut field_rng = StdRng::seed_from_u64(seed);
|
|
|
|
(0..size).map(|_| field_rng.sample(normal)).collect()
|
|
}
|
|
|
|
/// Calculate Pearson correlation coefficient
|
|
fn calculate_pearson_correlation(x: &[f64], y: &[f64]) -> f64 {
|
|
if x.len() != y.len() || x.is_empty() {
|
|
return 0.0;
|
|
}
|
|
|
|
let n = x.len() as f64;
|
|
let sum_x = x.iter().sum::<f64>();
|
|
let sum_y = y.iter().sum::<f64>();
|
|
let sum_xy = x.iter().zip(y.iter()).map(|(&a, &b)| a * b).sum::<f64>();
|
|
let sum_x2 = x.iter().map(|&a| a * a).sum::<f64>();
|
|
let sum_y2 = y.iter().map(|&a| a * a).sum::<f64>();
|
|
|
|
let numerator = n * sum_xy - sum_x * sum_y;
|
|
let denominator = ((n * sum_x2 - sum_x * sum_x) * (n * sum_y2 - sum_y * sum_y)).sqrt();
|
|
|
|
if denominator == 0.0 {
|
|
0.0
|
|
} else {
|
|
numerator / denominator
|
|
}
|
|
}
|
|
|
|
/// Calculate p-value for correlation significance testing
|
|
fn calculate_correlation_p_value(r: f64, n: usize) -> f64 {
|
|
if n < 3 {
|
|
return 1.0;
|
|
}
|
|
|
|
let t = r * ((n - 2) as f64 / (1.0 - r * r)).sqrt();
|
|
let df = n - 2;
|
|
|
|
// Simplified p-value calculation using t-distribution approximation
|
|
let p_value = 2.0 * (1.0 - Self::student_t_cdf(t.abs(), df));
|
|
p_value.max(0.0).min(1.0)
|
|
}
|
|
|
|
/// Approximate CDF of Student's t-distribution
|
|
fn student_t_cdf(t: f64, df: usize) -> f64 {
|
|
if df == 0 {
|
|
return 0.5;
|
|
}
|
|
|
|
// Simple approximation for t-distribution CDF
|
|
let x = t / (t * t + df as f64).sqrt();
|
|
0.5 + 0.5 * x * (1.0 + x * x / 4.0) / (1.0 + x * x / 2.0)
|
|
}
|
|
|
|
fn calculate_missing_analysis(&self) -> MissingValueAnalysis {
|
|
let mut field_missing_pct = HashMap::new();
|
|
let mut total_missing = 0;
|
|
let mut total_values = 0;
|
|
let mut missing_correlations = HashMap::new();
|
|
|
|
for (field_name, profile) in &self.column_profiles {
|
|
let missing_pct = if profile.count + profile.null_count > 0 {
|
|
profile.null_count as f64 / (profile.count + profile.null_count) as f64 * 100.0
|
|
} else {
|
|
0.0
|
|
};
|
|
|
|
field_missing_pct.insert(field_name.clone(), missing_pct);
|
|
total_missing += profile.null_count;
|
|
total_values += profile.count + profile.null_count;
|
|
}
|
|
|
|
let overall_missing_pct = if total_values > 0 {
|
|
total_missing as f64 / total_values as f64 * 100.0
|
|
} else {
|
|
0.0
|
|
};
|
|
|
|
// Calculate missing patterns - identify common combinations of missing fields
|
|
let missing_patterns = self.identify_missing_patterns();
|
|
|
|
// Calculate correlations between missing values in different fields
|
|
for (field1, pct1) in &field_missing_pct {
|
|
for (field2, pct2) in &field_missing_pct {
|
|
if field1 != field2 {
|
|
// Simplified correlation calculation based on missing percentages
|
|
// In production, this would analyze actual missing patterns in the data
|
|
let correlation = Self::calculate_missing_correlation(*pct1, *pct2);
|
|
missing_correlations.insert(format!("{}-{}", field1, field2), correlation);
|
|
}
|
|
}
|
|
}
|
|
|
|
MissingValueAnalysis {
|
|
overall_missing_pct,
|
|
field_missing_pct,
|
|
missing_patterns,
|
|
missing_correlations,
|
|
}
|
|
}
|
|
|
|
/// Identify common patterns of missing values across fields
|
|
fn identify_missing_patterns(&self) -> Vec<MissingPattern> {
|
|
let mut patterns = Vec::new();
|
|
|
|
// Generate common missing patterns based on field relationships
|
|
let field_names: Vec<String> = self.column_profiles.keys().cloned().collect();
|
|
|
|
// Single field missing patterns
|
|
for field_name in &field_names {
|
|
if let Some(profile) = self.column_profiles.get(field_name) {
|
|
let missing_count = profile.null_count;
|
|
let total_count = profile.count + profile.null_count;
|
|
|
|
if missing_count > 0 && total_count > 0 {
|
|
patterns.push(MissingPattern {
|
|
missing_fields: vec![field_name.clone()],
|
|
count: missing_count,
|
|
percentage: (missing_count as f64 / total_count as f64) * 100.0,
|
|
});
|
|
}
|
|
}
|
|
}
|
|
|
|
// Identify potential multi-field missing patterns
|
|
// This is a simplified version - production code would analyze actual data
|
|
if field_names.len() > 1 {
|
|
let estimated_total_records = self.record_count.max(1);
|
|
|
|
// Common pattern: related fields missing together
|
|
for i in 0..field_names.len() {
|
|
for j in (i + 1)..field_names.len() {
|
|
let field1 = &field_names[i];
|
|
let field2 = &field_names[j];
|
|
|
|
// Estimate joint missing based on individual missing rates
|
|
if let (Some(profile1), Some(profile2)) = (
|
|
self.column_profiles.get(field1),
|
|
self.column_profiles.get(field2),
|
|
) {
|
|
let missing_rate1 = profile1.null_count as f64
|
|
/ (profile1.count + profile1.null_count) as f64;
|
|
let missing_rate2 = profile2.null_count as f64
|
|
/ (profile2.count + profile2.null_count) as f64;
|
|
|
|
// Estimate joint missing (simplified)
|
|
let joint_missing_rate = (missing_rate1 * missing_rate2).min(0.5);
|
|
let joint_count =
|
|
(joint_missing_rate * estimated_total_records as f64) as usize;
|
|
|
|
if joint_count > 0 {
|
|
patterns.push(MissingPattern {
|
|
missing_fields: vec![field1.clone(), field2.clone()],
|
|
count: joint_count,
|
|
percentage: joint_missing_rate * 100.0,
|
|
});
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
// Sort patterns by frequency
|
|
patterns.sort_by(|a, b| b.count.cmp(&a.count));
|
|
patterns.truncate(10); // Keep top 10 patterns
|
|
|
|
patterns
|
|
}
|
|
|
|
/// Calculate correlation between missing values in two fields
|
|
fn calculate_missing_correlation(pct1: f64, pct2: f64) -> f64 {
|
|
// Simplified correlation based on missing percentages
|
|
// In production, this would calculate actual Phi coefficient or Cramér's V
|
|
|
|
let normalized_pct1 = pct1 / 100.0;
|
|
let normalized_pct2 = pct2 / 100.0;
|
|
|
|
// Simple correlation approximation
|
|
let mean = (normalized_pct1 + normalized_pct2) / 2.0;
|
|
let variance = ((normalized_pct1 - mean).powi(2) + (normalized_pct2 - mean).powi(2)) / 2.0;
|
|
|
|
if variance == 0.0 {
|
|
1.0 // Perfect correlation when both have same missing rate
|
|
} else {
|
|
let covariance = (normalized_pct1 - mean) * (normalized_pct2 - mean);
|
|
covariance / variance
|
|
}
|
|
}
|
|
}
|
|
|
|
impl Default for DataProfile {
|
|
fn default() -> Self {
|
|
Self::new()
|
|
}
|
|
}
|
|
|
|
impl StatisticalProfile {
|
|
/// Create a new statistical profile
|
|
pub fn new() -> Self {
|
|
Self {
|
|
field_statistics: HashMap::new(),
|
|
dataset_statistics: DatasetStatistics {
|
|
total_records: 0,
|
|
total_fields: 0,
|
|
completeness: 0.0,
|
|
type_distribution: HashMap::new(),
|
|
memory_usage_bytes: 0,
|
|
},
|
|
}
|
|
}
|
|
|
|
/// Add a numerical value to a field's statistics
|
|
pub fn add_numeric_value(&mut self, field_name: &str, value: f64) {
|
|
let stats = self
|
|
.field_statistics
|
|
.entry(field_name.to_string())
|
|
.or_insert_with(|| FieldStatistics::new(field_name));
|
|
|
|
if let Some(ref mut numerical) = stats.numerical {
|
|
numerical.add_value(value);
|
|
} else {
|
|
let mut numerical = NumericalStatistics::new();
|
|
numerical.add_value(value);
|
|
stats.numerical = Some(numerical);
|
|
}
|
|
}
|
|
|
|
/// Finalize all statistical calculations
|
|
pub fn finalize(&mut self) {
|
|
for stats in self.field_statistics.values_mut() {
|
|
stats.finalize();
|
|
}
|
|
}
|
|
}
|
|
|
|
impl Default for StatisticalProfile {
|
|
fn default() -> Self {
|
|
Self::new()
|
|
}
|
|
}
|
|
|
|
impl FieldStatistics {
|
|
/// Create new field statistics
|
|
pub fn new(field_name: &str) -> Self {
|
|
Self {
|
|
field_name: field_name.to_string(),
|
|
data_type: "unknown".to_string(),
|
|
count: 0,
|
|
null_count: 0,
|
|
null_percentage: 0.0,
|
|
numerical: None,
|
|
string: None,
|
|
temporal: None,
|
|
distribution: None,
|
|
}
|
|
}
|
|
|
|
/// Finalize statistics calculations
|
|
pub fn finalize(&mut self) {
|
|
let total = self.count + self.null_count;
|
|
self.null_percentage = if total > 0 {
|
|
self.null_count as f64 / total as f64 * 100.0
|
|
} else {
|
|
0.0
|
|
};
|
|
|
|
if let Some(ref mut numerical) = self.numerical {
|
|
numerical.finalize();
|
|
}
|
|
}
|
|
}
|
|
|
|
impl NumericalStatistics {
|
|
/// Create new numerical statistics tracker
|
|
pub fn new() -> Self {
|
|
Self {
|
|
mean: 0.0,
|
|
median: 0.0,
|
|
mode: None,
|
|
std_dev: 0.0,
|
|
variance: 0.0,
|
|
min: f64::INFINITY,
|
|
max: f64::NEG_INFINITY,
|
|
range: 0.0,
|
|
skewness: 0.0,
|
|
kurtosis: 0.0,
|
|
q1: 0.0,
|
|
q3: 0.0,
|
|
iqr: 0.0,
|
|
percentiles: HashMap::new(),
|
|
outliers: Vec::new(),
|
|
unique_count: 0,
|
|
value_counts: HashMap::new(),
|
|
}
|
|
}
|
|
|
|
/// Add a value to the statistics
|
|
pub fn add_value(&mut self, value: f64) {
|
|
if value < self.min {
|
|
self.min = value;
|
|
}
|
|
if value > self.max {
|
|
self.max = value;
|
|
}
|
|
|
|
// Track value frequency for mode calculation
|
|
let value_key = format!("{:.6}", value);
|
|
*self.value_counts.entry(value_key).or_insert(0) += 1;
|
|
}
|
|
|
|
/// Finalize calculations (called when all values have been added)
|
|
pub fn finalize(&mut self) {
|
|
self.range = self.max - self.min;
|
|
|
|
// Find mode (most frequent value)
|
|
if let Some((mode_str, _)) = self.value_counts.iter().max_by_key(|(_, count)| *count) {
|
|
if let Ok(mode_val) = mode_str.parse::<f64>() {
|
|
self.mode = Some(mode_val);
|
|
}
|
|
}
|
|
|
|
self.unique_count = self.value_counts.len();
|
|
}
|
|
|
|
/// Calculate comprehensive statistics from a vector of values
|
|
pub fn from_values(values: &[f64]) -> Result<Self> {
|
|
if values.is_empty() {
|
|
return Ok(Self::new());
|
|
}
|
|
|
|
let mut stats = Self::new();
|
|
let mut sorted_values = values.to_vec();
|
|
sorted_values.sort_by(|a, b| a.total_cmp(b));
|
|
|
|
// Basic statistics
|
|
stats.min = *sorted_values.first().unwrap();
|
|
stats.max = *sorted_values.last().unwrap();
|
|
stats.range = stats.max - stats.min;
|
|
stats.mean = StatisticalExtensions::mean(values);
|
|
stats.median = sorted_values.median();
|
|
|
|
// Variance and standard deviation
|
|
stats.variance = StatisticalExtensions::variance(values);
|
|
stats.std_dev = stats.variance.sqrt();
|
|
|
|
// Quartiles
|
|
stats.q1 = sorted_values.quantile(0.25);
|
|
stats.q3 = sorted_values.quantile(0.75);
|
|
stats.iqr = stats.q3 - stats.q1;
|
|
|
|
// Percentiles
|
|
let percentile_points = [1, 5, 10, 25, 50, 75, 90, 95, 99];
|
|
for &p in &percentile_points {
|
|
let quantile = p as f64 / 100.0;
|
|
stats
|
|
.percentiles
|
|
.insert(p, sorted_values.quantile(quantile));
|
|
}
|
|
|
|
// Skewness calculation
|
|
stats.skewness = Self::calculate_skewness(values, stats.mean, stats.std_dev);
|
|
|
|
// Kurtosis calculation
|
|
stats.kurtosis = Self::calculate_kurtosis(values, stats.mean, stats.std_dev);
|
|
|
|
// Outliers using IQR method
|
|
stats.outliers = Self::detect_outliers_iqr(&sorted_values, stats.q1, stats.q3);
|
|
|
|
// Value counts for mode
|
|
let mut value_counts = HashMap::new();
|
|
for &value in values {
|
|
let value_key = format!("{:.6}", value);
|
|
*value_counts.entry(value_key).or_insert(0) += 1;
|
|
}
|
|
stats.value_counts = value_counts;
|
|
|
|
// Find mode
|
|
if let Some((mode_str, _)) = stats.value_counts.iter().max_by_key(|(_, count)| *count) {
|
|
if let Ok(mode_val) = mode_str.parse::<f64>() {
|
|
stats.mode = Some(mode_val);
|
|
}
|
|
}
|
|
|
|
stats.unique_count = stats.value_counts.len();
|
|
|
|
Ok(stats)
|
|
}
|
|
|
|
fn calculate_skewness(values: &[f64], mean: f64, std_dev: f64) -> f64 {
|
|
if std_dev == 0.0 || values.len() < 3 {
|
|
return 0.0;
|
|
}
|
|
|
|
let n = values.len() as f64;
|
|
let sum_cubed = values
|
|
.iter()
|
|
.map(|&x| ((x - mean) / std_dev).powi(3))
|
|
.sum::<f64>();
|
|
|
|
(n / ((n - 1.0) * (n - 2.0))) * sum_cubed
|
|
}
|
|
|
|
fn calculate_kurtosis(values: &[f64], mean: f64, std_dev: f64) -> f64 {
|
|
if std_dev == 0.0 || values.len() < 4 {
|
|
return 0.0;
|
|
}
|
|
|
|
let n = values.len() as f64;
|
|
let sum_fourth = values
|
|
.iter()
|
|
.map(|&x| ((x - mean) / std_dev).powi(4))
|
|
.sum::<f64>();
|
|
|
|
let kurtosis = (n * (n + 1.0) / ((n - 1.0) * (n - 2.0) * (n - 3.0))) * sum_fourth;
|
|
kurtosis - 3.0 * (n - 1.0).powi(2) / ((n - 2.0) * (n - 3.0)) // Excess kurtosis
|
|
}
|
|
|
|
fn detect_outliers_iqr(sorted_values: &[f64], q1: f64, q3: f64) -> Vec<f64> {
|
|
let iqr = q3 - q1;
|
|
let lower_bound = q1 - 1.5 * iqr;
|
|
let upper_bound = q3 + 1.5 * iqr;
|
|
|
|
sorted_values
|
|
.iter()
|
|
.filter(|&&x| x < lower_bound || x > upper_bound)
|
|
.copied()
|
|
.collect()
|
|
}
|
|
}
|
|
|
|
impl Default for NumericalStatistics {
|
|
fn default() -> Self {
|
|
Self::new()
|
|
}
|
|
}
|
|
|
|
impl ColumnProfile {
|
|
/// Create a new column profile
|
|
pub fn new(name: String) -> Self {
|
|
Self {
|
|
name: name.clone(),
|
|
data_type: "unknown".to_string(),
|
|
count: 0,
|
|
null_count: 0,
|
|
unique_count: 0,
|
|
quality_score: 0.0,
|
|
sample_values: Vec::new(),
|
|
statistics: FieldStatistics::new(&name),
|
|
}
|
|
}
|
|
|
|
/// Add a value to this column profile
|
|
pub fn add_value(&mut self, value: &DataValue) -> Result<()> {
|
|
if value.is_null() {
|
|
self.null_count += 1;
|
|
} else {
|
|
self.count += 1;
|
|
self.data_type = value.type_name().to_string();
|
|
|
|
// Add to sample values (keep only first 10)
|
|
if self.sample_values.len() < 10 {
|
|
if let Some(str_val) = value.as_string() {
|
|
self.sample_values.push(str_val);
|
|
}
|
|
}
|
|
}
|
|
|
|
Ok(())
|
|
}
|
|
|
|
/// Finalize the column profile calculations
|
|
pub fn finalize(&mut self) -> Result<()> {
|
|
// Calculate quality score
|
|
let total = self.count + self.null_count;
|
|
let completeness = if total > 0 {
|
|
self.count as f64 / total as f64
|
|
} else {
|
|
1.0
|
|
};
|
|
|
|
// Simple quality score based on completeness
|
|
self.quality_score = completeness;
|
|
|
|
// Update unique count (simplified - in real implementation would track unique values)
|
|
self.unique_count = self.sample_values.len().min(self.count);
|
|
|
|
self.statistics.finalize();
|
|
Ok(())
|
|
}
|
|
}
|
|
|
|
impl MissingValueAnalysis {
|
|
/// Create a new missing value analysis
|
|
pub fn new() -> Self {
|
|
Self {
|
|
overall_missing_pct: 0.0,
|
|
field_missing_pct: HashMap::new(),
|
|
missing_patterns: Vec::new(),
|
|
missing_correlations: HashMap::new(),
|
|
}
|
|
}
|
|
}
|
|
|
|
impl Default for MissingValueAnalysis {
|
|
fn default() -> Self {
|
|
Self::new()
|
|
}
|
|
}
|
|
|
|
impl CorrelationStrength {
|
|
/// Determine correlation strength from coefficient
|
|
pub fn from_coefficient(r: f64) -> Self {
|
|
let abs_r = r.abs();
|
|
match abs_r {
|
|
x if x >= 0.9 => CorrelationStrength::VeryStrong,
|
|
x if x >= 0.7 => CorrelationStrength::Strong,
|
|
x if x >= 0.5 => CorrelationStrength::Moderate,
|
|
x if x >= 0.3 => CorrelationStrength::Weak,
|
|
_ => CorrelationStrength::VeryWeak,
|
|
}
|
|
}
|
|
}
|
|
|
|
#[cfg(test)]
|
|
mod tests {
|
|
use super::*;
|
|
|
|
#[test]
|
|
fn test_numerical_statistics_creation() {
|
|
let values = vec![1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0, 9.0, 10.0];
|
|
let stats = NumericalStatistics::from_values(&values).unwrap();
|
|
|
|
assert_eq!(stats.min, 1.0);
|
|
assert_eq!(stats.max, 10.0);
|
|
assert_eq!(stats.mean, 5.5);
|
|
assert_eq!(stats.median, 5.5);
|
|
assert_eq!(stats.range, 9.0);
|
|
}
|
|
|
|
#[test]
|
|
fn test_data_profile_creation() {
|
|
let mut profile = DataProfile::new();
|
|
assert_eq!(profile.record_count, 0);
|
|
assert_eq!(profile.field_count, 0);
|
|
|
|
let mut fields = HashMap::new();
|
|
fields.insert("age".to_string(), DataValue::Int(25));
|
|
fields.insert("name".to_string(), DataValue::String("John".to_string()));
|
|
|
|
profile.add_record(&fields).unwrap();
|
|
assert_eq!(profile.record_count, 1);
|
|
assert_eq!(profile.field_count, 2);
|
|
}
|
|
|
|
#[test]
|
|
fn test_missing_value_analysis() {
|
|
let mut profile = DataProfile::new();
|
|
|
|
let mut fields1 = HashMap::new();
|
|
fields1.insert("age".to_string(), DataValue::Int(25));
|
|
fields1.insert("name".to_string(), DataValue::Null);
|
|
|
|
let mut fields2 = HashMap::new();
|
|
fields2.insert("age".to_string(), DataValue::Null);
|
|
fields2.insert("name".to_string(), DataValue::String("Jane".to_string()));
|
|
|
|
profile.add_record(&fields1).unwrap();
|
|
profile.add_record(&fields2).unwrap();
|
|
profile.finalize().unwrap();
|
|
|
|
assert!(profile.missing_analysis.overall_missing_pct > 0.0);
|
|
}
|
|
|
|
#[test]
|
|
fn test_column_profile() {
|
|
let mut profile = ColumnProfile::new("test".to_string());
|
|
profile.add_value(&DataValue::Int(42)).unwrap();
|
|
profile.add_value(&DataValue::Null).unwrap();
|
|
profile.finalize().unwrap();
|
|
|
|
assert_eq!(profile.count, 1);
|
|
assert_eq!(profile.null_count, 1);
|
|
assert_eq!(profile.quality_score, 0.5);
|
|
}
|
|
|
|
#[test]
|
|
fn test_correlation_strength() {
|
|
assert_eq!(
|
|
std::mem::discriminant(&CorrelationStrength::from_coefficient(0.95)),
|
|
std::mem::discriminant(&CorrelationStrength::VeryStrong)
|
|
);
|
|
assert_eq!(
|
|
std::mem::discriminant(&CorrelationStrength::from_coefficient(0.8)),
|
|
std::mem::discriminant(&CorrelationStrength::Strong)
|
|
);
|
|
assert_eq!(
|
|
std::mem::discriminant(&CorrelationStrength::from_coefficient(0.6)),
|
|
std::mem::discriminant(&CorrelationStrength::Moderate)
|
|
);
|
|
assert_eq!(
|
|
std::mem::discriminant(&CorrelationStrength::from_coefficient(0.4)),
|
|
std::mem::discriminant(&CorrelationStrength::Weak)
|
|
);
|
|
assert_eq!(
|
|
std::mem::discriminant(&CorrelationStrength::from_coefficient(0.2)),
|
|
std::mem::discriminant(&CorrelationStrength::VeryWeak)
|
|
);
|
|
}
|
|
}
|