//! Data quality scoring system with comprehensive metrics //! //! This module provides comprehensive data quality scoring including completeness, //! uniqueness, validity, consistency, and timeliness metrics with configurable //! weights and scoring algorithms. use std::collections::{HashMap, HashSet}; use chrono::{DateTime, Utc}; use serde::{Deserialize, Serialize}; use crate::{DataRecord, Result}; /// Comprehensive data quality score with multiple dimensions #[derive(Debug, Clone, Serialize, Deserialize)] pub struct QualityScore { /// Overall composite quality score (0.0 - 1.0) pub overall: f64, /// Individual quality dimension scores pub dimensions: QualityDimensions, /// Weights used for calculating overall score pub weights: QualityWeights, /// Timestamp when score was calculated pub timestamp: DateTime, /// Additional metadata about the scoring process pub metadata: QualityMetadata, } /// Individual quality dimension scores #[derive(Debug, Clone, Serialize, Deserialize)] pub struct QualityDimensions { /// Completeness score (data availability) pub completeness: CompletenessScore, /// Uniqueness score (duplicate detection) pub uniqueness: UniquenessScore, /// Validity score (format and constraint compliance) pub validity: ValidityScore, /// Consistency score (cross-field validation) pub consistency: ConsistencyScore, /// Timeliness score (data freshness) pub timeliness: TimelinessScore, /// Accuracy score (correctness assessment) pub accuracy: AccuracyScore, /// Integrity score (referential integrity) pub integrity: IntegrityScore, } /// Completeness scoring - measures data availability and missing values #[derive(Debug, Clone, Serialize, Deserialize)] pub struct CompletenessScore { /// Overall completeness score (0.0 - 1.0) pub score: f64, /// Per-field completeness scores pub field_scores: HashMap, /// Total number of expected values pub total_expected: usize, /// Total number of non-null values pub total_present: usize, /// Missing value patterns pub missing_patterns: Vec, /// Critical fields that are missing pub critical_missing: Vec, } /// Uniqueness scoring - measures duplicate detection and data uniqueness #[derive(Debug, Clone, Serialize, Deserialize)] pub struct UniquenessScore { /// Overall uniqueness score (0.0 - 1.0) pub score: f64, /// Per-field uniqueness scores pub field_scores: HashMap, /// Duplicate detection results pub duplicates: DuplicateAnalysis, /// Unique identifier analysis pub identifier_analysis: IdentifierAnalysis, } /// Validity scoring - measures format and constraint compliance #[derive(Debug, Clone, Serialize, Deserialize)] pub struct ValidityScore { /// Overall validity score (0.0 - 1.0) pub score: f64, /// Per-field validity scores pub field_scores: HashMap, /// Format validation results pub format_validation: FormatValidation, /// Constraint validation results pub constraint_validation: ConstraintValidation, /// Data type consistency pub type_consistency: TypeConsistency, } /// Consistency scoring - measures cross-field validation and logical consistency #[derive(Debug, Clone, Serialize, Deserialize)] pub struct ConsistencyScore { /// Overall consistency score (0.0 - 1.0) pub score: f64, /// Cross-field validation results pub cross_field_results: HashMap, /// Business rule violations pub business_rule_violations: Vec, /// Logical consistency checks pub logical_consistency: LogicalConsistency, } /// Timeliness scoring - measures data freshness and temporal validity #[derive(Debug, Clone, Serialize, Deserialize)] pub struct TimelinessScore { /// Overall timeliness score (0.0 - 1.0) pub score: f64, /// Age of data in hours pub data_age_hours: f64, /// Expected maximum age for full score pub expected_max_age_hours: f64, /// Temporal patterns analysis pub temporal_patterns: TemporalPatterns, /// Freshness per data source pub source_freshness: HashMap, } /// Accuracy scoring - measures correctness of data values #[derive(Debug, Clone, Serialize, Deserialize)] pub struct AccuracyScore { /// Overall accuracy score (0.0 - 1.0) pub score: f64, /// Reference data comparison results pub reference_comparison: ReferenceComparison, /// Statistical outlier analysis pub outlier_analysis: OutlierAnalysis, /// Domain-specific accuracy checks pub domain_checks: HashMap, } /// Integrity scoring - measures referential and structural integrity #[derive(Debug, Clone, Serialize, Deserialize)] pub struct IntegrityScore { /// Overall integrity score (0.0 - 1.0) pub score: f64, /// Referential integrity results pub referential_integrity: ReferentialIntegrity, /// Entity integrity results pub entity_integrity: EntityIntegrity, /// Domain integrity results pub domain_integrity: DomainIntegrity, } /// Weights for different quality dimensions #[derive(Debug, Clone, Serialize, Deserialize)] pub struct QualityWeights { /// Completeness weight (default: 0.25) pub completeness: f64, /// Uniqueness weight (default: 0.15) pub uniqueness: f64, /// Validity weight (default: 0.25) pub validity: f64, /// Consistency weight (default: 0.15) pub consistency: f64, /// Timeliness weight (default: 0.10) pub timeliness: f64, /// Accuracy weight (default: 0.05) pub accuracy: f64, /// Integrity weight (default: 0.05) pub integrity: f64, } /// Metadata about the quality scoring process #[derive(Debug, Clone, Serialize, Deserialize)] pub struct QualityMetadata { /// Number of records analyzed pub records_analyzed: usize, /// Number of fields analyzed pub fields_analyzed: usize, /// Scoring algorithm version pub algorithm_version: String, /// Configuration used for scoring pub configuration: ScoringConfiguration, /// Performance metrics pub performance: ScoringPerformance, } /// Configuration for quality scoring #[derive(Debug, Clone, Serialize, Deserialize)] pub struct ScoringConfiguration { /// Enable strict mode (more stringent scoring) pub strict_mode: bool, /// Custom business rules pub business_rules: Vec, /// Reference data sources pub reference_sources: Vec, /// Temporal validity windows pub temporal_windows: HashMap, } /// Performance metrics for the scoring process #[derive(Debug, Clone, Serialize, Deserialize)] pub struct ScoringPerformance { /// Time taken for scoring in milliseconds pub duration_ms: u64, /// Memory used during scoring in bytes pub memory_used_bytes: usize, /// Number of validation rules applied pub rules_applied: usize, } /// Missing value pattern analysis #[derive(Debug, Clone, Serialize, Deserialize)] pub struct MissingPattern { /// Fields involved in the pattern pub fields: Vec, /// Number of occurrences pub count: usize, /// Pattern description pub description: String, } /// Duplicate analysis results #[derive(Debug, Clone, Serialize, Deserialize)] pub struct DuplicateAnalysis { /// Total number of duplicates found pub total_duplicates: usize, /// Duplicate groups (records that are duplicates of each other) pub duplicate_groups: Vec, /// Fuzzy duplicate analysis pub fuzzy_duplicates: FuzzyDuplicateAnalysis, } /// Group of duplicate records #[derive(Debug, Clone, Serialize, Deserialize)] pub struct DuplicateGroup { /// Representative record ID pub representative_id: String, /// IDs of duplicate records pub duplicate_ids: Vec, /// Similarity score between records pub similarity_score: f64, /// Fields that are identical pub identical_fields: Vec, } /// Fuzzy duplicate analysis #[derive(Debug, Clone, Serialize, Deserialize)] pub struct FuzzyDuplicateAnalysis { /// Potential fuzzy duplicates pub potential_duplicates: Vec, /// Similarity threshold used pub similarity_threshold: f64, /// Matching algorithm used pub matching_algorithm: String, } /// Group of potentially similar records #[derive(Debug, Clone, Serialize, Deserialize)] pub struct FuzzyDuplicateGroup { /// Record IDs in the group pub record_ids: Vec, /// Maximum similarity score in the group pub max_similarity: f64, /// Fields used for comparison pub comparison_fields: Vec, } /// Unique identifier analysis #[derive(Debug, Clone, Serialize, Deserialize)] pub struct IdentifierAnalysis { /// Potential unique identifiers found pub potential_identifiers: Vec, /// Composite key candidates pub composite_keys: Vec>, /// Identifier quality scores pub identifier_scores: HashMap, } /// Format validation results #[derive(Debug, Clone, Serialize, Deserialize)] pub struct FormatValidation { /// Per-field format compliance pub field_compliance: HashMap, /// Format violations found pub violations: Vec, /// Standard format adherence pub standard_adherence: HashMap, } /// Format validation violation #[derive(Debug, Clone, Serialize, Deserialize)] pub struct FormatViolation { /// Field name where violation occurred pub field_name: String, /// Expected format pub expected_format: String, /// Actual value pub actual_value: String, /// Violation description pub description: String, } /// Constraint validation results #[derive(Debug, Clone, Serialize, Deserialize)] pub struct ConstraintValidation { /// Range constraint violations pub range_violations: Vec, /// Length constraint violations pub length_violations: Vec, /// Custom constraint violations pub custom_violations: Vec, } /// Constraint violation details #[derive(Debug, Clone, Serialize, Deserialize)] pub struct ConstraintViolation { /// Field name pub field_name: String, /// Constraint type pub constraint_type: String, /// Constraint description pub constraint: String, /// Actual value pub actual_value: String, /// Severity level pub severity: String, } /// Type consistency analysis #[derive(Debug, Clone, Serialize, Deserialize)] pub struct TypeConsistency { /// Per-field type consistency scores pub field_consistency: HashMap, /// Type conflicts found pub type_conflicts: Vec, /// Inferred types vs. declared types pub type_alignment: HashMap, } /// Type conflict information #[derive(Debug, Clone, Serialize, Deserialize)] pub struct TypeConflict { /// Field name pub field_name: String, /// Expected type pub expected_type: String, /// Actual types found pub actual_types: Vec, /// Conflict frequency pub frequency: usize, } /// Type alignment between inferred and declared #[derive(Debug, Clone, Serialize, Deserialize)] pub struct TypeAlignment { /// Declared type pub declared_type: String, /// Inferred type pub inferred_type: String, /// Alignment score (0.0 - 1.0) pub alignment_score: f64, } /// Business rule violation #[derive(Debug, Clone, Serialize, Deserialize)] pub struct BusinessRuleViolation { /// Rule identifier pub rule_id: String, /// Rule description pub rule_description: String, /// Fields involved pub fields_involved: Vec, /// Violation description pub violation_description: String, /// Severity level pub severity: String, } /// Logical consistency checks #[derive(Debug, Clone, Serialize, Deserialize)] pub struct LogicalConsistency { /// Mathematical consistency checks pub mathematical_consistency: f64, /// Temporal consistency checks pub temporal_consistency: f64, /// Hierarchical consistency checks pub hierarchical_consistency: f64, } /// Temporal patterns in data #[derive(Debug, Clone, Serialize, Deserialize)] pub struct TemporalPatterns { /// Data arrival patterns pub arrival_patterns: HashMap, /// Update frequency analysis pub update_frequency: UpdateFrequency, /// Seasonal patterns pub seasonal_patterns: HashMap, } /// Update frequency analysis #[derive(Debug, Clone, Serialize, Deserialize)] pub struct UpdateFrequency { /// Average time between updates in hours pub avg_update_interval_hours: f64, /// Update frequency distribution pub frequency_distribution: HashMap, /// Expected vs. actual update frequency pub frequency_deviation: f64, } /// Reference data comparison results #[derive(Debug, Clone, Serialize, Deserialize)] pub struct ReferenceComparison { /// Comparison results per reference source pub source_comparisons: HashMap, /// Fields validated against reference data pub validated_fields: Vec, /// Reference data match rate pub match_rate: f64, } /// Statistical outlier analysis #[derive(Debug, Clone, Serialize, Deserialize)] pub struct OutlierAnalysis { /// Outliers detected per field pub field_outliers: HashMap>, /// Outlier detection methods used pub detection_methods: Vec, /// Overall outlier rate pub outlier_rate: f64, } /// Information about a detected outlier #[derive(Debug, Clone, Serialize, Deserialize)] pub struct OutlierInfo { /// Value that is an outlier pub value: String, /// Outlier score (higher = more extreme) pub score: f64, /// Detection method used pub method: String, /// Context information pub context: HashMap, } /// Referential integrity analysis #[derive(Debug, Clone, Serialize, Deserialize)] pub struct ReferentialIntegrity { /// Foreign key violations pub foreign_key_violations: Vec, /// Orphaned records pub orphaned_records: Vec, /// Overall integrity score pub integrity_score: f64, } /// Referential integrity violation #[derive(Debug, Clone, Serialize, Deserialize)] pub struct ReferentialViolation { /// Child table/field pub child_field: String, /// Parent table/field pub parent_field: String, /// Violating values pub violating_values: Vec, /// Violation count pub violation_count: usize, } /// Entity integrity analysis #[derive(Debug, Clone, Serialize, Deserialize)] pub struct EntityIntegrity { /// Primary key violations pub primary_key_violations: Vec, /// Null primary key instances pub null_primary_keys: usize, /// Duplicate primary key instances pub duplicate_primary_keys: usize, } /// Domain integrity analysis #[derive(Debug, Clone, Serialize, Deserialize)] pub struct DomainIntegrity { /// Domain constraint violations per field pub constraint_violations: HashMap, /// Check constraint violations pub check_constraint_violations: Vec, /// Overall domain integrity score pub integrity_score: f64, } /// Check constraint violation #[derive(Debug, Clone, Serialize, Deserialize)] pub struct CheckConstraintViolation { /// Constraint name pub constraint_name: String, /// Field name pub field_name: String, /// Violating value pub violating_value: String, /// Constraint description pub constraint_description: String, } impl QualityScore { /// Create a new quality score with default configuration pub fn new() -> Self { Self { overall: 0.0, dimensions: QualityDimensions::default(), weights: QualityWeights::default(), timestamp: Utc::now(), metadata: QualityMetadata::default(), } } /// Calculate overall quality score from dimension scores pub fn calculate_overall(&mut self) { let weights = &self.weights; let dims = &self.dimensions; self.overall = weights.completeness * dims.completeness.score + weights.uniqueness * dims.uniqueness.score + weights.validity * dims.validity.score + weights.consistency * dims.consistency.score + weights.timeliness * dims.timeliness.score + weights.accuracy * dims.accuracy.score + weights.integrity * dims.integrity.score; // Ensure score is between 0 and 1 self.overall = self.overall.max(0.0).min(1.0); } /// Get the overall quality score pub fn overall_score(&self) -> f64 { self.overall } /// Set completeness score pub fn set_completeness(&mut self, score: f64) { self.dimensions.completeness.score = score.max(0.0).min(1.0); self.calculate_overall(); } /// Set uniqueness score pub fn set_uniqueness(&mut self, score: f64) { self.dimensions.uniqueness.score = score.max(0.0).min(1.0); self.calculate_overall(); } /// Set validity score pub fn set_validity(&mut self, score: f64) { self.dimensions.validity.score = score.max(0.0).min(1.0); self.calculate_overall(); } /// Set consistency score pub fn set_consistency(&mut self, score: f64) { self.dimensions.consistency.score = score.max(0.0).min(1.0); self.calculate_overall(); } /// Set timeliness score pub fn set_timeliness(&mut self, score: f64) { self.dimensions.timeliness.score = score.max(0.0).min(1.0); self.calculate_overall(); } /// Get quality grade as string pub fn quality_grade(&self) -> &'static str { match self.overall { score if score >= 0.9 => "Excellent", score if score >= 0.8 => "Good", score if score >= 0.7 => "Acceptable", score if score >= 0.6 => "Poor", _ => "Very Poor", } } /// Calculate quality score for a set of records pub fn calculate_for_records(records: &[DataRecord]) -> Result { let mut quality_score = Self::new(); let start_time = std::time::Instant::now(); if records.is_empty() { return Ok(quality_score); } // Calculate completeness score let completeness = Self::calculate_completeness(records)?; quality_score.dimensions.completeness = completeness; // Calculate uniqueness score let uniqueness = Self::calculate_uniqueness(records)?; quality_score.dimensions.uniqueness = uniqueness; // Calculate validity score (simplified) let validity = Self::calculate_validity(records)?; quality_score.dimensions.validity = validity; // Calculate consistency score (simplified) let consistency = Self::calculate_consistency(records)?; quality_score.dimensions.consistency = consistency; // Calculate timeliness score let timeliness = Self::calculate_timeliness(records)?; quality_score.dimensions.timeliness = timeliness; // Set default scores for accuracy and integrity quality_score.dimensions.accuracy.score = 0.9; // Placeholder quality_score.dimensions.integrity.score = 0.95; // Placeholder // Calculate overall score quality_score.calculate_overall(); // Update metadata quality_score.metadata.records_analyzed = records.len(); quality_score.metadata.performance.duration_ms = start_time.elapsed().as_millis() as u64; Ok(quality_score) } fn calculate_completeness(records: &[DataRecord]) -> Result { let mut field_scores = HashMap::new(); let mut total_expected = 0; let mut total_present = 0; let mut all_fields = HashSet::new(); // Collect all field names for record in records { for field_name in record.fields.keys() { all_fields.insert(field_name.clone()); } } // Calculate completeness for each field for field_name in &all_fields { let mut present_count = 0; let record_count = records.len(); for record in records { if let Some(value) = record.fields.get(field_name) { if !value.is_null() { present_count += 1; } } } let field_completeness = if record_count > 0 { present_count as f64 / record_count as f64 } else { 1.0 }; field_scores.insert(field_name.clone(), field_completeness); total_expected += record_count; total_present += present_count; } let overall_completeness = if total_expected > 0 { total_present as f64 / total_expected as f64 } else { 1.0 }; Ok(CompletenessScore { score: overall_completeness, field_scores, total_expected, total_present, missing_patterns: vec![], // Would be calculated with pattern analysis critical_missing: vec![], // Would be determined based on business rules }) } fn calculate_uniqueness(records: &[DataRecord]) -> Result { let mut field_scores = HashMap::new(); let mut all_fields = HashSet::new(); // Collect all field names for record in records { for field_name in record.fields.keys() { all_fields.insert(field_name.clone()); } } // Calculate uniqueness for each field for field_name in &all_fields { let mut values = HashMap::new(); let mut total_values = 0; for record in records { if let Some(value) = record.fields.get(field_name) { if !value.is_null() { let value_str = format!("{:?}", value); *values.entry(value_str).or_insert(0) += 1; total_values += 1; } } } let unique_count = values.len(); let field_uniqueness = if total_values > 0 { unique_count as f64 / total_values as f64 } else { 1.0 }; field_scores.insert(field_name.clone(), field_uniqueness); } let overall_uniqueness = if !field_scores.is_empty() { field_scores.values().sum::() / field_scores.len() as f64 } else { 1.0 }; Ok(UniquenessScore { score: overall_uniqueness, field_scores, duplicates: DuplicateAnalysis { total_duplicates: 0, duplicate_groups: vec![], fuzzy_duplicates: FuzzyDuplicateAnalysis { potential_duplicates: vec![], similarity_threshold: 0.8, matching_algorithm: "Levenshtein".to_string(), }, }, identifier_analysis: IdentifierAnalysis { potential_identifiers: vec![], composite_keys: vec![], identifier_scores: HashMap::new(), }, }) } fn calculate_validity(_records: &[DataRecord]) -> Result { // Simplified validity calculation // In a full implementation, this would validate formats, constraints, etc. Ok(ValidityScore { score: 0.85, // Placeholder field_scores: HashMap::new(), format_validation: FormatValidation { field_compliance: HashMap::new(), violations: vec![], standard_adherence: HashMap::new(), }, constraint_validation: ConstraintValidation { range_violations: vec![], length_violations: vec![], custom_violations: vec![], }, type_consistency: TypeConsistency { field_consistency: HashMap::new(), type_conflicts: vec![], type_alignment: HashMap::new(), }, }) } fn calculate_consistency(_records: &[DataRecord]) -> Result { // Simplified consistency calculation Ok(ConsistencyScore { score: 0.9, // Placeholder cross_field_results: HashMap::new(), business_rule_violations: vec![], logical_consistency: LogicalConsistency { mathematical_consistency: 0.95, temporal_consistency: 0.9, hierarchical_consistency: 0.85, }, }) } fn calculate_timeliness(records: &[DataRecord]) -> Result { let now = Utc::now(); let mut total_age_hours = 0.0; let mut count = 0; for record in records { let age_duration = now.signed_duration_since(record.timestamp); let age_hours = age_duration.num_hours() as f64; total_age_hours += age_hours; count += 1; } let avg_age_hours = if count > 0 { total_age_hours / count as f64 } else { 0.0 }; // Score decreases with age, full score for data less than 24 hours old let expected_max_age_hours = 24.0; let timeliness_score = if avg_age_hours <= expected_max_age_hours { 1.0 } else { (expected_max_age_hours / avg_age_hours).min(1.0).max(0.0) }; Ok(TimelinessScore { score: timeliness_score, data_age_hours: avg_age_hours, expected_max_age_hours, temporal_patterns: TemporalPatterns { arrival_patterns: HashMap::new(), update_frequency: UpdateFrequency { avg_update_interval_hours: avg_age_hours, frequency_distribution: HashMap::new(), frequency_deviation: 0.0, }, seasonal_patterns: HashMap::new(), }, source_freshness: HashMap::new(), }) } } impl Default for QualityScore { fn default() -> Self { Self::new() } } impl Default for QualityDimensions { fn default() -> Self { Self { completeness: CompletenessScore::default(), uniqueness: UniquenessScore::default(), validity: ValidityScore::default(), consistency: ConsistencyScore::default(), timeliness: TimelinessScore::default(), accuracy: AccuracyScore::default(), integrity: IntegrityScore::default(), } } } impl Default for QualityWeights { fn default() -> Self { Self { completeness: 0.25, uniqueness: 0.15, validity: 0.25, consistency: 0.15, timeliness: 0.10, accuracy: 0.05, integrity: 0.05, } } } impl Default for QualityMetadata { fn default() -> Self { Self { records_analyzed: 0, fields_analyzed: 0, algorithm_version: "1.0.0".to_string(), configuration: ScoringConfiguration::default(), performance: ScoringPerformance::default(), } } } impl Default for ScoringConfiguration { fn default() -> Self { Self { strict_mode: false, business_rules: vec![], reference_sources: vec![], temporal_windows: HashMap::new(), } } } impl Default for ScoringPerformance { fn default() -> Self { Self { duration_ms: 0, memory_used_bytes: 0, rules_applied: 0, } } } // Default implementations for all score types impl Default for CompletenessScore { fn default() -> Self { Self { score: 0.0, field_scores: HashMap::new(), total_expected: 0, total_present: 0, missing_patterns: vec![], critical_missing: vec![], } } } impl Default for UniquenessScore { fn default() -> Self { Self { score: 0.0, field_scores: HashMap::new(), duplicates: DuplicateAnalysis { total_duplicates: 0, duplicate_groups: vec![], fuzzy_duplicates: FuzzyDuplicateAnalysis { potential_duplicates: vec![], similarity_threshold: 0.8, matching_algorithm: "Levenshtein".to_string(), }, }, identifier_analysis: IdentifierAnalysis { potential_identifiers: vec![], composite_keys: vec![], identifier_scores: HashMap::new(), }, } } } impl Default for ValidityScore { fn default() -> Self { Self { score: 0.0, field_scores: HashMap::new(), format_validation: FormatValidation { field_compliance: HashMap::new(), violations: vec![], standard_adherence: HashMap::new(), }, constraint_validation: ConstraintValidation { range_violations: vec![], length_violations: vec![], custom_violations: vec![], }, type_consistency: TypeConsistency { field_consistency: HashMap::new(), type_conflicts: vec![], type_alignment: HashMap::new(), }, } } } impl Default for ConsistencyScore { fn default() -> Self { Self { score: 0.0, cross_field_results: HashMap::new(), business_rule_violations: vec![], logical_consistency: LogicalConsistency { mathematical_consistency: 0.0, temporal_consistency: 0.0, hierarchical_consistency: 0.0, }, } } } impl Default for TimelinessScore { fn default() -> Self { Self { score: 0.0, data_age_hours: 0.0, expected_max_age_hours: 24.0, temporal_patterns: TemporalPatterns { arrival_patterns: HashMap::new(), update_frequency: UpdateFrequency { avg_update_interval_hours: 0.0, frequency_distribution: HashMap::new(), frequency_deviation: 0.0, }, seasonal_patterns: HashMap::new(), }, source_freshness: HashMap::new(), } } } impl Default for AccuracyScore { fn default() -> Self { Self { score: 0.0, reference_comparison: ReferenceComparison { source_comparisons: HashMap::new(), validated_fields: vec![], match_rate: 0.0, }, outlier_analysis: OutlierAnalysis { field_outliers: HashMap::new(), detection_methods: vec![], outlier_rate: 0.0, }, domain_checks: HashMap::new(), } } } impl Default for IntegrityScore { fn default() -> Self { Self { score: 0.0, referential_integrity: ReferentialIntegrity { foreign_key_violations: vec![], orphaned_records: vec![], integrity_score: 0.0, }, entity_integrity: EntityIntegrity { primary_key_violations: vec![], null_primary_keys: 0, duplicate_primary_keys: 0, }, domain_integrity: DomainIntegrity { constraint_violations: HashMap::new(), check_constraint_violations: vec![], integrity_score: 0.0, }, } } } #[cfg(test)] mod tests { use super::*; use crate::DataValue; use chrono::Duration; use std::collections::HashMap; #[test] fn test_quality_score_creation() { let quality_score = QualityScore::new(); assert_eq!(quality_score.overall_score(), 0.0); assert_eq!(quality_score.quality_grade(), "Very Poor"); } #[test] fn test_quality_score_calculation() { let mut quality_score = QualityScore::new(); quality_score.set_completeness(0.9); quality_score.set_uniqueness(0.8); quality_score.set_validity(0.95); quality_score.set_consistency(0.85); quality_score.set_timeliness(0.7); assert!(quality_score.overall_score() > 0.0); assert!(quality_score.overall_score() <= 1.0); } #[test] fn test_quality_grade_calculation() { let mut quality_score = QualityScore::new(); quality_score.overall = 0.95; assert_eq!(quality_score.quality_grade(), "Excellent"); quality_score.overall = 0.85; assert_eq!(quality_score.quality_grade(), "Good"); quality_score.overall = 0.75; assert_eq!(quality_score.quality_grade(), "Acceptable"); quality_score.overall = 0.65; assert_eq!(quality_score.quality_grade(), "Poor"); quality_score.overall = 0.5; assert_eq!(quality_score.quality_grade(), "Very Poor"); } #[test] fn test_completeness_calculation() { let mut records = vec![]; // Record 1: all fields present let mut fields1 = HashMap::new(); fields1.insert("name".to_string(), DataValue::String("John".to_string())); fields1.insert("age".to_string(), DataValue::Int(30)); fields1.insert( "email".to_string(), DataValue::String("john@example.com".to_string()), ); records.push(DataRecord { id: "1".to_string(), timestamp: Utc::now(), fields: fields1, metadata: HashMap::new(), }); // Record 2: one field missing let mut fields2 = HashMap::new(); fields2.insert("name".to_string(), DataValue::String("Jane".to_string())); fields2.insert("age".to_string(), DataValue::Int(25)); fields2.insert("email".to_string(), DataValue::Null); records.push(DataRecord { id: "2".to_string(), timestamp: Utc::now(), fields: fields2, metadata: HashMap::new(), }); let quality_score = QualityScore::calculate_for_records(&records).unwrap(); // Should have some completeness score less than 1.0 due to missing email assert!(quality_score.dimensions.completeness.score < 1.0); assert!(quality_score.dimensions.completeness.score > 0.0); } #[test] fn test_uniqueness_calculation() { let mut records = vec![]; // Record 1 let mut fields1 = HashMap::new(); fields1.insert("id".to_string(), DataValue::Int(1)); fields1.insert("name".to_string(), DataValue::String("John".to_string())); records.push(DataRecord { id: "1".to_string(), timestamp: Utc::now(), fields: fields1, metadata: HashMap::new(), }); // Record 2 (unique) let mut fields2 = HashMap::new(); fields2.insert("id".to_string(), DataValue::Int(2)); fields2.insert("name".to_string(), DataValue::String("Jane".to_string())); records.push(DataRecord { id: "2".to_string(), timestamp: Utc::now(), fields: fields2, metadata: HashMap::new(), }); let quality_score = QualityScore::calculate_for_records(&records).unwrap(); // Should have perfect uniqueness since all values are unique assert!(quality_score.dimensions.uniqueness.score > 0.0); } #[test] fn test_timeliness_calculation() { let mut records = vec![]; // Recent record let mut fields1 = HashMap::new(); fields1.insert("data".to_string(), DataValue::String("recent".to_string())); records.push(DataRecord { id: "1".to_string(), timestamp: Utc::now(), fields: fields1, metadata: HashMap::new(), }); // Older record let mut fields2 = HashMap::new(); fields2.insert("data".to_string(), DataValue::String("old".to_string())); records.push(DataRecord { id: "2".to_string(), timestamp: Utc::now() - Duration::days(2), fields: fields2, metadata: HashMap::new(), }); let quality_score = QualityScore::calculate_for_records(&records).unwrap(); // Should have reduced timeliness due to old record assert!(quality_score.dimensions.timeliness.score <= 1.0); assert!(quality_score.dimensions.timeliness.score >= 0.0); } }