1174 lines
37 KiB
Rust
1174 lines
37 KiB
Rust
//! Data quality scoring system with comprehensive metrics
|
|
//!
|
|
//! This module provides comprehensive data quality scoring including completeness,
|
|
//! uniqueness, validity, consistency, and timeliness metrics with configurable
|
|
//! weights and scoring algorithms.
|
|
|
|
use std::collections::{HashMap, HashSet};
|
|
|
|
use chrono::{DateTime, Utc};
|
|
use serde::{Deserialize, Serialize};
|
|
|
|
use crate::{DataRecord, Result};
|
|
|
|
/// Comprehensive data quality score with multiple dimensions
|
|
#[derive(Debug, Clone, Serialize, Deserialize)]
|
|
pub struct QualityScore {
|
|
/// Overall composite quality score (0.0 - 1.0)
|
|
pub overall: f64,
|
|
/// Individual quality dimension scores
|
|
pub dimensions: QualityDimensions,
|
|
/// Weights used for calculating overall score
|
|
pub weights: QualityWeights,
|
|
/// Timestamp when score was calculated
|
|
pub timestamp: DateTime<Utc>,
|
|
/// Additional metadata about the scoring process
|
|
pub metadata: QualityMetadata,
|
|
}
|
|
|
|
/// Individual quality dimension scores
|
|
#[derive(Debug, Clone, Serialize, Deserialize)]
|
|
pub struct QualityDimensions {
|
|
/// Completeness score (data availability)
|
|
pub completeness: CompletenessScore,
|
|
/// Uniqueness score (duplicate detection)
|
|
pub uniqueness: UniquenessScore,
|
|
/// Validity score (format and constraint compliance)
|
|
pub validity: ValidityScore,
|
|
/// Consistency score (cross-field validation)
|
|
pub consistency: ConsistencyScore,
|
|
/// Timeliness score (data freshness)
|
|
pub timeliness: TimelinessScore,
|
|
/// Accuracy score (correctness assessment)
|
|
pub accuracy: AccuracyScore,
|
|
/// Integrity score (referential integrity)
|
|
pub integrity: IntegrityScore,
|
|
}
|
|
|
|
/// Completeness scoring - measures data availability and missing values
|
|
#[derive(Debug, Clone, Serialize, Deserialize)]
|
|
pub struct CompletenessScore {
|
|
/// Overall completeness score (0.0 - 1.0)
|
|
pub score: f64,
|
|
/// Per-field completeness scores
|
|
pub field_scores: HashMap<String, f64>,
|
|
/// Total number of expected values
|
|
pub total_expected: usize,
|
|
/// Total number of non-null values
|
|
pub total_present: usize,
|
|
/// Missing value patterns
|
|
pub missing_patterns: Vec<MissingPattern>,
|
|
/// Critical fields that are missing
|
|
pub critical_missing: Vec<String>,
|
|
}
|
|
|
|
/// Uniqueness scoring - measures duplicate detection and data uniqueness
|
|
#[derive(Debug, Clone, Serialize, Deserialize)]
|
|
pub struct UniquenessScore {
|
|
/// Overall uniqueness score (0.0 - 1.0)
|
|
pub score: f64,
|
|
/// Per-field uniqueness scores
|
|
pub field_scores: HashMap<String, f64>,
|
|
/// Duplicate detection results
|
|
pub duplicates: DuplicateAnalysis,
|
|
/// Unique identifier analysis
|
|
pub identifier_analysis: IdentifierAnalysis,
|
|
}
|
|
|
|
/// Validity scoring - measures format and constraint compliance
|
|
#[derive(Debug, Clone, Serialize, Deserialize)]
|
|
pub struct ValidityScore {
|
|
/// Overall validity score (0.0 - 1.0)
|
|
pub score: f64,
|
|
/// Per-field validity scores
|
|
pub field_scores: HashMap<String, f64>,
|
|
/// Format validation results
|
|
pub format_validation: FormatValidation,
|
|
/// Constraint validation results
|
|
pub constraint_validation: ConstraintValidation,
|
|
/// Data type consistency
|
|
pub type_consistency: TypeConsistency,
|
|
}
|
|
|
|
/// Consistency scoring - measures cross-field validation and logical consistency
|
|
#[derive(Debug, Clone, Serialize, Deserialize)]
|
|
pub struct ConsistencyScore {
|
|
/// Overall consistency score (0.0 - 1.0)
|
|
pub score: f64,
|
|
/// Cross-field validation results
|
|
pub cross_field_results: HashMap<String, f64>,
|
|
/// Business rule violations
|
|
pub business_rule_violations: Vec<BusinessRuleViolation>,
|
|
/// Logical consistency checks
|
|
pub logical_consistency: LogicalConsistency,
|
|
}
|
|
|
|
/// Timeliness scoring - measures data freshness and temporal validity
|
|
#[derive(Debug, Clone, Serialize, Deserialize)]
|
|
pub struct TimelinessScore {
|
|
/// Overall timeliness score (0.0 - 1.0)
|
|
pub score: f64,
|
|
/// Age of data in hours
|
|
pub data_age_hours: f64,
|
|
/// Expected maximum age for full score
|
|
pub expected_max_age_hours: f64,
|
|
/// Temporal patterns analysis
|
|
pub temporal_patterns: TemporalPatterns,
|
|
/// Freshness per data source
|
|
pub source_freshness: HashMap<String, f64>,
|
|
}
|
|
|
|
/// Accuracy scoring - measures correctness of data values
|
|
#[derive(Debug, Clone, Serialize, Deserialize)]
|
|
pub struct AccuracyScore {
|
|
/// Overall accuracy score (0.0 - 1.0)
|
|
pub score: f64,
|
|
/// Reference data comparison results
|
|
pub reference_comparison: ReferenceComparison,
|
|
/// Statistical outlier analysis
|
|
pub outlier_analysis: OutlierAnalysis,
|
|
/// Domain-specific accuracy checks
|
|
pub domain_checks: HashMap<String, f64>,
|
|
}
|
|
|
|
/// Integrity scoring - measures referential and structural integrity
|
|
#[derive(Debug, Clone, Serialize, Deserialize)]
|
|
pub struct IntegrityScore {
|
|
/// Overall integrity score (0.0 - 1.0)
|
|
pub score: f64,
|
|
/// Referential integrity results
|
|
pub referential_integrity: ReferentialIntegrity,
|
|
/// Entity integrity results
|
|
pub entity_integrity: EntityIntegrity,
|
|
/// Domain integrity results
|
|
pub domain_integrity: DomainIntegrity,
|
|
}
|
|
|
|
/// Weights for different quality dimensions
|
|
#[derive(Debug, Clone, Serialize, Deserialize)]
|
|
pub struct QualityWeights {
|
|
/// Completeness weight (default: 0.25)
|
|
pub completeness: f64,
|
|
/// Uniqueness weight (default: 0.15)
|
|
pub uniqueness: f64,
|
|
/// Validity weight (default: 0.25)
|
|
pub validity: f64,
|
|
/// Consistency weight (default: 0.15)
|
|
pub consistency: f64,
|
|
/// Timeliness weight (default: 0.10)
|
|
pub timeliness: f64,
|
|
/// Accuracy weight (default: 0.05)
|
|
pub accuracy: f64,
|
|
/// Integrity weight (default: 0.05)
|
|
pub integrity: f64,
|
|
}
|
|
|
|
/// Metadata about the quality scoring process
|
|
#[derive(Debug, Clone, Serialize, Deserialize)]
|
|
pub struct QualityMetadata {
|
|
/// Number of records analyzed
|
|
pub records_analyzed: usize,
|
|
/// Number of fields analyzed
|
|
pub fields_analyzed: usize,
|
|
/// Scoring algorithm version
|
|
pub algorithm_version: String,
|
|
/// Configuration used for scoring
|
|
pub configuration: ScoringConfiguration,
|
|
/// Performance metrics
|
|
pub performance: ScoringPerformance,
|
|
}
|
|
|
|
/// Configuration for quality scoring
|
|
#[derive(Debug, Clone, Serialize, Deserialize)]
|
|
pub struct ScoringConfiguration {
|
|
/// Enable strict mode (more stringent scoring)
|
|
pub strict_mode: bool,
|
|
/// Custom business rules
|
|
pub business_rules: Vec<String>,
|
|
/// Reference data sources
|
|
pub reference_sources: Vec<String>,
|
|
/// Temporal validity windows
|
|
pub temporal_windows: HashMap<String, i64>,
|
|
}
|
|
|
|
/// Performance metrics for the scoring process
|
|
#[derive(Debug, Clone, Serialize, Deserialize)]
|
|
pub struct ScoringPerformance {
|
|
/// Time taken for scoring in milliseconds
|
|
pub duration_ms: u64,
|
|
/// Memory used during scoring in bytes
|
|
pub memory_used_bytes: usize,
|
|
/// Number of validation rules applied
|
|
pub rules_applied: usize,
|
|
}
|
|
|
|
/// Missing value pattern analysis
|
|
#[derive(Debug, Clone, Serialize, Deserialize)]
|
|
pub struct MissingPattern {
|
|
/// Fields involved in the pattern
|
|
pub fields: Vec<String>,
|
|
/// Number of occurrences
|
|
pub count: usize,
|
|
/// Pattern description
|
|
pub description: String,
|
|
}
|
|
|
|
/// Duplicate analysis results
|
|
#[derive(Debug, Clone, Serialize, Deserialize)]
|
|
pub struct DuplicateAnalysis {
|
|
/// Total number of duplicates found
|
|
pub total_duplicates: usize,
|
|
/// Duplicate groups (records that are duplicates of each other)
|
|
pub duplicate_groups: Vec<DuplicateGroup>,
|
|
/// Fuzzy duplicate analysis
|
|
pub fuzzy_duplicates: FuzzyDuplicateAnalysis,
|
|
}
|
|
|
|
/// Group of duplicate records
|
|
#[derive(Debug, Clone, Serialize, Deserialize)]
|
|
pub struct DuplicateGroup {
|
|
/// Representative record ID
|
|
pub representative_id: String,
|
|
/// IDs of duplicate records
|
|
pub duplicate_ids: Vec<String>,
|
|
/// Similarity score between records
|
|
pub similarity_score: f64,
|
|
/// Fields that are identical
|
|
pub identical_fields: Vec<String>,
|
|
}
|
|
|
|
/// Fuzzy duplicate analysis
|
|
#[derive(Debug, Clone, Serialize, Deserialize)]
|
|
pub struct FuzzyDuplicateAnalysis {
|
|
/// Potential fuzzy duplicates
|
|
pub potential_duplicates: Vec<FuzzyDuplicateGroup>,
|
|
/// Similarity threshold used
|
|
pub similarity_threshold: f64,
|
|
/// Matching algorithm used
|
|
pub matching_algorithm: String,
|
|
}
|
|
|
|
/// Group of potentially similar records
|
|
#[derive(Debug, Clone, Serialize, Deserialize)]
|
|
pub struct FuzzyDuplicateGroup {
|
|
/// Record IDs in the group
|
|
pub record_ids: Vec<String>,
|
|
/// Maximum similarity score in the group
|
|
pub max_similarity: f64,
|
|
/// Fields used for comparison
|
|
pub comparison_fields: Vec<String>,
|
|
}
|
|
|
|
/// Unique identifier analysis
|
|
#[derive(Debug, Clone, Serialize, Deserialize)]
|
|
pub struct IdentifierAnalysis {
|
|
/// Potential unique identifiers found
|
|
pub potential_identifiers: Vec<String>,
|
|
/// Composite key candidates
|
|
pub composite_keys: Vec<Vec<String>>,
|
|
/// Identifier quality scores
|
|
pub identifier_scores: HashMap<String, f64>,
|
|
}
|
|
|
|
/// Format validation results
|
|
#[derive(Debug, Clone, Serialize, Deserialize)]
|
|
pub struct FormatValidation {
|
|
/// Per-field format compliance
|
|
pub field_compliance: HashMap<String, f64>,
|
|
/// Format violations found
|
|
pub violations: Vec<FormatViolation>,
|
|
/// Standard format adherence
|
|
pub standard_adherence: HashMap<String, f64>,
|
|
}
|
|
|
|
/// Format validation violation
|
|
#[derive(Debug, Clone, Serialize, Deserialize)]
|
|
pub struct FormatViolation {
|
|
/// Field name where violation occurred
|
|
pub field_name: String,
|
|
/// Expected format
|
|
pub expected_format: String,
|
|
/// Actual value
|
|
pub actual_value: String,
|
|
/// Violation description
|
|
pub description: String,
|
|
}
|
|
|
|
/// Constraint validation results
|
|
#[derive(Debug, Clone, Serialize, Deserialize)]
|
|
pub struct ConstraintValidation {
|
|
/// Range constraint violations
|
|
pub range_violations: Vec<ConstraintViolation>,
|
|
/// Length constraint violations
|
|
pub length_violations: Vec<ConstraintViolation>,
|
|
/// Custom constraint violations
|
|
pub custom_violations: Vec<ConstraintViolation>,
|
|
}
|
|
|
|
/// Constraint violation details
|
|
#[derive(Debug, Clone, Serialize, Deserialize)]
|
|
pub struct ConstraintViolation {
|
|
/// Field name
|
|
pub field_name: String,
|
|
/// Constraint type
|
|
pub constraint_type: String,
|
|
/// Constraint description
|
|
pub constraint: String,
|
|
/// Actual value
|
|
pub actual_value: String,
|
|
/// Severity level
|
|
pub severity: String,
|
|
}
|
|
|
|
/// Type consistency analysis
|
|
#[derive(Debug, Clone, Serialize, Deserialize)]
|
|
pub struct TypeConsistency {
|
|
/// Per-field type consistency scores
|
|
pub field_consistency: HashMap<String, f64>,
|
|
/// Type conflicts found
|
|
pub type_conflicts: Vec<TypeConflict>,
|
|
/// Inferred types vs. declared types
|
|
pub type_alignment: HashMap<String, TypeAlignment>,
|
|
}
|
|
|
|
/// Type conflict information
|
|
#[derive(Debug, Clone, Serialize, Deserialize)]
|
|
pub struct TypeConflict {
|
|
/// Field name
|
|
pub field_name: String,
|
|
/// Expected type
|
|
pub expected_type: String,
|
|
/// Actual types found
|
|
pub actual_types: Vec<String>,
|
|
/// Conflict frequency
|
|
pub frequency: usize,
|
|
}
|
|
|
|
/// Type alignment between inferred and declared
|
|
#[derive(Debug, Clone, Serialize, Deserialize)]
|
|
pub struct TypeAlignment {
|
|
/// Declared type
|
|
pub declared_type: String,
|
|
/// Inferred type
|
|
pub inferred_type: String,
|
|
/// Alignment score (0.0 - 1.0)
|
|
pub alignment_score: f64,
|
|
}
|
|
|
|
/// Business rule violation
|
|
#[derive(Debug, Clone, Serialize, Deserialize)]
|
|
pub struct BusinessRuleViolation {
|
|
/// Rule identifier
|
|
pub rule_id: String,
|
|
/// Rule description
|
|
pub rule_description: String,
|
|
/// Fields involved
|
|
pub fields_involved: Vec<String>,
|
|
/// Violation description
|
|
pub violation_description: String,
|
|
/// Severity level
|
|
pub severity: String,
|
|
}
|
|
|
|
/// Logical consistency checks
|
|
#[derive(Debug, Clone, Serialize, Deserialize)]
|
|
pub struct LogicalConsistency {
|
|
/// Mathematical consistency checks
|
|
pub mathematical_consistency: f64,
|
|
/// Temporal consistency checks
|
|
pub temporal_consistency: f64,
|
|
/// Hierarchical consistency checks
|
|
pub hierarchical_consistency: f64,
|
|
}
|
|
|
|
/// Temporal patterns in data
|
|
#[derive(Debug, Clone, Serialize, Deserialize)]
|
|
pub struct TemporalPatterns {
|
|
/// Data arrival patterns
|
|
pub arrival_patterns: HashMap<String, usize>,
|
|
/// Update frequency analysis
|
|
pub update_frequency: UpdateFrequency,
|
|
/// Seasonal patterns
|
|
pub seasonal_patterns: HashMap<String, f64>,
|
|
}
|
|
|
|
/// Update frequency analysis
|
|
#[derive(Debug, Clone, Serialize, Deserialize)]
|
|
pub struct UpdateFrequency {
|
|
/// Average time between updates in hours
|
|
pub avg_update_interval_hours: f64,
|
|
/// Update frequency distribution
|
|
pub frequency_distribution: HashMap<String, usize>,
|
|
/// Expected vs. actual update frequency
|
|
pub frequency_deviation: f64,
|
|
}
|
|
|
|
/// Reference data comparison results
|
|
#[derive(Debug, Clone, Serialize, Deserialize)]
|
|
pub struct ReferenceComparison {
|
|
/// Comparison results per reference source
|
|
pub source_comparisons: HashMap<String, f64>,
|
|
/// Fields validated against reference data
|
|
pub validated_fields: Vec<String>,
|
|
/// Reference data match rate
|
|
pub match_rate: f64,
|
|
}
|
|
|
|
/// Statistical outlier analysis
|
|
#[derive(Debug, Clone, Serialize, Deserialize)]
|
|
pub struct OutlierAnalysis {
|
|
/// Outliers detected per field
|
|
pub field_outliers: HashMap<String, Vec<OutlierInfo>>,
|
|
/// Outlier detection methods used
|
|
pub detection_methods: Vec<String>,
|
|
/// Overall outlier rate
|
|
pub outlier_rate: f64,
|
|
}
|
|
|
|
/// Information about a detected outlier
|
|
#[derive(Debug, Clone, Serialize, Deserialize)]
|
|
pub struct OutlierInfo {
|
|
/// Value that is an outlier
|
|
pub value: String,
|
|
/// Outlier score (higher = more extreme)
|
|
pub score: f64,
|
|
/// Detection method used
|
|
pub method: String,
|
|
/// Context information
|
|
pub context: HashMap<String, String>,
|
|
}
|
|
|
|
/// Referential integrity analysis
|
|
#[derive(Debug, Clone, Serialize, Deserialize)]
|
|
pub struct ReferentialIntegrity {
|
|
/// Foreign key violations
|
|
pub foreign_key_violations: Vec<ReferentialViolation>,
|
|
/// Orphaned records
|
|
pub orphaned_records: Vec<String>,
|
|
/// Overall integrity score
|
|
pub integrity_score: f64,
|
|
}
|
|
|
|
/// Referential integrity violation
|
|
#[derive(Debug, Clone, Serialize, Deserialize)]
|
|
pub struct ReferentialViolation {
|
|
/// Child table/field
|
|
pub child_field: String,
|
|
/// Parent table/field
|
|
pub parent_field: String,
|
|
/// Violating values
|
|
pub violating_values: Vec<String>,
|
|
/// Violation count
|
|
pub violation_count: usize,
|
|
}
|
|
|
|
/// Entity integrity analysis
|
|
#[derive(Debug, Clone, Serialize, Deserialize)]
|
|
pub struct EntityIntegrity {
|
|
/// Primary key violations
|
|
pub primary_key_violations: Vec<String>,
|
|
/// Null primary key instances
|
|
pub null_primary_keys: usize,
|
|
/// Duplicate primary key instances
|
|
pub duplicate_primary_keys: usize,
|
|
}
|
|
|
|
/// Domain integrity analysis
|
|
#[derive(Debug, Clone, Serialize, Deserialize)]
|
|
pub struct DomainIntegrity {
|
|
/// Domain constraint violations per field
|
|
pub constraint_violations: HashMap<String, usize>,
|
|
/// Check constraint violations
|
|
pub check_constraint_violations: Vec<CheckConstraintViolation>,
|
|
/// Overall domain integrity score
|
|
pub integrity_score: f64,
|
|
}
|
|
|
|
/// Check constraint violation
|
|
#[derive(Debug, Clone, Serialize, Deserialize)]
|
|
pub struct CheckConstraintViolation {
|
|
/// Constraint name
|
|
pub constraint_name: String,
|
|
/// Field name
|
|
pub field_name: String,
|
|
/// Violating value
|
|
pub violating_value: String,
|
|
/// Constraint description
|
|
pub constraint_description: String,
|
|
}
|
|
|
|
impl QualityScore {
|
|
/// Create a new quality score with default configuration
|
|
pub fn new() -> Self {
|
|
Self {
|
|
overall: 0.0,
|
|
dimensions: QualityDimensions::default(),
|
|
weights: QualityWeights::default(),
|
|
timestamp: Utc::now(),
|
|
metadata: QualityMetadata::default(),
|
|
}
|
|
}
|
|
|
|
/// Calculate overall quality score from dimension scores
|
|
pub fn calculate_overall(&mut self) {
|
|
let weights = &self.weights;
|
|
let dims = &self.dimensions;
|
|
|
|
self.overall = weights.completeness * dims.completeness.score
|
|
+ weights.uniqueness * dims.uniqueness.score
|
|
+ weights.validity * dims.validity.score
|
|
+ weights.consistency * dims.consistency.score
|
|
+ weights.timeliness * dims.timeliness.score
|
|
+ weights.accuracy * dims.accuracy.score
|
|
+ weights.integrity * dims.integrity.score;
|
|
|
|
// Ensure score is between 0 and 1
|
|
self.overall = self.overall.max(0.0).min(1.0);
|
|
}
|
|
|
|
/// Get the overall quality score
|
|
pub fn overall_score(&self) -> f64 {
|
|
self.overall
|
|
}
|
|
|
|
/// Set completeness score
|
|
pub fn set_completeness(&mut self, score: f64) {
|
|
self.dimensions.completeness.score = score.max(0.0).min(1.0);
|
|
self.calculate_overall();
|
|
}
|
|
|
|
/// Set uniqueness score
|
|
pub fn set_uniqueness(&mut self, score: f64) {
|
|
self.dimensions.uniqueness.score = score.max(0.0).min(1.0);
|
|
self.calculate_overall();
|
|
}
|
|
|
|
/// Set validity score
|
|
pub fn set_validity(&mut self, score: f64) {
|
|
self.dimensions.validity.score = score.max(0.0).min(1.0);
|
|
self.calculate_overall();
|
|
}
|
|
|
|
/// Set consistency score
|
|
pub fn set_consistency(&mut self, score: f64) {
|
|
self.dimensions.consistency.score = score.max(0.0).min(1.0);
|
|
self.calculate_overall();
|
|
}
|
|
|
|
/// Set timeliness score
|
|
pub fn set_timeliness(&mut self, score: f64) {
|
|
self.dimensions.timeliness.score = score.max(0.0).min(1.0);
|
|
self.calculate_overall();
|
|
}
|
|
|
|
/// Get quality grade as string
|
|
pub fn quality_grade(&self) -> &'static str {
|
|
match self.overall {
|
|
score if score >= 0.9 => "Excellent",
|
|
score if score >= 0.8 => "Good",
|
|
score if score >= 0.7 => "Acceptable",
|
|
score if score >= 0.6 => "Poor",
|
|
_ => "Very Poor",
|
|
}
|
|
}
|
|
|
|
/// Calculate quality score for a set of records
|
|
pub fn calculate_for_records(records: &[DataRecord]) -> Result<Self> {
|
|
let mut quality_score = Self::new();
|
|
let start_time = std::time::Instant::now();
|
|
|
|
if records.is_empty() {
|
|
return Ok(quality_score);
|
|
}
|
|
|
|
// Calculate completeness score
|
|
let completeness = Self::calculate_completeness(records)?;
|
|
quality_score.dimensions.completeness = completeness;
|
|
|
|
// Calculate uniqueness score
|
|
let uniqueness = Self::calculate_uniqueness(records)?;
|
|
quality_score.dimensions.uniqueness = uniqueness;
|
|
|
|
// Calculate validity score (simplified)
|
|
let validity = Self::calculate_validity(records)?;
|
|
quality_score.dimensions.validity = validity;
|
|
|
|
// Calculate consistency score (simplified)
|
|
let consistency = Self::calculate_consistency(records)?;
|
|
quality_score.dimensions.consistency = consistency;
|
|
|
|
// Calculate timeliness score
|
|
let timeliness = Self::calculate_timeliness(records)?;
|
|
quality_score.dimensions.timeliness = timeliness;
|
|
|
|
// Set default scores for accuracy and integrity
|
|
quality_score.dimensions.accuracy.score = 0.9; // Placeholder
|
|
quality_score.dimensions.integrity.score = 0.95; // Placeholder
|
|
|
|
// Calculate overall score
|
|
quality_score.calculate_overall();
|
|
|
|
// Update metadata
|
|
quality_score.metadata.records_analyzed = records.len();
|
|
quality_score.metadata.performance.duration_ms = start_time.elapsed().as_millis() as u64;
|
|
|
|
Ok(quality_score)
|
|
}
|
|
|
|
fn calculate_completeness(records: &[DataRecord]) -> Result<CompletenessScore> {
|
|
let mut field_scores = HashMap::new();
|
|
let mut total_expected = 0;
|
|
let mut total_present = 0;
|
|
let mut all_fields = HashSet::new();
|
|
|
|
// Collect all field names
|
|
for record in records {
|
|
for field_name in record.fields.keys() {
|
|
all_fields.insert(field_name.clone());
|
|
}
|
|
}
|
|
|
|
// Calculate completeness for each field
|
|
for field_name in &all_fields {
|
|
let mut present_count = 0;
|
|
let record_count = records.len();
|
|
|
|
for record in records {
|
|
if let Some(value) = record.fields.get(field_name) {
|
|
if !value.is_null() {
|
|
present_count += 1;
|
|
}
|
|
}
|
|
}
|
|
|
|
let field_completeness = if record_count > 0 {
|
|
present_count as f64 / record_count as f64
|
|
} else {
|
|
1.0
|
|
};
|
|
|
|
field_scores.insert(field_name.clone(), field_completeness);
|
|
total_expected += record_count;
|
|
total_present += present_count;
|
|
}
|
|
|
|
let overall_completeness = if total_expected > 0 {
|
|
total_present as f64 / total_expected as f64
|
|
} else {
|
|
1.0
|
|
};
|
|
|
|
Ok(CompletenessScore {
|
|
score: overall_completeness,
|
|
field_scores,
|
|
total_expected,
|
|
total_present,
|
|
missing_patterns: vec![], // Would be calculated with pattern analysis
|
|
critical_missing: vec![], // Would be determined based on business rules
|
|
})
|
|
}
|
|
|
|
fn calculate_uniqueness(records: &[DataRecord]) -> Result<UniquenessScore> {
|
|
let mut field_scores = HashMap::new();
|
|
let mut all_fields = HashSet::new();
|
|
|
|
// Collect all field names
|
|
for record in records {
|
|
for field_name in record.fields.keys() {
|
|
all_fields.insert(field_name.clone());
|
|
}
|
|
}
|
|
|
|
// Calculate uniqueness for each field
|
|
for field_name in &all_fields {
|
|
let mut values = HashMap::new();
|
|
let mut total_values = 0;
|
|
|
|
for record in records {
|
|
if let Some(value) = record.fields.get(field_name) {
|
|
if !value.is_null() {
|
|
let value_str = format!("{:?}", value);
|
|
*values.entry(value_str).or_insert(0) += 1;
|
|
total_values += 1;
|
|
}
|
|
}
|
|
}
|
|
|
|
let unique_count = values.len();
|
|
let field_uniqueness = if total_values > 0 {
|
|
unique_count as f64 / total_values as f64
|
|
} else {
|
|
1.0
|
|
};
|
|
|
|
field_scores.insert(field_name.clone(), field_uniqueness);
|
|
}
|
|
|
|
let overall_uniqueness = if !field_scores.is_empty() {
|
|
field_scores.values().sum::<f64>() / field_scores.len() as f64
|
|
} else {
|
|
1.0
|
|
};
|
|
|
|
Ok(UniquenessScore {
|
|
score: overall_uniqueness,
|
|
field_scores,
|
|
duplicates: DuplicateAnalysis {
|
|
total_duplicates: 0,
|
|
duplicate_groups: vec![],
|
|
fuzzy_duplicates: FuzzyDuplicateAnalysis {
|
|
potential_duplicates: vec![],
|
|
similarity_threshold: 0.8,
|
|
matching_algorithm: "Levenshtein".to_string(),
|
|
},
|
|
},
|
|
identifier_analysis: IdentifierAnalysis {
|
|
potential_identifiers: vec![],
|
|
composite_keys: vec![],
|
|
identifier_scores: HashMap::new(),
|
|
},
|
|
})
|
|
}
|
|
|
|
fn calculate_validity(_records: &[DataRecord]) -> Result<ValidityScore> {
|
|
// Simplified validity calculation
|
|
// In a full implementation, this would validate formats, constraints, etc.
|
|
Ok(ValidityScore {
|
|
score: 0.85, // Placeholder
|
|
field_scores: HashMap::new(),
|
|
format_validation: FormatValidation {
|
|
field_compliance: HashMap::new(),
|
|
violations: vec![],
|
|
standard_adherence: HashMap::new(),
|
|
},
|
|
constraint_validation: ConstraintValidation {
|
|
range_violations: vec![],
|
|
length_violations: vec![],
|
|
custom_violations: vec![],
|
|
},
|
|
type_consistency: TypeConsistency {
|
|
field_consistency: HashMap::new(),
|
|
type_conflicts: vec![],
|
|
type_alignment: HashMap::new(),
|
|
},
|
|
})
|
|
}
|
|
|
|
fn calculate_consistency(_records: &[DataRecord]) -> Result<ConsistencyScore> {
|
|
// Simplified consistency calculation
|
|
Ok(ConsistencyScore {
|
|
score: 0.9, // Placeholder
|
|
cross_field_results: HashMap::new(),
|
|
business_rule_violations: vec![],
|
|
logical_consistency: LogicalConsistency {
|
|
mathematical_consistency: 0.95,
|
|
temporal_consistency: 0.9,
|
|
hierarchical_consistency: 0.85,
|
|
},
|
|
})
|
|
}
|
|
|
|
fn calculate_timeliness(records: &[DataRecord]) -> Result<TimelinessScore> {
|
|
let now = Utc::now();
|
|
let mut total_age_hours = 0.0;
|
|
let mut count = 0;
|
|
|
|
for record in records {
|
|
let age_duration = now.signed_duration_since(record.timestamp);
|
|
let age_hours = age_duration.num_hours() as f64;
|
|
total_age_hours += age_hours;
|
|
count += 1;
|
|
}
|
|
|
|
let avg_age_hours = if count > 0 {
|
|
total_age_hours / count as f64
|
|
} else {
|
|
0.0
|
|
};
|
|
|
|
// Score decreases with age, full score for data less than 24 hours old
|
|
let expected_max_age_hours = 24.0;
|
|
let timeliness_score = if avg_age_hours <= expected_max_age_hours {
|
|
1.0
|
|
} else {
|
|
(expected_max_age_hours / avg_age_hours).min(1.0).max(0.0)
|
|
};
|
|
|
|
Ok(TimelinessScore {
|
|
score: timeliness_score,
|
|
data_age_hours: avg_age_hours,
|
|
expected_max_age_hours,
|
|
temporal_patterns: TemporalPatterns {
|
|
arrival_patterns: HashMap::new(),
|
|
update_frequency: UpdateFrequency {
|
|
avg_update_interval_hours: avg_age_hours,
|
|
frequency_distribution: HashMap::new(),
|
|
frequency_deviation: 0.0,
|
|
},
|
|
seasonal_patterns: HashMap::new(),
|
|
},
|
|
source_freshness: HashMap::new(),
|
|
})
|
|
}
|
|
}
|
|
|
|
impl Default for QualityScore {
|
|
fn default() -> Self {
|
|
Self::new()
|
|
}
|
|
}
|
|
|
|
impl Default for QualityDimensions {
|
|
fn default() -> Self {
|
|
Self {
|
|
completeness: CompletenessScore::default(),
|
|
uniqueness: UniquenessScore::default(),
|
|
validity: ValidityScore::default(),
|
|
consistency: ConsistencyScore::default(),
|
|
timeliness: TimelinessScore::default(),
|
|
accuracy: AccuracyScore::default(),
|
|
integrity: IntegrityScore::default(),
|
|
}
|
|
}
|
|
}
|
|
|
|
impl Default for QualityWeights {
|
|
fn default() -> Self {
|
|
Self {
|
|
completeness: 0.25,
|
|
uniqueness: 0.15,
|
|
validity: 0.25,
|
|
consistency: 0.15,
|
|
timeliness: 0.10,
|
|
accuracy: 0.05,
|
|
integrity: 0.05,
|
|
}
|
|
}
|
|
}
|
|
|
|
impl Default for QualityMetadata {
|
|
fn default() -> Self {
|
|
Self {
|
|
records_analyzed: 0,
|
|
fields_analyzed: 0,
|
|
algorithm_version: "1.0.0".to_string(),
|
|
configuration: ScoringConfiguration::default(),
|
|
performance: ScoringPerformance::default(),
|
|
}
|
|
}
|
|
}
|
|
|
|
impl Default for ScoringConfiguration {
|
|
fn default() -> Self {
|
|
Self {
|
|
strict_mode: false,
|
|
business_rules: vec![],
|
|
reference_sources: vec![],
|
|
temporal_windows: HashMap::new(),
|
|
}
|
|
}
|
|
}
|
|
|
|
impl Default for ScoringPerformance {
|
|
fn default() -> Self {
|
|
Self {
|
|
duration_ms: 0,
|
|
memory_used_bytes: 0,
|
|
rules_applied: 0,
|
|
}
|
|
}
|
|
}
|
|
|
|
// Default implementations for all score types
|
|
impl Default for CompletenessScore {
|
|
fn default() -> Self {
|
|
Self {
|
|
score: 0.0,
|
|
field_scores: HashMap::new(),
|
|
total_expected: 0,
|
|
total_present: 0,
|
|
missing_patterns: vec![],
|
|
critical_missing: vec![],
|
|
}
|
|
}
|
|
}
|
|
|
|
impl Default for UniquenessScore {
|
|
fn default() -> Self {
|
|
Self {
|
|
score: 0.0,
|
|
field_scores: HashMap::new(),
|
|
duplicates: DuplicateAnalysis {
|
|
total_duplicates: 0,
|
|
duplicate_groups: vec![],
|
|
fuzzy_duplicates: FuzzyDuplicateAnalysis {
|
|
potential_duplicates: vec![],
|
|
similarity_threshold: 0.8,
|
|
matching_algorithm: "Levenshtein".to_string(),
|
|
},
|
|
},
|
|
identifier_analysis: IdentifierAnalysis {
|
|
potential_identifiers: vec![],
|
|
composite_keys: vec![],
|
|
identifier_scores: HashMap::new(),
|
|
},
|
|
}
|
|
}
|
|
}
|
|
|
|
impl Default for ValidityScore {
|
|
fn default() -> Self {
|
|
Self {
|
|
score: 0.0,
|
|
field_scores: HashMap::new(),
|
|
format_validation: FormatValidation {
|
|
field_compliance: HashMap::new(),
|
|
violations: vec![],
|
|
standard_adherence: HashMap::new(),
|
|
},
|
|
constraint_validation: ConstraintValidation {
|
|
range_violations: vec![],
|
|
length_violations: vec![],
|
|
custom_violations: vec![],
|
|
},
|
|
type_consistency: TypeConsistency {
|
|
field_consistency: HashMap::new(),
|
|
type_conflicts: vec![],
|
|
type_alignment: HashMap::new(),
|
|
},
|
|
}
|
|
}
|
|
}
|
|
|
|
impl Default for ConsistencyScore {
|
|
fn default() -> Self {
|
|
Self {
|
|
score: 0.0,
|
|
cross_field_results: HashMap::new(),
|
|
business_rule_violations: vec![],
|
|
logical_consistency: LogicalConsistency {
|
|
mathematical_consistency: 0.0,
|
|
temporal_consistency: 0.0,
|
|
hierarchical_consistency: 0.0,
|
|
},
|
|
}
|
|
}
|
|
}
|
|
|
|
impl Default for TimelinessScore {
|
|
fn default() -> Self {
|
|
Self {
|
|
score: 0.0,
|
|
data_age_hours: 0.0,
|
|
expected_max_age_hours: 24.0,
|
|
temporal_patterns: TemporalPatterns {
|
|
arrival_patterns: HashMap::new(),
|
|
update_frequency: UpdateFrequency {
|
|
avg_update_interval_hours: 0.0,
|
|
frequency_distribution: HashMap::new(),
|
|
frequency_deviation: 0.0,
|
|
},
|
|
seasonal_patterns: HashMap::new(),
|
|
},
|
|
source_freshness: HashMap::new(),
|
|
}
|
|
}
|
|
}
|
|
|
|
impl Default for AccuracyScore {
|
|
fn default() -> Self {
|
|
Self {
|
|
score: 0.0,
|
|
reference_comparison: ReferenceComparison {
|
|
source_comparisons: HashMap::new(),
|
|
validated_fields: vec![],
|
|
match_rate: 0.0,
|
|
},
|
|
outlier_analysis: OutlierAnalysis {
|
|
field_outliers: HashMap::new(),
|
|
detection_methods: vec![],
|
|
outlier_rate: 0.0,
|
|
},
|
|
domain_checks: HashMap::new(),
|
|
}
|
|
}
|
|
}
|
|
|
|
impl Default for IntegrityScore {
|
|
fn default() -> Self {
|
|
Self {
|
|
score: 0.0,
|
|
referential_integrity: ReferentialIntegrity {
|
|
foreign_key_violations: vec![],
|
|
orphaned_records: vec![],
|
|
integrity_score: 0.0,
|
|
},
|
|
entity_integrity: EntityIntegrity {
|
|
primary_key_violations: vec![],
|
|
null_primary_keys: 0,
|
|
duplicate_primary_keys: 0,
|
|
},
|
|
domain_integrity: DomainIntegrity {
|
|
constraint_violations: HashMap::new(),
|
|
check_constraint_violations: vec![],
|
|
integrity_score: 0.0,
|
|
},
|
|
}
|
|
}
|
|
}
|
|
|
|
#[cfg(test)]
|
|
mod tests {
|
|
use super::*;
|
|
use crate::DataValue;
|
|
use chrono::Duration;
|
|
use std::collections::HashMap;
|
|
|
|
#[test]
|
|
fn test_quality_score_creation() {
|
|
let quality_score = QualityScore::new();
|
|
assert_eq!(quality_score.overall_score(), 0.0);
|
|
assert_eq!(quality_score.quality_grade(), "Very Poor");
|
|
}
|
|
|
|
#[test]
|
|
fn test_quality_score_calculation() {
|
|
let mut quality_score = QualityScore::new();
|
|
quality_score.set_completeness(0.9);
|
|
quality_score.set_uniqueness(0.8);
|
|
quality_score.set_validity(0.95);
|
|
quality_score.set_consistency(0.85);
|
|
quality_score.set_timeliness(0.7);
|
|
|
|
assert!(quality_score.overall_score() > 0.0);
|
|
assert!(quality_score.overall_score() <= 1.0);
|
|
}
|
|
|
|
#[test]
|
|
fn test_quality_grade_calculation() {
|
|
let mut quality_score = QualityScore::new();
|
|
|
|
quality_score.overall = 0.95;
|
|
assert_eq!(quality_score.quality_grade(), "Excellent");
|
|
|
|
quality_score.overall = 0.85;
|
|
assert_eq!(quality_score.quality_grade(), "Good");
|
|
|
|
quality_score.overall = 0.75;
|
|
assert_eq!(quality_score.quality_grade(), "Acceptable");
|
|
|
|
quality_score.overall = 0.65;
|
|
assert_eq!(quality_score.quality_grade(), "Poor");
|
|
|
|
quality_score.overall = 0.5;
|
|
assert_eq!(quality_score.quality_grade(), "Very Poor");
|
|
}
|
|
|
|
#[test]
|
|
fn test_completeness_calculation() {
|
|
let mut records = vec![];
|
|
|
|
// Record 1: all fields present
|
|
let mut fields1 = HashMap::new();
|
|
fields1.insert("name".to_string(), DataValue::String("John".to_string()));
|
|
fields1.insert("age".to_string(), DataValue::Int(30));
|
|
fields1.insert(
|
|
"email".to_string(),
|
|
DataValue::String("[email protected]".to_string()),
|
|
);
|
|
|
|
records.push(DataRecord {
|
|
id: "1".to_string(),
|
|
timestamp: Utc::now(),
|
|
fields: fields1,
|
|
metadata: HashMap::new(),
|
|
});
|
|
|
|
// Record 2: one field missing
|
|
let mut fields2 = HashMap::new();
|
|
fields2.insert("name".to_string(), DataValue::String("Jane".to_string()));
|
|
fields2.insert("age".to_string(), DataValue::Int(25));
|
|
fields2.insert("email".to_string(), DataValue::Null);
|
|
|
|
records.push(DataRecord {
|
|
id: "2".to_string(),
|
|
timestamp: Utc::now(),
|
|
fields: fields2,
|
|
metadata: HashMap::new(),
|
|
});
|
|
|
|
let quality_score = QualityScore::calculate_for_records(&records).unwrap();
|
|
|
|
// Should have some completeness score less than 1.0 due to missing email
|
|
assert!(quality_score.dimensions.completeness.score < 1.0);
|
|
assert!(quality_score.dimensions.completeness.score > 0.0);
|
|
}
|
|
|
|
#[test]
|
|
fn test_uniqueness_calculation() {
|
|
let mut records = vec![];
|
|
|
|
// Record 1
|
|
let mut fields1 = HashMap::new();
|
|
fields1.insert("id".to_string(), DataValue::Int(1));
|
|
fields1.insert("name".to_string(), DataValue::String("John".to_string()));
|
|
|
|
records.push(DataRecord {
|
|
id: "1".to_string(),
|
|
timestamp: Utc::now(),
|
|
fields: fields1,
|
|
metadata: HashMap::new(),
|
|
});
|
|
|
|
// Record 2 (unique)
|
|
let mut fields2 = HashMap::new();
|
|
fields2.insert("id".to_string(), DataValue::Int(2));
|
|
fields2.insert("name".to_string(), DataValue::String("Jane".to_string()));
|
|
|
|
records.push(DataRecord {
|
|
id: "2".to_string(),
|
|
timestamp: Utc::now(),
|
|
fields: fields2,
|
|
metadata: HashMap::new(),
|
|
});
|
|
|
|
let quality_score = QualityScore::calculate_for_records(&records).unwrap();
|
|
|
|
// Should have perfect uniqueness since all values are unique
|
|
assert!(quality_score.dimensions.uniqueness.score > 0.0);
|
|
}
|
|
|
|
#[test]
|
|
fn test_timeliness_calculation() {
|
|
let mut records = vec![];
|
|
|
|
// Recent record
|
|
let mut fields1 = HashMap::new();
|
|
fields1.insert("data".to_string(), DataValue::String("recent".to_string()));
|
|
|
|
records.push(DataRecord {
|
|
id: "1".to_string(),
|
|
timestamp: Utc::now(),
|
|
fields: fields1,
|
|
metadata: HashMap::new(),
|
|
});
|
|
|
|
// Older record
|
|
let mut fields2 = HashMap::new();
|
|
fields2.insert("data".to_string(), DataValue::String("old".to_string()));
|
|
|
|
records.push(DataRecord {
|
|
id: "2".to_string(),
|
|
timestamp: Utc::now() - Duration::days(2),
|
|
fields: fields2,
|
|
metadata: HashMap::new(),
|
|
});
|
|
|
|
let quality_score = QualityScore::calculate_for_records(&records).unwrap();
|
|
|
|
// Should have reduced timeliness due to old record
|
|
assert!(quality_score.dimensions.timeliness.score <= 1.0);
|
|
assert!(quality_score.dimensions.timeliness.score >= 0.0);
|
|
}
|
|
}
|