278 lines
8.7 KiB
Rust
278 lines
8.7 KiB
Rust
//! Comprehensive data validation with statistical profiling, anomaly detection, and real-time pipeline
|
|
//!
|
|
//! This crate provides enterprise-grade data validation capabilities including:
|
|
//! - Rule-based validation engine with composable validation rules
|
|
//! - Statistical profiling with comprehensive descriptive statistics
|
|
//! - Data quality scoring (completeness, uniqueness, validity, consistency)
|
|
//! - Anomaly detection using z-score, IQR, and isolation forest algorithms
|
|
//! - Schema management with inference, evolution tracking, and drift detection
|
|
//! - Data lineage tracking and impact analysis with dependency graphs
|
|
//! - Real-time validation pipeline with streaming support
|
|
//! - Custom validation rules framework with extensible plugin system
|
|
//!
|
|
//! # Examples
|
|
//!
|
|
//! ```rust
|
|
//! use rtx_data_validation::{
|
|
//! ValidationEngine, ValidationRule, DataProfile, QualityScore,
|
|
//! AnomalyDetector, SchemaManager, LineageTracker
|
|
//! };
|
|
//!
|
|
//! // Create a validation engine with statistical profiling
|
|
//! let mut engine = ValidationEngine::builder()
|
|
//! .with_statistical_validation(true)
|
|
//! .with_anomaly_detection(true)
|
|
//! .build()?;
|
|
//!
|
|
//! // Add validation rules
|
|
//! engine.add_rule(ValidationRule::range("age", 0.0, 120.0));
|
|
//! engine.add_rule(ValidationRule::not_null("email"));
|
|
//! engine.add_rule(ValidationRule::format("email", r"^[^@]+@[^@]+\.[^@]+$"));
|
|
//!
|
|
//! // Validate data and get quality scores
|
|
//! let data = serde_json::json!({
|
|
//! "age": 25,
|
|
//! "email": "[email protected]",
|
|
//! "score": 85.5
|
|
//! });
|
|
//!
|
|
//! let result = engine.validate(&data)?;
|
|
//! let quality_score = engine.calculate_quality_score(&data)?;
|
|
//! let profile = engine.generate_profile(&data)?;
|
|
//!
|
|
//! println!("Validation passed: {}", result.is_valid());
|
|
//! println!("Quality score: {:.2}", quality_score.overall_score());
|
|
//! println!("Data profile: {:?}", profile);
|
|
//! # Ok::<(), Box<dyn std::error::Error>>(())
|
|
//! ```
|
|
|
|
#![allow(clippy::missing_errors_doc)]
|
|
|
|
use chrono::{DateTime, Utc};
|
|
use serde::{Deserialize, Serialize};
|
|
use std::collections::HashMap;
|
|
|
|
// Core modules
|
|
pub mod anomaly;
|
|
pub mod engine;
|
|
pub mod lineage;
|
|
pub mod metrics;
|
|
pub mod pipeline;
|
|
pub mod profile;
|
|
pub mod quality;
|
|
pub mod rules;
|
|
pub mod schema;
|
|
|
|
// Re-exports for convenient access
|
|
pub use anomaly::{
|
|
AnomalyDetector, AnomalyResult, DetectionMethod, IQRDetector, IsolationForestDetector,
|
|
ZScoreDetector,
|
|
};
|
|
pub use engine::{ValidationEngine, ValidationEngineBuilder};
|
|
pub use lineage::{DataLineage, DependencyGraph, ImpactAnalysis, LineageGraph, LineageTracker};
|
|
pub use metrics::{PerformanceMetrics, ValidationMetrics};
|
|
pub use pipeline::{BatchValidator, PipelineConfig, StreamingValidator, ValidationPipeline};
|
|
pub use profile::{
|
|
ColumnProfile, CorrelationMatrix, DataProfile, DistributionAnalysis, StatisticalProfile,
|
|
};
|
|
pub use quality::{
|
|
CompletenessScore, ConsistencyScore, QualityScore, TimelinessScore, UniquenessScore,
|
|
ValidityScore,
|
|
};
|
|
pub use rules::{
|
|
CrossFieldRule, NumericRule, RuleResult, RuleType, RuleViolation, StringRule, TemporalRule,
|
|
ValidationRule,
|
|
};
|
|
pub use schema::{
|
|
DataType, SchemaDiff, SchemaEvolution, SchemaInference, SchemaManager, SchemaValidation,
|
|
};
|
|
|
|
/// Comprehensive error types for data validation operations
|
|
#[derive(Debug, thiserror::Error)]
|
|
pub enum ValidationError {
|
|
/// Rule validation error with context
|
|
#[error("Validation rule '{rule}' failed: {message}")]
|
|
RuleViolation { rule: String, message: String },
|
|
|
|
/// Schema validation error
|
|
#[error("Schema validation error: {0}")]
|
|
Schema(String),
|
|
|
|
/// Statistical computation error
|
|
#[error("Statistical computation error: {0}")]
|
|
Statistics(String),
|
|
|
|
/// Anomaly detection error
|
|
#[error("Anomaly detection error: {0}")]
|
|
Anomaly(String),
|
|
|
|
/// Data lineage tracking error
|
|
#[error("Lineage tracking error: {0}")]
|
|
Lineage(String),
|
|
|
|
/// Pipeline processing error
|
|
#[error("Pipeline processing error: {0}")]
|
|
Pipeline(String),
|
|
|
|
/// Configuration error
|
|
#[error("Configuration error: {0}")]
|
|
Config(String),
|
|
|
|
/// Data format error
|
|
#[error("Data format error: {0}")]
|
|
Format(String),
|
|
|
|
/// I/O operation error
|
|
#[error("I/O error: {0}")]
|
|
Io(#[from] std::io::Error),
|
|
|
|
/// Serialization/deserialization error
|
|
#[error("Serialization error: {0}")]
|
|
Serde(#[from] serde_json::Error),
|
|
|
|
/// Generic error for other cases
|
|
#[error("Validation error: {0}")]
|
|
Other(String),
|
|
}
|
|
|
|
/// Result type for validation operations
|
|
pub type Result<T> = std::result::Result<T, ValidationError>;
|
|
|
|
/// Core data structure representing a data record for validation
|
|
#[derive(Debug, Clone, Serialize, Deserialize)]
|
|
pub struct DataRecord {
|
|
/// Unique identifier for the record
|
|
pub id: String,
|
|
/// Timestamp when the record was created
|
|
pub timestamp: DateTime<Utc>,
|
|
/// The actual data fields as key-value pairs
|
|
pub fields: HashMap<String, DataValue>,
|
|
/// Metadata associated with the record
|
|
pub metadata: HashMap<String, String>,
|
|
}
|
|
|
|
/// Enum representing different types of data values
|
|
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
|
|
pub enum DataValue {
|
|
/// Null/missing value
|
|
Null,
|
|
/// Boolean value
|
|
Bool(bool),
|
|
/// Integer value
|
|
Int(i64),
|
|
/// Floating point value
|
|
Float(f64),
|
|
/// String value
|
|
String(String),
|
|
/// Array of values
|
|
Array(Vec<DataValue>),
|
|
/// Object/map of key-value pairs
|
|
Object(HashMap<String, DataValue>),
|
|
/// Timestamp value
|
|
Timestamp(DateTime<Utc>),
|
|
}
|
|
|
|
impl DataValue {
|
|
/// Check if the value is null/missing
|
|
pub fn is_null(&self) -> bool {
|
|
matches!(self, DataValue::Null)
|
|
}
|
|
|
|
/// Get the type name of the value
|
|
pub fn type_name(&self) -> &'static str {
|
|
match self {
|
|
DataValue::Null => "null",
|
|
DataValue::Bool(_) => "boolean",
|
|
DataValue::Int(_) => "integer",
|
|
DataValue::Float(_) => "float",
|
|
DataValue::String(_) => "string",
|
|
DataValue::Array(_) => "array",
|
|
DataValue::Object(_) => "object",
|
|
DataValue::Timestamp(_) => "timestamp",
|
|
}
|
|
}
|
|
|
|
/// Convert to f64 if possible for numerical operations
|
|
pub fn as_f64(&self) -> Option<f64> {
|
|
match self {
|
|
DataValue::Int(i) => Some(*i as f64),
|
|
DataValue::Float(f) => Some(*f),
|
|
_ => None,
|
|
}
|
|
}
|
|
|
|
/// Convert to string representation
|
|
pub fn as_string(&self) -> Option<String> {
|
|
match self {
|
|
DataValue::String(s) => Some(s.clone()),
|
|
DataValue::Int(i) => Some(i.to_string()),
|
|
DataValue::Float(f) => Some(f.to_string()),
|
|
DataValue::Bool(b) => Some(b.to_string()),
|
|
_ => None,
|
|
}
|
|
}
|
|
}
|
|
|
|
impl From<serde_json::Value> for DataValue {
|
|
fn from(value: serde_json::Value) -> Self {
|
|
match value {
|
|
serde_json::Value::Null => DataValue::Null,
|
|
serde_json::Value::Bool(b) => DataValue::Bool(b),
|
|
serde_json::Value::Number(n) => {
|
|
if let Some(i) = n.as_i64() {
|
|
DataValue::Int(i)
|
|
} else if let Some(f) = n.as_f64() {
|
|
DataValue::Float(f)
|
|
} else {
|
|
DataValue::Null
|
|
}
|
|
}
|
|
serde_json::Value::String(s) => DataValue::String(s),
|
|
serde_json::Value::Array(arr) => {
|
|
DataValue::Array(arr.into_iter().map(DataValue::from).collect())
|
|
}
|
|
serde_json::Value::Object(obj) => DataValue::Object(
|
|
obj.into_iter()
|
|
.map(|(k, v)| (k, DataValue::from(v)))
|
|
.collect(),
|
|
),
|
|
}
|
|
}
|
|
}
|
|
|
|
/// Configuration for validation operations
|
|
#[derive(Debug, Clone, Serialize, Deserialize)]
|
|
pub struct ValidationConfig {
|
|
/// Enable statistical validation
|
|
pub statistical_validation: bool,
|
|
/// Enable schema enforcement
|
|
pub schema_enforcement: bool,
|
|
/// Enable drift detection
|
|
pub drift_detection: bool,
|
|
/// Enable anomaly detection
|
|
pub anomaly_detection: bool,
|
|
/// Enable lineage tracking
|
|
pub lineage_tracking: bool,
|
|
/// Maximum number of validation errors to collect
|
|
pub max_errors: usize,
|
|
/// Validation timeout in milliseconds
|
|
pub timeout_ms: u64,
|
|
/// Enable performance metrics collection
|
|
pub collect_metrics: bool,
|
|
}
|
|
|
|
impl Default for ValidationConfig {
|
|
fn default() -> Self {
|
|
Self {
|
|
statistical_validation: true,
|
|
schema_enforcement: true,
|
|
drift_detection: false,
|
|
anomaly_detection: false,
|
|
lineage_tracking: false,
|
|
max_errors: 100,
|
|
timeout_ms: 5000,
|
|
collect_metrics: true,
|
|
}
|
|
}
|
|
}
|