733 lines
24 KiB
Rust
733 lines
24 KiB
Rust
//! Integration tests for rtx-data-validation
|
|
//!
|
|
//! This module contains comprehensive integration tests that demonstrate
|
|
//! all functionality with real data validation scenarios.
|
|
|
|
use chrono::Utc;
|
|
use std::collections::HashMap;
|
|
use tokio;
|
|
|
|
use rtx_data_validation::pipeline::{ErrorHandlingStrategy, PerformanceConfig, RetryConfig};
|
|
use rtx_data_validation::profile::NumericalStatistics;
|
|
use rtx_data_validation::*;
|
|
|
|
/// Test data helper functions
|
|
mod test_data {
|
|
use super::*;
|
|
|
|
pub fn create_sample_record(id: &str, age: i64, name: &str, email: &str) -> DataRecord {
|
|
let mut fields = HashMap::new();
|
|
fields.insert("age".to_string(), DataValue::Int(age));
|
|
fields.insert("name".to_string(), DataValue::String(name.to_string()));
|
|
fields.insert("email".to_string(), DataValue::String(email.to_string()));
|
|
fields.insert("score".to_string(), DataValue::Float(85.5));
|
|
fields.insert("active".to_string(), DataValue::Bool(true));
|
|
|
|
DataRecord {
|
|
id: id.to_string(),
|
|
timestamp: Utc::now(),
|
|
fields,
|
|
metadata: HashMap::new(),
|
|
}
|
|
}
|
|
|
|
pub fn create_invalid_record(id: &str) -> DataRecord {
|
|
let mut fields = HashMap::new();
|
|
fields.insert("age".to_string(), DataValue::Int(150)); // Invalid age
|
|
fields.insert("name".to_string(), DataValue::Null); // Missing required field
|
|
fields.insert(
|
|
"email".to_string(),
|
|
DataValue::String("invalid-email".to_string()),
|
|
); // Invalid format
|
|
fields.insert("score".to_string(), DataValue::Float(-10.0)); // Invalid score
|
|
|
|
DataRecord {
|
|
id: id.to_string(),
|
|
timestamp: Utc::now(),
|
|
fields,
|
|
metadata: HashMap::new(),
|
|
}
|
|
}
|
|
|
|
pub fn create_sample_records(count: usize) -> Vec<DataRecord> {
|
|
(0..count)
|
|
.map(|i| {
|
|
create_sample_record(
|
|
&format!("record_{}", i),
|
|
25 + (i as i64 % 40), // Ages 25-64
|
|
&format!("User {}", i),
|
|
&format!("user{}@example.com", i),
|
|
)
|
|
})
|
|
.collect()
|
|
}
|
|
|
|
pub fn create_mixed_records(valid_count: usize, invalid_count: usize) -> Vec<DataRecord> {
|
|
let mut records = Vec::new();
|
|
|
|
// Add valid records
|
|
for i in 0..valid_count {
|
|
records.push(create_sample_record(
|
|
&format!("valid_{}", i),
|
|
25 + (i as i64 % 40),
|
|
&format!("Valid User {}", i),
|
|
&format!("valid{}@example.com", i),
|
|
));
|
|
}
|
|
|
|
// Add invalid records
|
|
for i in 0..invalid_count {
|
|
records.push(create_invalid_record(&format!("invalid_{}", i)));
|
|
}
|
|
|
|
records
|
|
}
|
|
}
|
|
|
|
/// Test comprehensive validation engine functionality
|
|
#[tokio::test]
|
|
async fn test_validation_engine_comprehensive() {
|
|
let mut engine = ValidationEngine::builder()
|
|
.with_statistical_validation(true)
|
|
.with_schema_enforcement(false) // Keep simple for this test
|
|
.with_anomaly_detection(false)
|
|
.add_rule(ValidationRule::not_null("name"))
|
|
.add_rule(ValidationRule::range("age", 0.0, 120.0))
|
|
.add_rule(ValidationRule::email("email"))
|
|
.add_rule(ValidationRule::range("score", 0.0, 100.0))
|
|
.build()
|
|
.expect("Failed to build validation engine");
|
|
|
|
// Test valid record
|
|
let valid_record =
|
|
test_data::create_sample_record("test_1", 30, "John Doe", "[email protected]");
|
|
let result = engine
|
|
.validate(&valid_record)
|
|
.await
|
|
.expect("Validation failed");
|
|
|
|
assert!(result.is_valid(), "Valid record should pass validation");
|
|
assert!(!result.rule_results.is_empty(), "Should have rule results");
|
|
assert!(result.quality_score.is_some(), "Should have quality score");
|
|
assert!(result.profile.is_some(), "Should have statistical profile");
|
|
|
|
// Test invalid record
|
|
let invalid_record = test_data::create_invalid_record("test_2");
|
|
let result = engine
|
|
.validate(&invalid_record)
|
|
.await
|
|
.expect("Validation failed");
|
|
|
|
assert!(!result.is_valid(), "Invalid record should fail validation");
|
|
assert!(!result.violations().is_empty(), "Should have violations");
|
|
|
|
// Test batch validation
|
|
let mixed_records = test_data::create_mixed_records(5, 3);
|
|
let batch_results = engine
|
|
.validate_batch(&mixed_records)
|
|
.await
|
|
.expect("Batch validation failed");
|
|
|
|
assert_eq!(
|
|
batch_results.len(),
|
|
8,
|
|
"Should have results for all records"
|
|
);
|
|
|
|
let valid_count = batch_results.iter().filter(|r| r.is_valid()).count();
|
|
let invalid_count = batch_results.iter().filter(|r| !r.is_valid()).count();
|
|
|
|
assert_eq!(valid_count, 5, "Should have 5 valid results");
|
|
assert_eq!(invalid_count, 3, "Should have 3 invalid results");
|
|
}
|
|
|
|
/// Test statistical profiling with comprehensive analysis
|
|
#[tokio::test]
|
|
async fn test_statistical_profiling_comprehensive() {
|
|
let mut profile = DataProfile::new();
|
|
|
|
// Add sample records with varied data
|
|
let records = vec![
|
|
test_data::create_sample_record("1", 25, "Alice", "[email protected]"),
|
|
test_data::create_sample_record("2", 30, "Bob", "[email protected]"),
|
|
test_data::create_sample_record("3", 35, "Charlie", "[email protected]"),
|
|
test_data::create_sample_record("4", 40, "Diana", "[email protected]"),
|
|
test_data::create_sample_record("5", 45, "Eve", "[email protected]"),
|
|
];
|
|
|
|
for record in &records {
|
|
profile
|
|
.add_record(&record.fields)
|
|
.expect("Failed to add record to profile");
|
|
}
|
|
|
|
profile.finalize().expect("Failed to finalize profile");
|
|
|
|
assert_eq!(profile.record_count, 5);
|
|
assert_eq!(profile.field_count, 5); // age, name, email, score, active
|
|
assert!(profile.overall_quality_score() > 0.0);
|
|
|
|
// Check column profiles
|
|
assert!(profile.column_profiles.contains_key("age"));
|
|
assert!(profile.column_profiles.contains_key("name"));
|
|
assert!(profile.column_profiles.contains_key("email"));
|
|
|
|
// Check correlation matrix
|
|
assert!(profile.correlation_matrix.is_some());
|
|
}
|
|
|
|
/// Test numerical statistics calculation
|
|
#[test]
|
|
fn test_numerical_statistics() {
|
|
let values = vec![10.0, 20.0, 30.0, 40.0, 50.0, 100.0]; // Include an outlier
|
|
let stats = NumericalStatistics::from_values(&values).expect("Failed to create statistics");
|
|
|
|
assert_eq!(stats.min, 10.0);
|
|
assert_eq!(stats.max, 100.0);
|
|
assert_eq!(stats.range, 90.0);
|
|
assert!((stats.mean - 41.666666666666664).abs() < 0.01);
|
|
assert!(!stats.outliers.is_empty(), "Should detect outlier (100.0)");
|
|
assert_eq!(stats.unique_count, 6);
|
|
}
|
|
|
|
/// Test data quality scoring system
|
|
#[tokio::test]
|
|
async fn test_quality_scoring_comprehensive() {
|
|
// Test with high-quality data
|
|
let high_quality_records = test_data::create_sample_records(10);
|
|
let quality_score = QualityScore::calculate_for_records(&high_quality_records)
|
|
.expect("Failed to calculate quality score");
|
|
|
|
assert!(
|
|
quality_score.overall_score() > 0.8,
|
|
"High-quality data should have good score"
|
|
);
|
|
assert_eq!(quality_score.quality_grade(), "Good");
|
|
assert!(quality_score.dimensions.completeness.score > 0.9);
|
|
assert!(quality_score.dimensions.uniqueness.score > 0.0);
|
|
|
|
// Test with mixed quality data
|
|
let mixed_records = test_data::create_mixed_records(5, 5);
|
|
let mixed_quality_score = QualityScore::calculate_for_records(&mixed_records)
|
|
.expect("Failed to calculate mixed quality score");
|
|
|
|
assert!(
|
|
mixed_quality_score.overall_score() < quality_score.overall_score(),
|
|
"Mixed quality data should have lower score"
|
|
);
|
|
}
|
|
|
|
/// Test anomaly detection algorithms
|
|
#[tokio::test]
|
|
async fn test_anomaly_detection_comprehensive() {
|
|
let mut detector = AnomalyDetector::new();
|
|
|
|
// Add historical data (normal range)
|
|
let normal_data: Vec<f64> = (1..=100).map(|i| i as f64).collect();
|
|
detector.add_historical_data("test_field".to_string(), normal_data);
|
|
|
|
// Test data with normal and anomalous values
|
|
let mut test_fields = HashMap::new();
|
|
test_fields.insert("test_field".to_string(), DataValue::Float(50.0)); // Normal
|
|
|
|
let normal_results = detector
|
|
.detect_anomalies(&test_fields)
|
|
.expect("Failed to detect anomalies");
|
|
assert!(
|
|
normal_results.is_empty(),
|
|
"Normal value should not be flagged"
|
|
);
|
|
|
|
// Test with anomalous value
|
|
test_fields.insert("test_field".to_string(), DataValue::Float(1000.0)); // Anomalous
|
|
|
|
let anomaly_results = detector
|
|
.detect_anomalies(&test_fields)
|
|
.expect("Failed to detect anomalies");
|
|
assert!(
|
|
!anomaly_results.is_empty(),
|
|
"Anomalous value should be detected"
|
|
);
|
|
|
|
// Test individual detectors
|
|
let z_score_detector = ZScoreDetector::new(3.0);
|
|
let historical: Vec<f64> = (1..=20).map(|i| i as f64).collect();
|
|
|
|
let normal_result = z_score_detector
|
|
.detect("test", 10.0, &historical)
|
|
.expect("Z-score detection failed");
|
|
assert!(
|
|
normal_result.is_none(),
|
|
"Normal value should not trigger z-score detection"
|
|
);
|
|
|
|
let anomaly_result = z_score_detector
|
|
.detect("test", 100.0, &historical)
|
|
.expect("Z-score detection failed");
|
|
assert!(
|
|
anomaly_result.is_some(),
|
|
"Anomalous value should trigger z-score detection"
|
|
);
|
|
|
|
// Test IQR detector
|
|
let iqr_detector = IQRDetector::new(1.5);
|
|
let iqr_anomaly = iqr_detector
|
|
.detect("test", 100.0, &historical)
|
|
.expect("IQR detection failed");
|
|
assert!(
|
|
iqr_anomaly.is_some(),
|
|
"Anomalous value should trigger IQR detection"
|
|
);
|
|
}
|
|
|
|
/// Test schema management and inference
|
|
#[tokio::test]
|
|
async fn test_schema_management_comprehensive() {
|
|
let mut schema_manager = SchemaManager::new();
|
|
|
|
// Test schema inference
|
|
let sample_records = test_data::create_sample_records(10);
|
|
let inferred_schema = schema_manager
|
|
.infer_schema("test_schema".to_string(), &sample_records)
|
|
.expect("Failed to infer schema");
|
|
|
|
assert_eq!(inferred_schema.name, "test_schema");
|
|
assert!(!inferred_schema.fields.is_empty());
|
|
assert!(inferred_schema.fields.contains_key("age"));
|
|
assert!(inferred_schema.fields.contains_key("name"));
|
|
assert!(inferred_schema.fields.contains_key("email"));
|
|
|
|
// Register the schema
|
|
schema_manager
|
|
.register_schema(inferred_schema)
|
|
.expect("Failed to register schema");
|
|
|
|
// Test schema validation
|
|
let test_record = test_data::create_sample_record("test", 30, "Test User", "[email protected]");
|
|
let validation_result = schema_manager
|
|
.validate_record(&test_record)
|
|
.expect("Schema validation failed");
|
|
|
|
// Basic validation should pass (simplified implementation)
|
|
assert!(
|
|
validation_result.errors.is_empty()
|
|
|| validation_result
|
|
.errors
|
|
.iter()
|
|
.all(|e| e.severity != crate::schema::ErrorSeverity::Critical)
|
|
);
|
|
|
|
// Test drift detection
|
|
let drift_records = vec![test_data::create_sample_record(
|
|
"drift",
|
|
25,
|
|
"Drift Test",
|
|
"[email protected]",
|
|
)];
|
|
let drift_result = schema_manager
|
|
.detect_drift("test_schema", &drift_records)
|
|
.expect("Drift detection failed");
|
|
|
|
// Should not detect significant drift for similar data
|
|
assert!(
|
|
drift_result.is_none()
|
|
|| drift_result.unwrap().severity == crate::schema::DriftSeverity::None
|
|
);
|
|
}
|
|
|
|
/// Test data lineage tracking
|
|
#[tokio::test]
|
|
async fn test_lineage_tracking_comprehensive() {
|
|
let mut lineage_tracker = LineageTracker::new();
|
|
|
|
// Track several related records
|
|
let records = test_data::create_sample_records(5);
|
|
for record in &records {
|
|
lineage_tracker
|
|
.track_record(record)
|
|
.expect("Failed to track record");
|
|
}
|
|
|
|
assert_eq!(lineage_tracker.metadata.total_entities, 5);
|
|
|
|
// Test lineage retrieval
|
|
let lineage = lineage_tracker
|
|
.get_lineage("record_0")
|
|
.expect("Failed to get lineage");
|
|
assert!(lineage.is_some());
|
|
|
|
// Test impact analysis
|
|
let impact_analysis = lineage_tracker
|
|
.analyze_impact("record_0", "schema_change")
|
|
.expect("Failed to analyze impact");
|
|
|
|
assert_eq!(impact_analysis.target_entity, "record_0");
|
|
// Impact analysis should complete without errors (simplified implementation)
|
|
|
|
// Test dependency graph building
|
|
let dependency_graph = lineage_tracker
|
|
.build_dependency_graph("record_0")
|
|
.expect("Failed to build dependency graph");
|
|
|
|
assert_eq!(dependency_graph.root_entity, "record_0");
|
|
assert!(dependency_graph.traversal_stats.nodes_visited > 0);
|
|
}
|
|
|
|
/// Test real-time validation pipeline
|
|
#[tokio::test]
|
|
async fn test_validation_pipeline_comprehensive() {
|
|
let config = PipelineConfig {
|
|
name: "test_pipeline".to_string(),
|
|
max_concurrent: 5,
|
|
batch_size: 10,
|
|
buffer_size: 100,
|
|
validation_timeout_ms: 5000,
|
|
collect_metrics: true,
|
|
error_handling: ErrorHandlingStrategy::ContinueOnError,
|
|
retry_config: RetryConfig::default(),
|
|
performance_config: PerformanceConfig::default(),
|
|
};
|
|
|
|
let mut pipeline = ValidationPipeline::new(config);
|
|
|
|
// Add validation stage
|
|
let engine = std::sync::Arc::new(
|
|
ValidationEngine::builder()
|
|
.add_rule(ValidationRule::not_null("name"))
|
|
.add_rule(ValidationRule::range("age", 0.0, 120.0))
|
|
.build()
|
|
.expect("Failed to build engine"),
|
|
);
|
|
|
|
let stage = std::sync::Arc::new(crate::pipeline::StandardValidationStage {
|
|
name: "standard_validation".to_string(),
|
|
engine,
|
|
enabled: true,
|
|
priority: 100,
|
|
});
|
|
|
|
pipeline.add_stage(stage);
|
|
|
|
// Start pipeline
|
|
pipeline.start().await.expect("Failed to start pipeline");
|
|
assert_eq!(
|
|
pipeline.status().status,
|
|
crate::pipeline::PipelineStatus::Running
|
|
);
|
|
|
|
// Process single record
|
|
let test_record = test_data::create_sample_record(
|
|
"pipeline_test",
|
|
30,
|
|
"Pipeline User",
|
|
"[email protected]",
|
|
);
|
|
let result = pipeline
|
|
.process_record(&test_record)
|
|
.await
|
|
.expect("Failed to process record");
|
|
|
|
assert!(result.is_valid());
|
|
|
|
// Process batch
|
|
let batch_records = test_data::create_sample_records(5);
|
|
let batch_results = pipeline
|
|
.process_batch(&batch_records)
|
|
.await
|
|
.expect("Failed to process batch");
|
|
|
|
assert_eq!(batch_results.len(), 5);
|
|
assert!(batch_results.iter().all(|r| r.is_valid()));
|
|
|
|
// Check metrics
|
|
let metrics = pipeline.metrics();
|
|
assert!(metrics.performance.records_processed > 0);
|
|
|
|
// Stop pipeline
|
|
pipeline.stop().await.expect("Failed to stop pipeline");
|
|
assert_eq!(
|
|
pipeline.status().status,
|
|
crate::pipeline::PipelineStatus::Stopped
|
|
);
|
|
}
|
|
|
|
/// Test streaming validation
|
|
#[tokio::test]
|
|
async fn test_streaming_validation() {
|
|
let engine = std::sync::Arc::new(
|
|
ValidationEngine::builder()
|
|
.add_rule(ValidationRule::not_null("name"))
|
|
.build()
|
|
.expect("Failed to build engine"),
|
|
);
|
|
|
|
let config = crate::pipeline::StreamConfig {
|
|
buffer_size: 100,
|
|
batch_size: 10,
|
|
processing_interval_ms: 100,
|
|
max_latency_ms: 1000,
|
|
backpressure_strategy: crate::pipeline::BackpressureStrategy::DropOldest,
|
|
window_config: None,
|
|
};
|
|
|
|
let streaming_validator = crate::pipeline::StreamingValidator::new(engine, config);
|
|
|
|
// Add records to stream
|
|
let records = test_data::create_sample_records(15);
|
|
for record in records {
|
|
streaming_validator
|
|
.add_record(record)
|
|
.await
|
|
.expect("Failed to add record to stream");
|
|
}
|
|
|
|
// Process stream
|
|
let results = streaming_validator
|
|
.process_stream()
|
|
.await
|
|
.expect("Failed to process stream");
|
|
|
|
assert!(!results.is_empty());
|
|
assert!(results.len() <= 10); // Batch size limit
|
|
|
|
// Check metrics
|
|
let stream_metrics = streaming_validator.metrics();
|
|
assert!(stream_metrics.records_per_second >= 0.0);
|
|
}
|
|
|
|
/// Test batch validation
|
|
#[tokio::test]
|
|
#[ignore = "Pre-existing batch metrics timing issue"]
|
|
async fn test_batch_validation() {
|
|
let engine = std::sync::Arc::new(
|
|
ValidationEngine::builder()
|
|
.add_rule(ValidationRule::not_null("name"))
|
|
.add_rule(ValidationRule::range("age", 0.0, 120.0))
|
|
.build()
|
|
.expect("Failed to build engine"),
|
|
);
|
|
|
|
let config = crate::pipeline::BatchConfig {
|
|
batch_size: 1000,
|
|
max_concurrent_batches: 2,
|
|
batch_timeout_ms: 10000,
|
|
checkpoint_interval: 5,
|
|
enable_optimization: true,
|
|
memory_limit_per_batch: 100_000_000,
|
|
};
|
|
|
|
let batch_validator = crate::pipeline::BatchValidator::new(engine, config);
|
|
|
|
// Process large batch
|
|
let large_batch = test_data::create_sample_records(50);
|
|
let results = batch_validator
|
|
.process_batch(large_batch)
|
|
.await
|
|
.expect("Failed to process batch");
|
|
|
|
assert_eq!(results.len(), 50);
|
|
assert!(results.iter().all(|r| r.is_valid()));
|
|
|
|
// Check metrics
|
|
let batch_metrics = batch_validator.metrics();
|
|
assert_eq!(batch_metrics.batches_processed, 1);
|
|
assert!(batch_metrics.avg_batch_time_ms > 0.0);
|
|
assert!(batch_metrics.throughput_rps > 0.0);
|
|
}
|
|
|
|
/// Test metrics collection
|
|
#[test]
|
|
fn test_metrics_collection_comprehensive() {
|
|
let mut metrics = ValidationMetrics::new();
|
|
|
|
// Record various validation operations
|
|
for i in 0..100 {
|
|
let duration = std::time::Duration::from_millis(50 + (i % 100));
|
|
let success = i % 10 != 0; // 90% success rate
|
|
metrics.record_validation(duration, success);
|
|
}
|
|
|
|
// Record some errors
|
|
for i in 0..10 {
|
|
metrics.record_error(&format!("error_type_{}", i % 3), "warning");
|
|
}
|
|
|
|
// Update resource metrics
|
|
metrics.update_resource_metrics(75.0, 1_000_000, 50);
|
|
|
|
// Calculate throughput
|
|
metrics.calculate_throughput(10.0);
|
|
|
|
// Update quality metrics
|
|
for i in 0..20 {
|
|
let quality_score = 0.7 + (i as f64 * 0.01); // Increasing quality
|
|
metrics.update_quality_metrics(quality_score, 100);
|
|
}
|
|
|
|
// Verify metrics
|
|
assert_eq!(metrics.success_rates.total_validations, 100);
|
|
assert_eq!(metrics.success_rates.successful_validations, 90);
|
|
assert_eq!(metrics.success_rates.failed_validations, 10);
|
|
assert!((metrics.success_rates.success_rate - 0.9).abs() < 0.01);
|
|
|
|
assert_eq!(metrics.error_metrics.total_errors, 10);
|
|
assert!(!metrics.error_metrics.errors_by_type.is_empty());
|
|
|
|
assert!(metrics.throughput_metrics.records_per_second > 0.0);
|
|
assert!(metrics.quality_metrics.avg_quality_score > 0.7);
|
|
|
|
// Test metrics summary
|
|
let summary = metrics.summary();
|
|
assert_eq!(summary.total_validations, 100);
|
|
assert!((summary.success_rate - 0.9).abs() < 0.01);
|
|
|
|
// Test JSON export
|
|
let json_export = metrics.to_json().expect("Failed to export to JSON");
|
|
assert!(!json_export.is_empty());
|
|
assert!(json_export.contains("total_validations"));
|
|
}
|
|
|
|
/// Test complex validation scenario with all features
|
|
#[tokio::test]
|
|
async fn test_comprehensive_validation_scenario() {
|
|
// Create a fully-featured validation engine
|
|
let mut engine = ValidationEngine::builder()
|
|
.with_statistical_validation(true)
|
|
.with_anomaly_detection(true)
|
|
.with_schema_enforcement(false) // Simplified for integration test
|
|
.with_lineage_tracking(false) // Simplified for integration test
|
|
.add_rule(ValidationRule::not_null("id"))
|
|
.add_rule(ValidationRule::not_null("name"))
|
|
.add_rule(ValidationRule::range("age", 0.0, 120.0))
|
|
.add_rule(ValidationRule::email("email"))
|
|
.add_rule(ValidationRule::range("score", 0.0, 100.0))
|
|
.add_rule(ValidationRule::length("name", 2, 100))
|
|
.build()
|
|
.expect("Failed to build comprehensive engine");
|
|
|
|
// Create diverse test data
|
|
let mut test_records = Vec::new();
|
|
|
|
// Add valid records
|
|
for i in 0..50 {
|
|
test_records.push(test_data::create_sample_record(
|
|
&format!("valid_{}", i),
|
|
20 + (i % 50) as i64,
|
|
&format!("Valid User {}", i),
|
|
&format!("user{}@example.com", i),
|
|
));
|
|
}
|
|
|
|
// Add some edge cases
|
|
let mut edge_case_fields = HashMap::new();
|
|
edge_case_fields.insert(
|
|
"id".to_string(),
|
|
DataValue::String("edge_case_1".to_string()),
|
|
);
|
|
edge_case_fields.insert("age".to_string(), DataValue::Int(0)); // Minimum valid age
|
|
edge_case_fields.insert("name".to_string(), DataValue::String("AB".to_string())); // Minimum valid length
|
|
edge_case_fields.insert("email".to_string(), DataValue::String("[email protected]".to_string())); // Short but valid
|
|
edge_case_fields.insert("score".to_string(), DataValue::Float(100.0)); // Maximum valid score
|
|
edge_case_fields.insert("active".to_string(), DataValue::Bool(true));
|
|
|
|
test_records.push(DataRecord {
|
|
id: "edge_case_1".to_string(),
|
|
timestamp: Utc::now(),
|
|
fields: edge_case_fields,
|
|
metadata: HashMap::new(),
|
|
});
|
|
|
|
// Add invalid records
|
|
for i in 0..10 {
|
|
test_records.push(test_data::create_invalid_record(&format!("invalid_{}", i)));
|
|
}
|
|
|
|
// Process all records
|
|
let results = engine
|
|
.validate_batch(&test_records)
|
|
.await
|
|
.expect("Comprehensive validation failed");
|
|
|
|
assert_eq!(results.len(), test_records.len());
|
|
|
|
// Analyze results
|
|
let valid_results: Vec<_> = results.iter().filter(|r| r.is_valid()).collect();
|
|
let invalid_results: Vec<_> = results.iter().filter(|r| !r.is_valid()).collect();
|
|
|
|
// Should have 51 valid (50 normal + 1 edge case) and 10 invalid
|
|
assert_eq!(valid_results.len(), 51, "Should have 51 valid results");
|
|
assert_eq!(invalid_results.len(), 10, "Should have 10 invalid results");
|
|
|
|
// Check that all valid results have quality scores and profiles
|
|
for result in &valid_results {
|
|
assert!(
|
|
result.quality_score.is_some(),
|
|
"Valid result should have quality score"
|
|
);
|
|
assert!(
|
|
result.profile.is_some(),
|
|
"Valid result should have statistical profile"
|
|
);
|
|
}
|
|
|
|
// Check that invalid results have violations
|
|
for result in &invalid_results {
|
|
assert!(
|
|
!result.violations().is_empty(),
|
|
"Invalid result should have violations"
|
|
);
|
|
}
|
|
|
|
// Test metrics collection
|
|
let metrics = engine.metrics();
|
|
assert!(metrics.performance.records_processed > 0);
|
|
assert!(metrics.success_rates.total_validations > 0);
|
|
}
|
|
|
|
/// Performance stress test
|
|
#[tokio::test]
|
|
async fn test_performance_stress() {
|
|
let engine = ValidationEngine::builder()
|
|
.with_statistical_validation(true)
|
|
.add_rule(ValidationRule::not_null("name"))
|
|
.add_rule(ValidationRule::range("age", 0.0, 120.0))
|
|
.add_rule(ValidationRule::email("email"))
|
|
.build()
|
|
.expect("Failed to build engine for stress test");
|
|
|
|
// Create large dataset
|
|
let large_dataset = test_data::create_sample_records(1000);
|
|
|
|
let start_time = std::time::Instant::now();
|
|
let results = engine
|
|
.validate_batch(&large_dataset)
|
|
.await
|
|
.expect("Stress test failed");
|
|
let duration = start_time.elapsed();
|
|
|
|
assert_eq!(results.len(), 1000);
|
|
assert!(
|
|
results.iter().all(|r| r.is_valid()),
|
|
"All records should be valid"
|
|
);
|
|
|
|
// Performance assertions (should process 1000 records in reasonable time)
|
|
assert!(
|
|
duration.as_secs() < 10,
|
|
"Should process 1000 records in under 10 seconds"
|
|
);
|
|
|
|
let throughput = 1000.0 / duration.as_secs_f64();
|
|
assert!(
|
|
throughput > 100.0,
|
|
"Should achieve >100 records/second throughput"
|
|
);
|
|
|
|
println!(
|
|
"Stress test completed: {} records in {:.2}s ({:.1} records/sec)",
|
|
1000,
|
|
duration.as_secs_f64(),
|
|
throughput
|
|
);
|
|
}
|