2283 lines
70 KiB
Rust
2283 lines
70 KiB
Rust
|
|
use std::collections::{HashMap, BTreeMap, VecDeque};
|
|
use std::sync::Arc;
|
|
use std::time::{Duration, Instant};
|
|
use serde::{Deserialize, Serialize};
|
|
use tokio::sync::{RwLock, Mutex};
|
|
use tracing::{info, instrument};
|
|
use uuid::Uuid;
|
|
use chrono::{DateTime, Utc};
|
|
use sha2::{Sha256, Digest};
|
|
|
|
use crate::{
|
|
Result, EtlError, DataRecord, DataValue,
|
|
state::StateManager, monitoring::EtlMetrics,
|
|
};
|
|
|
|
/// Configuration for incremental processing
|
|
#[derive(Debug, Clone, Serialize, Deserialize)]
|
|
pub struct IncrementalConfig {
|
|
/// Change detection strategy
|
|
pub change_detection: ChangeDetectionStrategy,
|
|
/// Checkpoint configuration
|
|
pub checkpoint_config: CheckpointConfig,
|
|
/// Deduplication strategy
|
|
pub deduplication: DeduplicationStrategy,
|
|
/// State retention policy
|
|
pub state_retention: StateRetentionPolicy,
|
|
/// Batch processing configuration
|
|
pub batch_config: BatchConfig,
|
|
/// Memory management configuration
|
|
pub memory_config: MemoryConfig,
|
|
/// Parallel processing configuration
|
|
pub parallelism: ParallelismConfig,
|
|
}
|
|
|
|
impl Default for IncrementalConfig {
|
|
fn default() -> Self {
|
|
Self {
|
|
change_detection: ChangeDetectionStrategy::TimestampBased {
|
|
column: "updated_at".to_string(),
|
|
format: TimestampFormat::Iso8601,
|
|
},
|
|
checkpoint_config: CheckpointConfig::default(),
|
|
deduplication: DeduplicationStrategy::Hash {
|
|
hash_columns: vec!["id".to_string()],
|
|
hash_algorithm: HashAlgorithm::Sha256,
|
|
},
|
|
state_retention: StateRetentionPolicy::TimeBased {
|
|
retention_period: Duration::from_secs(7 * 24 * 3600), // 7 days
|
|
},
|
|
batch_config: BatchConfig::default(),
|
|
memory_config: MemoryConfig::default(),
|
|
parallelism: ParallelismConfig::default(),
|
|
}
|
|
}
|
|
}
|
|
|
|
/// Incremental processor for handling change data capture and delta processing
|
|
pub struct IncrementalProcessor {
|
|
/// Configuration
|
|
config: IncrementalConfig,
|
|
/// State manager for persistence
|
|
state_manager: Arc<StateManager>,
|
|
/// Metrics collector
|
|
metrics: Arc<EtlMetrics>,
|
|
/// Change detector
|
|
change_detector: Arc<ChangeDetector>,
|
|
/// Checkpoint manager
|
|
checkpoint_manager: Arc<CheckpointManager>,
|
|
/// Deduplicator
|
|
deduplicator: Arc<Deduplicator>,
|
|
/// State tracker for incremental processing
|
|
state_tracker: Arc<RwLock<StateTracker>>,
|
|
/// Watermark manager
|
|
watermark_manager: Arc<WatermarkManager>,
|
|
/// Merge processor
|
|
merge_processor: Arc<MergeProcessor>,
|
|
}
|
|
|
|
/// Strategies for detecting changes in data
|
|
#[derive(Debug, Clone, Serialize, Deserialize)]
|
|
pub enum ChangeDetectionStrategy {
|
|
/// Timestamp-based change detection
|
|
TimestampBased {
|
|
/// Column name containing timestamp
|
|
column: String,
|
|
/// Timestamp format
|
|
format: TimestampFormat,
|
|
},
|
|
/// Version-based change detection
|
|
VersionBased {
|
|
/// Column name containing version
|
|
column: String,
|
|
/// Version comparison strategy
|
|
comparison: VersionComparison,
|
|
},
|
|
/// Hash-based change detection
|
|
HashBased {
|
|
/// Columns to include in hash
|
|
columns: Vec<String>,
|
|
/// Hash algorithm to use
|
|
algorithm: HashAlgorithm,
|
|
},
|
|
/// Log-based change detection (CDC)
|
|
LogBased {
|
|
/// Log source configuration
|
|
log_source: LogSourceConfig,
|
|
},
|
|
/// Trigger-based change detection
|
|
TriggerBased {
|
|
/// Trigger configuration
|
|
trigger_config: TriggerConfig,
|
|
},
|
|
}
|
|
|
|
/// Timestamp format options
|
|
#[derive(Debug, Clone, Serialize, Deserialize)]
|
|
pub enum TimestampFormat {
|
|
/// ISO 8601 format
|
|
Iso8601,
|
|
/// Unix timestamp (seconds)
|
|
UnixSeconds,
|
|
/// Unix timestamp (milliseconds)
|
|
UnixMilliseconds,
|
|
/// Custom format string
|
|
Custom(String),
|
|
}
|
|
|
|
/// Version comparison strategies
|
|
#[derive(Debug, Clone, Serialize, Deserialize)]
|
|
pub enum VersionComparison {
|
|
/// Numeric comparison
|
|
Numeric,
|
|
/// Semantic versioning
|
|
Semantic,
|
|
/// String comparison
|
|
Lexicographic,
|
|
}
|
|
|
|
/// Hash algorithms for change detection
|
|
#[derive(Debug, Clone, Serialize, Deserialize)]
|
|
pub enum HashAlgorithm {
|
|
/// MD5 hash
|
|
Md5,
|
|
/// SHA-256 hash
|
|
Sha256,
|
|
/// SHA-512 hash
|
|
Sha512,
|
|
/// CRC32 checksum
|
|
Crc32,
|
|
}
|
|
|
|
/// Log source configuration for CDC
|
|
#[derive(Debug, Clone, Serialize, Deserialize)]
|
|
pub struct LogSourceConfig {
|
|
/// Log source type
|
|
pub source_type: LogSourceType,
|
|
/// Connection configuration
|
|
pub connection: LogConnectionConfig,
|
|
/// Filter configuration
|
|
pub filters: Vec<LogFilter>,
|
|
}
|
|
|
|
/// Types of log sources for CDC
|
|
#[derive(Debug, Clone, Serialize, Deserialize)]
|
|
pub enum LogSourceType {
|
|
/// Database transaction log
|
|
DatabaseLog {
|
|
/// Database type
|
|
db_type: DatabaseType,
|
|
},
|
|
/// Kafka changelog topic
|
|
KafkaChangelog {
|
|
/// Topic name
|
|
topic: String,
|
|
},
|
|
/// File-based changelog
|
|
FileChangelog {
|
|
/// File pattern
|
|
pattern: String,
|
|
},
|
|
}
|
|
|
|
/// Database types for log-based CDC
|
|
#[derive(Debug, Clone, Serialize, Deserialize)]
|
|
pub enum DatabaseType {
|
|
/// `PostgreSQL`
|
|
PostgreSQL,
|
|
/// `MySQL`
|
|
MySQL,
|
|
/// SQL Server
|
|
SqlServer,
|
|
/// Oracle
|
|
Oracle,
|
|
/// `MongoDB`
|
|
MongoDB,
|
|
}
|
|
|
|
/// Connection configuration for log sources
|
|
#[derive(Debug, Clone, Serialize, Deserialize)]
|
|
pub struct LogConnectionConfig {
|
|
/// Connection string
|
|
pub connection_string: String,
|
|
/// Authentication configuration
|
|
pub auth: Option<AuthConfig>,
|
|
/// Connection pool configuration
|
|
pub pool_config: Option<PoolConfig>,
|
|
}
|
|
|
|
/// Authentication configuration
|
|
#[derive(Debug, Clone, Serialize, Deserialize)]
|
|
pub struct AuthConfig {
|
|
/// Username
|
|
pub username: String,
|
|
/// Password
|
|
pub password: String,
|
|
/// Additional authentication parameters
|
|
pub params: HashMap<String, String>,
|
|
}
|
|
|
|
/// Connection pool configuration
|
|
#[derive(Debug, Clone, Serialize, Deserialize)]
|
|
pub struct PoolConfig {
|
|
/// Maximum connections
|
|
pub max_connections: u32,
|
|
/// Minimum connections
|
|
pub min_connections: u32,
|
|
/// Connection timeout
|
|
pub connection_timeout: Duration,
|
|
}
|
|
|
|
/// Log filter configuration
|
|
#[derive(Debug, Clone, Serialize, Deserialize)]
|
|
pub struct LogFilter {
|
|
/// Filter type
|
|
pub filter_type: LogFilterType,
|
|
/// Filter expression
|
|
pub expression: String,
|
|
}
|
|
|
|
/// Types of log filters
|
|
#[derive(Debug, Clone, Serialize, Deserialize)]
|
|
pub enum LogFilterType {
|
|
/// Table/collection filter
|
|
Table,
|
|
/// Operation filter (INSERT, UPDATE, DELETE)
|
|
Operation,
|
|
/// Column filter
|
|
Column,
|
|
/// Custom filter
|
|
Custom,
|
|
}
|
|
|
|
/// Trigger configuration for change detection
|
|
#[derive(Debug, Clone, Serialize, Deserialize)]
|
|
pub struct TriggerConfig {
|
|
/// Trigger source
|
|
pub source: TriggerSource,
|
|
/// Trigger conditions
|
|
pub conditions: Vec<TriggerCondition>,
|
|
}
|
|
|
|
/// Trigger source types
|
|
#[derive(Debug, Clone, Serialize, Deserialize)]
|
|
pub enum TriggerSource {
|
|
/// Database trigger
|
|
Database {
|
|
/// Table name
|
|
table: String,
|
|
/// Trigger events
|
|
events: Vec<TriggerEvent>,
|
|
},
|
|
/// File system watcher
|
|
FileSystem {
|
|
/// Path to watch
|
|
path: String,
|
|
/// File patterns
|
|
patterns: Vec<String>,
|
|
},
|
|
/// External API webhook
|
|
Webhook {
|
|
/// Webhook endpoint
|
|
endpoint: String,
|
|
},
|
|
}
|
|
|
|
/// Database trigger events
|
|
#[derive(Debug, Clone, Serialize, Deserialize)]
|
|
pub enum TriggerEvent {
|
|
/// Insert event
|
|
Insert,
|
|
/// Update event
|
|
Update,
|
|
/// Delete event
|
|
Delete,
|
|
}
|
|
|
|
/// Trigger condition
|
|
#[derive(Debug, Clone, Serialize, Deserialize)]
|
|
pub struct TriggerCondition {
|
|
/// Column name
|
|
pub column: String,
|
|
/// Condition operator
|
|
pub operator: ComparisonOperator,
|
|
/// Value to compare against
|
|
pub value: DataValue,
|
|
}
|
|
|
|
/// Comparison operators for conditions
|
|
#[derive(Debug, Clone, Serialize, Deserialize)]
|
|
pub enum ComparisonOperator {
|
|
/// Equal
|
|
Eq,
|
|
/// Not equal
|
|
Ne,
|
|
/// Greater than
|
|
Gt,
|
|
/// Greater than or equal
|
|
Gte,
|
|
/// Less than
|
|
Lt,
|
|
/// Less than or equal
|
|
Lte,
|
|
/// In list
|
|
In,
|
|
/// Not in list
|
|
NotIn,
|
|
/// Like pattern
|
|
Like,
|
|
/// Regex match
|
|
Regex,
|
|
}
|
|
|
|
/// Configuration for checkpointing
|
|
#[derive(Debug, Clone, Serialize, Deserialize)]
|
|
pub struct CheckpointConfig {
|
|
/// Checkpoint strategy
|
|
pub strategy: CheckpointStrategy,
|
|
/// Checkpoint interval
|
|
pub interval: Duration,
|
|
/// Maximum checkpoint size
|
|
pub max_size_bytes: usize,
|
|
/// Checkpoint retention
|
|
pub retention: CheckpointRetention,
|
|
/// Compression configuration
|
|
pub compression: Option<CompressionConfig>,
|
|
}
|
|
|
|
impl Default for CheckpointConfig {
|
|
fn default() -> Self {
|
|
Self {
|
|
strategy: CheckpointStrategy::Periodic,
|
|
interval: Duration::from_secs(300), // 5 minutes
|
|
max_size_bytes: 100 * 1024 * 1024, // 100 MB
|
|
retention: CheckpointRetention::Count(10),
|
|
compression: Some(CompressionConfig {
|
|
algorithm: CompressionAlgorithm::Gzip,
|
|
level: 6,
|
|
}),
|
|
}
|
|
}
|
|
}
|
|
|
|
/// Checkpoint strategies
|
|
#[derive(Debug, Clone, Serialize, Deserialize)]
|
|
pub enum CheckpointStrategy {
|
|
/// Periodic checkpointing
|
|
Periodic,
|
|
/// Size-based checkpointing
|
|
SizeBased {
|
|
/// Threshold size in bytes
|
|
threshold_bytes: usize,
|
|
},
|
|
/// Count-based checkpointing
|
|
CountBased {
|
|
/// Record count threshold
|
|
threshold_count: usize,
|
|
},
|
|
/// Time and size based
|
|
Hybrid {
|
|
/// Time threshold
|
|
time_threshold: Duration,
|
|
/// Size threshold
|
|
size_threshold: usize,
|
|
},
|
|
}
|
|
|
|
/// Checkpoint retention policies
|
|
#[derive(Debug, Clone, Serialize, Deserialize)]
|
|
pub enum CheckpointRetention {
|
|
/// Keep a specific number of checkpoints
|
|
Count(usize),
|
|
/// Keep checkpoints for a specific duration
|
|
Time(Duration),
|
|
/// Keep checkpoints based on size limit
|
|
Size(usize),
|
|
}
|
|
|
|
/// Compression configuration
|
|
#[derive(Debug, Clone, Serialize, Deserialize)]
|
|
pub struct CompressionConfig {
|
|
/// Compression algorithm
|
|
pub algorithm: CompressionAlgorithm,
|
|
/// Compression level
|
|
pub level: u32,
|
|
}
|
|
|
|
/// Compression algorithms
|
|
#[derive(Debug, Clone, Serialize, Deserialize)]
|
|
pub enum CompressionAlgorithm {
|
|
/// Gzip compression
|
|
Gzip,
|
|
/// LZ4 compression
|
|
Lz4,
|
|
/// Zstd compression
|
|
Zstd,
|
|
/// Snappy compression
|
|
Snappy,
|
|
}
|
|
|
|
/// Deduplication strategies
|
|
#[derive(Debug, Clone, Serialize, Deserialize)]
|
|
pub enum DeduplicationStrategy {
|
|
/// Hash-based deduplication
|
|
Hash {
|
|
/// Columns to include in hash
|
|
hash_columns: Vec<String>,
|
|
/// Hash algorithm
|
|
hash_algorithm: HashAlgorithm,
|
|
},
|
|
/// Key-based deduplication
|
|
Key {
|
|
/// Key columns
|
|
key_columns: Vec<String>,
|
|
},
|
|
/// Bloom filter based deduplication
|
|
BloomFilter {
|
|
/// Expected number of elements
|
|
expected_elements: usize,
|
|
/// False positive probability
|
|
false_positive_rate: f64,
|
|
},
|
|
/// No deduplication
|
|
None,
|
|
}
|
|
|
|
/// State retention policies
|
|
#[derive(Debug, Clone, Serialize, Deserialize)]
|
|
pub enum StateRetentionPolicy {
|
|
/// Time-based retention
|
|
TimeBased {
|
|
/// Retention period
|
|
retention_period: Duration,
|
|
},
|
|
/// Size-based retention
|
|
SizeBased {
|
|
/// Maximum state size in bytes
|
|
max_size_bytes: usize,
|
|
},
|
|
/// Count-based retention
|
|
CountBased {
|
|
/// Maximum number of state entries
|
|
max_count: usize,
|
|
},
|
|
/// No retention (keep forever)
|
|
None,
|
|
}
|
|
|
|
/// Batch processing configuration
|
|
#[derive(Debug, Clone, Serialize, Deserialize)]
|
|
pub struct BatchConfig {
|
|
/// Batch size (number of records)
|
|
pub batch_size: usize,
|
|
/// Maximum batch processing time
|
|
pub max_batch_time: Duration,
|
|
/// Batch timeout
|
|
pub batch_timeout: Duration,
|
|
/// Enable adaptive batching
|
|
pub adaptive_batching: bool,
|
|
}
|
|
|
|
impl Default for BatchConfig {
|
|
fn default() -> Self {
|
|
Self {
|
|
batch_size: 1000,
|
|
max_batch_time: Duration::from_secs(30),
|
|
batch_timeout: Duration::from_secs(60),
|
|
adaptive_batching: true,
|
|
}
|
|
}
|
|
}
|
|
|
|
/// Memory management configuration
|
|
#[derive(Debug, Clone, Serialize, Deserialize)]
|
|
pub struct MemoryConfig {
|
|
/// Maximum memory usage in bytes
|
|
pub max_memory_bytes: usize,
|
|
/// Memory pressure threshold (0.0 to 1.0)
|
|
pub pressure_threshold: f64,
|
|
/// Enable memory compaction
|
|
pub enable_compaction: bool,
|
|
/// Garbage collection interval
|
|
pub gc_interval: Duration,
|
|
}
|
|
|
|
impl Default for MemoryConfig {
|
|
fn default() -> Self {
|
|
Self {
|
|
max_memory_bytes: 1024 * 1024 * 1024, // 1 GB
|
|
pressure_threshold: 0.8,
|
|
enable_compaction: true,
|
|
gc_interval: Duration::from_secs(60),
|
|
}
|
|
}
|
|
}
|
|
|
|
/// Parallelism configuration
|
|
#[derive(Debug, Clone, Serialize, Deserialize)]
|
|
pub struct ParallelismConfig {
|
|
/// Number of parallel workers
|
|
pub worker_count: usize,
|
|
/// Work stealing enabled
|
|
pub work_stealing: bool,
|
|
/// Load balancing strategy
|
|
pub load_balancing: LoadBalancingStrategy,
|
|
}
|
|
|
|
impl Default for ParallelismConfig {
|
|
fn default() -> Self {
|
|
Self {
|
|
worker_count: num_cpus::get(),
|
|
work_stealing: true,
|
|
load_balancing: LoadBalancingStrategy::RoundRobin,
|
|
}
|
|
}
|
|
}
|
|
|
|
/// Load balancing strategies
|
|
#[derive(Debug, Clone, Serialize, Deserialize)]
|
|
pub enum LoadBalancingStrategy {
|
|
/// Round-robin assignment
|
|
RoundRobin,
|
|
/// Least connections
|
|
LeastConnections,
|
|
/// Hash-based assignment
|
|
Hash {
|
|
/// Hash key column
|
|
key_column: String,
|
|
},
|
|
/// Random assignment
|
|
Random,
|
|
}
|
|
|
|
/// Change detector for identifying data changes
|
|
#[derive(Debug)]
|
|
pub struct ChangeDetector {
|
|
/// Detection strategy
|
|
strategy: ChangeDetectionStrategy,
|
|
/// State tracker
|
|
state_tracker: Arc<RwLock<ChangeState>>,
|
|
}
|
|
|
|
/// State for tracking changes
|
|
#[derive(Debug)]
|
|
pub struct ChangeState {
|
|
/// Last processed timestamps per source
|
|
last_timestamps: HashMap<String, DateTime<Utc>>,
|
|
/// Last processed versions per source
|
|
last_versions: HashMap<String, String>,
|
|
/// Hash signatures for change detection
|
|
hash_signatures: HashMap<String, String>,
|
|
/// Change log entries
|
|
change_log: BTreeMap<DateTime<Utc>, ChangeEntry>,
|
|
}
|
|
|
|
/// Entry in the change log
|
|
#[derive(Debug, Clone, Serialize, Deserialize)]
|
|
pub struct ChangeEntry {
|
|
/// Change identifier
|
|
pub change_id: String,
|
|
/// Change type
|
|
pub change_type: ChangeType,
|
|
/// Source record identifier
|
|
pub source_record_id: String,
|
|
/// Change timestamp
|
|
pub timestamp: DateTime<Utc>,
|
|
/// Changed fields
|
|
pub changed_fields: Vec<String>,
|
|
/// Old values (for updates)
|
|
pub old_values: Option<HashMap<String, DataValue>>,
|
|
/// New values
|
|
pub new_values: HashMap<String, DataValue>,
|
|
}
|
|
|
|
/// Types of changes
|
|
#[derive(Debug, Clone, Serialize, Deserialize)]
|
|
pub enum ChangeType {
|
|
/// Insert operation
|
|
Insert,
|
|
/// Update operation
|
|
Update,
|
|
/// Delete operation
|
|
Delete,
|
|
/// Upsert operation
|
|
Upsert,
|
|
}
|
|
|
|
/// Checkpoint manager for state persistence
|
|
#[derive(Debug)]
|
|
pub struct CheckpointManager {
|
|
/// Checkpoint configuration
|
|
config: CheckpointConfig,
|
|
/// State manager
|
|
state_manager: Arc<StateManager>,
|
|
/// Active checkpoints
|
|
active_checkpoints: Arc<RwLock<HashMap<String, Checkpoint>>>,
|
|
}
|
|
|
|
/// Checkpoint representing a point-in-time state
|
|
#[derive(Debug, Clone, Serialize, Deserialize)]
|
|
pub struct Checkpoint {
|
|
/// Checkpoint identifier
|
|
pub checkpoint_id: String,
|
|
/// Checkpoint timestamp
|
|
pub timestamp: DateTime<Utc>,
|
|
/// Source watermarks at checkpoint time
|
|
pub watermarks: HashMap<String, DateTime<Utc>>,
|
|
/// Processing state snapshot
|
|
pub state_snapshot: StateSnapshot,
|
|
/// Checkpoint metadata
|
|
pub metadata: CheckpointMetadata,
|
|
}
|
|
|
|
/// Snapshot of processing state
|
|
#[derive(Debug, Clone, Serialize, Deserialize)]
|
|
pub struct StateSnapshot {
|
|
/// Processed record counts per source
|
|
pub record_counts: HashMap<String, u64>,
|
|
/// Bytes processed per source
|
|
pub bytes_processed: HashMap<String, u64>,
|
|
/// Last processed positions
|
|
pub positions: HashMap<String, ProcessingPosition>,
|
|
/// State checksums for validation
|
|
pub checksums: HashMap<String, String>,
|
|
}
|
|
|
|
/// Processing position for resuming from checkpoint
|
|
#[derive(Debug, Clone, Serialize, Deserialize)]
|
|
pub struct ProcessingPosition {
|
|
/// Source identifier
|
|
pub source_id: String,
|
|
/// Position type
|
|
pub position_type: PositionType,
|
|
/// Position value
|
|
pub position_value: String,
|
|
/// Position timestamp
|
|
pub timestamp: DateTime<Utc>,
|
|
}
|
|
|
|
/// Types of processing positions
|
|
#[derive(Debug, Clone, Serialize, Deserialize)]
|
|
pub enum PositionType {
|
|
/// Timestamp-based position
|
|
Timestamp,
|
|
/// Sequence number position
|
|
SequenceNumber,
|
|
/// Log sequence number
|
|
LogSequenceNumber,
|
|
/// File offset position
|
|
FileOffset,
|
|
/// Custom position type
|
|
Custom(String),
|
|
}
|
|
|
|
/// Checkpoint metadata
|
|
#[derive(Debug, Clone, Serialize, Deserialize)]
|
|
pub struct CheckpointMetadata {
|
|
/// Checkpoint size in bytes
|
|
pub size_bytes: usize,
|
|
/// Compression used
|
|
pub compression: Option<CompressionAlgorithm>,
|
|
/// Checksum for validation
|
|
pub checksum: String,
|
|
/// Creation duration
|
|
pub creation_duration_ms: u64,
|
|
}
|
|
|
|
/// Deduplicator for removing duplicate records
|
|
#[derive(Debug)]
|
|
pub struct Deduplicator {
|
|
/// Deduplication strategy
|
|
strategy: DeduplicationStrategy,
|
|
/// Deduplication cache
|
|
cache: Arc<RwLock<DeduplicationCache>>,
|
|
}
|
|
|
|
/// Cache for deduplication
|
|
#[derive(Debug)]
|
|
pub struct DeduplicationCache {
|
|
/// Hash-based cache
|
|
hash_cache: HashMap<String, CacheEntry>,
|
|
/// Key-based cache
|
|
key_cache: HashMap<String, CacheEntry>,
|
|
/// Bloom filter for approximate deduplication
|
|
bloom_filter: Option<BloomFilter>,
|
|
}
|
|
|
|
/// Cache entry for deduplication
|
|
#[derive(Debug, Clone)]
|
|
pub struct CacheEntry {
|
|
/// Record hash or key
|
|
pub identifier: String,
|
|
/// First seen timestamp
|
|
pub first_seen: DateTime<Utc>,
|
|
/// Last seen timestamp
|
|
pub last_seen: DateTime<Utc>,
|
|
/// Occurrence count
|
|
pub count: u64,
|
|
}
|
|
|
|
/// Bloom filter for approximate deduplication
|
|
#[derive(Debug)]
|
|
pub struct BloomFilter {
|
|
/// Bit array
|
|
bits: Vec<bool>,
|
|
/// Hash functions count
|
|
hash_functions: usize,
|
|
/// Expected elements
|
|
expected_elements: usize,
|
|
/// Current element count
|
|
current_count: usize,
|
|
}
|
|
|
|
/// State tracker for incremental processing
|
|
#[derive(Debug)]
|
|
pub struct StateTracker {
|
|
/// Processing state per source
|
|
source_states: HashMap<String, SourceState>,
|
|
/// Global processing statistics
|
|
global_stats: ProcessingStats,
|
|
/// Error tracking
|
|
error_tracker: ErrorTracker,
|
|
}
|
|
|
|
/// State for individual data source
|
|
#[derive(Debug)]
|
|
pub struct SourceState {
|
|
/// Source identifier
|
|
pub source_id: String,
|
|
/// Last processing watermark
|
|
pub watermark: Option<DateTime<Utc>>,
|
|
/// Records processed count
|
|
pub records_processed: u64,
|
|
/// Bytes processed count
|
|
pub bytes_processed: u64,
|
|
/// Last error encountered
|
|
pub last_error: Option<ProcessingError>,
|
|
/// Processing lag metrics
|
|
pub lag_metrics: LagMetrics,
|
|
}
|
|
|
|
/// Processing statistics
|
|
#[derive(Debug, Clone, Serialize, Deserialize)]
|
|
pub struct ProcessingStats {
|
|
/// Total records processed
|
|
pub total_records: u64,
|
|
/// Total bytes processed
|
|
pub total_bytes: u64,
|
|
/// Processing start time
|
|
pub start_time: DateTime<Utc>,
|
|
/// Last processing time
|
|
pub last_processing_time: Option<DateTime<Utc>>,
|
|
/// Processing rate (records per second)
|
|
pub processing_rate: f64,
|
|
/// Throughput (bytes per second)
|
|
pub throughput: f64,
|
|
}
|
|
|
|
/// Error tracking for processing issues
|
|
#[derive(Debug)]
|
|
pub struct ErrorTracker {
|
|
/// Error history
|
|
pub error_history: VecDeque<ProcessingError>,
|
|
/// Error counts by type
|
|
pub error_counts: HashMap<String, u64>,
|
|
/// Last error time
|
|
pub last_error_time: Option<DateTime<Utc>>,
|
|
}
|
|
|
|
/// Processing error information
|
|
#[derive(Debug, Clone, Serialize, Deserialize)]
|
|
pub struct ProcessingError {
|
|
/// Error identifier
|
|
pub error_id: String,
|
|
/// Error type
|
|
pub error_type: String,
|
|
/// Error message
|
|
pub message: String,
|
|
/// Error timestamp
|
|
pub timestamp: DateTime<Utc>,
|
|
/// Source that caused the error
|
|
pub source_id: Option<String>,
|
|
/// Record that caused the error
|
|
pub record_id: Option<String>,
|
|
/// Stack trace
|
|
pub stack_trace: Option<String>,
|
|
}
|
|
|
|
/// Lag metrics for processing delay tracking
|
|
#[derive(Debug, Clone, Serialize, Deserialize)]
|
|
pub struct LagMetrics {
|
|
/// Current lag in milliseconds
|
|
pub current_lag_ms: u64,
|
|
/// Average lag in milliseconds
|
|
pub average_lag_ms: f64,
|
|
/// Maximum lag seen
|
|
pub max_lag_ms: u64,
|
|
/// Lag trend (increasing/decreasing)
|
|
pub trend: LagTrend,
|
|
}
|
|
|
|
/// Lag trend indicators
|
|
#[derive(Debug, Clone, Serialize, Deserialize)]
|
|
pub enum LagTrend {
|
|
/// Lag is increasing
|
|
Increasing,
|
|
/// Lag is decreasing
|
|
Decreasing,
|
|
/// Lag is stable
|
|
Stable,
|
|
}
|
|
|
|
/// Watermark manager for incremental processing
|
|
#[derive(Debug)]
|
|
pub struct WatermarkManager {
|
|
/// Watermarks per source
|
|
source_watermarks: Arc<RwLock<HashMap<String, DateTime<Utc>>>>,
|
|
/// Global watermark
|
|
global_watermark: Arc<RwLock<Option<DateTime<Utc>>>>,
|
|
/// Watermark update listeners
|
|
listeners: Arc<Mutex<Vec<tokio::sync::mpsc::UnboundedSender<WatermarkUpdate>>>>,
|
|
}
|
|
|
|
/// Watermark update notification
|
|
#[derive(Debug, Clone)]
|
|
pub struct WatermarkUpdate {
|
|
/// Source identifier
|
|
pub source_id: String,
|
|
/// New watermark
|
|
pub watermark: DateTime<Utc>,
|
|
/// Update timestamp
|
|
pub update_time: DateTime<Utc>,
|
|
}
|
|
|
|
/// Merge processor for combining incremental changes
|
|
#[derive(Debug)]
|
|
pub struct MergeProcessor {
|
|
/// Merge strategies by table/collection
|
|
strategies: HashMap<String, MergeStrategy>,
|
|
/// Conflict resolution rules
|
|
conflict_rules: Vec<ConflictResolutionRule>,
|
|
}
|
|
|
|
/// Strategies for merging data
|
|
#[derive(Debug, Clone, Serialize, Deserialize)]
|
|
pub enum MergeStrategy {
|
|
/// Append new records only
|
|
Append,
|
|
/// Update existing records, insert new ones
|
|
Upsert {
|
|
/// Key columns for matching
|
|
key_columns: Vec<String>,
|
|
},
|
|
/// Overwrite entire dataset
|
|
Overwrite,
|
|
/// Merge with conflict resolution
|
|
Merge {
|
|
/// Key columns for matching
|
|
key_columns: Vec<String>,
|
|
/// Conflict resolution strategy
|
|
conflict_resolution: ConflictResolution,
|
|
},
|
|
/// Time-based merge (keep latest)
|
|
TimeBased {
|
|
/// Timestamp column
|
|
timestamp_column: String,
|
|
},
|
|
}
|
|
|
|
/// Conflict resolution strategies
|
|
#[derive(Debug, Clone, Serialize, Deserialize)]
|
|
pub enum ConflictResolution {
|
|
/// Keep source value (incoming data wins)
|
|
Source,
|
|
/// Keep target value (existing data wins)
|
|
Target,
|
|
/// Use last writer wins
|
|
LastWriter,
|
|
/// Use timestamp-based resolution
|
|
Timestamp {
|
|
/// Timestamp column
|
|
column: String,
|
|
},
|
|
/// Custom resolution logic
|
|
Custom(String),
|
|
}
|
|
|
|
/// Rule for resolving conflicts during merge
|
|
#[derive(Debug, Clone, Serialize, Deserialize)]
|
|
pub struct ConflictResolutionRule {
|
|
/// Rule identifier
|
|
pub rule_id: String,
|
|
/// Table/collection pattern
|
|
pub table_pattern: String,
|
|
/// Column pattern
|
|
pub column_pattern: Option<String>,
|
|
/// Resolution strategy
|
|
pub resolution: ConflictResolution,
|
|
/// Rule priority (higher numbers take precedence)
|
|
pub priority: i32,
|
|
}
|
|
|
|
impl IncrementalProcessor {
|
|
/// Create a new incremental processor
|
|
pub async fn new(
|
|
config: IncrementalConfig,
|
|
state_manager: Arc<StateManager>,
|
|
metrics: Arc<EtlMetrics>,
|
|
) -> Result<Self> {
|
|
let change_detector = Arc::new(ChangeDetector::new(config.change_detection.clone()).await?);
|
|
let checkpoint_manager = Arc::new(CheckpointManager::new(
|
|
config.checkpoint_config.clone(),
|
|
state_manager.clone(),
|
|
));
|
|
let deduplicator = Arc::new(Deduplicator::new(config.deduplication.clone()).await?);
|
|
let state_tracker = Arc::new(RwLock::new(StateTracker::new()));
|
|
let watermark_manager = Arc::new(WatermarkManager::new());
|
|
let merge_processor = Arc::new(MergeProcessor::new());
|
|
|
|
Ok(Self {
|
|
config,
|
|
state_manager,
|
|
metrics,
|
|
change_detector,
|
|
checkpoint_manager,
|
|
deduplicator,
|
|
state_tracker,
|
|
watermark_manager,
|
|
merge_processor,
|
|
})
|
|
}
|
|
|
|
/// Detect changes since last checkpoint
|
|
#[instrument(skip(self))]
|
|
pub async fn detect_changes(
|
|
&self,
|
|
source: &str,
|
|
checkpoint: Option<Checkpoint>,
|
|
) -> Result<ChangeSet> {
|
|
info!("Detecting changes for source: {}", source);
|
|
|
|
let start_position = if let Some(ref cp) = checkpoint {
|
|
cp.state_snapshot.positions.get(source).cloned()
|
|
} else {
|
|
None
|
|
};
|
|
|
|
self.change_detector.detect_changes(source, start_position).await
|
|
}
|
|
|
|
/// Process incremental changes
|
|
#[instrument(skip(self, changes))]
|
|
pub async fn process_changes(&self, changes: ChangeSet) -> Result<ProcessedChanges> {
|
|
let mut processed = ProcessedChanges {
|
|
change_set_id: changes.change_set_id.clone(),
|
|
processed_count: 0,
|
|
skipped_count: 0,
|
|
error_count: 0,
|
|
processing_time: Instant::now(),
|
|
records: Vec::new(),
|
|
errors: Vec::new(),
|
|
};
|
|
|
|
for change_batch in &changes.batches {
|
|
let batch_result = self.process_change_batch(change_batch.clone()).await?;
|
|
processed.processed_count += batch_result.processed_count;
|
|
processed.skipped_count += batch_result.skipped_count;
|
|
processed.error_count += batch_result.error_count;
|
|
processed.records.extend(batch_result.records);
|
|
processed.errors.extend(batch_result.errors);
|
|
}
|
|
|
|
// Update watermarks
|
|
self.update_watermarks(&changes).await?;
|
|
|
|
Ok(processed)
|
|
}
|
|
|
|
/// Process a single batch of changes
|
|
async fn process_change_batch(&self, batch: ChangeBatch) -> Result<ProcessedChanges> {
|
|
let mut result = ProcessedChanges {
|
|
change_set_id: batch.batch_id.clone(),
|
|
processed_count: 0,
|
|
skipped_count: 0,
|
|
error_count: 0,
|
|
processing_time: Instant::now(),
|
|
records: Vec::new(),
|
|
errors: Vec::new(),
|
|
};
|
|
|
|
for change in batch.changes {
|
|
match self.process_single_change(change).await {
|
|
Ok(record) => {
|
|
result.records.push(record);
|
|
result.processed_count += 1;
|
|
}
|
|
Err(e) => {
|
|
result.errors.push(ProcessingError {
|
|
error_id: Uuid::new_v4().to_string(),
|
|
error_type: "processing_error".to_string(),
|
|
message: e.to_string(),
|
|
timestamp: Utc::now(),
|
|
source_id: Some(batch.source_id.clone()),
|
|
record_id: None,
|
|
stack_trace: None,
|
|
});
|
|
result.error_count += 1;
|
|
}
|
|
}
|
|
}
|
|
|
|
Ok(result)
|
|
}
|
|
|
|
/// Process a single change entry
|
|
async fn process_single_change(&self, change: ChangeEntry) -> Result<DataRecord> {
|
|
// Apply deduplication
|
|
if !self.deduplicator.should_process(&change).await? {
|
|
return Err(EtlError::IncrementalProcessing("Record is duplicate".to_string()));
|
|
}
|
|
|
|
// Convert change to data record
|
|
let record = DataRecord {
|
|
id: Uuid::new_v4(),
|
|
event_time: change.timestamp,
|
|
process_time: Utc::now(),
|
|
partition_key: Some(change.source_record_id.clone()),
|
|
data: crate::DataPayload::Structured(change.new_values),
|
|
metadata: crate::RecordMetadata {
|
|
source: "incremental_processor".to_string(),
|
|
lineage: Vec::new(),
|
|
quality_scores: None,
|
|
attributes: HashMap::new(),
|
|
schema_version: None,
|
|
checksum: None,
|
|
},
|
|
};
|
|
|
|
Ok(record)
|
|
}
|
|
|
|
/// Update watermarks based on processed changes
|
|
async fn update_watermarks(&self, changes: &ChangeSet) -> Result<()> {
|
|
for batch in &changes.batches {
|
|
if let Some(max_timestamp) = batch.max_timestamp {
|
|
self.watermark_manager.update_watermark(&batch.source_id, max_timestamp).await?;
|
|
}
|
|
}
|
|
Ok(())
|
|
}
|
|
|
|
/// Create a checkpoint of current processing state
|
|
#[instrument(skip(self))]
|
|
pub async fn create_checkpoint(&self, checkpoint_id: &str) -> Result<Checkpoint> {
|
|
info!("Creating checkpoint: {}", checkpoint_id);
|
|
let start_time = Instant::now();
|
|
|
|
let watermarks = self.watermark_manager.get_all_watermarks().await;
|
|
let state_tracker = self.state_tracker.read().await;
|
|
|
|
let positions: HashMap<String, ProcessingPosition> = state_tracker
|
|
.source_states
|
|
.iter()
|
|
.map(|(source_id, state)| {
|
|
(
|
|
source_id.clone(),
|
|
ProcessingPosition {
|
|
source_id: source_id.clone(),
|
|
position_type: PositionType::Timestamp,
|
|
position_value: state.watermark
|
|
.unwrap_or_else(Utc::now)
|
|
.to_rfc3339(),
|
|
timestamp: Utc::now(),
|
|
}
|
|
)
|
|
})
|
|
.collect();
|
|
|
|
let record_counts: HashMap<String, u64> = state_tracker
|
|
.source_states
|
|
.iter()
|
|
.map(|(source_id, state)| (source_id.clone(), state.records_processed))
|
|
.collect();
|
|
|
|
let bytes_processed: HashMap<String, u64> = state_tracker
|
|
.source_states
|
|
.iter()
|
|
.map(|(source_id, state)| (source_id.clone(), state.bytes_processed))
|
|
.collect();
|
|
|
|
// Calculate checksums for each source state
|
|
let checksums: HashMap<String, String> = state_tracker
|
|
.source_states
|
|
.iter()
|
|
.map(|(source_id, state)| {
|
|
let mut hasher = Sha256::new();
|
|
hasher.update(source_id.as_bytes());
|
|
hasher.update(state.records_processed.to_le_bytes());
|
|
if let Some(watermark) = state.watermark {
|
|
hasher.update(watermark.to_rfc3339().as_bytes());
|
|
}
|
|
hasher.update(state.bytes_processed.to_le_bytes());
|
|
let hash = format!("{:x}", hasher.finalize());
|
|
(source_id.clone(), hash)
|
|
})
|
|
.collect();
|
|
|
|
let state_snapshot = StateSnapshot {
|
|
record_counts,
|
|
bytes_processed,
|
|
positions,
|
|
checksums,
|
|
};
|
|
|
|
// Calculate checkpoint size by serializing it
|
|
let snapshot_bytes = serde_json::to_vec(&state_snapshot)
|
|
.map_err(EtlError::Serde)?;
|
|
let watermarks_bytes = serde_json::to_vec(&watermarks)
|
|
.map_err(EtlError::Serde)?;
|
|
let total_size = snapshot_bytes.len() + watermarks_bytes.len() + checkpoint_id.len();
|
|
|
|
// Calculate checkpoint checksum
|
|
let mut checkpoint_hasher = Sha256::new();
|
|
checkpoint_hasher.update(checkpoint_id.as_bytes());
|
|
checkpoint_hasher.update(&snapshot_bytes);
|
|
checkpoint_hasher.update(&watermarks_bytes);
|
|
let checkpoint_checksum = format!("{:x}", checkpoint_hasher.finalize());
|
|
|
|
let creation_duration = start_time.elapsed();
|
|
|
|
let checkpoint = Checkpoint {
|
|
checkpoint_id: checkpoint_id.to_string(),
|
|
timestamp: Utc::now(),
|
|
watermarks,
|
|
state_snapshot,
|
|
metadata: CheckpointMetadata {
|
|
size_bytes: total_size,
|
|
compression: self.config.checkpoint_config.compression.as_ref().map(|c| c.algorithm.clone()),
|
|
checksum: checkpoint_checksum,
|
|
creation_duration_ms: creation_duration.as_millis() as u64,
|
|
},
|
|
};
|
|
|
|
// Persist checkpoint
|
|
self.checkpoint_manager.save_checkpoint(&checkpoint).await?;
|
|
|
|
Ok(checkpoint)
|
|
}
|
|
|
|
/// Load a checkpoint and restore processing state
|
|
#[instrument(skip(self))]
|
|
pub async fn load_checkpoint(&self, checkpoint_id: &str) -> Result<Checkpoint> {
|
|
info!("Loading checkpoint: {}", checkpoint_id);
|
|
|
|
let checkpoint = self.checkpoint_manager.load_checkpoint(checkpoint_id).await?;
|
|
|
|
// Restore watermarks
|
|
for (source_id, watermark) in &checkpoint.watermarks {
|
|
self.watermark_manager.update_watermark(source_id, *watermark).await?;
|
|
}
|
|
|
|
// Restore state tracker
|
|
{
|
|
let mut state_tracker = self.state_tracker.write().await;
|
|
for (source_id, position) in &checkpoint.state_snapshot.positions {
|
|
let record_count = checkpoint.state_snapshot.record_counts
|
|
.get(source_id)
|
|
.copied()
|
|
.unwrap_or(0);
|
|
|
|
let bytes_count = checkpoint.state_snapshot.bytes_processed
|
|
.get(source_id)
|
|
.copied()
|
|
.unwrap_or(0);
|
|
|
|
state_tracker.source_states.insert(
|
|
source_id.clone(),
|
|
SourceState {
|
|
source_id: source_id.clone(),
|
|
watermark: Some(position.timestamp),
|
|
records_processed: record_count,
|
|
bytes_processed: bytes_count,
|
|
last_error: None,
|
|
lag_metrics: LagMetrics {
|
|
current_lag_ms: 0,
|
|
average_lag_ms: 0.0,
|
|
max_lag_ms: 0,
|
|
trend: LagTrend::Stable,
|
|
},
|
|
}
|
|
);
|
|
}
|
|
}
|
|
|
|
info!("Checkpoint loaded successfully: {}", checkpoint_id);
|
|
Ok(checkpoint)
|
|
}
|
|
|
|
/// Apply merge strategy to integrate incremental changes
|
|
#[instrument(skip(self, changes))]
|
|
pub async fn merge_changes(
|
|
&self,
|
|
target: &str,
|
|
changes: ProcessedChanges,
|
|
strategy: MergeStrategy,
|
|
) -> Result<MergeResult> {
|
|
self.merge_processor.merge(target, changes, strategy).await
|
|
}
|
|
}
|
|
|
|
/// Set of changes detected from a source
|
|
#[derive(Debug, Clone, Serialize, Deserialize)]
|
|
pub struct ChangeSet {
|
|
/// Change set identifier
|
|
pub change_set_id: String,
|
|
/// Source identifier
|
|
pub source_id: String,
|
|
/// Detection timestamp
|
|
pub detection_time: DateTime<Utc>,
|
|
/// Time range of changes
|
|
pub time_range: (DateTime<Utc>, DateTime<Utc>),
|
|
/// Total number of changes
|
|
pub total_changes: usize,
|
|
/// Changes grouped into batches
|
|
pub batches: Vec<ChangeBatch>,
|
|
}
|
|
|
|
/// Batch of changes for processing
|
|
#[derive(Debug, Clone, Serialize, Deserialize)]
|
|
pub struct ChangeBatch {
|
|
/// Batch identifier
|
|
pub batch_id: String,
|
|
/// Source identifier
|
|
pub source_id: String,
|
|
/// Changes in this batch
|
|
pub changes: Vec<ChangeEntry>,
|
|
/// Batch sequence number
|
|
pub sequence: u64,
|
|
/// Maximum timestamp in batch
|
|
pub max_timestamp: Option<DateTime<Utc>>,
|
|
/// Minimum timestamp in batch
|
|
pub min_timestamp: Option<DateTime<Utc>>,
|
|
}
|
|
|
|
/// Result of processing changes
|
|
#[derive(Debug)]
|
|
pub struct ProcessedChanges {
|
|
/// Change set identifier
|
|
pub change_set_id: String,
|
|
/// Number of successfully processed records
|
|
pub processed_count: u64,
|
|
/// Number of skipped records
|
|
pub skipped_count: u64,
|
|
/// Number of error records
|
|
pub error_count: u64,
|
|
/// Processing start time
|
|
pub processing_time: Instant,
|
|
/// Processed records
|
|
pub records: Vec<DataRecord>,
|
|
/// Processing errors
|
|
pub errors: Vec<ProcessingError>,
|
|
}
|
|
|
|
/// Result of merge operation
|
|
#[derive(Debug, Clone, Serialize, Deserialize)]
|
|
pub struct MergeResult {
|
|
/// Target identifier
|
|
pub target: String,
|
|
/// Merge strategy used
|
|
pub strategy: MergeStrategy,
|
|
/// Number of records inserted
|
|
pub inserted_count: u64,
|
|
/// Number of records updated
|
|
pub updated_count: u64,
|
|
/// Number of records deleted
|
|
pub deleted_count: u64,
|
|
/// Number of conflicts resolved
|
|
pub conflicts_resolved: u64,
|
|
/// Merge duration
|
|
pub duration_ms: u64,
|
|
/// Merge errors
|
|
pub errors: Vec<String>,
|
|
}
|
|
|
|
// Implementation stubs for the various components
|
|
impl ChangeDetector {
|
|
pub async fn new(strategy: ChangeDetectionStrategy) -> Result<Self> {
|
|
Ok(Self {
|
|
strategy,
|
|
state_tracker: Arc::new(RwLock::new(ChangeState {
|
|
last_timestamps: HashMap::new(),
|
|
last_versions: HashMap::new(),
|
|
hash_signatures: HashMap::new(),
|
|
change_log: BTreeMap::new(),
|
|
})),
|
|
})
|
|
}
|
|
|
|
pub async fn detect_changes(&self, source: &str, position: Option<ProcessingPosition>) -> Result<ChangeSet> {
|
|
let detection_start = Utc::now();
|
|
let mut changes = Vec::new();
|
|
|
|
// Get the starting position for change detection
|
|
let start_position = position.unwrap_or_else(|| ProcessingPosition {
|
|
source_id: source.to_string(),
|
|
position_type: PositionType::Timestamp,
|
|
position_value: Utc::now().to_rfc3339(),
|
|
timestamp: Utc::now(),
|
|
});
|
|
|
|
// Perform change detection based on strategy
|
|
match &self.strategy {
|
|
ChangeDetectionStrategy::TimestampBased { column, format } => {
|
|
changes = self.detect_timestamp_changes(source, column, format, &start_position).await?;
|
|
},
|
|
ChangeDetectionStrategy::HashBased { columns, algorithm } => {
|
|
changes = self.detect_hash_changes(source, columns, algorithm, &start_position).await?;
|
|
},
|
|
ChangeDetectionStrategy::VersionBased { column, comparison } => {
|
|
changes = self.detect_version_changes(source, column, comparison, &start_position).await?;
|
|
},
|
|
ChangeDetectionStrategy::LogBased { log_source } => {
|
|
changes = self.detect_log_changes(source, log_source, &start_position).await?;
|
|
},
|
|
ChangeDetectionStrategy::TriggerBased { trigger_config } => {
|
|
changes = self.detect_trigger_changes(source, trigger_config, &start_position).await?;
|
|
},
|
|
}
|
|
|
|
// Group changes into batches
|
|
let batches = self.group_changes_into_batches(changes, 1000).await?;
|
|
let total_changes = batches.iter().map(|b| b.changes.len()).sum();
|
|
|
|
// Update state tracker
|
|
self.update_detection_state(source, &batches).await?;
|
|
|
|
// Calculate time range
|
|
let min_time = batches.iter()
|
|
.filter_map(|b| b.min_timestamp)
|
|
.min()
|
|
.unwrap_or(detection_start);
|
|
let max_time = batches.iter()
|
|
.filter_map(|b| b.max_timestamp)
|
|
.max()
|
|
.unwrap_or(detection_start);
|
|
|
|
Ok(ChangeSet {
|
|
change_set_id: Uuid::new_v4().to_string(),
|
|
source_id: source.to_string(),
|
|
detection_time: detection_start,
|
|
time_range: (min_time, max_time),
|
|
total_changes,
|
|
batches,
|
|
})
|
|
}
|
|
|
|
/// Detect changes based on timestamp comparison
|
|
async fn detect_timestamp_changes(
|
|
&self,
|
|
source: &str,
|
|
column: &str,
|
|
format: &TimestampFormat,
|
|
start_position: &ProcessingPosition,
|
|
) -> Result<Vec<ChangeEntry>> {
|
|
let mut changes = Vec::new();
|
|
|
|
// Get the last processed timestamp for this source
|
|
let state = self.state_tracker.read().await;
|
|
let last_timestamp = state.last_timestamps.get(source)
|
|
.copied()
|
|
.unwrap_or_else(|| {
|
|
// Parse start position timestamp
|
|
DateTime::parse_from_rfc3339(&start_position.position_value).map_or_else(|_| Utc::now(), |dt| dt.with_timezone(&Utc))
|
|
});
|
|
drop(state);
|
|
|
|
// Simulate querying data source for records modified after last_timestamp
|
|
// In a real implementation, this would query the actual data source
|
|
let new_records = self.simulate_timestamp_query(source, column, format, last_timestamp).await?;
|
|
|
|
// Convert records to change entries
|
|
for record in new_records {
|
|
let change_id = Uuid::new_v4().to_string();
|
|
let change_type = if record.contains_key("_operation") {
|
|
match record.get("_operation").and_then(super::DataValue::as_string) {
|
|
Some(op) if op == "INSERT" => ChangeType::Insert,
|
|
Some(op) if op == "UPDATE" => ChangeType::Update,
|
|
Some(op) if op == "DELETE" => ChangeType::Delete,
|
|
_ => ChangeType::Upsert,
|
|
}
|
|
} else {
|
|
ChangeType::Upsert
|
|
};
|
|
|
|
let timestamp = record.get(column)
|
|
.and_then(|v| match v {
|
|
DataValue::Timestamp(dt) => Some(*dt),
|
|
DataValue::String(s) => DateTime::parse_from_rfc3339(s).ok().map(|dt| dt.with_timezone(&Utc)),
|
|
_ => None,
|
|
})
|
|
.unwrap_or_else(Utc::now);
|
|
|
|
changes.push(ChangeEntry {
|
|
change_id,
|
|
change_type,
|
|
source_record_id: record.get("id")
|
|
.and_then(super::DataValue::as_string)
|
|
.unwrap_or_else(|| Uuid::new_v4().to_string()),
|
|
timestamp,
|
|
changed_fields: record.keys().cloned().collect(),
|
|
old_values: None, // Would be populated for updates
|
|
new_values: record,
|
|
});
|
|
}
|
|
|
|
Ok(changes)
|
|
}
|
|
|
|
/// Detect changes based on hash comparison
|
|
async fn detect_hash_changes(
|
|
&self,
|
|
source: &str,
|
|
columns: &[String],
|
|
algorithm: &HashAlgorithm,
|
|
_start_position: &ProcessingPosition,
|
|
) -> Result<Vec<ChangeEntry>> {
|
|
let mut changes = Vec::new();
|
|
|
|
// Get current hash signatures
|
|
let state = self.state_tracker.read().await;
|
|
let existing_signatures = state.hash_signatures.clone();
|
|
drop(state);
|
|
|
|
// Simulate querying current data and calculating hashes
|
|
let current_records = self.simulate_hash_query(source, columns).await?;
|
|
|
|
for record in current_records {
|
|
let record_id = record.get("id")
|
|
.and_then(super::DataValue::as_string)
|
|
.unwrap_or_else(|| Uuid::new_v4().to_string());
|
|
|
|
// Calculate hash for specified columns
|
|
let hash_value = self.calculate_record_hash(&record, columns, algorithm);
|
|
let signature_key = format!("{source}:{record_id}");
|
|
|
|
// Check if hash has changed
|
|
let has_changed = existing_signatures.get(&signature_key) != Some(&hash_value);
|
|
|
|
if has_changed {
|
|
changes.push(ChangeEntry {
|
|
change_id: Uuid::new_v4().to_string(),
|
|
change_type: if existing_signatures.contains_key(&signature_key) {
|
|
ChangeType::Update
|
|
} else {
|
|
ChangeType::Insert
|
|
},
|
|
source_record_id: record_id,
|
|
timestamp: Utc::now(),
|
|
changed_fields: columns.to_vec(),
|
|
old_values: None,
|
|
new_values: record,
|
|
});
|
|
}
|
|
}
|
|
|
|
Ok(changes)
|
|
}
|
|
|
|
/// Detect changes based on version comparison
|
|
async fn detect_version_changes(
|
|
&self,
|
|
source: &str,
|
|
column: &str,
|
|
comparison: &VersionComparison,
|
|
_start_position: &ProcessingPosition,
|
|
) -> Result<Vec<ChangeEntry>> {
|
|
let mut changes = Vec::new();
|
|
|
|
// Get last processed versions
|
|
let state = self.state_tracker.read().await;
|
|
let last_versions = state.last_versions.clone();
|
|
drop(state);
|
|
|
|
// Simulate querying for version changes
|
|
let records = self.simulate_version_query(source, column).await?;
|
|
|
|
for record in records {
|
|
let record_id = record.get("id")
|
|
.and_then(super::DataValue::as_string)
|
|
.unwrap_or_else(|| Uuid::new_v4().to_string());
|
|
|
|
let current_version = record.get(column)
|
|
.and_then(super::DataValue::as_string)
|
|
.unwrap_or_else(|| "0".to_string());
|
|
|
|
let version_key = format!("{source}:{record_id}");
|
|
let last_version = last_versions.get(&version_key);
|
|
|
|
let has_changed = match (last_version, comparison) {
|
|
(Some(last), VersionComparison::Numeric) => {
|
|
let last_num: f64 = last.parse().unwrap_or(0.0);
|
|
let current_num: f64 = current_version.parse().unwrap_or(0.0);
|
|
current_num > last_num
|
|
},
|
|
(Some(last), VersionComparison::Lexicographic) => {
|
|
current_version > *last
|
|
},
|
|
(Some(last), VersionComparison::Semantic) => {
|
|
self.compare_semantic_versions(¤t_version, last) > 0
|
|
},
|
|
(None, _) => true, // New record
|
|
};
|
|
|
|
if has_changed {
|
|
changes.push(ChangeEntry {
|
|
change_id: Uuid::new_v4().to_string(),
|
|
change_type: if last_version.is_some() {
|
|
ChangeType::Update
|
|
} else {
|
|
ChangeType::Insert
|
|
},
|
|
source_record_id: record_id,
|
|
timestamp: Utc::now(),
|
|
changed_fields: record.keys().cloned().collect(),
|
|
old_values: None,
|
|
new_values: record,
|
|
});
|
|
}
|
|
}
|
|
|
|
Ok(changes)
|
|
}
|
|
|
|
/// Detect changes from log-based sources (CDC)
|
|
async fn detect_log_changes(
|
|
&self,
|
|
_source: &str,
|
|
log_source: &LogSourceConfig,
|
|
_start_position: &ProcessingPosition,
|
|
) -> Result<Vec<ChangeEntry>> {
|
|
let mut changes = Vec::new();
|
|
|
|
// Simulate reading from CDC log source
|
|
match &log_source.source_type {
|
|
LogSourceType::DatabaseLog { db_type } => {
|
|
changes = self.read_database_log(db_type, log_source).await?;
|
|
},
|
|
LogSourceType::KafkaChangelog { topic } => {
|
|
changes = self.read_kafka_changelog(topic, log_source).await?;
|
|
},
|
|
LogSourceType::FileChangelog { pattern } => {
|
|
changes = self.read_file_changelog(pattern, log_source).await?;
|
|
},
|
|
}
|
|
|
|
Ok(changes)
|
|
}
|
|
|
|
/// Detect changes from trigger-based sources
|
|
async fn detect_trigger_changes(
|
|
&self,
|
|
_source: &str,
|
|
_trigger_config: &TriggerConfig,
|
|
_start_position: &ProcessingPosition,
|
|
) -> Result<Vec<ChangeEntry>> {
|
|
// Simplified implementation - would integrate with trigger systems
|
|
Ok(vec![])
|
|
}
|
|
|
|
// Helper methods for simulating data source queries
|
|
async fn simulate_timestamp_query(
|
|
&self,
|
|
_source: &str,
|
|
_column: &str,
|
|
_format: &TimestampFormat,
|
|
_after: DateTime<Utc>,
|
|
) -> Result<Vec<HashMap<String, DataValue>>> {
|
|
// Simulate returning some changed records
|
|
let mut records = Vec::new();
|
|
let now = Utc::now();
|
|
|
|
for i in 0..5 {
|
|
let mut record = HashMap::new();
|
|
record.insert("id".to_string(), DataValue::String(format!("record_{i}")));
|
|
record.insert("updated_at".to_string(), DataValue::Timestamp(now));
|
|
record.insert("data".to_string(), DataValue::String(format!("data_value_{i}")));
|
|
records.push(record);
|
|
}
|
|
|
|
Ok(records)
|
|
}
|
|
|
|
async fn simulate_hash_query(
|
|
&self,
|
|
_source: &str,
|
|
_columns: &[String],
|
|
) -> Result<Vec<HashMap<String, DataValue>>> {
|
|
// Simulate returning current records for hash comparison
|
|
let mut records = Vec::new();
|
|
|
|
for i in 0..3 {
|
|
let mut record = HashMap::new();
|
|
record.insert("id".to_string(), DataValue::String(format!("record_{i}")));
|
|
record.insert("name".to_string(), DataValue::String(format!("name_{i}")));
|
|
record.insert("value".to_string(), DataValue::Int(i64::from(i) * 10));
|
|
records.push(record);
|
|
}
|
|
|
|
Ok(records)
|
|
}
|
|
|
|
async fn simulate_version_query(
|
|
&self,
|
|
_source: &str,
|
|
_column: &str,
|
|
) -> Result<Vec<HashMap<String, DataValue>>> {
|
|
// Simulate returning records with version information
|
|
let mut records = Vec::new();
|
|
|
|
for i in 0..3 {
|
|
let mut record = HashMap::new();
|
|
record.insert("id".to_string(), DataValue::String(format!("record_{i}")));
|
|
record.insert("version".to_string(), DataValue::String(format!("1.{i}.0")));
|
|
record.insert("data".to_string(), DataValue::String(format!("versioned_data_{i}")));
|
|
records.push(record);
|
|
}
|
|
|
|
Ok(records)
|
|
}
|
|
|
|
/// Calculate hash for a record using specified algorithm
|
|
fn calculate_record_hash(
|
|
&self,
|
|
record: &HashMap<String, DataValue>,
|
|
columns: &[String],
|
|
algorithm: &HashAlgorithm,
|
|
) -> String {
|
|
use sha2::{Sha256, Digest};
|
|
|
|
// Extract values for specified columns in deterministic order
|
|
let mut values = Vec::new();
|
|
for column in columns {
|
|
if let Some(value) = record.get(column) {
|
|
values.push(serde_json::to_string(value).unwrap_or_default());
|
|
}
|
|
}
|
|
|
|
let combined = values.join("|");
|
|
|
|
match algorithm {
|
|
HashAlgorithm::Sha256 => {
|
|
let mut hasher = Sha256::new();
|
|
hasher.update(combined.as_bytes());
|
|
format!("{:x}", hasher.finalize())
|
|
},
|
|
HashAlgorithm::Md5 => {
|
|
// Simplified - would use actual MD5
|
|
format!("md5_{}", combined.len())
|
|
},
|
|
HashAlgorithm::Sha512 => {
|
|
// Simplified - would use actual SHA-512
|
|
format!("sha512_{}", combined.len())
|
|
},
|
|
HashAlgorithm::Crc32 => {
|
|
// Simplified - would use actual CRC32
|
|
format!("crc32_{}", combined.len())
|
|
},
|
|
}
|
|
}
|
|
|
|
/// Compare semantic versions
|
|
fn compare_semantic_versions(&self, version1: &str, version2: &str) -> i32 {
|
|
let parse_version = |v: &str| -> Vec<u32> {
|
|
v.split('.')
|
|
.map(|part| part.parse().unwrap_or(0))
|
|
.collect()
|
|
};
|
|
|
|
let v1 = parse_version(version1);
|
|
let v2 = parse_version(version2);
|
|
|
|
for i in 0..std::cmp::max(v1.len(), v2.len()) {
|
|
let part1 = v1.get(i).unwrap_or(&0);
|
|
let part2 = v2.get(i).unwrap_or(&0);
|
|
|
|
match part1.cmp(part2) {
|
|
std::cmp::Ordering::Greater => return 1,
|
|
std::cmp::Ordering::Less => return -1,
|
|
std::cmp::Ordering::Equal => continue,
|
|
}
|
|
}
|
|
|
|
0
|
|
}
|
|
|
|
/// Group changes into processing batches
|
|
async fn group_changes_into_batches(
|
|
&self,
|
|
changes: Vec<ChangeEntry>,
|
|
batch_size: usize,
|
|
) -> Result<Vec<ChangeBatch>> {
|
|
let mut batches = Vec::new();
|
|
let mut current_batch = Vec::new();
|
|
let mut sequence = 0u64;
|
|
|
|
for change in changes {
|
|
current_batch.push(change);
|
|
|
|
if current_batch.len() >= batch_size {
|
|
let min_timestamp = current_batch.iter().map(|c| c.timestamp).min();
|
|
let max_timestamp = current_batch.iter().map(|c| c.timestamp).max();
|
|
|
|
batches.push(ChangeBatch {
|
|
batch_id: Uuid::new_v4().to_string(),
|
|
source_id: current_batch[0].source_record_id.clone(),
|
|
changes: current_batch.clone(),
|
|
sequence,
|
|
min_timestamp,
|
|
max_timestamp,
|
|
});
|
|
|
|
current_batch.clear();
|
|
sequence += 1;
|
|
}
|
|
}
|
|
|
|
// Handle remaining changes
|
|
if !current_batch.is_empty() {
|
|
let min_timestamp = current_batch.iter().map(|c| c.timestamp).min();
|
|
let max_timestamp = current_batch.iter().map(|c| c.timestamp).max();
|
|
|
|
batches.push(ChangeBatch {
|
|
batch_id: Uuid::new_v4().to_string(),
|
|
source_id: current_batch[0].source_record_id.clone(),
|
|
changes: current_batch,
|
|
sequence,
|
|
min_timestamp,
|
|
max_timestamp,
|
|
});
|
|
}
|
|
|
|
Ok(batches)
|
|
}
|
|
|
|
/// Update detection state after processing changes
|
|
async fn update_detection_state(&self, source: &str, batches: &[ChangeBatch]) -> Result<()> {
|
|
let mut state = self.state_tracker.write().await;
|
|
|
|
// Update last processed timestamp
|
|
if let Some(latest_timestamp) = batches.iter()
|
|
.filter_map(|b| b.max_timestamp)
|
|
.max() {
|
|
state.last_timestamps.insert(source.to_string(), latest_timestamp);
|
|
}
|
|
|
|
// Update version tracking
|
|
for batch in batches {
|
|
for change in &batch.changes {
|
|
if let Some(version) = change.new_values.get("version")
|
|
.and_then(super::DataValue::as_string) {
|
|
let key = format!("{}:{}", source, change.source_record_id);
|
|
state.last_versions.insert(key, version);
|
|
}
|
|
|
|
// Log change entry
|
|
state.change_log.insert(change.timestamp, change.clone());
|
|
}
|
|
}
|
|
|
|
// Cleanup old entries (keep last 1000)
|
|
while state.change_log.len() > 1000 {
|
|
if let Some(oldest_key) = state.change_log.keys().next().copied() {
|
|
state.change_log.remove(&oldest_key);
|
|
}
|
|
}
|
|
|
|
Ok(())
|
|
}
|
|
|
|
// CDC source reading methods (simplified implementations)
|
|
async fn read_database_log(
|
|
&self,
|
|
_db_type: &DatabaseType,
|
|
_config: &LogSourceConfig,
|
|
) -> Result<Vec<ChangeEntry>> {
|
|
// Simulate reading from database transaction log
|
|
Ok(vec![])
|
|
}
|
|
|
|
async fn read_kafka_changelog(
|
|
&self,
|
|
_topic: &str,
|
|
_config: &LogSourceConfig,
|
|
) -> Result<Vec<ChangeEntry>> {
|
|
// Simulate reading from Kafka changelog topic
|
|
Ok(vec![])
|
|
}
|
|
|
|
async fn read_file_changelog(
|
|
&self,
|
|
_pattern: &str,
|
|
_config: &LogSourceConfig,
|
|
) -> Result<Vec<ChangeEntry>> {
|
|
// Simulate reading from file-based changelog
|
|
Ok(vec![])
|
|
}
|
|
}
|
|
|
|
impl CheckpointManager {
|
|
#[must_use]
|
|
pub fn new(config: CheckpointConfig, state_manager: Arc<StateManager>) -> Self {
|
|
Self {
|
|
config,
|
|
state_manager,
|
|
active_checkpoints: Arc::new(RwLock::new(HashMap::new())),
|
|
}
|
|
}
|
|
|
|
pub async fn save_checkpoint(&self, _checkpoint: &Checkpoint) -> Result<()> {
|
|
// Implementation would save checkpoint to persistent storage
|
|
Ok(())
|
|
}
|
|
|
|
pub async fn load_checkpoint(&self, _checkpoint_id: &str) -> Result<Checkpoint> {
|
|
// Implementation would load checkpoint from persistent storage
|
|
Err(EtlError::Other("Checkpoint not found".to_string()))
|
|
}
|
|
}
|
|
|
|
impl Deduplicator {
|
|
pub async fn new(strategy: DeduplicationStrategy) -> Result<Self> {
|
|
let bloom_filter = match &strategy {
|
|
DeduplicationStrategy::BloomFilter { expected_elements, false_positive_rate } => {
|
|
Some(BloomFilter::new(*expected_elements, *false_positive_rate))
|
|
},
|
|
_ => None,
|
|
};
|
|
|
|
Ok(Self {
|
|
strategy,
|
|
cache: Arc::new(RwLock::new(DeduplicationCache {
|
|
hash_cache: HashMap::new(),
|
|
key_cache: HashMap::new(),
|
|
bloom_filter,
|
|
})),
|
|
})
|
|
}
|
|
|
|
pub async fn should_process(&self, change: &ChangeEntry) -> Result<bool> {
|
|
match &self.strategy {
|
|
DeduplicationStrategy::Hash { hash_columns, hash_algorithm } => {
|
|
self.check_hash_deduplication(change, hash_columns, hash_algorithm).await
|
|
},
|
|
DeduplicationStrategy::Key { key_columns } => {
|
|
self.check_key_deduplication(change, key_columns).await
|
|
},
|
|
DeduplicationStrategy::BloomFilter { .. } => {
|
|
self.check_bloom_filter_deduplication(change).await
|
|
},
|
|
DeduplicationStrategy::None => Ok(true),
|
|
}
|
|
}
|
|
|
|
/// Check hash-based deduplication
|
|
async fn check_hash_deduplication(
|
|
&self,
|
|
change: &ChangeEntry,
|
|
hash_columns: &[String],
|
|
hash_algorithm: &HashAlgorithm,
|
|
) -> Result<bool> {
|
|
let hash_value = self.calculate_change_hash(change, hash_columns, hash_algorithm);
|
|
let mut cache = self.cache.write().await;
|
|
|
|
let identifier = format!("{}:{}", change.source_record_id, hash_value);
|
|
let current_time = Utc::now();
|
|
|
|
if let Some(entry) = cache.hash_cache.get_mut(&identifier) {
|
|
// Found duplicate
|
|
entry.last_seen = current_time;
|
|
entry.count += 1;
|
|
Ok(false) // Don't process duplicate
|
|
} else {
|
|
// New entry
|
|
cache.hash_cache.insert(identifier, CacheEntry {
|
|
identifier: hash_value,
|
|
first_seen: current_time,
|
|
last_seen: current_time,
|
|
count: 1,
|
|
});
|
|
Ok(true) // Process new entry
|
|
}
|
|
}
|
|
|
|
/// Check key-based deduplication
|
|
async fn check_key_deduplication(
|
|
&self,
|
|
change: &ChangeEntry,
|
|
key_columns: &[String],
|
|
) -> Result<bool> {
|
|
let key_value = self.extract_key_value(change, key_columns);
|
|
let mut cache = self.cache.write().await;
|
|
|
|
let current_time = Utc::now();
|
|
|
|
if let Some(entry) = cache.key_cache.get_mut(&key_value) {
|
|
// Check if this is truly a duplicate or an update
|
|
// For updates, we should process if the content has changed
|
|
if let ChangeType::Update = change.change_type {
|
|
entry.last_seen = current_time;
|
|
entry.count += 1;
|
|
Ok(true) // Process updates
|
|
} else {
|
|
entry.last_seen = current_time;
|
|
entry.count += 1;
|
|
Ok(false) // Don't process duplicates
|
|
}
|
|
} else {
|
|
// New entry
|
|
cache.key_cache.insert(key_value.clone(), CacheEntry {
|
|
identifier: key_value,
|
|
first_seen: current_time,
|
|
last_seen: current_time,
|
|
count: 1,
|
|
});
|
|
Ok(true) // Process new entry
|
|
}
|
|
}
|
|
|
|
/// Check bloom filter-based deduplication
|
|
async fn check_bloom_filter_deduplication(&self, change: &ChangeEntry) -> Result<bool> {
|
|
let mut cache = self.cache.write().await;
|
|
|
|
if let Some(ref mut bloom_filter) = cache.bloom_filter {
|
|
let identifier = format!("{}:{}", change.source_record_id, change.timestamp.timestamp());
|
|
|
|
if bloom_filter.contains(&identifier) {
|
|
// Possibly seen before (false positives possible)
|
|
// For critical applications, combine with exact cache
|
|
Ok(false)
|
|
} else {
|
|
// Definitely not seen before
|
|
bloom_filter.insert(&identifier);
|
|
Ok(true)
|
|
}
|
|
} else {
|
|
// No bloom filter configured
|
|
Ok(true)
|
|
}
|
|
}
|
|
|
|
/// Calculate hash for change entry
|
|
fn calculate_change_hash(
|
|
&self,
|
|
change: &ChangeEntry,
|
|
hash_columns: &[String],
|
|
algorithm: &HashAlgorithm,
|
|
) -> String {
|
|
use sha2::{Sha256, Digest};
|
|
|
|
// Create deterministic representation of the change
|
|
let mut values = Vec::new();
|
|
|
|
// Include change metadata
|
|
values.push(change.source_record_id.clone());
|
|
values.push(change.change_type.to_string());
|
|
|
|
// Include specified columns
|
|
for column in hash_columns {
|
|
if let Some(value) = change.new_values.get(column) {
|
|
values.push(serde_json::to_string(value).unwrap_or_default());
|
|
}
|
|
}
|
|
|
|
let combined = values.join("|");
|
|
|
|
match algorithm {
|
|
HashAlgorithm::Sha256 => {
|
|
let mut hasher = Sha256::new();
|
|
hasher.update(combined.as_bytes());
|
|
format!("{:x}", hasher.finalize())
|
|
},
|
|
HashAlgorithm::Md5 => {
|
|
// Simplified MD5 implementation
|
|
format!("md5_{:x}", self.simple_hash(&combined))
|
|
},
|
|
HashAlgorithm::Sha512 => {
|
|
// Simplified SHA-512 implementation
|
|
format!("sha512_{:x}", self.simple_hash(&combined))
|
|
},
|
|
HashAlgorithm::Crc32 => {
|
|
// Simplified CRC32 implementation
|
|
format!("crc32_{:x}", self.simple_hash(&combined))
|
|
},
|
|
}
|
|
}
|
|
|
|
/// Extract key value from change entry
|
|
fn extract_key_value(&self, change: &ChangeEntry, key_columns: &[String]) -> String {
|
|
let mut key_parts = Vec::new();
|
|
|
|
for column in key_columns {
|
|
if let Some(value) = change.new_values.get(column) {
|
|
key_parts.push(serde_json::to_string(value).unwrap_or_default());
|
|
} else {
|
|
key_parts.push("NULL".to_string());
|
|
}
|
|
}
|
|
|
|
key_parts.join(":")
|
|
}
|
|
|
|
/// Simple hash function for fallback implementations
|
|
fn simple_hash(&self, input: &str) -> u64 {
|
|
use std::collections::hash_map::DefaultHasher;
|
|
use std::hash::{Hash, Hasher};
|
|
|
|
let mut hasher = DefaultHasher::new();
|
|
input.hash(&mut hasher);
|
|
hasher.finish()
|
|
}
|
|
|
|
/// Clean up old cache entries to prevent memory leaks
|
|
pub async fn cleanup_cache(&self, retention_period: Duration) -> Result<()> {
|
|
let mut cache = self.cache.write().await;
|
|
let cutoff_time = Utc::now() - retention_period;
|
|
|
|
// Clean hash cache
|
|
cache.hash_cache.retain(|_, entry| entry.last_seen > cutoff_time);
|
|
|
|
// Clean key cache
|
|
cache.key_cache.retain(|_, entry| entry.last_seen > cutoff_time);
|
|
|
|
Ok(())
|
|
}
|
|
|
|
/// Get deduplication statistics
|
|
pub async fn get_statistics(&self) -> DeduplicationStatistics {
|
|
let cache = self.cache.read().await;
|
|
|
|
let hash_entries = cache.hash_cache.len();
|
|
let key_entries = cache.key_cache.len();
|
|
let total_hash_hits = cache.hash_cache.values().map(|e| e.count).sum::<u64>();
|
|
let total_key_hits = cache.key_cache.values().map(|e| e.count).sum::<u64>();
|
|
|
|
DeduplicationStatistics {
|
|
hash_cache_entries: hash_entries,
|
|
key_cache_entries: key_entries,
|
|
total_hash_duplicates: total_hash_hits,
|
|
total_key_duplicates: total_key_hits,
|
|
bloom_filter_entries: cache.bloom_filter.as_ref()
|
|
.map_or(0, |bf| bf.current_count),
|
|
}
|
|
}
|
|
}
|
|
|
|
/// Statistics for deduplication operations
|
|
#[derive(Debug, Clone, Serialize, Deserialize)]
|
|
pub struct DeduplicationStatistics {
|
|
/// Number of entries in hash cache
|
|
pub hash_cache_entries: usize,
|
|
/// Number of entries in key cache
|
|
pub key_cache_entries: usize,
|
|
/// Total duplicate hits in hash cache
|
|
pub total_hash_duplicates: u64,
|
|
/// Total duplicate hits in key cache
|
|
pub total_key_duplicates: u64,
|
|
/// Number of entries in bloom filter
|
|
pub bloom_filter_entries: usize,
|
|
}
|
|
|
|
impl ChangeType {
|
|
fn to_string(&self) -> String {
|
|
match self {
|
|
Self::Insert => "INSERT".to_string(),
|
|
Self::Update => "UPDATE".to_string(),
|
|
Self::Delete => "DELETE".to_string(),
|
|
Self::Upsert => "UPSERT".to_string(),
|
|
}
|
|
}
|
|
}
|
|
|
|
impl BloomFilter {
|
|
/// Create a new bloom filter
|
|
#[must_use]
|
|
pub fn new(expected_elements: usize, false_positive_rate: f64) -> Self {
|
|
let bit_array_size = Self::calculate_bit_array_size(expected_elements, false_positive_rate);
|
|
let hash_functions = Self::calculate_hash_functions(bit_array_size, expected_elements);
|
|
|
|
Self {
|
|
bits: vec![false; bit_array_size],
|
|
hash_functions,
|
|
expected_elements,
|
|
current_count: 0,
|
|
}
|
|
}
|
|
|
|
/// Calculate optimal bit array size
|
|
fn calculate_bit_array_size(expected_elements: usize, false_positive_rate: f64) -> usize {
|
|
let m = -(expected_elements as f64 * false_positive_rate.ln()) / (2.0_f64.ln().powi(2));
|
|
m.ceil() as usize
|
|
}
|
|
|
|
/// Calculate optimal number of hash functions
|
|
fn calculate_hash_functions(bit_array_size: usize, expected_elements: usize) -> usize {
|
|
let k = (bit_array_size as f64 / expected_elements as f64) * 2.0_f64.ln();
|
|
(k.ceil() as usize).max(1)
|
|
}
|
|
|
|
/// Insert an element into the bloom filter
|
|
pub fn insert(&mut self, element: &str) {
|
|
let hashes = self.hash_element(element);
|
|
|
|
for &hash_value in &hashes {
|
|
let index = hash_value % self.bits.len();
|
|
self.bits[index] = true;
|
|
}
|
|
|
|
self.current_count += 1;
|
|
}
|
|
|
|
/// Check if an element might be in the filter
|
|
#[must_use]
|
|
pub fn contains(&self, element: &str) -> bool {
|
|
let hashes = self.hash_element(element);
|
|
|
|
for &hash_value in &hashes {
|
|
let index = hash_value % self.bits.len();
|
|
if !self.bits[index] {
|
|
return false; // Definitely not in set
|
|
}
|
|
}
|
|
|
|
true // Might be in set (or false positive)
|
|
}
|
|
|
|
/// Generate multiple hash values for an element
|
|
fn hash_element(&self, element: &str) -> Vec<usize> {
|
|
use std::collections::hash_map::DefaultHasher;
|
|
use std::hash::{Hash, Hasher};
|
|
|
|
let mut hashes = Vec::with_capacity(self.hash_functions);
|
|
|
|
// Use multiple hash functions based on a single hash with different salts
|
|
let mut hasher = DefaultHasher::new();
|
|
element.hash(&mut hasher);
|
|
let primary_hash = hasher.finish();
|
|
|
|
for i in 0..self.hash_functions {
|
|
let hash = primary_hash.wrapping_add(i as u64 * 0x9E3779B9);
|
|
hashes.push(hash as usize);
|
|
}
|
|
|
|
hashes
|
|
}
|
|
|
|
/// Get current false positive probability
|
|
#[must_use]
|
|
pub fn current_false_positive_rate(&self) -> f64 {
|
|
if self.current_count == 0 {
|
|
return 0.0;
|
|
}
|
|
|
|
let filled_bits = self.bits.iter().filter(|&&bit| bit).count();
|
|
let probability = filled_bits as f64 / self.bits.len() as f64;
|
|
probability.powi(self.hash_functions as i32)
|
|
}
|
|
|
|
/// Clear the bloom filter
|
|
pub fn clear(&mut self) {
|
|
self.bits.fill(false);
|
|
self.current_count = 0;
|
|
}
|
|
}
|
|
|
|
impl StateTracker {
|
|
#[must_use]
|
|
pub fn new() -> Self {
|
|
Self {
|
|
source_states: HashMap::new(),
|
|
global_stats: ProcessingStats {
|
|
total_records: 0,
|
|
total_bytes: 0,
|
|
start_time: Utc::now(),
|
|
last_processing_time: None,
|
|
processing_rate: 0.0,
|
|
throughput: 0.0,
|
|
},
|
|
error_tracker: ErrorTracker {
|
|
error_history: VecDeque::new(),
|
|
error_counts: HashMap::new(),
|
|
last_error_time: None,
|
|
},
|
|
}
|
|
}
|
|
}
|
|
|
|
impl WatermarkManager {
|
|
#[must_use]
|
|
pub fn new() -> Self {
|
|
Self {
|
|
source_watermarks: Arc::new(RwLock::new(HashMap::new())),
|
|
global_watermark: Arc::new(RwLock::new(None)),
|
|
listeners: Arc::new(Mutex::new(Vec::new())),
|
|
}
|
|
}
|
|
|
|
pub async fn update_watermark(&self, source_id: &str, watermark: DateTime<Utc>) -> Result<()> {
|
|
let mut watermarks = self.source_watermarks.write().await;
|
|
watermarks.insert(source_id.to_string(), watermark);
|
|
Ok(())
|
|
}
|
|
|
|
pub async fn get_all_watermarks(&self) -> HashMap<String, DateTime<Utc>> {
|
|
self.source_watermarks.read().await.clone()
|
|
}
|
|
}
|
|
|
|
impl MergeProcessor {
|
|
#[must_use]
|
|
pub fn new() -> Self {
|
|
Self {
|
|
strategies: HashMap::new(),
|
|
conflict_rules: Vec::new(),
|
|
}
|
|
}
|
|
|
|
pub async fn merge(
|
|
&self,
|
|
_target: &str,
|
|
_changes: ProcessedChanges,
|
|
strategy: MergeStrategy,
|
|
) -> Result<MergeResult> {
|
|
// Simplified implementation - would contain actual merge logic
|
|
Ok(MergeResult {
|
|
target: _target.to_string(),
|
|
strategy,
|
|
inserted_count: 0,
|
|
updated_count: 0,
|
|
deleted_count: 0,
|
|
conflicts_resolved: 0,
|
|
duration_ms: 0,
|
|
errors: Vec::new(),
|
|
})
|
|
}
|
|
}
|
|
|
|
impl Default for StateTracker {
|
|
fn default() -> Self {
|
|
Self::new()
|
|
}
|
|
}
|
|
|
|
impl Default for WatermarkManager {
|
|
fn default() -> Self {
|
|
Self::new()
|
|
}
|
|
}
|
|
|
|
impl Default for MergeProcessor {
|
|
fn default() -> Self {
|
|
Self::new()
|
|
}
|
|
} |