Initial commit

crates/data/rtx-etl/src/lineage/schema.rs

@@ -0,0 +1,287 @@
+//! Schema information and evolution tracking.
+
+use chrono::{DateTime, Utc};
+use serde::{Deserialize, Serialize};
+use std::collections::HashMap;
+
+use crate::DataValue;
+
+/// Schema information for lineage nodes
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub struct SchemaInfo {
+    /// Schema identifier
+    pub schema_id: String,
+    /// Schema name
+    pub name: String,
+    /// Schema version
+    pub version: String,
+    /// Fields in the schema
+    pub fields: Vec<FieldInfo>,
+    /// Schema creation time
+    pub created_at: DateTime<Utc>,
+    /// Schema checksum for change detection
+    pub checksum: String,
+}
+
+/// Field information within a schema
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub struct FieldInfo {
+    /// Field name
+    pub name: String,
+    /// Data type
+    pub data_type: String,
+    /// Whether field is nullable
+    pub nullable: bool,
+    /// Field description
+    pub description: Option<String>,
+    /// Field constraints
+    pub constraints: Vec<FieldConstraint>,
+    /// Field tags
+    pub tags: Vec<String>,
+}
+
+/// Field constraints
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub enum FieldConstraint {
+    /// Primary key constraint
+    PrimaryKey,
+    /// Foreign key constraint
+    ForeignKey {
+        /// Referenced table
+        referenced_table: String,
+        /// Referenced column
+        referenced_column: String,
+    },
+    /// Unique constraint
+    Unique,
+    /// Not null constraint
+    NotNull,
+    /// Check constraint
+    Check {
+        /// Constraint expression
+        expression: String,
+    },
+    /// Default value constraint
+    Default {
+        /// Default value
+        value: DataValue,
+    },
+}
+
+/// Schema evolution tracker
+#[derive(Debug)]
+pub struct SchemaEvolutionTracker {
+    /// Schema versions by dataset
+    pub(crate) schema_versions: HashMap<String, Vec<SchemaVersion>>,
+    /// Schema compatibility matrix
+    pub(crate) compatibility_matrix: HashMap<(String, String), CompatibilityInfo>,
+    /// Evolution rules
+    pub(crate) evolution_rules: Vec<EvolutionRule>,
+}
+
+impl SchemaEvolutionTracker {
+    #[must_use]
+    pub fn new() -> Self {
+        Self {
+            schema_versions: HashMap::new(),
+            compatibility_matrix: HashMap::new(),
+            evolution_rules: Vec::new(),
+        }
+    }
+}
+
+impl Default for SchemaEvolutionTracker {
+    fn default() -> Self {
+        Self::new()
+    }
+}
+
+/// Schema version information
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub struct SchemaVersion {
+    /// Version identifier
+    pub version_id: String,
+    /// Version number
+    pub version: String,
+    /// Schema information
+    pub schema: SchemaInfo,
+    /// Changes from previous version
+    pub changes: Vec<SchemaChange>,
+    /// Backward compatibility status
+    pub backward_compatible: bool,
+    /// Forward compatibility status
+    pub forward_compatible: bool,
+}
+
+/// Types of schema changes
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub enum SchemaChange {
+    /// Field added
+    FieldAdded {
+        /// Field information
+        field: FieldInfo,
+    },
+    /// Field removed
+    FieldRemoved {
+        /// Field name
+        field_name: String,
+    },
+    /// Field modified
+    FieldModified {
+        /// Field name
+        field_name: String,
+        /// Old field information
+        old_field: FieldInfo,
+        /// New field information
+        new_field: FieldInfo,
+    },
+    /// Field renamed
+    FieldRenamed {
+        /// Old field name
+        old_name: String,
+        /// New field name
+        new_name: String,
+    },
+    /// Constraint added
+    ConstraintAdded {
+        /// Field name
+        field_name: String,
+        /// Constraint
+        constraint: FieldConstraint,
+    },
+    /// Constraint removed
+    ConstraintRemoved {
+        /// Field name
+        field_name: String,
+        /// Constraint
+        constraint: FieldConstraint,
+    },
+}
+
+/// Schema compatibility information
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub struct CompatibilityInfo {
+    /// Whether schemas are compatible
+    pub compatible: bool,
+    /// Compatibility level
+    pub compatibility_level: CompatibilityLevel,
+    /// Compatibility issues
+    pub issues: Vec<CompatibilityIssue>,
+    /// Migration suggestions
+    pub migration_suggestions: Vec<String>,
+}
+
+/// Levels of schema compatibility
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub enum CompatibilityLevel {
+    /// Fully compatible
+    Full,
+    /// Backward compatible
+    Backward,
+    /// Forward compatible
+    Forward,
+    /// Partially compatible
+    Partial,
+    /// Incompatible
+    None,
+}
+
+/// Schema compatibility issues
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub struct CompatibilityIssue {
+    /// Issue type
+    pub issue_type: CompatibilityIssueType,
+    /// Issue description
+    pub description: String,
+    /// Severity level
+    pub severity: IssueSeverity,
+    /// Affected fields
+    pub affected_fields: Vec<String>,
+}
+
+/// Types of compatibility issues
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub enum CompatibilityIssueType {
+    /// Missing field
+    MissingField,
+    /// Type mismatch
+    TypeMismatch,
+    /// Constraint violation
+    ConstraintViolation,
+    /// Nullable change
+    NullabilityChange,
+    /// Renamed field
+    RenamedField,
+}
+
+/// Issue severity levels
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub enum IssueSeverity {
+    /// Critical issue
+    Critical,
+    /// Warning issue
+    Warning,
+    /// Info issue
+    Info,
+}
+
+/// Schema evolution rules
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub struct EvolutionRule {
+    /// Rule identifier
+    pub rule_id: String,
+    /// Rule name
+    pub name: String,
+    /// Rule condition
+    pub condition: EvolutionCondition,
+    /// Rule action
+    pub action: EvolutionAction,
+    /// Rule priority
+    pub priority: i32,
+}
+
+/// Conditions for schema evolution rules
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub enum EvolutionCondition {
+    /// Field added condition
+    FieldAdded {
+        /// Field pattern
+        field_pattern: String,
+    },
+    /// Field removed condition
+    FieldRemoved {
+        /// Field pattern
+        field_pattern: String,
+    },
+    /// Type changed condition
+    TypeChanged {
+        /// Field pattern
+        field_pattern: String,
+        /// Old type pattern
+        old_type: String,
+        /// New type pattern
+        new_type: String,
+    },
+}
+
+/// Actions for schema evolution rules
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub enum EvolutionAction {
+    /// Allow the change
+    Allow,
+    /// Block the change
+    Block {
+        /// Reason for blocking
+        reason: String,
+    },
+    /// Warn about the change
+    Warn {
+        /// Warning message
+        message: String,
+    },
+    /// Suggest migration
+    SuggestMigration {
+        /// Migration steps
+        steps: Vec<String>,
+    },
+}