Initial commit

2026-03-04 00:08:42 +00:00
commit 4d88dc0584
4449 changed files with 1556714 additions and 0 deletions
@@ -0,0 +1,187 @@
+//! Data source and sink connectors
+
+use crate::{DataRecord, Result};
+use serde::{Deserialize, Serialize};
+use std::collections::HashMap;
+
+/// Data source configuration
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub struct DataSource {
+    pub id: String,
+    pub source_type: DataSourceType,
+    pub config: ConnectorConfig,
+}
+
+/// Data sink configuration
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub struct DataSink {
+    pub id: String,
+    pub sink_type: DataSinkType,
+    pub config: ConnectorConfig,
+}
+
+/// Data source types
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub enum DataSourceType {
+    Database(DatabaseConfig),
+    File(FileConfig),
+    Stream(StreamConfig),
+    Api(ApiConfig),
+}
+
+/// Data sink types
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub enum DataSinkType {
+    Database(DatabaseConfig),
+    File(FileConfig),
+    Stream(StreamConfig),
+    Api(ApiConfig),
+}
+
+/// Database configuration
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub struct DatabaseConfig {
+    pub connection_string: String,
+    pub table: String,
+    pub schema: Option<String>,
+}
+
+/// File configuration
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub struct FileConfig {
+    pub path: String,
+    pub format: FileFormat,
+    pub compression: Option<CompressionType>,
+}
+
+/// Stream configuration
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub struct StreamConfig {
+    pub topic: String,
+    pub bootstrap_servers: String,
+    pub consumer_group: Option<String>,
+}
+
+/// API configuration
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub struct ApiConfig {
+    pub endpoint: String,
+    pub method: HttpMethod,
+    pub headers: HashMap<String, String>,
+}
+
+/// File formats
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub enum FileFormat {
+    Json,
+    Csv,
+    Parquet,
+    Avro,
+}
+
+/// Compression types
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub enum CompressionType {
+    Gzip,
+    Snappy,
+    Lz4,
+}
+
+/// HTTP methods
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub enum HttpMethod {
+    Get,
+    Post,
+    Put,
+    Delete,
+}
+
+/// Connector configuration
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub struct ConnectorConfig {
+    pub properties: HashMap<String, String>,
+    pub batch_size: Option<usize>,
+    pub timeout_ms: Option<u64>,
+}
+
+/// Generic connector trait
+pub trait Connector {
+    fn connect(&self) -> impl std::future::Future<Output = Result<()>> + Send;
+    fn disconnect(&self) -> impl std::future::Future<Output = Result<()>> + Send;
+}
+
+/// File connector
+pub struct FileConnector {
+    config: FileConfig,
+}
+
+/// Database connector
+pub struct DatabaseConnector {
+    config: DatabaseConfig,
+}
+
+/// Stream connector
+pub struct StreamConnector {
+    config: StreamConfig,
+}
+
+/// Stream source trait
+pub trait StreamSource {
+    fn read(&self) -> impl std::future::Future<Output = Result<Vec<DataRecord>>> + Send;
+}
+
+/// Stream sink trait
+pub trait StreamSink {
+    fn write(
+        &self,
+        records: Vec<DataRecord>,
+    ) -> impl std::future::Future<Output = Result<()>> + Send;
+}
+
+impl DataSource {
+    #[must_use]
+    pub fn database(connection_string: &str) -> Self {
+        Self {
+            id: uuid::Uuid::new_v4().to_string(),
+            source_type: DataSourceType::Database(DatabaseConfig {
+                connection_string: connection_string.to_string(),
+                table: "default".to_string(),
+                schema: None,
+            }),
+            config: ConnectorConfig {
+                properties: HashMap::new(),
+                batch_size: Some(1000),
+                timeout_ms: Some(30000),
+            },
+        }
+    }
+
+    #[must_use]
+    pub fn id(&self) -> &str {
+        &self.id
+    }
+}
+
+impl DataSink {
+    #[must_use]
+    pub fn parquet(path: &str) -> Self {
+        Self {
+            id: uuid::Uuid::new_v4().to_string(),
+            sink_type: DataSinkType::File(FileConfig {
+                path: path.to_string(),
+                format: FileFormat::Parquet,
+                compression: Some(CompressionType::Snappy),
+            }),
+            config: ConnectorConfig {
+                properties: HashMap::new(),
+                batch_size: Some(10000),
+                timeout_ms: Some(60000),
+            },
+        }
+    }
+
+    #[must_use]
+    pub fn id(&self) -> &str {
+        &self.id
+    }
+}