Initial commit

crates/production/rtx-monitoring/src/lib.rs

@@ -0,0 +1,305 @@
+//! # RTX Monitoring - Observability and Metrics Collection
+//!
+//! RTX Monitoring provides comprehensive observability for RustyTorch production
+//! deployments, including metrics collection, distributed tracing, health checks,
+//! and alerting.
+//!
+//! ## Core Features
+//!
+//! - Prometheus metrics collection and export
+//! - OpenTelemetry distributed tracing
+//! - System and application health monitoring
+//! - Custom metrics and dashboards
+//! - Alert management and notification
+//! - Performance monitoring and SLI/SLO tracking
+
+#![deny(clippy::unwrap_used)]
+#![cfg_attr(test, allow(clippy::unwrap_used))]
+
+pub mod alerts;
+pub mod collector;
+pub mod error;
+pub mod health;
+pub mod metrics;
+pub mod telemetry;
+pub mod trace_utils;
+
+pub use alerts::{
+    AlertCondition, AlertManager, AlertRule, AlertSeverity, AlertState, ChannelType, Notification,
+    NotificationChannel, presets as alert_presets,
+};
+pub use collector::{CollectorConfig, MetricsCollector, SystemMetrics};
+pub use error::{MonitoringError, MonitoringResult};
+pub use health::{HealthCheck, HealthManager, HealthStatus};
+pub use metrics::{
+    BATCH_SIZE_BUCKETS, CustomMetric, INFERENCE_LATENCY_BUCKETS, InferenceMetrics, MetricRegistry,
+    MetricType, TrainingMetrics,
+};
+pub use telemetry::{
+    SpanContext, SpanData, SpanEvent, SpanGuard, SpanStatus, TelemetryManager, TraceConfig,
+    generate_span_id, generate_trace_id,
+};
+
+use std::sync::Arc;
+// Import tracing macros with alias to avoid conflict with local tracing module
+
+/// Shared monitoring system instance
+pub type SharedMonitoringSystem = Arc<MonitoringSystem>;
+
+/// Main monitoring system that coordinates all observability components
+#[derive(Debug)]
+pub struct MonitoringSystem {
+    /// Metrics collector
+    pub metrics: Arc<MetricsCollector>,
+    /// Health manager
+    pub health: Arc<HealthManager>,
+    /// Alert manager
+    pub alerts: Arc<AlertManager>,
+    /// Telemetry manager
+    pub telemetry: Arc<TelemetryManager>,
+    /// System configuration
+    config: MonitoringConfig,
+}
+
+/// Configuration for the monitoring system
+#[derive(Debug, Clone)]
+pub struct MonitoringConfig {
+    /// Enable Prometheus metrics
+    pub enable_prometheus: bool,
+    /// Enable OpenTelemetry tracing
+    pub enable_telemetry: bool,
+    /// Enable health checks
+    pub enable_health_checks: bool,
+    /// Enable alerting
+    pub enable_alerting: bool,
+    /// Metrics collection interval
+    pub collection_interval: std::time::Duration,
+    /// Health check interval
+    pub health_check_interval: std::time::Duration,
+    /// Metrics server port
+    pub metrics_port: u16,
+    /// Health check port
+    pub health_port: u16,
+}
+
+impl Default for MonitoringConfig {
+    fn default() -> Self {
+        Self {
+            enable_prometheus: true,
+            enable_telemetry: true,
+            enable_health_checks: true,
+            enable_alerting: false,
+            collection_interval: std::time::Duration::from_secs(30),
+            health_check_interval: std::time::Duration::from_secs(10),
+            metrics_port: 9090,
+            health_port: 8080,
+        }
+    }
+}
+
+impl MonitoringSystem {
+    /// Create a new monitoring system
+    pub async fn new(config: MonitoringConfig) -> MonitoringResult<Self> {
+        let metrics = Arc::new(
+            MetricsCollector::new(CollectorConfig {
+                enable_prometheus: config.enable_prometheus,
+                collection_interval: config.collection_interval,
+                ..Default::default()
+            })
+            .await?,
+        );
+
+        let health = Arc::new(HealthManager::new().await?);
+        let alerts = Arc::new(AlertManager::new().await?);
+        let telemetry = Arc::new(TelemetryManager::new(TraceConfig::default()).await?);
+
+        let enable_prometheus = config.enable_prometheus;
+        let enable_health_checks = config.enable_health_checks;
+
+        let system = Self {
+            metrics,
+            health,
+            alerts,
+            telemetry,
+            config,
+        };
+
+        // Start background tasks
+        if enable_prometheus {
+            system.start_metrics_server().await?;
+        }
+
+        if enable_health_checks {
+            system.start_health_checks().await?;
+        }
+
+        Ok(system)
+    }
+
+    /// Start the metrics HTTP server
+    async fn start_metrics_server(&self) -> MonitoringResult<()> {
+        let metrics = self.metrics.clone();
+        let port = self.config.metrics_port;
+
+        tokio::spawn(async move {
+            if let Err(e) = run_metrics_server(metrics, port).await {
+                tracing::error!("Metrics server error: {}", e);
+            }
+        });
+
+        Ok(())
+    }
+
+    /// Start health check background tasks
+    async fn start_health_checks(&self) -> MonitoringResult<()> {
+        let health = self.health.clone();
+        let interval = self.config.health_check_interval;
+
+        tokio::spawn(async move {
+            let mut ticker = tokio::time::interval(interval);
+            loop {
+                ticker.tick().await;
+                if let Err(e) = health.run_all_checks().await {
+                    tracing::error!("Health check error: {}", e);
+                }
+            }
+        });
+
+        Ok(())
+    }
+
+    /// Get system metrics summary
+    pub async fn get_metrics_summary(&self) -> MonitoringResult<MetricsSummary> {
+        let system_metrics = self.metrics.collect_system_metrics().await?;
+        let health_status = self.health.overall_status().await?;
+        let active_alerts = self.alerts.active_alerts().await?;
+
+        Ok(MetricsSummary {
+            system_metrics,
+            health_status,
+            active_alerts: active_alerts.len(),
+            uptime: self.get_uptime(),
+        })
+    }
+
+    /// Get system uptime
+    fn get_uptime(&self) -> std::time::Duration {
+        // Placeholder implementation
+        std::time::Duration::from_secs(3600)
+    }
+}
+
+/// Summary of system metrics
+#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)]
+pub struct MetricsSummary {
+    pub system_metrics: SystemMetrics,
+    pub health_status: HealthStatus,
+    pub active_alerts: usize,
+    pub uptime: std::time::Duration,
+}
+
+/// Run the metrics HTTP server
+async fn run_metrics_server(metrics: Arc<MetricsCollector>, port: u16) -> MonitoringResult<()> {
+    use axum::{Router, routing::get};
+
+    let app = Router::new().route(
+        "/metrics",
+        get(move || async move {
+            match metrics.export_prometheus().await {
+                Ok(metrics_text) => axum::response::Response::builder()
+                    .header("Content-Type", "text/plain; version=0.0.4; charset=utf-8")
+                    .body(metrics_text)
+                    .expect("failed to build successful metrics response"),
+                Err(e) => {
+                    tracing::error!("Failed to export Prometheus metrics: {}", e);
+                    axum::response::Response::builder()
+                        .status(500)
+                        .body("Internal Server Error".to_string())
+                        .expect("failed to build error response")
+                }
+            }
+        }),
+    );
+
+    let addr = format!("0.0.0.0:{port}");
+    let listener =
+        tokio::net::TcpListener::bind(&addr)
+            .await
+            .map_err(|e| MonitoringError::ConfigError {
+                details: format!("Failed to bind metrics server to {addr}: {e}"),
+            })?;
+
+    tracing::info!("Metrics server listening on {}", addr);
+
+    axum::serve(listener, app)
+        .await
+        .map_err(|e| MonitoringError::ServerError {
+            details: e.to_string(),
+        })?;
+
+    Ok(())
+}
+
+/// Create a production-ready monitoring system
+pub async fn create_production_monitoring() -> MonitoringResult<MonitoringSystem> {
+    let config = MonitoringConfig {
+        enable_prometheus: true,
+        enable_telemetry: true,
+        enable_health_checks: true,
+        enable_alerting: true,
+        collection_interval: std::time::Duration::from_secs(15),
+        health_check_interval: std::time::Duration::from_secs(5),
+        metrics_port: 9090,
+        health_port: 8080,
+    };
+
+    MonitoringSystem::new(config).await
+}
+
+/// Create a development monitoring system
+pub async fn create_development_monitoring() -> MonitoringResult<MonitoringSystem> {
+    let config = MonitoringConfig {
+        enable_prometheus: true,
+        enable_telemetry: false,
+        enable_health_checks: true,
+        enable_alerting: false,
+        collection_interval: std::time::Duration::from_secs(60),
+        health_check_interval: std::time::Duration::from_secs(30),
+        metrics_port: 9090,
+        health_port: 8080,
+    };
+
+    MonitoringSystem::new(config).await
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[tokio::test]
+    async fn test_monitoring_system_creation() {
+        let config = MonitoringConfig::default();
+        let system = MonitoringSystem::new(config).await.unwrap();
+
+        assert!(system.metrics.is_enabled());
+        assert!(system.health.is_enabled());
+    }
+
+    #[tokio::test]
+    async fn test_production_monitoring_setup() {
+        let system = create_production_monitoring().await.unwrap();
+        assert!(system.config.enable_prometheus);
+        assert!(system.config.enable_telemetry);
+        assert!(system.config.enable_health_checks);
+        assert!(system.config.enable_alerting);
+    }
+
+    #[tokio::test]
+    async fn test_development_monitoring_setup() {
+        let system = create_development_monitoring().await.unwrap();
+        assert!(system.config.enable_prometheus);
+        assert!(!system.config.enable_telemetry);
+        assert!(system.config.enable_health_checks);
+        assert!(!system.config.enable_alerting);
+    }
+}