//! # RTX Monitoring - Observability and Metrics Collection //! //! RTX Monitoring provides comprehensive observability for RustyTorch production //! deployments, including metrics collection, distributed tracing, health checks, //! and alerting. //! //! ## Core Features //! //! - Prometheus metrics collection and export //! - OpenTelemetry distributed tracing //! - System and application health monitoring //! - Custom metrics and dashboards //! - Alert management and notification //! - Performance monitoring and SLI/SLO tracking #![deny(clippy::unwrap_used)] #![cfg_attr(test, allow(clippy::unwrap_used))] pub mod alerts; pub mod collector; pub mod error; pub mod health; pub mod metrics; pub mod telemetry; pub mod trace_utils; pub use alerts::{ AlertCondition, AlertManager, AlertRule, AlertSeverity, AlertState, ChannelType, Notification, NotificationChannel, presets as alert_presets, }; pub use collector::{CollectorConfig, MetricsCollector, SystemMetrics}; pub use error::{MonitoringError, MonitoringResult}; pub use health::{HealthCheck, HealthManager, HealthStatus}; pub use metrics::{ BATCH_SIZE_BUCKETS, CustomMetric, INFERENCE_LATENCY_BUCKETS, InferenceMetrics, MetricRegistry, MetricType, TrainingMetrics, }; pub use telemetry::{ SpanContext, SpanData, SpanEvent, SpanGuard, SpanStatus, TelemetryManager, TraceConfig, generate_span_id, generate_trace_id, }; use std::sync::Arc; // Import tracing macros with alias to avoid conflict with local tracing module /// Shared monitoring system instance pub type SharedMonitoringSystem = Arc; /// Main monitoring system that coordinates all observability components #[derive(Debug)] pub struct MonitoringSystem { /// Metrics collector pub metrics: Arc, /// Health manager pub health: Arc, /// Alert manager pub alerts: Arc, /// Telemetry manager pub telemetry: Arc, /// System configuration config: MonitoringConfig, } /// Configuration for the monitoring system #[derive(Debug, Clone)] pub struct MonitoringConfig { /// Enable Prometheus metrics pub enable_prometheus: bool, /// Enable OpenTelemetry tracing pub enable_telemetry: bool, /// Enable health checks pub enable_health_checks: bool, /// Enable alerting pub enable_alerting: bool, /// Metrics collection interval pub collection_interval: std::time::Duration, /// Health check interval pub health_check_interval: std::time::Duration, /// Metrics server port pub metrics_port: u16, /// Health check port pub health_port: u16, } impl Default for MonitoringConfig { fn default() -> Self { Self { enable_prometheus: true, enable_telemetry: true, enable_health_checks: true, enable_alerting: false, collection_interval: std::time::Duration::from_secs(30), health_check_interval: std::time::Duration::from_secs(10), metrics_port: 9090, health_port: 8080, } } } impl MonitoringSystem { /// Create a new monitoring system pub async fn new(config: MonitoringConfig) -> MonitoringResult { let metrics = Arc::new( MetricsCollector::new(CollectorConfig { enable_prometheus: config.enable_prometheus, collection_interval: config.collection_interval, ..Default::default() }) .await?, ); let health = Arc::new(HealthManager::new().await?); let alerts = Arc::new(AlertManager::new().await?); let telemetry = Arc::new(TelemetryManager::new(TraceConfig::default()).await?); let enable_prometheus = config.enable_prometheus; let enable_health_checks = config.enable_health_checks; let system = Self { metrics, health, alerts, telemetry, config, }; // Start background tasks if enable_prometheus { system.start_metrics_server().await?; } if enable_health_checks { system.start_health_checks().await?; } Ok(system) } /// Start the metrics HTTP server async fn start_metrics_server(&self) -> MonitoringResult<()> { let metrics = self.metrics.clone(); let port = self.config.metrics_port; tokio::spawn(async move { if let Err(e) = run_metrics_server(metrics, port).await { tracing::error!("Metrics server error: {}", e); } }); Ok(()) } /// Start health check background tasks async fn start_health_checks(&self) -> MonitoringResult<()> { let health = self.health.clone(); let interval = self.config.health_check_interval; tokio::spawn(async move { let mut ticker = tokio::time::interval(interval); loop { ticker.tick().await; if let Err(e) = health.run_all_checks().await { tracing::error!("Health check error: {}", e); } } }); Ok(()) } /// Get system metrics summary pub async fn get_metrics_summary(&self) -> MonitoringResult { let system_metrics = self.metrics.collect_system_metrics().await?; let health_status = self.health.overall_status().await?; let active_alerts = self.alerts.active_alerts().await?; Ok(MetricsSummary { system_metrics, health_status, active_alerts: active_alerts.len(), uptime: self.get_uptime(), }) } /// Get system uptime fn get_uptime(&self) -> std::time::Duration { // Placeholder implementation std::time::Duration::from_secs(3600) } } /// Summary of system metrics #[derive(Debug, Clone, serde::Serialize, serde::Deserialize)] pub struct MetricsSummary { pub system_metrics: SystemMetrics, pub health_status: HealthStatus, pub active_alerts: usize, pub uptime: std::time::Duration, } /// Run the metrics HTTP server async fn run_metrics_server(metrics: Arc, port: u16) -> MonitoringResult<()> { use axum::{Router, routing::get}; let app = Router::new().route( "/metrics", get(move || async move { match metrics.export_prometheus().await { Ok(metrics_text) => axum::response::Response::builder() .header("Content-Type", "text/plain; version=0.0.4; charset=utf-8") .body(metrics_text) .expect("failed to build successful metrics response"), Err(e) => { tracing::error!("Failed to export Prometheus metrics: {}", e); axum::response::Response::builder() .status(500) .body("Internal Server Error".to_string()) .expect("failed to build error response") } } }), ); let addr = format!("0.0.0.0:{port}"); let listener = tokio::net::TcpListener::bind(&addr) .await .map_err(|e| MonitoringError::ConfigError { details: format!("Failed to bind metrics server to {addr}: {e}"), })?; tracing::info!("Metrics server listening on {}", addr); axum::serve(listener, app) .await .map_err(|e| MonitoringError::ServerError { details: e.to_string(), })?; Ok(()) } /// Create a production-ready monitoring system pub async fn create_production_monitoring() -> MonitoringResult { let config = MonitoringConfig { enable_prometheus: true, enable_telemetry: true, enable_health_checks: true, enable_alerting: true, collection_interval: std::time::Duration::from_secs(15), health_check_interval: std::time::Duration::from_secs(5), metrics_port: 9090, health_port: 8080, }; MonitoringSystem::new(config).await } /// Create a development monitoring system pub async fn create_development_monitoring() -> MonitoringResult { let config = MonitoringConfig { enable_prometheus: true, enable_telemetry: false, enable_health_checks: true, enable_alerting: false, collection_interval: std::time::Duration::from_secs(60), health_check_interval: std::time::Duration::from_secs(30), metrics_port: 9090, health_port: 8080, }; MonitoringSystem::new(config).await } #[cfg(test)] mod tests { use super::*; #[tokio::test] async fn test_monitoring_system_creation() { let config = MonitoringConfig::default(); let system = MonitoringSystem::new(config).await.unwrap(); assert!(system.metrics.is_enabled()); assert!(system.health.is_enabled()); } #[tokio::test] async fn test_production_monitoring_setup() { let system = create_production_monitoring().await.unwrap(); assert!(system.config.enable_prometheus); assert!(system.config.enable_telemetry); assert!(system.config.enable_health_checks); assert!(system.config.enable_alerting); } #[tokio::test] async fn test_development_monitoring_setup() { let system = create_development_monitoring().await.unwrap(); assert!(system.config.enable_prometheus); assert!(!system.config.enable_telemetry); assert!(system.config.enable_health_checks); assert!(!system.config.enable_alerting); } }