rustytorch/crates/production/rtx-monitoring/src/lib.rs

//! # RTX Monitoring - Observability and Metrics Collection
//!
//! RTX Monitoring provides comprehensive observability for RustyTorch production
//! deployments, including metrics collection, distributed tracing, health checks,
//! and alerting.
//!
//! ## Core Features
//!
//! - Prometheus metrics collection and export
//! - OpenTelemetry distributed tracing
//! - System and application health monitoring
//! - Custom metrics and dashboards
//! - Alert management and notification
//! - Performance monitoring and SLI/SLO tracking

#![deny(clippy::unwrap_used)]
#![cfg_attr(test, allow(clippy::unwrap_used))]

pub mod alerts;
pub mod collector;
pub mod error;
pub mod health;
pub mod metrics;
pub mod telemetry;
pub mod trace_utils;

pub use alerts::{
    AlertCondition, AlertManager, AlertRule, AlertSeverity, AlertState, ChannelType, Notification,
    NotificationChannel, presets as alert_presets,
};
pub use collector::{CollectorConfig, MetricsCollector, SystemMetrics};
pub use error::{MonitoringError, MonitoringResult};
pub use health::{HealthCheck, HealthManager, HealthStatus};
pub use metrics::{
    BATCH_SIZE_BUCKETS, CustomMetric, INFERENCE_LATENCY_BUCKETS, InferenceMetrics, MetricRegistry,
    MetricType, TrainingMetrics,
};
pub use telemetry::{
    SpanContext, SpanData, SpanEvent, SpanGuard, SpanStatus, TelemetryManager, TraceConfig,
    generate_span_id, generate_trace_id,
};

use std::sync::Arc;
// Import tracing macros with alias to avoid conflict with local tracing module

/// Shared monitoring system instance
pub type SharedMonitoringSystem = Arc<MonitoringSystem>;

/// Main monitoring system that coordinates all observability components
#[derive(Debug)]
pub struct MonitoringSystem {
    /// Metrics collector
    pub metrics: Arc<MetricsCollector>,
    /// Health manager
    pub health: Arc<HealthManager>,
    /// Alert manager
    pub alerts: Arc<AlertManager>,
    /// Telemetry manager
    pub telemetry: Arc<TelemetryManager>,
    /// System configuration
    config: MonitoringConfig,
}

/// Configuration for the monitoring system
#[derive(Debug, Clone)]
pub struct MonitoringConfig {
    /// Enable Prometheus metrics
    pub enable_prometheus: bool,
    /// Enable OpenTelemetry tracing
    pub enable_telemetry: bool,
    /// Enable health checks
    pub enable_health_checks: bool,
    /// Enable alerting
    pub enable_alerting: bool,
    /// Metrics collection interval
    pub collection_interval: std::time::Duration,
    /// Health check interval
    pub health_check_interval: std::time::Duration,
    /// Metrics server port
    pub metrics_port: u16,
    /// Health check port
    pub health_port: u16,
}

impl Default for MonitoringConfig {
    fn default() -> Self {
        Self {
            enable_prometheus: true,
            enable_telemetry: true,
            enable_health_checks: true,
            enable_alerting: false,
            collection_interval: std::time::Duration::from_secs(30),
            health_check_interval: std::time::Duration::from_secs(10),
            metrics_port: 9090,
            health_port: 8080,
        }
    }
}

impl MonitoringSystem {
    /// Create a new monitoring system
    pub async fn new(config: MonitoringConfig) -> MonitoringResult<Self> {
        let metrics = Arc::new(
            MetricsCollector::new(CollectorConfig {
                enable_prometheus: config.enable_prometheus,
                collection_interval: config.collection_interval,
                ..Default::default()
            })
            .await?,
        );

        let health = Arc::new(HealthManager::new().await?);
        let alerts = Arc::new(AlertManager::new().await?);
        let telemetry = Arc::new(TelemetryManager::new(TraceConfig::default()).await?);

        let enable_prometheus = config.enable_prometheus;
        let enable_health_checks = config.enable_health_checks;

        let system = Self {
            metrics,
            health,
            alerts,
            telemetry,
            config,
        };

        // Start background tasks
        if enable_prometheus {
            system.start_metrics_server().await?;
        }

        if enable_health_checks {
            system.start_health_checks().await?;
        }

        Ok(system)
    }

    /// Start the metrics HTTP server
    async fn start_metrics_server(&self) -> MonitoringResult<()> {
        let metrics = self.metrics.clone();
        let port = self.config.metrics_port;

        tokio::spawn(async move {
            if let Err(e) = run_metrics_server(metrics, port).await {
                tracing::error!("Metrics server error: {}", e);
            }
        });

        Ok(())
    }

    /// Start health check background tasks
    async fn start_health_checks(&self) -> MonitoringResult<()> {
        let health = self.health.clone();
        let interval = self.config.health_check_interval;

        tokio::spawn(async move {
            let mut ticker = tokio::time::interval(interval);
            loop {
                ticker.tick().await;
                if let Err(e) = health.run_all_checks().await {
                    tracing::error!("Health check error: {}", e);
                }
            }
        });

        Ok(())
    }

    /// Get system metrics summary
    pub async fn get_metrics_summary(&self) -> MonitoringResult<MetricsSummary> {
        let system_metrics = self.metrics.collect_system_metrics().await?;
        let health_status = self.health.overall_status().await?;
        let active_alerts = self.alerts.active_alerts().await?;

        Ok(MetricsSummary {
            system_metrics,
            health_status,
            active_alerts: active_alerts.len(),
            uptime: self.get_uptime(),
        })
    }

    /// Get system uptime
    fn get_uptime(&self) -> std::time::Duration {
        // Placeholder implementation
        std::time::Duration::from_secs(3600)
    }
}

/// Summary of system metrics
#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)]
pub struct MetricsSummary {
    pub system_metrics: SystemMetrics,
    pub health_status: HealthStatus,
    pub active_alerts: usize,
    pub uptime: std::time::Duration,
}

/// Run the metrics HTTP server
async fn run_metrics_server(metrics: Arc<MetricsCollector>, port: u16) -> MonitoringResult<()> {
    use axum::{Router, routing::get};

    let app = Router::new().route(
        "/metrics",
        get(move || async move {
            match metrics.export_prometheus().await {
                Ok(metrics_text) => axum::response::Response::builder()
                    .header("Content-Type", "text/plain; version=0.0.4; charset=utf-8")
                    .body(metrics_text)
                    .expect("failed to build successful metrics response"),
                Err(e) => {
                    tracing::error!("Failed to export Prometheus metrics: {}", e);
                    axum::response::Response::builder()
                        .status(500)
                        .body("Internal Server Error".to_string())
                        .expect("failed to build error response")
                }
            }
        }),
    );

    let addr = format!("0.0.0.0:{port}");
    let listener =
        tokio::net::TcpListener::bind(&addr)
            .await
            .map_err(|e| MonitoringError::ConfigError {
                details: format!("Failed to bind metrics server to {addr}: {e}"),
            })?;

    tracing::info!("Metrics server listening on {}", addr);

    axum::serve(listener, app)
        .await
        .map_err(|e| MonitoringError::ServerError {
            details: e.to_string(),
        })?;

    Ok(())
}

/// Create a production-ready monitoring system
pub async fn create_production_monitoring() -> MonitoringResult<MonitoringSystem> {
    let config = MonitoringConfig {
        enable_prometheus: true,
        enable_telemetry: true,
        enable_health_checks: true,
        enable_alerting: true,
        collection_interval: std::time::Duration::from_secs(15),
        health_check_interval: std::time::Duration::from_secs(5),
        metrics_port: 9090,
        health_port: 8080,
    };

    MonitoringSystem::new(config).await
}

/// Create a development monitoring system
pub async fn create_development_monitoring() -> MonitoringResult<MonitoringSystem> {
    let config = MonitoringConfig {
        enable_prometheus: true,
        enable_telemetry: false,
        enable_health_checks: true,
        enable_alerting: false,
        collection_interval: std::time::Duration::from_secs(60),
        health_check_interval: std::time::Duration::from_secs(30),
        metrics_port: 9090,
        health_port: 8080,
    };

    MonitoringSystem::new(config).await
}

#[cfg(test)]
mod tests {
    use super::*;

    #[tokio::test]
    async fn test_monitoring_system_creation() {
        let config = MonitoringConfig::default();
        let system = MonitoringSystem::new(config).await.unwrap();

        assert!(system.metrics.is_enabled());
        assert!(system.health.is_enabled());
    }

    #[tokio::test]
    async fn test_production_monitoring_setup() {
        let system = create_production_monitoring().await.unwrap();
        assert!(system.config.enable_prometheus);
        assert!(system.config.enable_telemetry);
        assert!(system.config.enable_health_checks);
        assert!(system.config.enable_alerting);
    }

    #[tokio::test]
    async fn test_development_monitoring_setup() {
        let system = create_development_monitoring().await.unwrap();
        assert!(system.config.enable_prometheus);
        assert!(!system.config.enable_telemetry);
        assert!(system.config.enable_health_checks);
        assert!(!system.config.enable_alerting);
    }
}