//! Alert management and notification system. //! //! This module provides alerting capabilities for RustyTorch++ including //! alert rule evaluation, notification dispatch, and alert history tracking. use crate::MonitoringResult; use chrono::{DateTime, Duration, Utc}; use parking_lot::RwLock; use serde::{Deserialize, Serialize}; use std::collections::HashMap; use std::sync::Arc; /// Alert severity levels. #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)] pub enum AlertSeverity { /// Critical alert - immediate action required Critical, /// Warning - attention needed soon Warning, /// Info - informational alert Info, } impl AlertSeverity { /// Get string representation pub fn as_str(&self) -> &'static str { match self { Self::Critical => "critical", Self::Warning => "warning", Self::Info => "info", } } } /// Alert state for tracking firing status #[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] pub enum AlertState { /// Alert is currently firing Firing, /// Alert was firing but is now resolved Resolved, /// Alert is pending (condition met but waiting for duration) Pending, } /// Alert rule definition. #[derive(Debug, Clone, Serialize, Deserialize)] pub struct AlertRule { /// Unique rule name pub name: String, /// Alert severity pub severity: AlertSeverity, /// Human-readable description pub description: String, /// Metric name to evaluate pub metric: String, /// Threshold condition pub condition: AlertCondition, /// Duration the condition must hold before firing pub duration: std::time::Duration, /// Labels to add to the alert pub labels: HashMap, /// Annotations (additional metadata) pub annotations: HashMap, } impl AlertRule { /// Create a new alert rule pub fn new( name: &str, metric: &str, condition: AlertCondition, severity: AlertSeverity, ) -> Self { Self { name: name.to_string(), severity, description: String::new(), metric: metric.to_string(), condition, duration: std::time::Duration::from_secs(0), labels: HashMap::new(), annotations: HashMap::new(), } } /// Set description pub fn with_description(mut self, desc: &str) -> Self { self.description = desc.to_string(); self } /// Set duration before firing pub fn with_duration(mut self, duration: std::time::Duration) -> Self { self.duration = duration; self } /// Add label pub fn with_label(mut self, key: &str, value: &str) -> Self { self.labels.insert(key.to_string(), value.to_string()); self } /// Add annotation pub fn with_annotation(mut self, key: &str, value: &str) -> Self { self.annotations.insert(key.to_string(), value.to_string()); self } } /// Alert condition types #[derive(Debug, Clone, Serialize, Deserialize)] pub enum AlertCondition { /// Value is greater than threshold GreaterThan(f64), /// Value is less than threshold LessThan(f64), /// Value is greater than or equal to threshold GreaterThanOrEqual(f64), /// Value is less than or equal to threshold LessThanOrEqual(f64), /// Value equals threshold Equal(f64), /// Value is not equal to threshold NotEqual(f64), /// Value is absent (no data) Absent, } impl AlertCondition { /// Evaluate the condition against a value pub fn evaluate(&self, value: Option) -> bool { match (self, value) { (Self::GreaterThan(threshold), Some(v)) => v > *threshold, (Self::LessThan(threshold), Some(v)) => v < *threshold, (Self::GreaterThanOrEqual(threshold), Some(v)) => v >= *threshold, (Self::LessThanOrEqual(threshold), Some(v)) => v <= *threshold, (Self::Equal(threshold), Some(v)) => (v - threshold).abs() < f64::EPSILON, (Self::NotEqual(threshold), Some(v)) => (v - threshold).abs() >= f64::EPSILON, (Self::Absent, None) => true, (Self::Absent, Some(_)) => false, (_, None) => false, } } } /// Notification configuration. #[derive(Debug, Clone, Serialize, Deserialize)] pub struct Notification { /// Unique notification ID pub id: String, /// Alert message pub message: String, /// Alert severity pub severity: AlertSeverity, /// When the notification was created pub timestamp: DateTime, /// Alert state pub state: AlertState, /// Rule name that triggered this notification pub rule_name: String, /// Labels from the alert pub labels: HashMap, /// Current metric value pub value: Option, } impl Notification { /// Create a new notification from a rule pub fn from_rule(rule: &AlertRule, state: AlertState, value: Option) -> Self { Self { id: uuid::Uuid::new_v4().to_string(), message: rule.description.clone(), severity: rule.severity, timestamp: Utc::now(), state, rule_name: rule.name.clone(), labels: rule.labels.clone(), value, } } } /// Notification channel for dispatching alerts #[derive(Debug, Clone, Serialize, Deserialize)] pub struct NotificationChannel { /// Channel name pub name: String, /// Channel type pub channel_type: ChannelType, /// Channel configuration pub config: HashMap, /// Minimum severity to send pub min_severity: AlertSeverity, } /// Types of notification channels #[derive(Debug, Clone, Serialize, Deserialize)] pub enum ChannelType { /// Webhook (HTTP POST) Webhook, /// Email Email, /// Slack Slack, /// PagerDuty PagerDuty, /// Console/Log output Console, } /// Active alert tracking #[derive(Debug, Clone)] struct ActiveAlert { rule: AlertRule, first_firing: DateTime, last_evaluation: DateTime, state: AlertState, value: Option, } /// Alert manager for handling alerts and notifications. #[derive(Debug)] pub struct AlertManager { /// Registered alert rules rules: Arc>>, /// Active alerts active_alerts: Arc>>, /// Notification history notifications: Arc>>, /// Notification channels channels: Arc>>, /// Maximum notifications to keep in history max_history: usize, /// Whether the manager is enabled enabled: bool, } impl AlertManager { /// Create a new alert manager pub async fn new() -> MonitoringResult { Ok(Self { rules: Arc::new(RwLock::new(HashMap::new())), active_alerts: Arc::new(RwLock::new(HashMap::new())), notifications: Arc::new(RwLock::new(Vec::new())), channels: Arc::new(RwLock::new(Vec::new())), max_history: 10000, enabled: true, }) } /// Register an alert rule pub fn register_rule(&self, rule: AlertRule) { self.rules.write().insert(rule.name.clone(), rule); } /// Remove an alert rule pub fn remove_rule(&self, name: &str) { self.rules.write().remove(name); self.active_alerts.write().remove(name); } /// Add a notification channel pub fn add_channel(&self, channel: NotificationChannel) { self.channels.write().push(channel); } /// Evaluate a metric value against all rules pub async fn evaluate(&self, metric: &str, value: Option) -> MonitoringResult<()> { let rules = self.rules.read(); let now = Utc::now(); for rule in rules.values() { if rule.metric != metric { continue; } let condition_met = rule.condition.evaluate(value); let mut active_alerts = self.active_alerts.write(); if condition_met { if let Some(active) = active_alerts.get_mut(&rule.name) { // Already tracking this alert active.last_evaluation = now; active.value = value; // Check if duration has passed if active.state == AlertState::Pending { let elapsed = now - active.first_firing; if elapsed >= Duration::from_std(rule.duration).unwrap_or(Duration::zero()) { active.state = AlertState::Firing; self.fire_alert(rule, value).await?; } } } else { // New alert let state = if rule.duration.is_zero() { AlertState::Firing } else { AlertState::Pending }; active_alerts.insert( rule.name.clone(), ActiveAlert { rule: rule.clone(), first_firing: now, last_evaluation: now, state, value, }, ); if state == AlertState::Firing { drop(active_alerts); self.fire_alert(rule, value).await?; } } } else { // Condition no longer met if let Some(active) = active_alerts.remove(&rule.name) && active.state == AlertState::Firing { // Send resolved notification drop(active_alerts); self.resolve_alert(rule, value).await?; } } } Ok(()) } /// Fire an alert async fn fire_alert(&self, rule: &AlertRule, value: Option) -> MonitoringResult<()> { let notification = Notification::from_rule(rule, AlertState::Firing, value); // Add to history { let mut notifications = self.notifications.write(); if notifications.len() >= self.max_history { notifications.remove(0); } notifications.push(notification.clone()); } // Dispatch to channels self.dispatch_notification(¬ification).await?; tracing::warn!( rule = %rule.name, severity = %rule.severity.as_str(), value = ?value, "Alert fired: {}", rule.description ); Ok(()) } /// Resolve an alert async fn resolve_alert(&self, rule: &AlertRule, value: Option) -> MonitoringResult<()> { let notification = Notification::from_rule(rule, AlertState::Resolved, value); // Add to history { let mut notifications = self.notifications.write(); if notifications.len() >= self.max_history { notifications.remove(0); } notifications.push(notification.clone()); } // Dispatch to channels self.dispatch_notification(¬ification).await?; tracing::info!( rule = %rule.name, "Alert resolved: {}", rule.description ); Ok(()) } /// Dispatch notification to all configured channels async fn dispatch_notification(&self, notification: &Notification) -> MonitoringResult<()> { let channels = self.channels.read(); for channel in channels.iter() { // Check severity filter let should_send = match (channel.min_severity, notification.severity) { (AlertSeverity::Critical, AlertSeverity::Critical) => true, (AlertSeverity::Warning, AlertSeverity::Critical | AlertSeverity::Warning) => true, (AlertSeverity::Info, _) => true, _ => false, }; if !should_send { continue; } match channel.channel_type { ChannelType::Console => { tracing::info!( channel = %channel.name, severity = %notification.severity.as_str(), state = ?notification.state, "Alert: {} - {}", notification.rule_name, notification.message ); } ChannelType::Webhook => { // In a real implementation, this would make an HTTP POST if let Some(url) = channel.config.get("url") { tracing::debug!("Would send webhook to: {}", url); } } ChannelType::Slack => { // In a real implementation, this would send to Slack if let Some(webhook) = channel.config.get("webhook_url") { tracing::debug!("Would send Slack notification to: {}", webhook); } } ChannelType::PagerDuty => { // In a real implementation, this would trigger PagerDuty if let Some(key) = channel.config.get("routing_key") { tracing::debug!("Would trigger PagerDuty with key: {}", key); } } ChannelType::Email => { // Email would be sent via lettre if the feature is enabled if let Some(to) = channel.config.get("to") { tracing::debug!("Would send email to: {}", to); } } } } Ok(()) } /// Get all active alerts pub async fn active_alerts(&self) -> MonitoringResult> { let active = self.active_alerts.read(); let notifications: Vec = active .values() .filter(|a| a.state == AlertState::Firing) .map(|a| Notification::from_rule(&a.rule, a.state, a.value)) .collect(); Ok(notifications) } /// Get notification history pub fn notification_history(&self, limit: usize) -> Vec { let notifications = self.notifications.read(); notifications.iter().rev().take(limit).cloned().collect() } /// Get alert counts by severity pub fn alert_counts(&self) -> HashMap { let active = self.active_alerts.read(); let mut counts = HashMap::new(); for alert in active.values() { if alert.state == AlertState::Firing { *counts.entry(alert.rule.severity).or_insert(0) += 1; } } counts } /// Check if the alert manager is enabled pub fn is_enabled(&self) -> bool { self.enabled } /// Get registered rule count pub fn rule_count(&self) -> usize { self.rules.read().len() } } /// Pre-defined alert rules for common ML scenarios pub mod presets { use super::{AlertCondition, AlertRule, AlertSeverity}; /// High GPU memory usage alert pub fn high_gpu_memory(threshold_percent: f64) -> AlertRule { AlertRule::new( "high_gpu_memory", "rtx_gpu_memory_percent", AlertCondition::GreaterThan(threshold_percent), AlertSeverity::Warning, ) .with_description(&format!("GPU memory usage exceeds {threshold_percent}%")) .with_duration(std::time::Duration::from_secs(60)) } /// High inference latency alert pub fn high_inference_latency(threshold_secs: f64) -> AlertRule { AlertRule::new( "high_inference_latency", "rtx_inference_latency_p99", AlertCondition::GreaterThan(threshold_secs), AlertSeverity::Warning, ) .with_description(&format!("P99 inference latency exceeds {threshold_secs}s")) .with_duration(std::time::Duration::from_secs(300)) } /// High error rate alert pub fn high_error_rate(threshold_percent: f64) -> AlertRule { AlertRule::new( "high_error_rate", "rtx_inference_error_rate", AlertCondition::GreaterThan(threshold_percent), AlertSeverity::Critical, ) .with_description(&format!("Error rate exceeds {threshold_percent}%")) .with_duration(std::time::Duration::from_secs(60)) } /// Model not loaded alert pub fn model_not_loaded() -> AlertRule { AlertRule::new( "model_not_loaded", "rtx_model_loaded", AlertCondition::Equal(0.0), AlertSeverity::Critical, ) .with_description("No model is loaded") .with_duration(std::time::Duration::from_secs(0)) } /// Queue depth high alert pub fn queue_depth_high(threshold: f64) -> AlertRule { AlertRule::new( "queue_depth_high", "rtx_queue_depth", AlertCondition::GreaterThan(threshold), AlertSeverity::Warning, ) .with_description(&format!("Request queue depth exceeds {threshold}")) .with_duration(std::time::Duration::from_secs(120)) } } #[cfg(test)] mod tests { use super::*; #[test] fn test_alert_condition_evaluate() { assert!(AlertCondition::GreaterThan(50.0).evaluate(Some(60.0))); assert!(!AlertCondition::GreaterThan(50.0).evaluate(Some(40.0))); assert!(AlertCondition::LessThan(50.0).evaluate(Some(40.0))); assert!(AlertCondition::Absent.evaluate(None)); assert!(!AlertCondition::Absent.evaluate(Some(1.0))); } #[test] fn test_alert_rule_builder() { let rule = AlertRule::new( "test_rule", "test_metric", AlertCondition::GreaterThan(90.0), AlertSeverity::Critical, ) .with_description("Test description") .with_duration(std::time::Duration::from_secs(60)) .with_label("env", "prod") .with_annotation("runbook", "https://docs.example.com/runbook"); assert_eq!(rule.name, "test_rule"); assert_eq!(rule.severity, AlertSeverity::Critical); assert_eq!(rule.labels.get("env"), Some(&"prod".to_string())); } #[tokio::test] async fn test_alert_manager() { let manager = AlertManager::new().await.unwrap(); // Register a rule let rule = AlertRule::new( "test_alert", "test_metric", AlertCondition::GreaterThan(80.0), AlertSeverity::Warning, ); manager.register_rule(rule); assert_eq!(manager.rule_count(), 1); // Evaluate - should not fire manager.evaluate("test_metric", Some(50.0)).await.unwrap(); let active = manager.active_alerts().await.unwrap(); assert!(active.is_empty()); // Evaluate - should fire manager.evaluate("test_metric", Some(90.0)).await.unwrap(); let active = manager.active_alerts().await.unwrap(); assert_eq!(active.len(), 1); // Evaluate - should resolve manager.evaluate("test_metric", Some(50.0)).await.unwrap(); let active = manager.active_alerts().await.unwrap(); assert!(active.is_empty()); } #[test] fn test_preset_rules() { let rule = presets::high_gpu_memory(90.0); assert_eq!(rule.name, "high_gpu_memory"); assert_eq!(rule.severity, AlertSeverity::Warning); let rule = presets::high_error_rate(5.0); assert_eq!(rule.severity, AlertSeverity::Critical); } #[test] fn test_notification_from_rule() { let rule = AlertRule::new( "test", "metric", AlertCondition::GreaterThan(1.0), AlertSeverity::Warning, ) .with_description("Test alert"); let notification = Notification::from_rule(&rule, AlertState::Firing, Some(2.0)); assert_eq!(notification.rule_name, "test"); assert_eq!(notification.state, AlertState::Firing); assert_eq!(notification.value, Some(2.0)); } }