rustytorch/crates/production/rtx-monitoring/src/alerts.rs

//! Alert management and notification system.
//!
//! This module provides alerting capabilities for RustyTorch++ including
//! alert rule evaluation, notification dispatch, and alert history tracking.

use crate::MonitoringResult;
use chrono::{DateTime, Duration, Utc};
use parking_lot::RwLock;
use serde::{Deserialize, Serialize};
use std::collections::HashMap;
use std::sync::Arc;

/// Alert severity levels.
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)]
pub enum AlertSeverity {
    /// Critical alert - immediate action required
    Critical,
    /// Warning - attention needed soon
    Warning,
    /// Info - informational alert
    Info,
}

impl AlertSeverity {
    /// Get string representation
    pub fn as_str(&self) -> &'static str {
        match self {
            Self::Critical => "critical",
            Self::Warning => "warning",
            Self::Info => "info",
        }
    }
}

/// Alert state for tracking firing status
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
pub enum AlertState {
    /// Alert is currently firing
    Firing,
    /// Alert was firing but is now resolved
    Resolved,
    /// Alert is pending (condition met but waiting for duration)
    Pending,
}

/// Alert rule definition.
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct AlertRule {
    /// Unique rule name
    pub name: String,
    /// Alert severity
    pub severity: AlertSeverity,
    /// Human-readable description
    pub description: String,
    /// Metric name to evaluate
    pub metric: String,
    /// Threshold condition
    pub condition: AlertCondition,
    /// Duration the condition must hold before firing
    pub duration: std::time::Duration,
    /// Labels to add to the alert
    pub labels: HashMap<String, String>,
    /// Annotations (additional metadata)
    pub annotations: HashMap<String, String>,
}

impl AlertRule {
    /// Create a new alert rule
    pub fn new(
        name: &str,
        metric: &str,
        condition: AlertCondition,
        severity: AlertSeverity,
    ) -> Self {
        Self {
            name: name.to_string(),
            severity,
            description: String::new(),
            metric: metric.to_string(),
            condition,
            duration: std::time::Duration::from_secs(0),
            labels: HashMap::new(),
            annotations: HashMap::new(),
        }
    }

    /// Set description
    pub fn with_description(mut self, desc: &str) -> Self {
        self.description = desc.to_string();
        self
    }

    /// Set duration before firing
    pub fn with_duration(mut self, duration: std::time::Duration) -> Self {
        self.duration = duration;
        self
    }

    /// Add label
    pub fn with_label(mut self, key: &str, value: &str) -> Self {
        self.labels.insert(key.to_string(), value.to_string());
        self
    }

    /// Add annotation
    pub fn with_annotation(mut self, key: &str, value: &str) -> Self {
        self.annotations.insert(key.to_string(), value.to_string());
        self
    }
}

/// Alert condition types
#[derive(Debug, Clone, Serialize, Deserialize)]
pub enum AlertCondition {
    /// Value is greater than threshold
    GreaterThan(f64),
    /// Value is less than threshold
    LessThan(f64),
    /// Value is greater than or equal to threshold
    GreaterThanOrEqual(f64),
    /// Value is less than or equal to threshold
    LessThanOrEqual(f64),
    /// Value equals threshold
    Equal(f64),
    /// Value is not equal to threshold
    NotEqual(f64),
    /// Value is absent (no data)
    Absent,
}

impl AlertCondition {
    /// Evaluate the condition against a value
    pub fn evaluate(&self, value: Option<f64>) -> bool {
        match (self, value) {
            (Self::GreaterThan(threshold), Some(v)) => v > *threshold,
            (Self::LessThan(threshold), Some(v)) => v < *threshold,
            (Self::GreaterThanOrEqual(threshold), Some(v)) => v >= *threshold,
            (Self::LessThanOrEqual(threshold), Some(v)) => v <= *threshold,
            (Self::Equal(threshold), Some(v)) => (v - threshold).abs() < f64::EPSILON,
            (Self::NotEqual(threshold), Some(v)) => (v - threshold).abs() >= f64::EPSILON,
            (Self::Absent, None) => true,
            (Self::Absent, Some(_)) => false,
            (_, None) => false,
        }
    }
}

/// Notification configuration.
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct Notification {
    /// Unique notification ID
    pub id: String,
    /// Alert message
    pub message: String,
    /// Alert severity
    pub severity: AlertSeverity,
    /// When the notification was created
    pub timestamp: DateTime<Utc>,
    /// Alert state
    pub state: AlertState,
    /// Rule name that triggered this notification
    pub rule_name: String,
    /// Labels from the alert
    pub labels: HashMap<String, String>,
    /// Current metric value
    pub value: Option<f64>,
}

impl Notification {
    /// Create a new notification from a rule
    pub fn from_rule(rule: &AlertRule, state: AlertState, value: Option<f64>) -> Self {
        Self {
            id: uuid::Uuid::new_v4().to_string(),
            message: rule.description.clone(),
            severity: rule.severity,
            timestamp: Utc::now(),
            state,
            rule_name: rule.name.clone(),
            labels: rule.labels.clone(),
            value,
        }
    }
}

/// Notification channel for dispatching alerts
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct NotificationChannel {
    /// Channel name
    pub name: String,
    /// Channel type
    pub channel_type: ChannelType,
    /// Channel configuration
    pub config: HashMap<String, String>,
    /// Minimum severity to send
    pub min_severity: AlertSeverity,
}

/// Types of notification channels
#[derive(Debug, Clone, Serialize, Deserialize)]
pub enum ChannelType {
    /// Webhook (HTTP POST)
    Webhook,
    /// Email
    Email,
    /// Slack
    Slack,
    /// PagerDuty
    PagerDuty,
    /// Console/Log output
    Console,
}

/// Active alert tracking
#[derive(Debug, Clone)]
struct ActiveAlert {
    rule: AlertRule,
    first_firing: DateTime<Utc>,
    last_evaluation: DateTime<Utc>,
    state: AlertState,
    value: Option<f64>,
}

/// Alert manager for handling alerts and notifications.
#[derive(Debug)]
pub struct AlertManager {
    /// Registered alert rules
    rules: Arc<RwLock<HashMap<String, AlertRule>>>,
    /// Active alerts
    active_alerts: Arc<RwLock<HashMap<String, ActiveAlert>>>,
    /// Notification history
    notifications: Arc<RwLock<Vec<Notification>>>,
    /// Notification channels
    channels: Arc<RwLock<Vec<NotificationChannel>>>,
    /// Maximum notifications to keep in history
    max_history: usize,
    /// Whether the manager is enabled
    enabled: bool,
}

impl AlertManager {
    /// Create a new alert manager
    pub async fn new() -> MonitoringResult<Self> {
        Ok(Self {
            rules: Arc::new(RwLock::new(HashMap::new())),
            active_alerts: Arc::new(RwLock::new(HashMap::new())),
            notifications: Arc::new(RwLock::new(Vec::new())),
            channels: Arc::new(RwLock::new(Vec::new())),
            max_history: 10000,
            enabled: true,
        })
    }

    /// Register an alert rule
    pub fn register_rule(&self, rule: AlertRule) {
        self.rules.write().insert(rule.name.clone(), rule);
    }

    /// Remove an alert rule
    pub fn remove_rule(&self, name: &str) {
        self.rules.write().remove(name);
        self.active_alerts.write().remove(name);
    }

    /// Add a notification channel
    pub fn add_channel(&self, channel: NotificationChannel) {
        self.channels.write().push(channel);
    }

    /// Evaluate a metric value against all rules
    pub async fn evaluate(&self, metric: &str, value: Option<f64>) -> MonitoringResult<()> {
        let rules = self.rules.read();
        let now = Utc::now();

        for rule in rules.values() {
            if rule.metric != metric {
                continue;
            }

            let condition_met = rule.condition.evaluate(value);
            let mut active_alerts = self.active_alerts.write();

            if condition_met {
                if let Some(active) = active_alerts.get_mut(&rule.name) {
                    // Already tracking this alert
                    active.last_evaluation = now;
                    active.value = value;

                    // Check if duration has passed
                    if active.state == AlertState::Pending {
                        let elapsed = now - active.first_firing;
                        if elapsed >= Duration::from_std(rule.duration).unwrap_or(Duration::zero())
                        {
                            active.state = AlertState::Firing;
                            self.fire_alert(rule, value).await?;
                        }
                    }
                } else {
                    // New alert
                    let state = if rule.duration.is_zero() {
                        AlertState::Firing
                    } else {
                        AlertState::Pending
                    };

                    active_alerts.insert(
                        rule.name.clone(),
                        ActiveAlert {
                            rule: rule.clone(),
                            first_firing: now,
                            last_evaluation: now,
                            state,
                            value,
                        },
                    );

                    if state == AlertState::Firing {
                        drop(active_alerts);
                        self.fire_alert(rule, value).await?;
                    }
                }
            } else {
                // Condition no longer met
                if let Some(active) = active_alerts.remove(&rule.name)
                    && active.state == AlertState::Firing
                {
                    // Send resolved notification
                    drop(active_alerts);
                    self.resolve_alert(rule, value).await?;
                }
            }
        }

        Ok(())
    }

    /// Fire an alert
    async fn fire_alert(&self, rule: &AlertRule, value: Option<f64>) -> MonitoringResult<()> {
        let notification = Notification::from_rule(rule, AlertState::Firing, value);

        // Add to history
        {
            let mut notifications = self.notifications.write();
            if notifications.len() >= self.max_history {
                notifications.remove(0);
            }
            notifications.push(notification.clone());
        }

        // Dispatch to channels
        self.dispatch_notification(&notification).await?;

        tracing::warn!(
            rule = %rule.name,
            severity = %rule.severity.as_str(),
            value = ?value,
            "Alert fired: {}",
            rule.description
        );

        Ok(())
    }

    /// Resolve an alert
    async fn resolve_alert(&self, rule: &AlertRule, value: Option<f64>) -> MonitoringResult<()> {
        let notification = Notification::from_rule(rule, AlertState::Resolved, value);

        // Add to history
        {
            let mut notifications = self.notifications.write();
            if notifications.len() >= self.max_history {
                notifications.remove(0);
            }
            notifications.push(notification.clone());
        }

        // Dispatch to channels
        self.dispatch_notification(&notification).await?;

        tracing::info!(
            rule = %rule.name,
            "Alert resolved: {}",
            rule.description
        );

        Ok(())
    }

    /// Dispatch notification to all configured channels
    async fn dispatch_notification(&self, notification: &Notification) -> MonitoringResult<()> {
        let channels = self.channels.read();

        for channel in channels.iter() {
            // Check severity filter
            let should_send = match (channel.min_severity, notification.severity) {
                (AlertSeverity::Critical, AlertSeverity::Critical) => true,
                (AlertSeverity::Warning, AlertSeverity::Critical | AlertSeverity::Warning) => true,
                (AlertSeverity::Info, _) => true,
                _ => false,
            };

            if !should_send {
                continue;
            }

            match channel.channel_type {
                ChannelType::Console => {
                    tracing::info!(
                        channel = %channel.name,
                        severity = %notification.severity.as_str(),
                        state = ?notification.state,
                        "Alert: {} - {}",
                        notification.rule_name,
                        notification.message
                    );
                }
                ChannelType::Webhook => {
                    // In a real implementation, this would make an HTTP POST
                    if let Some(url) = channel.config.get("url") {
                        tracing::debug!("Would send webhook to: {}", url);
                    }
                }
                ChannelType::Slack => {
                    // In a real implementation, this would send to Slack
                    if let Some(webhook) = channel.config.get("webhook_url") {
                        tracing::debug!("Would send Slack notification to: {}", webhook);
                    }
                }
                ChannelType::PagerDuty => {
                    // In a real implementation, this would trigger PagerDuty
                    if let Some(key) = channel.config.get("routing_key") {
                        tracing::debug!("Would trigger PagerDuty with key: {}", key);
                    }
                }
                ChannelType::Email => {
                    // Email would be sent via lettre if the feature is enabled
                    if let Some(to) = channel.config.get("to") {
                        tracing::debug!("Would send email to: {}", to);
                    }
                }
            }
        }

        Ok(())
    }

    /// Get all active alerts
    pub async fn active_alerts(&self) -> MonitoringResult<Vec<Notification>> {
        let active = self.active_alerts.read();
        let notifications: Vec<Notification> = active
            .values()
            .filter(|a| a.state == AlertState::Firing)
            .map(|a| Notification::from_rule(&a.rule, a.state, a.value))
            .collect();
        Ok(notifications)
    }

    /// Get notification history
    pub fn notification_history(&self, limit: usize) -> Vec<Notification> {
        let notifications = self.notifications.read();
        notifications.iter().rev().take(limit).cloned().collect()
    }

    /// Get alert counts by severity
    pub fn alert_counts(&self) -> HashMap<AlertSeverity, usize> {
        let active = self.active_alerts.read();
        let mut counts = HashMap::new();

        for alert in active.values() {
            if alert.state == AlertState::Firing {
                *counts.entry(alert.rule.severity).or_insert(0) += 1;
            }
        }

        counts
    }

    /// Check if the alert manager is enabled
    pub fn is_enabled(&self) -> bool {
        self.enabled
    }

    /// Get registered rule count
    pub fn rule_count(&self) -> usize {
        self.rules.read().len()
    }
}

/// Pre-defined alert rules for common ML scenarios
pub mod presets {
    use super::{AlertCondition, AlertRule, AlertSeverity};

    /// High GPU memory usage alert
    pub fn high_gpu_memory(threshold_percent: f64) -> AlertRule {
        AlertRule::new(
            "high_gpu_memory",
            "rtx_gpu_memory_percent",
            AlertCondition::GreaterThan(threshold_percent),
            AlertSeverity::Warning,
        )
        .with_description(&format!("GPU memory usage exceeds {threshold_percent}%"))
        .with_duration(std::time::Duration::from_secs(60))
    }

    /// High inference latency alert
    pub fn high_inference_latency(threshold_secs: f64) -> AlertRule {
        AlertRule::new(
            "high_inference_latency",
            "rtx_inference_latency_p99",
            AlertCondition::GreaterThan(threshold_secs),
            AlertSeverity::Warning,
        )
        .with_description(&format!("P99 inference latency exceeds {threshold_secs}s"))
        .with_duration(std::time::Duration::from_secs(300))
    }

    /// High error rate alert
    pub fn high_error_rate(threshold_percent: f64) -> AlertRule {
        AlertRule::new(
            "high_error_rate",
            "rtx_inference_error_rate",
            AlertCondition::GreaterThan(threshold_percent),
            AlertSeverity::Critical,
        )
        .with_description(&format!("Error rate exceeds {threshold_percent}%"))
        .with_duration(std::time::Duration::from_secs(60))
    }

    /// Model not loaded alert
    pub fn model_not_loaded() -> AlertRule {
        AlertRule::new(
            "model_not_loaded",
            "rtx_model_loaded",
            AlertCondition::Equal(0.0),
            AlertSeverity::Critical,
        )
        .with_description("No model is loaded")
        .with_duration(std::time::Duration::from_secs(0))
    }

    /// Queue depth high alert
    pub fn queue_depth_high(threshold: f64) -> AlertRule {
        AlertRule::new(
            "queue_depth_high",
            "rtx_queue_depth",
            AlertCondition::GreaterThan(threshold),
            AlertSeverity::Warning,
        )
        .with_description(&format!("Request queue depth exceeds {threshold}"))
        .with_duration(std::time::Duration::from_secs(120))
    }
}

#[cfg(test)]
mod tests {
    use super::*;

    #[test]
    fn test_alert_condition_evaluate() {
        assert!(AlertCondition::GreaterThan(50.0).evaluate(Some(60.0)));
        assert!(!AlertCondition::GreaterThan(50.0).evaluate(Some(40.0)));
        assert!(AlertCondition::LessThan(50.0).evaluate(Some(40.0)));
        assert!(AlertCondition::Absent.evaluate(None));
        assert!(!AlertCondition::Absent.evaluate(Some(1.0)));
    }

    #[test]
    fn test_alert_rule_builder() {
        let rule = AlertRule::new(
            "test_rule",
            "test_metric",
            AlertCondition::GreaterThan(90.0),
            AlertSeverity::Critical,
        )
        .with_description("Test description")
        .with_duration(std::time::Duration::from_secs(60))
        .with_label("env", "prod")
        .with_annotation("runbook", "https://docs.example.com/runbook");

        assert_eq!(rule.name, "test_rule");
        assert_eq!(rule.severity, AlertSeverity::Critical);
        assert_eq!(rule.labels.get("env"), Some(&"prod".to_string()));
    }

    #[tokio::test]
    async fn test_alert_manager() {
        let manager = AlertManager::new().await.unwrap();

        // Register a rule
        let rule = AlertRule::new(
            "test_alert",
            "test_metric",
            AlertCondition::GreaterThan(80.0),
            AlertSeverity::Warning,
        );
        manager.register_rule(rule);

        assert_eq!(manager.rule_count(), 1);

        // Evaluate - should not fire
        manager.evaluate("test_metric", Some(50.0)).await.unwrap();
        let active = manager.active_alerts().await.unwrap();
        assert!(active.is_empty());

        // Evaluate - should fire
        manager.evaluate("test_metric", Some(90.0)).await.unwrap();
        let active = manager.active_alerts().await.unwrap();
        assert_eq!(active.len(), 1);

        // Evaluate - should resolve
        manager.evaluate("test_metric", Some(50.0)).await.unwrap();
        let active = manager.active_alerts().await.unwrap();
        assert!(active.is_empty());
    }

    #[test]
    fn test_preset_rules() {
        let rule = presets::high_gpu_memory(90.0);
        assert_eq!(rule.name, "high_gpu_memory");
        assert_eq!(rule.severity, AlertSeverity::Warning);

        let rule = presets::high_error_rate(5.0);
        assert_eq!(rule.severity, AlertSeverity::Critical);
    }

    #[test]
    fn test_notification_from_rule() {
        let rule = AlertRule::new(
            "test",
            "metric",
            AlertCondition::GreaterThan(1.0),
            AlertSeverity::Warning,
        )
        .with_description("Test alert");

        let notification = Notification::from_rule(&rule, AlertState::Firing, Some(2.0));
        assert_eq!(notification.rule_name, "test");
        assert_eq!(notification.state, AlertState::Firing);
        assert_eq!(notification.value, Some(2.0));
    }
}