Initial commit
This commit is contained in:
redclawsystems
@@ -0,0 +1,640 @@
|
||||
//! Alert management and notification system.
|
||||
//!
|
||||
//! This module provides alerting capabilities for RustyTorch++ including
|
||||
//! alert rule evaluation, notification dispatch, and alert history tracking.
|
||||
|
||||
use crate::MonitoringResult;
|
||||
use chrono::{DateTime, Duration, Utc};
|
||||
use parking_lot::RwLock;
|
||||
use serde::{Deserialize, Serialize};
|
||||
use std::collections::HashMap;
|
||||
use std::sync::Arc;
|
||||
|
||||
/// Alert severity levels.
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)]
|
||||
pub enum AlertSeverity {
|
||||
/// Critical alert - immediate action required
|
||||
Critical,
|
||||
/// Warning - attention needed soon
|
||||
Warning,
|
||||
/// Info - informational alert
|
||||
Info,
|
||||
}
|
||||
|
||||
impl AlertSeverity {
|
||||
/// Get string representation
|
||||
pub fn as_str(&self) -> &'static str {
|
||||
match self {
|
||||
Self::Critical => "critical",
|
||||
Self::Warning => "warning",
|
||||
Self::Info => "info",
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Alert state for tracking firing status
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
|
||||
pub enum AlertState {
|
||||
/// Alert is currently firing
|
||||
Firing,
|
||||
/// Alert was firing but is now resolved
|
||||
Resolved,
|
||||
/// Alert is pending (condition met but waiting for duration)
|
||||
Pending,
|
||||
}
|
||||
|
||||
/// Alert rule definition.
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct AlertRule {
|
||||
/// Unique rule name
|
||||
pub name: String,
|
||||
/// Alert severity
|
||||
pub severity: AlertSeverity,
|
||||
/// Human-readable description
|
||||
pub description: String,
|
||||
/// Metric name to evaluate
|
||||
pub metric: String,
|
||||
/// Threshold condition
|
||||
pub condition: AlertCondition,
|
||||
/// Duration the condition must hold before firing
|
||||
pub duration: std::time::Duration,
|
||||
/// Labels to add to the alert
|
||||
pub labels: HashMap<String, String>,
|
||||
/// Annotations (additional metadata)
|
||||
pub annotations: HashMap<String, String>,
|
||||
}
|
||||
|
||||
impl AlertRule {
|
||||
/// Create a new alert rule
|
||||
pub fn new(
|
||||
name: &str,
|
||||
metric: &str,
|
||||
condition: AlertCondition,
|
||||
severity: AlertSeverity,
|
||||
) -> Self {
|
||||
Self {
|
||||
name: name.to_string(),
|
||||
severity,
|
||||
description: String::new(),
|
||||
metric: metric.to_string(),
|
||||
condition,
|
||||
duration: std::time::Duration::from_secs(0),
|
||||
labels: HashMap::new(),
|
||||
annotations: HashMap::new(),
|
||||
}
|
||||
}
|
||||
|
||||
/// Set description
|
||||
pub fn with_description(mut self, desc: &str) -> Self {
|
||||
self.description = desc.to_string();
|
||||
self
|
||||
}
|
||||
|
||||
/// Set duration before firing
|
||||
pub fn with_duration(mut self, duration: std::time::Duration) -> Self {
|
||||
self.duration = duration;
|
||||
self
|
||||
}
|
||||
|
||||
/// Add label
|
||||
pub fn with_label(mut self, key: &str, value: &str) -> Self {
|
||||
self.labels.insert(key.to_string(), value.to_string());
|
||||
self
|
||||
}
|
||||
|
||||
/// Add annotation
|
||||
pub fn with_annotation(mut self, key: &str, value: &str) -> Self {
|
||||
self.annotations.insert(key.to_string(), value.to_string());
|
||||
self
|
||||
}
|
||||
}
|
||||
|
||||
/// Alert condition types
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub enum AlertCondition {
|
||||
/// Value is greater than threshold
|
||||
GreaterThan(f64),
|
||||
/// Value is less than threshold
|
||||
LessThan(f64),
|
||||
/// Value is greater than or equal to threshold
|
||||
GreaterThanOrEqual(f64),
|
||||
/// Value is less than or equal to threshold
|
||||
LessThanOrEqual(f64),
|
||||
/// Value equals threshold
|
||||
Equal(f64),
|
||||
/// Value is not equal to threshold
|
||||
NotEqual(f64),
|
||||
/// Value is absent (no data)
|
||||
Absent,
|
||||
}
|
||||
|
||||
impl AlertCondition {
|
||||
/// Evaluate the condition against a value
|
||||
pub fn evaluate(&self, value: Option<f64>) -> bool {
|
||||
match (self, value) {
|
||||
(Self::GreaterThan(threshold), Some(v)) => v > *threshold,
|
||||
(Self::LessThan(threshold), Some(v)) => v < *threshold,
|
||||
(Self::GreaterThanOrEqual(threshold), Some(v)) => v >= *threshold,
|
||||
(Self::LessThanOrEqual(threshold), Some(v)) => v <= *threshold,
|
||||
(Self::Equal(threshold), Some(v)) => (v - threshold).abs() < f64::EPSILON,
|
||||
(Self::NotEqual(threshold), Some(v)) => (v - threshold).abs() >= f64::EPSILON,
|
||||
(Self::Absent, None) => true,
|
||||
(Self::Absent, Some(_)) => false,
|
||||
(_, None) => false,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Notification configuration.
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct Notification {
|
||||
/// Unique notification ID
|
||||
pub id: String,
|
||||
/// Alert message
|
||||
pub message: String,
|
||||
/// Alert severity
|
||||
pub severity: AlertSeverity,
|
||||
/// When the notification was created
|
||||
pub timestamp: DateTime<Utc>,
|
||||
/// Alert state
|
||||
pub state: AlertState,
|
||||
/// Rule name that triggered this notification
|
||||
pub rule_name: String,
|
||||
/// Labels from the alert
|
||||
pub labels: HashMap<String, String>,
|
||||
/// Current metric value
|
||||
pub value: Option<f64>,
|
||||
}
|
||||
|
||||
impl Notification {
|
||||
/// Create a new notification from a rule
|
||||
pub fn from_rule(rule: &AlertRule, state: AlertState, value: Option<f64>) -> Self {
|
||||
Self {
|
||||
id: uuid::Uuid::new_v4().to_string(),
|
||||
message: rule.description.clone(),
|
||||
severity: rule.severity,
|
||||
timestamp: Utc::now(),
|
||||
state,
|
||||
rule_name: rule.name.clone(),
|
||||
labels: rule.labels.clone(),
|
||||
value,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Notification channel for dispatching alerts
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct NotificationChannel {
|
||||
/// Channel name
|
||||
pub name: String,
|
||||
/// Channel type
|
||||
pub channel_type: ChannelType,
|
||||
/// Channel configuration
|
||||
pub config: HashMap<String, String>,
|
||||
/// Minimum severity to send
|
||||
pub min_severity: AlertSeverity,
|
||||
}
|
||||
|
||||
/// Types of notification channels
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub enum ChannelType {
|
||||
/// Webhook (HTTP POST)
|
||||
Webhook,
|
||||
/// Email
|
||||
Email,
|
||||
/// Slack
|
||||
Slack,
|
||||
/// PagerDuty
|
||||
PagerDuty,
|
||||
/// Console/Log output
|
||||
Console,
|
||||
}
|
||||
|
||||
/// Active alert tracking
|
||||
#[derive(Debug, Clone)]
|
||||
struct ActiveAlert {
|
||||
rule: AlertRule,
|
||||
first_firing: DateTime<Utc>,
|
||||
last_evaluation: DateTime<Utc>,
|
||||
state: AlertState,
|
||||
value: Option<f64>,
|
||||
}
|
||||
|
||||
/// Alert manager for handling alerts and notifications.
|
||||
#[derive(Debug)]
|
||||
pub struct AlertManager {
|
||||
/// Registered alert rules
|
||||
rules: Arc<RwLock<HashMap<String, AlertRule>>>,
|
||||
/// Active alerts
|
||||
active_alerts: Arc<RwLock<HashMap<String, ActiveAlert>>>,
|
||||
/// Notification history
|
||||
notifications: Arc<RwLock<Vec<Notification>>>,
|
||||
/// Notification channels
|
||||
channels: Arc<RwLock<Vec<NotificationChannel>>>,
|
||||
/// Maximum notifications to keep in history
|
||||
max_history: usize,
|
||||
/// Whether the manager is enabled
|
||||
enabled: bool,
|
||||
}
|
||||
|
||||
impl AlertManager {
|
||||
/// Create a new alert manager
|
||||
pub async fn new() -> MonitoringResult<Self> {
|
||||
Ok(Self {
|
||||
rules: Arc::new(RwLock::new(HashMap::new())),
|
||||
active_alerts: Arc::new(RwLock::new(HashMap::new())),
|
||||
notifications: Arc::new(RwLock::new(Vec::new())),
|
||||
channels: Arc::new(RwLock::new(Vec::new())),
|
||||
max_history: 10000,
|
||||
enabled: true,
|
||||
})
|
||||
}
|
||||
|
||||
/// Register an alert rule
|
||||
pub fn register_rule(&self, rule: AlertRule) {
|
||||
self.rules.write().insert(rule.name.clone(), rule);
|
||||
}
|
||||
|
||||
/// Remove an alert rule
|
||||
pub fn remove_rule(&self, name: &str) {
|
||||
self.rules.write().remove(name);
|
||||
self.active_alerts.write().remove(name);
|
||||
}
|
||||
|
||||
/// Add a notification channel
|
||||
pub fn add_channel(&self, channel: NotificationChannel) {
|
||||
self.channels.write().push(channel);
|
||||
}
|
||||
|
||||
/// Evaluate a metric value against all rules
|
||||
pub async fn evaluate(&self, metric: &str, value: Option<f64>) -> MonitoringResult<()> {
|
||||
let rules = self.rules.read();
|
||||
let now = Utc::now();
|
||||
|
||||
for rule in rules.values() {
|
||||
if rule.metric != metric {
|
||||
continue;
|
||||
}
|
||||
|
||||
let condition_met = rule.condition.evaluate(value);
|
||||
let mut active_alerts = self.active_alerts.write();
|
||||
|
||||
if condition_met {
|
||||
if let Some(active) = active_alerts.get_mut(&rule.name) {
|
||||
// Already tracking this alert
|
||||
active.last_evaluation = now;
|
||||
active.value = value;
|
||||
|
||||
// Check if duration has passed
|
||||
if active.state == AlertState::Pending {
|
||||
let elapsed = now - active.first_firing;
|
||||
if elapsed >= Duration::from_std(rule.duration).unwrap_or(Duration::zero())
|
||||
{
|
||||
active.state = AlertState::Firing;
|
||||
self.fire_alert(rule, value).await?;
|
||||
}
|
||||
}
|
||||
} else {
|
||||
// New alert
|
||||
let state = if rule.duration.is_zero() {
|
||||
AlertState::Firing
|
||||
} else {
|
||||
AlertState::Pending
|
||||
};
|
||||
|
||||
active_alerts.insert(
|
||||
rule.name.clone(),
|
||||
ActiveAlert {
|
||||
rule: rule.clone(),
|
||||
first_firing: now,
|
||||
last_evaluation: now,
|
||||
state,
|
||||
value,
|
||||
},
|
||||
);
|
||||
|
||||
if state == AlertState::Firing {
|
||||
drop(active_alerts);
|
||||
self.fire_alert(rule, value).await?;
|
||||
}
|
||||
}
|
||||
} else {
|
||||
// Condition no longer met
|
||||
if let Some(active) = active_alerts.remove(&rule.name)
|
||||
&& active.state == AlertState::Firing
|
||||
{
|
||||
// Send resolved notification
|
||||
drop(active_alerts);
|
||||
self.resolve_alert(rule, value).await?;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Fire an alert
|
||||
async fn fire_alert(&self, rule: &AlertRule, value: Option<f64>) -> MonitoringResult<()> {
|
||||
let notification = Notification::from_rule(rule, AlertState::Firing, value);
|
||||
|
||||
// Add to history
|
||||
{
|
||||
let mut notifications = self.notifications.write();
|
||||
if notifications.len() >= self.max_history {
|
||||
notifications.remove(0);
|
||||
}
|
||||
notifications.push(notification.clone());
|
||||
}
|
||||
|
||||
// Dispatch to channels
|
||||
self.dispatch_notification(¬ification).await?;
|
||||
|
||||
tracing::warn!(
|
||||
rule = %rule.name,
|
||||
severity = %rule.severity.as_str(),
|
||||
value = ?value,
|
||||
"Alert fired: {}",
|
||||
rule.description
|
||||
);
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Resolve an alert
|
||||
async fn resolve_alert(&self, rule: &AlertRule, value: Option<f64>) -> MonitoringResult<()> {
|
||||
let notification = Notification::from_rule(rule, AlertState::Resolved, value);
|
||||
|
||||
// Add to history
|
||||
{
|
||||
let mut notifications = self.notifications.write();
|
||||
if notifications.len() >= self.max_history {
|
||||
notifications.remove(0);
|
||||
}
|
||||
notifications.push(notification.clone());
|
||||
}
|
||||
|
||||
// Dispatch to channels
|
||||
self.dispatch_notification(¬ification).await?;
|
||||
|
||||
tracing::info!(
|
||||
rule = %rule.name,
|
||||
"Alert resolved: {}",
|
||||
rule.description
|
||||
);
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Dispatch notification to all configured channels
|
||||
async fn dispatch_notification(&self, notification: &Notification) -> MonitoringResult<()> {
|
||||
let channels = self.channels.read();
|
||||
|
||||
for channel in channels.iter() {
|
||||
// Check severity filter
|
||||
let should_send = match (channel.min_severity, notification.severity) {
|
||||
(AlertSeverity::Critical, AlertSeverity::Critical) => true,
|
||||
(AlertSeverity::Warning, AlertSeverity::Critical | AlertSeverity::Warning) => true,
|
||||
(AlertSeverity::Info, _) => true,
|
||||
_ => false,
|
||||
};
|
||||
|
||||
if !should_send {
|
||||
continue;
|
||||
}
|
||||
|
||||
match channel.channel_type {
|
||||
ChannelType::Console => {
|
||||
tracing::info!(
|
||||
channel = %channel.name,
|
||||
severity = %notification.severity.as_str(),
|
||||
state = ?notification.state,
|
||||
"Alert: {} - {}",
|
||||
notification.rule_name,
|
||||
notification.message
|
||||
);
|
||||
}
|
||||
ChannelType::Webhook => {
|
||||
// In a real implementation, this would make an HTTP POST
|
||||
if let Some(url) = channel.config.get("url") {
|
||||
tracing::debug!("Would send webhook to: {}", url);
|
||||
}
|
||||
}
|
||||
ChannelType::Slack => {
|
||||
// In a real implementation, this would send to Slack
|
||||
if let Some(webhook) = channel.config.get("webhook_url") {
|
||||
tracing::debug!("Would send Slack notification to: {}", webhook);
|
||||
}
|
||||
}
|
||||
ChannelType::PagerDuty => {
|
||||
// In a real implementation, this would trigger PagerDuty
|
||||
if let Some(key) = channel.config.get("routing_key") {
|
||||
tracing::debug!("Would trigger PagerDuty with key: {}", key);
|
||||
}
|
||||
}
|
||||
ChannelType::Email => {
|
||||
// Email would be sent via lettre if the feature is enabled
|
||||
if let Some(to) = channel.config.get("to") {
|
||||
tracing::debug!("Would send email to: {}", to);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Get all active alerts
|
||||
pub async fn active_alerts(&self) -> MonitoringResult<Vec<Notification>> {
|
||||
let active = self.active_alerts.read();
|
||||
let notifications: Vec<Notification> = active
|
||||
.values()
|
||||
.filter(|a| a.state == AlertState::Firing)
|
||||
.map(|a| Notification::from_rule(&a.rule, a.state, a.value))
|
||||
.collect();
|
||||
Ok(notifications)
|
||||
}
|
||||
|
||||
/// Get notification history
|
||||
pub fn notification_history(&self, limit: usize) -> Vec<Notification> {
|
||||
let notifications = self.notifications.read();
|
||||
notifications.iter().rev().take(limit).cloned().collect()
|
||||
}
|
||||
|
||||
/// Get alert counts by severity
|
||||
pub fn alert_counts(&self) -> HashMap<AlertSeverity, usize> {
|
||||
let active = self.active_alerts.read();
|
||||
let mut counts = HashMap::new();
|
||||
|
||||
for alert in active.values() {
|
||||
if alert.state == AlertState::Firing {
|
||||
*counts.entry(alert.rule.severity).or_insert(0) += 1;
|
||||
}
|
||||
}
|
||||
|
||||
counts
|
||||
}
|
||||
|
||||
/// Check if the alert manager is enabled
|
||||
pub fn is_enabled(&self) -> bool {
|
||||
self.enabled
|
||||
}
|
||||
|
||||
/// Get registered rule count
|
||||
pub fn rule_count(&self) -> usize {
|
||||
self.rules.read().len()
|
||||
}
|
||||
}
|
||||
|
||||
/// Pre-defined alert rules for common ML scenarios
|
||||
pub mod presets {
|
||||
use super::{AlertRule, AlertCondition, AlertSeverity};
|
||||
|
||||
/// High GPU memory usage alert
|
||||
pub fn high_gpu_memory(threshold_percent: f64) -> AlertRule {
|
||||
AlertRule::new(
|
||||
"high_gpu_memory",
|
||||
"rtx_gpu_memory_percent",
|
||||
AlertCondition::GreaterThan(threshold_percent),
|
||||
AlertSeverity::Warning,
|
||||
)
|
||||
.with_description(&format!("GPU memory usage exceeds {threshold_percent}%"))
|
||||
.with_duration(std::time::Duration::from_secs(60))
|
||||
}
|
||||
|
||||
/// High inference latency alert
|
||||
pub fn high_inference_latency(threshold_secs: f64) -> AlertRule {
|
||||
AlertRule::new(
|
||||
"high_inference_latency",
|
||||
"rtx_inference_latency_p99",
|
||||
AlertCondition::GreaterThan(threshold_secs),
|
||||
AlertSeverity::Warning,
|
||||
)
|
||||
.with_description(&format!("P99 inference latency exceeds {threshold_secs}s"))
|
||||
.with_duration(std::time::Duration::from_secs(300))
|
||||
}
|
||||
|
||||
/// High error rate alert
|
||||
pub fn high_error_rate(threshold_percent: f64) -> AlertRule {
|
||||
AlertRule::new(
|
||||
"high_error_rate",
|
||||
"rtx_inference_error_rate",
|
||||
AlertCondition::GreaterThan(threshold_percent),
|
||||
AlertSeverity::Critical,
|
||||
)
|
||||
.with_description(&format!("Error rate exceeds {threshold_percent}%"))
|
||||
.with_duration(std::time::Duration::from_secs(60))
|
||||
}
|
||||
|
||||
/// Model not loaded alert
|
||||
pub fn model_not_loaded() -> AlertRule {
|
||||
AlertRule::new(
|
||||
"model_not_loaded",
|
||||
"rtx_model_loaded",
|
||||
AlertCondition::Equal(0.0),
|
||||
AlertSeverity::Critical,
|
||||
)
|
||||
.with_description("No model is loaded")
|
||||
.with_duration(std::time::Duration::from_secs(0))
|
||||
}
|
||||
|
||||
/// Queue depth high alert
|
||||
pub fn queue_depth_high(threshold: f64) -> AlertRule {
|
||||
AlertRule::new(
|
||||
"queue_depth_high",
|
||||
"rtx_queue_depth",
|
||||
AlertCondition::GreaterThan(threshold),
|
||||
AlertSeverity::Warning,
|
||||
)
|
||||
.with_description(&format!("Request queue depth exceeds {threshold}"))
|
||||
.with_duration(std::time::Duration::from_secs(120))
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn test_alert_condition_evaluate() {
|
||||
assert!(AlertCondition::GreaterThan(50.0).evaluate(Some(60.0)));
|
||||
assert!(!AlertCondition::GreaterThan(50.0).evaluate(Some(40.0)));
|
||||
assert!(AlertCondition::LessThan(50.0).evaluate(Some(40.0)));
|
||||
assert!(AlertCondition::Absent.evaluate(None));
|
||||
assert!(!AlertCondition::Absent.evaluate(Some(1.0)));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_alert_rule_builder() {
|
||||
let rule = AlertRule::new(
|
||||
"test_rule",
|
||||
"test_metric",
|
||||
AlertCondition::GreaterThan(90.0),
|
||||
AlertSeverity::Critical,
|
||||
)
|
||||
.with_description("Test description")
|
||||
.with_duration(std::time::Duration::from_secs(60))
|
||||
.with_label("env", "prod")
|
||||
.with_annotation("runbook", "https://docs.example.com/runbook");
|
||||
|
||||
assert_eq!(rule.name, "test_rule");
|
||||
assert_eq!(rule.severity, AlertSeverity::Critical);
|
||||
assert_eq!(rule.labels.get("env"), Some(&"prod".to_string()));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_alert_manager() {
|
||||
let manager = AlertManager::new().await.unwrap();
|
||||
|
||||
// Register a rule
|
||||
let rule = AlertRule::new(
|
||||
"test_alert",
|
||||
"test_metric",
|
||||
AlertCondition::GreaterThan(80.0),
|
||||
AlertSeverity::Warning,
|
||||
);
|
||||
manager.register_rule(rule);
|
||||
|
||||
assert_eq!(manager.rule_count(), 1);
|
||||
|
||||
// Evaluate - should not fire
|
||||
manager.evaluate("test_metric", Some(50.0)).await.unwrap();
|
||||
let active = manager.active_alerts().await.unwrap();
|
||||
assert!(active.is_empty());
|
||||
|
||||
// Evaluate - should fire
|
||||
manager.evaluate("test_metric", Some(90.0)).await.unwrap();
|
||||
let active = manager.active_alerts().await.unwrap();
|
||||
assert_eq!(active.len(), 1);
|
||||
|
||||
// Evaluate - should resolve
|
||||
manager.evaluate("test_metric", Some(50.0)).await.unwrap();
|
||||
let active = manager.active_alerts().await.unwrap();
|
||||
assert!(active.is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_preset_rules() {
|
||||
let rule = presets::high_gpu_memory(90.0);
|
||||
assert_eq!(rule.name, "high_gpu_memory");
|
||||
assert_eq!(rule.severity, AlertSeverity::Warning);
|
||||
|
||||
let rule = presets::high_error_rate(5.0);
|
||||
assert_eq!(rule.severity, AlertSeverity::Critical);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_notification_from_rule() {
|
||||
let rule = AlertRule::new(
|
||||
"test",
|
||||
"metric",
|
||||
AlertCondition::GreaterThan(1.0),
|
||||
AlertSeverity::Warning,
|
||||
)
|
||||
.with_description("Test alert");
|
||||
|
||||
let notification = Notification::from_rule(&rule, AlertState::Firing, Some(2.0));
|
||||
assert_eq!(notification.rule_name, "test");
|
||||
assert_eq!(notification.state, AlertState::Firing);
|
||||
assert_eq!(notification.value, Some(2.0));
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user