1048 lines
35 KiB
Rust
1048 lines
35 KiB
Rust
//! SLO monitoring and 99.95% availability module
|
|
//! Implements comprehensive Service Level Objective tracking with real-time monitoring,
|
|
//! error budget management, burn rate calculation, and automated alerting
|
|
|
|
use crate::{PlatformConfig, PlatformResult, error::SloError};
|
|
use chrono::{DateTime, Utc};
|
|
use serde::{Deserialize, Serialize};
|
|
use std::collections::{HashMap, VecDeque};
|
|
use std::sync::{Arc, RwLock};
|
|
use std::time::{Duration, SystemTime};
|
|
use tokio::sync::{Mutex, RwLock as AsyncRwLock};
|
|
use tokio::time::Interval;
|
|
use tracing::info;
|
|
use uuid::Uuid;
|
|
|
|
const MAX_DATAPOINTS_PER_WINDOW: usize = 10_000;
|
|
const HISTORY_RETENTION_HOURS: u64 = 7 * 24;
|
|
|
|
/// SLA targets configuration
|
|
#[derive(Debug, Clone, Serialize, Deserialize)]
|
|
pub struct SlaTargets {
|
|
pub availability_percentage: f64,
|
|
pub max_latency_p50_ms: u64,
|
|
pub max_latency_p99_ms: u64,
|
|
pub max_latency_p999_ms: u64,
|
|
pub max_error_rate_percentage: f64,
|
|
pub error_budget_burn_rate_threshold: f64,
|
|
pub downtime_budget_minutes_per_month: f64,
|
|
}
|
|
|
|
impl Default for SlaTargets {
|
|
fn default() -> Self {
|
|
Self {
|
|
availability_percentage: 99.95,
|
|
max_latency_p50_ms: 100,
|
|
max_latency_p99_ms: 500,
|
|
max_latency_p999_ms: 2000,
|
|
max_error_rate_percentage: 0.1,
|
|
error_budget_burn_rate_threshold: 2.0,
|
|
downtime_budget_minutes_per_month: 43.8, // (100 - 99.95) / 100 * SECONDS_PER_MONTH / 60
|
|
}
|
|
}
|
|
}
|
|
|
|
/// Availability metrics for a region or global
|
|
#[derive(Debug, Clone, Serialize, Deserialize)]
|
|
pub struct AvailabilityMetrics {
|
|
pub current_percentage: f64,
|
|
pub total_requests: u64,
|
|
pub successful_requests: u64,
|
|
pub failed_requests: u64,
|
|
pub sla_violated: bool,
|
|
pub last_updated: DateTime<Utc>,
|
|
}
|
|
|
|
/// Latency percentile metrics
|
|
#[derive(Debug, Clone, Serialize, Deserialize)]
|
|
pub struct LatencyMetrics {
|
|
pub p50_ms: u64,
|
|
pub p99_ms: u64,
|
|
pub p999_ms: u64,
|
|
pub sla_violated_p50: bool,
|
|
pub sla_violated_p99: bool,
|
|
pub sla_violated_p999: bool,
|
|
pub sample_count: u64,
|
|
pub last_updated: DateTime<Utc>,
|
|
}
|
|
|
|
/// Error budget tracking
|
|
#[derive(Debug, Clone, Serialize, Deserialize)]
|
|
pub struct ErrorBudget {
|
|
pub total_budget_minutes: f64,
|
|
pub consumed_minutes: f64,
|
|
pub remaining_percentage: f64,
|
|
pub burn_rate_1h: f64,
|
|
pub burn_rate_24h: f64,
|
|
pub last_updated: DateTime<Utc>,
|
|
}
|
|
|
|
/// Multi-window error rate tracking
|
|
#[derive(Debug, Clone, Serialize, Deserialize)]
|
|
pub struct ErrorRateMetrics {
|
|
pub window_1m: f64,
|
|
pub window_5m: f64,
|
|
pub window_1h: f64,
|
|
pub sla_violated_1m: bool,
|
|
pub sla_violated_5m: bool,
|
|
pub sla_violated_1h: bool,
|
|
pub last_updated: DateTime<Utc>,
|
|
}
|
|
|
|
/// Alert severity levels
|
|
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
|
|
pub enum AlertSeverity {
|
|
Info,
|
|
Warning,
|
|
Critical,
|
|
}
|
|
|
|
/// Alert action types
|
|
#[derive(Debug, Clone, Serialize, Deserialize)]
|
|
pub struct AlertAction {
|
|
pub action_type: String,
|
|
pub target: String,
|
|
pub executed_at: DateTime<Utc>,
|
|
}
|
|
|
|
/// SLO violation alert
|
|
#[derive(Debug, Clone, Serialize, Deserialize)]
|
|
pub struct SloViolation {
|
|
pub id: Uuid,
|
|
pub alert_type: String,
|
|
pub severity: AlertSeverity,
|
|
pub region: Option<String>,
|
|
pub metric: String,
|
|
pub current_value: f64,
|
|
pub threshold: f64,
|
|
pub message: String,
|
|
pub escalated: bool,
|
|
pub notification_attempts: u32,
|
|
pub actions_taken: Vec<AlertAction>,
|
|
pub created_at: DateTime<Utc>,
|
|
pub resolved_at: Option<DateTime<Utc>>,
|
|
}
|
|
|
|
/// Alert configuration
|
|
#[derive(Debug, Clone, Serialize, Deserialize)]
|
|
pub struct AlertConfiguration {
|
|
pub warning_threshold_percentage: f64,
|
|
pub critical_threshold_percentage: f64,
|
|
pub escalation_delay_minutes: u64,
|
|
pub notification_channels: Vec<String>,
|
|
}
|
|
|
|
/// Burn rate configuration
|
|
#[derive(Debug, Clone, Serialize, Deserialize)]
|
|
pub struct BurnRateConfig {
|
|
pub short_window_minutes: u64,
|
|
pub long_window_minutes: u64,
|
|
pub alert_threshold: f64,
|
|
}
|
|
|
|
/// Service level objective configuration
|
|
#[derive(Debug, Clone, Serialize, Deserialize)]
|
|
pub struct ServiceLevelObjective {
|
|
pub metric_name: String,
|
|
pub target_value: f64,
|
|
pub comparison: String,
|
|
pub window_duration: Duration,
|
|
}
|
|
|
|
/// SLO configuration for the monitor
|
|
#[derive(Debug, Clone, Serialize, Deserialize)]
|
|
pub struct SloConfiguration {
|
|
pub availability_slo: ServiceLevelObjective,
|
|
pub latency_slos: Vec<ServiceLevelObjective>,
|
|
pub error_rate_slo: ServiceLevelObjective,
|
|
pub burn_rate_config: BurnRateConfig,
|
|
pub alert_config: AlertConfiguration,
|
|
}
|
|
|
|
/// Monitoring window for time-series data
|
|
#[derive(Debug, Clone, Serialize, Deserialize)]
|
|
pub enum MonitoringWindow {
|
|
OneMinute,
|
|
FiveMinutes,
|
|
OneHour,
|
|
TwentyFourHours,
|
|
}
|
|
|
|
/// Percentile metrics calculation
|
|
#[derive(Debug, Clone, Serialize, Deserialize)]
|
|
pub struct PercentileMetrics {
|
|
pub p50: f64,
|
|
pub p99: f64,
|
|
pub p999: f64,
|
|
}
|
|
|
|
/// Regional SLO metrics
|
|
#[derive(Debug, Clone, Serialize, Deserialize)]
|
|
pub struct RegionalSlo {
|
|
pub region_id: String,
|
|
pub availability_percentage: f64,
|
|
pub latency_p99_ms: u64,
|
|
pub error_rate_percentage: f64,
|
|
pub sla_violated: bool,
|
|
pub last_updated: DateTime<Utc>,
|
|
}
|
|
|
|
/// Global SLO metrics (aggregate)
|
|
#[derive(Debug, Clone, Serialize, Deserialize)]
|
|
pub struct GlobalSlo {
|
|
pub availability_percentage: f64,
|
|
pub latency_p99_ms: u64,
|
|
pub error_rate_percentage: f64,
|
|
pub sla_violated: bool,
|
|
pub regions_violating: Vec<String>,
|
|
pub last_updated: DateTime<Utc>,
|
|
}
|
|
|
|
/// SLO breach prediction result
|
|
#[derive(Debug, Clone, Serialize, Deserialize)]
|
|
pub struct PredictionModel {
|
|
pub breach_probability: f64,
|
|
pub estimated_time_to_breach: Duration,
|
|
pub predicted_metric: String,
|
|
pub confidence_interval: (f64, f64),
|
|
pub model_accuracy: f64,
|
|
}
|
|
|
|
/// Alert rule configuration
|
|
#[derive(Debug, Clone, Serialize, Deserialize)]
|
|
pub struct AlertRule {
|
|
pub id: Uuid,
|
|
pub name: String,
|
|
pub condition: String,
|
|
pub threshold: f64,
|
|
pub duration: Duration,
|
|
pub severity: AlertSeverity,
|
|
pub enabled: bool,
|
|
}
|
|
|
|
/// Time-stamped data point for metrics
|
|
#[derive(Debug, Clone)]
|
|
struct DataPoint {
|
|
timestamp: SystemTime,
|
|
value: f64,
|
|
success: bool,
|
|
latency_ms: u64,
|
|
}
|
|
|
|
/// Time-series data storage for a region
|
|
#[derive(Debug)]
|
|
struct RegionMetrics {
|
|
datapoints: VecDeque<DataPoint>,
|
|
total_requests: u64,
|
|
total_successes: u64,
|
|
total_failures: u64,
|
|
downtime_minutes: f64,
|
|
latency_samples: VecDeque<u64>,
|
|
}
|
|
|
|
impl RegionMetrics {
|
|
fn new() -> Self {
|
|
Self {
|
|
datapoints: VecDeque::with_capacity(MAX_DATAPOINTS_PER_WINDOW),
|
|
total_requests: 0,
|
|
total_successes: 0,
|
|
total_failures: 0,
|
|
downtime_minutes: 0.0,
|
|
latency_samples: VecDeque::with_capacity(MAX_DATAPOINTS_PER_WINDOW),
|
|
}
|
|
}
|
|
|
|
fn add_request(&mut self, success: bool, latency: Duration, timestamp: SystemTime) {
|
|
let latency_ms = latency.as_millis() as u64;
|
|
|
|
let datapoint = DataPoint {
|
|
timestamp,
|
|
value: if success { 1.0 } else { 0.0 },
|
|
success,
|
|
latency_ms,
|
|
};
|
|
|
|
// Add to time series with capacity management
|
|
if self.datapoints.len() >= MAX_DATAPOINTS_PER_WINDOW {
|
|
self.datapoints.pop_front();
|
|
}
|
|
self.datapoints.push_back(datapoint);
|
|
|
|
// Update counters
|
|
self.total_requests += 1;
|
|
if success {
|
|
self.total_successes += 1;
|
|
} else {
|
|
self.total_failures += 1;
|
|
}
|
|
|
|
// Add latency sample
|
|
if self.latency_samples.len() >= MAX_DATAPOINTS_PER_WINDOW {
|
|
self.latency_samples.pop_front();
|
|
}
|
|
self.latency_samples.push_back(latency_ms);
|
|
}
|
|
|
|
fn add_downtime(&mut self, duration: Duration) {
|
|
self.downtime_minutes += duration.as_secs_f64() / 60.0;
|
|
}
|
|
|
|
fn cleanup_old_data(&mut self, retention_duration: Duration) {
|
|
let cutoff_time = SystemTime::now() - retention_duration;
|
|
|
|
while let Some(front) = self.datapoints.front() {
|
|
if front.timestamp < cutoff_time {
|
|
self.datapoints.pop_front();
|
|
} else {
|
|
break;
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
/// Metrics collector for gathering system metrics
|
|
#[derive(Debug, Clone)]
|
|
pub struct MetricsCollector {
|
|
region_metrics: Arc<RwLock<HashMap<String, RegionMetrics>>>,
|
|
global_metrics: Arc<RwLock<RegionMetrics>>,
|
|
}
|
|
|
|
impl Default for MetricsCollector {
|
|
fn default() -> Self {
|
|
Self::new()
|
|
}
|
|
}
|
|
|
|
impl MetricsCollector {
|
|
pub fn new() -> Self {
|
|
Self {
|
|
region_metrics: Arc::new(RwLock::new(HashMap::new())),
|
|
global_metrics: Arc::new(RwLock::new(RegionMetrics::new())),
|
|
}
|
|
}
|
|
}
|
|
|
|
/// Main SLO monitor implementation
|
|
#[derive(Debug)]
|
|
pub struct SloMonitor {
|
|
config: PlatformConfig,
|
|
sla_targets: SlaTargets,
|
|
metrics_collector: MetricsCollector,
|
|
active_alerts: Arc<AsyncRwLock<HashMap<Uuid, SloViolation>>>,
|
|
alert_config: Arc<AsyncRwLock<Option<AlertConfiguration>>>,
|
|
running: Arc<std::sync::atomic::AtomicBool>,
|
|
cleanup_interval: Arc<Mutex<Option<Interval>>>,
|
|
}
|
|
|
|
impl SloMonitor {
|
|
/// Helper to get region metrics with error handling
|
|
fn get_region_metrics(
|
|
&self,
|
|
_region: &str,
|
|
) -> PlatformResult<std::sync::RwLockReadGuard<'_, HashMap<String, RegionMetrics>>> {
|
|
self.metrics_collector
|
|
.region_metrics
|
|
.read()
|
|
.map_err(|_| crate::PlatformError::Internal {
|
|
message: "Failed to acquire region metrics lock".to_string(),
|
|
})
|
|
}
|
|
|
|
/// Helper to get mutable region metrics
|
|
fn get_region_metrics_mut(
|
|
&self,
|
|
_region: &str,
|
|
) -> PlatformResult<std::sync::RwLockWriteGuard<'_, HashMap<String, RegionMetrics>>> {
|
|
self.metrics_collector
|
|
.region_metrics
|
|
.write()
|
|
.map_err(|_| crate::PlatformError::Internal {
|
|
message: "Failed to acquire region metrics lock".to_string(),
|
|
})
|
|
}
|
|
|
|
/// Create new SLO monitor instance
|
|
pub async fn new(config: &PlatformConfig) -> PlatformResult<Self> {
|
|
info!("Initializing SLO Monitor with 99.95% availability target");
|
|
|
|
let sla_targets = config.sla_targets.clone();
|
|
let metrics_collector = MetricsCollector::new();
|
|
let active_alerts = Arc::new(AsyncRwLock::new(HashMap::new()));
|
|
let alert_config = Arc::new(AsyncRwLock::new(None));
|
|
let running = Arc::new(std::sync::atomic::AtomicBool::new(false));
|
|
let cleanup_interval = Arc::new(Mutex::new(None));
|
|
|
|
Ok(Self {
|
|
config: config.clone(),
|
|
sla_targets,
|
|
metrics_collector,
|
|
active_alerts,
|
|
alert_config,
|
|
running,
|
|
cleanup_interval,
|
|
})
|
|
}
|
|
|
|
/// Start the SLO monitoring service
|
|
pub async fn start(&mut self) -> PlatformResult<()> {
|
|
info!("Starting SLO Monitor");
|
|
|
|
self.running
|
|
.store(true, std::sync::atomic::Ordering::SeqCst);
|
|
|
|
// Start periodic cleanup task
|
|
let cleanup_task = tokio::time::interval(Duration::from_secs(3600)); // Cleanup every hour
|
|
*self.cleanup_interval.lock().await = Some(cleanup_task);
|
|
|
|
let metrics_collector = self.metrics_collector.clone();
|
|
let running = self.running.clone();
|
|
|
|
// Spawn background cleanup task
|
|
tokio::spawn(async move {
|
|
let mut interval = tokio::time::interval(Duration::from_secs(3600));
|
|
|
|
while running.load(std::sync::atomic::Ordering::SeqCst) {
|
|
interval.tick().await;
|
|
|
|
// Cleanup old data
|
|
let retention_duration = Duration::from_secs(HISTORY_RETENTION_HOURS * 3600);
|
|
|
|
if let Ok(mut regions) = metrics_collector.region_metrics.write() {
|
|
for (_, metrics) in regions.iter_mut() {
|
|
metrics.cleanup_old_data(retention_duration);
|
|
}
|
|
}
|
|
|
|
if let Ok(mut global) = metrics_collector.global_metrics.write() {
|
|
global.cleanup_old_data(retention_duration);
|
|
}
|
|
}
|
|
});
|
|
|
|
info!("SLO Monitor started successfully");
|
|
Ok(())
|
|
}
|
|
|
|
/// Shutdown the SLO monitoring service
|
|
pub async fn shutdown(&mut self) -> PlatformResult<()> {
|
|
info!("Shutting down SLO Monitor");
|
|
|
|
self.running
|
|
.store(false, std::sync::atomic::Ordering::SeqCst);
|
|
|
|
// Clear cleanup interval
|
|
*self.cleanup_interval.lock().await = None;
|
|
|
|
info!("SLO Monitor shutdown complete");
|
|
Ok(())
|
|
}
|
|
|
|
/// Get configured SLA targets
|
|
pub fn get_sla_targets(&self) -> &SlaTargets {
|
|
&self.sla_targets
|
|
}
|
|
|
|
/// Record a successful request
|
|
pub async fn record_request_success(
|
|
&self,
|
|
region: &str,
|
|
latency: Duration,
|
|
) -> PlatformResult<()> {
|
|
self.record_request_success_with_timestamp(region, latency, SystemTime::now())
|
|
.await
|
|
}
|
|
|
|
/// Record a failed request
|
|
pub async fn record_request_failure(
|
|
&self,
|
|
region: &str,
|
|
latency: Duration,
|
|
) -> PlatformResult<()> {
|
|
self.record_request_failure_with_timestamp(region, latency, SystemTime::now())
|
|
.await
|
|
}
|
|
|
|
/// Record a successful request with custom timestamp
|
|
pub async fn record_request_success_with_timestamp(
|
|
&self,
|
|
region: &str,
|
|
latency: Duration,
|
|
timestamp: SystemTime,
|
|
) -> PlatformResult<()> {
|
|
if !self.running.load(std::sync::atomic::Ordering::SeqCst) {
|
|
return Err(crate::PlatformError::Slo(SloError::MonitoringFailure {
|
|
component: "SloMonitor not running".to_string(),
|
|
}));
|
|
}
|
|
|
|
// Update regional and global metrics
|
|
{
|
|
let mut regions = self.get_region_metrics_mut(region)?;
|
|
regions
|
|
.entry(region.to_string())
|
|
.or_insert_with(RegionMetrics::new)
|
|
.add_request(true, latency, timestamp);
|
|
}
|
|
{
|
|
let mut global = self.metrics_collector.global_metrics.write().map_err(|_| {
|
|
crate::PlatformError::Internal {
|
|
message: "Failed to acquire global metrics lock".to_string(),
|
|
}
|
|
})?;
|
|
global.add_request(true, latency, timestamp);
|
|
}
|
|
|
|
Ok(())
|
|
}
|
|
|
|
/// Record a failed request with custom timestamp
|
|
pub async fn record_request_failure_with_timestamp(
|
|
&self,
|
|
region: &str,
|
|
latency: Duration,
|
|
timestamp: SystemTime,
|
|
) -> PlatformResult<()> {
|
|
if !self.running.load(std::sync::atomic::Ordering::SeqCst) {
|
|
return Err(crate::PlatformError::Slo(SloError::MonitoringFailure {
|
|
component: "SloMonitor not running".to_string(),
|
|
}));
|
|
}
|
|
|
|
// Update regional and global metrics
|
|
{
|
|
let mut regions = self.get_region_metrics_mut(region)?;
|
|
regions
|
|
.entry(region.to_string())
|
|
.or_insert_with(RegionMetrics::new)
|
|
.add_request(false, latency, timestamp);
|
|
}
|
|
{
|
|
let mut global = self.metrics_collector.global_metrics.write().map_err(|_| {
|
|
crate::PlatformError::Internal {
|
|
message: "Failed to acquire global metrics lock".to_string(),
|
|
}
|
|
})?;
|
|
global.add_request(false, latency, timestamp);
|
|
}
|
|
|
|
Ok(())
|
|
}
|
|
|
|
/// Record downtime for a region
|
|
pub async fn record_downtime(&self, region: &str, duration: Duration) -> PlatformResult<()> {
|
|
// Update regional and global metrics
|
|
{
|
|
let mut regions = self.get_region_metrics_mut(region)?;
|
|
regions
|
|
.entry(region.to_string())
|
|
.or_insert_with(RegionMetrics::new)
|
|
.add_downtime(duration);
|
|
}
|
|
{
|
|
let mut global = self.metrics_collector.global_metrics.write().map_err(|_| {
|
|
crate::PlatformError::Internal {
|
|
message: "Failed to acquire global metrics lock".to_string(),
|
|
}
|
|
})?;
|
|
global.add_downtime(duration);
|
|
}
|
|
|
|
Ok(())
|
|
}
|
|
|
|
/// Get availability metrics for a region
|
|
pub async fn get_availability_metrics(
|
|
&self,
|
|
region: &str,
|
|
) -> PlatformResult<AvailabilityMetrics> {
|
|
let regions = self.get_region_metrics(region)?;
|
|
let region_metrics = regions.get(region).ok_or_else(|| {
|
|
crate::PlatformError::Slo(SloError::MonitoringFailure {
|
|
component: format!("Region {region} not found"),
|
|
})
|
|
})?;
|
|
|
|
let current_percentage = if region_metrics.total_requests > 0 {
|
|
(region_metrics.total_successes as f64 / region_metrics.total_requests as f64) * 100.0
|
|
} else {
|
|
100.0
|
|
};
|
|
|
|
Ok(AvailabilityMetrics {
|
|
current_percentage,
|
|
total_requests: region_metrics.total_requests,
|
|
successful_requests: region_metrics.total_successes,
|
|
failed_requests: region_metrics.total_failures,
|
|
sla_violated: current_percentage < self.sla_targets.availability_percentage,
|
|
last_updated: Utc::now(),
|
|
})
|
|
}
|
|
|
|
/// Get availability metrics for a specific time period
|
|
pub async fn get_availability_metrics_for_period(
|
|
&self,
|
|
region: &str,
|
|
period: Duration,
|
|
) -> PlatformResult<AvailabilityMetrics> {
|
|
let regions = self.metrics_collector.region_metrics.read().map_err(|_| {
|
|
crate::PlatformError::Internal {
|
|
message: "Failed to acquire region metrics lock".to_string(),
|
|
}
|
|
})?;
|
|
|
|
let region_metrics = regions.get(region).ok_or_else(|| {
|
|
crate::PlatformError::Slo(SloError::MonitoringFailure {
|
|
component: format!("Region {region} not found"),
|
|
})
|
|
})?;
|
|
|
|
let cutoff_time = SystemTime::now() - period;
|
|
let mut total_requests = 0u64;
|
|
let mut successful_requests = 0u64;
|
|
|
|
for datapoint in ®ion_metrics.datapoints {
|
|
if datapoint.timestamp >= cutoff_time {
|
|
total_requests += 1;
|
|
if datapoint.success {
|
|
successful_requests += 1;
|
|
}
|
|
}
|
|
}
|
|
|
|
let failed_requests = total_requests - successful_requests;
|
|
let current_percentage = if total_requests > 0 {
|
|
(successful_requests as f64 / total_requests as f64) * 100.0
|
|
} else {
|
|
100.0
|
|
};
|
|
|
|
let sla_violated = current_percentage < self.sla_targets.availability_percentage;
|
|
|
|
Ok(AvailabilityMetrics {
|
|
current_percentage,
|
|
total_requests,
|
|
successful_requests,
|
|
failed_requests,
|
|
sla_violated,
|
|
last_updated: Utc::now(),
|
|
})
|
|
}
|
|
|
|
/// Get latency metrics for a region
|
|
pub async fn get_latency_metrics(&self, region: &str) -> PlatformResult<LatencyMetrics> {
|
|
let regions = self.metrics_collector.region_metrics.read().map_err(|_| {
|
|
crate::PlatformError::Internal {
|
|
message: "Failed to acquire region metrics lock".to_string(),
|
|
}
|
|
})?;
|
|
|
|
let region_metrics = regions.get(region).ok_or_else(|| {
|
|
crate::PlatformError::Slo(SloError::MonitoringFailure {
|
|
component: format!("Region {region} not found"),
|
|
})
|
|
})?;
|
|
|
|
let sample_count = region_metrics.latency_samples.len() as u64;
|
|
|
|
if sample_count == 0 {
|
|
return Ok(LatencyMetrics {
|
|
p50_ms: 0,
|
|
p99_ms: 0,
|
|
p999_ms: 0,
|
|
sla_violated_p50: false,
|
|
sla_violated_p99: false,
|
|
sla_violated_p999: false,
|
|
sample_count: 0,
|
|
last_updated: Utc::now(),
|
|
});
|
|
}
|
|
|
|
let mut samples: Vec<u64> = region_metrics.latency_samples.iter().copied().collect();
|
|
samples.sort_unstable();
|
|
|
|
let p50_idx = (sample_count as f64 * 0.5).ceil() as usize - 1;
|
|
let p99_idx = (sample_count as f64 * 0.99).ceil() as usize - 1;
|
|
let p999_idx = (sample_count as f64 * 0.999).ceil() as usize - 1;
|
|
|
|
let p50_ms = samples.get(p50_idx).copied().unwrap_or(0);
|
|
let p99_ms = samples.get(p99_idx).copied().unwrap_or(0);
|
|
let p999_ms = samples.get(p999_idx).copied().unwrap_or(0);
|
|
|
|
let sla_violated_p50 = p50_ms > self.sla_targets.max_latency_p50_ms;
|
|
let sla_violated_p99 = p99_ms > self.sla_targets.max_latency_p99_ms;
|
|
let sla_violated_p999 = p999_ms > self.sla_targets.max_latency_p999_ms;
|
|
|
|
Ok(LatencyMetrics {
|
|
p50_ms,
|
|
p99_ms,
|
|
p999_ms,
|
|
sla_violated_p50,
|
|
sla_violated_p99,
|
|
sla_violated_p999,
|
|
sample_count,
|
|
last_updated: Utc::now(),
|
|
})
|
|
}
|
|
|
|
/// Get error budget for a region
|
|
pub async fn get_error_budget(&self, region: &str) -> PlatformResult<ErrorBudget> {
|
|
let regions = self.metrics_collector.region_metrics.read().map_err(|_| {
|
|
crate::PlatformError::Internal {
|
|
message: "Failed to acquire region metrics lock".to_string(),
|
|
}
|
|
})?;
|
|
|
|
let region_metrics = regions.get(region).ok_or_else(|| {
|
|
crate::PlatformError::Slo(SloError::MonitoringFailure {
|
|
component: format!("Region {region} not found"),
|
|
})
|
|
})?;
|
|
|
|
let total_budget_minutes = self.sla_targets.downtime_budget_minutes_per_month;
|
|
let consumed_minutes = region_metrics.downtime_minutes;
|
|
let remaining_percentage =
|
|
((total_budget_minutes - consumed_minutes) / total_budget_minutes * 100.0).max(0.0);
|
|
|
|
// Calculate burn rates (simplified)
|
|
let burn_rate_1h = if consumed_minutes > 0.0 {
|
|
consumed_minutes / (30.0 * 24.0 * 60.0) * 100.0
|
|
} else {
|
|
0.0
|
|
};
|
|
let burn_rate_24h = burn_rate_1h;
|
|
|
|
Ok(ErrorBudget {
|
|
total_budget_minutes,
|
|
consumed_minutes,
|
|
remaining_percentage,
|
|
burn_rate_1h,
|
|
burn_rate_24h,
|
|
last_updated: Utc::now(),
|
|
})
|
|
}
|
|
|
|
/// Calculate burn rate for a region over a time window
|
|
pub async fn calculate_burn_rate(&self, region: &str, window: Duration) -> PlatformResult<f64> {
|
|
let regions = self.metrics_collector.region_metrics.read().map_err(|_| {
|
|
crate::PlatformError::Internal {
|
|
message: "Failed to acquire region metrics lock".to_string(),
|
|
}
|
|
})?;
|
|
|
|
let region_metrics = regions.get(region).ok_or_else(|| {
|
|
crate::PlatformError::Slo(SloError::MonitoringFailure {
|
|
component: format!("Region {region} not found"),
|
|
})
|
|
})?;
|
|
|
|
let cutoff_time = SystemTime::now() - window;
|
|
let mut total_requests = 0u64;
|
|
let mut failed_requests = 0u64;
|
|
|
|
for datapoint in ®ion_metrics.datapoints {
|
|
if datapoint.timestamp >= cutoff_time {
|
|
total_requests += 1;
|
|
if !datapoint.success {
|
|
failed_requests += 1;
|
|
}
|
|
}
|
|
}
|
|
|
|
let error_rate = if total_requests > 0 {
|
|
(failed_requests as f64 / total_requests as f64) * 100.0
|
|
} else {
|
|
0.0
|
|
};
|
|
|
|
// Burn rate = actual error rate / SLA error rate allowance
|
|
let burn_rate = error_rate / self.sla_targets.max_error_rate_percentage;
|
|
|
|
// Generate alert if burn rate exceeds threshold
|
|
if burn_rate > self.sla_targets.error_budget_burn_rate_threshold {
|
|
let alert = SloViolation {
|
|
id: Uuid::new_v4(),
|
|
alert_type: "burn_rate_high".to_string(),
|
|
severity: AlertSeverity::Warning,
|
|
region: Some(region.to_string()),
|
|
metric: "error_rate".to_string(),
|
|
current_value: error_rate,
|
|
threshold: self.sla_targets.max_error_rate_percentage,
|
|
message: format!("High burn rate detected: {burn_rate:.2}x"),
|
|
escalated: false,
|
|
notification_attempts: 1,
|
|
actions_taken: vec![],
|
|
created_at: Utc::now(),
|
|
resolved_at: None,
|
|
};
|
|
|
|
let mut alerts = self.active_alerts.write().await;
|
|
alerts.insert(alert.id, alert);
|
|
}
|
|
|
|
Ok(burn_rate)
|
|
}
|
|
|
|
/// Get error rates for multiple time windows
|
|
pub async fn get_error_rates(&self, region: &str) -> PlatformResult<ErrorRateMetrics> {
|
|
let regions = self.metrics_collector.region_metrics.read().map_err(|_| {
|
|
crate::PlatformError::Internal {
|
|
message: "Failed to acquire region metrics lock".to_string(),
|
|
}
|
|
})?;
|
|
|
|
let region_metrics = regions.get(region).ok_or_else(|| {
|
|
crate::PlatformError::Slo(SloError::MonitoringFailure {
|
|
component: format!("Region {region} not found"),
|
|
})
|
|
})?;
|
|
|
|
let now = SystemTime::now();
|
|
let windows = [
|
|
(Duration::from_secs(60), "1m"),
|
|
(Duration::from_secs(300), "5m"),
|
|
(Duration::from_secs(3600), "1h"),
|
|
];
|
|
|
|
let mut error_rates = vec![];
|
|
for (window_duration, _name) in &windows {
|
|
let cutoff_time = now - *window_duration;
|
|
let mut total = 0u64;
|
|
let mut failures = 0u64;
|
|
|
|
for datapoint in ®ion_metrics.datapoints {
|
|
if datapoint.timestamp >= cutoff_time {
|
|
total += 1;
|
|
if !datapoint.success {
|
|
failures += 1;
|
|
}
|
|
}
|
|
}
|
|
|
|
let error_rate = if total > 0 {
|
|
(failures as f64 / total as f64) * 100.0
|
|
} else {
|
|
0.0
|
|
};
|
|
|
|
error_rates.push(error_rate);
|
|
}
|
|
|
|
let window_1m = error_rates.first().copied().unwrap_or(0.0);
|
|
let window_5m = error_rates.get(1).copied().unwrap_or(0.0);
|
|
let window_1h = error_rates.get(2).copied().unwrap_or(0.0);
|
|
|
|
let sla_violated_1m = window_1m > self.sla_targets.max_error_rate_percentage;
|
|
let sla_violated_5m = window_5m > self.sla_targets.max_error_rate_percentage;
|
|
let sla_violated_1h = window_1h > self.sla_targets.max_error_rate_percentage;
|
|
|
|
Ok(ErrorRateMetrics {
|
|
window_1m,
|
|
window_5m,
|
|
window_1h,
|
|
sla_violated_1m,
|
|
sla_violated_5m,
|
|
sla_violated_1h,
|
|
last_updated: Utc::now(),
|
|
})
|
|
}
|
|
|
|
/// Configure alerting
|
|
pub async fn configure_alerts(&mut self, config: AlertConfiguration) -> PlatformResult<()> {
|
|
let mut alert_config = self.alert_config.write().await;
|
|
*alert_config = Some(config);
|
|
Ok(())
|
|
}
|
|
|
|
/// Get active alerts
|
|
pub async fn get_active_alerts(&self) -> PlatformResult<Vec<SloViolation>> {
|
|
let alerts = self.active_alerts.read().await;
|
|
Ok(alerts.values().cloned().collect())
|
|
}
|
|
|
|
/// Predict SLO breach
|
|
pub async fn predict_slo_breach(
|
|
&self,
|
|
region: &str,
|
|
horizon: Duration,
|
|
) -> PlatformResult<PredictionModel> {
|
|
// Simplified prediction based on recent trend
|
|
let regions = self.metrics_collector.region_metrics.read().map_err(|_| {
|
|
crate::PlatformError::Internal {
|
|
message: "Failed to acquire region metrics lock".to_string(),
|
|
}
|
|
})?;
|
|
|
|
let region_metrics = regions.get(region).ok_or_else(|| {
|
|
crate::PlatformError::Slo(SloError::MonitoringFailure {
|
|
component: format!("Region {region} not found"),
|
|
})
|
|
})?;
|
|
|
|
let recent_window = Duration::from_secs(3600); // Last hour
|
|
let cutoff_time = SystemTime::now() - recent_window;
|
|
|
|
let mut recent_error_rate = 0.0;
|
|
let mut recent_total = 0u64;
|
|
let mut recent_failures = 0u64;
|
|
|
|
for datapoint in ®ion_metrics.datapoints {
|
|
if datapoint.timestamp >= cutoff_time {
|
|
recent_total += 1;
|
|
if !datapoint.success {
|
|
recent_failures += 1;
|
|
}
|
|
}
|
|
}
|
|
|
|
if recent_total > 0 {
|
|
recent_error_rate = (recent_failures as f64 / recent_total as f64) * 100.0;
|
|
}
|
|
|
|
let breach_probability = if recent_error_rate > self.sla_targets.max_error_rate_percentage {
|
|
0.8 // High probability if already violating
|
|
} else {
|
|
recent_error_rate / self.sla_targets.max_error_rate_percentage * 0.5
|
|
};
|
|
|
|
let estimated_time_to_breach = if breach_probability > 0.7 {
|
|
Duration::from_secs(3600) // 1 hour
|
|
} else {
|
|
horizon
|
|
};
|
|
|
|
// Generate predictive alert if high probability
|
|
if breach_probability > 0.7 {
|
|
let alert = SloViolation {
|
|
id: Uuid::new_v4(),
|
|
alert_type: "slo_breach_predicted".to_string(),
|
|
severity: AlertSeverity::Warning,
|
|
region: Some(region.to_string()),
|
|
metric: "availability".to_string(),
|
|
current_value: recent_error_rate,
|
|
threshold: self.sla_targets.max_error_rate_percentage,
|
|
message: format!(
|
|
"SLO breach predicted with {:.1}% probability",
|
|
breach_probability * 100.0
|
|
),
|
|
escalated: false,
|
|
notification_attempts: 1,
|
|
actions_taken: vec![],
|
|
created_at: Utc::now(),
|
|
resolved_at: None,
|
|
};
|
|
|
|
let mut alerts = self.active_alerts.write().await;
|
|
alerts.insert(alert.id, alert);
|
|
}
|
|
|
|
Ok(PredictionModel {
|
|
breach_probability,
|
|
estimated_time_to_breach,
|
|
predicted_metric: "availability".to_string(),
|
|
confidence_interval: (breach_probability - 0.1, breach_probability + 0.1),
|
|
model_accuracy: 0.75,
|
|
})
|
|
}
|
|
|
|
/// Get regional SLO metrics
|
|
pub async fn get_regional_slo(&self, region: &str) -> PlatformResult<RegionalSlo> {
|
|
let availability = self.get_availability_metrics(region).await?;
|
|
let latency = self.get_latency_metrics(region).await?;
|
|
let error_rates = self.get_error_rates(region).await?;
|
|
|
|
let sla_violated =
|
|
availability.sla_violated || latency.sla_violated_p99 || error_rates.sla_violated_1h;
|
|
|
|
Ok(RegionalSlo {
|
|
region_id: region.to_string(),
|
|
availability_percentage: availability.current_percentage,
|
|
latency_p99_ms: latency.p99_ms,
|
|
error_rate_percentage: error_rates.window_1h,
|
|
sla_violated,
|
|
last_updated: Utc::now(),
|
|
})
|
|
}
|
|
|
|
/// Get global SLO metrics (aggregate across all regions)
|
|
pub async fn get_global_slo(&self) -> PlatformResult<GlobalSlo> {
|
|
let global = self.metrics_collector.global_metrics.read().map_err(|_| {
|
|
crate::PlatformError::Internal {
|
|
message: "Failed to acquire global metrics lock".to_string(),
|
|
}
|
|
})?;
|
|
|
|
let total_requests = global.total_requests;
|
|
let successful_requests = global.total_successes;
|
|
|
|
let availability_percentage = if total_requests > 0 {
|
|
(successful_requests as f64 / total_requests as f64) * 100.0
|
|
} else {
|
|
100.0
|
|
};
|
|
|
|
// Calculate global latency P99 (simplified)
|
|
let mut samples: Vec<u64> = global.latency_samples.iter().copied().collect();
|
|
samples.sort_unstable();
|
|
let sample_count = samples.len();
|
|
let latency_p99_ms = if sample_count > 0 {
|
|
let p99_idx = (sample_count as f64 * 0.99).ceil() as usize - 1;
|
|
samples.get(p99_idx).copied().unwrap_or(0)
|
|
} else {
|
|
0
|
|
};
|
|
|
|
// Global error rate calculation
|
|
let error_rate_percentage = if total_requests > 0 {
|
|
((total_requests - successful_requests) as f64 / total_requests as f64) * 100.0
|
|
} else {
|
|
0.0
|
|
};
|
|
|
|
let sla_violated = availability_percentage < self.sla_targets.availability_percentage
|
|
|| latency_p99_ms > self.sla_targets.max_latency_p99_ms
|
|
|| error_rate_percentage > self.sla_targets.max_error_rate_percentage;
|
|
|
|
Ok(GlobalSlo {
|
|
availability_percentage,
|
|
latency_p99_ms,
|
|
error_rate_percentage,
|
|
sla_violated,
|
|
regions_violating: vec![], // Would need to check each region
|
|
last_updated: Utc::now(),
|
|
})
|
|
}
|
|
|
|
/// Export metrics in Prometheus format
|
|
pub async fn export_prometheus_metrics(&self) -> PlatformResult<String> {
|
|
let mut output = String::new();
|
|
let global_slo = self.get_global_slo().await?;
|
|
let error_budget = self
|
|
.get_error_budget("global")
|
|
.await
|
|
.unwrap_or(ErrorBudget {
|
|
total_budget_minutes: self.sla_targets.downtime_budget_minutes_per_month,
|
|
consumed_minutes: 0.0,
|
|
remaining_percentage: 100.0,
|
|
burn_rate_1h: 0.0,
|
|
burn_rate_24h: 0.0,
|
|
last_updated: Utc::now(),
|
|
});
|
|
|
|
output.push_str(&format!(
|
|
"slo_availability_percentage{{region=\"global\"}} {}\nslo_latency_p99_seconds{{region=\"global\"}} {}\nslo_latency_p50_seconds{{region=\"global\"}} {}\nslo_error_budget_remaining_percentage{{region=\"global\"}} {}\nslo_burn_rate{{region=\"global\"}} {}\n",
|
|
global_slo.availability_percentage, global_slo.latency_p99_ms as f64 / 1000.0, 0.0, error_budget.remaining_percentage, error_budget.burn_rate_1h
|
|
));
|
|
|
|
let regions = self.get_region_metrics("global")?;
|
|
for (region_name, _) in regions.iter() {
|
|
if let Ok(regional_slo) = self.get_regional_slo(region_name).await {
|
|
output.push_str(&format!(
|
|
"slo_availability_percentage{{region=\"{}\"}} {}\nslo_latency_p99_seconds{{region=\"{}\"}} {}\n",
|
|
region_name, regional_slo.availability_percentage, region_name, regional_slo.latency_p99_ms as f64 / 1000.0
|
|
));
|
|
}
|
|
}
|
|
Ok(output)
|
|
}
|
|
}
|