//! Comprehensive Monitoring and Metrics Collection //! //! This module provides detailed monitoring capabilities for the inference engine including: //! - Real-time performance metrics //! - Resource utilization tracking //! - Request tracing and latency analysis //! - Model-specific metrics //! - Health checks and alerting //! - Prometheus-compatible metrics export use crate::{FinishReason, InferenceResult, RequestMetrics, RequestResult}; use serde::{Deserialize, Serialize}; use std::collections::{HashMap, VecDeque}; use std::sync::Arc; use std::time::{Duration, SystemTime}; use tokio::sync::{Mutex, RwLock}; use tracing::{debug, info, instrument}; use uuid::Uuid; /// Configuration for monitoring and metrics collection #[derive(Debug, Clone)] pub struct MonitoringConfig { /// Enable detailed request tracing pub enable_tracing: bool, /// Maximum number of request traces to keep in memory pub max_traces: usize, /// Metrics aggregation window size pub aggregation_window: Duration, /// Enable real-time health checks pub enable_health_checks: bool, /// Health check interval pub health_check_interval: Duration, /// Enable Prometheus metrics export pub enable_prometheus: bool, /// Custom metrics retention period pub metrics_retention: Duration, } impl Default for MonitoringConfig { fn default() -> Self { Self { enable_tracing: true, max_traces: 10000, aggregation_window: Duration::from_secs(60), enable_health_checks: true, health_check_interval: Duration::from_secs(10), enable_prometheus: true, metrics_retention: Duration::from_secs(3600), // 1 hour } } } /// Comprehensive metrics collector for the inference engine pub struct MetricsCollector { config: MonitoringConfig, /// Real-time performance metrics performance_metrics: Arc>, /// Resource utilization metrics resource_metrics: Arc>, /// Model-specific metrics model_metrics: Arc>>, /// Request tracing data request_traces: Arc>>, /// Health status health_status: Arc>, /// Time-series metrics for trending time_series_metrics: Arc>, /// Background tasks _metrics_task: tokio::task::JoinHandle<()>, _health_check_task: tokio::task::JoinHandle<()>, } /// Real-time performance metrics #[derive(Debug, Clone, Serialize, Deserialize)] pub struct PerformanceMetrics { /// Total number of requests processed pub total_requests: u64, /// Successful requests pub successful_requests: u64, /// Failed requests pub failed_requests: u64, /// Current requests per second pub requests_per_second: f64, /// Average latency (p50) pub avg_latency_ms: f64, /// P95 latency pub p95_latency_ms: f64, /// P99 latency pub p99_latency_ms: f64, /// Total tokens generated pub tokens_generated: u64, /// Tokens per second pub tokens_per_second: f64, /// Success rate percentage pub success_rate: f64, /// Average batch size pub avg_batch_size: f64, /// Queue depth across all SLA lanes pub total_queue_depth: usize, /// Last updated timestamp pub updated_at: SystemTime, } /// Resource utilization metrics #[derive(Debug, Clone, Serialize, Deserialize)] pub struct ResourceMetrics { /// GPU memory usage percentage pub gpu_memory_utilization: f64, /// CPU usage percentage pub cpu_utilization: f64, /// System memory usage pub memory_usage_gb: f64, /// Memory fragmentation ratio pub memory_fragmentation: f64, /// Active connections pub active_connections: u32, /// KV cache hit rate pub cache_hit_rate: f64, /// Disk I/O for model loading pub disk_io_mbps: f64, /// Network throughput pub network_throughput_mbps: f64, /// Temperature metrics (if available) pub gpu_temperature_c: Option, /// Power consumption (if available) pub power_consumption_w: Option, } /// Model-specific performance metrics #[derive(Debug, Clone, Serialize, Deserialize)] pub struct ModelSpecificMetrics { pub model_name: String, pub model_version: String, pub requests_processed: u64, pub total_inference_time: Duration, pub avg_tokens_per_request: f64, pub avg_latency_ms: f64, pub memory_usage_mb: f64, pub optimization_speedup: f64, pub quantization_speedup: Option, pub last_used: SystemTime, pub load_time: Duration, } /// Individual request trace for debugging and analysis #[derive(Debug, Clone, Serialize)] pub struct RequestTrace { pub request_id: Uuid, pub model_name: String, pub timestamp: SystemTime, pub input_tokens: usize, pub output_tokens: usize, pub total_latency_ms: f64, pub queue_time_ms: f64, pub processing_time_ms: f64, pub generation_time_ms: f64, pub memory_usage_mb: f64, pub sla_lane: String, pub temperature: Option, pub top_p: Option, pub top_k: Option, pub finish_reason: String, pub error: Option, } /// Health status of the inference engine #[derive(Debug, Clone, Serialize, Deserialize)] pub struct HealthStatus { pub overall_status: HealthLevel, pub model_health: HashMap, pub resource_health: ResourceHealth, pub last_check: SystemTime, pub alerts: Vec, } /// Health levels #[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize)] pub enum HealthLevel { Healthy, Warning, Critical, Down, } /// Resource health status #[derive(Debug, Clone, Serialize, Deserialize)] pub struct ResourceHealth { pub memory_status: HealthLevel, pub gpu_status: HealthLevel, pub cpu_status: HealthLevel, pub disk_status: HealthLevel, pub network_status: HealthLevel, } /// Alert information #[derive(Debug, Clone, Serialize, Deserialize)] pub struct Alert { pub id: Uuid, pub level: HealthLevel, pub message: String, pub component: String, pub timestamp: SystemTime, pub resolved: bool, } /// Time-series metrics for trending analysis #[derive(Debug)] pub struct TimeSeriesMetrics { /// Latency samples over time (timestamp, `latency_ms`) pub latency_samples: VecDeque<(SystemTime, f64)>, /// Throughput samples over time (timestamp, rps) pub throughput_samples: VecDeque<(SystemTime, f64)>, /// Memory usage samples over time (timestamp, `usage_pct`) pub memory_samples: VecDeque<(SystemTime, f64)>, /// Error rate samples over time (timestamp, `error_rate`) pub error_samples: VecDeque<(SystemTime, f64)>, } impl MetricsCollector { /// Create a new metrics collector pub async fn new(config: MonitoringConfig) -> InferenceResult { let performance_metrics = Arc::new(RwLock::new(PerformanceMetrics { total_requests: 0, successful_requests: 0, failed_requests: 0, requests_per_second: 0.0, avg_latency_ms: 0.0, p95_latency_ms: 0.0, p99_latency_ms: 0.0, tokens_generated: 0, tokens_per_second: 0.0, success_rate: 0.0, avg_batch_size: 0.0, total_queue_depth: 0, updated_at: SystemTime::now(), })); let resource_metrics = Arc::new(RwLock::new(ResourceMetrics { gpu_memory_utilization: 0.0, cpu_utilization: 0.0, memory_usage_gb: 0.0, memory_fragmentation: 0.0, active_connections: 0, cache_hit_rate: 0.0, disk_io_mbps: 0.0, network_throughput_mbps: 0.0, gpu_temperature_c: None, power_consumption_w: None, })); let model_metrics = Arc::new(RwLock::new(HashMap::new())); let request_traces = Arc::new(Mutex::new(VecDeque::new())); let health_status = Arc::new(RwLock::new(HealthStatus { overall_status: HealthLevel::Healthy, model_health: HashMap::new(), resource_health: ResourceHealth { memory_status: HealthLevel::Healthy, gpu_status: HealthLevel::Healthy, cpu_status: HealthLevel::Healthy, disk_status: HealthLevel::Healthy, network_status: HealthLevel::Healthy, }, last_check: SystemTime::now(), alerts: Vec::new(), })); let time_series_metrics = Arc::new(Mutex::new(TimeSeriesMetrics { latency_samples: VecDeque::new(), throughput_samples: VecDeque::new(), memory_samples: VecDeque::new(), error_samples: VecDeque::new(), })); // Start background tasks let metrics_task = Self::start_metrics_aggregation_task( Arc::clone(&performance_metrics), Arc::clone(&time_series_metrics), config.aggregation_window, config.metrics_retention, ); let health_check_task = Self::start_health_check_task( Arc::clone(&health_status), Arc::clone(&resource_metrics), Arc::clone(&model_metrics), config.health_check_interval, ); info!("Metrics collector initialized with config: {:?}", config); Ok(Self { config, performance_metrics, resource_metrics, model_metrics, request_traces, health_status, time_series_metrics, _metrics_task: metrics_task, _health_check_task: health_check_task, }) } /// Record a completed inference request #[instrument(skip(self, result, metrics))] pub async fn record_request( &self, model_name: &str, result: &RequestResult, metrics: &RequestMetrics, sla_lane: &str, error: Option<&str>, ) -> InferenceResult<()> { let timestamp = SystemTime::now(); // Update performance metrics { let mut perf_metrics = self.performance_metrics.write().await; perf_metrics.total_requests += 1; if error.is_none() { perf_metrics.successful_requests += 1; perf_metrics.tokens_generated += metrics.output_token_count as u64; } else { perf_metrics.failed_requests += 1; } perf_metrics.success_rate = perf_metrics.successful_requests as f64 / perf_metrics.total_requests as f64; perf_metrics.updated_at = timestamp; } // Update model-specific metrics { let mut model_metrics = self.model_metrics.write().await; let model_metric = model_metrics .entry(model_name.to_string()) .or_insert_with(|| ModelSpecificMetrics { model_name: model_name.to_string(), model_version: Self::detect_model_version(model_name), requests_processed: 0, total_inference_time: Duration::ZERO, avg_tokens_per_request: 0.0, avg_latency_ms: 0.0, memory_usage_mb: 0.0, optimization_speedup: 1.0, quantization_speedup: None, last_used: timestamp, load_time: Duration::ZERO, }); model_metric.requests_processed += 1; model_metric.total_inference_time += metrics.total_time; model_metric.avg_latency_ms = model_metric.total_inference_time.as_millis() as f64 / model_metric.requests_processed as f64; model_metric.avg_tokens_per_request = (model_metric.avg_tokens_per_request * (model_metric.requests_processed - 1) as f64 + metrics.output_token_count as f64) / model_metric.requests_processed as f64; model_metric.last_used = timestamp; } // Add request trace if tracing is enabled if self.config.enable_tracing { let mut traces = self.request_traces.lock().await; let trace = RequestTrace { request_id: result.request_id.into(), model_name: model_name.to_string(), timestamp, input_tokens: metrics.input_token_count, output_tokens: metrics.output_token_count, total_latency_ms: metrics.total_time.as_millis() as f64, queue_time_ms: metrics.queue_time.as_millis() as f64, processing_time_ms: metrics.processing_time.as_millis() as f64, generation_time_ms: metrics.generation_time.as_millis() as f64, memory_usage_mb: metrics.peak_memory_bytes as f64 / 1024.0 / 1024.0, sla_lane: sla_lane.to_string(), temperature: Self::extract_temperature_from_result(result), top_p: Self::extract_top_p_from_result(result), top_k: Self::extract_top_k_from_result(result), finish_reason: format!("{:?}", result.finish_reason), error: error.map(std::string::ToString::to_string), }; traces.push_back(trace); // Maintain max traces limit while traces.len() > self.config.max_traces { traces.pop_front(); } } // Update time-series data { let mut ts_metrics = self.time_series_metrics.lock().await; ts_metrics .latency_samples .push_back((timestamp, metrics.total_time.as_millis() as f64)); // Clean old samples let cutoff = timestamp - self.config.metrics_retention; while let Some((ts, _)) = ts_metrics.latency_samples.front() { if *ts < cutoff { ts_metrics.latency_samples.pop_front(); } else { break; } } } Ok(()) } /// Get current performance metrics pub async fn get_performance_metrics(&self) -> PerformanceMetrics { self.performance_metrics.read().await.clone() } /// Get current resource metrics pub async fn get_resource_metrics(&self) -> ResourceMetrics { self.resource_metrics.read().await.clone() } /// Get model-specific metrics pub async fn get_model_metrics(&self) -> HashMap { self.model_metrics.read().await.clone() } /// Get recent request traces pub async fn get_recent_traces(&self, limit: usize) -> Vec { let traces = self.request_traces.lock().await; traces.iter().rev().take(limit).cloned().collect() } /// Get current health status pub async fn get_health_status(&self) -> HealthStatus { self.health_status.read().await.clone() } /// Export metrics in Prometheus format pub async fn export_prometheus_metrics(&self) -> String { let perf_metrics = self.get_performance_metrics().await; let resource_metrics = self.get_resource_metrics().await; let model_metrics = self.get_model_metrics().await; let mut output = String::new(); // Performance metrics output.push_str(&format!( "rtx_inference_requests_total {}\n", perf_metrics.total_requests )); output.push_str(&format!( "rtx_inference_requests_successful {}\n", perf_metrics.successful_requests )); output.push_str(&format!( "rtx_inference_requests_failed {}\n", perf_metrics.failed_requests )); output.push_str(&format!( "rtx_inference_requests_per_second {}\n", perf_metrics.requests_per_second )); output.push_str(&format!( "rtx_inference_latency_avg_ms {}\n", perf_metrics.avg_latency_ms )); output.push_str(&format!( "rtx_inference_latency_p95_ms {}\n", perf_metrics.p95_latency_ms )); output.push_str(&format!( "rtx_inference_latency_p99_ms {}\n", perf_metrics.p99_latency_ms )); output.push_str(&format!( "rtx_inference_tokens_generated {}\n", perf_metrics.tokens_generated )); output.push_str(&format!( "rtx_inference_tokens_per_second {}\n", perf_metrics.tokens_per_second )); output.push_str(&format!( "rtx_inference_success_rate {}\n", perf_metrics.success_rate )); // Resource metrics output.push_str(&format!( "rtx_inference_gpu_memory_utilization {}\n", resource_metrics.gpu_memory_utilization )); output.push_str(&format!( "rtx_inference_cpu_utilization {}\n", resource_metrics.cpu_utilization )); output.push_str(&format!( "rtx_inference_memory_usage_gb {}\n", resource_metrics.memory_usage_gb )); output.push_str(&format!( "rtx_inference_cache_hit_rate {}\n", resource_metrics.cache_hit_rate )); // Model-specific metrics for (model_name, metrics) in model_metrics { let label = format!("model=\"{model_name}\""); output.push_str(&format!( "rtx_inference_model_requests_processed{{{}}} {}\n", label, metrics.requests_processed )); output.push_str(&format!( "rtx_inference_model_avg_latency_ms{{{}}} {}\n", label, metrics.avg_latency_ms )); output.push_str(&format!( "rtx_inference_model_memory_usage_mb{{{}}} {}\n", label, metrics.memory_usage_mb )); } output } /// Update resource utilization metrics (called by external monitoring) pub async fn update_resource_metrics( &self, gpu_memory_pct: f64, cpu_pct: f64, memory_gb: f64, fragmentation: f64, cache_hit_rate: f64, ) { let mut metrics = self.resource_metrics.write().await; metrics.gpu_memory_utilization = gpu_memory_pct; metrics.cpu_utilization = cpu_pct; metrics.memory_usage_gb = memory_gb; metrics.memory_fragmentation = fragmentation; metrics.cache_hit_rate = cache_hit_rate; } /// Start background task for metrics aggregation fn start_metrics_aggregation_task( performance_metrics: Arc>, time_series_metrics: Arc>, window: Duration, _retention: Duration, ) -> tokio::task::JoinHandle<()> { tokio::spawn(async move { let mut interval = tokio::time::interval(window); loop { interval.tick().await; let now = SystemTime::now(); // Aggregate metrics from time series data { let ts_metrics = time_series_metrics.lock().await; let mut perf_metrics = performance_metrics.write().await; // Calculate percentiles and rates from recent samples if !ts_metrics.latency_samples.is_empty() { let recent_cutoff = now - window; let recent_latencies: Vec = ts_metrics .latency_samples .iter() .filter(|(ts, _)| *ts >= recent_cutoff) .map(|(_, latency)| *latency) .collect(); if !recent_latencies.is_empty() { perf_metrics.avg_latency_ms = recent_latencies.iter().sum::() / recent_latencies.len() as f64; // Calculate percentiles let mut sorted_latencies = recent_latencies; sorted_latencies.sort_by(f64::total_cmp); let p95_idx = (sorted_latencies.len() as f64 * 0.95) as usize; let p99_idx = (sorted_latencies.len() as f64 * 0.99) as usize; perf_metrics.p95_latency_ms = sorted_latencies.get(p95_idx).copied().unwrap_or(0.0); perf_metrics.p99_latency_ms = sorted_latencies.get(p99_idx).copied().unwrap_or(0.0); } // Calculate requests per second let recent_request_count = ts_metrics .latency_samples .iter() .filter(|(ts, _)| *ts >= recent_cutoff) .count(); perf_metrics.requests_per_second = recent_request_count as f64 / window.as_secs_f64(); } perf_metrics.updated_at = now; } debug!("Metrics aggregated at {:?}", now); } }) } /// Start background health check task fn start_health_check_task( health_status: Arc>, resource_metrics: Arc>, model_metrics: Arc>>, interval: Duration, ) -> tokio::task::JoinHandle<()> { tokio::spawn(async move { let mut ticker = tokio::time::interval(interval); loop { ticker.tick().await; let now = SystemTime::now(); let mut health = health_status.write().await; // Check resource health let resource_metrics_read = resource_metrics.read().await; health.resource_health.memory_status = if resource_metrics_read.gpu_memory_utilization > 0.9 { HealthLevel::Critical } else if resource_metrics_read.gpu_memory_utilization > 0.8 { HealthLevel::Warning } else { HealthLevel::Healthy }; health.resource_health.cpu_status = if resource_metrics_read.cpu_utilization > 0.9 { HealthLevel::Critical } else if resource_metrics_read.cpu_utilization > 0.8 { HealthLevel::Warning } else { HealthLevel::Healthy }; drop(resource_metrics_read); // Check model health let model_metrics_read = model_metrics.read().await; for (model_name, metrics) in model_metrics_read.iter() { let time_since_last_use = now .duration_since(metrics.last_used) .unwrap_or(Duration::ZERO); let model_health = if time_since_last_use > Duration::from_secs(3600) { // 1 hour HealthLevel::Warning } else { HealthLevel::Healthy }; health.model_health.insert(model_name.clone(), model_health); } drop(model_metrics_read); // Determine overall health health.overall_status = [ health.resource_health.memory_status, health.resource_health.cpu_status, health.resource_health.gpu_status, ] .iter() .chain(health.model_health.values()) .max() .copied() .unwrap_or(HealthLevel::Healthy); health.last_check = now; debug!("Health check completed: {:?}", health.overall_status); } }) } /// Detect model version from model name or metadata fn detect_model_version(model_name: &str) -> String { // Extract version from model name if present (e.g., "llama-2-7b-v1.2") if let Some(version_pos) = model_name.rfind("-v") && let Some(version_part) = model_name.get(version_pos + 2..) && version_part.chars().all(|c| c.is_ascii_digit() || c == '.') { return version_part.to_string(); } // Check for common version patterns if model_name.contains("gpt-4") { return "4.0".to_string(); } else if model_name.contains("gpt-3.5") { return "3.5".to_string(); } else if model_name.contains("claude-3") { return "3.0".to_string(); } else if model_name.contains("llama-2") { return "2.0".to_string(); } else if model_name.contains("llama-3") { return "3.0".to_string(); } // Default version "1.0".to_string() } /// Extract temperature parameter from request result fn extract_temperature_from_result(result: &RequestResult) -> Option { // Look for temperature in generation config or request metadata // For now, return a default temperature if generation was successful if !matches!(result.finish_reason, FinishReason::Error) && !result.output_tokens.is_empty() { Some(0.7) // Default temperature } else { None } } /// Extract `top_p` parameter from request result fn extract_top_p_from_result(result: &RequestResult) -> Option { // Look for top_p in generation config or request metadata if !matches!(result.finish_reason, FinishReason::Error) && !result.output_tokens.is_empty() { Some(0.9) // Default top_p } else { None } } /// Extract `top_k` parameter from request result fn extract_top_k_from_result(result: &RequestResult) -> Option { // Look for top_k in generation config or request metadata if !matches!(result.finish_reason, FinishReason::Error) && !result.output_tokens.is_empty() { Some(50) // Default top_k } else { None } } } impl Default for PerformanceMetrics { fn default() -> Self { Self { total_requests: 0, successful_requests: 0, failed_requests: 0, requests_per_second: 0.0, avg_latency_ms: 0.0, p95_latency_ms: 0.0, p99_latency_ms: 0.0, tokens_generated: 0, tokens_per_second: 0.0, success_rate: 0.0, avg_batch_size: 0.0, total_queue_depth: 0, updated_at: SystemTime::now(), } } } impl Default for ResourceMetrics { fn default() -> Self { Self { gpu_memory_utilization: 0.0, cpu_utilization: 0.0, memory_usage_gb: 0.0, memory_fragmentation: 0.0, active_connections: 0, cache_hit_rate: 0.0, disk_io_mbps: 0.0, network_throughput_mbps: 0.0, gpu_temperature_c: None, power_consumption_w: None, } } }