Files
rustytorch/crates/production/rtx-inference/src/monitoring.rs
T
2026-03-04 00:08:42 +00:00

786 lines
27 KiB
Rust

//! Comprehensive Monitoring and Metrics Collection
//!
//! This module provides detailed monitoring capabilities for the inference engine including:
//! - Real-time performance metrics
//! - Resource utilization tracking
//! - Request tracing and latency analysis
//! - Model-specific metrics
//! - Health checks and alerting
//! - Prometheus-compatible metrics export
use crate::{FinishReason, InferenceResult, RequestMetrics, RequestResult};
use serde::{Deserialize, Serialize};
use std::collections::{HashMap, VecDeque};
use std::sync::Arc;
use std::time::{Duration, SystemTime};
use tokio::sync::{Mutex, RwLock};
use tracing::{debug, info, instrument};
use uuid::Uuid;
/// Configuration for monitoring and metrics collection
#[derive(Debug, Clone)]
pub struct MonitoringConfig {
/// Enable detailed request tracing
pub enable_tracing: bool,
/// Maximum number of request traces to keep in memory
pub max_traces: usize,
/// Metrics aggregation window size
pub aggregation_window: Duration,
/// Enable real-time health checks
pub enable_health_checks: bool,
/// Health check interval
pub health_check_interval: Duration,
/// Enable Prometheus metrics export
pub enable_prometheus: bool,
/// Custom metrics retention period
pub metrics_retention: Duration,
}
impl Default for MonitoringConfig {
fn default() -> Self {
Self {
enable_tracing: true,
max_traces: 10000,
aggregation_window: Duration::from_secs(60),
enable_health_checks: true,
health_check_interval: Duration::from_secs(10),
enable_prometheus: true,
metrics_retention: Duration::from_secs(3600), // 1 hour
}
}
}
/// Comprehensive metrics collector for the inference engine
pub struct MetricsCollector {
config: MonitoringConfig,
/// Real-time performance metrics
performance_metrics: Arc<RwLock<PerformanceMetrics>>,
/// Resource utilization metrics
resource_metrics: Arc<RwLock<ResourceMetrics>>,
/// Model-specific metrics
model_metrics: Arc<RwLock<HashMap<String, ModelSpecificMetrics>>>,
/// Request tracing data
request_traces: Arc<Mutex<VecDeque<RequestTrace>>>,
/// Health status
health_status: Arc<RwLock<HealthStatus>>,
/// Time-series metrics for trending
time_series_metrics: Arc<Mutex<TimeSeriesMetrics>>,
/// Background tasks
_metrics_task: tokio::task::JoinHandle<()>,
_health_check_task: tokio::task::JoinHandle<()>,
}
/// Real-time performance metrics
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct PerformanceMetrics {
/// Total number of requests processed
pub total_requests: u64,
/// Successful requests
pub successful_requests: u64,
/// Failed requests
pub failed_requests: u64,
/// Current requests per second
pub requests_per_second: f64,
/// Average latency (p50)
pub avg_latency_ms: f64,
/// P95 latency
pub p95_latency_ms: f64,
/// P99 latency
pub p99_latency_ms: f64,
/// Total tokens generated
pub tokens_generated: u64,
/// Tokens per second
pub tokens_per_second: f64,
/// Success rate percentage
pub success_rate: f64,
/// Average batch size
pub avg_batch_size: f64,
/// Queue depth across all SLA lanes
pub total_queue_depth: usize,
/// Last updated timestamp
pub updated_at: SystemTime,
}
/// Resource utilization metrics
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct ResourceMetrics {
/// GPU memory usage percentage
pub gpu_memory_utilization: f64,
/// CPU usage percentage
pub cpu_utilization: f64,
/// System memory usage
pub memory_usage_gb: f64,
/// Memory fragmentation ratio
pub memory_fragmentation: f64,
/// Active connections
pub active_connections: u32,
/// KV cache hit rate
pub cache_hit_rate: f64,
/// Disk I/O for model loading
pub disk_io_mbps: f64,
/// Network throughput
pub network_throughput_mbps: f64,
/// Temperature metrics (if available)
pub gpu_temperature_c: Option<f32>,
/// Power consumption (if available)
pub power_consumption_w: Option<f32>,
}
/// Model-specific performance metrics
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct ModelSpecificMetrics {
pub model_name: String,
pub model_version: String,
pub requests_processed: u64,
pub total_inference_time: Duration,
pub avg_tokens_per_request: f64,
pub avg_latency_ms: f64,
pub memory_usage_mb: f64,
pub optimization_speedup: f64,
pub quantization_speedup: Option<f64>,
pub last_used: SystemTime,
pub load_time: Duration,
}
/// Individual request trace for debugging and analysis
#[derive(Debug, Clone, Serialize)]
pub struct RequestTrace {
pub request_id: Uuid,
pub model_name: String,
pub timestamp: SystemTime,
pub input_tokens: usize,
pub output_tokens: usize,
pub total_latency_ms: f64,
pub queue_time_ms: f64,
pub processing_time_ms: f64,
pub generation_time_ms: f64,
pub memory_usage_mb: f64,
pub sla_lane: String,
pub temperature: Option<f32>,
pub top_p: Option<f32>,
pub top_k: Option<u32>,
pub finish_reason: String,
pub error: Option<String>,
}
/// Health status of the inference engine
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct HealthStatus {
pub overall_status: HealthLevel,
pub model_health: HashMap<String, HealthLevel>,
pub resource_health: ResourceHealth,
pub last_check: SystemTime,
pub alerts: Vec<Alert>,
}
/// Health levels
#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize)]
pub enum HealthLevel {
Healthy,
Warning,
Critical,
Down,
}
/// Resource health status
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct ResourceHealth {
pub memory_status: HealthLevel,
pub gpu_status: HealthLevel,
pub cpu_status: HealthLevel,
pub disk_status: HealthLevel,
pub network_status: HealthLevel,
}
/// Alert information
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct Alert {
pub id: Uuid,
pub level: HealthLevel,
pub message: String,
pub component: String,
pub timestamp: SystemTime,
pub resolved: bool,
}
/// Time-series metrics for trending analysis
#[derive(Debug)]
pub struct TimeSeriesMetrics {
/// Latency samples over time (timestamp, `latency_ms`)
pub latency_samples: VecDeque<(SystemTime, f64)>,
/// Throughput samples over time (timestamp, rps)
pub throughput_samples: VecDeque<(SystemTime, f64)>,
/// Memory usage samples over time (timestamp, `usage_pct`)
pub memory_samples: VecDeque<(SystemTime, f64)>,
/// Error rate samples over time (timestamp, `error_rate`)
pub error_samples: VecDeque<(SystemTime, f64)>,
}
impl MetricsCollector {
/// Create a new metrics collector
pub async fn new(config: MonitoringConfig) -> InferenceResult<Self> {
let performance_metrics = Arc::new(RwLock::new(PerformanceMetrics {
total_requests: 0,
successful_requests: 0,
failed_requests: 0,
requests_per_second: 0.0,
avg_latency_ms: 0.0,
p95_latency_ms: 0.0,
p99_latency_ms: 0.0,
tokens_generated: 0,
tokens_per_second: 0.0,
success_rate: 0.0,
avg_batch_size: 0.0,
total_queue_depth: 0,
updated_at: SystemTime::now(),
}));
let resource_metrics = Arc::new(RwLock::new(ResourceMetrics {
gpu_memory_utilization: 0.0,
cpu_utilization: 0.0,
memory_usage_gb: 0.0,
memory_fragmentation: 0.0,
active_connections: 0,
cache_hit_rate: 0.0,
disk_io_mbps: 0.0,
network_throughput_mbps: 0.0,
gpu_temperature_c: None,
power_consumption_w: None,
}));
let model_metrics = Arc::new(RwLock::new(HashMap::new()));
let request_traces = Arc::new(Mutex::new(VecDeque::new()));
let health_status = Arc::new(RwLock::new(HealthStatus {
overall_status: HealthLevel::Healthy,
model_health: HashMap::new(),
resource_health: ResourceHealth {
memory_status: HealthLevel::Healthy,
gpu_status: HealthLevel::Healthy,
cpu_status: HealthLevel::Healthy,
disk_status: HealthLevel::Healthy,
network_status: HealthLevel::Healthy,
},
last_check: SystemTime::now(),
alerts: Vec::new(),
}));
let time_series_metrics = Arc::new(Mutex::new(TimeSeriesMetrics {
latency_samples: VecDeque::new(),
throughput_samples: VecDeque::new(),
memory_samples: VecDeque::new(),
error_samples: VecDeque::new(),
}));
// Start background tasks
let metrics_task = Self::start_metrics_aggregation_task(
Arc::clone(&performance_metrics),
Arc::clone(&time_series_metrics),
config.aggregation_window,
config.metrics_retention,
);
let health_check_task = Self::start_health_check_task(
Arc::clone(&health_status),
Arc::clone(&resource_metrics),
Arc::clone(&model_metrics),
config.health_check_interval,
);
info!("Metrics collector initialized with config: {:?}", config);
Ok(Self {
config,
performance_metrics,
resource_metrics,
model_metrics,
request_traces,
health_status,
time_series_metrics,
_metrics_task: metrics_task,
_health_check_task: health_check_task,
})
}
/// Record a completed inference request
#[instrument(skip(self, result, metrics))]
pub async fn record_request(
&self,
model_name: &str,
result: &RequestResult,
metrics: &RequestMetrics,
sla_lane: &str,
error: Option<&str>,
) -> InferenceResult<()> {
let timestamp = SystemTime::now();
// Update performance metrics
{
let mut perf_metrics = self.performance_metrics.write().await;
perf_metrics.total_requests += 1;
if error.is_none() {
perf_metrics.successful_requests += 1;
perf_metrics.tokens_generated += metrics.output_token_count as u64;
} else {
perf_metrics.failed_requests += 1;
}
perf_metrics.success_rate =
perf_metrics.successful_requests as f64 / perf_metrics.total_requests as f64;
perf_metrics.updated_at = timestamp;
}
// Update model-specific metrics
{
let mut model_metrics = self.model_metrics.write().await;
let model_metric = model_metrics
.entry(model_name.to_string())
.or_insert_with(|| ModelSpecificMetrics {
model_name: model_name.to_string(),
model_version: Self::detect_model_version(model_name),
requests_processed: 0,
total_inference_time: Duration::ZERO,
avg_tokens_per_request: 0.0,
avg_latency_ms: 0.0,
memory_usage_mb: 0.0,
optimization_speedup: 1.0,
quantization_speedup: None,
last_used: timestamp,
load_time: Duration::ZERO,
});
model_metric.requests_processed += 1;
model_metric.total_inference_time += metrics.total_time;
model_metric.avg_latency_ms = model_metric.total_inference_time.as_millis() as f64
/ model_metric.requests_processed as f64;
model_metric.avg_tokens_per_request = (model_metric.avg_tokens_per_request
* (model_metric.requests_processed - 1) as f64
+ metrics.output_token_count as f64)
/ model_metric.requests_processed as f64;
model_metric.last_used = timestamp;
}
// Add request trace if tracing is enabled
if self.config.enable_tracing {
let mut traces = self.request_traces.lock().await;
let trace = RequestTrace {
request_id: result.request_id.into(),
model_name: model_name.to_string(),
timestamp,
input_tokens: metrics.input_token_count,
output_tokens: metrics.output_token_count,
total_latency_ms: metrics.total_time.as_millis() as f64,
queue_time_ms: metrics.queue_time.as_millis() as f64,
processing_time_ms: metrics.processing_time.as_millis() as f64,
generation_time_ms: metrics.generation_time.as_millis() as f64,
memory_usage_mb: metrics.peak_memory_bytes as f64 / 1024.0 / 1024.0,
sla_lane: sla_lane.to_string(),
temperature: Self::extract_temperature_from_result(result),
top_p: Self::extract_top_p_from_result(result),
top_k: Self::extract_top_k_from_result(result),
finish_reason: format!("{:?}", result.finish_reason),
error: error.map(std::string::ToString::to_string),
};
traces.push_back(trace);
// Maintain max traces limit
while traces.len() > self.config.max_traces {
traces.pop_front();
}
}
// Update time-series data
{
let mut ts_metrics = self.time_series_metrics.lock().await;
ts_metrics
.latency_samples
.push_back((timestamp, metrics.total_time.as_millis() as f64));
// Clean old samples
let cutoff = timestamp - self.config.metrics_retention;
while let Some((ts, _)) = ts_metrics.latency_samples.front() {
if *ts < cutoff {
ts_metrics.latency_samples.pop_front();
} else {
break;
}
}
}
Ok(())
}
/// Get current performance metrics
pub async fn get_performance_metrics(&self) -> PerformanceMetrics {
self.performance_metrics.read().await.clone()
}
/// Get current resource metrics
pub async fn get_resource_metrics(&self) -> ResourceMetrics {
self.resource_metrics.read().await.clone()
}
/// Get model-specific metrics
pub async fn get_model_metrics(&self) -> HashMap<String, ModelSpecificMetrics> {
self.model_metrics.read().await.clone()
}
/// Get recent request traces
pub async fn get_recent_traces(&self, limit: usize) -> Vec<RequestTrace> {
let traces = self.request_traces.lock().await;
traces.iter().rev().take(limit).cloned().collect()
}
/// Get current health status
pub async fn get_health_status(&self) -> HealthStatus {
self.health_status.read().await.clone()
}
/// Export metrics in Prometheus format
pub async fn export_prometheus_metrics(&self) -> String {
let perf_metrics = self.get_performance_metrics().await;
let resource_metrics = self.get_resource_metrics().await;
let model_metrics = self.get_model_metrics().await;
let mut output = String::new();
// Performance metrics
output.push_str(&format!(
"rtx_inference_requests_total {}\n",
perf_metrics.total_requests
));
output.push_str(&format!(
"rtx_inference_requests_successful {}\n",
perf_metrics.successful_requests
));
output.push_str(&format!(
"rtx_inference_requests_failed {}\n",
perf_metrics.failed_requests
));
output.push_str(&format!(
"rtx_inference_requests_per_second {}\n",
perf_metrics.requests_per_second
));
output.push_str(&format!(
"rtx_inference_latency_avg_ms {}\n",
perf_metrics.avg_latency_ms
));
output.push_str(&format!(
"rtx_inference_latency_p95_ms {}\n",
perf_metrics.p95_latency_ms
));
output.push_str(&format!(
"rtx_inference_latency_p99_ms {}\n",
perf_metrics.p99_latency_ms
));
output.push_str(&format!(
"rtx_inference_tokens_generated {}\n",
perf_metrics.tokens_generated
));
output.push_str(&format!(
"rtx_inference_tokens_per_second {}\n",
perf_metrics.tokens_per_second
));
output.push_str(&format!(
"rtx_inference_success_rate {}\n",
perf_metrics.success_rate
));
// Resource metrics
output.push_str(&format!(
"rtx_inference_gpu_memory_utilization {}\n",
resource_metrics.gpu_memory_utilization
));
output.push_str(&format!(
"rtx_inference_cpu_utilization {}\n",
resource_metrics.cpu_utilization
));
output.push_str(&format!(
"rtx_inference_memory_usage_gb {}\n",
resource_metrics.memory_usage_gb
));
output.push_str(&format!(
"rtx_inference_cache_hit_rate {}\n",
resource_metrics.cache_hit_rate
));
// Model-specific metrics
for (model_name, metrics) in model_metrics {
let label = format!("model=\"{model_name}\"");
output.push_str(&format!(
"rtx_inference_model_requests_processed{{{}}} {}\n",
label, metrics.requests_processed
));
output.push_str(&format!(
"rtx_inference_model_avg_latency_ms{{{}}} {}\n",
label, metrics.avg_latency_ms
));
output.push_str(&format!(
"rtx_inference_model_memory_usage_mb{{{}}} {}\n",
label, metrics.memory_usage_mb
));
}
output
}
/// Update resource utilization metrics (called by external monitoring)
pub async fn update_resource_metrics(
&self,
gpu_memory_pct: f64,
cpu_pct: f64,
memory_gb: f64,
fragmentation: f64,
cache_hit_rate: f64,
) {
let mut metrics = self.resource_metrics.write().await;
metrics.gpu_memory_utilization = gpu_memory_pct;
metrics.cpu_utilization = cpu_pct;
metrics.memory_usage_gb = memory_gb;
metrics.memory_fragmentation = fragmentation;
metrics.cache_hit_rate = cache_hit_rate;
}
/// Start background task for metrics aggregation
fn start_metrics_aggregation_task(
performance_metrics: Arc<RwLock<PerformanceMetrics>>,
time_series_metrics: Arc<Mutex<TimeSeriesMetrics>>,
window: Duration,
_retention: Duration,
) -> tokio::task::JoinHandle<()> {
tokio::spawn(async move {
let mut interval = tokio::time::interval(window);
loop {
interval.tick().await;
let now = SystemTime::now();
// Aggregate metrics from time series data
{
let ts_metrics = time_series_metrics.lock().await;
let mut perf_metrics = performance_metrics.write().await;
// Calculate percentiles and rates from recent samples
if !ts_metrics.latency_samples.is_empty() {
let recent_cutoff = now - window;
let recent_latencies: Vec<f64> = ts_metrics
.latency_samples
.iter()
.filter(|(ts, _)| *ts >= recent_cutoff)
.map(|(_, latency)| *latency)
.collect();
if !recent_latencies.is_empty() {
perf_metrics.avg_latency_ms = recent_latencies.iter().sum::<f64>()
/ recent_latencies.len() as f64;
// Calculate percentiles
let mut sorted_latencies = recent_latencies;
sorted_latencies.sort_by(f64::total_cmp);
let p95_idx = (sorted_latencies.len() as f64 * 0.95) as usize;
let p99_idx = (sorted_latencies.len() as f64 * 0.99) as usize;
perf_metrics.p95_latency_ms =
sorted_latencies.get(p95_idx).copied().unwrap_or(0.0);
perf_metrics.p99_latency_ms =
sorted_latencies.get(p99_idx).copied().unwrap_or(0.0);
}
// Calculate requests per second
let recent_request_count = ts_metrics
.latency_samples
.iter()
.filter(|(ts, _)| *ts >= recent_cutoff)
.count();
perf_metrics.requests_per_second =
recent_request_count as f64 / window.as_secs_f64();
}
perf_metrics.updated_at = now;
}
debug!("Metrics aggregated at {:?}", now);
}
})
}
/// Start background health check task
fn start_health_check_task(
health_status: Arc<RwLock<HealthStatus>>,
resource_metrics: Arc<RwLock<ResourceMetrics>>,
model_metrics: Arc<RwLock<HashMap<String, ModelSpecificMetrics>>>,
interval: Duration,
) -> tokio::task::JoinHandle<()> {
tokio::spawn(async move {
let mut ticker = tokio::time::interval(interval);
loop {
ticker.tick().await;
let now = SystemTime::now();
let mut health = health_status.write().await;
// Check resource health
let resource_metrics_read = resource_metrics.read().await;
health.resource_health.memory_status =
if resource_metrics_read.gpu_memory_utilization > 0.9 {
HealthLevel::Critical
} else if resource_metrics_read.gpu_memory_utilization > 0.8 {
HealthLevel::Warning
} else {
HealthLevel::Healthy
};
health.resource_health.cpu_status = if resource_metrics_read.cpu_utilization > 0.9 {
HealthLevel::Critical
} else if resource_metrics_read.cpu_utilization > 0.8 {
HealthLevel::Warning
} else {
HealthLevel::Healthy
};
drop(resource_metrics_read);
// Check model health
let model_metrics_read = model_metrics.read().await;
for (model_name, metrics) in model_metrics_read.iter() {
let time_since_last_use = now
.duration_since(metrics.last_used)
.unwrap_or(Duration::ZERO);
let model_health = if time_since_last_use > Duration::from_secs(3600) {
// 1 hour
HealthLevel::Warning
} else {
HealthLevel::Healthy
};
health.model_health.insert(model_name.clone(), model_health);
}
drop(model_metrics_read);
// Determine overall health
health.overall_status = [
health.resource_health.memory_status,
health.resource_health.cpu_status,
health.resource_health.gpu_status,
]
.iter()
.chain(health.model_health.values())
.max()
.copied()
.unwrap_or(HealthLevel::Healthy);
health.last_check = now;
debug!("Health check completed: {:?}", health.overall_status);
}
})
}
/// Detect model version from model name or metadata
fn detect_model_version(model_name: &str) -> String {
// Extract version from model name if present (e.g., "llama-2-7b-v1.2")
if let Some(version_pos) = model_name.rfind("-v")
&& let Some(version_part) = model_name.get(version_pos + 2..)
&& version_part.chars().all(|c| c.is_ascii_digit() || c == '.')
{
return version_part.to_string();
}
// Check for common version patterns
if model_name.contains("gpt-4") {
return "4.0".to_string();
} else if model_name.contains("gpt-3.5") {
return "3.5".to_string();
} else if model_name.contains("claude-3") {
return "3.0".to_string();
} else if model_name.contains("llama-2") {
return "2.0".to_string();
} else if model_name.contains("llama-3") {
return "3.0".to_string();
}
// Default version
"1.0".to_string()
}
/// Extract temperature parameter from request result
fn extract_temperature_from_result(result: &RequestResult) -> Option<f32> {
// Look for temperature in generation config or request metadata
// For now, return a default temperature if generation was successful
if !matches!(result.finish_reason, FinishReason::Error) && !result.output_tokens.is_empty()
{
Some(0.7) // Default temperature
} else {
None
}
}
/// Extract `top_p` parameter from request result
fn extract_top_p_from_result(result: &RequestResult) -> Option<f32> {
// Look for top_p in generation config or request metadata
if !matches!(result.finish_reason, FinishReason::Error) && !result.output_tokens.is_empty()
{
Some(0.9) // Default top_p
} else {
None
}
}
/// Extract `top_k` parameter from request result
fn extract_top_k_from_result(result: &RequestResult) -> Option<u32> {
// Look for top_k in generation config or request metadata
if !matches!(result.finish_reason, FinishReason::Error) && !result.output_tokens.is_empty()
{
Some(50) // Default top_k
} else {
None
}
}
}
impl Default for PerformanceMetrics {
fn default() -> Self {
Self {
total_requests: 0,
successful_requests: 0,
failed_requests: 0,
requests_per_second: 0.0,
avg_latency_ms: 0.0,
p95_latency_ms: 0.0,
p99_latency_ms: 0.0,
tokens_generated: 0,
tokens_per_second: 0.0,
success_rate: 0.0,
avg_batch_size: 0.0,
total_queue_depth: 0,
updated_at: SystemTime::now(),
}
}
}
impl Default for ResourceMetrics {
fn default() -> Self {
Self {
gpu_memory_utilization: 0.0,
cpu_utilization: 0.0,
memory_usage_gb: 0.0,
memory_fragmentation: 0.0,
active_connections: 0,
cache_hit_rate: 0.0,
disk_io_mbps: 0.0,
network_throughput_mbps: 0.0,
gpu_temperature_c: None,
power_consumption_w: None,
}
}
}