786 lines
27 KiB
Rust
786 lines
27 KiB
Rust
//! Comprehensive Monitoring and Metrics Collection
|
|
//!
|
|
//! This module provides detailed monitoring capabilities for the inference engine including:
|
|
//! - Real-time performance metrics
|
|
//! - Resource utilization tracking
|
|
//! - Request tracing and latency analysis
|
|
//! - Model-specific metrics
|
|
//! - Health checks and alerting
|
|
//! - Prometheus-compatible metrics export
|
|
|
|
use crate::{FinishReason, InferenceResult, RequestMetrics, RequestResult};
|
|
use serde::{Deserialize, Serialize};
|
|
use std::collections::{HashMap, VecDeque};
|
|
use std::sync::Arc;
|
|
use std::time::{Duration, SystemTime};
|
|
use tokio::sync::{Mutex, RwLock};
|
|
use tracing::{debug, info, instrument};
|
|
use uuid::Uuid;
|
|
|
|
/// Configuration for monitoring and metrics collection
|
|
#[derive(Debug, Clone)]
|
|
pub struct MonitoringConfig {
|
|
/// Enable detailed request tracing
|
|
pub enable_tracing: bool,
|
|
/// Maximum number of request traces to keep in memory
|
|
pub max_traces: usize,
|
|
/// Metrics aggregation window size
|
|
pub aggregation_window: Duration,
|
|
/// Enable real-time health checks
|
|
pub enable_health_checks: bool,
|
|
/// Health check interval
|
|
pub health_check_interval: Duration,
|
|
/// Enable Prometheus metrics export
|
|
pub enable_prometheus: bool,
|
|
/// Custom metrics retention period
|
|
pub metrics_retention: Duration,
|
|
}
|
|
|
|
impl Default for MonitoringConfig {
|
|
fn default() -> Self {
|
|
Self {
|
|
enable_tracing: true,
|
|
max_traces: 10000,
|
|
aggregation_window: Duration::from_secs(60),
|
|
enable_health_checks: true,
|
|
health_check_interval: Duration::from_secs(10),
|
|
enable_prometheus: true,
|
|
metrics_retention: Duration::from_secs(3600), // 1 hour
|
|
}
|
|
}
|
|
}
|
|
|
|
/// Comprehensive metrics collector for the inference engine
|
|
pub struct MetricsCollector {
|
|
config: MonitoringConfig,
|
|
|
|
/// Real-time performance metrics
|
|
performance_metrics: Arc<RwLock<PerformanceMetrics>>,
|
|
|
|
/// Resource utilization metrics
|
|
resource_metrics: Arc<RwLock<ResourceMetrics>>,
|
|
|
|
/// Model-specific metrics
|
|
model_metrics: Arc<RwLock<HashMap<String, ModelSpecificMetrics>>>,
|
|
|
|
/// Request tracing data
|
|
request_traces: Arc<Mutex<VecDeque<RequestTrace>>>,
|
|
|
|
/// Health status
|
|
health_status: Arc<RwLock<HealthStatus>>,
|
|
|
|
/// Time-series metrics for trending
|
|
time_series_metrics: Arc<Mutex<TimeSeriesMetrics>>,
|
|
|
|
/// Background tasks
|
|
_metrics_task: tokio::task::JoinHandle<()>,
|
|
_health_check_task: tokio::task::JoinHandle<()>,
|
|
}
|
|
|
|
/// Real-time performance metrics
|
|
#[derive(Debug, Clone, Serialize, Deserialize)]
|
|
pub struct PerformanceMetrics {
|
|
/// Total number of requests processed
|
|
pub total_requests: u64,
|
|
/// Successful requests
|
|
pub successful_requests: u64,
|
|
/// Failed requests
|
|
pub failed_requests: u64,
|
|
/// Current requests per second
|
|
pub requests_per_second: f64,
|
|
/// Average latency (p50)
|
|
pub avg_latency_ms: f64,
|
|
/// P95 latency
|
|
pub p95_latency_ms: f64,
|
|
/// P99 latency
|
|
pub p99_latency_ms: f64,
|
|
/// Total tokens generated
|
|
pub tokens_generated: u64,
|
|
/// Tokens per second
|
|
pub tokens_per_second: f64,
|
|
/// Success rate percentage
|
|
pub success_rate: f64,
|
|
/// Average batch size
|
|
pub avg_batch_size: f64,
|
|
/// Queue depth across all SLA lanes
|
|
pub total_queue_depth: usize,
|
|
/// Last updated timestamp
|
|
pub updated_at: SystemTime,
|
|
}
|
|
|
|
/// Resource utilization metrics
|
|
#[derive(Debug, Clone, Serialize, Deserialize)]
|
|
pub struct ResourceMetrics {
|
|
/// GPU memory usage percentage
|
|
pub gpu_memory_utilization: f64,
|
|
/// CPU usage percentage
|
|
pub cpu_utilization: f64,
|
|
/// System memory usage
|
|
pub memory_usage_gb: f64,
|
|
/// Memory fragmentation ratio
|
|
pub memory_fragmentation: f64,
|
|
/// Active connections
|
|
pub active_connections: u32,
|
|
/// KV cache hit rate
|
|
pub cache_hit_rate: f64,
|
|
/// Disk I/O for model loading
|
|
pub disk_io_mbps: f64,
|
|
/// Network throughput
|
|
pub network_throughput_mbps: f64,
|
|
/// Temperature metrics (if available)
|
|
pub gpu_temperature_c: Option<f32>,
|
|
/// Power consumption (if available)
|
|
pub power_consumption_w: Option<f32>,
|
|
}
|
|
|
|
/// Model-specific performance metrics
|
|
#[derive(Debug, Clone, Serialize, Deserialize)]
|
|
pub struct ModelSpecificMetrics {
|
|
pub model_name: String,
|
|
pub model_version: String,
|
|
pub requests_processed: u64,
|
|
pub total_inference_time: Duration,
|
|
pub avg_tokens_per_request: f64,
|
|
pub avg_latency_ms: f64,
|
|
pub memory_usage_mb: f64,
|
|
pub optimization_speedup: f64,
|
|
pub quantization_speedup: Option<f64>,
|
|
pub last_used: SystemTime,
|
|
pub load_time: Duration,
|
|
}
|
|
|
|
/// Individual request trace for debugging and analysis
|
|
#[derive(Debug, Clone, Serialize)]
|
|
pub struct RequestTrace {
|
|
pub request_id: Uuid,
|
|
pub model_name: String,
|
|
pub timestamp: SystemTime,
|
|
pub input_tokens: usize,
|
|
pub output_tokens: usize,
|
|
pub total_latency_ms: f64,
|
|
pub queue_time_ms: f64,
|
|
pub processing_time_ms: f64,
|
|
pub generation_time_ms: f64,
|
|
pub memory_usage_mb: f64,
|
|
pub sla_lane: String,
|
|
pub temperature: Option<f32>,
|
|
pub top_p: Option<f32>,
|
|
pub top_k: Option<u32>,
|
|
pub finish_reason: String,
|
|
pub error: Option<String>,
|
|
}
|
|
|
|
/// Health status of the inference engine
|
|
#[derive(Debug, Clone, Serialize, Deserialize)]
|
|
pub struct HealthStatus {
|
|
pub overall_status: HealthLevel,
|
|
pub model_health: HashMap<String, HealthLevel>,
|
|
pub resource_health: ResourceHealth,
|
|
pub last_check: SystemTime,
|
|
pub alerts: Vec<Alert>,
|
|
}
|
|
|
|
/// Health levels
|
|
#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize)]
|
|
pub enum HealthLevel {
|
|
Healthy,
|
|
Warning,
|
|
Critical,
|
|
Down,
|
|
}
|
|
|
|
/// Resource health status
|
|
#[derive(Debug, Clone, Serialize, Deserialize)]
|
|
pub struct ResourceHealth {
|
|
pub memory_status: HealthLevel,
|
|
pub gpu_status: HealthLevel,
|
|
pub cpu_status: HealthLevel,
|
|
pub disk_status: HealthLevel,
|
|
pub network_status: HealthLevel,
|
|
}
|
|
|
|
/// Alert information
|
|
#[derive(Debug, Clone, Serialize, Deserialize)]
|
|
pub struct Alert {
|
|
pub id: Uuid,
|
|
pub level: HealthLevel,
|
|
pub message: String,
|
|
pub component: String,
|
|
pub timestamp: SystemTime,
|
|
pub resolved: bool,
|
|
}
|
|
|
|
/// Time-series metrics for trending analysis
|
|
#[derive(Debug)]
|
|
pub struct TimeSeriesMetrics {
|
|
/// Latency samples over time (timestamp, `latency_ms`)
|
|
pub latency_samples: VecDeque<(SystemTime, f64)>,
|
|
/// Throughput samples over time (timestamp, rps)
|
|
pub throughput_samples: VecDeque<(SystemTime, f64)>,
|
|
/// Memory usage samples over time (timestamp, `usage_pct`)
|
|
pub memory_samples: VecDeque<(SystemTime, f64)>,
|
|
/// Error rate samples over time (timestamp, `error_rate`)
|
|
pub error_samples: VecDeque<(SystemTime, f64)>,
|
|
}
|
|
|
|
impl MetricsCollector {
|
|
/// Create a new metrics collector
|
|
pub async fn new(config: MonitoringConfig) -> InferenceResult<Self> {
|
|
let performance_metrics = Arc::new(RwLock::new(PerformanceMetrics {
|
|
total_requests: 0,
|
|
successful_requests: 0,
|
|
failed_requests: 0,
|
|
requests_per_second: 0.0,
|
|
avg_latency_ms: 0.0,
|
|
p95_latency_ms: 0.0,
|
|
p99_latency_ms: 0.0,
|
|
tokens_generated: 0,
|
|
tokens_per_second: 0.0,
|
|
success_rate: 0.0,
|
|
avg_batch_size: 0.0,
|
|
total_queue_depth: 0,
|
|
updated_at: SystemTime::now(),
|
|
}));
|
|
|
|
let resource_metrics = Arc::new(RwLock::new(ResourceMetrics {
|
|
gpu_memory_utilization: 0.0,
|
|
cpu_utilization: 0.0,
|
|
memory_usage_gb: 0.0,
|
|
memory_fragmentation: 0.0,
|
|
active_connections: 0,
|
|
cache_hit_rate: 0.0,
|
|
disk_io_mbps: 0.0,
|
|
network_throughput_mbps: 0.0,
|
|
gpu_temperature_c: None,
|
|
power_consumption_w: None,
|
|
}));
|
|
|
|
let model_metrics = Arc::new(RwLock::new(HashMap::new()));
|
|
let request_traces = Arc::new(Mutex::new(VecDeque::new()));
|
|
let health_status = Arc::new(RwLock::new(HealthStatus {
|
|
overall_status: HealthLevel::Healthy,
|
|
model_health: HashMap::new(),
|
|
resource_health: ResourceHealth {
|
|
memory_status: HealthLevel::Healthy,
|
|
gpu_status: HealthLevel::Healthy,
|
|
cpu_status: HealthLevel::Healthy,
|
|
disk_status: HealthLevel::Healthy,
|
|
network_status: HealthLevel::Healthy,
|
|
},
|
|
last_check: SystemTime::now(),
|
|
alerts: Vec::new(),
|
|
}));
|
|
let time_series_metrics = Arc::new(Mutex::new(TimeSeriesMetrics {
|
|
latency_samples: VecDeque::new(),
|
|
throughput_samples: VecDeque::new(),
|
|
memory_samples: VecDeque::new(),
|
|
error_samples: VecDeque::new(),
|
|
}));
|
|
|
|
// Start background tasks
|
|
let metrics_task = Self::start_metrics_aggregation_task(
|
|
Arc::clone(&performance_metrics),
|
|
Arc::clone(&time_series_metrics),
|
|
config.aggregation_window,
|
|
config.metrics_retention,
|
|
);
|
|
|
|
let health_check_task = Self::start_health_check_task(
|
|
Arc::clone(&health_status),
|
|
Arc::clone(&resource_metrics),
|
|
Arc::clone(&model_metrics),
|
|
config.health_check_interval,
|
|
);
|
|
|
|
info!("Metrics collector initialized with config: {:?}", config);
|
|
|
|
Ok(Self {
|
|
config,
|
|
performance_metrics,
|
|
resource_metrics,
|
|
model_metrics,
|
|
request_traces,
|
|
health_status,
|
|
time_series_metrics,
|
|
_metrics_task: metrics_task,
|
|
_health_check_task: health_check_task,
|
|
})
|
|
}
|
|
|
|
/// Record a completed inference request
|
|
#[instrument(skip(self, result, metrics))]
|
|
pub async fn record_request(
|
|
&self,
|
|
model_name: &str,
|
|
result: &RequestResult,
|
|
metrics: &RequestMetrics,
|
|
sla_lane: &str,
|
|
error: Option<&str>,
|
|
) -> InferenceResult<()> {
|
|
let timestamp = SystemTime::now();
|
|
|
|
// Update performance metrics
|
|
{
|
|
let mut perf_metrics = self.performance_metrics.write().await;
|
|
perf_metrics.total_requests += 1;
|
|
|
|
if error.is_none() {
|
|
perf_metrics.successful_requests += 1;
|
|
perf_metrics.tokens_generated += metrics.output_token_count as u64;
|
|
} else {
|
|
perf_metrics.failed_requests += 1;
|
|
}
|
|
|
|
perf_metrics.success_rate =
|
|
perf_metrics.successful_requests as f64 / perf_metrics.total_requests as f64;
|
|
perf_metrics.updated_at = timestamp;
|
|
}
|
|
|
|
// Update model-specific metrics
|
|
{
|
|
let mut model_metrics = self.model_metrics.write().await;
|
|
let model_metric = model_metrics
|
|
.entry(model_name.to_string())
|
|
.or_insert_with(|| ModelSpecificMetrics {
|
|
model_name: model_name.to_string(),
|
|
model_version: Self::detect_model_version(model_name),
|
|
requests_processed: 0,
|
|
total_inference_time: Duration::ZERO,
|
|
avg_tokens_per_request: 0.0,
|
|
avg_latency_ms: 0.0,
|
|
memory_usage_mb: 0.0,
|
|
optimization_speedup: 1.0,
|
|
quantization_speedup: None,
|
|
last_used: timestamp,
|
|
load_time: Duration::ZERO,
|
|
});
|
|
|
|
model_metric.requests_processed += 1;
|
|
model_metric.total_inference_time += metrics.total_time;
|
|
model_metric.avg_latency_ms = model_metric.total_inference_time.as_millis() as f64
|
|
/ model_metric.requests_processed as f64;
|
|
model_metric.avg_tokens_per_request = (model_metric.avg_tokens_per_request
|
|
* (model_metric.requests_processed - 1) as f64
|
|
+ metrics.output_token_count as f64)
|
|
/ model_metric.requests_processed as f64;
|
|
model_metric.last_used = timestamp;
|
|
}
|
|
|
|
// Add request trace if tracing is enabled
|
|
if self.config.enable_tracing {
|
|
let mut traces = self.request_traces.lock().await;
|
|
|
|
let trace = RequestTrace {
|
|
request_id: result.request_id.into(),
|
|
model_name: model_name.to_string(),
|
|
timestamp,
|
|
input_tokens: metrics.input_token_count,
|
|
output_tokens: metrics.output_token_count,
|
|
total_latency_ms: metrics.total_time.as_millis() as f64,
|
|
queue_time_ms: metrics.queue_time.as_millis() as f64,
|
|
processing_time_ms: metrics.processing_time.as_millis() as f64,
|
|
generation_time_ms: metrics.generation_time.as_millis() as f64,
|
|
memory_usage_mb: metrics.peak_memory_bytes as f64 / 1024.0 / 1024.0,
|
|
sla_lane: sla_lane.to_string(),
|
|
temperature: Self::extract_temperature_from_result(result),
|
|
top_p: Self::extract_top_p_from_result(result),
|
|
top_k: Self::extract_top_k_from_result(result),
|
|
finish_reason: format!("{:?}", result.finish_reason),
|
|
error: error.map(std::string::ToString::to_string),
|
|
};
|
|
|
|
traces.push_back(trace);
|
|
|
|
// Maintain max traces limit
|
|
while traces.len() > self.config.max_traces {
|
|
traces.pop_front();
|
|
}
|
|
}
|
|
|
|
// Update time-series data
|
|
{
|
|
let mut ts_metrics = self.time_series_metrics.lock().await;
|
|
ts_metrics
|
|
.latency_samples
|
|
.push_back((timestamp, metrics.total_time.as_millis() as f64));
|
|
|
|
// Clean old samples
|
|
let cutoff = timestamp - self.config.metrics_retention;
|
|
while let Some((ts, _)) = ts_metrics.latency_samples.front() {
|
|
if *ts < cutoff {
|
|
ts_metrics.latency_samples.pop_front();
|
|
} else {
|
|
break;
|
|
}
|
|
}
|
|
}
|
|
|
|
Ok(())
|
|
}
|
|
|
|
/// Get current performance metrics
|
|
pub async fn get_performance_metrics(&self) -> PerformanceMetrics {
|
|
self.performance_metrics.read().await.clone()
|
|
}
|
|
|
|
/// Get current resource metrics
|
|
pub async fn get_resource_metrics(&self) -> ResourceMetrics {
|
|
self.resource_metrics.read().await.clone()
|
|
}
|
|
|
|
/// Get model-specific metrics
|
|
pub async fn get_model_metrics(&self) -> HashMap<String, ModelSpecificMetrics> {
|
|
self.model_metrics.read().await.clone()
|
|
}
|
|
|
|
/// Get recent request traces
|
|
pub async fn get_recent_traces(&self, limit: usize) -> Vec<RequestTrace> {
|
|
let traces = self.request_traces.lock().await;
|
|
traces.iter().rev().take(limit).cloned().collect()
|
|
}
|
|
|
|
/// Get current health status
|
|
pub async fn get_health_status(&self) -> HealthStatus {
|
|
self.health_status.read().await.clone()
|
|
}
|
|
|
|
/// Export metrics in Prometheus format
|
|
pub async fn export_prometheus_metrics(&self) -> String {
|
|
let perf_metrics = self.get_performance_metrics().await;
|
|
let resource_metrics = self.get_resource_metrics().await;
|
|
let model_metrics = self.get_model_metrics().await;
|
|
|
|
let mut output = String::new();
|
|
|
|
// Performance metrics
|
|
output.push_str(&format!(
|
|
"rtx_inference_requests_total {}\n",
|
|
perf_metrics.total_requests
|
|
));
|
|
output.push_str(&format!(
|
|
"rtx_inference_requests_successful {}\n",
|
|
perf_metrics.successful_requests
|
|
));
|
|
output.push_str(&format!(
|
|
"rtx_inference_requests_failed {}\n",
|
|
perf_metrics.failed_requests
|
|
));
|
|
output.push_str(&format!(
|
|
"rtx_inference_requests_per_second {}\n",
|
|
perf_metrics.requests_per_second
|
|
));
|
|
output.push_str(&format!(
|
|
"rtx_inference_latency_avg_ms {}\n",
|
|
perf_metrics.avg_latency_ms
|
|
));
|
|
output.push_str(&format!(
|
|
"rtx_inference_latency_p95_ms {}\n",
|
|
perf_metrics.p95_latency_ms
|
|
));
|
|
output.push_str(&format!(
|
|
"rtx_inference_latency_p99_ms {}\n",
|
|
perf_metrics.p99_latency_ms
|
|
));
|
|
output.push_str(&format!(
|
|
"rtx_inference_tokens_generated {}\n",
|
|
perf_metrics.tokens_generated
|
|
));
|
|
output.push_str(&format!(
|
|
"rtx_inference_tokens_per_second {}\n",
|
|
perf_metrics.tokens_per_second
|
|
));
|
|
output.push_str(&format!(
|
|
"rtx_inference_success_rate {}\n",
|
|
perf_metrics.success_rate
|
|
));
|
|
|
|
// Resource metrics
|
|
output.push_str(&format!(
|
|
"rtx_inference_gpu_memory_utilization {}\n",
|
|
resource_metrics.gpu_memory_utilization
|
|
));
|
|
output.push_str(&format!(
|
|
"rtx_inference_cpu_utilization {}\n",
|
|
resource_metrics.cpu_utilization
|
|
));
|
|
output.push_str(&format!(
|
|
"rtx_inference_memory_usage_gb {}\n",
|
|
resource_metrics.memory_usage_gb
|
|
));
|
|
output.push_str(&format!(
|
|
"rtx_inference_cache_hit_rate {}\n",
|
|
resource_metrics.cache_hit_rate
|
|
));
|
|
|
|
// Model-specific metrics
|
|
for (model_name, metrics) in model_metrics {
|
|
let label = format!("model=\"{model_name}\"");
|
|
output.push_str(&format!(
|
|
"rtx_inference_model_requests_processed{{{}}} {}\n",
|
|
label, metrics.requests_processed
|
|
));
|
|
output.push_str(&format!(
|
|
"rtx_inference_model_avg_latency_ms{{{}}} {}\n",
|
|
label, metrics.avg_latency_ms
|
|
));
|
|
output.push_str(&format!(
|
|
"rtx_inference_model_memory_usage_mb{{{}}} {}\n",
|
|
label, metrics.memory_usage_mb
|
|
));
|
|
}
|
|
|
|
output
|
|
}
|
|
|
|
/// Update resource utilization metrics (called by external monitoring)
|
|
pub async fn update_resource_metrics(
|
|
&self,
|
|
gpu_memory_pct: f64,
|
|
cpu_pct: f64,
|
|
memory_gb: f64,
|
|
fragmentation: f64,
|
|
cache_hit_rate: f64,
|
|
) {
|
|
let mut metrics = self.resource_metrics.write().await;
|
|
metrics.gpu_memory_utilization = gpu_memory_pct;
|
|
metrics.cpu_utilization = cpu_pct;
|
|
metrics.memory_usage_gb = memory_gb;
|
|
metrics.memory_fragmentation = fragmentation;
|
|
metrics.cache_hit_rate = cache_hit_rate;
|
|
}
|
|
|
|
/// Start background task for metrics aggregation
|
|
fn start_metrics_aggregation_task(
|
|
performance_metrics: Arc<RwLock<PerformanceMetrics>>,
|
|
time_series_metrics: Arc<Mutex<TimeSeriesMetrics>>,
|
|
window: Duration,
|
|
_retention: Duration,
|
|
) -> tokio::task::JoinHandle<()> {
|
|
tokio::spawn(async move {
|
|
let mut interval = tokio::time::interval(window);
|
|
|
|
loop {
|
|
interval.tick().await;
|
|
|
|
let now = SystemTime::now();
|
|
|
|
// Aggregate metrics from time series data
|
|
{
|
|
let ts_metrics = time_series_metrics.lock().await;
|
|
let mut perf_metrics = performance_metrics.write().await;
|
|
|
|
// Calculate percentiles and rates from recent samples
|
|
if !ts_metrics.latency_samples.is_empty() {
|
|
let recent_cutoff = now - window;
|
|
let recent_latencies: Vec<f64> = ts_metrics
|
|
.latency_samples
|
|
.iter()
|
|
.filter(|(ts, _)| *ts >= recent_cutoff)
|
|
.map(|(_, latency)| *latency)
|
|
.collect();
|
|
|
|
if !recent_latencies.is_empty() {
|
|
perf_metrics.avg_latency_ms = recent_latencies.iter().sum::<f64>()
|
|
/ recent_latencies.len() as f64;
|
|
|
|
// Calculate percentiles
|
|
let mut sorted_latencies = recent_latencies;
|
|
sorted_latencies.sort_by(f64::total_cmp);
|
|
|
|
let p95_idx = (sorted_latencies.len() as f64 * 0.95) as usize;
|
|
let p99_idx = (sorted_latencies.len() as f64 * 0.99) as usize;
|
|
|
|
perf_metrics.p95_latency_ms =
|
|
sorted_latencies.get(p95_idx).copied().unwrap_or(0.0);
|
|
perf_metrics.p99_latency_ms =
|
|
sorted_latencies.get(p99_idx).copied().unwrap_or(0.0);
|
|
}
|
|
|
|
// Calculate requests per second
|
|
let recent_request_count = ts_metrics
|
|
.latency_samples
|
|
.iter()
|
|
.filter(|(ts, _)| *ts >= recent_cutoff)
|
|
.count();
|
|
perf_metrics.requests_per_second =
|
|
recent_request_count as f64 / window.as_secs_f64();
|
|
}
|
|
|
|
perf_metrics.updated_at = now;
|
|
}
|
|
|
|
debug!("Metrics aggregated at {:?}", now);
|
|
}
|
|
})
|
|
}
|
|
|
|
/// Start background health check task
|
|
fn start_health_check_task(
|
|
health_status: Arc<RwLock<HealthStatus>>,
|
|
resource_metrics: Arc<RwLock<ResourceMetrics>>,
|
|
model_metrics: Arc<RwLock<HashMap<String, ModelSpecificMetrics>>>,
|
|
interval: Duration,
|
|
) -> tokio::task::JoinHandle<()> {
|
|
tokio::spawn(async move {
|
|
let mut ticker = tokio::time::interval(interval);
|
|
|
|
loop {
|
|
ticker.tick().await;
|
|
|
|
let now = SystemTime::now();
|
|
let mut health = health_status.write().await;
|
|
|
|
// Check resource health
|
|
let resource_metrics_read = resource_metrics.read().await;
|
|
health.resource_health.memory_status =
|
|
if resource_metrics_read.gpu_memory_utilization > 0.9 {
|
|
HealthLevel::Critical
|
|
} else if resource_metrics_read.gpu_memory_utilization > 0.8 {
|
|
HealthLevel::Warning
|
|
} else {
|
|
HealthLevel::Healthy
|
|
};
|
|
|
|
health.resource_health.cpu_status = if resource_metrics_read.cpu_utilization > 0.9 {
|
|
HealthLevel::Critical
|
|
} else if resource_metrics_read.cpu_utilization > 0.8 {
|
|
HealthLevel::Warning
|
|
} else {
|
|
HealthLevel::Healthy
|
|
};
|
|
drop(resource_metrics_read);
|
|
|
|
// Check model health
|
|
let model_metrics_read = model_metrics.read().await;
|
|
for (model_name, metrics) in model_metrics_read.iter() {
|
|
let time_since_last_use = now
|
|
.duration_since(metrics.last_used)
|
|
.unwrap_or(Duration::ZERO);
|
|
let model_health = if time_since_last_use > Duration::from_secs(3600) {
|
|
// 1 hour
|
|
HealthLevel::Warning
|
|
} else {
|
|
HealthLevel::Healthy
|
|
};
|
|
health.model_health.insert(model_name.clone(), model_health);
|
|
}
|
|
drop(model_metrics_read);
|
|
|
|
// Determine overall health
|
|
health.overall_status = [
|
|
health.resource_health.memory_status,
|
|
health.resource_health.cpu_status,
|
|
health.resource_health.gpu_status,
|
|
]
|
|
.iter()
|
|
.chain(health.model_health.values())
|
|
.max()
|
|
.copied()
|
|
.unwrap_or(HealthLevel::Healthy);
|
|
|
|
health.last_check = now;
|
|
|
|
debug!("Health check completed: {:?}", health.overall_status);
|
|
}
|
|
})
|
|
}
|
|
|
|
/// Detect model version from model name or metadata
|
|
fn detect_model_version(model_name: &str) -> String {
|
|
// Extract version from model name if present (e.g., "llama-2-7b-v1.2")
|
|
if let Some(version_pos) = model_name.rfind("-v")
|
|
&& let Some(version_part) = model_name.get(version_pos + 2..)
|
|
&& version_part.chars().all(|c| c.is_ascii_digit() || c == '.')
|
|
{
|
|
return version_part.to_string();
|
|
}
|
|
|
|
// Check for common version patterns
|
|
if model_name.contains("gpt-4") {
|
|
return "4.0".to_string();
|
|
} else if model_name.contains("gpt-3.5") {
|
|
return "3.5".to_string();
|
|
} else if model_name.contains("claude-3") {
|
|
return "3.0".to_string();
|
|
} else if model_name.contains("llama-2") {
|
|
return "2.0".to_string();
|
|
} else if model_name.contains("llama-3") {
|
|
return "3.0".to_string();
|
|
}
|
|
|
|
// Default version
|
|
"1.0".to_string()
|
|
}
|
|
|
|
/// Extract temperature parameter from request result
|
|
fn extract_temperature_from_result(result: &RequestResult) -> Option<f32> {
|
|
// Look for temperature in generation config or request metadata
|
|
// For now, return a default temperature if generation was successful
|
|
if !matches!(result.finish_reason, FinishReason::Error) && !result.output_tokens.is_empty()
|
|
{
|
|
Some(0.7) // Default temperature
|
|
} else {
|
|
None
|
|
}
|
|
}
|
|
|
|
/// Extract `top_p` parameter from request result
|
|
fn extract_top_p_from_result(result: &RequestResult) -> Option<f32> {
|
|
// Look for top_p in generation config or request metadata
|
|
if !matches!(result.finish_reason, FinishReason::Error) && !result.output_tokens.is_empty()
|
|
{
|
|
Some(0.9) // Default top_p
|
|
} else {
|
|
None
|
|
}
|
|
}
|
|
|
|
/// Extract `top_k` parameter from request result
|
|
fn extract_top_k_from_result(result: &RequestResult) -> Option<u32> {
|
|
// Look for top_k in generation config or request metadata
|
|
if !matches!(result.finish_reason, FinishReason::Error) && !result.output_tokens.is_empty()
|
|
{
|
|
Some(50) // Default top_k
|
|
} else {
|
|
None
|
|
}
|
|
}
|
|
}
|
|
|
|
impl Default for PerformanceMetrics {
|
|
fn default() -> Self {
|
|
Self {
|
|
total_requests: 0,
|
|
successful_requests: 0,
|
|
failed_requests: 0,
|
|
requests_per_second: 0.0,
|
|
avg_latency_ms: 0.0,
|
|
p95_latency_ms: 0.0,
|
|
p99_latency_ms: 0.0,
|
|
tokens_generated: 0,
|
|
tokens_per_second: 0.0,
|
|
success_rate: 0.0,
|
|
avg_batch_size: 0.0,
|
|
total_queue_depth: 0,
|
|
updated_at: SystemTime::now(),
|
|
}
|
|
}
|
|
}
|
|
|
|
impl Default for ResourceMetrics {
|
|
fn default() -> Self {
|
|
Self {
|
|
gpu_memory_utilization: 0.0,
|
|
cpu_utilization: 0.0,
|
|
memory_usage_gb: 0.0,
|
|
memory_fragmentation: 0.0,
|
|
active_connections: 0,
|
|
cache_hit_rate: 0.0,
|
|
disk_io_mbps: 0.0,
|
|
network_throughput_mbps: 0.0,
|
|
gpu_temperature_c: None,
|
|
power_consumption_w: None,
|
|
}
|
|
}
|
|
}
|