443 lines
14 KiB
Rust
443 lines
14 KiB
Rust
//! Error types for the inference runtime
|
|
//!
|
|
//! This module defines all error types used throughout the inference system,
|
|
//! providing comprehensive error handling with proper context and recovery information.
|
|
|
|
use std::time::Duration;
|
|
use thiserror::Error;
|
|
use uuid::Uuid;
|
|
|
|
/// Comprehensive error type for inference operations
|
|
#[derive(Error, Debug)]
|
|
pub enum InferenceError {
|
|
/// Request validation failed
|
|
#[error("Invalid request: {message}")]
|
|
InvalidRequest { message: String },
|
|
|
|
/// Queue is at maximum capacity
|
|
#[error("Request queue is full (capacity: {capacity}, current: {current})")]
|
|
QueueFull { capacity: usize, current: usize },
|
|
|
|
/// Request not found in system
|
|
#[error("Request not found: {request_id}")]
|
|
RequestNotFound { request_id: Uuid },
|
|
|
|
/// Request has timed out
|
|
#[error("Request timed out after {elapsed:?} (deadline: {deadline:?})")]
|
|
RequestTimeout {
|
|
elapsed: Duration,
|
|
deadline: Duration,
|
|
},
|
|
|
|
/// Request was cancelled
|
|
#[error("Request was cancelled: {request_id}")]
|
|
RequestCancelled { request_id: Uuid },
|
|
|
|
/// Scheduler is in degradation mode
|
|
#[error("Scheduler degradation mode active (reason: {reason})")]
|
|
SchedulerDegradation { reason: String },
|
|
|
|
/// Batch formation failed
|
|
#[error("Failed to form batch: {reason}")]
|
|
BatchFormationFailed { reason: String },
|
|
|
|
/// SLA violation occurred
|
|
#[error("SLA violation for lane '{lane}': {violation_type}")]
|
|
SlaViolation {
|
|
lane: String,
|
|
violation_type: String,
|
|
},
|
|
|
|
/// Memory allocation failed
|
|
#[error("Memory allocation failed: {bytes} bytes requested, {available} available")]
|
|
OutOfMemory { bytes: usize, available: usize },
|
|
|
|
/// KV cache operation failed
|
|
#[error("KV cache error: {operation} failed - {reason}")]
|
|
KvCacheError { operation: String, reason: String },
|
|
|
|
/// Page allocation failed
|
|
#[error("Page allocation failed: requested {pages} pages, tier {tier}")]
|
|
PageAllocationFailed { pages: usize, tier: String },
|
|
|
|
/// Cache tier migration failed
|
|
#[error("Migration failed: {source_tier} -> {target_tier} - {reason}")]
|
|
MigrationFailed {
|
|
source_tier: String,
|
|
target_tier: String,
|
|
reason: String,
|
|
},
|
|
|
|
/// Eviction operation failed
|
|
#[error("Eviction failed: policy {policy}, pages {pages} - {reason}")]
|
|
EvictionFailed {
|
|
policy: String,
|
|
pages: usize,
|
|
reason: String,
|
|
},
|
|
|
|
/// Model not found or not loaded
|
|
#[error("Model not found: {model_name}")]
|
|
ModelNotFound { model_name: String },
|
|
|
|
/// Model loading failed
|
|
#[error("Model loading failed: {model_name} - {reason}")]
|
|
ModelLoadingFailed { model_name: String, reason: String },
|
|
|
|
/// Quantization operation failed
|
|
#[error("Quantization failed: {operation} from {from_dtype} to {to_dtype} - {reason}")]
|
|
QuantizationFailed {
|
|
operation: String,
|
|
from_dtype: String,
|
|
to_dtype: String,
|
|
reason: String,
|
|
},
|
|
|
|
/// Kernel synthesis failed
|
|
#[error("Kernel synthesis failed: {operation} - {reason}")]
|
|
SynthesisFailed { operation: String, reason: String },
|
|
|
|
/// Device operation failed
|
|
#[error("Device error: {device_type} - {operation} failed: {reason}")]
|
|
DeviceError {
|
|
device_type: String,
|
|
operation: String,
|
|
reason: String,
|
|
},
|
|
|
|
/// Tensor operation failed
|
|
#[error("Tensor operation failed: {operation} - {reason}")]
|
|
TensorError { operation: String, reason: String },
|
|
|
|
/// Serialization/deserialization failed
|
|
#[error("Serialization error: {operation} - {reason}")]
|
|
SerializationError { operation: String, reason: String },
|
|
|
|
/// Network/communication error
|
|
#[error("Network error: {endpoint} - {reason}")]
|
|
NetworkError { endpoint: String, reason: String },
|
|
|
|
/// Configuration error
|
|
#[error("Configuration error: {parameter} - {reason}")]
|
|
ConfigurationError { parameter: String, reason: String },
|
|
|
|
/// Runtime panic recovery
|
|
#[error("Runtime panic recovered: {location} - {message}")]
|
|
PanicRecovered { location: String, message: String },
|
|
|
|
/// Internal system error (should be rare)
|
|
#[error("Internal system error: {context} - {reason}")]
|
|
InternalError { context: String, reason: String },
|
|
|
|
/// Wrapped errors from dependencies (boxed to avoid Clone requirement)
|
|
#[error("Runtime error: {0}")]
|
|
Runtime(Box<rtx_runtime::error::RuntimeError>),
|
|
|
|
#[error("Tensor error: {0}")]
|
|
Tensor(Box<rtx_tensor::error::TensorError>),
|
|
|
|
#[error("Synthesis error: {0}")]
|
|
Synthesis(Box<rtx_synthesis::error::SynthesisError>),
|
|
|
|
#[error("IO error: {0}")]
|
|
Io(Box<std::io::Error>),
|
|
|
|
#[error("JSON serialization error: {0}")]
|
|
Json(Box<serde_json::Error>),
|
|
|
|
#[error("Async task error: {0}")]
|
|
Join(Box<tokio::task::JoinError>),
|
|
|
|
/// Feature not implemented yet
|
|
#[error("Not implemented: {feature}")]
|
|
NotImplemented { feature: String },
|
|
|
|
/// Resource exhaustion (too many concurrent operations)
|
|
#[error("Resource exhausted: {resource} - {reason}")]
|
|
ResourceExhausted { resource: String, reason: String },
|
|
}
|
|
|
|
impl InferenceError {
|
|
/// Create an invalid request error
|
|
pub fn invalid_request(message: impl Into<String>) -> Self {
|
|
Self::InvalidRequest {
|
|
message: message.into(),
|
|
}
|
|
}
|
|
|
|
/// Create a queue full error
|
|
#[must_use]
|
|
pub fn queue_full(capacity: usize, current: usize) -> Self {
|
|
Self::QueueFull { capacity, current }
|
|
}
|
|
|
|
/// Create a request not found error
|
|
pub fn request_not_found(request_id: impl Into<Uuid>) -> Self {
|
|
Self::RequestNotFound {
|
|
request_id: request_id.into(),
|
|
}
|
|
}
|
|
|
|
/// Create a request timeout error
|
|
#[must_use]
|
|
pub fn request_timeout(elapsed: Duration, deadline: Duration) -> Self {
|
|
Self::RequestTimeout { elapsed, deadline }
|
|
}
|
|
|
|
/// Create a KV cache error
|
|
pub fn kv_cache_error(operation: impl Into<String>, reason: impl Into<String>) -> Self {
|
|
Self::KvCacheError {
|
|
operation: operation.into(),
|
|
reason: reason.into(),
|
|
}
|
|
}
|
|
|
|
/// Create a model not found error
|
|
pub fn model_not_found(model_name: impl Into<String>) -> Self {
|
|
Self::ModelNotFound {
|
|
model_name: model_name.into(),
|
|
}
|
|
}
|
|
|
|
/// Create an internal error (should be used sparingly)
|
|
pub fn internal_error(context: impl Into<String>, reason: impl Into<String>) -> Self {
|
|
Self::InternalError {
|
|
context: context.into(),
|
|
reason: reason.into(),
|
|
}
|
|
}
|
|
|
|
/// Create a not implemented error
|
|
pub fn not_implemented(feature: impl Into<String>) -> Self {
|
|
Self::NotImplemented {
|
|
feature: feature.into(),
|
|
}
|
|
}
|
|
|
|
/// Create a resource exhausted error
|
|
pub fn resource_exhausted(resource: impl Into<String>) -> Self {
|
|
Self::ResourceExhausted {
|
|
resource: resource.into(),
|
|
reason: "Maximum concurrent operations exceeded".to_string(),
|
|
}
|
|
}
|
|
|
|
/// Check if error is recoverable
|
|
#[must_use]
|
|
pub fn is_recoverable(&self) -> bool {
|
|
match self {
|
|
// Unrecoverable errors
|
|
Self::InternalError { .. }
|
|
| Self::PanicRecovered { .. }
|
|
| Self::ConfigurationError { .. }
|
|
| Self::NotImplemented { .. } => false,
|
|
|
|
// Potentially recoverable with retry
|
|
Self::OutOfMemory { .. }
|
|
| Self::NetworkError { .. }
|
|
| Self::DeviceError { .. }
|
|
| Self::KvCacheError { .. } => true,
|
|
|
|
// Request-level errors (recoverable at system level)
|
|
Self::InvalidRequest { .. }
|
|
| Self::RequestNotFound { .. }
|
|
| Self::RequestTimeout { .. }
|
|
| Self::RequestCancelled { .. } => true,
|
|
|
|
// Resource exhaustion (may recover over time)
|
|
Self::QueueFull { .. }
|
|
| Self::PageAllocationFailed { .. }
|
|
| Self::SchedulerDegradation { .. }
|
|
| Self::ResourceExhausted { .. } => true,
|
|
|
|
// Wrapped errors - simplified check
|
|
Self::Runtime(_) => true,
|
|
Self::Tensor(_) => true,
|
|
Self::Synthesis(_) => true,
|
|
Self::Io(_) => true,
|
|
Self::Json(_) => false,
|
|
Self::Join(_) => false,
|
|
|
|
_ => true, // Default to recoverable for new error types
|
|
}
|
|
}
|
|
|
|
/// Get error severity level
|
|
#[must_use]
|
|
pub fn severity(&self) -> ErrorSeverity {
|
|
match self {
|
|
// Critical errors requiring immediate attention
|
|
Self::InternalError { .. }
|
|
| Self::PanicRecovered { .. }
|
|
| Self::NotImplemented { .. } => ErrorSeverity::Critical,
|
|
|
|
// High severity - significant impact
|
|
Self::ModelLoadingFailed { .. }
|
|
| Self::DeviceError { .. }
|
|
| Self::SchedulerDegradation { .. } => ErrorSeverity::High,
|
|
|
|
// Medium severity - operational issues
|
|
Self::OutOfMemory { .. }
|
|
| Self::SlaViolation { .. }
|
|
| Self::KvCacheError { .. }
|
|
| Self::NetworkError { .. }
|
|
| Self::ResourceExhausted { .. } => ErrorSeverity::Medium,
|
|
|
|
// Low severity - request-level issues
|
|
Self::InvalidRequest { .. }
|
|
| Self::RequestNotFound { .. }
|
|
| Self::RequestTimeout { .. }
|
|
| Self::RequestCancelled { .. }
|
|
| Self::QueueFull { .. } => ErrorSeverity::Low,
|
|
|
|
_ => ErrorSeverity::Medium,
|
|
}
|
|
}
|
|
|
|
/// Get suggested retry delay
|
|
#[must_use]
|
|
pub fn retry_delay(&self) -> Option<Duration> {
|
|
match self {
|
|
// No retry for permanent failures
|
|
Self::InvalidRequest { .. }
|
|
| Self::RequestCancelled { .. }
|
|
| Self::ConfigurationError { .. }
|
|
| Self::InternalError { .. }
|
|
| Self::NotImplemented { .. } => None,
|
|
|
|
// Quick retry for transient issues
|
|
Self::NetworkError { .. } | Self::DeviceError { .. } => {
|
|
Some(Duration::from_millis(100))
|
|
}
|
|
|
|
// Longer delay for resource exhaustion
|
|
Self::OutOfMemory { .. }
|
|
| Self::QueueFull { .. }
|
|
| Self::PageAllocationFailed { .. }
|
|
| Self::ResourceExhausted { .. } => Some(Duration::from_millis(1000)),
|
|
|
|
// Very long delay for degradation mode
|
|
Self::SchedulerDegradation { .. } => Some(Duration::from_secs(5)),
|
|
|
|
_ => Some(Duration::from_millis(500)), // Default retry delay
|
|
}
|
|
}
|
|
}
|
|
|
|
/// Error severity levels for monitoring and alerting
|
|
#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)]
|
|
pub enum ErrorSeverity {
|
|
Low,
|
|
Medium,
|
|
High,
|
|
Critical,
|
|
}
|
|
|
|
impl std::fmt::Display for ErrorSeverity {
|
|
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
|
match self {
|
|
Self::Low => write!(f, "LOW"),
|
|
Self::Medium => write!(f, "MEDIUM"),
|
|
Self::High => write!(f, "HIGH"),
|
|
Self::Critical => write!(f, "CRITICAL"),
|
|
}
|
|
}
|
|
}
|
|
|
|
/// Result type alias for inference operations
|
|
pub type InferenceResult<T> = Result<T, InferenceError>;
|
|
|
|
// Manual From implementations for boxed errors
|
|
impl From<rtx_runtime::error::RuntimeError> for InferenceError {
|
|
fn from(err: rtx_runtime::error::RuntimeError) -> Self {
|
|
Self::Runtime(Box::new(err))
|
|
}
|
|
}
|
|
|
|
impl From<rtx_tensor::error::TensorError> for InferenceError {
|
|
fn from(err: rtx_tensor::error::TensorError) -> Self {
|
|
Self::Tensor(Box::new(err))
|
|
}
|
|
}
|
|
|
|
impl From<rtx_synthesis::error::SynthesisError> for InferenceError {
|
|
fn from(err: rtx_synthesis::error::SynthesisError) -> Self {
|
|
Self::Synthesis(Box::new(err))
|
|
}
|
|
}
|
|
|
|
impl From<std::io::Error> for InferenceError {
|
|
fn from(err: std::io::Error) -> Self {
|
|
Self::Io(Box::new(err))
|
|
}
|
|
}
|
|
|
|
impl From<serde_json::Error> for InferenceError {
|
|
fn from(err: serde_json::Error) -> Self {
|
|
Self::Json(Box::new(err))
|
|
}
|
|
}
|
|
|
|
impl From<tokio::task::JoinError> for InferenceError {
|
|
fn from(err: tokio::task::JoinError) -> Self {
|
|
Self::Join(Box::new(err))
|
|
}
|
|
}
|
|
|
|
/// Convenience macro for creating inference errors
|
|
#[macro_export]
|
|
macro_rules! inference_error {
|
|
($variant:ident, $($arg:expr),*) => {
|
|
$crate::error::InferenceError::$variant { $($arg),* }
|
|
};
|
|
}
|
|
|
|
/// Convenience macro for creating invalid request errors
|
|
#[macro_export]
|
|
macro_rules! invalid_request {
|
|
($msg:expr) => {
|
|
$crate::error::InferenceError::invalid_request($msg)
|
|
};
|
|
($fmt:expr, $($arg:tt)*) => {
|
|
$crate::error::InferenceError::invalid_request(format!($fmt, $($arg)*))
|
|
};
|
|
}
|
|
|
|
#[cfg(test)]
|
|
mod tests {
|
|
use super::*;
|
|
|
|
#[test]
|
|
fn test_error_creation() {
|
|
let error = InferenceError::invalid_request("test message");
|
|
assert!(matches!(error, InferenceError::InvalidRequest { .. }));
|
|
}
|
|
|
|
#[test]
|
|
fn test_error_recoverability() {
|
|
let recoverable = InferenceError::queue_full(10, 10);
|
|
assert!(recoverable.is_recoverable());
|
|
|
|
let unrecoverable = InferenceError::internal_error("test", "reason");
|
|
assert!(!unrecoverable.is_recoverable());
|
|
}
|
|
|
|
#[test]
|
|
fn test_error_severity() {
|
|
let low = InferenceError::invalid_request("test");
|
|
assert_eq!(low.severity(), ErrorSeverity::Low);
|
|
|
|
let critical = InferenceError::internal_error("test", "reason");
|
|
assert_eq!(critical.severity(), ErrorSeverity::Critical);
|
|
}
|
|
|
|
#[test]
|
|
fn test_retry_delay() {
|
|
let no_retry = InferenceError::invalid_request("test");
|
|
assert_eq!(no_retry.retry_delay(), None);
|
|
|
|
let quick_retry = InferenceError::kv_cache_error("read", "timeout");
|
|
assert!(quick_retry.retry_delay().unwrap() > Duration::ZERO);
|
|
}
|
|
}
|