//! Error types for the inference runtime //! //! This module defines all error types used throughout the inference system, //! providing comprehensive error handling with proper context and recovery information. use std::time::Duration; use thiserror::Error; use uuid::Uuid; /// Comprehensive error type for inference operations #[derive(Error, Debug)] pub enum InferenceError { /// Request validation failed #[error("Invalid request: {message}")] InvalidRequest { message: String }, /// Queue is at maximum capacity #[error("Request queue is full (capacity: {capacity}, current: {current})")] QueueFull { capacity: usize, current: usize }, /// Request not found in system #[error("Request not found: {request_id}")] RequestNotFound { request_id: Uuid }, /// Request has timed out #[error("Request timed out after {elapsed:?} (deadline: {deadline:?})")] RequestTimeout { elapsed: Duration, deadline: Duration, }, /// Request was cancelled #[error("Request was cancelled: {request_id}")] RequestCancelled { request_id: Uuid }, /// Scheduler is in degradation mode #[error("Scheduler degradation mode active (reason: {reason})")] SchedulerDegradation { reason: String }, /// Batch formation failed #[error("Failed to form batch: {reason}")] BatchFormationFailed { reason: String }, /// SLA violation occurred #[error("SLA violation for lane '{lane}': {violation_type}")] SlaViolation { lane: String, violation_type: String, }, /// Memory allocation failed #[error("Memory allocation failed: {bytes} bytes requested, {available} available")] OutOfMemory { bytes: usize, available: usize }, /// KV cache operation failed #[error("KV cache error: {operation} failed - {reason}")] KvCacheError { operation: String, reason: String }, /// Page allocation failed #[error("Page allocation failed: requested {pages} pages, tier {tier}")] PageAllocationFailed { pages: usize, tier: String }, /// Cache tier migration failed #[error("Migration failed: {source_tier} -> {target_tier} - {reason}")] MigrationFailed { source_tier: String, target_tier: String, reason: String, }, /// Eviction operation failed #[error("Eviction failed: policy {policy}, pages {pages} - {reason}")] EvictionFailed { policy: String, pages: usize, reason: String, }, /// Model not found or not loaded #[error("Model not found: {model_name}")] ModelNotFound { model_name: String }, /// Model loading failed #[error("Model loading failed: {model_name} - {reason}")] ModelLoadingFailed { model_name: String, reason: String }, /// Quantization operation failed #[error("Quantization failed: {operation} from {from_dtype} to {to_dtype} - {reason}")] QuantizationFailed { operation: String, from_dtype: String, to_dtype: String, reason: String, }, /// Kernel synthesis failed #[error("Kernel synthesis failed: {operation} - {reason}")] SynthesisFailed { operation: String, reason: String }, /// Device operation failed #[error("Device error: {device_type} - {operation} failed: {reason}")] DeviceError { device_type: String, operation: String, reason: String, }, /// Tensor operation failed #[error("Tensor operation failed: {operation} - {reason}")] TensorError { operation: String, reason: String }, /// Serialization/deserialization failed #[error("Serialization error: {operation} - {reason}")] SerializationError { operation: String, reason: String }, /// Network/communication error #[error("Network error: {endpoint} - {reason}")] NetworkError { endpoint: String, reason: String }, /// Configuration error #[error("Configuration error: {parameter} - {reason}")] ConfigurationError { parameter: String, reason: String }, /// Runtime panic recovery #[error("Runtime panic recovered: {location} - {message}")] PanicRecovered { location: String, message: String }, /// Internal system error (should be rare) #[error("Internal system error: {context} - {reason}")] InternalError { context: String, reason: String }, /// Wrapped errors from dependencies (boxed to avoid Clone requirement) #[error("Runtime error: {0}")] Runtime(Box), #[error("Tensor error: {0}")] Tensor(Box), #[error("Synthesis error: {0}")] Synthesis(Box), #[error("IO error: {0}")] Io(Box), #[error("JSON serialization error: {0}")] Json(Box), #[error("Async task error: {0}")] Join(Box), /// Feature not implemented yet #[error("Not implemented: {feature}")] NotImplemented { feature: String }, /// Resource exhaustion (too many concurrent operations) #[error("Resource exhausted: {resource} - {reason}")] ResourceExhausted { resource: String, reason: String }, } impl InferenceError { /// Create an invalid request error pub fn invalid_request(message: impl Into) -> Self { Self::InvalidRequest { message: message.into(), } } /// Create a queue full error #[must_use] pub fn queue_full(capacity: usize, current: usize) -> Self { Self::QueueFull { capacity, current } } /// Create a request not found error pub fn request_not_found(request_id: impl Into) -> Self { Self::RequestNotFound { request_id: request_id.into(), } } /// Create a request timeout error #[must_use] pub fn request_timeout(elapsed: Duration, deadline: Duration) -> Self { Self::RequestTimeout { elapsed, deadline } } /// Create a KV cache error pub fn kv_cache_error(operation: impl Into, reason: impl Into) -> Self { Self::KvCacheError { operation: operation.into(), reason: reason.into(), } } /// Create a model not found error pub fn model_not_found(model_name: impl Into) -> Self { Self::ModelNotFound { model_name: model_name.into(), } } /// Create an internal error (should be used sparingly) pub fn internal_error(context: impl Into, reason: impl Into) -> Self { Self::InternalError { context: context.into(), reason: reason.into(), } } /// Create a not implemented error pub fn not_implemented(feature: impl Into) -> Self { Self::NotImplemented { feature: feature.into(), } } /// Create a resource exhausted error pub fn resource_exhausted(resource: impl Into) -> Self { Self::ResourceExhausted { resource: resource.into(), reason: "Maximum concurrent operations exceeded".to_string(), } } /// Check if error is recoverable #[must_use] pub fn is_recoverable(&self) -> bool { match self { // Unrecoverable errors Self::InternalError { .. } | Self::PanicRecovered { .. } | Self::ConfigurationError { .. } | Self::NotImplemented { .. } => false, // Potentially recoverable with retry Self::OutOfMemory { .. } | Self::NetworkError { .. } | Self::DeviceError { .. } | Self::KvCacheError { .. } => true, // Request-level errors (recoverable at system level) Self::InvalidRequest { .. } | Self::RequestNotFound { .. } | Self::RequestTimeout { .. } | Self::RequestCancelled { .. } => true, // Resource exhaustion (may recover over time) Self::QueueFull { .. } | Self::PageAllocationFailed { .. } | Self::SchedulerDegradation { .. } | Self::ResourceExhausted { .. } => true, // Wrapped errors - simplified check Self::Runtime(_) => true, Self::Tensor(_) => true, Self::Synthesis(_) => true, Self::Io(_) => true, Self::Json(_) => false, Self::Join(_) => false, _ => true, // Default to recoverable for new error types } } /// Get error severity level #[must_use] pub fn severity(&self) -> ErrorSeverity { match self { // Critical errors requiring immediate attention Self::InternalError { .. } | Self::PanicRecovered { .. } | Self::NotImplemented { .. } => ErrorSeverity::Critical, // High severity - significant impact Self::ModelLoadingFailed { .. } | Self::DeviceError { .. } | Self::SchedulerDegradation { .. } => ErrorSeverity::High, // Medium severity - operational issues Self::OutOfMemory { .. } | Self::SlaViolation { .. } | Self::KvCacheError { .. } | Self::NetworkError { .. } | Self::ResourceExhausted { .. } => ErrorSeverity::Medium, // Low severity - request-level issues Self::InvalidRequest { .. } | Self::RequestNotFound { .. } | Self::RequestTimeout { .. } | Self::RequestCancelled { .. } | Self::QueueFull { .. } => ErrorSeverity::Low, _ => ErrorSeverity::Medium, } } /// Get suggested retry delay #[must_use] pub fn retry_delay(&self) -> Option { match self { // No retry for permanent failures Self::InvalidRequest { .. } | Self::RequestCancelled { .. } | Self::ConfigurationError { .. } | Self::InternalError { .. } | Self::NotImplemented { .. } => None, // Quick retry for transient issues Self::NetworkError { .. } | Self::DeviceError { .. } => { Some(Duration::from_millis(100)) } // Longer delay for resource exhaustion Self::OutOfMemory { .. } | Self::QueueFull { .. } | Self::PageAllocationFailed { .. } | Self::ResourceExhausted { .. } => Some(Duration::from_millis(1000)), // Very long delay for degradation mode Self::SchedulerDegradation { .. } => Some(Duration::from_secs(5)), _ => Some(Duration::from_millis(500)), // Default retry delay } } } /// Error severity levels for monitoring and alerting #[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)] pub enum ErrorSeverity { Low, Medium, High, Critical, } impl std::fmt::Display for ErrorSeverity { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { match self { Self::Low => write!(f, "LOW"), Self::Medium => write!(f, "MEDIUM"), Self::High => write!(f, "HIGH"), Self::Critical => write!(f, "CRITICAL"), } } } /// Result type alias for inference operations pub type InferenceResult = Result; // Manual From implementations for boxed errors impl From for InferenceError { fn from(err: rtx_runtime::error::RuntimeError) -> Self { Self::Runtime(Box::new(err)) } } impl From for InferenceError { fn from(err: rtx_tensor::error::TensorError) -> Self { Self::Tensor(Box::new(err)) } } impl From for InferenceError { fn from(err: rtx_synthesis::error::SynthesisError) -> Self { Self::Synthesis(Box::new(err)) } } impl From for InferenceError { fn from(err: std::io::Error) -> Self { Self::Io(Box::new(err)) } } impl From for InferenceError { fn from(err: serde_json::Error) -> Self { Self::Json(Box::new(err)) } } impl From for InferenceError { fn from(err: tokio::task::JoinError) -> Self { Self::Join(Box::new(err)) } } /// Convenience macro for creating inference errors #[macro_export] macro_rules! inference_error { ($variant:ident, $($arg:expr),*) => { $crate::error::InferenceError::$variant { $($arg),* } }; } /// Convenience macro for creating invalid request errors #[macro_export] macro_rules! invalid_request { ($msg:expr) => { $crate::error::InferenceError::invalid_request($msg) }; ($fmt:expr, $($arg:tt)*) => { $crate::error::InferenceError::invalid_request(format!($fmt, $($arg)*)) }; } #[cfg(test)] mod tests { use super::*; #[test] fn test_error_creation() { let error = InferenceError::invalid_request("test message"); assert!(matches!(error, InferenceError::InvalidRequest { .. })); } #[test] fn test_error_recoverability() { let recoverable = InferenceError::queue_full(10, 10); assert!(recoverable.is_recoverable()); let unrecoverable = InferenceError::internal_error("test", "reason"); assert!(!unrecoverable.is_recoverable()); } #[test] fn test_error_severity() { let low = InferenceError::invalid_request("test"); assert_eq!(low.severity(), ErrorSeverity::Low); let critical = InferenceError::internal_error("test", "reason"); assert_eq!(critical.severity(), ErrorSeverity::Critical); } #[test] fn test_retry_delay() { let no_retry = InferenceError::invalid_request("test"); assert_eq!(no_retry.retry_delay(), None); let quick_retry = InferenceError::kv_cache_error("read", "timeout"); assert!(quick_retry.retry_delay().unwrap() > Duration::ZERO); } }