82 lines
2.6 KiB
Rust
82 lines
2.6 KiB
Rust
//! `RustyTorch`++ Serving API
|
|
//!
|
|
//! HTTP/gRPC serving layer for the rtx-inference runtime providing:
|
|
//! - `RESTful` API endpoints for model inference
|
|
//! - Health check endpoint for service monitoring
|
|
//! - Model listing endpoint for available models
|
|
//! - Inference endpoint with streaming support
|
|
//! - Integration with rtx-inference crate
|
|
//!
|
|
//! Built with Axum for high-performance async HTTP handling.
|
|
|
|
#![deny(clippy::unwrap_used)]
|
|
#![cfg_attr(test, allow(clippy::unwrap_used))]
|
|
#![allow(clippy::module_name_repetitions)]
|
|
#![allow(clippy::too_many_lines)]
|
|
|
|
pub mod advanced_server;
|
|
pub mod billing;
|
|
pub mod cache;
|
|
pub mod continuous_batch;
|
|
pub mod error;
|
|
pub mod grammar_sampling;
|
|
pub mod grpc;
|
|
pub mod health;
|
|
pub mod inference;
|
|
pub mod inference_cached;
|
|
pub mod load_balancer;
|
|
pub mod models;
|
|
pub mod multi_model;
|
|
pub mod queue_management;
|
|
pub mod rate_limiting;
|
|
pub mod resilience;
|
|
pub mod sampling_strategies;
|
|
pub mod server;
|
|
pub mod streaming;
|
|
pub mod structured_generation;
|
|
pub mod validation;
|
|
pub mod websocket;
|
|
|
|
// Re-export key types
|
|
pub use error::{ApiError, ApiResult};
|
|
pub use health::HealthStatus;
|
|
pub use server::{ServerConfig, ServingServer};
|
|
|
|
// Re-export advanced LLM features
|
|
pub use advanced_server::{AdvancedServerConfig, AdvancedServingServer};
|
|
pub use billing::{BillingManager, BudgetConfig, CostBreakdown};
|
|
pub use continuous_batch::{
|
|
ActiveBatch, BatchRequest, BatchingError, BatchingStats, ContinuousBatchingConfig,
|
|
ContinuousBatchingController, FinishReason, Priority, RequestState,
|
|
};
|
|
pub use grammar_sampling::{BnfParser, ContextFreeGrammar, GrammarGuidedSampler};
|
|
pub use multi_model::{ModelConfig, ModelType, MultiModelManager};
|
|
pub use queue_management::{QueueManager, QueuedRequest, RequestPriority};
|
|
pub use rate_limiting::{RateLimitManager, TokenCost, UserTier};
|
|
pub use resilience::{
|
|
CircuitBreaker, CircuitBreakerConfig, CircuitBreakerError, CircuitBreakerStats, CircuitState,
|
|
ResilienceError, ResilienceHandler, RetryConfig, RetryError, RetryWithBackoff, TimeoutError,
|
|
with_timeout,
|
|
};
|
|
pub use sampling_strategies::{AdvancedSampler, SamplingConfig, SamplingStrategy};
|
|
pub use streaming::{StreamConfig, StreamEvent, StreamManager, StreamType};
|
|
pub use structured_generation::{
|
|
JsonSchema, OutputFormat, StructuredGenerationManager, ValidationResult,
|
|
};
|
|
pub use validation::{
|
|
InputValidator, ValidationConfig, ValidationError, ValidationResult as InputValidationResult,
|
|
};
|
|
|
|
/// Version information
|
|
pub const VERSION: &str = env!("CARGO_PKG_VERSION");
|
|
|
|
#[cfg(test)]
|
|
mod tests {
|
|
use super::*;
|
|
|
|
#[test]
|
|
fn test_version() {
|
|
assert!(!VERSION.is_empty());
|
|
}
|
|
}
|