fix(production): wire real inference path through engine, serving, and streaming
- rtx-inference: sample_next_token now copies the actual logits from the forward pass (Tensor::to_vec, last-token slice) instead of sampling from a fabricated all-zero vector; request metrics report measured queue/processing times instead of hardcoded constants. - rtx-serving-api: depends on rtx-inference; /v1/completions dispatches to a shared InferenceEngine (byte-level tokenization until a real tokenizer is threaded through) and returns 503 when no engine is loaded instead of mock text. ServingServer::with_engine attaches one. - rtx-streaming: depends on rtx-inference; generate_tokens delegates to an attached backend engine and errors without one instead of emitting "token_N" placeholders; tokenization is byte-level, not position-mod. - speculative decoding: document the orchestration (speculative/) vs implementation (medusa.rs/lookahead.rs) layering; CLAUDE.md no longer claims a standalone rtx-speculative-decoding crate. Co-Authored-By: Claude Fable 5 <[email protected]>
This commit is contained in:
@@ -34,13 +34,31 @@ impl Default for ServerConfig {
|
||||
/// Serving server
|
||||
pub struct ServingServer {
|
||||
config: ServerConfig,
|
||||
state: inference::AppState,
|
||||
}
|
||||
|
||||
impl ServingServer {
|
||||
/// Create a new serving server
|
||||
/// Create a new serving server (no engine loaded; inference returns 503)
|
||||
#[must_use]
|
||||
pub fn new(config: ServerConfig) -> Self {
|
||||
Self { config }
|
||||
Self {
|
||||
config,
|
||||
state: inference::AppState::default(),
|
||||
}
|
||||
}
|
||||
|
||||
/// Create a serving server backed by a live inference engine
|
||||
#[must_use]
|
||||
pub fn with_engine(
|
||||
config: ServerConfig,
|
||||
engine: std::sync::Arc<tokio::sync::RwLock<rtx_inference::InferenceEngine>>,
|
||||
) -> Self {
|
||||
Self {
|
||||
config,
|
||||
state: inference::AppState {
|
||||
engine: Some(engine),
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
/// Build the Axum router with all routes
|
||||
@@ -68,6 +86,7 @@ impl ServingServer {
|
||||
.layer(TimeoutLayer::new(Duration::from_secs(
|
||||
self.config.timeout_seconds,
|
||||
)))
|
||||
.with_state(self.state.clone())
|
||||
}
|
||||
|
||||
/// Start the server
|
||||
|
||||
Reference in New Issue
Block a user