fix(production): wire real inference path through engine, serving, and streaming

- rtx-inference: sample_next_token now copies the actual logits from the
  forward pass (Tensor::to_vec, last-token slice) instead of sampling
  from a fabricated all-zero vector; request metrics report measured
  queue/processing times instead of hardcoded constants.
- rtx-serving-api: depends on rtx-inference; /v1/completions dispatches
  to a shared InferenceEngine (byte-level tokenization until a real
  tokenizer is threaded through) and returns 503 when no engine is
  loaded instead of mock text. ServingServer::with_engine attaches one.
- rtx-streaming: depends on rtx-inference; generate_tokens delegates to
  an attached backend engine and errors without one instead of emitting
  "token_N" placeholders; tokenization is byte-level, not position-mod.
- speculative decoding: document the orchestration (speculative/) vs
  implementation (medusa.rs/lookahead.rs) layering; CLAUDE.md no longer
  claims a standalone rtx-speculative-decoding crate.

Co-Authored-By: Claude Fable 5 <[email protected]>

crates/production/rtx-inference/src/engine/inference.rs

@@ -7,7 +7,7 @@ use rtx_tensor::{Device, Tensor};
 use tokio_stream::Stream;
 use tracing::debug;
 
-use crate::{InferenceRequest, InferenceResult};
+use crate::{InferenceError, InferenceRequest, InferenceResult};
 
 use super::forward_pass::ForwardPass;
 use super::model::LoadedModel;
@@ -126,9 +126,16 @@ impl TokenGenerator {
         top_k: u32,
     ) -> InferenceResult<u32> {
         // Extract logits for the last token (assuming batch_size = 1)
-        // to_vec1 doesn't exist, create workaround to get data as Vec<f32>
-        let vocab_size = logits.shape().dims().last().copied().unwrap_or(50000);
-        let logits_data = vec![0.0f32; vocab_size]; // Placeholder - in production would copy from GPU
+        let dims = logits.shape().dims().to_vec();
+        let vocab_size = *dims.last().ok_or_else(|| InferenceError::InvalidRequest {
+            message: "logits tensor has no dimensions".to_string(),
+        })?;
+        let all_logits = logits.to_vec().map_err(|e| InferenceError::InvalidRequest {
+            message: format!("failed to copy logits to host: {e}"),
+        })?;
+        // For [batch, seq, vocab] (or [seq, vocab]) take the final vocab_size slice:
+        // the last token's logits.
+        let logits_data: Vec<f32> = all_logits[all_logits.len().saturating_sub(vocab_size)..].to_vec();
 
         // Apply temperature scaling
         let scaled_logits: Vec<f32> = if temperature != 1.0 && temperature > 0.0 {

crates/production/rtx-inference/src/engine/mod.rs

@@ -282,9 +282,11 @@ impl InferenceEngine {
             let mut request_manager = self.request_manager.lock().await;
             request_manager.submit_request(request.clone()).await?;
         }
+        let queue_time = start_time.elapsed();
 
         // Get model for inference
         let model = self.get_loaded_model(model_name).await?;
+        let processing_time = start_time.elapsed().saturating_sub(queue_time);
 
         // Process the request
         let output_tokens =
@@ -299,18 +301,18 @@ impl InferenceEngine {
 
         // Create metrics
         let metrics = RequestMetrics {
-            queue_time: Duration::from_millis(1),      // Simplified
-            processing_time: Duration::from_millis(5), // Simplified
-            generation_time: total_time
-                .checked_sub(Duration::from_millis(6))
-                .unwrap_or(Duration::ZERO),
+            queue_time,
+            processing_time,
+            generation_time: total_time.saturating_sub(queue_time + processing_time),
             total_time,
             input_token_count: request.input_tokens.len(),
             output_token_count: output_tokens.len(),
             tokens_per_second: output_tokens.len() as f64 / total_time.as_secs_f64(),
             peak_memory_bytes: TokenGenerator::estimate_memory_usage(&request),
-            kv_cache_hits: 0,   // Simplified
-            kv_cache_misses: 0, // Simplified
+            // KV-cache instrumentation is not threaded through this path yet;
+            // 0 means "not measured", not "no hits".
+            kv_cache_hits: 0,
+            kv_cache_misses: 0,
         };
 
         // Update global metrics