osobhandClaude Fable 5
64ade03ab9
- rtx-inference: sample_next_token now copies the actual logits from the forward pass (Tensor::to_vec, last-token slice) instead of sampling from a fabricated all-zero vector; request metrics report measured queue/processing times instead of hardcoded constants. - rtx-serving-api: depends on rtx-inference; /v1/completions dispatches to a shared InferenceEngine (byte-level tokenization until a real tokenizer is threaded through) and returns 503 when no engine is loaded instead of mock text. ServingServer::with_engine attaches one. - rtx-streaming: depends on rtx-inference; generate_tokens delegates to an attached backend engine and errors without one instead of emitting "token_N" placeholders; tokenization is byte-level, not position-mod. - speculative decoding: document the orchestration (speculative/) vs implementation (medusa.rs/lookahead.rs) layering; CLAUDE.md no longer claims a standalone rtx-speculative-decoding crate. Co-Authored-By: Claude Fable 5 <[email protected]>
107 lines
2.2 KiB
TOML
107 lines
2.2 KiB
TOML
[package]
|
|
name = "rtx-serving-api"
|
|
version = "1.0.0"
|
|
edition.workspace = true
|
|
rust-version = "1.92"
|
|
authors = ["Claude AI Assistant"]
|
|
license = "MIT"
|
|
repository = "https://github.com/rustytorch/rustytorch"
|
|
description = "HTTP/gRPC serving layer for rtx-inference runtime with REST APIs and streaming support"
|
|
|
|
[dependencies]
|
|
rtx-inference = { path = "../rtx-inference" }
|
|
|
|
# HTTP server framework
|
|
axum = { version = "0.7", features = ["http2", "ws", "tokio"] }
|
|
tower = { version = "0.4", features = ["full"] }
|
|
tower-http = { version = "0.5", features = ["cors", "trace", "timeout"] }
|
|
hyper = { version = "1.0", features = ["full"] }
|
|
|
|
# Async runtime
|
|
tokio = { version = "1.0", features = ["full", "rt-multi-thread"] }
|
|
async-trait = "0.1"
|
|
futures-util = "0.3"
|
|
futures = "0.3"
|
|
|
|
# Serialization for JSON APIs
|
|
serde = { version = "1.0", features = ["derive"] }
|
|
serde_json = "1.0"
|
|
|
|
# Error handling
|
|
anyhow = "1.0"
|
|
thiserror = "2"
|
|
|
|
# HTTP utilities
|
|
mime = "0.3"
|
|
headers = "0.4"
|
|
|
|
# Logging and telemetry
|
|
tracing = "0.1"
|
|
tracing-subscriber = "0.3"
|
|
|
|
# UUID for request tracking
|
|
uuid = { version = "1.0", features = ["v4", "serde"] }
|
|
|
|
# Time utilities
|
|
chrono = { version = "0.4", features = ["serde"] }
|
|
|
|
# Context caching dependencies
|
|
lru = "0.12"
|
|
dashmap = "5.5"
|
|
parking_lot = "0.12"
|
|
memmap2 = "0.9"
|
|
bincode = "1.3"
|
|
zstd = "0.13"
|
|
crossbeam = "0.8"
|
|
rayon = "1.8"
|
|
|
|
# Configuration and serialization
|
|
toml = "0.8"
|
|
serde_yaml = "0.9"
|
|
num_cpus = "1.16"
|
|
|
|
# Metrics and monitoring
|
|
prometheus = "0.13"
|
|
histogram = "0.6"
|
|
|
|
# gRPC support
|
|
tonic = "0.13"
|
|
prost = "0.13"
|
|
tokio-stream = "0.1"
|
|
|
|
# WebSocket support
|
|
tokio-tungstenite = "0.21"
|
|
|
|
# HTTP body utilities
|
|
bytes = "1.6"
|
|
|
|
# Advanced LLM features dependencies
|
|
regex = "1.10"
|
|
rand = "0.8"
|
|
async-stream = "0.3"
|
|
|
|
# Additional serialization (zstd already included above)
|
|
|
|
[dev-dependencies]
|
|
tokio-test = "0.4"
|
|
axum-test = "14.0"
|
|
proptest = "1.4"
|
|
criterion = { version = "0.5", features = ["html_reports"] }
|
|
tempfile = "3.8"
|
|
futures = "0.3"
|
|
|
|
[[bench]]
|
|
name = "cache_performance"
|
|
harness = false
|
|
|
|
[[bin]]
|
|
name = "rtx-serving-api"
|
|
path = "src/main.rs"
|
|
|
|
[features]
|
|
default = []
|
|
grpc = [] # Future gRPC support
|
|
metrics = ["tracing-subscriber/env-filter"]
|
|
[lints]
|
|
workspace = true
|