//! Generic LLM client abstraction for the Rust Unmute conversational stack. //! //! Provides a uniform [`LlmClient`] trait with `generate_stream` returning //! a stream of text tokens. Implementations: //! //! - [`OpenAiCompatibleClient`] — works with OpenAI's Chat Completions //! API and any compatible endpoint (Z.AI, vLLM, llama.cpp's HTTP server, //! LiteLLM, etc.). Streams via Server-Sent Events. //! //! This is the bridge layer in the STT → LLM → TTS conversational pipeline. //! Token-level streaming is critical: TTS can start speaking the assistant //! response as soon as the first token arrives, instead of waiting for the //! full LLM completion. //! //! ## Example //! //! ```no_run //! use rtx_csm::llm_client::{ChatMessage, GenConfig, LlmClient, OpenAiCompatibleClient, Role}; //! use futures_util::StreamExt; //! //! # async fn run() -> anyhow::Result<()> { //! let client = OpenAiCompatibleClient::new( //! "https://api.openai.com/v1", //! std::env::var("OPENAI_API_KEY")?, //! "gpt-4o-mini", //! ); //! let messages = vec![ //! ChatMessage::system("You are a concise assistant."), //! ChatMessage::user("Say hello in one word."), //! ]; //! let mut stream = client.generate_stream(messages, GenConfig::default()).await?; //! while let Some(tok) = stream.next().await { //! print!("{}", tok?); //! } //! # Ok(()) } //! ``` use crate::error::{CsmError, Result}; use async_trait::async_trait; use eventsource_stream::Eventsource; use futures_util::stream::{Stream, StreamExt}; use serde::{Deserialize, Serialize}; use std::pin::Pin; /// Boxed stream of text tokens. Each token is one chunk of the assistant /// response — typically a sub-word from the LLM tokenizer. Concatenate to /// rebuild the full message; pipe to TTS as they arrive for streaming UX. pub type TokenStream = Pin> + Send>>; #[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq)] #[serde(rename_all = "lowercase")] pub enum Role { System, User, Assistant, } #[derive(Debug, Clone, Serialize, Deserialize)] pub struct ChatMessage { pub role: Role, pub content: String, } impl ChatMessage { pub fn system(content: impl Into) -> Self { Self { role: Role::System, content: content.into(), } } pub fn user(content: impl Into) -> Self { Self { role: Role::User, content: content.into(), } } pub fn assistant(content: impl Into) -> Self { Self { role: Role::Assistant, content: content.into(), } } } #[derive(Debug, Clone)] pub struct GenConfig { /// Maximum tokens in the response. pub max_tokens: Option, /// Sampling temperature; 0.0 = greedy. pub temperature: f32, /// Top-p nucleus cutoff. 1.0 = disabled. pub top_p: f32, /// Optional stop sequences. pub stop: Vec, /// Provider-specific extra fields merged into the JSON request body /// at top level. Used for things the OpenAI schema doesn't expose: /// /// - **Z.AI thinking-disabled** (skips reasoning_content for /// voice-AI use): `{"thinking": {"type": "disabled"}}`. Without /// this, glm-4.5/4.6/4.7 burn most of `max_tokens` on /// reasoning_content and emit empty `content`. /// - **Anthropic thinking budget** (when proxied through an /// OpenAI-compatible Anthropic shim): `{"thinking": {"type": /// "enabled", "budget_tokens": 1024}}`. /// - **vLLM guided decoding**: `{"guided_json": {...}}`. pub extra_body: serde_json::Map, } impl Default for GenConfig { fn default() -> Self { Self { max_tokens: Some(512), temperature: 0.7, top_p: 1.0, stop: Vec::new(), extra_body: serde_json::Map::new(), } } } #[async_trait] pub trait LlmClient: Send + Sync { /// Generate a streaming response. The returned stream yields text /// chunks as the model produces them. Each chunk is typically one or /// a few tokens; concatenate for the full response. async fn generate_stream( &self, messages: Vec, config: GenConfig, ) -> Result; /// Convenience: collect the full response. Default impl folds the /// stream into a single String. Implementations are free to override /// for non-streaming endpoints. async fn generate(&self, messages: Vec, config: GenConfig) -> Result { let mut stream = self.generate_stream(messages, config).await?; let mut out = String::new(); while let Some(chunk) = stream.next().await { out.push_str(&chunk?); } Ok(out) } } // -- OpenAI-compatible Chat Completions impl ------------------------------ /// HTTP client for OpenAI-compatible Chat Completions endpoints. /// Tested patterns: /// - OpenAI: base="https://api.openai.com/v1", model="gpt-4o-mini" /// - Z.AI: base="https://api.z.ai/api/coding/paas/v4", model="glm-4.6" /// - vLLM: base="http://localhost:8000/v1", model="" /// - llama.cpp: base="http://localhost:8080/v1", model="" #[derive(Debug, Clone)] pub struct OpenAiCompatibleClient { base_url: String, api_key: String, model: String, http: reqwest::Client, } impl OpenAiCompatibleClient { pub fn new( base_url: impl Into, api_key: impl Into, model: impl Into, ) -> Self { Self { base_url: base_url.into(), api_key: api_key.into(), model: model.into(), http: reqwest::Client::builder() .timeout(std::time::Duration::from_secs(120)) .build() .expect("reqwest client init"), } } } // -- Wire types: OpenAI Chat Completions request/response ----------------- #[derive(Serialize)] struct ChatRequest<'a> { model: &'a str, messages: &'a [ChatMessage], stream: bool, #[serde(skip_serializing_if = "Option::is_none")] max_tokens: Option, temperature: f32, top_p: f32, #[serde(skip_serializing_if = "Vec::is_empty")] stop: Vec, } #[derive(Deserialize)] struct ChatStreamEvent { choices: Vec, } #[derive(Deserialize)] struct ChatStreamChoice { delta: ChatStreamDelta, /// Captured for completeness; OpenAI sets to `"stop"`/`"length"` on /// the last event. We don't currently surface it to callers. #[serde(default)] #[allow(dead_code)] finish_reason: Option, } #[derive(Deserialize, Default)] struct ChatStreamDelta { #[serde(default)] content: Option, } #[async_trait] impl LlmClient for OpenAiCompatibleClient { async fn generate_stream( &self, messages: Vec, config: GenConfig, ) -> Result { let url = format!("{}/chat/completions", self.base_url.trim_end_matches('/')); let req = ChatRequest { model: &self.model, messages: &messages, stream: true, max_tokens: config.max_tokens, temperature: config.temperature, top_p: config.top_p, stop: config.stop.clone(), }; // Serialize the typed request, then merge in extra_body at the // top level so provider-specific fields (e.g., Z.AI's `thinking`) // ride alongside the OpenAI schema. extra_body is empty in the // default case so this is a no-op. let mut body = serde_json::to_value(&req) .map_err(|e| CsmError::Config(format!("LLM serialize: {e}")))?; if !config.extra_body.is_empty() && let Some(obj) = body.as_object_mut() { for (k, v) in config.extra_body.iter() { obj.insert(k.clone(), v.clone()); } } let response = self .http .post(&url) .bearer_auth(&self.api_key) .header("accept", "text/event-stream") .json(&body) .send() .await .map_err(|e| CsmError::Config(format!("LLM request: {e}")))?; if !response.status().is_success() { let status = response.status(); let body = response.text().await.unwrap_or_default(); return Err(CsmError::Config(format!("LLM HTTP {status}: {body}"))); } // SSE stream: each event has data: \n\n. Final event is data: [DONE]. let bytes_stream = response.bytes_stream(); let event_stream = bytes_stream.eventsource(); let token_stream = event_stream.filter_map(|ev| async move { match ev { Err(e) => Some(Err(CsmError::Config(format!("SSE: {e}")))), Ok(event) => { let data = event.data; if data.trim() == "[DONE]" { None } else { match serde_json::from_str::(&data) { Ok(parsed) => parsed .choices .into_iter() .next() .and_then(|c| c.delta.content) .filter(|s| !s.is_empty()) .map(Ok), Err(e) => { Some(Err(CsmError::Config(format!("SSE parse: {e} body={data}")))) } } } } } }); Ok(Box::pin(token_stream)) } } #[cfg(test)] mod tests { use super::*; #[test] fn chat_message_constructors() { assert_eq!(ChatMessage::system("s").role, Role::System); assert_eq!(ChatMessage::user("u").role, Role::User); assert_eq!(ChatMessage::assistant("a").role, Role::Assistant); } #[test] fn role_serializes_lowercase() { let m = ChatMessage::user("hi"); let json = serde_json::to_string(&m).unwrap(); assert!(json.contains("\"role\":\"user\""), "got {json}"); } #[test] fn gen_config_defaults_sane() { let c = GenConfig::default(); assert_eq!(c.max_tokens, Some(512)); assert!((c.temperature - 0.7).abs() < 1e-6); assert_eq!(c.top_p, 1.0); assert!(c.stop.is_empty()); } }