//! Opt-in tests against real inference endpoints. Activated with //! `CM_LIVE_LLM=1` plus `ANTHROPIC_API_KEY` (Anthropic) or //! `CM_OPENAI_COMPAT_URL` (e.g. a local Ollama at //! `http://127.0.0.1:11434/v1` with `CM_OPENAI_COMPAT_MODEL` set). //! CI runs these in a dedicated credentialed job; the default suite uses //! the scripted provider, which exercises the identical seam. use cm_llm::{ AnthropicProvider, ChatMessage, ChatRequest, ChatRole, ContentPart, LlmEvent, LlmProvider, OpenAiCompatProvider, }; use futures::StreamExt; fn live_enabled() -> bool { std::env::var("CM_LIVE_LLM").as_deref() == Ok("1") } fn simple_request(model: &str) -> ChatRequest { ChatRequest { system: "Answer in exactly one short sentence.".into(), messages: vec![ChatMessage { role: ChatRole::User, parts: vec![ContentPart::text("Say the word 'pong'.")], }], tools: vec![], model: model.into(), max_tokens: 64, web_search: false, } } async fn collect_text(provider: &dyn LlmProvider, request: ChatRequest) -> String { let mut stream = provider.stream(request).await.expect("stream opens"); let mut text = String::new(); while let Some(event) = stream.next().await { if let LlmEvent::TextDelta(t) = event.expect("stream event") { text.push_str(&t); } } text } #[tokio::test] async fn anthropic_streams_text() { if !live_enabled() { eprintln!("skipped: set CM_LIVE_LLM=1 to run"); return; } let Ok(key) = std::env::var("ANTHROPIC_API_KEY") else { eprintln!("skipped: ANTHROPIC_API_KEY not set"); return; }; let provider = AnthropicProvider::new(key); let text = collect_text(&provider, simple_request("claude-haiku-4-5-20251001")).await; assert!(text.to_lowercase().contains("pong"), "got: {text}"); } #[tokio::test] async fn openai_compat_streams_text() { if !live_enabled() { eprintln!("skipped: set CM_LIVE_LLM=1 to run"); return; } let Ok(url) = std::env::var("CM_OPENAI_COMPAT_URL") else { eprintln!("skipped: CM_OPENAI_COMPAT_URL not set"); return; }; let model = std::env::var("CM_OPENAI_COMPAT_MODEL").unwrap_or_else(|_| "qwen2.5:0.5b".into()); let provider = OpenAiCompatProvider::new(url, None); let text = collect_text(&provider, simple_request(&model)).await; assert!(text.to_lowercase().contains("pong"), "got: {text}"); } /// The plan's #1 risk, validated on the REAL wire: a tool-use turn comes /// back as a ToolUse event, the ToolResult goes back up, and the model /// completes — the provider-neutral round trip holds against Anthropic's /// actual streaming format, including usage accounting. #[tokio::test] async fn anthropic_tool_round_trip_with_usage() { if !live_enabled() { eprintln!("skipped: set CM_LIVE_LLM=1 to run"); return; } let Ok(key) = std::env::var("ANTHROPIC_API_KEY") else { eprintln!("skipped: ANTHROPIC_API_KEY not set"); return; }; let provider = AnthropicProvider::new(key); let clock_tool = cm_llm::ToolDescriptor { name: "clock.now".into(), description: "Returns the current UTC time.".into(), input_schema: serde_json::json!({"type": "object", "properties": {}}), }; let mut request = ChatRequest { system: "You have a clock tool. When asked the time you MUST call it.".into(), messages: vec![ChatMessage { role: ChatRole::User, parts: vec![ContentPart::text("What time is it right now?")], }], tools: vec![clock_tool], model: "claude-haiku-4-5-20251001".into(), max_tokens: 300, web_search: false, }; // Leg 1: the model must emit a real ToolUse with an id. let mut stream = provider .stream(request.clone()) .await .expect("stream opens"); let mut tool_use: Option<(String, String)> = None; let mut usage_seen = false; while let Some(event) = stream.next().await { match event.expect("clean event") { LlmEvent::ToolUse { id, name, .. } => tool_use = Some((id, name)), LlmEvent::Usage { input_tokens, output_tokens, } => { assert!(input_tokens > 0 && output_tokens > 0); usage_seen = true; } _ => {} } } let (tool_id, tool_name) = tool_use.expect("model called the tool"); assert_eq!(tool_name, "clock.now"); assert!(usage_seen, "usage must arrive on the wire"); // Leg 2: ship the ToolResult back exactly as the runtime checkpoint // would after a suspension — the reassembly the §15 path depends on. request.messages.push(ChatMessage { role: ChatRole::Assistant, parts: vec![ContentPart::ToolUse { id: tool_id.clone(), name: tool_name, input: serde_json::json!({}), }], }); request.messages.push(ChatMessage { role: ChatRole::User, parts: vec![ContentPart::ToolResult { tool_use_id: tool_id, content: serde_json::json!({"utc": "2026-06-10T17:00:00Z"}), }], }); let text = collect_text(&provider, request).await; assert!( text.contains("17:00") || text.to_lowercase().contains("5"), "model used the tool result: {text}" ); }