clawmates/crates/cm-llm/tests/live.rs

//! Opt-in tests against real inference endpoints. Activated with
//! `CM_LIVE_LLM=1` plus `ANTHROPIC_API_KEY` (Anthropic) or
//! `CM_OPENAI_COMPAT_URL` (e.g. a local Ollama at
//! `http://127.0.0.1:11434/v1` with `CM_OPENAI_COMPAT_MODEL` set).
//! CI runs these in a dedicated credentialed job; the default suite uses
//! the scripted provider, which exercises the identical seam.

use cm_llm::{
    AnthropicProvider, ChatMessage, ChatRequest, ChatRole, ContentPart, LlmEvent, LlmProvider,
    OpenAiCompatProvider,
};
use futures::StreamExt;

fn live_enabled() -> bool {
    std::env::var("CM_LIVE_LLM").as_deref() == Ok("1")
}

fn simple_request(model: &str) -> ChatRequest {
    ChatRequest {
        system: "Answer in exactly one short sentence.".into(),
        messages: vec![ChatMessage {
            role: ChatRole::User,
            parts: vec![ContentPart::text("Say the word 'pong'.")],
        }],
        tools: vec![],
        model: model.into(),
        max_tokens: 64,
        web_search: false,
    }
}

async fn collect_text(provider: &dyn LlmProvider, request: ChatRequest) -> String {
    let mut stream = provider.stream(request).await.expect("stream opens");
    let mut text = String::new();
    while let Some(event) = stream.next().await {
        if let LlmEvent::TextDelta(t) = event.expect("stream event") {
            text.push_str(&t);
        }
    }
    text
}

#[tokio::test]
async fn anthropic_streams_text() {
    if !live_enabled() {
        eprintln!("skipped: set CM_LIVE_LLM=1 to run");
        return;
    }
    let Ok(key) = std::env::var("ANTHROPIC_API_KEY") else {
        eprintln!("skipped: ANTHROPIC_API_KEY not set");
        return;
    };
    let provider = AnthropicProvider::new(key);
    let text = collect_text(&provider, simple_request("claude-haiku-4-5-20251001")).await;
    assert!(text.to_lowercase().contains("pong"), "got: {text}");
}

#[tokio::test]
async fn openai_compat_streams_text() {
    if !live_enabled() {
        eprintln!("skipped: set CM_LIVE_LLM=1 to run");
        return;
    }
    let Ok(url) = std::env::var("CM_OPENAI_COMPAT_URL") else {
        eprintln!("skipped: CM_OPENAI_COMPAT_URL not set");
        return;
    };
    let model = std::env::var("CM_OPENAI_COMPAT_MODEL").unwrap_or_else(|_| "qwen2.5:0.5b".into());
    let provider = OpenAiCompatProvider::new(url, None);
    let text = collect_text(&provider, simple_request(&model)).await;
    assert!(text.to_lowercase().contains("pong"), "got: {text}");
}

/// The plan's #1 risk, validated on the REAL wire: a tool-use turn comes
/// back as a ToolUse event, the ToolResult goes back up, and the model
/// completes — the provider-neutral round trip holds against Anthropic's
/// actual streaming format, including usage accounting.
#[tokio::test]
async fn anthropic_tool_round_trip_with_usage() {
    if !live_enabled() {
        eprintln!("skipped: set CM_LIVE_LLM=1 to run");
        return;
    }
    let Ok(key) = std::env::var("ANTHROPIC_API_KEY") else {
        eprintln!("skipped: ANTHROPIC_API_KEY not set");
        return;
    };
    let provider = AnthropicProvider::new(key);
    let clock_tool = cm_llm::ToolDescriptor {
        name: "clock.now".into(),
        description: "Returns the current UTC time.".into(),
        input_schema: serde_json::json!({"type": "object", "properties": {}}),
    };
    let mut request = ChatRequest {
        system: "You have a clock tool. When asked the time you MUST call it.".into(),
        messages: vec![ChatMessage {
            role: ChatRole::User,
            parts: vec![ContentPart::text("What time is it right now?")],
        }],
        tools: vec![clock_tool],
        model: "claude-haiku-4-5-20251001".into(),
        max_tokens: 300,
        web_search: false,
    };

    // Leg 1: the model must emit a real ToolUse with an id.
    let mut stream = provider
        .stream(request.clone())
        .await
        .expect("stream opens");
    let mut tool_use: Option<(String, String)> = None;
    let mut usage_seen = false;
    while let Some(event) = stream.next().await {
        match event.expect("clean event") {
            LlmEvent::ToolUse { id, name, .. } => tool_use = Some((id, name)),
            LlmEvent::Usage {
                input_tokens,
                output_tokens,
            } => {
                assert!(input_tokens > 0 && output_tokens > 0);
                usage_seen = true;
            }
            _ => {}
        }
    }
    let (tool_id, tool_name) = tool_use.expect("model called the tool");
    assert_eq!(tool_name, "clock.now");
    assert!(usage_seen, "usage must arrive on the wire");

    // Leg 2: ship the ToolResult back exactly as the runtime checkpoint
    // would after a suspension — the reassembly the §15 path depends on.
    request.messages.push(ChatMessage {
        role: ChatRole::Assistant,
        parts: vec![ContentPart::ToolUse {
            id: tool_id.clone(),
            name: tool_name,
            input: serde_json::json!({}),
        }],
    });
    request.messages.push(ChatMessage {
        role: ChatRole::User,
        parts: vec![ContentPart::ToolResult {
            tool_use_id: tool_id,
            content: serde_json::json!({"utc": "2026-06-10T17:00:00Z"}),
        }],
    });
    let text = collect_text(&provider, request).await;
    assert!(
        text.contains("17:00") || text.to_lowercase().contains("5"),
        "model used the tool result: {text}"
    );
}