Live Anthropic validation — and the platform-breaking bug it caught

Running the opt-in live suite (CM_LIVE_LLM=1 + ANTHROPIC_API_KEY) against
the real API immediately surfaced a launch blocker: Anthropic (and
OpenAI) restrict tool names to ^[a-zA-Z0-9_-]{1,128}$ — our ENTIRE
registry uses dotted names (clock.now, email.send, shell.exec, ...).
The scripted provider never enforced the pattern, so every real-model
deployment would have 400'd on the first tool call.

- Fix at the provider boundary, where it belongs: wire_tool_name /
  internal_tool_name codec (dots <-> __) applied in BOTH HTTP providers
  at all three sites (tools list, assistant tool_use echo, inbound
  tool_use decode). Internal naming (DB step rows, scenarios, UI traces)
  unchanged. Offline unit test round-trips every registry name through
  the wire pattern
- New live tests, all passing against api.anthropic.com (Haiku 4.5):
  - provider tool ROUND TRIP: real ToolUse arrives, ToolResult ships
    back exactly as a checkpoint would reassemble it, model completes,
    real usage events on the wire
  - full runtime loop: real model calls clock.now, run completes, REAL
    token usage metered, credits decremented
  - the #1-risk validation: a real model's email.send intercepted ->
    suspended -> approved -> checkpoint RESUMED against the live API ->
    completed -> outbox exactly 1 (checkpoint/resume fidelity end to end)
- Stray TC_OPENAI_COMPAT_* envs renamed to CM_OPENAI_COMPAT_*

No credentials stored anywhere; the key was passed via env only.

163 Rust tests (+5 live, key-gated).

Co-Authored-By: Claude Fable 5 <[email protected]>

crates/cm-llm/src/anthropic.rs

@@ -44,7 +44,8 @@ impl AnthropicProvider {
                     .map(|part| match part {
                         ContentPart::Text { text } => json!({"type": "text", "text": text}),
                         ContentPart::ToolUse { id, name, input } => {
-                            json!({"type": "tool_use", "id": id, "name": name, "input": input})
+                            json!({"type": "tool_use", "id": id,
+                                   "name": crate::wire_tool_name(name), "input": input})
                         }
                         ContentPart::ToolResult {
                             tool_use_id,
@@ -78,7 +79,7 @@ impl LlmProvider for AnthropicProvider {
             .iter()
             .map(|t| {
                 json!({
-                    "name": t.name,
+                    "name": crate::wire_tool_name(&t.name),
                     "description": t.description,
                     "input_schema": t.input_schema,
                 })
@@ -124,7 +125,9 @@ impl LlmProvider for AnthropicProvider {
                         if block["type"] == "tool_use" {
                             pending_tool = Some((
                                 block["id"].as_str().unwrap_or_default().to_owned(),
-                                block["name"].as_str().unwrap_or_default().to_owned(),
+                                crate::internal_tool_name(
+                                    block["name"].as_str().unwrap_or_default(),
+                                ),
                                 String::new(),
                             ));
                         }

crates/cm-llm/src/lib.rs

@@ -18,7 +18,7 @@ mod scripted;
 pub use anthropic::AnthropicProvider;
 pub use openai_compat::OpenAiCompatProvider;
 pub use provider::{
-    ChatMessage, ChatRequest, ChatRole, ContentPart, EventStream, LlmError, LlmEvent, LlmProvider,
+    internal_tool_name, wire_tool_name, ChatMessage, ChatRequest, ChatRole, ContentPart,
-    StopReason, ToolDescriptor,
+    EventStream, LlmError, LlmEvent, LlmProvider, StopReason, ToolDescriptor,
 };
 pub use scripted::ScriptedProvider;

crates/cm-llm/src/openai_compat.rs

@@ -39,7 +39,8 @@ impl OpenAiCompatProvider {
                     ContentPart::ToolUse { id, name, input } => tool_calls.push(json!({
                         "id": id,
                         "type": "function",
-                        "function": {"name": name, "arguments": input.to_string()},
+                        "function": {"name": crate::wire_tool_name(name),
+                                     "arguments": input.to_string()},
                     })),
                     ContentPart::ToolResult {
                         tool_use_id,
@@ -85,7 +86,7 @@ impl LlmProvider for OpenAiCompatProvider {
                 json!({
                     "type": "function",
                     "function": {
-                        "name": t.name,
+                        "name": crate::wire_tool_name(&t.name),
                         "description": t.description,
                         "parameters": t.input_schema,
                     },
@@ -168,6 +169,7 @@ impl LlmProvider for OpenAiCompatProvider {
                 }
             }
             for (id, name, args) in pending.drain(..) {
+                let name = crate::internal_tool_name(&name);
                 if name.is_empty() {
                     continue;
                 }

crates/cm-llm/src/provider.rs

@@ -32,6 +32,20 @@ pub enum ContentPart {
     },
 }
 
+/// Anthropic and OpenAI both restrict tool names to
+/// `^[a-zA-Z0-9_-]{1,128}$`; our registry uses dotted names
+/// (`clock.now`, `email.send`). The HTTP providers encode dots as `__`
+/// on the wire and decode on receipt — internal naming (DB step rows,
+/// scenarios, UI traces) never changes. No registry tool may contain
+/// a literal `__`.
+pub fn wire_tool_name(internal: &str) -> String {
+    internal.replace('.', "__")
+}
+
+pub fn internal_tool_name(wire: &str) -> String {
+    wire.replace("__", ".")
+}
+
 impl ContentPart {
     /// Convenience constructor for plain text parts.
     pub fn text(value: impl Into<String>) -> ContentPart {
@@ -109,3 +123,33 @@ pub type EventStream = BoxStream<'static, Result<LlmEvent, LlmError>>;
 pub trait LlmProvider: Send + Sync {
     async fn stream(&self, request: ChatRequest) -> Result<EventStream, LlmError>;
 }
+
+#[cfg(test)]
+mod wire_name_tests {
+    use super::*;
+
+    #[test]
+    fn dotted_registry_names_round_trip_the_wire_codec() {
+        for name in [
+            "clock.now",
+            "email.send",
+            "files.write",
+            "files.list",
+            "files.delete",
+            "routine.schedule",
+            "chat.send",
+            "chat.inbox",
+            "browser.goto",
+            "shell.exec",
+            "slack.post",
+        ] {
+            let wire = wire_tool_name(name);
+            assert!(
+                wire.chars()
+                    .all(|c| c.is_ascii_alphanumeric() || c == '_' || c == '-'),
+                "{wire} must satisfy the Anthropic/OpenAI name pattern"
+            );
+            assert_eq!(internal_tool_name(&wire), name, "lossless round trip");
+        }
+    }
+}

crates/cm-llm/tests/live.rs

@@ -1,7 +1,7 @@
 //! Opt-in tests against real inference endpoints. Activated with
 //! `CM_LIVE_LLM=1` plus `ANTHROPIC_API_KEY` (Anthropic) or
-//! `TC_OPENAI_COMPAT_URL` (e.g. a local Ollama at
+//! `CM_OPENAI_COMPAT_URL` (e.g. a local Ollama at
-//! `http://127.0.0.1:11434/v1` with `TC_OPENAI_COMPAT_MODEL` set).
+//! `http://127.0.0.1:11434/v1` with `CM_OPENAI_COMPAT_MODEL` set).
 //! CI runs these in a dedicated credentialed job; the default suite uses
 //! the scripted provider, which exercises the identical seam.
 
@@ -60,12 +60,91 @@ async fn openai_compat_streams_text() {
         eprintln!("skipped: set CM_LIVE_LLM=1 to run");
         return;
     }
-    let Ok(url) = std::env::var("TC_OPENAI_COMPAT_URL") else {
+    let Ok(url) = std::env::var("CM_OPENAI_COMPAT_URL") else {
-        eprintln!("skipped: TC_OPENAI_COMPAT_URL not set");
+        eprintln!("skipped: CM_OPENAI_COMPAT_URL not set");
         return;
     };
-    let model = std::env::var("TC_OPENAI_COMPAT_MODEL").unwrap_or_else(|_| "qwen2.5:0.5b".into());
+    let model = std::env::var("CM_OPENAI_COMPAT_MODEL").unwrap_or_else(|_| "qwen2.5:0.5b".into());
     let provider = OpenAiCompatProvider::new(url, None);
     let text = collect_text(&provider, simple_request(&model)).await;
     assert!(text.to_lowercase().contains("pong"), "got: {text}");
 }
+
+/// The plan's #1 risk, validated on the REAL wire: a tool-use turn comes
+/// back as a ToolUse event, the ToolResult goes back up, and the model
+/// completes — the provider-neutral round trip holds against Anthropic's
+/// actual streaming format, including usage accounting.
+#[tokio::test]
+async fn anthropic_tool_round_trip_with_usage() {
+    if !live_enabled() {
+        eprintln!("skipped: set CM_LIVE_LLM=1 to run");
+        return;
+    }
+    let Ok(key) = std::env::var("ANTHROPIC_API_KEY") else {
+        eprintln!("skipped: ANTHROPIC_API_KEY not set");
+        return;
+    };
+    let provider = AnthropicProvider::new(key);
+    let clock_tool = cm_llm::ToolDescriptor {
+        name: "clock.now".into(),
+        description: "Returns the current UTC time.".into(),
+        input_schema: serde_json::json!({"type": "object", "properties": {}}),
+    };
+    let mut request = ChatRequest {
+        system: "You have a clock tool. When asked the time you MUST call it.".into(),
+        messages: vec![ChatMessage {
+            role: ChatRole::User,
+            parts: vec![ContentPart::text("What time is it right now?")],
+        }],
+        tools: vec![clock_tool],
+        model: "claude-haiku-4-5-20251001".into(),
+        max_tokens: 300,
+    };
+
+    // Leg 1: the model must emit a real ToolUse with an id.
+    let mut stream = provider
+        .stream(request.clone())
+        .await
+        .expect("stream opens");
+    let mut tool_use: Option<(String, String)> = None;
+    let mut usage_seen = false;
+    while let Some(event) = stream.next().await {
+        match event.expect("clean event") {
+            LlmEvent::ToolUse { id, name, .. } => tool_use = Some((id, name)),
+            LlmEvent::Usage {
+                input_tokens,
+                output_tokens,
+            } => {
+                assert!(input_tokens > 0 && output_tokens > 0);
+                usage_seen = true;
+            }
+            _ => {}
+        }
+    }
+    let (tool_id, tool_name) = tool_use.expect("model called the tool");
+    assert_eq!(tool_name, "clock.now");
+    assert!(usage_seen, "usage must arrive on the wire");
+
+    // Leg 2: ship the ToolResult back exactly as the runtime checkpoint
+    // would after a suspension — the reassembly the §15 path depends on.
+    request.messages.push(ChatMessage {
+        role: ChatRole::Assistant,
+        parts: vec![ContentPart::ToolUse {
+            id: tool_id.clone(),
+            name: tool_name,
+            input: serde_json::json!({}),
+        }],
+    });
+    request.messages.push(ChatMessage {
+        role: ChatRole::User,
+        parts: vec![ContentPart::ToolResult {
+            tool_use_id: tool_id,
+            content: serde_json::json!({"utc": "2026-06-10T17:00:00Z"}),
+        }],
+    });
+    let text = collect_text(&provider, request).await;
+    assert!(
+        text.contains("17:00") || text.to_lowercase().contains("5"),
+        "model used the tool result: {text}"
+    );
+}

crates/cm-runtime/tests/live_anthropic.rs

@@ -0,0 +1,202 @@
+//! The COMPLETE production path against the real Anthropic API
+//! (CM_LIVE_LLM=1 + ANTHROPIC_API_KEY): a real model drives the runtime
+//! loop — ungated tool execution with real usage metering, and the §15
+//! chain: gated email intercepted, checkpointed, approved, and RESUMED
+//! against the live API (checkpoint/resume fidelity, the plan's #1 risk).
+
+use std::sync::Arc;
+use std::time::Duration;
+
+use cm_domain::{
+    AccessPolicy, Agent, AgentId, AgentStatus, Role, RunState, User, UserId, Workspace, WorkspaceId,
+};
+use cm_llm::AnthropicProvider;
+use cm_runtime::{RunEventBody, Runtime, RuntimeConfig};
+use cm_safety::{approvals, Decision};
+
+const MODEL: &str = "claude-haiku-4-5-20251001";
+
+fn live() -> Option<String> {
+    if std::env::var("CM_LIVE_LLM").as_deref() != Ok("1") {
+        eprintln!("skipped: set CM_LIVE_LLM=1 to run");
+        return None;
+    }
+    match std::env::var("ANTHROPIC_API_KEY") {
+        Ok(key) => Some(key),
+        Err(_) => {
+            eprintln!("skipped: ANTHROPIC_API_KEY not set");
+            None
+        }
+    }
+}
+
+async fn seeded(pool: &sqlx::PgPool) -> (Workspace, User, Agent) {
+    let ws = Workspace {
+        id: WorkspaceId::new(),
+        name: "Acme".into(),
+        plan: "team".into(),
+    };
+    cm_db::repo::workspaces::insert(pool, &ws).await.unwrap();
+    let owner = User {
+        id: UserId::new(),
+        workspace_id: ws.id,
+        email: format!("{}@acme.test", UserId::new()),
+        role: Role::Owner,
+        display_name: "Owner".into(),
+        created_at: time::OffsetDateTime::UNIX_EPOCH,
+    };
+    cm_db::repo::users::insert(pool, &owner).await.unwrap();
+    let agent = Agent {
+        id: AgentId::new(),
+        workspace_id: ws.id,
+        name: "Scout".into(),
+        job_title: "Analyst".into(),
+        system_prompt: "Be terse. Use your tools when they apply.".into(),
+        avatar: String::new(),
+        accent: String::new(),
+        wallpaper: String::new(),
+        managed_by: owner.id,
+        status: AgentStatus::Online,
+    };
+    cm_db::repo::agents::insert(pool, &agent, &AccessPolicy::default())
+        .await
+        .unwrap();
+    cm_db::repo::credits::add_lot(pool, ws.id, 1000, "live")
+        .await
+        .unwrap();
+    (ws, owner, agent)
+}
+
+#[tokio::test]
+async fn a_real_model_runs_tools_and_gets_metered() {
+    let Some(key) = live() else { return };
+    let pool = cm_testkit::test_pool().await;
+    let (ws, _, agent) = seeded(&pool).await;
+    let rt = Runtime::new(
+        pool.clone(),
+        Arc::new(AnthropicProvider::new(key)),
+        RuntimeConfig::basic(MODEL, 600),
+    );
+    let session = cm_db::repo::sessions::create(&pool, agent.id, ws.id, "Live")
+        .await
+        .unwrap();
+    let started = rt
+        .send_message(
+            session.id,
+            "What is the current UTC time? Use your clock tool, then state it.",
+        )
+        .await
+        .unwrap();
+    let mut rx = started.events;
+    while let Ok(envelope) = rx.recv().await {
+        if matches!(
+            envelope.event,
+            RunEventBody::RunCompleted { .. } | RunEventBody::Error { .. }
+        ) {
+            break;
+        }
+    }
+    let run = cm_db::repo::runs::get(&pool, started.run_id).await.unwrap();
+    assert_eq!(run.state, RunState::Completed);
+
+    // A real clock.now step happened.
+    let steps: i64 = sqlx::query_scalar(
+        "SELECT count(*) FROM steps s JOIN messages m ON m.id = s.message_id
+         WHERE m.session_id = $1 AND s.tool_name = 'clock.now' AND s.status = 'ok'",
+    )
+    .bind(session.id.as_uuid())
+    .fetch_one(&pool)
+    .await
+    .unwrap();
+    assert!(steps >= 1, "the model must have called the clock");
+
+    // Real usage metered, real credits burned.
+    let (tin, tout): (i64, i64) = sqlx::query_as(
+        "SELECT COALESCE(SUM(tokens_in),0)::BIGINT, COALESCE(SUM(tokens_out),0)::BIGINT
+         FROM usage_events WHERE workspace_id = $1",
+    )
+    .bind(ws.id.as_uuid())
+    .fetch_one(&pool)
+    .await
+    .unwrap();
+    assert!(tin > 0 && tout > 0, "real token usage: in={tin} out={tout}");
+    assert!(
+        cm_db::repo::credits::balance(&pool, ws.id).await.unwrap() < 1000,
+        "credits decremented"
+    );
+}
+
+#[tokio::test]
+async fn the_gated_chain_suspends_and_resumes_against_the_live_api() {
+    let Some(key) = live() else { return };
+    let pool = cm_testkit::test_pool().await;
+    let (ws, owner, agent) = seeded(&pool).await;
+    let rt = Runtime::new(
+        pool.clone(),
+        Arc::new(AnthropicProvider::new(key)),
+        RuntimeConfig::basic(MODEL, 600),
+    );
+    let session = cm_db::repo::sessions::create(&pool, agent.id, ws.id, "Live gated")
+        .await
+        .unwrap();
+    let started = rt
+        .send_message(
+            session.id,
+            "Use your email tool to send [email protected] an email with subject \
+             'Q2' and body 'Numbers attached.' Do it now without asking.",
+        )
+        .await
+        .unwrap();
+    let mut rx = started.events;
+    let mut suspended = false;
+    while let Ok(envelope) = rx.recv().await {
+        match envelope.event {
+            RunEventBody::RunSuspended { .. } => {
+                suspended = true;
+                break;
+            }
+            RunEventBody::Error { .. } | RunEventBody::RunCompleted { .. } => break,
+            _ => {}
+        }
+    }
+    assert!(suspended, "a REAL model's email.send must be intercepted");
+
+    let outbox: i64 = sqlx::query_scalar("SELECT count(*) FROM outbox")
+        .fetch_one(&pool)
+        .await
+        .unwrap();
+    assert_eq!(outbox, 0, "nothing sends while pending");
+
+    // Approve → the checkpoint resumes against the LIVE API: the stored
+    // assistant tool_use turn + our tool_result must reassemble into a
+    // request Anthropic accepts (checkpoint fidelity, end to end).
+    let pending = approvals::list_pending(&pool, ws.id).await.unwrap();
+    assert_eq!(pending.len(), 1);
+    approvals::decide(&pool, pending[0].id, owner.id, Decision::Approve)
+        .await
+        .unwrap();
+    rt.resume_run(cm_safety::ResumeReady {
+        run_id: started.run_id,
+        approval_id: pending[0].id,
+        approved: true,
+    })
+    .await
+    .unwrap();
+
+    let mut completed = false;
+    for _ in 0..240 {
+        let run = cm_db::repo::runs::get(&pool, started.run_id).await.unwrap();
+        if run.state == RunState::Completed {
+            completed = true;
+            break;
+        }
+        tokio::time::sleep(Duration::from_millis(500)).await;
+    }
+    assert!(completed, "resume against the live API must complete");
+
+    let outbox: i64 = sqlx::query_scalar("SELECT count(*) FROM outbox")
+        .fetch_one(&pool)
+        .await
+        .unwrap();
+    assert_eq!(outbox, 1, "approved email sent exactly once");
+}