feat(llm-proxy): mission containers reach their models through a key-adding proxy (flagged)

Container-tier missions held the platform's provider keys in their environment,
readable by an agent with Bash and public egress. With CLAWMATES_LLM_PROXY=1
(and CLAWMATES_LLM_PROXY_SECRET), a container instead holds a per-mission token
where each key was, ANTHROPIC_BASE_URL points at the server's proxy on :8089
(not published, not routed by Traefik), and the GLM/Kimi hops' base URLs are
rewritten in the mission's config copy. The proxy verifies the token
(HMAC(secret, mission_id) — stateless, survives redeploys), refuses it unless
the mission is running, swaps in the real credential and streams the response.

Spiked first on gw-04: Claude Code on a subscription OAuth token, given only a
placeholder and a base URL, sent nothing but POST /v1/messages there and
answered once the placeholder was swapped. Off by default; no behaviour change
until enabled.

Co-Authored-By: Claude Opus 5.5 (1M context) <[email protected]>

crates/bins/clawmates-server/src/main.rs

@@ -452,6 +452,9 @@ async fn run() -> Result<(), String> {
     // evaluator skip a judge at 95% for the fallback. Ten minutes: the windows
     // are hours and days long, and each poll is one tiny GET per provider.
     cm_api::judge_quota::spawn_poller(std::time::Duration::from_secs(600));
+    // Mission containers reach their models through this, holding a
+    // per-mission token instead of provider keys. Off unless configured.
+    cm_api::llm_proxy::spawn(pool.clone());
     // Nightly: check upstream for newer dev-tool releases (claude/kimi/ollama).
     cm_api::tool_versions::spawn_latest_checker(
         pool.clone(),

crates/cm-api/Cargo.toml

@@ -25,7 +25,7 @@ futures = "0.3"
 serde = { workspace = true }
 serde_json = { workspace = true }
 sqlx = { workspace = true }
-reqwest = { version = "0.12", default-features = false, features = ["json", "rustls-tls"] }
+reqwest = { version = "0.12", default-features = false, features = ["json", "rustls-tls", "stream"] }
 cm-auth = { path = "../cm-auth" }
 cm-billing = { path = "../cm-billing" }
 cm-brain = { path = "../cm-brain" }

crates/cm-api/src/lib.rs

@@ -27,6 +27,7 @@ pub mod microvm_executor;
 pub mod microvm_turn_executor;
 pub mod continuous_research;
 pub mod delivery_secrets;
+pub mod llm_proxy;
 pub mod mission_delivery;
 pub mod podcast;
 pub mod mission_events;

crates/cm-api/src/llm_proxy.rs

@@ -0,0 +1,371 @@
+//! Model calls from mission containers, with the real credential added here.
+//!
+//! Container-tier missions ran Claude Code with the platform's provider keys in
+//! their environment (`mission_runtime::forwarded_provider_env`), readable by
+//! an agent that has Bash and public egress. `delivery_secrets` stops those keys
+//! leaving through a delivery; nothing stopped a `curl` carrying one.
+//!
+//! With this on (`CLAWMATES_LLM_PROXY=1`), a mission container holds a
+//! per-mission TOKEN where each key used to be, and `ANTHROPIC_BASE_URL` (plus
+//! the GLM/Kimi hops' base URLs in the mission's config copy) point here. This
+//! swaps the token for the real credential and forwards the request unchanged,
+//! streaming the answer back. The agent never holds a real key.
+//!
+//! Measured before building (2026-09-23 spike on gw-04): Claude Code logged in
+//! with a subscription OAuth token, given only a placeholder and a base URL,
+//! sent nothing but `POST /v1/messages` to it and answered correctly once the
+//! placeholder was swapped. Nothing bypassed the base URL.
+//!
+//! # Who can use it
+//!
+//! Its own listener, [`PORT`], which is neither published nor routed by
+//! Traefik: reachable only from containers on the server's Docker networks. A
+//! token is `HMAC(secret, mission_id)` — stateless, so it survives a server
+//! redeploy under a running mission — and is honoured only while that mission
+//! is `running`. A token that escapes is worth one mission's model calls, from
+//! inside the network, until the mission ends.
+
+use axum::body::Body;
+use axum::extract::{Path, State};
+use axum::http::{HeaderMap, Method, StatusCode, Uri};
+use axum::response::Response;
+use hmac::{Hmac, Mac};
+use sha2::Sha256;
+use sqlx::PgPool;
+use uuid::Uuid;
+
+/// The proxy's own port inside the server container.
+pub const PORT: u16 = 8089;
+
+const TOKEN_PREFIX: &str = "cmlp";
+
+/// On only when asked for AND a secret exists to sign tokens with. A flag
+/// without a secret would mint tokens nobody can verify, and every mission
+/// would fail to reach its model.
+pub fn enabled() -> bool {
+    std::env::var("CLAWMATES_LLM_PROXY").is_ok_and(|v| v.trim() == "1") && secret().is_some()
+}
+
+fn secret() -> Option<Vec<u8>> {
+    std::env::var("CLAWMATES_LLM_PROXY_SECRET")
+        .ok()
+        .map(|s| s.trim().to_string())
+        .filter(|s| s.len() >= 32)
+        .map(String::into_bytes)
+}
+
+fn sign(secret: &[u8], mission_id: Uuid) -> String {
+    let mut mac = Hmac::<Sha256>::new_from_slice(secret).expect("hmac takes any key length");
+    mac.update(mission_id.as_bytes());
+    hex::encode(&mac.finalize().into_bytes()[..20])
+}
+
+/// The token a mission's container gets in place of every provider key.
+pub fn token_for_with(secret: &[u8], mission_id: Uuid) -> String {
+    format!("{TOKEN_PREFIX}.{}.{}", mission_id.simple(), sign(secret, mission_id))
+}
+
+pub fn token_for(mission_id: Uuid) -> Option<String> {
+    secret().map(|s| token_for_with(&s, mission_id))
+}
+
+/// The mission a token was minted for, if the signature holds.
+pub fn verify_with(secret: &[u8], token: &str) -> Option<Uuid> {
+    let mut parts = token.trim().splitn(3, '.');
+    if parts.next()? != TOKEN_PREFIX {
+        return None;
+    }
+    let mission = Uuid::parse_str(parts.next()?).ok()?;
+    let sig = parts.next()?;
+    let want = sign(secret, mission);
+    // Constant-time compare: the signature is the whole credential.
+    let ok = sig.len() == want.len()
+        && sig.bytes().zip(want.bytes()).fold(0u8, |a, (x, y)| a | (x ^ y)) == 0;
+    ok.then_some(mission)
+}
+
+/// Where a mission container reaches the proxy: the server's own hostname on
+/// the Docker network (the same self-configuring rule as the skills door's
+/// `api_origin`), overridable for other deployments.
+pub fn base_url() -> Option<String> {
+    if let Ok(v) = std::env::var("CLAWMATES_LLM_PROXY_URL") {
+        if !v.trim().is_empty() {
+            return Some(v.trim().trim_end_matches('/').to_string());
+        }
+    }
+    let host = std::env::var("HOSTNAME").ok()?;
+    let host = host.trim();
+    (!host.is_empty()).then(|| format!("http://{host}:{PORT}"))
+}
+
+/// One upstream: where it lives and the real credential it takes.
+#[derive(Debug, PartialEq, Eq)]
+pub struct Upstream {
+    pub base: &'static str,
+    /// `(header, value)`.
+    pub auth: (&'static str, String),
+}
+
+/// The upstream for a route, with the credential read from the server's env.
+/// `None` for an unknown route or a provider whose key is not set here.
+pub fn upstream(provider: &str) -> Option<Upstream> {
+    let env = |k: &str| std::env::var(k).ok().map(|v| v.trim().to_string()).filter(|v| !v.is_empty());
+    match provider {
+        "anthropic" => match crate::mission_runtime::runtime_auth_mode() {
+            crate::mission_runtime::RuntimeAuth::Subscription => Some(Upstream {
+                base: "https://api.anthropic.com",
+                auth: ("authorization", format!("Bearer {}", env("CLAUDE_CODE_OAUTH_TOKEN")?)),
+            }),
+            crate::mission_runtime::RuntimeAuth::ApiKey => Some(Upstream {
+                base: "https://api.anthropic.com",
+                auth: ("x-api-key", env("ANTHROPIC_API_KEY")?),
+            }),
+        },
+        "glm" => Some(Upstream {
+            base: "https://api.z.ai/api/anthropic",
+            auth: ("authorization", format!("Bearer {}", env("ZAI_API_KEY")?)),
+        }),
+        "kimi" => Some(Upstream {
+            base: "https://api.kimi.com/coding",
+            auth: ("authorization", format!("Bearer {}", env("KIMI_API_KEY")?)),
+        }),
+        _ => None,
+    }
+}
+
+/// The token a request presents, as Claude Code sends it: `Authorization:
+/// Bearer` for an OAuth or auth-token credential, `x-api-key` for an API key.
+fn presented_token(headers: &HeaderMap) -> Option<String> {
+    if let Some(v) = headers.get("authorization").and_then(|v| v.to_str().ok()) {
+        return Some(v.trim().trim_start_matches("Bearer ").trim().to_string());
+    }
+    headers
+        .get("x-api-key")
+        .and_then(|v| v.to_str().ok())
+        .map(|v| v.trim().to_string())
+}
+
+/// Request headers that must not be forwarded: the placeholder credential,
+/// and the hop-by-hop / length headers the client rebuilds.
+const DROP_REQUEST: &[&str] = &[
+    "authorization",
+    "x-api-key",
+    "host",
+    "content-length",
+    "connection",
+    "accept-encoding",
+    "transfer-encoding",
+];
+const DROP_RESPONSE: &[&str] = &["content-length", "transfer-encoding", "connection", "content-encoding"];
+
+fn deny(status: StatusCode, why: &str) -> Response {
+    Response::builder()
+        .status(status)
+        .header("content-type", "application/json")
+        .body(Body::from(
+            serde_json::json!({"type":"error","error":{"type":"clawmates_llm_proxy","message":why}})
+                .to_string(),
+        ))
+        .unwrap_or_default()
+}
+
+#[derive(Clone)]
+struct ProxyState {
+    pool: PgPool,
+    client: reqwest::Client,
+}
+
+async fn handle(
+    State(st): State<ProxyState>,
+    Path((provider, rest)): Path<(String, String)>,
+    method: Method,
+    uri: Uri,
+    headers: HeaderMap,
+    body: axum::body::Bytes,
+) -> Response {
+    let Some(secret) = secret() else {
+        return deny(StatusCode::SERVICE_UNAVAILABLE, "proxy has no signing secret");
+    };
+    let Some(mission) = presented_token(&headers).and_then(|t| verify_with(&secret, &t)) else {
+        return deny(StatusCode::UNAUTHORIZED, "not a valid mission token");
+    };
+    let running: bool = sqlx::query_scalar("SELECT status = 'running' FROM missions WHERE id = $1")
+        .bind(mission)
+        .fetch_optional(&st.pool)
+        .await
+        .ok()
+        .flatten()
+        .unwrap_or(false);
+    if !running {
+        return deny(StatusCode::FORBIDDEN, "mission is not running");
+    }
+    let Some(up) = upstream(&provider) else {
+        return deny(StatusCode::NOT_FOUND, "unknown or unconfigured provider");
+    };
+    let query = uri.query().map(|q| format!("?{q}")).unwrap_or_default();
+    let url = format!("{}/{rest}{query}", up.base);
+    let mut req = st.client.request(method, &url);
+    for (k, v) in headers.iter() {
+        if !DROP_REQUEST.contains(&k.as_str()) {
+            req = req.header(k, v);
+        }
+    }
+    req = req.header(up.auth.0, up.auth.1).body(body);
+    let resp = match req.send().await {
+        Ok(r) => r,
+        Err(e) => {
+            eprintln!("llm_proxy: mission {mission} → {provider}: {e}");
+            return deny(StatusCode::BAD_GATEWAY, "upstream unreachable");
+        }
+    };
+    let mut out = Response::builder().status(resp.status().as_u16());
+    for (k, v) in resp.headers().iter() {
+        if !DROP_RESPONSE.contains(&k.as_str()) {
+            out = out.header(k.as_str(), v.as_bytes());
+        }
+    }
+    out.body(Body::from_stream(resp.bytes_stream()))
+        .unwrap_or_else(|_| deny(StatusCode::BAD_GATEWAY, "could not relay the response"))
+}
+
+/// Serve the proxy on [`PORT`]. A no-op unless [`enabled`].
+pub fn spawn(pool: PgPool) {
+    if !enabled() {
+        eprintln!("llm_proxy: off (CLAWMATES_LLM_PROXY != 1 or no CLAWMATES_LLM_PROXY_SECRET) — mission containers hold provider keys");
+        return;
+    }
+    tokio::spawn(async move {
+        let client = reqwest::Client::builder()
+            // A long agent turn streams for many minutes; the CLI's own
+            // API_TIMEOUT_MS is 50 minutes.
+            .timeout(std::time::Duration::from_secs(3000))
+            .build()
+            .expect("reqwest client");
+        let app = axum::Router::new()
+            .route("/{provider}/{*rest}", axum::routing::any(handle))
+            .with_state(ProxyState { pool, client });
+        match tokio::net::TcpListener::bind(("0.0.0.0", PORT)).await {
+            Ok(l) => {
+                eprintln!("llm_proxy: listening on :{PORT} — mission containers get tokens, not keys");
+                if let Err(e) = axum::serve(l, app).await {
+                    eprintln!("llm_proxy: stopped: {e}");
+                }
+            }
+            Err(e) => eprintln!("llm_proxy: could not bind :{PORT}: {e}"),
+        }
+    });
+}
+
+/// Point a mission config's GLM and Kimi hops at the proxy. Their base URLs are
+/// literal in the seed config (the default hop takes `ANTHROPIC_BASE_URL` from
+/// the container env), so they are rewritten in the mission's own copy.
+/// Returns the edited document and how many hops were redirected.
+pub fn route_config_through(raw: &str, proxy: &str) -> Result<(String, usize), String> {
+    let mut doc = raw
+        .parse::<toml_edit::DocumentMut>()
+        .map_err(|e| format!("parse runtime config.toml: {e}"))?;
+    let mut n = 0;
+    for hop in ["glm", "kimi"] {
+        let Some(env) = doc
+            .get_mut("providers")
+            .and_then(|p| p.get_mut("models"))
+            .and_then(|m| m.get_mut("claude_cli"))
+            .and_then(|c| c.get_mut(hop))
+            .and_then(|h| h.get_mut("env"))
+            .and_then(|e| e.as_table_like_mut())
+        else {
+            continue;
+        };
+        if env.get("ANTHROPIC_BASE_URL").is_some() {
+            env.insert("ANTHROPIC_BASE_URL", toml_edit::value(format!("{proxy}/{hop}")));
+            n += 1;
+        }
+    }
+    Ok((doc.to_string(), n))
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    const S: &[u8] = b"0123456789abcdef0123456789abcdef-test";
+
+    #[test]
+    fn a_token_verifies_to_its_own_mission() {
+        let m = Uuid::now_v7();
+        assert_eq!(verify_with(S, &token_for_with(S, m)), Some(m));
+    }
+
+    #[test]
+    fn a_tampered_or_foreign_token_is_refused() {
+        let m = Uuid::now_v7();
+        let t = token_for_with(S, m);
+        // Another mission's id with this signature.
+        let other = Uuid::now_v7();
+        let forged = t.replace(&m.simple().to_string(), &other.simple().to_string());
+        assert_eq!(verify_with(S, &forged), None);
+        // Signed with a different secret.
+        assert_eq!(verify_with(b"another-secret-another-secret-xx", &t), None);
+        // Garbage and a real provider key are not tokens.
+        assert_eq!(verify_with(S, "sk-ant-oat01-whatever"), None);
+        assert_eq!(verify_with(S, ""), None);
+    }
+
+    /// The token must never look like, or contain, a real key — it is what the
+    /// agent can read now.
+    #[test]
+    fn a_token_names_its_mission_and_nothing_else() {
+        let m = Uuid::now_v7();
+        let t = token_for_with(S, m);
+        assert!(t.starts_with("cmlp."), "{t}");
+        assert!(t.contains(&m.simple().to_string()));
+    }
+
+    #[test]
+    fn the_placeholder_is_what_gets_checked_either_way_claude_sends_it() {
+        let mut h = HeaderMap::new();
+        h.insert("authorization", "Bearer cmlp.x.y".parse().unwrap());
+        assert_eq!(presented_token(&h).as_deref(), Some("cmlp.x.y"));
+        let mut h = HeaderMap::new();
+        h.insert("x-api-key", "cmlp.a.b".parse().unwrap());
+        assert_eq!(presented_token(&h).as_deref(), Some("cmlp.a.b"));
+    }
+
+    /// The placeholder never travels upstream: both credential headers are
+    /// dropped before the real one is added.
+    #[test]
+    fn the_placeholder_is_never_forwarded() {
+        assert!(DROP_REQUEST.contains(&"authorization") && DROP_REQUEST.contains(&"x-api-key"));
+    }
+
+    #[test]
+    fn only_known_providers_route() {
+        assert!(upstream("evil.example").is_none());
+        assert!(upstream("").is_none());
+    }
+
+    #[test]
+    fn the_glm_and_kimi_hops_are_redirected_and_nothing_else_moves() {
+        let raw = r#"
+[providers.models.claude_cli.default]
+model = "claude-sonnet-4-6"
+env = { CLAUDE_CODE_OAUTH_TOKEN = "$CLAUDE_CODE_OAUTH_TOKEN" }
+
+[providers.models.claude_cli.glm]
+model = "glm-4.7"
+env = { HOME = "/zeroclaw-data/glm-home", ANTHROPIC_BASE_URL = "https://api.z.ai/api/anthropic", ANTHROPIC_AUTH_TOKEN = "$ZAI_API_KEY" }
+
+[providers.models.claude_cli.kimi]
+model = "kimi-for-coding"
+env = { HOME = "/zeroclaw-data/kimi-home", ANTHROPIC_BASE_URL = "https://api.kimi.com/coding", ANTHROPIC_AUTH_TOKEN = "$KIMI_API_KEY" }
+"#;
+        let (out, n) = route_config_through(raw, "http://srv:8089").unwrap();
+        assert_eq!(n, 2);
+        assert!(out.contains(r#"ANTHROPIC_BASE_URL = "http://srv:8089/glm""#), "{out}");
+        assert!(out.contains(r#"ANTHROPIC_BASE_URL = "http://srv:8089/kimi""#), "{out}");
+        assert!(!out.contains("api.z.ai") && !out.contains("api.kimi.com"), "{out}");
+        // The credential REFERENCES are untouched; the env behind them changes.
+        assert!(out.contains(r#"ANTHROPIC_AUTH_TOKEN = "$ZAI_API_KEY""#));
+        assert!(out.contains(r#"HOME = "/zeroclaw-data/glm-home""#));
+    }
+}

crates/cm-api/src/mission_runtime.rs

@@ -779,8 +779,30 @@ impl MissionRuntimeProvisioner {
         // The other three are unrelated providers (Gemini/Groq/OpenAI) with no
         // subscription equivalent, so they forward in both modes.
         let auth_mode = runtime_auth_mode();
+        // With the LLM proxy on, the container gets a per-mission token where
+        // each key would be, and Claude Code's base URL points at the proxy,
+        // which adds the real credential. See `llm_proxy`. Without a reachable
+        // proxy address the keys forward as before, loudly.
+        let proxied = match (crate::llm_proxy::enabled(), crate::llm_proxy::base_url(), crate::llm_proxy::token_for(mission_id)) {
+            (true, Some(base), Some(token)) => Some((base, token)),
+            (true, _, _) => {
+                eprintln!(
+                    "mission_runtime: CLAWMATES_LLM_PROXY is on but the proxy address or token \
+                     could not be derived — mission {mission_id} gets the real provider keys"
+                );
+                None
+            }
+            _ => None,
+        };
         for (key, v) in forwarded_provider_env(auth_mode) {
-            env.push(format!("{key}={v}"));
+            match &proxied {
+                Some((_, token)) => env.push(format!("{key}={token}")),
+                None => env.push(format!("{key}={v}")),
+            }
+        }
+        if let Some((base, _)) = &proxied {
+            env.push(format!("ANTHROPIC_BASE_URL={base}/anthropic"));
+            eprintln!("mission_runtime: mission {mission_id} reaches its models through {base} — no provider key in the container");
         }
         eprintln!(
             "mission_runtime: mission {mission_id} container auth mode = {} \
@@ -1016,6 +1038,22 @@ impl MissionRuntimeProvisioner {
         // pin never applied, its agents never saw `/mission/repo`, and phase 0
         // completed having written nothing. The tar API has no argv limit, so
         // the failure mode is gone rather than merely further away.
+        // The GLM and Kimi hops carry literal base URLs in the config; point
+        // them at the proxy too, or a fallback would send the placeholder token
+        // straight to the provider and fail.
+        let edited = match (crate::llm_proxy::enabled(), crate::llm_proxy::base_url()) {
+            (true, Some(base)) => {
+                let (routed, n) = crate::llm_proxy::route_config_through(&edited, &base)?;
+                if n < 2 {
+                    eprintln!(
+                        "mission_runtime: routed only {n} of 2 fallback hops through the LLM proxy \
+                         for mission {mission_id} — an unrouted hop will fail rather than leak"
+                    );
+                }
+                routed
+            }
+            _ => edited,
+        };
         crate::mission_fs::put_file(&self.docker, &name, CONFIG_PATH, edited.as_bytes())
             .await
             .map_err(|e| format!("write runtime config.toml: {e}"))?;