clawmates/crates/bins/clawmates-node/src/egress.rs

//! Host side of a microVM's only route out: an HTTP `CONNECT` proxy on a Unix
//! socket, one per VM.
//!
//! # Why the guest has no network card
//!
//! It could have had one. A TAP device plus NAT is what the Firecracker
//! write-ups do, and it was measured against this before being rejected:
//!
//! - `ip tuntap add` is **denied to the daemon user** (needs `CAP_NET_ADMIN`), so
//!   TAP would need root to pre-provision devices at setup time — the same
//!   privilege detour the loop-mounted rootfs already forced.
//! - tank's `FORWARD` policy is `DROP` with Docker and Tailscale chains, so rules
//!   would have to be *inserted* at position 1; appended ones die silently.
//! - a leaked TAP device is a new class of host litter to reap.
//!
//! Against that, `CONNECT` needs no privilege at all, and it is better on the
//! merits: the client hands us the **hostname**, so resolution happens here and
//! the guest needs no DNS or `resolv.conf`; the allow-list is by name rather than
//! by address; and nothing in the guest can reach the network except through this
//! function. That is what the isolation plan's egress restriction actually asked
//! for, and it is strictly tighter than the mission container's present full
//! egress on `clawmates_edge`.
//!
//! The design rests on one measured fact: **`claude` honours `HTTPS_PROXY`**.
//! With the proxy pointed at a closed port, `claude -p` fails with
//! `ConnectionRefused` instead of answering. (That could only be measured in a
//! container — inside a VM the CLI collapses every failure into `Execution
//! error`.)
//!
//! # Shape
//!
//! Firecracker's convention for a guest-initiated connection is that the **host**
//! listens on `<uds_path>_<port>`. The guest's agent pumps bytes from
//! `127.0.0.1:3128` to vsock port 9002 and parses nothing, so all policy is here
//! and a compromised guest cannot argue with it.

use std::path::{Path, PathBuf};
use std::sync::Arc;

use tokio::io::{AsyncReadExt, AsyncWriteExt, BufReader};
use tokio::net::{TcpStream, UnixListener, UnixStream};

/// Port the guest dials. Must match `fcagent`'s `EGRESS_PORT`.
pub const EGRESS_PORT: u32 = 9002;


/// What every backend gets, whatever it is.
const COMMON_ALLOW: &[&str] = &["git.redclaw.dev"];

/// The model host a backend's CLI must reach, and NOTHING else.
///
/// Per backend rather than a union, and that is not tidiness. MEASURED on tank:
/// a `glm` VM completed a whole mission with `api.anthropic.com` denied at this
/// proxy, dialling only `api.z.ai` — Claude Code's calls to anthropic.com are
/// its own telemetry, not its completions. So a GLM VM has no need of Anthropic
/// at all, and a union allow-list would let a credential mix-up reach the wrong
/// provider's endpoint instead of failing at a closed door.
///
/// The measurement also settled something a self-report could not: that same
/// agent, served only by z.ai, still described itself as "Claude Opus 5". A
/// model's account of which model it is has no evidential value here; the
/// proxy's log of which host it dialled does.
fn provider_hosts(backend: Option<&str>) -> &'static [&'static str] {
    match backend {
        // `canary-claude` is the same provider, from a candidate CLI image —
        // see `mission_runtime::microvm_credential_for`, which must grant it the
        // same credential. A backend is defined in TWO maps: the credential one
        // on the server and this one on the node. Adding it to only the first is
        // exactly what happened here: the mission launched, the VM booted, the
        // agent ran, and the turn died on
        // "403 api.anthropic.com is not on the egress allow-list" — which is the
        // fail-closed branch below working correctly.
        None | Some("") | Some("default") | Some("claude") | Some("canary-claude") => {
            &["api.anthropic.com", ".anthropic.com"]
        }
        Some("glm") => &["api.z.ai"],
        // The Kimi CODE service, which is where an `sk-kimi-` key is valid —
        // NOT `api.moonshot.ai`, whose Anthropic endpoint exists but belongs to
        // a different account namespace and rejects that key. Only the host the
        // `agent-kimi` image bakes in.
        Some("kimi") => &["api.kimi.com"],
        // A locally-hosted model reaches NOTHING through this proxy. Its route
        // is `crate::local_model` — a vsock pipe to the node's own loopback,
        // with no destination in the protocol — so the correct allow-list here
        // is the empty one, and it falls through to the branch below.
        //
        // Spelled out rather than left implicit because the temptation was to
        // widen this proxy instead: an entry here would have meant relaxing the
        // 443-only rule AND the IP-literal refusal, both of which exist because
        // a unit test caught them being bypassed.
        // Fail closed: a backend nobody taught this function about reaches the
        // forge and no model API. It cannot silently borrow another provider's
        // door, which is the failure this split exists to prevent.
        Some(_) => &[],
    }
}

/// Parse the allow-list once per VM.
///
/// An empty `CLAWMATES_FC_EGRESS_ALLOW` means **deny everything**, not "fall back
/// to the default": an operator who blanked it asked for no egress, and quietly
/// restoring the default would hand a mission the network they just took away.
/// The allow-list for a VM running `backend`.
///
/// An explicit `CLAWMATES_FC_EGRESS_ALLOW` still wins outright: an operator who
/// set it asked for exactly that list, and quietly adding a provider host to it
/// would widen a boundary they had drawn on purpose.
fn allow_list_for(backend: Option<&str>) -> Vec<String> {
    match std::env::var("CLAWMATES_FC_EGRESS_ALLOW") {
        Ok(raw) => raw
            .split(',')
            .map(|s| s.trim().to_ascii_lowercase())
            .filter(|s| !s.is_empty())
            .collect(),
        Err(_) => COMMON_ALLOW
            .iter()
            .chain(provider_hosts(backend).iter())
            .map(|s| s.to_string())
            .collect(),
    }
}

/// Is `host` allowed?
///
/// Case-insensitive, port already stripped. A leading `.` in an entry matches
/// that domain and its subdomains; anything else must match exactly. Deliberately
/// not a substring test — `api.anthropic.com.evil.test` contains the allowed name
/// and must not pass.
fn host_allowed(host: &str, allow: &[String]) -> bool {
    let host = host.trim().trim_end_matches('.').to_ascii_lowercase();
    if host.is_empty() {
        return false;
    }
    // A hostname is letters, digits, dots and hyphens — nothing else. This is
    // load-bearing, not hygiene: `evil.test/api.anthropic.com` ends with an
    // allowed suffix and would otherwise PASS the match below. A unit test found
    // it. Rejecting the character class also refuses IP literals, so an address
    // cannot be used to sidestep a list written in names.
    if !host
        .chars()
        .all(|c| c.is_ascii_alphanumeric() || c == '.' || c == '-')
    {
        return false;
    }
    allow.iter().any(|a| match a.strip_prefix('.') {
        Some(domain) => host == domain || host.ends_with(&format!(".{domain}")),
        None => host == *a,
    })
}

/// Split `host:port` from a CONNECT target.
///
/// Only 443 is allowed. Permitting arbitrary ports would turn the proxy into a
/// general-purpose tunnel to anything the allow-list happens to name, which is a
/// different and much larger promise than "the agent can reach its API".
fn parse_target(target: &str) -> Result<(String, u16), String> {
    let (host, port) = target
        .rsplit_once(':')
        .ok_or_else(|| format!("CONNECT target {target:?} has no port"))?;
    let port: u16 = port
        .trim()
        .parse()
        .map_err(|_| format!("CONNECT target {target:?} has a non-numeric port"))?;
    if port != 443 {
        return Err(format!("port {port} is not permitted (only 443)"));
    }
    // Strip IPv6 brackets so the allow-list sees the same text either way.
    let host = host.trim().trim_start_matches('[').trim_end_matches(']');
    Ok((host.to_string(), port))
}

/// What happened to one connection. Returned so the caller can log it and the
/// selftest can assert on it.
#[derive(Debug, PartialEq, Eq)]
pub enum Verdict {
    Allowed(String),
    Denied(String),
    Malformed(String),
}

/// One header line, with a cap.
///
/// `read_line` has no limit, and a guest that never sends a newline would make
/// the host allocate until it died. Read byte-wise instead — the reads come out
/// of the BufReader, so this is cheap for lines this size, and it keeps ONE
/// reader over the connection, which matters (see `serve`).
async fn read_line_capped(reader: &mut BufReader<UnixStream>, cap: usize) -> Result<String, String> {
    let mut out = Vec::new();
    loop {
        match reader.read_u8().await {
            Ok(b'\n') => break,
            Ok(b) => out.push(b),
            // EOF mid-line: return what we have and let the caller judge it.
            Err(e) if e.kind() == std::io::ErrorKind::UnexpectedEof => break,
            Err(e) => return Err(format!("read: {e}")),
        }
        if out.len() > cap {
            return Err(format!("a request line longer than {cap} bytes"));
        }
    }
    Ok(String::from_utf8_lossy(&out)
        .trim_end_matches('\r')
        .to_string())
}

/// Serve one tunnelled connection.
async fn serve(stream: UnixStream, allow: Arc<Vec<String>>) -> Verdict {
    // ONE reader for the whole request. Wrapping the stream a second time would
    // discard whatever the first reader had already buffered — including the
    // first bytes of the TLS handshake — and the tunnel would come up looking
    // fine and then stall on a corrupt stream.
    let mut reader = BufReader::new(stream);

    let line = match read_line_capped(&mut reader, 8 * 1024).await {
        Ok(l) if !l.trim().is_empty() => l,
        Ok(_) => return Verdict::Malformed("no request line".into()),
        Err(e) => return Verdict::Malformed(e),
    };

    let mut parts = line.split_whitespace();
    let method = parts.next().unwrap_or_default().to_ascii_uppercase();
    let target = parts.next().unwrap_or_default().to_string();

    if method != "CONNECT" {
        // Plain HTTP would mean proxying a request we would then have to rewrite,
        // and everything a mission needs is TLS. Refused with a status, so the
        // client reports something better than a closed socket.
        let _ = reply(reader.get_mut(), 405, "only CONNECT is supported").await;
        return Verdict::Malformed(format!("method {method}"));
    }

    let (host, port) = match parse_target(&target) {
        Ok(v) => v,
        Err(e) => {
            let _ = reply(reader.get_mut(), 400, &e).await;
            return Verdict::Malformed(e);
        }
    };

    if !host_allowed(&host, &allow) {
        // 403 rather than a silent drop: a denial that looks like a network
        // timeout is indistinguishable from a hung agent, and this codebase has
        // paid for that confusion more than once.
        let _ = reply(
            reader.get_mut(),
            403,
            &format!("{host} is not on the egress allow-list"),
        )
        .await;
        return Verdict::Denied(host);
    }

    // Consume the remaining request headers: they belong to the CONNECT, not to
    // the tunnel.
    loop {
        match read_line_capped(&mut reader, 8 * 1024).await {
            Ok(h) if h.trim().is_empty() => break,
            Ok(_) => {}
            Err(e) => return Verdict::Malformed(e),
        }
    }

    let mut upstream = match TcpStream::connect((host.as_str(), port)).await {
        Ok(s) => s,
        Err(e) => {
            let _ = reply(reader.get_mut(), 502, &format!("connect {host}:{port}: {e}")).await;
            return Verdict::Denied(host);
        }
    };

    if reply(reader.get_mut(), 200, "Connection established")
        .await
        .is_err()
    {
        return Verdict::Denied(host);
    }

    // Anything already buffered past the headers is tunnel payload — a client
    // that pipelined its first TLS bytes would otherwise lose them.
    let pending = reader.buffer().to_vec();
    let mut stream = reader.into_inner();
    if !pending.is_empty() && upstream.write_all(&pending).await.is_err() {
        return Verdict::Denied(host);
    }

    // Bytes both ways until either side is done. Errors are not worth reporting:
    // a closed connection is the normal end of a tunnel.
    let _ = tokio::io::copy_bidirectional(&mut stream, &mut upstream).await;
    Verdict::Allowed(host)
}

async fn reply(s: &mut UnixStream, code: u16, text: &str) -> std::io::Result<()> {
    let reason = if code == 200 {
        "Connection established"
    } else {
        "Forbidden"
    };
    // The body carries the reason for a non-200 so it reaches the agent's own
    // error output, where whoever is reading a failed mission will see it.
    let body = if code == 200 { String::new() } else { format!("{text}\n") };
    let head = format!(
        "HTTP/1.1 {code} {reason}\r\nContent-Length: {}\r\nConnection: close\r\n\r\n",
        body.len()
    );
    s.write_all(head.as_bytes()).await?;
    if !body.is_empty() {
        s.write_all(body.as_bytes()).await?;
    }
    s.flush().await
}

/// Start this VM's proxy. Returns the socket path and the task serving it.
///
/// Bound **before** firecracker starts, because a guest that dials before the
/// host is listening gets a connection refused it will not retry.
pub fn start(
    uds: &Path,
    vm_id: &str,
    backend: Option<&str>,
) -> Result<(PathBuf, tokio::task::JoinHandle<()>), String> {
    let path = PathBuf::from(format!("{}_{}", uds.display(), EGRESS_PORT));
    // Firecracker does not clean these up any more than it cleans up its own
    // socket, and a stale file makes bind fail with EADDRINUSE.
    let _ = std::fs::remove_file(&path);
    let listener =
        UnixListener::bind(&path).map_err(|e| format!("bind {}: {e}", path.display()))?;

    let allow = Arc::new(allow_list_for(backend));
    eprintln!(
        "microvm {vm_id}: egress proxy on {} allowing {:?}",
        path.display(),
        allow
    );
    let vm = vm_id.to_string();
    let task = tokio::spawn(async move {
        loop {
            match listener.accept().await {
                Ok((s, _)) => {
                    let allow = allow.clone();
                    let vm = vm.clone();
                    tokio::spawn(async move {
                        match serve(s, allow).await {
                            // Logged at every outcome: this is the audit trail of
                            // everything a mission reached, and a denial that is
                            // not logged is a mystery hang later.
                            Verdict::Allowed(h) => eprintln!("microvm {vm}: egress -> {h}"),
                            Verdict::Denied(h) => {
                                eprintln!("microvm {vm}: egress DENIED {h}")
                            }
                            Verdict::Malformed(w) => {
                                eprintln!("microvm {vm}: egress malformed request ({w})")
                            }
                        }
                    });
                }
                Err(e) => {
                    eprintln!("microvm {vm}: egress accept failed: {e}");
                    return;
                }
            }
        }
    });
    Ok((path, task))
}

#[cfg(test)]
mod tests {
    /// A backend is defined in TWO places — the server's credential map and this
    /// egress map — and granting it one without the other produces a mission
    /// that launches, boots, runs, and dies on a 403 from our own proxy.
    ///
    /// Measured exactly that way: `canary-claude` was credentialed on the server
    /// and unknown here, and the turn failed with
    /// "api.anthropic.com is not on the egress allow-list".
    #[test]
    fn the_canary_backend_reaches_the_same_provider_as_claude() {
        assert_eq!(
            provider_hosts(Some("canary-claude")),
            provider_hosts(Some("claude")),
            "a canary of the Claude image must reach Anthropic, or it tests nothing"
        );
        // And the fail-closed branch must still hold for anything unknown: this
        // is what stops a new backend silently borrowing another provider's door.
        assert!(provider_hosts(Some("canary-something-else")).is_empty());
        assert!(provider_hosts(Some("definitely-not-built")).is_empty());
    }

    use super::*;

    fn allow() -> Vec<String> {
        allow_list_for(None)
    }

    /// MEASURED on tank, not assumed: a `glm` VM ran a whole mission to
    /// completion with `api.anthropic.com` denied at this proxy, dialling only
    /// `api.z.ai`. So Anthropic's host is not something a GLM agent needs — and
    /// a VM that cannot reach it cannot send z.ai's key there, or Anthropic's
    /// subscription token to z.ai, whatever a credential bug does upstream.
    #[test]
    fn each_backend_reaches_its_own_provider_and_no_other() {
        let claude = allow_list_for(Some("claude"));
        assert!(claude.iter().any(|h| h == "api.anthropic.com"), "{claude:?}");
        assert!(!claude.iter().any(|h| h == "api.z.ai"), "{claude:?}");

        let glm = allow_list_for(Some("glm"));
        assert!(glm.iter().any(|h| h == "api.z.ai"), "{glm:?}");
        assert!(
            !glm.iter().any(|h| h.contains("anthropic")),
            "a GLM VM must not be able to reach Anthropic: {glm:?}"
        );

        // Both still reach the forge — delivery is host-side, but a mission that
        // clones or fetches needs it.
        for l in [&claude, &glm] {
            assert!(l.iter().any(|h| h == "git.redclaw.dev"), "{l:?}");
        }

        let kimi = allow_list_for(Some("kimi"));
        assert!(kimi.iter().any(|h| h == "api.kimi.com"), "{kimi:?}");
        for other in ["api.z.ai", "api.anthropic.com"] {
            assert!(!kimi.iter().any(|h| h == other), "{kimi:?}");
        }

        // An unknown backend gets no model API at all rather than borrowing
        // somebody's: it cannot run anyway, and failing at a closed door beats
        // reaching the wrong endpoint with a credential.
        let unknown = allow_list_for(Some("rootfs-opus"));
        assert_eq!(unknown, vec!["git.redclaw.dev".to_string()], "{unknown:?}");
    }

    #[test]
    fn the_model_api_and_the_forge_are_reachable() {
        for h in ["api.anthropic.com", "git.redclaw.dev", "API.Anthropic.COM"] {
            assert!(host_allowed(h, &allow()), "{h} must be allowed");
        }
    }

    /// The check is a match, never a substring test. A name that merely CONTAINS
    /// an allowed one is a different host controlled by someone else.
    #[test]
    fn a_lookalike_host_is_not_allowed() {
        for h in [
            "api.anthropic.com.evil.test",
            "notapi.anthropic.com.attacker.io",
            // These contain an allowed suffix but are not that host. The first
            // PASSED before the character-class check was added — a unit test
            // found it, not review.
            "evil.test/api.anthropic.com",
            "[email protected]",
            "api.anthropic.com:443",
            "git.redclaw.dev.evil.test",
            "example.com",
            "",
            "   ",
        ] {
            assert!(!host_allowed(h, &allow()), "{h} must NOT be allowed");
        }
    }

    /// A raw address must not sidestep a list written in names.
    /// A local-model backend gets NO egress, and the 443 rule is untouched.
    ///
    /// The alternative design routed the node's Ollama through this proxy, which
    /// would have meant permitting port 11434 and an address the guest names.
    /// Both are refused here, still, and a `local-ornith` VM reaches the forge
    /// and nothing else — its model lives on the other socket entirely.
    #[test]
    fn a_local_model_backend_gets_no_egress_and_no_new_port() {
        let allow = allow_list_for(Some("local-ornith"));
        assert!(
            allow.iter().all(|a| a == "git.redclaw.dev"),
            "a local backend must reach only the forge, got {allow:?}"
        );
        for h in ["api.anthropic.com", "api.z.ai", "api.kimi.com", "127.0.0.1"] {
            assert!(!host_allowed(h, &allow), "{h} must NOT be reachable");
        }
        // The rules this design exists to avoid loosening.
        assert!(parse_target("anything:11434").is_err());
        assert!(parse_target("127.0.0.1:443").is_ok_and(|(h, _)| !host_allowed(&h, &allow)));
    }

    #[test]
    fn an_ip_literal_is_not_allowed() {
        let a = vec![".anthropic.com".to_string()];
        assert!(!host_allowed("[::1]", &a));
        assert!(!host_allowed("2606:4700::1111", &a));
    }

    /// A trailing dot is the same host to a resolver, so it must be to us.
    #[test]
    fn a_trailing_dot_does_not_bypass_the_list() {
        assert!(host_allowed("api.anthropic.com.", &allow()));
    }

    /// A `.domain` entry covers subdomains, and only real subdomains.
    #[test]
    fn a_dot_prefixed_entry_matches_subdomains_only() {
        let a = vec![".example.com".to_string()];
        assert!(host_allowed("a.example.com", &a));
        assert!(host_allowed("example.com", &a));
        assert!(!host_allowed("notexample.com", &a));
        assert!(!host_allowed("example.com.evil.test", &a));
    }

    /// Blanking the allow-list means no egress. Falling back to the default
    /// would hand a mission the network an operator had just taken away.
    #[test]
    fn an_empty_allow_list_denies_everything() {
        let none: Vec<String> = vec![];
        assert!(!host_allowed("api.anthropic.com", &none));
    }

    /// Only 443. Anything else turns the proxy into a general-purpose tunnel to
    /// whatever the allow-list happens to name.
    #[test]
    fn only_https_is_tunnelled() {
        assert_eq!(parse_target("api.anthropic.com:443").unwrap().1, 443);
        for bad in [
            "api.anthropic.com:22",
            "api.anthropic.com:80",
            "api.anthropic.com",
            "api.anthropic.com:not-a-port",
        ] {
            assert!(parse_target(bad).is_err(), "{bad} must be refused");
        }
    }
}