Merge: B4.6 microVM egress via vsock CONNECT proxy with a hostname allow-list

crates/bins/clawmates-node/src/egress.rs

@@ -0,0 +1,397 @@
+//! Host side of a microVM's only route out: an HTTP `CONNECT` proxy on a Unix
+//! socket, one per VM.
+//!
+//! # Why the guest has no network card
+//!
+//! It could have had one. A TAP device plus NAT is what the Firecracker
+//! write-ups do, and it was measured against this before being rejected:
+//!
+//! - `ip tuntap add` is **denied to the daemon user** (needs `CAP_NET_ADMIN`), so
+//!   TAP would need root to pre-provision devices at setup time — the same
+//!   privilege detour the loop-mounted rootfs already forced.
+//! - tank's `FORWARD` policy is `DROP` with Docker and Tailscale chains, so rules
+//!   would have to be *inserted* at position 1; appended ones die silently.
+//! - a leaked TAP device is a new class of host litter to reap.
+//!
+//! Against that, `CONNECT` needs no privilege at all, and it is better on the
+//! merits: the client hands us the **hostname**, so resolution happens here and
+//! the guest needs no DNS or `resolv.conf`; the allow-list is by name rather than
+//! by address; and nothing in the guest can reach the network except through this
+//! function. That is what the isolation plan's egress restriction actually asked
+//! for, and it is strictly tighter than the mission container's present full
+//! egress on `clawmates_edge`.
+//!
+//! The design rests on one measured fact: **`claude` honours `HTTPS_PROXY`**.
+//! With the proxy pointed at a closed port, `claude -p` fails with
+//! `ConnectionRefused` instead of answering. (That could only be measured in a
+//! container — inside a VM the CLI collapses every failure into `Execution
+//! error`.)
+//!
+//! # Shape
+//!
+//! Firecracker's convention for a guest-initiated connection is that the **host**
+//! listens on `<uds_path>_<port>`. The guest's agent pumps bytes from
+//! `127.0.0.1:3128` to vsock port 9002 and parses nothing, so all policy is here
+//! and a compromised guest cannot argue with it.
+
+use std::path::{Path, PathBuf};
+use std::sync::Arc;
+
+use tokio::io::{AsyncReadExt, AsyncWriteExt, BufReader};
+use tokio::net::{TcpStream, UnixListener, UnixStream};
+
+/// Port the guest dials. Must match `fcagent`'s `EGRESS_PORT`.
+pub const EGRESS_PORT: u32 = 9002;
+
+/// Hosts a mission may reach, unless `CLAWMATES_FC_EGRESS_ALLOW` overrides it.
+///
+/// Narrow on purpose: this is the whole egress surface of a mission, and the
+/// point of the exercise is that it is smaller than "the internet". An entry
+/// beginning with `.` matches subdomains.
+const DEFAULT_ALLOW: &[&str] = &[
+    // The model API. Without it there is no agent.
+    "api.anthropic.com",
+    ".anthropic.com",
+    // Our forge: clone, push, PRs.
+    "git.redclaw.dev",
+];
+
+/// Parse the allow-list once per VM.
+///
+/// An empty `CLAWMATES_FC_EGRESS_ALLOW` means **deny everything**, not "fall back
+/// to the default": an operator who blanked it asked for no egress, and quietly
+/// restoring the default would hand a mission the network they just took away.
+fn allow_list() -> Vec<String> {
+    match std::env::var("CLAWMATES_FC_EGRESS_ALLOW") {
+        Ok(raw) => raw
+            .split(',')
+            .map(|s| s.trim().to_ascii_lowercase())
+            .filter(|s| !s.is_empty())
+            .collect(),
+        Err(_) => DEFAULT_ALLOW.iter().map(|s| s.to_string()).collect(),
+    }
+}
+
+/// Is `host` allowed?
+///
+/// Case-insensitive, port already stripped. A leading `.` in an entry matches
+/// that domain and its subdomains; anything else must match exactly. Deliberately
+/// not a substring test — `api.anthropic.com.evil.test` contains the allowed name
+/// and must not pass.
+fn host_allowed(host: &str, allow: &[String]) -> bool {
+    let host = host.trim().trim_end_matches('.').to_ascii_lowercase();
+    if host.is_empty() {
+        return false;
+    }
+    // A hostname is letters, digits, dots and hyphens — nothing else. This is
+    // load-bearing, not hygiene: `evil.test/api.anthropic.com` ends with an
+    // allowed suffix and would otherwise PASS the match below. A unit test found
+    // it. Rejecting the character class also refuses IP literals, so an address
+    // cannot be used to sidestep a list written in names.
+    if !host
+        .chars()
+        .all(|c| c.is_ascii_alphanumeric() || c == '.' || c == '-')
+    {
+        return false;
+    }
+    allow.iter().any(|a| match a.strip_prefix('.') {
+        Some(domain) => host == domain || host.ends_with(&format!(".{domain}")),
+        None => host == *a,
+    })
+}
+
+/// Split `host:port` from a CONNECT target.
+///
+/// Only 443 is allowed. Permitting arbitrary ports would turn the proxy into a
+/// general-purpose tunnel to anything the allow-list happens to name, which is a
+/// different and much larger promise than "the agent can reach its API".
+fn parse_target(target: &str) -> Result<(String, u16), String> {
+    let (host, port) = target
+        .rsplit_once(':')
+        .ok_or_else(|| format!("CONNECT target {target:?} has no port"))?;
+    let port: u16 = port
+        .trim()
+        .parse()
+        .map_err(|_| format!("CONNECT target {target:?} has a non-numeric port"))?;
+    if port != 443 {
+        return Err(format!("port {port} is not permitted (only 443)"));
+    }
+    // Strip IPv6 brackets so the allow-list sees the same text either way.
+    let host = host.trim().trim_start_matches('[').trim_end_matches(']');
+    Ok((host.to_string(), port))
+}
+
+/// What happened to one connection. Returned so the caller can log it and the
+/// selftest can assert on it.
+#[derive(Debug, PartialEq, Eq)]
+pub enum Verdict {
+    Allowed(String),
+    Denied(String),
+    Malformed(String),
+}
+
+/// One header line, with a cap.
+///
+/// `read_line` has no limit, and a guest that never sends a newline would make
+/// the host allocate until it died. Read byte-wise instead — the reads come out
+/// of the BufReader, so this is cheap for lines this size, and it keeps ONE
+/// reader over the connection, which matters (see `serve`).
+async fn read_line_capped(reader: &mut BufReader<UnixStream>, cap: usize) -> Result<String, String> {
+    let mut out = Vec::new();
+    loop {
+        match reader.read_u8().await {
+            Ok(b'\n') => break,
+            Ok(b) => out.push(b),
+            // EOF mid-line: return what we have and let the caller judge it.
+            Err(e) if e.kind() == std::io::ErrorKind::UnexpectedEof => break,
+            Err(e) => return Err(format!("read: {e}")),
+        }
+        if out.len() > cap {
+            return Err(format!("a request line longer than {cap} bytes"));
+        }
+    }
+    Ok(String::from_utf8_lossy(&out)
+        .trim_end_matches('\r')
+        .to_string())
+}
+
+/// Serve one tunnelled connection.
+async fn serve(stream: UnixStream, allow: Arc<Vec<String>>) -> Verdict {
+    // ONE reader for the whole request. Wrapping the stream a second time would
+    // discard whatever the first reader had already buffered — including the
+    // first bytes of the TLS handshake — and the tunnel would come up looking
+    // fine and then stall on a corrupt stream.
+    let mut reader = BufReader::new(stream);
+
+    let line = match read_line_capped(&mut reader, 8 * 1024).await {
+        Ok(l) if !l.trim().is_empty() => l,
+        Ok(_) => return Verdict::Malformed("no request line".into()),
+        Err(e) => return Verdict::Malformed(e),
+    };
+
+    let mut parts = line.split_whitespace();
+    let method = parts.next().unwrap_or_default().to_ascii_uppercase();
+    let target = parts.next().unwrap_or_default().to_string();
+
+    if method != "CONNECT" {
+        // Plain HTTP would mean proxying a request we would then have to rewrite,
+        // and everything a mission needs is TLS. Refused with a status, so the
+        // client reports something better than a closed socket.
+        let _ = reply(reader.get_mut(), 405, "only CONNECT is supported").await;
+        return Verdict::Malformed(format!("method {method}"));
+    }
+
+    let (host, port) = match parse_target(&target) {
+        Ok(v) => v,
+        Err(e) => {
+            let _ = reply(reader.get_mut(), 400, &e).await;
+            return Verdict::Malformed(e);
+        }
+    };
+
+    if !host_allowed(&host, &allow) {
+        // 403 rather than a silent drop: a denial that looks like a network
+        // timeout is indistinguishable from a hung agent, and this codebase has
+        // paid for that confusion more than once.
+        let _ = reply(
+            reader.get_mut(),
+            403,
+            &format!("{host} is not on the egress allow-list"),
+        )
+        .await;
+        return Verdict::Denied(host);
+    }
+
+    // Consume the remaining request headers: they belong to the CONNECT, not to
+    // the tunnel.
+    loop {
+        match read_line_capped(&mut reader, 8 * 1024).await {
+            Ok(h) if h.trim().is_empty() => break,
+            Ok(_) => {}
+            Err(e) => return Verdict::Malformed(e),
+        }
+    }
+
+    let mut upstream = match TcpStream::connect((host.as_str(), port)).await {
+        Ok(s) => s,
+        Err(e) => {
+            let _ = reply(reader.get_mut(), 502, &format!("connect {host}:{port}: {e}")).await;
+            return Verdict::Denied(host);
+        }
+    };
+
+    if reply(reader.get_mut(), 200, "Connection established")
+        .await
+        .is_err()
+    {
+        return Verdict::Denied(host);
+    }
+
+    // Anything already buffered past the headers is tunnel payload — a client
+    // that pipelined its first TLS bytes would otherwise lose them.
+    let pending = reader.buffer().to_vec();
+    let mut stream = reader.into_inner();
+    if !pending.is_empty() && upstream.write_all(&pending).await.is_err() {
+        return Verdict::Denied(host);
+    }
+
+    // Bytes both ways until either side is done. Errors are not worth reporting:
+    // a closed connection is the normal end of a tunnel.
+    let _ = tokio::io::copy_bidirectional(&mut stream, &mut upstream).await;
+    Verdict::Allowed(host)
+}
+
+async fn reply(s: &mut UnixStream, code: u16, text: &str) -> std::io::Result<()> {
+    let reason = if code == 200 {
+        "Connection established"
+    } else {
+        "Forbidden"
+    };
+    // The body carries the reason for a non-200 so it reaches the agent's own
+    // error output, where whoever is reading a failed mission will see it.
+    let body = if code == 200 { String::new() } else { format!("{text}\n") };
+    let head = format!(
+        "HTTP/1.1 {code} {reason}\r\nContent-Length: {}\r\nConnection: close\r\n\r\n",
+        body.len()
+    );
+    s.write_all(head.as_bytes()).await?;
+    if !body.is_empty() {
+        s.write_all(body.as_bytes()).await?;
+    }
+    s.flush().await
+}
+
+/// Start this VM's proxy. Returns the socket path and the task serving it.
+///
+/// Bound **before** firecracker starts, because a guest that dials before the
+/// host is listening gets a connection refused it will not retry.
+pub fn start(uds: &Path, vm_id: &str) -> Result<(PathBuf, tokio::task::JoinHandle<()>), String> {
+    let path = PathBuf::from(format!("{}_{}", uds.display(), EGRESS_PORT));
+    // Firecracker does not clean these up any more than it cleans up its own
+    // socket, and a stale file makes bind fail with EADDRINUSE.
+    let _ = std::fs::remove_file(&path);
+    let listener =
+        UnixListener::bind(&path).map_err(|e| format!("bind {}: {e}", path.display()))?;
+
+    let allow = Arc::new(allow_list());
+    eprintln!(
+        "microvm {vm_id}: egress proxy on {} allowing {:?}",
+        path.display(),
+        allow
+    );
+    let vm = vm_id.to_string();
+    let task = tokio::spawn(async move {
+        loop {
+            match listener.accept().await {
+                Ok((s, _)) => {
+                    let allow = allow.clone();
+                    let vm = vm.clone();
+                    tokio::spawn(async move {
+                        match serve(s, allow).await {
+                            // Logged at every outcome: this is the audit trail of
+                            // everything a mission reached, and a denial that is
+                            // not logged is a mystery hang later.
+                            Verdict::Allowed(h) => eprintln!("microvm {vm}: egress -> {h}"),
+                            Verdict::Denied(h) => {
+                                eprintln!("microvm {vm}: egress DENIED {h}")
+                            }
+                            Verdict::Malformed(w) => {
+                                eprintln!("microvm {vm}: egress malformed request ({w})")
+                            }
+                        }
+                    });
+                }
+                Err(e) => {
+                    eprintln!("microvm {vm}: egress accept failed: {e}");
+                    return;
+                }
+            }
+        }
+    });
+    Ok((path, task))
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    fn allow() -> Vec<String> {
+        DEFAULT_ALLOW.iter().map(|s| s.to_string()).collect()
+    }
+
+    #[test]
+    fn the_model_api_and_the_forge_are_reachable() {
+        for h in ["api.anthropic.com", "git.redclaw.dev", "API.Anthropic.COM"] {
+            assert!(host_allowed(h, &allow()), "{h} must be allowed");
+        }
+    }
+
+    /// The check is a match, never a substring test. A name that merely CONTAINS
+    /// an allowed one is a different host controlled by someone else.
+    #[test]
+    fn a_lookalike_host_is_not_allowed() {
+        for h in [
+            "api.anthropic.com.evil.test",
+            "notapi.anthropic.com.attacker.io",
+            // These contain an allowed suffix but are not that host. The first
+            // PASSED before the character-class check was added — a unit test
+            // found it, not review.
+            "evil.test/api.anthropic.com",
+            "[email protected]",
+            "api.anthropic.com:443",
+            "git.redclaw.dev.evil.test",
+            "example.com",
+            "",
+            "   ",
+        ] {
+            assert!(!host_allowed(h, &allow()), "{h} must NOT be allowed");
+        }
+    }
+
+    /// A raw address must not sidestep a list written in names.
+    #[test]
+    fn an_ip_literal_is_not_allowed() {
+        let a = vec![".anthropic.com".to_string()];
+        assert!(!host_allowed("[::1]", &a));
+        assert!(!host_allowed("2606:4700::1111", &a));
+    }
+
+    /// A trailing dot is the same host to a resolver, so it must be to us.
+    #[test]
+    fn a_trailing_dot_does_not_bypass_the_list() {
+        assert!(host_allowed("api.anthropic.com.", &allow()));
+    }
+
+    /// A `.domain` entry covers subdomains, and only real subdomains.
+    #[test]
+    fn a_dot_prefixed_entry_matches_subdomains_only() {
+        let a = vec![".example.com".to_string()];
+        assert!(host_allowed("a.example.com", &a));
+        assert!(host_allowed("example.com", &a));
+        assert!(!host_allowed("notexample.com", &a));
+        assert!(!host_allowed("example.com.evil.test", &a));
+    }
+
+    /// Blanking the allow-list means no egress. Falling back to the default
+    /// would hand a mission the network an operator had just taken away.
+    #[test]
+    fn an_empty_allow_list_denies_everything() {
+        let none: Vec<String> = vec![];
+        assert!(!host_allowed("api.anthropic.com", &none));
+    }
+
+    /// Only 443. Anything else turns the proxy into a general-purpose tunnel to
+    /// whatever the allow-list happens to name.
+    #[test]
+    fn only_https_is_tunnelled() {
+        assert_eq!(parse_target("api.anthropic.com:443").unwrap().1, 443);
+        for bad in [
+            "api.anthropic.com:22",
+            "api.anthropic.com:80",
+            "api.anthropic.com",
+            "api.anthropic.com:not-a-port",
+        ] {
+            assert!(parse_target(bad).is_err(), "{bad} must be refused");
+        }
+    }
+}

crates/bins/clawmates-node/src/main.rs

@@ -19,6 +19,7 @@ use sysinfo::{Disks, System};
 use tokio::sync::{mpsc, Mutex};
 use tokio_tungstenite::tungstenite::Message;
 
+mod egress;
 mod microvm;
 mod rtc;
 

crates/bins/clawmates-node/src/microvm.rs

@@ -52,6 +52,12 @@ pub struct Vm {
     pgid: i32,
     workdir: PathBuf,
     uds: PathBuf,
+    /// This VM's egress proxy (see [`crate::egress`]) and its socket. Held so
+    /// destroy can abort the task: a listener outliving its VM would accept a
+    /// connection from the NEXT VM to reuse the path and proxy it under the dead
+    /// one's identity.
+    egress: Option<tokio::task::JoinHandle<()>>,
+    egress_uds: PathBuf,
 }
 
 pub type Vms = Arc<Mutex<HashMap<String, Vm>>>;
@@ -245,6 +251,19 @@ pub async fn create(
         .await
         .map_err(|e| format!("write {}: {e}", cfg_path.display()))?;
 
+    // Bound BEFORE firecracker starts: a guest that dials the host before the
+    // host is listening gets a connection refused it will not retry.
+    let (egress_uds, egress_task) = match crate::egress::start(&uds, vm_id) {
+        Ok((p, t)) => (p, Some(t)),
+        // Not fatal — a VM is still useful for work that needs no network — but
+        // it must be visible. `create`'s reply says whether egress exists, and
+        // the guest's own `proxy` flag says whether the guest end came up.
+        Err(e) => {
+            eprintln!("microvm {vm_id}: NO EGRESS ({e}) — the guest cannot reach any network");
+            (PathBuf::from(format!("{}_{}", uds.display(), crate::egress::EGRESS_PORT)), None)
+        }
+    };
+
     let log = std::fs::File::create(workdir.join("console.log"))
         .map_err(|e| format!("create console.log: {e}"))?;
     let errlog = log
@@ -268,16 +287,23 @@ pub async fn create(
     // setsid's child IS the new group leader, and its pid is the pgid.
     let pgid = child.id().ok_or("firecracker exited immediately")? as i32;
 
+    let host_egress = egress_task.is_some();
     let vm = Vm {
         pgid,
         workdir: workdir.clone(),
         uds: uds.clone(),
+        egress: egress_task,
+        egress_uds: egress_uds.clone(),
     };
 
     // Poll for the agent. 10s is generous: the measured boot-to-agent is under
     // a second, so anything near the ceiling means something is wrong.
     let deadline = std::time::Instant::now() + std::time::Duration::from_secs(10);
     let mut last = String::new();
+    // Assigned on the only path that reaches the use below; the timeout path
+    // returns. Declared without a default so a future edit cannot make "we never
+    // asked" read as "the guest said no".
+    let guest_egress;
     loop {
         if std::time::Instant::now() > deadline {
             let console = tokio::fs::read_to_string(workdir.join("console.log"))
@@ -291,7 +317,13 @@ pub async fn create(
             ));
         }
         match rpc(&uds, &json!({"op": "ping"})).await {
-            Ok(v) if v.get("ok").and_then(Value::as_bool) == Some(true) => break,
+            Ok(v) if v.get("ok").and_then(Value::as_bool) == Some(true) => {
+                // The guest reports whether ITS end of the tunnel is listening.
+                // Both ends must be up for the VM to have egress, and only the
+                // guest knows whether its image could bring loopback up.
+                guest_egress = v.get("proxy").and_then(Value::as_bool) == Some(true);
+                break;
+            }
             Ok(v) => last = v.to_string(),
             Err(e) => last = e,
         }
@@ -306,6 +338,12 @@ pub async fn create(
         // Which image actually booted, not which was asked for. A mission
         // artifact that records the request cannot show that the wrong VM ran.
         "rootfs": golden.display().to_string(),
+        // Egress needs BOTH ends. Reported rather than assumed so a caller that
+        // requires the network can refuse the VM up front, instead of a mission
+        // discovering it as an agent that cannot reach its API.
+        "egress": host_egress && guest_egress,
+        "egress_host": host_egress,
+        "egress_guest": guest_egress,
     }))
 }
 
@@ -381,13 +419,22 @@ async fn kill_group(pgid: i32) {
 pub async fn destroy(vms: &Vms, vm_id: &str) -> Result<Value, String> {
     check_id(vm_id)?;
     let vm = vms.lock().await.remove(vm_id);
-    let (pgid, workdir, uds) = match vm {
+    let (pgid, workdir, uds, egress_uds) = match vm {
-        Some(v) => (Some(v.pgid), v.workdir, v.uds),
+        Some(v) => {
-        // Not registered: still clean the path, so a VM created by a previous
+            // Abort first: a live listener would keep accepting on a path the
+            // next VM is about to reuse.
+            if let Some(t) = v.egress {
+                t.abort();
+            }
+            (Some(v.pgid), v.workdir, v.uds, v.egress_uds)
+        }
+        // Not registered: still clean the paths, so a VM created by a previous
         // incarnation of the daemon can be reaped rather than orphaned forever.
         None => {
             let wd = work_root().join("vms").join(vm_id);
-            (None, wd.clone(), wd.join("v.sock"))
+            let uds = wd.join("v.sock");
+            let eg = PathBuf::from(format!("{}_{}", uds.display(), crate::egress::EGRESS_PORT));
+            (None, wd.clone(), uds, eg)
         }
     };
     if let Some(pgid) = pgid {
@@ -396,6 +443,9 @@ pub async fn destroy(vms: &Vms, vm_id: &str) -> Result<Value, String> {
     // Give the group a moment to die before removing the files it has open.
     tokio::time::sleep(std::time::Duration::from_millis(100)).await;
     let _ = tokio::fs::remove_file(&uds).await;
+    // Same trap as firecracker's own socket: nothing unlinks these for us, and a
+    // stale file makes the next bind fail with EADDRINUSE.
+    let _ = tokio::fs::remove_file(&egress_uds).await;
     let removed = tokio::fs::remove_dir_all(&workdir).await.is_ok();
     Ok(json!({ "vm_id": vm_id, "killed": pgid.is_some(), "workdir_removed": removed }))
 }
@@ -517,18 +567,21 @@ pub async fn selftest() -> bool {
     }
 
     let started = std::time::Instant::now();
-    match create(&vms, id, 2, 1024, backend.as_deref()).await {
+    let created = match create(&vms, id, 2, 1024, backend.as_deref()).await {
-        Ok(v) => check(
+        Ok(v) => {
+            check(
                 true,
                 &format!("create ({} ms) {}", started.elapsed().as_millis(), v["vm_id"]),
                 String::new(),
-        ),
+            );
+            v
+        }
         Err(e) => {
             check(false, "create", e);
             println!("\n1 or more checks failed");
             return false;
         }
-    }
+    };
 
     // Inject a tar the way the mission checkout will travel.
     let mut tar = tar::Builder::new(Vec::new());
@@ -608,6 +661,77 @@ pub async fn selftest() -> bool {
         format!("{r:?}"),
     );
 
+    // Egress. Both ends of the tunnel must be up, and `create` says so rather
+    // than leaving the caller to find out from an agent that cannot reach its
+    // API. Only asserted for a backend whose image is expected to have iproute2.
+    if backend.is_some() {
+        check(
+            created["egress"] == json!(true),
+            "the VM reports working egress (both ends of the tunnel)",
+            format!(
+                "host={} guest={}",
+                created["egress_host"], created["egress_guest"]
+            ),
+        );
+
+        // The allow-list has to actually let the model API through — this is the
+        // check that says a mission could run — and it goes over the real tunnel:
+        // guest loopback → vsock → host proxy → TLS to Anthropic. `-sS -o
+        // /dev/null -w %{http_code}` because any 2xx/4xx from the API proves the
+        // connection completed; only a transport failure gives no code at all.
+        let r = exec(
+            &vms,
+            id,
+            "curl -sS -o /dev/null -w '%{http_code}' --max-time 25 https://api.anthropic.com/v1/messages",
+            None,
+            40,
+            None,
+        )
+        .await;
+        let code = r
+            .as_ref()
+            .map(|v| v["stdout"].as_str().unwrap_or_default().trim().to_string())
+            .unwrap_or_default();
+        check(
+            code.len() == 3 && code != "000",
+            "an allow-listed host is reachable from inside the VM",
+            format!("http_code={code:?} {r:?}"),
+        );
+
+        // And the denial must fire. Without this the allow-list is decoration:
+        // a proxy that allows everything passes the check above just as well.
+        let r = exec(
+            &vms,
+            id,
+            "curl -sS -o /dev/null -w '%{http_code}' --max-time 25 https://example.com",
+            None,
+            40,
+            None,
+        )
+        .await;
+        let out = r
+            .as_ref()
+            .map(|v| {
+                format!(
+                    "{}{}",
+                    v["stdout"].as_str().unwrap_or_default(),
+                    v["stderr"].as_str().unwrap_or_default()
+                )
+            })
+            .unwrap_or_default();
+        // The refusal must come from THE PROXY, not from a dead network. When
+        // this check merely asserted "did not reach it", it passed while the
+        // guest had no proxy env at all and curl was failing with "Could not
+        // resolve host" — a green result for a broken tunnel, which is the exact
+        // failure shape this project keeps paying for. `403` is the status the
+        // proxy returns after CONNECT for a host off the list.
+        check(
+            out.contains("403") || out.contains("allow-list"),
+            "a host that is NOT allow-listed is refused BY THE PROXY",
+            format!("expected a 403 from the proxy, got: {out:?}"),
+        );
+    }
+
     // Work produced in the guest must come back out.
     let _ = exec(&vms, id, "echo PRODUCED-OK > /work/out.txt", None, 30, None).await;
     let r = collect(&vms, id, "/work").await;
@@ -662,6 +786,43 @@ pub async fn selftest() -> bool {
                 "the guest provides git",
                 format!("{r:?}"),
             );
+
+            // The whole track in one check: a real agent turn, in a VM with no
+            // network card, reaching the API through the vsock tunnel on the
+            // subscription credential. Everything above can pass while this
+            // fails, which is why it is asked separately.
+            //
+            // Gated on a token being present, and SKIPPED loudly when it is not —
+            // it spends a small amount of subscription budget, so it must be a
+            // deliberate act rather than something every infra check does.
+            match std::env::var("CLAUDE_CODE_OAUTH_TOKEN") {
+                Ok(t) if !t.trim().is_empty() => {
+                    let r = exec(
+                        &vms,
+                        id,
+                        "cd /work && claude -p 'Reply with exactly: VM-OK'",
+                        None,
+                        180,
+                        // Subscription only. An ANTHROPIC_API_KEY would outrank
+                        // this token and bill the API instead of the plan.
+                        Some(&json!({ "CLAUDE_CODE_OAUTH_TOKEN": t })),
+                    )
+                    .await;
+                    let said = r
+                        .as_ref()
+                        .map(|v| v["stdout"].as_str().unwrap_or_default().to_string())
+                        .unwrap_or_default();
+                    check(
+                        said.contains("VM-OK"),
+                        "the agent completes a real turn inside the VM (subscription auth)",
+                        format!("said {said:?} — {r:?}"),
+                    );
+                }
+                _ => println!(
+                    "SKIP  no CLAUDE_CODE_OAUTH_TOKEN in the environment — the agent's \
+                     own turn is UNPROVEN by this run"
+                ),
+            }
         }
         None => println!(
             "SKIP  no agent CLI is required of backend {} — its contents are unchecked",

crates/bins/fcagent/src/main.rs

@@ -25,15 +25,39 @@
 //! and `crates/cm-api/src/microvm_client.rs` speak this and needed no change.
 
 use std::io::{Read, Write};
+use std::net::TcpListener;
 use std::os::unix::process::CommandExt;
 use std::path::Path;
 use std::process::{Command, Stdio};
+use std::sync::atomic::{AtomicBool, Ordering};
 use std::time::{Duration, Instant};
 
 use base64::Engine;
 use serde_json::{json, Value};
 
 const PORT: u32 = 9001;
+/// Guest-side egress proxy. The VM has **no network interface at all** — see
+/// `microvm.rs`, whose machine config declares no `network-interfaces` — so an
+/// agent CLI cannot reach the model API on its own. It reaches it by honouring
+/// `HTTPS_PROXY`, which is measured, not assumed: with the proxy pointed at a
+/// closed port, `claude -p` fails with `ConnectionRefused` instead of answering.
+///
+/// This listener is a dumb byte pump. It parses nothing and enforces nothing:
+/// the `CONNECT` request travels verbatim to the host, which speaks HTTP CONNECT
+/// and owns the allow-list. Keeping policy on the host means nothing running in
+/// the guest — including a compromised agent — can talk it into a different
+/// answer.
+const PROXY_PORT: u16 = 3128;
+/// Host-side vsock port the tunnel lands on. Firecracker's convention for a
+/// guest-initiated connection is that the HOST listens on `<uds_path>_<port>`.
+const EGRESS_PORT: u32 = 9002;
+/// `VMADDR_CID_HOST` — the hypervisor side of the vsock.
+const HOST_CID: u32 = 2;
+
+/// Whether the egress proxy is actually listening. Reported by `ping` so the
+/// host can refuse to hand a mission to a VM with no way out, rather than
+/// discovering it as an agent that hangs.
+static PROXY_UP: AtomicBool = AtomicBool::new(false);
 /// Cap on a single request. A hostile or broken host must not be able to make
 /// pid 1 allocate without bound and get the VM OOM-killed.
 const MAX_REQUEST: u32 = 512 * 1024 * 1024;
@@ -59,6 +83,8 @@ fn main() {
             .status();
     }
 
+    start_egress_proxy();
+
     let listener = match vsock::VsockListener::bind_with_cid_port(libc_vmaddr_cid_any(), PORT) {
         Ok(l) => l,
         Err(e) => {
@@ -94,6 +120,99 @@ fn libc_vmaddr_cid_any() -> u32 {
     u32::MAX
 }
 
+/// Bring up loopback and start the egress tunnel.
+///
+/// Loopback is not optional and not free: the guest's `lo` exists but starts
+/// **down**, and while it is down a listener on 127.0.0.1 *binds successfully*
+/// and then refuses every connection with `ENETUNREACH`. A bind-only check would
+/// have reported a working proxy. So `lo` goes up first, via `ip` — which is why
+/// `iproute2` is in the agent images.
+///
+/// Failure here is recorded, not fatal: exec still works, so a VM is still
+/// useful for work that needs no network. It is reported through `ping` so the
+/// host can decide, instead of a mission discovering it as an agent that hangs.
+fn start_egress_proxy() {
+    // Absolute paths, not `Command::new("ip")`. This process is pid 1, so its
+    // PATH is whatever the kernel handed it — and when PATH is unset, `execvp`
+    // falls back to a default that does NOT include `/usr/sbin`, which is exactly
+    // where Debian puts `ip`. Searching by name would fail on an image that has
+    // it, and the symptom would be a VM with no egress and no explanation.
+    const IP_CANDIDATES: &[&str] = &["/usr/sbin/ip", "/sbin/ip", "/usr/bin/ip", "/bin/ip"];
+    let Some(ip) = IP_CANDIDATES.iter().find(|p| Path::new(p).exists()) else {
+        eprintln!(
+            "FC-AGENT-NO-PROXY no `ip` binary in {IP_CANDIDATES:?} — no egress; \
+             add iproute2 to this image"
+        );
+        return;
+    };
+    match Command::new(ip).args(["link", "set", "lo", "up"]).status() {
+        Ok(s) if s.success() => {}
+        other => {
+            eprintln!("FC-AGENT-NO-PROXY `{ip} link set lo up` failed ({other:?}) — no egress");
+            return;
+        }
+    }
+
+    let listener = match TcpListener::bind(("127.0.0.1", PROXY_PORT)) {
+        Ok(l) => l,
+        Err(e) => {
+            eprintln!("FC-AGENT-NO-PROXY could not listen on 127.0.0.1:{PROXY_PORT}: {e}");
+            return;
+        }
+    };
+    PROXY_UP.store(true, Ordering::Relaxed);
+    println!("FC-AGENT-PROXY listening on 127.0.0.1:{PROXY_PORT} -> vsock {EGRESS_PORT}");
+    let _ = std::io::stdout().flush();
+
+    std::thread::spawn(move || {
+        for c in listener.incoming() {
+            match c {
+                // One thread per connection. An agent CLI opens several at once,
+                // and serving them in sequence would look like a hang.
+                Ok(tcp) => {
+                    std::thread::spawn(move || {
+                        if let Err(e) = tunnel(tcp) {
+                            eprintln!("FC-AGENT-PROXY-ERROR {e}");
+                        }
+                    });
+                }
+                Err(e) => eprintln!("FC-AGENT-PROXY-ERROR accept: {e}"),
+            }
+        }
+    });
+}
+
+/// Splice one TCP connection onto a fresh vsock connection to the host.
+///
+/// No parsing: whatever the client sent — `CONNECT host:443`, or an absolute-form
+/// request — is the host's business. The host answers with real HTTP, so a
+/// refusal reaches the client as a status code rather than a dropped socket.
+fn tunnel(tcp: std::net::TcpStream) -> Result<(), String> {
+    let vs = vsock::VsockStream::connect_with_cid_port(HOST_CID, EGRESS_PORT)
+        .map_err(|e| format!("vsock connect to host:{EGRESS_PORT}: {e}"))?;
+
+    let (mut tcp_r, mut tcp_w) = (
+        tcp.try_clone().map_err(|e| format!("clone tcp: {e}"))?,
+        tcp,
+    );
+    let (mut vs_r, mut vs_w) = (
+        vs.try_clone().map_err(|e| format!("clone vsock: {e}"))?,
+        vs,
+    );
+
+    // Each direction gets its own thread, and each shuts its peer's write side
+    // down when it ends. Without the shutdown the other half blocks forever on a
+    // half-closed connection and the CLI waits out its own timeout.
+    let up = std::thread::spawn(move || {
+        let _ = std::io::copy(&mut tcp_r, &mut vs_w);
+        let _ = vs_w.shutdown(std::net::Shutdown::Write);
+    });
+    let _ = std::io::copy(&mut vs_r, &mut tcp_w);
+    let _ = tcp_w.shutdown(std::net::Shutdown::Write);
+    let _ = up.join();
+    Ok(())
+}
+
 fn serve_one(s: &mut vsock::VsockStream) -> Result<(), String> {
     let mut len = [0u8; 4];
     s.read_exact(&mut len)
@@ -127,7 +246,14 @@ fn reply(s: &mut vsock::VsockStream, v: &Value) -> Result<(), String> {
 fn handle(req: &Value) -> Value {
     let op = req.get("op").and_then(Value::as_str).unwrap_or_default();
     match op {
-        "ping" => json!({ "ok": true, "pid": std::process::id() }),
+        "ping" => json!({
+            "ok": true,
+            "pid": std::process::id(),
+            // The host refuses to run a mission in a VM with no way out; this is
+            // how it knows. Reported rather than assumed because the image, not
+            // this binary, decides whether loopback can come up.
+            "proxy": PROXY_UP.load(Ordering::Relaxed),
+        }),
         "exec" => op_exec(req),
         "put" => op_put(req),
         "get" => op_get(req),
@@ -152,18 +278,21 @@ fn handle(req: &Value) -> Value {
 /// Errors name the key and never the value: the value is the secret, and an
 /// error string travels back over the wire and into logs.
 fn env_pairs(req: &Value) -> Result<Vec<(String, String)>, String> {
-    let Some(env) = req.get("env") else {
+    // Absent or `null` means the caller sent no variables of its own — which is
-        return Ok(Vec::new());
+    // NOT the same as "this command needs no environment". Both cases still get
+    // the proxy address below; returning early here meant every exec that passed
+    // no env ran with no HTTPS_PROXY, and the symptom was `curl` reporting
+    // "Could not resolve host" from a guest that had a working tunnel.
+    let empty = serde_json::Map::new();
+    let map = match req.get("env") {
+        None => &empty,
+        Some(v) if v.is_null() => &empty,
+        // Anything else that is not an object is a caller bug.
+        Some(v) => v
+            .as_object()
+            .ok_or("exec env must be an object of name → string")?,
     };
-    // `null` means "nothing to add" — that is what a caller with no credentials
+    let mut out = Vec::with_capacity(map.len() + 3);
-    // serialises. Anything else that is not an object is a caller bug.
-    if env.is_null() {
-        return Ok(Vec::new());
-    }
-    let Some(map) = env.as_object() else {
-        return Err("exec env must be an object of name → string".into());
-    };
-    let mut out = Vec::with_capacity(map.len());
     for (k, v) in map {
         let Some(val) = v.as_str() else {
             return Err(format!("exec env {k}: value must be a string"));
@@ -182,7 +311,38 @@ fn env_pairs(req: &Value) -> Result<Vec<(String, String)>, String> {
         }
         out.push((k.clone(), val.to_string()));
     }
-    Ok(out)
+
+    Ok(with_proxy_env(out, PROXY_UP.load(Ordering::Relaxed)))
+}
+
+/// Add the proxy variables the guest's own listener serves.
+///
+/// The agent runs the proxy, so the agent declares where it is. Deriving this on
+/// the host would mean two places agreeing on a port number, and the one that
+/// drifts is the one nobody tests.
+///
+/// Explicit caller values win: a caller can still point a command elsewhere or
+/// switch the proxy off for it. Matched case-insensitively because the lowercase
+/// spellings are equally conventional and a duplicate would leave which one
+/// applies up to the shell.
+fn with_proxy_env(mut env: Vec<(String, String)>, proxy_up: bool) -> Vec<(String, String)> {
+    if !proxy_up {
+        return env;
+    }
+    let addr = format!("http://127.0.0.1:{PROXY_PORT}");
+    for (k, v) in [
+        ("HTTPS_PROXY", addr.as_str()),
+        ("HTTP_PROXY", addr.as_str()),
+        // Without this the client would ask the proxy to reach the proxy.
+        ("NO_PROXY", "localhost,127.0.0.1"),
+    ] {
+        // `eq_ignore_ascii_case` covers the lowercase spelling, which is equally
+        // conventional; setting both would leave which one applies to the client.
+        if !env.iter().any(|(have, _)| have.eq_ignore_ascii_case(k)) {
+            env.push((k.to_string(), v.to_string()));
+        }
+    }
+    env
 }
 
 fn op_exec(req: &Value) -> Value {
@@ -347,6 +507,48 @@ fn op_get(req: &Value) -> Value {
 mod tests {
     use super::*;
 
+    /// The CLI reaches the API only by honouring HTTPS_PROXY (measured: with the
+    /// proxy at a closed port, `claude -p` fails ConnectionRefused instead of
+    /// answering), so a VM whose proxy is up must hand it the address.
+    #[test]
+    fn the_proxy_address_is_declared_when_the_proxy_is_up() {
+        let env = with_proxy_env(vec![], true);
+        let get = |k: &str| {
+            env.iter()
+                .find(|(a, _)| a == k)
+                .map(|(_, v)| v.as_str())
+                .unwrap_or("")
+        };
+        assert_eq!(get("HTTPS_PROXY"), "http://127.0.0.1:3128");
+        assert_eq!(get("HTTP_PROXY"), "http://127.0.0.1:3128");
+        // Otherwise the client asks the proxy to reach the proxy.
+        assert!(get("NO_PROXY").contains("127.0.0.1"));
+    }
+
+    /// And a VM with no proxy must not claim one: pointing a CLI at a listener
+    /// that is not there turns "no egress" into a connection error mid-run
+    /// instead of a fact the host can check before it starts.
+    #[test]
+    fn no_proxy_address_is_declared_when_the_proxy_is_down() {
+        assert!(with_proxy_env(vec![], false).is_empty());
+    }
+
+    /// An explicit value from the caller wins, in either spelling — otherwise
+    /// both would be set and which one applies would be up to the client.
+    #[test]
+    fn an_explicit_proxy_setting_is_not_overridden() {
+        let env = with_proxy_env(
+            vec![("https_proxy".into(), "http://elsewhere:8080".into())],
+            true,
+        );
+        let proxies: Vec<&str> = env
+            .iter()
+            .filter(|(k, _)| k.eq_ignore_ascii_case("https_proxy"))
+            .map(|(_, v)| v.as_str())
+            .collect();
+        assert_eq!(proxies, vec!["http://elsewhere:8080"]);
+    }
+
     /// The credential has to actually reach the command. This is the whole
     /// point of the op, and the failure it prevents is silent: a `claude -p`
     /// with no token hangs rather than erroring.

images/agent-toolchain/Dockerfile

@@ -30,8 +30,15 @@ FROM debian:bookworm-slim
 # Node 22 (Kimi Code needs >= 22.19; Claude Code is fine on it) is the runtime
 # for every agent CLI, so it belongs to the shared base rather than to any one
 # of them.
+#
+# `iproute2` is load-bearing on the microVM path, not a convenience. The guest
+# has no network interface at all; its only route out is the agent's proxy on
+# 127.0.0.1, and the guest's `lo` starts DOWN. While it is down a listener on
+# loopback *binds successfully* and then refuses every connection with
+# ENETUNREACH — so without `ip link set lo up` the VM has no egress and nothing
+# says so. Nothing else in the image needs it.
 RUN apt-get update && apt-get install -y --no-install-recommends \
-        ca-certificates curl gnupg git jq less procps \
+        ca-certificates curl gnupg git jq less procps iproute2 \
     && curl -fsSL https://deb.nodesource.com/setup_22.x | bash - \
     && apt-get install -y --no-install-recommends nodejs \
     && npm cache clean --force \