Post-1.0: Localhost seccomp on K8s, warm sandbox pool, server HPA

- K8sDriver.with_localhost_seccomp(profile): sandbox pods run under the
  STRICT allowlist instead of the runtime default. Proven live on kind:
  the harness installs the profile onto the node, a pod runs ordinary
  work as uid 10001, and unshare is kernel-denied inside the pod — the
  same probe the Docker suite uses, now passing on both targets
- Helm: sandbox.seccomp=localhost renders a DaemonSet that installs the
  chart-shipped profile into /var/lib/kubelet/seccomp on every node
  (ConfigMap + hostPath); ci/check-helm.sh enforces the chart copy stays
  byte-identical to images/seccomp/agent-profile.json and asserts the
  hardened render (DaemonSet + profile + HPA)
- server HPA (autoscaling/v2, CPU target) behind
  server.autoscaling.enabled
- SandboxManager.warm(n): a background warmer keeps n pre-provisioned
  sandboxes ready so an agent's first exec skips container startup;
  unhealthy pool entries are discarded, reuse never drains the pool,
  shutdown destroys assigned AND pooled. [sandbox] warm_pool config
  (default 0). Real-Docker test: prefill -> assign -> refill -> reuse ->
  clean shutdown

160 Rust tests + 4 live kind tests.

Co-Authored-By: Claude Fable 5 <[email protected]>

crates/tc-runtime/src/sandboxes.rs

@@ -22,6 +22,10 @@ pub struct SandboxManager {
     image: String,
     egress: bool,
     handles: Mutex<HashMap<AgentId, SandboxHandle>>,
+    /// Pre-provisioned, unassigned sandboxes (the warm pool).
+    pool: Mutex<Vec<SandboxHandle>>,
+    /// Warm-pool target; the background warmer keeps `pool` at this size.
+    warm_target: std::sync::atomic::AtomicUsize,
 }
 
 impl SandboxManager {
@@ -31,9 +35,59 @@ impl SandboxManager {
             image: image.to_owned(),
             egress: false,
             handles: Mutex::new(HashMap::new()),
+            pool: Mutex::new(Vec::new()),
+            warm_target: std::sync::atomic::AtomicUsize::new(0),
         }
     }
 
+    /// Starts the background warmer: keeps `target` sandboxes
+    /// pre-provisioned so an agent's first exec skips container startup.
+    pub fn warm(self: Arc<Self>, target: usize) -> Arc<Self> {
+        self.warm_target
+            .store(target, std::sync::atomic::Ordering::Relaxed);
+        let manager = self.clone();
+        tokio::spawn(async move {
+            loop {
+                let target = manager
+                    .warm_target
+                    .load(std::sync::atomic::Ordering::Relaxed);
+                if target == 0 {
+                    break;
+                }
+                let deficit = target.saturating_sub(manager.pool.lock().await.len());
+                for _ in 0..deficit {
+                    match manager.provision_one().await {
+                        Ok(handle) => manager.pool.lock().await.push(handle),
+                        Err(_) => break,
+                    }
+                }
+                tokio::time::sleep(std::time::Duration::from_millis(250)).await;
+            }
+        });
+        self
+    }
+
+    /// Unassigned warm sandboxes currently ready.
+    pub async fn pool_size(&self) -> usize {
+        self.pool.lock().await.len()
+    }
+
+    async fn provision_one(&self) -> Result<SandboxHandle, String> {
+        let short = uuid::Uuid::now_v7().simple().to_string();
+        let spec = SandboxSpec {
+            name: format!("tc-agent-{}", &short[short.len() - 12..]),
+            image: self.image.clone(),
+            memory_bytes: 512 * 1024 * 1024,
+            nano_cpus: 1_000_000_000,
+            pids_limit: 256,
+            egress: self.egress,
+        };
+        self.driver
+            .provision(&spec)
+            .await
+            .map_err(|e| format!("sandbox provision failed: {e}"))
+    }
+
     /// The browser-container variant: egress on, no credentials inside,
     /// all output tainted `web` by the calling tool.
     pub fn with_egress(mut self) -> SandboxManager {
@@ -51,20 +105,20 @@ impl SandboxManager {
             None => false,
         };
         if !alive {
-            let short = uuid::Uuid::now_v7().simple().to_string();
-            let spec = SandboxSpec {
-                name: format!("tc-agent-{}", &short[short.len() - 12..]),
-                image: self.image.clone(),
-                memory_bytes: 512 * 1024 * 1024,
-                nano_cpus: 1_000_000_000,
-                pids_limit: 256,
-                egress: self.egress,
+            // A warm sandbox if one is ready (and still healthy);
+            // otherwise provision inline.
+            let mut assigned = None;
+            while let Some(candidate) = self.pool.lock().await.pop() {
+                if self.driver.health(&candidate).await.unwrap_or(false) {
+                    assigned = Some(candidate);
+                    break;
+                }
+                let _ = self.driver.destroy(&candidate).await;
+            }
+            let handle = match assigned {
+                Some(handle) => handle,
+                None => self.provision_one().await?,
             };
-            let handle = self
-                .driver
-                .provision(&spec)
-                .await
-                .map_err(|e| format!("sandbox provision failed: {e}"))?;
             handles.insert(agent_id, handle);
         }
         let handle = handles.get(&agent_id).expect("just ensured");
@@ -74,11 +128,19 @@ impl SandboxManager {
             .map_err(|e| format!("sandbox exec failed: {e}"))
     }
 
-    /// Destroys every sandbox this manager provisioned.
+    /// Destroys every sandbox this manager provisioned — assigned and
+    /// pooled — and stops the warmer.
     pub async fn shutdown(&self) {
+        self.warm_target
+            .store(0, std::sync::atomic::Ordering::Relaxed);
         let mut handles = self.handles.lock().await;
         for (_, handle) in handles.drain() {
             let _ = self.driver.destroy(&handle).await;
         }
+        drop(handles);
+        let mut pool = self.pool.lock().await;
+        for handle in pool.drain(..) {
+            let _ = self.driver.destroy(&handle).await;
+        }
     }
 }

crates/tc-runtime/tests/warm_pool.rs

@@ -0,0 +1,80 @@
+//! Warm sandbox pool: pre-provisioned containers absorb the first-exec
+//! latency. Real Docker — the pool fills in the background, an exec
+//! takes a sandbox from it, and the warmer restores the target.
+
+use std::process::Command;
+use std::sync::Arc;
+use std::time::Duration;
+
+use tc_domain::AgentId;
+use tc_runtime::SandboxManager;
+use tc_sandbox::DockerDriver;
+
+const IMAGE: &str = "teamclaw/agent-base:dev";
+
+fn ensure_image() {
+    let exists = Command::new("docker")
+        .args(["image", "inspect", IMAGE])
+        .output()
+        .expect("docker available")
+        .status
+        .success();
+    if !exists {
+        let root = env!("CARGO_MANIFEST_DIR");
+        let status = Command::new("docker")
+            .args([
+                "build",
+                "-t",
+                IMAGE,
+                "-f",
+                &format!("{root}/../../images/agent-base/Dockerfile"),
+                &format!("{root}/../../images/agent-base"),
+            ])
+            .status()
+            .expect("docker build runs");
+        assert!(status.success());
+    }
+}
+
+async fn pool_reaches(manager: &SandboxManager, target: usize) {
+    for _ in 0..120 {
+        if manager.pool_size().await == target {
+            return;
+        }
+        tokio::time::sleep(Duration::from_millis(250)).await;
+    }
+    panic!(
+        "pool never reached {target} (now {})",
+        manager.pool_size().await
+    );
+}
+
+#[tokio::test]
+async fn the_pool_prefills_assigns_and_refills() {
+    ensure_image();
+    let driver: Arc<dyn tc_sandbox::SandboxDriver> =
+        Arc::new(DockerDriver::connect().expect("docker reachable"));
+    let manager = Arc::new(SandboxManager::new(driver, IMAGE)).warm(2);
+
+    // The warmer fills the pool without any exec happening.
+    pool_reaches(&manager, 2).await;
+
+    // An exec is served from the pool — and works.
+    let agent = AgentId::new();
+    let result = manager.exec(agent, "id -u").await.unwrap();
+    assert_eq!(result.stdout.trim(), "10001");
+
+    // The warmer restores the target while the agent keeps its sandbox.
+    pool_reaches(&manager, 2).await;
+    let again = manager.exec(agent, "echo still-mine").await.unwrap();
+    assert_eq!(again.stdout.trim(), "still-mine");
+    assert_eq!(
+        manager.pool_size().await,
+        2,
+        "reuse must not drain the pool"
+    );
+
+    // Shutdown destroys assigned AND pooled sandboxes.
+    manager.shutdown().await;
+    assert_eq!(manager.pool_size().await, 0);
+}