//! The K8s sandbox driver against a REAL kind cluster: the same kernel //! probes as the Docker suite (uid, capabilities, no-new-privs, rootfs), //! plus namespace policy objects. Run with `--features k8s-tests` after //! `kind create cluster --name clawmates-test` and a `kind load` of the //! agent image (the harness does both image steps itself). #![cfg(feature = "k8s-tests")] use std::process::Command; use cm_sandbox::{K8sDriver, SandboxDriver, SandboxSpec}; const IMAGE: &str = "clawmates/agent-base:dev"; const CLUSTER: &str = "clawmates-test"; const CONTEXT: &str = "kind-clawmates-test"; const NAMESPACE: &str = "clawmates-sandboxes-test"; fn ensure_image_in_kind() { let exists = Command::new("docker") .args(["image", "inspect", IMAGE]) .output() .expect("docker available") .status .success(); if !exists { let root = env!("CARGO_MANIFEST_DIR"); let status = Command::new("docker") .args([ "build", "-t", IMAGE, "-f", &format!("{root}/../../images/agent-base/Dockerfile"), &format!("{root}/../../images/agent-base"), ]) .status() .expect("docker build runs"); assert!(status.success(), "agent-base image build failed"); } let status = Command::new("kind") .args(["load", "docker-image", IMAGE, "--name", CLUSTER]) .status() .expect("kind available"); assert!(status.success(), "kind load failed"); } fn ensure_image_in_cluster(cluster: &str) { ensure_image_in_kind(); let status = Command::new("kind") .args(["load", "docker-image", IMAGE, "--name", cluster]) .status() .expect("kind available"); assert!(status.success(), "kind load into {cluster} failed"); } async fn spawn(suffix: &str) -> (K8sDriver, cm_sandbox::SandboxHandle) { ensure_image_in_kind(); let driver = K8sDriver::connect_with_context(NAMESPACE, CONTEXT) .await .expect("cluster reachable"); let spec = SandboxSpec { name: format!("tc-k8s-{suffix}-{}", std::process::id()), image: IMAGE.into(), memory_bytes: 256 * 1024 * 1024, nano_cpus: 1_000_000_000, pids_limit: 128, egress: false, kind: cm_sandbox::SandboxKind::Agent, mounts: Vec::new(), }; let handle = driver.provision(&spec).await.expect("pod provisions"); (driver, handle) } #[tokio::test] async fn pod_runs_hardened_with_writable_scratch_only() { let (driver, handle) = spawn("hard").await; let uid = driver.exec(&handle, &["id", "-u"]).await.unwrap(); assert_eq!(uid.stdout.trim(), "10001", "stderr: {}", uid.stderr); let caps = driver .exec(&handle, &["grep", "CapEff", "/proc/self/status"]) .await .unwrap(); let value = caps.stdout.split_whitespace().last().unwrap_or(""); assert_eq!(u64::from_str_radix(value, 16).unwrap(), 0); let nnp = driver .exec(&handle, &["grep", "NoNewPrivs", "/proc/self/status"]) .await .unwrap(); assert!(nnp.stdout.trim().ends_with('1'), "got: {}", nnp.stdout); let write_root = driver .exec(&handle, &["touch", "/etc/owned"]) .await .unwrap(); assert_ne!(write_root.exit_code, 0, "rootfs must reject writes"); let write_tmp = driver .exec(&handle, &["touch", "/tmp/scratch"]) .await .unwrap(); assert_eq!(write_tmp.exit_code, 0, "stderr: {}", write_tmp.stderr); let write_home = driver .exec(&handle, &["touch", "/home/agent/file"]) .await .unwrap(); assert_eq!(write_home.exit_code, 0, "stderr: {}", write_home.stderr); driver.destroy(&handle).await.unwrap(); } #[tokio::test] async fn namespace_carries_pss_restricted_and_default_deny_policy() { let (driver, handle) = spawn("policy").await; // Assert through the API: PSS label + the deny-all NetworkPolicy. // (Kernel-level egress enforcement needs a NetworkPolicy-capable CNI; // kind's default kindnet does not enforce โ€” production clusters do.) let options = kube::config::KubeConfigOptions { context: Some(CONTEXT.to_owned()), ..Default::default() }; let config = kube::Config::from_kubeconfig(&options).await.unwrap(); let client = kube::Client::try_from(config).unwrap(); let namespaces: kube::Api = kube::Api::all(client.clone()); let ns = namespaces.get(NAMESPACE).await.unwrap(); assert_eq!( ns.metadata .labels .unwrap() .get("pod-security.kubernetes.io/enforce") .map(String::as_str), Some("restricted") ); let policies: kube::Api = kube::Api::namespaced(client, NAMESPACE); let deny = policies.get("sandbox-default-deny").await.unwrap(); let spec = deny.spec.unwrap(); assert_eq!( spec.policy_types.unwrap(), vec!["Ingress".to_owned(), "Egress".to_owned()] ); assert!(spec.ingress.is_none() && spec.egress.is_none(), "deny-all"); driver.destroy(&handle).await.unwrap(); } #[tokio::test] async fn destroy_removes_the_pod_and_health_reflects_it() { let (driver, handle) = spawn("life").await; assert!(driver.health(&handle).await.unwrap()); driver.destroy(&handle).await.unwrap(); for _ in 0..60 { if !driver.health(&handle).await.unwrap() { return; } tokio::time::sleep(std::time::Duration::from_millis(500)).await; } panic!("pod never disappeared"); } /// With the strict allowlist installed on the node (the Helm DaemonSet's /// job; the harness drops it into the kind node), pods run under /// `Localhost` seccomp and the kernel refuses what the profile removed. #[tokio::test] async fn localhost_seccomp_profile_denies_unshare_inside_pods() { ensure_image_in_kind(); let status = Command::new("docker") .args([ "exec", "clawmates-test-control-plane", "mkdir", "-p", "/var/lib/kubelet/seccomp", ]) .status() .expect("kind node reachable"); assert!(status.success()); let root = env!("CARGO_MANIFEST_DIR"); let status = Command::new("docker") .args([ "cp", &format!("{root}/../../images/seccomp/agent-profile.json"), "clawmates-test-control-plane:/var/lib/kubelet/seccomp/clawmates-agent-profile.json", ]) .status() .expect("docker cp"); assert!(status.success()); let driver = K8sDriver::connect_with_context(NAMESPACE, CONTEXT) .await .expect("cluster reachable") .with_localhost_seccomp("clawmates-agent-profile.json"); let spec = SandboxSpec { name: format!("tc-k8s-seccomp-{}", std::process::id()), image: IMAGE.into(), memory_bytes: 256 * 1024 * 1024, nano_cpus: 1_000_000_000, pids_limit: 128, egress: false, kind: cm_sandbox::SandboxKind::Agent, mounts: Vec::new(), }; let handle = driver.provision(&spec).await.expect("pod provisions"); // Ordinary work runs... let ok = driver.exec(&handle, &["id", "-u"]).await.unwrap(); assert_eq!(ok.stdout.trim(), "10001"); // ...but the syscalls stripped from the allowlist are gone โ€” the // same kernel probe the Docker suite uses. let unshare = driver .exec(&handle, &["unshare", "--user", "true"]) .await .unwrap(); assert_ne!(unshare.exit_code, 0, "unshare must be denied: {unshare:?}"); driver.destroy(&handle).await.unwrap(); } /// The ยง15 egress claim, ENFORCED: on a NetworkPolicy-capable CNI /// (Calico via scripts/netpol-cluster.sh) the namespace's default-deny /// actually drops packets in the kernel โ€” outbound connects and DNS both /// fail inside the pod, while API-server exec still works (it is not pod /// network). kindnet (the default suite's cluster) accepts the policy /// object but never enforces it; this is the cluster where it bites. #[tokio::test] async fn calico_enforces_the_default_deny_egress() { const NETPOL_CONTEXT: &str = "kind-clawmates-netpol-test"; const NETPOL_CLUSTER: &str = "clawmates-netpol-test"; let have_cluster = Command::new("kind") .args(["get", "clusters"]) .output() .map(|out| String::from_utf8_lossy(&out.stdout).contains(NETPOL_CLUSTER)) .unwrap_or(false); if !have_cluster { eprintln!("skipped: run scripts/netpol-cluster.sh up first"); return; } ensure_image_in_cluster(NETPOL_CLUSTER); let driver = K8sDriver::connect_with_context(NAMESPACE, NETPOL_CONTEXT) .await .expect("calico cluster reachable"); let spec = SandboxSpec { name: format!("tc-netpol-{}", std::process::id()), image: IMAGE.into(), memory_bytes: 256 * 1024 * 1024, nano_cpus: 1_000_000_000, pids_limit: 128, egress: false, kind: cm_sandbox::SandboxKind::Agent, mounts: Vec::new(), }; let handle = driver.provision(&spec).await.expect("pod provisions"); // Exec works (API-server channel, not pod network). let ok = driver.exec(&handle, &["id", "-u"]).await.unwrap(); assert_eq!(ok.stdout.trim(), "10001"); // Raw outbound connect: dropped by Calico, not merely unconfigured. let direct = driver .exec( &handle, &["wget", "-T", "3", "-q", "-O", "-", "http://1.1.1.1"], ) .await .unwrap(); assert_ne!(direct.exit_code, 0, "egress to 1.1.1.1 must be dropped"); // DNS (UDP egress to cluster DNS) is denied too. let dns = driver .exec(&handle, &["nslookup", "anthropic.com"]) .await .unwrap(); assert_ne!(dns.exit_code, 0, "DNS egress must be dropped"); driver.destroy(&handle).await.unwrap(); }