clawmates/crates/cm-sandbox/tests/k8s_security.rs

//! The K8s sandbox driver against a REAL kind cluster: the same kernel
//! probes as the Docker suite (uid, capabilities, no-new-privs, rootfs),
//! plus namespace policy objects. Run with `--features k8s-tests` after
//! `kind create cluster --name clawmates-test` and a `kind load` of the
//! agent image (the harness does both image steps itself).
#![cfg(feature = "k8s-tests")]

use std::process::Command;

use cm_sandbox::{K8sDriver, SandboxDriver, SandboxSpec};

const IMAGE: &str = "clawmates/agent-base:dev";
const CLUSTER: &str = "clawmates-test";
const CONTEXT: &str = "kind-clawmates-test";
const NAMESPACE: &str = "clawmates-sandboxes-test";

fn ensure_image_in_kind() {
    let exists = Command::new("docker")
        .args(["image", "inspect", IMAGE])
        .output()
        .expect("docker available")
        .status
        .success();
    if !exists {
        let root = env!("CARGO_MANIFEST_DIR");
        let status = Command::new("docker")
            .args([
                "build",
                "-t",
                IMAGE,
                "-f",
                &format!("{root}/../../images/agent-base/Dockerfile"),
                &format!("{root}/../../images/agent-base"),
            ])
            .status()
            .expect("docker build runs");
        assert!(status.success(), "agent-base image build failed");
    }
    let status = Command::new("kind")
        .args(["load", "docker-image", IMAGE, "--name", CLUSTER])
        .status()
        .expect("kind available");
    assert!(status.success(), "kind load failed");
}

fn ensure_image_in_cluster(cluster: &str) {
    ensure_image_in_kind();
    let status = Command::new("kind")
        .args(["load", "docker-image", IMAGE, "--name", cluster])
        .status()
        .expect("kind available");
    assert!(status.success(), "kind load into {cluster} failed");
}

async fn spawn(suffix: &str) -> (K8sDriver, cm_sandbox::SandboxHandle) {
    ensure_image_in_kind();
    let driver = K8sDriver::connect_with_context(NAMESPACE, CONTEXT)
        .await
        .expect("cluster reachable");
    let spec = SandboxSpec {
        name: format!("tc-k8s-{suffix}-{}", std::process::id()),
        image: IMAGE.into(),
        memory_bytes: 256 * 1024 * 1024,
        nano_cpus: 1_000_000_000,
        pids_limit: 128,
        egress: false,
        kind: cm_sandbox::SandboxKind::Agent,
        mounts: Vec::new(),
    };
    let handle = driver.provision(&spec).await.expect("pod provisions");
    (driver, handle)
}

#[tokio::test]
async fn pod_runs_hardened_with_writable_scratch_only() {
    let (driver, handle) = spawn("hard").await;

    let uid = driver.exec(&handle, &["id", "-u"]).await.unwrap();
    assert_eq!(uid.stdout.trim(), "10001", "stderr: {}", uid.stderr);

    let caps = driver
        .exec(&handle, &["grep", "CapEff", "/proc/self/status"])
        .await
        .unwrap();
    let value = caps.stdout.split_whitespace().last().unwrap_or("");
    assert_eq!(u64::from_str_radix(value, 16).unwrap(), 0);

    let nnp = driver
        .exec(&handle, &["grep", "NoNewPrivs", "/proc/self/status"])
        .await
        .unwrap();
    assert!(nnp.stdout.trim().ends_with('1'), "got: {}", nnp.stdout);

    let write_root = driver
        .exec(&handle, &["touch", "/etc/owned"])
        .await
        .unwrap();
    assert_ne!(write_root.exit_code, 0, "rootfs must reject writes");

    let write_tmp = driver
        .exec(&handle, &["touch", "/tmp/scratch"])
        .await
        .unwrap();
    assert_eq!(write_tmp.exit_code, 0, "stderr: {}", write_tmp.stderr);
    let write_home = driver
        .exec(&handle, &["touch", "/home/agent/file"])
        .await
        .unwrap();
    assert_eq!(write_home.exit_code, 0, "stderr: {}", write_home.stderr);

    driver.destroy(&handle).await.unwrap();
}

#[tokio::test]
async fn namespace_carries_pss_restricted_and_default_deny_policy() {
    let (driver, handle) = spawn("policy").await;

    // Assert through the API: PSS label + the deny-all NetworkPolicy.
    // (Kernel-level egress enforcement needs a NetworkPolicy-capable CNI;
    // kind's default kindnet does not enforce — production clusters do.)
    let options = kube::config::KubeConfigOptions {
        context: Some(CONTEXT.to_owned()),
        ..Default::default()
    };
    let config = kube::Config::from_kubeconfig(&options).await.unwrap();
    let client = kube::Client::try_from(config).unwrap();
    let namespaces: kube::Api<k8s_openapi::api::core::v1::Namespace> =
        kube::Api::all(client.clone());
    let ns = namespaces.get(NAMESPACE).await.unwrap();
    assert_eq!(
        ns.metadata
            .labels
            .unwrap()
            .get("pod-security.kubernetes.io/enforce")
            .map(String::as_str),
        Some("restricted")
    );

    let policies: kube::Api<k8s_openapi::api::networking::v1::NetworkPolicy> =
        kube::Api::namespaced(client, NAMESPACE);
    let deny = policies.get("sandbox-default-deny").await.unwrap();
    let spec = deny.spec.unwrap();
    assert_eq!(
        spec.policy_types.unwrap(),
        vec!["Ingress".to_owned(), "Egress".to_owned()]
    );
    assert!(spec.ingress.is_none() && spec.egress.is_none(), "deny-all");

    driver.destroy(&handle).await.unwrap();
}

#[tokio::test]
async fn destroy_removes_the_pod_and_health_reflects_it() {
    let (driver, handle) = spawn("life").await;
    assert!(driver.health(&handle).await.unwrap());
    driver.destroy(&handle).await.unwrap();
    for _ in 0..60 {
        if !driver.health(&handle).await.unwrap() {
            return;
        }
        tokio::time::sleep(std::time::Duration::from_millis(500)).await;
    }
    panic!("pod never disappeared");
}

/// With the strict allowlist installed on the node (the Helm DaemonSet's
/// job; the harness drops it into the kind node), pods run under
/// `Localhost` seccomp and the kernel refuses what the profile removed.
#[tokio::test]
async fn localhost_seccomp_profile_denies_unshare_inside_pods() {
    ensure_image_in_kind();
    let status = Command::new("docker")
        .args([
            "exec",
            "clawmates-test-control-plane",
            "mkdir",
            "-p",
            "/var/lib/kubelet/seccomp",
        ])
        .status()
        .expect("kind node reachable");
    assert!(status.success());
    let root = env!("CARGO_MANIFEST_DIR");
    let status = Command::new("docker")
        .args([
            "cp",
            &format!("{root}/../../images/seccomp/agent-profile.json"),
            "clawmates-test-control-plane:/var/lib/kubelet/seccomp/clawmates-agent-profile.json",
        ])
        .status()
        .expect("docker cp");
    assert!(status.success());

    let driver = K8sDriver::connect_with_context(NAMESPACE, CONTEXT)
        .await
        .expect("cluster reachable")
        .with_localhost_seccomp("clawmates-agent-profile.json");
    let spec = SandboxSpec {
        name: format!("tc-k8s-seccomp-{}", std::process::id()),
        image: IMAGE.into(),
        memory_bytes: 256 * 1024 * 1024,
        nano_cpus: 1_000_000_000,
        pids_limit: 128,
        egress: false,
        kind: cm_sandbox::SandboxKind::Agent,
        mounts: Vec::new(),
    };
    let handle = driver.provision(&spec).await.expect("pod provisions");

    // Ordinary work runs...
    let ok = driver.exec(&handle, &["id", "-u"]).await.unwrap();
    assert_eq!(ok.stdout.trim(), "10001");

    // ...but the syscalls stripped from the allowlist are gone — the
    // same kernel probe the Docker suite uses.
    let unshare = driver
        .exec(&handle, &["unshare", "--user", "true"])
        .await
        .unwrap();
    assert_ne!(unshare.exit_code, 0, "unshare must be denied: {unshare:?}");

    driver.destroy(&handle).await.unwrap();
}

/// The §15 egress claim, ENFORCED: on a NetworkPolicy-capable CNI
/// (Calico via scripts/netpol-cluster.sh) the namespace's default-deny
/// actually drops packets in the kernel — outbound connects and DNS both
/// fail inside the pod, while API-server exec still works (it is not pod
/// network). kindnet (the default suite's cluster) accepts the policy
/// object but never enforces it; this is the cluster where it bites.
#[tokio::test]
async fn calico_enforces_the_default_deny_egress() {
    const NETPOL_CONTEXT: &str = "kind-clawmates-netpol-test";
    const NETPOL_CLUSTER: &str = "clawmates-netpol-test";
    let have_cluster = Command::new("kind")
        .args(["get", "clusters"])
        .output()
        .map(|out| String::from_utf8_lossy(&out.stdout).contains(NETPOL_CLUSTER))
        .unwrap_or(false);
    if !have_cluster {
        eprintln!("skipped: run scripts/netpol-cluster.sh up first");
        return;
    }
    ensure_image_in_cluster(NETPOL_CLUSTER);

    let driver = K8sDriver::connect_with_context(NAMESPACE, NETPOL_CONTEXT)
        .await
        .expect("calico cluster reachable");
    let spec = SandboxSpec {
        name: format!("tc-netpol-{}", std::process::id()),
        image: IMAGE.into(),
        memory_bytes: 256 * 1024 * 1024,
        nano_cpus: 1_000_000_000,
        pids_limit: 128,
        egress: false,
        kind: cm_sandbox::SandboxKind::Agent,
        mounts: Vec::new(),
    };
    let handle = driver.provision(&spec).await.expect("pod provisions");

    // Exec works (API-server channel, not pod network).
    let ok = driver.exec(&handle, &["id", "-u"]).await.unwrap();
    assert_eq!(ok.stdout.trim(), "10001");

    // Raw outbound connect: dropped by Calico, not merely unconfigured.
    let direct = driver
        .exec(
            &handle,
            &["wget", "-T", "3", "-q", "-O", "-", "http://1.1.1.1"],
        )
        .await
        .unwrap();
    assert_ne!(direct.exit_code, 0, "egress to 1.1.1.1 must be dropped");

    // DNS (UDP egress to cluster DNS) is denied too.
    let dns = driver
        .exec(&handle, &["nslookup", "anthropic.com"])
        .await
        .unwrap();
    assert_ne!(dns.exit_code, 0, "DNS egress must be dropped");

    driver.destroy(&handle).await.unwrap();
}