clawmates/crates/cm-sandbox/tests/k8s_security.rs

//! The K8s sandbox driver against a REAL kind cluster: the same kernel
//! probes as the Docker suite (uid, capabilities, no-new-privs, rootfs),
//! plus namespace policy objects. Run with `--features k8s-tests` after
//! `kind create cluster --name clawmates-test` and a `kind load` of the
//! agent image (the harness does both image steps itself).
#![cfg(feature = "k8s-tests")]

use std::process::Command;

use cm_sandbox::{K8sDriver, SandboxDriver, SandboxSpec};

const IMAGE: &str = "clawmates/agent-base:dev";
const CLUSTER: &str = "clawmates-test";
const NAMESPACE: &str = "clawmates-sandboxes-test";

fn ensure_image_in_kind() {
    let exists = Command::new("docker")
        .args(["image", "inspect", IMAGE])
        .output()
        .expect("docker available")
        .status
        .success();
    if !exists {
        let root = env!("CARGO_MANIFEST_DIR");
        let status = Command::new("docker")
            .args([
                "build",
                "-t",
                IMAGE,
                "-f",
                &format!("{root}/../../images/agent-base/Dockerfile"),
                &format!("{root}/../../images/agent-base"),
            ])
            .status()
            .expect("docker build runs");
        assert!(status.success(), "agent-base image build failed");
    }
    let status = Command::new("kind")
        .args(["load", "docker-image", IMAGE, "--name", CLUSTER])
        .status()
        .expect("kind available");
    assert!(status.success(), "kind load failed");
}

async fn spawn(suffix: &str) -> (K8sDriver, cm_sandbox::SandboxHandle) {
    ensure_image_in_kind();
    let driver = K8sDriver::connect(NAMESPACE)
        .await
        .expect("cluster reachable");
    let spec = SandboxSpec {
        name: format!("tc-k8s-{suffix}-{}", std::process::id()),
        image: IMAGE.into(),
        memory_bytes: 256 * 1024 * 1024,
        nano_cpus: 1_000_000_000,
        pids_limit: 128,
        egress: false,
    };
    let handle = driver.provision(&spec).await.expect("pod provisions");
    (driver, handle)
}

#[tokio::test]
async fn pod_runs_hardened_with_writable_scratch_only() {
    let (driver, handle) = spawn("hard").await;

    let uid = driver.exec(&handle, &["id", "-u"]).await.unwrap();
    assert_eq!(uid.stdout.trim(), "10001", "stderr: {}", uid.stderr);

    let caps = driver
        .exec(&handle, &["grep", "CapEff", "/proc/self/status"])
        .await
        .unwrap();
    let value = caps.stdout.split_whitespace().last().unwrap_or("");
    assert_eq!(u64::from_str_radix(value, 16).unwrap(), 0);

    let nnp = driver
        .exec(&handle, &["grep", "NoNewPrivs", "/proc/self/status"])
        .await
        .unwrap();
    assert!(nnp.stdout.trim().ends_with('1'), "got: {}", nnp.stdout);

    let write_root = driver
        .exec(&handle, &["touch", "/etc/owned"])
        .await
        .unwrap();
    assert_ne!(write_root.exit_code, 0, "rootfs must reject writes");

    let write_tmp = driver
        .exec(&handle, &["touch", "/tmp/scratch"])
        .await
        .unwrap();
    assert_eq!(write_tmp.exit_code, 0, "stderr: {}", write_tmp.stderr);
    let write_home = driver
        .exec(&handle, &["touch", "/home/agent/file"])
        .await
        .unwrap();
    assert_eq!(write_home.exit_code, 0, "stderr: {}", write_home.stderr);

    driver.destroy(&handle).await.unwrap();
}

#[tokio::test]
async fn namespace_carries_pss_restricted_and_default_deny_policy() {
    let (driver, handle) = spawn("policy").await;

    // Assert through the API: PSS label + the deny-all NetworkPolicy.
    // (Kernel-level egress enforcement needs a NetworkPolicy-capable CNI;
    // kind's default kindnet does not enforce — production clusters do.)
    let client = kube::Client::try_default().await.unwrap();
    let namespaces: kube::Api<k8s_openapi::api::core::v1::Namespace> =
        kube::Api::all(client.clone());
    let ns = namespaces.get(NAMESPACE).await.unwrap();
    assert_eq!(
        ns.metadata
            .labels
            .unwrap()
            .get("pod-security.kubernetes.io/enforce")
            .map(String::as_str),
        Some("restricted")
    );

    let policies: kube::Api<k8s_openapi::api::networking::v1::NetworkPolicy> =
        kube::Api::namespaced(client, NAMESPACE);
    let deny = policies.get("sandbox-default-deny").await.unwrap();
    let spec = deny.spec.unwrap();
    assert_eq!(
        spec.policy_types.unwrap(),
        vec!["Ingress".to_owned(), "Egress".to_owned()]
    );
    assert!(spec.ingress.is_none() && spec.egress.is_none(), "deny-all");

    driver.destroy(&handle).await.unwrap();
}

#[tokio::test]
async fn destroy_removes_the_pod_and_health_reflects_it() {
    let (driver, handle) = spawn("life").await;
    assert!(driver.health(&handle).await.unwrap());
    driver.destroy(&handle).await.unwrap();
    for _ in 0..60 {
        if !driver.health(&handle).await.unwrap() {
            return;
        }
        tokio::time::sleep(std::time::Duration::from_millis(500)).await;
    }
    panic!("pod never disappeared");
}

/// With the strict allowlist installed on the node (the Helm DaemonSet's
/// job; the harness drops it into the kind node), pods run under
/// `Localhost` seccomp and the kernel refuses what the profile removed.
#[tokio::test]
async fn localhost_seccomp_profile_denies_unshare_inside_pods() {
    ensure_image_in_kind();
    let status = Command::new("docker")
        .args([
            "exec",
            "clawmates-test-control-plane",
            "mkdir",
            "-p",
            "/var/lib/kubelet/seccomp",
        ])
        .status()
        .expect("kind node reachable");
    assert!(status.success());
    let root = env!("CARGO_MANIFEST_DIR");
    let status = Command::new("docker")
        .args([
            "cp",
            &format!("{root}/../../images/seccomp/agent-profile.json"),
            "clawmates-test-control-plane:/var/lib/kubelet/seccomp/clawmates-agent-profile.json",
        ])
        .status()
        .expect("docker cp");
    assert!(status.success());

    let driver = K8sDriver::connect(NAMESPACE)
        .await
        .expect("cluster reachable")
        .with_localhost_seccomp("clawmates-agent-profile.json");
    let spec = SandboxSpec {
        name: format!("tc-k8s-seccomp-{}", std::process::id()),
        image: IMAGE.into(),
        memory_bytes: 256 * 1024 * 1024,
        nano_cpus: 1_000_000_000,
        pids_limit: 128,
        egress: false,
    };
    let handle = driver.provision(&spec).await.expect("pod provisions");

    // Ordinary work runs...
    let ok = driver.exec(&handle, &["id", "-u"]).await.unwrap();
    assert_eq!(ok.stdout.trim(), "10001");

    // ...but the syscalls stripped from the allowlist are gone — the
    // same kernel probe the Docker suite uses.
    let unshare = driver
        .exec(&handle, &["unshare", "--user", "true"])
        .await
        .unwrap();
    assert_ne!(unshare.exit_code, 0, "unshare must be denied: {unshare:?}");

    driver.destroy(&handle).await.unwrap();
}