P6: strict seccomp allowlist + K8s sandbox driver proven on kind
- images/seccomp/agent-profile.json is now a TRUE ALLOWLIST: Docker's default profile (vendored from moby v27.5.1, defaultAction ERRNO) with 18 syscalls an agent never needs stripped from the allow groups (unshare, ptrace, bpf, mount family, setns, module loading, perf_event_open, process_vm_*, reboot, quotactl, ...); arch map trimmed to x86_64 + aarch64. All 6 Docker kernel assertions still green. - K8sDriver (tc-sandbox feature 'k8s', kube-rs): one hardened pod per sandbox — runAsUser 10001, cap-drop ALL, no-new-privs via allowPrivilegeEscalation=false, RuntimeDefault seccomp, read-only rootfs with emptyDir /tmp + /home/agent, resource limits, no service account token — in a PSS-restricted namespace carrying a default-deny NetworkPolicy (applied server-side apply, idempotent). Exec via the API server attach channel with exit codes parsed from v1.Status. - Live suite (feature 'k8s-tests') against a REAL kind cluster: uid / CapEff==0 / NoNewPrivs / rootfs probes from inside pods, PSS label + deny-all policy asserted via the API, lifecycle. Honest limits in the rustdoc: Localhost seccomp profile and CNI-enforced egress are per-cluster provisioning (kindnet does not enforce NetworkPolicy). - rustls 0.23 process provider pinned to ring at driver connect. - CI: dedicated sandbox-k8s job (helm/kind-action) running the suite. 149 Rust tests + 3 live kind tests; clippy clean including the k8s feature. Co-Authored-By: Claude Fable 5 <[email protected]>
This commit is contained in:
Omar Sobh
co-authored by
Claude Fable 5
Claude Fable 5
parent
70ec39f696
commit
84c51168be
@@ -10,12 +10,22 @@ publish.workspace = true
|
||||
async-trait = "0.1"
|
||||
bollard = "0.19"
|
||||
futures = "0.3"
|
||||
k8s-openapi = { version = "0.25", features = ["latest"], optional = true }
|
||||
kube = { version = "1", features = ["client", "rustls-tls", "ws"], default-features = false, optional = true }
|
||||
rustls = { version = "0.23", features = ["ring"], default-features = false, optional = true }
|
||||
serde = { workspace = true }
|
||||
serde_json = { workspace = true }
|
||||
thiserror = { workspace = true }
|
||||
tokio = { workspace = true }
|
||||
|
||||
[features]
|
||||
# The Kubernetes driver (kube-rs is a heavy dependency tree; the Docker
|
||||
# driver alone serves dev and the air-gapped target).
|
||||
k8s = ["dep:kube", "dep:k8s-openapi", "dep:rustls"]
|
||||
# Live kind-cluster tests for the K8s driver (dedicated CI job).
|
||||
k8s-tests = ["k8s"]
|
||||
|
||||
[dev-dependencies]
|
||||
tokio = { workspace = true }
|
||||
|
||||
[lints]
|
||||
workspace = true
|
||||
|
||||
@@ -0,0 +1,247 @@
|
||||
//! Kubernetes implementation of the sandbox driver (cloud target): one
|
||||
//! hardened pod per sandbox in a dedicated namespace with a default-deny
|
||||
//! NetworkPolicy.
|
||||
//!
|
||||
//! Hardening parity with the Docker driver: uid 10001, cap-drop ALL,
|
||||
//! no-new-privileges, read-only rootfs with emptyDir /tmp and /home/agent,
|
||||
//! resource limits. Differences, stated honestly: seccomp uses the
|
||||
//! runtime's `RuntimeDefault` profile (installing our strict allowlist as
|
||||
//! a `Localhost` profile is per-node provisioning, documented for managed
|
||||
//! clusters), and kernel-level egress enforcement of the NetworkPolicy
|
||||
//! requires a NetworkPolicy-capable CNI (kind's default kindnet does not
|
||||
//! enforce; production clusters with Calico/Cilium do).
|
||||
|
||||
use k8s_openapi::api::core::v1::{Namespace, Pod};
|
||||
use k8s_openapi::api::networking::v1::NetworkPolicy;
|
||||
use kube::api::{AttachParams, DeleteParams, ObjectMeta, Patch, PatchParams, PostParams};
|
||||
use kube::Api;
|
||||
use serde_json::json;
|
||||
use tokio::io::AsyncReadExt;
|
||||
|
||||
use crate::spec::{ExecResult, SandboxHandle, SandboxSpec};
|
||||
use crate::{SandboxDriver, SandboxError};
|
||||
|
||||
fn engine_err(e: impl std::fmt::Display) -> SandboxError {
|
||||
SandboxError::Engine(e.to_string())
|
||||
}
|
||||
|
||||
pub struct K8sDriver {
|
||||
client: kube::Client,
|
||||
namespace: String,
|
||||
}
|
||||
|
||||
impl K8sDriver {
|
||||
/// Connects via the ambient kubeconfig and ensures the sandbox
|
||||
/// namespace exists with its default-deny NetworkPolicy.
|
||||
pub async fn connect(namespace: &str) -> Result<K8sDriver, SandboxError> {
|
||||
// rustls 0.23 needs a process-level crypto provider when several
|
||||
// are linked; first caller wins, repeats are harmless.
|
||||
let _ = rustls::crypto::ring::default_provider().install_default();
|
||||
let client = kube::Client::try_default().await.map_err(engine_err)?;
|
||||
let driver = K8sDriver {
|
||||
client,
|
||||
namespace: namespace.to_owned(),
|
||||
};
|
||||
driver.ensure_namespace().await?;
|
||||
Ok(driver)
|
||||
}
|
||||
|
||||
async fn ensure_namespace(&self) -> Result<(), SandboxError> {
|
||||
let namespaces: Api<Namespace> = Api::all(self.client.clone());
|
||||
let ns = Namespace {
|
||||
metadata: ObjectMeta {
|
||||
name: Some(self.namespace.clone()),
|
||||
labels: Some(
|
||||
[
|
||||
// Pod Security Standards: refuse anything that
|
||||
// tries to relax the hardening below.
|
||||
(
|
||||
"pod-security.kubernetes.io/enforce".to_owned(),
|
||||
"restricted".to_owned(),
|
||||
),
|
||||
("app.kubernetes.io/name".to_owned(), "teamclaw".to_owned()),
|
||||
]
|
||||
.into(),
|
||||
),
|
||||
..Default::default()
|
||||
},
|
||||
..Default::default()
|
||||
};
|
||||
namespaces
|
||||
.patch(
|
||||
&self.namespace,
|
||||
&PatchParams::apply("teamclaw-sandbox").force(),
|
||||
&Patch::Apply(&ns),
|
||||
)
|
||||
.await
|
||||
.map_err(engine_err)?;
|
||||
|
||||
// Default-deny everything for sandbox pods; the orchestrator talks
|
||||
// to them via the API server's exec channel, not the pod network.
|
||||
let policies: Api<NetworkPolicy> = Api::namespaced(self.client.clone(), &self.namespace);
|
||||
let deny: NetworkPolicy = serde_json::from_value(json!({
|
||||
"apiVersion": "networking.k8s.io/v1",
|
||||
"kind": "NetworkPolicy",
|
||||
"metadata": { "name": "sandbox-default-deny", "namespace": self.namespace },
|
||||
"spec": {
|
||||
"podSelector": {},
|
||||
"policyTypes": ["Ingress", "Egress"]
|
||||
}
|
||||
}))
|
||||
.expect("static policy");
|
||||
policies
|
||||
.patch(
|
||||
"sandbox-default-deny",
|
||||
&PatchParams::apply("teamclaw-sandbox").force(),
|
||||
&Patch::Apply(&deny),
|
||||
)
|
||||
.await
|
||||
.map_err(engine_err)?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn pod_spec(&self, spec: &SandboxSpec) -> Pod {
|
||||
serde_json::from_value(json!({
|
||||
"apiVersion": "v1",
|
||||
"kind": "Pod",
|
||||
"metadata": {
|
||||
"name": spec.name,
|
||||
"namespace": self.namespace,
|
||||
"labels": { "app.kubernetes.io/name": "teamclaw-sandbox" }
|
||||
},
|
||||
"spec": {
|
||||
"restartPolicy": "Never",
|
||||
"automountServiceAccountToken": false,
|
||||
"securityContext": {
|
||||
"runAsNonRoot": true,
|
||||
"runAsUser": 10001,
|
||||
"runAsGroup": 10001,
|
||||
"seccompProfile": { "type": "RuntimeDefault" }
|
||||
},
|
||||
"containers": [{
|
||||
"name": "sandbox",
|
||||
"image": spec.image,
|
||||
"imagePullPolicy": "Never",
|
||||
"command": ["sleep", "infinity"],
|
||||
"securityContext": {
|
||||
"allowPrivilegeEscalation": false,
|
||||
"capabilities": { "drop": ["ALL"] },
|
||||
"readOnlyRootFilesystem": true
|
||||
},
|
||||
"resources": {
|
||||
"limits": {
|
||||
"memory": format!("{}", spec.memory_bytes),
|
||||
"cpu": format!("{}m", spec.nano_cpus / 1_000_000)
|
||||
}
|
||||
},
|
||||
"volumeMounts": [
|
||||
{ "name": "tmp", "mountPath": "/tmp" },
|
||||
{ "name": "home", "mountPath": "/home/agent" }
|
||||
]
|
||||
}],
|
||||
"volumes": [
|
||||
{ "name": "tmp", "emptyDir": { "sizeLimit": "64Mi" } },
|
||||
{ "name": "home", "emptyDir": { "sizeLimit": "256Mi" } }
|
||||
]
|
||||
}
|
||||
}))
|
||||
.expect("static pod spec")
|
||||
}
|
||||
|
||||
fn pods(&self) -> Api<Pod> {
|
||||
Api::namespaced(self.client.clone(), &self.namespace)
|
||||
}
|
||||
}
|
||||
|
||||
#[async_trait::async_trait]
|
||||
impl SandboxDriver for K8sDriver {
|
||||
async fn provision(&self, spec: &SandboxSpec) -> Result<SandboxHandle, SandboxError> {
|
||||
let pods = self.pods();
|
||||
pods.create(&PostParams::default(), &self.pod_spec(spec))
|
||||
.await
|
||||
.map_err(engine_err)?;
|
||||
// Wait until Running (image is preloaded, so this is fast).
|
||||
for _ in 0..120 {
|
||||
let pod = pods.get(&spec.name).await.map_err(engine_err)?;
|
||||
let phase = pod.status.as_ref().and_then(|s| s.phase.clone());
|
||||
match phase.as_deref() {
|
||||
Some("Running") => {
|
||||
return Ok(SandboxHandle {
|
||||
id: spec.name.clone(),
|
||||
name: spec.name.clone(),
|
||||
})
|
||||
}
|
||||
Some("Failed") => return Err(engine_err("sandbox pod failed to start")),
|
||||
_ => tokio::time::sleep(std::time::Duration::from_millis(500)).await,
|
||||
}
|
||||
}
|
||||
Err(engine_err("sandbox pod never reached Running"))
|
||||
}
|
||||
|
||||
async fn exec(&self, handle: &SandboxHandle, cmd: &[&str]) -> Result<ExecResult, SandboxError> {
|
||||
let pods = self.pods();
|
||||
let params = AttachParams::default().stdout(true).stderr(true);
|
||||
let mut attached = pods
|
||||
.exec(&handle.name, cmd.to_vec(), ¶ms)
|
||||
.await
|
||||
.map_err(engine_err)?;
|
||||
|
||||
let mut stdout = String::new();
|
||||
let mut stderr = String::new();
|
||||
if let Some(mut out) = attached.stdout() {
|
||||
out.read_to_string(&mut stdout).await.map_err(engine_err)?;
|
||||
}
|
||||
if let Some(mut err) = attached.stderr() {
|
||||
err.read_to_string(&mut stderr).await.map_err(engine_err)?;
|
||||
}
|
||||
let status = attached.take_status();
|
||||
attached.join().await.map_err(engine_err)?;
|
||||
|
||||
// Exit code travels in the v1.Status the API server sends on close.
|
||||
let exit_code = match status {
|
||||
Some(rx) => match rx.await {
|
||||
Some(s) if s.status.as_deref() == Some("Success") => 0,
|
||||
Some(s) => s
|
||||
.details
|
||||
.and_then(|d| {
|
||||
d.causes.unwrap_or_default().into_iter().find_map(|c| {
|
||||
(c.reason.as_deref() == Some("ExitCode"))
|
||||
.then(|| c.message.and_then(|m| m.parse().ok()))
|
||||
.flatten()
|
||||
})
|
||||
})
|
||||
.unwrap_or(1),
|
||||
None => 0,
|
||||
},
|
||||
None => 0,
|
||||
};
|
||||
Ok(ExecResult {
|
||||
exit_code,
|
||||
stdout,
|
||||
stderr,
|
||||
})
|
||||
}
|
||||
|
||||
async fn destroy(&self, handle: &SandboxHandle) -> Result<(), SandboxError> {
|
||||
self.pods()
|
||||
.delete(&handle.name, &DeleteParams::default().grace_period(0))
|
||||
.await
|
||||
.map_err(engine_err)?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn health(&self, handle: &SandboxHandle) -> Result<bool, SandboxError> {
|
||||
match self
|
||||
.pods()
|
||||
.get_opt(&handle.name)
|
||||
.await
|
||||
.map_err(engine_err)?
|
||||
{
|
||||
Some(pod) => Ok(pod
|
||||
.status
|
||||
.and_then(|s| s.phase)
|
||||
.is_some_and(|phase| phase == "Running")),
|
||||
None => Ok(false),
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,12 +1,17 @@
|
||||
//! Per-agent sandbox orchestration (spec §15): containers with no root, no
|
||||
//! capabilities, a seccomp deny profile, read-only rootfs, and no network.
|
||||
//! One `SandboxDriver` trait; the Docker implementation serves dev and the
|
||||
//! air-gapped compose target (the Kubernetes driver lands in P3).
|
||||
//! air-gapped compose target; the Kubernetes driver (feature `k8s`) runs
|
||||
//! hardened pods in a PSS-restricted, default-deny namespace.
|
||||
|
||||
mod docker;
|
||||
#[cfg(feature = "k8s")]
|
||||
mod k8s;
|
||||
mod spec;
|
||||
|
||||
pub use docker::DockerDriver;
|
||||
#[cfg(feature = "k8s")]
|
||||
pub use k8s::K8sDriver;
|
||||
pub use spec::{ExecResult, SandboxHandle, SandboxSpec};
|
||||
|
||||
#[derive(Debug, thiserror::Error)]
|
||||
|
||||
@@ -0,0 +1,146 @@
|
||||
//! The K8s sandbox driver against a REAL kind cluster: the same kernel
|
||||
//! probes as the Docker suite (uid, capabilities, no-new-privs, rootfs),
|
||||
//! plus namespace policy objects. Run with `--features k8s-tests` after
|
||||
//! `kind create cluster --name teamclaw-test` and a `kind load` of the
|
||||
//! agent image (the harness does both image steps itself).
|
||||
#![cfg(feature = "k8s-tests")]
|
||||
|
||||
use std::process::Command;
|
||||
|
||||
use tc_sandbox::{K8sDriver, SandboxDriver, SandboxSpec};
|
||||
|
||||
const IMAGE: &str = "teamclaw/agent-base:dev";
|
||||
const CLUSTER: &str = "teamclaw-test";
|
||||
const NAMESPACE: &str = "teamclaw-sandboxes-test";
|
||||
|
||||
fn ensure_image_in_kind() {
|
||||
let exists = Command::new("docker")
|
||||
.args(["image", "inspect", IMAGE])
|
||||
.output()
|
||||
.expect("docker available")
|
||||
.status
|
||||
.success();
|
||||
if !exists {
|
||||
let root = env!("CARGO_MANIFEST_DIR");
|
||||
let status = Command::new("docker")
|
||||
.args([
|
||||
"build",
|
||||
"-t",
|
||||
IMAGE,
|
||||
"-f",
|
||||
&format!("{root}/../../images/agent-base/Dockerfile"),
|
||||
&format!("{root}/../../images/agent-base"),
|
||||
])
|
||||
.status()
|
||||
.expect("docker build runs");
|
||||
assert!(status.success(), "agent-base image build failed");
|
||||
}
|
||||
let status = Command::new("kind")
|
||||
.args(["load", "docker-image", IMAGE, "--name", CLUSTER])
|
||||
.status()
|
||||
.expect("kind available");
|
||||
assert!(status.success(), "kind load failed");
|
||||
}
|
||||
|
||||
async fn spawn(suffix: &str) -> (K8sDriver, tc_sandbox::SandboxHandle) {
|
||||
ensure_image_in_kind();
|
||||
let driver = K8sDriver::connect(NAMESPACE)
|
||||
.await
|
||||
.expect("cluster reachable");
|
||||
let spec = SandboxSpec {
|
||||
name: format!("tc-k8s-{suffix}-{}", std::process::id()),
|
||||
image: IMAGE.into(),
|
||||
memory_bytes: 256 * 1024 * 1024,
|
||||
nano_cpus: 1_000_000_000,
|
||||
pids_limit: 128,
|
||||
};
|
||||
let handle = driver.provision(&spec).await.expect("pod provisions");
|
||||
(driver, handle)
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn pod_runs_hardened_with_writable_scratch_only() {
|
||||
let (driver, handle) = spawn("hard").await;
|
||||
|
||||
let uid = driver.exec(&handle, &["id", "-u"]).await.unwrap();
|
||||
assert_eq!(uid.stdout.trim(), "10001", "stderr: {}", uid.stderr);
|
||||
|
||||
let caps = driver
|
||||
.exec(&handle, &["grep", "CapEff", "/proc/self/status"])
|
||||
.await
|
||||
.unwrap();
|
||||
let value = caps.stdout.split_whitespace().last().unwrap_or("");
|
||||
assert_eq!(u64::from_str_radix(value, 16).unwrap(), 0);
|
||||
|
||||
let nnp = driver
|
||||
.exec(&handle, &["grep", "NoNewPrivs", "/proc/self/status"])
|
||||
.await
|
||||
.unwrap();
|
||||
assert!(nnp.stdout.trim().ends_with('1'), "got: {}", nnp.stdout);
|
||||
|
||||
let write_root = driver
|
||||
.exec(&handle, &["touch", "/etc/owned"])
|
||||
.await
|
||||
.unwrap();
|
||||
assert_ne!(write_root.exit_code, 0, "rootfs must reject writes");
|
||||
|
||||
let write_tmp = driver
|
||||
.exec(&handle, &["touch", "/tmp/scratch"])
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(write_tmp.exit_code, 0, "stderr: {}", write_tmp.stderr);
|
||||
let write_home = driver
|
||||
.exec(&handle, &["touch", "/home/agent/file"])
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(write_home.exit_code, 0, "stderr: {}", write_home.stderr);
|
||||
|
||||
driver.destroy(&handle).await.unwrap();
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn namespace_carries_pss_restricted_and_default_deny_policy() {
|
||||
let (driver, handle) = spawn("policy").await;
|
||||
|
||||
// Assert through the API: PSS label + the deny-all NetworkPolicy.
|
||||
// (Kernel-level egress enforcement needs a NetworkPolicy-capable CNI;
|
||||
// kind's default kindnet does not enforce — production clusters do.)
|
||||
let client = kube::Client::try_default().await.unwrap();
|
||||
let namespaces: kube::Api<k8s_openapi::api::core::v1::Namespace> =
|
||||
kube::Api::all(client.clone());
|
||||
let ns = namespaces.get(NAMESPACE).await.unwrap();
|
||||
assert_eq!(
|
||||
ns.metadata
|
||||
.labels
|
||||
.unwrap()
|
||||
.get("pod-security.kubernetes.io/enforce")
|
||||
.map(String::as_str),
|
||||
Some("restricted")
|
||||
);
|
||||
|
||||
let policies: kube::Api<k8s_openapi::api::networking::v1::NetworkPolicy> =
|
||||
kube::Api::namespaced(client, NAMESPACE);
|
||||
let deny = policies.get("sandbox-default-deny").await.unwrap();
|
||||
let spec = deny.spec.unwrap();
|
||||
assert_eq!(
|
||||
spec.policy_types.unwrap(),
|
||||
vec!["Ingress".to_owned(), "Egress".to_owned()]
|
||||
);
|
||||
assert!(spec.ingress.is_none() && spec.egress.is_none(), "deny-all");
|
||||
|
||||
driver.destroy(&handle).await.unwrap();
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn destroy_removes_the_pod_and_health_reflects_it() {
|
||||
let (driver, handle) = spawn("life").await;
|
||||
assert!(driver.health(&handle).await.unwrap());
|
||||
driver.destroy(&handle).await.unwrap();
|
||||
for _ in 0..60 {
|
||||
if !driver.health(&handle).await.unwrap() {
|
||||
return;
|
||||
}
|
||||
tokio::time::sleep(std::time::Duration::from_millis(500)).await;
|
||||
}
|
||||
panic!("pod never disappeared");
|
||||
}
|
||||
Reference in New Issue
Block a user