Compare commits
3
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
ac6bf72943 | ||
|
|
0d9498ec6e | ||
|
|
15e7608e4a |
@@ -160,6 +160,146 @@ const MISSIONS_HOST_ROOT: &str = "/var/lib/clawmates-missions";
|
||||
/// mission so this rarely bites. Long-term: copy-on-write per mission.
|
||||
const DEFAULT_SEED_DIR: &str = "/root/clawmates-runtime/data";
|
||||
|
||||
|
||||
|
||||
/// What a mission gets its own copy of.
|
||||
///
|
||||
/// An allow-list, not the whole directory. The seed dir is **1.7 GB** on
|
||||
/// gw-04 and 1.5 GB of that is a vestigial `.rustup` — a Rust toolchain that
|
||||
/// installed itself into the data dir back when `HOME=/zeroclaw-data` and the
|
||||
/// image had no toolchain. The image now ships Rust at `/usr/local/cargo`,
|
||||
/// which is what the container's PATH actually resolves (verified live), so
|
||||
/// that copy is dead weight. Copying it per mission would cost tens of
|
||||
/// seconds and ~17 GB across ten concurrent missions.
|
||||
///
|
||||
/// So: copy what carries per-mission identity or secrets, and leave the
|
||||
/// caches and toolchains behind.
|
||||
const SEEDED_PATHS: &[&str] = &[
|
||||
// The whole point: config.toml carries the §15 door bearer token, and
|
||||
// data/ holds sessions.db + devices.db. ~26 MB.
|
||||
".zeroclaw",
|
||||
// Door MCP config — also a bearer token.
|
||||
"clawmates-mcp.json",
|
||||
// Claude Code's own state and credentials (~16 MB). Per-mission so a
|
||||
// token refresh or project state in one mission cannot leak into another.
|
||||
".claude",
|
||||
".claude.json",
|
||||
// Per-CLI state for the alternate backends; small.
|
||||
".kimi-code",
|
||||
"glm-home",
|
||||
// The seeded agent library.
|
||||
"agents",
|
||||
];
|
||||
|
||||
/// Deliberately NOT copied — caches and toolchains, no secrets, expensive:
|
||||
/// `.rustup` (1.5 GB, vestigial), `.npm` (85 MB), `.cargo`, `.cache`,
|
||||
/// `.local`. A mission that needs them reads the image's copies.
|
||||
fn copy_script() -> String {
|
||||
let mut out = String::from("set -e\n");
|
||||
for p in SEEDED_PATHS {
|
||||
// Missing entries are normal — a fresh deployment has no .kimi-code
|
||||
// until Kimi is first used — so absence must not fail the copy.
|
||||
out.push_str(&format!(
|
||||
"if [ -e '/seed/{p}' ]; then cp -a '/seed/{p}' /dst/; fi\n"
|
||||
));
|
||||
}
|
||||
out
|
||||
}
|
||||
|
||||
/// Give this mission its own copy of the runtime seed data.
|
||||
///
|
||||
/// Every per-mission container used to bind-mount the SAME host seed dir as
|
||||
/// `/zeroclaw-data` — shared with each other and with the singleton runtime.
|
||||
/// That directory holds `config.toml`, which carries the §15 door bearer
|
||||
/// token, plus `sessions.db` and `devices.db`. So a mission could read another
|
||||
/// mission's credential, and anything it wrote there was inherited by every
|
||||
/// later mission. Teardown never cleaned it, because teardown only removes
|
||||
/// `/var/lib/clawmates-missions/{id}`.
|
||||
///
|
||||
/// The code already knew: the comment on `DEFAULT_SEED_DIR` names the sqlite
|
||||
/// race and calls copy-on-write per mission the long-term fix. This is that.
|
||||
///
|
||||
/// The copy runs in a throwaway container because cm-api cannot see the seed
|
||||
/// dir — it hands that host path to Docker but never mounts it itself. The
|
||||
/// runtime image is reused so nothing extra is pulled.
|
||||
///
|
||||
/// Failure is fatal to container creation on purpose. Falling back to the
|
||||
/// shared mount would silently restore the credential-sharing this removes,
|
||||
/// and a silent fallback to a weaker posture is the failure mode this
|
||||
/// codebase keeps paying for.
|
||||
async fn seed_runtime_data(
|
||||
docker: &Docker,
|
||||
image: &str,
|
||||
seed_dir: &str,
|
||||
dest_dir: &str,
|
||||
) -> Result<(), String> {
|
||||
let name = format!("cm-seed-{}", Uuid::now_v7().simple());
|
||||
let config = ContainerCreateBody {
|
||||
image: Some(image.to_string()),
|
||||
entrypoint: Some(vec!["/bin/sh".to_string()]),
|
||||
cmd: Some(vec!["-c".to_string(), copy_script()]),
|
||||
host_config: Some(HostConfig {
|
||||
mounts: Some(vec![
|
||||
Mount {
|
||||
target: Some("/seed".to_string()),
|
||||
source: Some(seed_dir.to_string()),
|
||||
typ: Some(MountTypeEnum::BIND),
|
||||
read_only: Some(true),
|
||||
..Default::default()
|
||||
},
|
||||
Mount {
|
||||
target: Some("/dst".to_string()),
|
||||
source: Some(dest_dir.to_string()),
|
||||
typ: Some(MountTypeEnum::BIND),
|
||||
read_only: Some(false),
|
||||
..Default::default()
|
||||
},
|
||||
]),
|
||||
auto_remove: Some(true),
|
||||
..Default::default()
|
||||
}),
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
docker
|
||||
.create_container(
|
||||
Some(CreateContainerOptions {
|
||||
name: Some(name.clone()),
|
||||
..Default::default()
|
||||
}),
|
||||
config,
|
||||
)
|
||||
.await
|
||||
.map_err(|e| format!("create seed copier: {e}"))?;
|
||||
docker
|
||||
.start_container(&name, None::<StartContainerOptions>)
|
||||
.await
|
||||
.map_err(|e| format!("start seed copier: {e}"))?;
|
||||
|
||||
// `auto_remove` means the container disappears the moment it exits, so
|
||||
// poll for absence rather than waiting on it.
|
||||
for _ in 0..120 {
|
||||
tokio::time::sleep(std::time::Duration::from_millis(250)).await;
|
||||
match docker
|
||||
.inspect_container(&name, None::<InspectContainerOptions>)
|
||||
.await
|
||||
{
|
||||
Err(_) => return Ok(()),
|
||||
Ok(info) => {
|
||||
let running = info
|
||||
.state
|
||||
.as_ref()
|
||||
.and_then(|st| st.running)
|
||||
.unwrap_or(false);
|
||||
if !running {
|
||||
return Ok(());
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
Err(format!("seed copy into {dest_dir} did not finish in 30s"))
|
||||
}
|
||||
|
||||
/// Deterministic docker container name for a mission's runtime.
|
||||
/// Uses the full UUID hex — UUIDv7 encodes time in the leading bytes,
|
||||
/// so a short prefix isn't guaranteed unique across missions minted
|
||||
@@ -257,6 +397,12 @@ impl MissionRuntimeProvisioner {
|
||||
let _ = tokio::fs::create_dir_all(&mission_dir).await;
|
||||
let seed_dir = std::env::var("CLAWMATES_RUNTIME_SEED_DIR")
|
||||
.unwrap_or_else(|_| DEFAULT_SEED_DIR.to_string());
|
||||
// Per-mission copy of the seed data. See `seed_runtime_data`: sharing
|
||||
// one directory meant sharing the door token and letting any mission
|
||||
// poison every later one.
|
||||
let runtime_data_dir = format!("{mission_dir}/runtime-data");
|
||||
let _ = tokio::fs::create_dir_all(&runtime_data_dir).await;
|
||||
seed_runtime_data(&self.docker, &self.image, &seed_dir, &runtime_data_dir).await?;
|
||||
let mounts = vec![
|
||||
// Mount just this mission's directory. Agents can navigate
|
||||
// its `/repo` subdir but never see other missions'.
|
||||
@@ -267,14 +413,14 @@ impl MissionRuntimeProvisioner {
|
||||
read_only: Some(false),
|
||||
..Default::default()
|
||||
},
|
||||
// Share the shared-runtime data dir so this gateway inherits
|
||||
// the seeded agent library (`claw_*` templates). We then
|
||||
// mint a per-mission pairing code via /admin/paircode/new
|
||||
// below — the mint writes into the shared devices.db but
|
||||
// the resulting token is unique to this mission.
|
||||
// This mission's OWN copy of the seeded agent library
|
||||
// (`claw_*` templates). Copied rather than shared, so its
|
||||
// config.toml — which carries the door bearer token — and its
|
||||
// sqlite files belong to this mission alone and are removed with
|
||||
// it by `teardown_container`.
|
||||
Mount {
|
||||
target: Some("/zeroclaw-data".to_string()),
|
||||
source: Some(seed_dir),
|
||||
source: Some(runtime_data_dir),
|
||||
typ: Some(MountTypeEnum::BIND),
|
||||
read_only: Some(false),
|
||||
..Default::default()
|
||||
@@ -990,4 +1136,75 @@ allowed_tools = ["file_read", "file_edit"]
|
||||
let url = endpoint_url("cm-runtime-mission-abc");
|
||||
assert_eq!(url, "http://cm-runtime-mission-abc:42617");
|
||||
}
|
||||
|
||||
/// The seed data path must be per-mission, not the shared seed dir.
|
||||
///
|
||||
/// Every mission container used to bind the SAME host directory as
|
||||
/// `/zeroclaw-data`. It holds `config.toml`, which carries the §15 door
|
||||
/// bearer token, plus `sessions.db`/`devices.db`. Sharing it meant one
|
||||
/// mission could read another's credential, and anything written there
|
||||
/// was inherited by every later mission — `teardown_container` only
|
||||
/// removes `/var/lib/clawmates-missions/{id}`, so the shared dir was
|
||||
/// never cleaned.
|
||||
///
|
||||
/// Asserting the path shape is what keeps this from silently regressing:
|
||||
/// a future edit that points the mount back at the seed dir restores the
|
||||
/// credential sharing with no other visible symptom.
|
||||
#[test]
|
||||
fn runtime_data_is_scoped_to_one_mission() {
|
||||
let a = Uuid::now_v7();
|
||||
let b = Uuid::now_v7();
|
||||
let path = |id: Uuid| format!("{MISSIONS_HOST_ROOT}/{id}/runtime-data");
|
||||
|
||||
assert_ne!(path(a), path(b), "two missions must not share runtime data");
|
||||
assert!(
|
||||
path(a).starts_with(&format!("{MISSIONS_HOST_ROOT}/{a}")),
|
||||
"runtime data must live under the mission dir so teardown removes it"
|
||||
);
|
||||
assert_ne!(
|
||||
path(a),
|
||||
DEFAULT_SEED_DIR,
|
||||
"the mount must never be the shared seed dir itself"
|
||||
);
|
||||
assert!(
|
||||
!path(a).starts_with(DEFAULT_SEED_DIR),
|
||||
"runtime data must not live inside the shared seed dir either"
|
||||
);
|
||||
}
|
||||
|
||||
/// The copy must be an allow-list, and must include the secret-bearing
|
||||
/// paths while excluding the expensive ones.
|
||||
///
|
||||
/// Measured on gw-04: the seed dir is 1.7 GB, of which 1.5 GB is a
|
||||
/// vestigial `.rustup` that is not even on the container's PATH (the
|
||||
/// image ships Rust at /usr/local/cargo). Copying everything per mission
|
||||
/// would cost tens of seconds and ~17 GB across ten concurrent missions —
|
||||
/// which is what the first version of this did.
|
||||
#[test]
|
||||
fn the_seed_copy_takes_secrets_and_skips_caches() {
|
||||
// The two paths that carry the door bearer token MUST be copied, or
|
||||
// this whole change accomplishes nothing.
|
||||
assert!(SEEDED_PATHS.contains(&".zeroclaw"));
|
||||
assert!(SEEDED_PATHS.contains(&"clawmates-mcp.json"));
|
||||
// Claude Code's credentials and state.
|
||||
assert!(SEEDED_PATHS.contains(&".claude"));
|
||||
|
||||
// The expensive, secret-free ones must NOT be.
|
||||
for cache in [".rustup", ".npm", ".cargo", ".cache"] {
|
||||
assert!(
|
||||
!SEEDED_PATHS.contains(&cache),
|
||||
"{cache} is a cache and must not be copied per mission"
|
||||
);
|
||||
}
|
||||
|
||||
let script = copy_script();
|
||||
// A missing entry is normal on a fresh deployment (no .kimi-code
|
||||
// until Kimi is first used) and must not fail the copy.
|
||||
assert!(script.contains("if [ -e "), "absent paths must be tolerated");
|
||||
assert!(script.contains("/seed/.zeroclaw"));
|
||||
assert!(!script.contains("/seed/.rustup"));
|
||||
for p in SEEDED_PATHS {
|
||||
assert!(script.contains(p), "{p} missing from the copy script");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user