osobh/clawmates
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix(runtime): funnel every reap path through purge_agent; sweep node-placed orphans

Browse Source 
Agent containers leaked two independent ways.

1. The four-step teardown (deprovision ZeroClaw -> reap_sandbox -> unlink
   .brain/.onion -> hard_purge) was inlined at three call sites and two had
   drifted. missions.rs::reap_mission_resources skipped reap_sandbox;
   topology_worker::maybe_teardown_ephemeral_team skipped it and the brain
   unlink; DELETE /api/claws/{id} (soft delete) released nothing at all, so an
   offline claw that can never run again kept its container and bind mount
   forever. All four now funnel through claws::purge_agent, with
   release_claw_resources for the soft-delete case (containers gone, rows kept).

2. Both orphan reapers listed only the local driver, so a container placed on a
   fleet node was invisible to the only backstop that could find it -- this is
   what accumulated 144 tc-agent-* orphans on one node. NodeDriverProvider gains
   node_ids() (backed by NodeHub::online_ids) and both reapers now sweep every
   connected node. The remote sweep is TTL-only on purpose: the boot pass runs
   with Duration::ZERO and would otherwise kill a container another instance is
   mid-provision on.

Why it was invisible: agent_containers.agent_id is ON DELETE CASCADE, so
hard_purge took the registry row with the agent and left the container
permanently unreferenceable.

Co-Authored-By: Claude Opus 5 (1M context) <[email protected]>
This commit is contained in:
Omar Sobh
2026-07-30 10:41:22 -07:00
co-authored by Claude Opus 5
parent a78f308eea
commit c573480955
6 changed files with 213 additions and 37 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/cm-api/src/routes/claws.rs
+67 -15
View File
@@ -170,6 +170,59 @@ pub(crate) fn brain_dir() -> std::path::PathBuf {
.unwrap_or_else(|_| std::env::temp_dir().join("clawmates-brains"))
}
/// What [`purge_agent`] actually managed to tear down, so callers can report
/// per-stage progress without each re-implementing the sequence.
pub(crate) struct AgentPurgeReport {
pub had_container: bool,
pub brain_gone: bool,
pub counts: Result<cm_db::repo::agents::PurgeCounts, cm_db::DbError>,
}
/// Release the host-side resources a claw holds without touching its rows:
/// deprovision the ZeroClaw runtime agent, then reap its sandbox / browser /
/// terminal containers (which also clears the `agent_containers` rows).
///
/// Split out from [`purge_agent`] because the soft-delete path wants the
/// containers gone but the data kept. Best-effort; returns whether a container
/// was actually attached.
pub(crate) async fn release_claw_resources(
runtime: &cm_runtime::Runtime,
provisioner: Option<&crate::runtime_provision::RuntimeProvisioner>,
id: AgentId,
) -> bool {
if let Some(p) = provisioner {
let _ = p.deprovision_claw(id.as_uuid()).await;
}
runtime.reap_sandbox(id).await
}
/// The full per-claw teardown, in FK-safe order: deprovision the ZeroClaw
/// runtime agent → reap the sandbox/browser/terminal containers → unlink the
/// `.brain`/`.onion` files → transactionally purge every DB row.
///
/// Every reap path funnels through here. Three call sites used to inline their
/// own variant of this sequence and two of them had silently drifted — skipping
/// `reap_sandbox`, so deleting a mission or tearing down an ephemeral team left
/// live `tc-agent-*` containers and orphan `agent_containers` rows behind.
/// Steps 1–3 are best-effort; only the DB purge can fail the call.
pub(crate) async fn purge_agent(
pool: &sqlx::PgPool,
runtime: &cm_runtime::Runtime,
provisioner: Option<&crate::runtime_provision::RuntimeProvisioner>,
id: AgentId,
) -> AgentPurgeReport {
let had_container = release_claw_resources(runtime, provisioner, id).await;
let brain = brain_dir();
let brain_gone = std::fs::remove_file(brain.join(format!("claw_{id}.h5"))).is_ok();
let _ = std::fs::remove_file(brain.join(format!("claw_{id}.h5.onion")));
let counts = cm_db::repo::agents::hard_purge(pool, id).await;
AgentPurgeReport {
had_container,
brain_gone,
counts,
}
}
/// Open (or first-create) the claw's brain and read it into a response. Seeds
/// the definition from Postgres on a fresh brain — mirrors the runtime's
/// first-touch seeding so the cards always have real data. Pure/sync.
@@ -1013,7 +1066,10 @@ pub async fn set_model(
}
/// DELETE /api/claws/{id} — destructive (§7.7): workspace owners or the
/// claw's manager only. Soft delete keeps rows for audit.
/// claw's manager only. Soft delete keeps rows for audit, but the claw's
/// host-side resources are released: a soft-deleted claw is `offline` and can
/// never run again, so leaving its container alive just burns the node's
/// memory and holds a workspace bind mount open indefinitely.
pub async fn delete(
State(state): State<AppState>,
Authed(user): Authed,
@@ -1023,6 +1079,8 @@ pub async fn delete(
if !user.role.is_owner() && agent.managed_by != user.user_id {
return Err(ApiError::Forbidden);
}
let provisioner = crate::runtime_provision::RuntimeProvisioner::from_env();
let had_container = release_claw_resources(&state.runtime, provisioner.as_ref(), id).await;
cm_db::repo::agents::soft_delete(&state.pool, id).await?;
cm_db::repo::audit::append(
&state.pool,
@@ -1031,7 +1089,7 @@ pub async fn delete(
"agent.deleted",
"agent",
&id.to_string(),
json!({"name": agent.name}),
json!({"name": agent.name, "container_reaped": had_container}),
)
.await?;
Ok(StatusCode::NO_CONTENT)
@@ -1116,21 +1174,15 @@ pub async fn batch_delete(
let name = agent.name.clone();
yield sse(json!({"stage":"start","pct":base,"label":format!("Removing {name}…")}));
// 1. Deprovision the ZeroClaw runtime agent (best-effort).
// Runtime → container → brain → DB, via the shared reaper. The
// whole sequence is sub-second, so the stage events are emitted
// from the report rather than interleaved.
yield sse(json!({"stage":"deprovision","pct":base,"label":format!("{name}: deprovisioning runtime…")}));
if let Some(p) = &provisioner {
let _ = p.deprovision_claw(id.as_uuid()).await;
}
// 2. Reap the sandbox/browser container if one is attached.
let had_container = state.runtime.reap_sandbox(id).await;
yield sse(json!({"stage":"container","pct":base,"label":format!("{name}: {}", if had_container { "reaped sandbox container" } else { "no container attached" })}));
// 3. Unlink the brain files.
let brain_gone = std::fs::remove_file(brain_dir().join(format!("claw_{id}.h5"))).is_ok();
let _ = std::fs::remove_file(brain_dir().join(format!("claw_{id}.h5.onion")));
yield sse(json!({"stage":"brain","pct":base,"label":format!("{name}: {}", if brain_gone { "deleted .brain file" } else { "no .brain file" })}));
// 4. Transactionally purge all DB rows + the agent itself.
let report = purge_agent(&state.pool, &state.runtime, provisioner.as_ref(), id).await;
yield sse(json!({"stage":"container","pct":base,"label":format!("{name}: {}", if report.had_container { "reaped sandbox container" } else { "no container attached" })}));
yield sse(json!({"stage":"brain","pct":base,"label":format!("{name}: {}", if report.brain_gone { "deleted .brain file" } else { "no .brain file" })}));
yield sse(json!({"stage":"purge","pct":base,"label":format!("{name}: purging data…")}));
match cm_db::repo::agents::hard_purge(&state.pool, id).await {
match report.counts {
Ok(c) => {
let _ = cm_db::repo::audit::append(
&state.pool, user.workspace_id, Actor::User(user.user_id),
crates/cm-api/src/routes/missions.rs
+11 -10
View File
@@ -437,18 +437,19 @@ async fn reap_mission_resources(state: &AppState, mission_id: Uuid) {
.unwrap_or_default()
};
// 2. Reap each claw: ZeroClaw config → .brain files → all DB rows.
// 2. Reap each claw: ZeroClaw config → sandbox container → .brain files →
// all DB rows. Shared with the batch-delete reaper so this path cannot
// drift back into skipping the container teardown.
let provisioner = crate::runtime_provision::RuntimeProvisioner::from_env();
for cid in &claw_ids {
if let Some(p) = &provisioner {
let _ = p.deprovision_claw(*cid).await;
}
let brain = crate::routes::claws::brain_dir();
let _ = std::fs::remove_file(brain.join(format!("claw_{cid}.h5")));
let _ = std::fs::remove_file(brain.join(format!("claw_{cid}.h5.onion")));
if let Err(e) =
cm_db::repo::agents::hard_purge(&state.pool, cm_domain::AgentId::from(*cid)).await
{
let report = crate::routes::claws::purge_agent(
&state.pool,
&state.runtime,
provisioner.as_ref(),
cm_domain::AgentId::from(*cid),
)
.await;
if let Err(e) = report.counts {
eprintln!("missions::delete: hard_purge claw {cid} failed (continuing): {e}");
}
}