fix(missions): reap all mission resources on delete (no hanging claws/files)
ci / gates (push) Successful in 24s
ci / rust (push) Failing after 10s
ci / frontend (push) Successful in 27s
ci / e2e (push) Skipped
ci / publish (push) Skipped

DELETE /api/missions/{id} was a bare `DELETE FROM missions` relying on FK
cascades that only cover mission-owned tables. Everything the mission
provisioned leaked: per-mission runtime container, host workspace dir,
teams (created lifecycle=permanent, so no cascade + skipped by the
ephemeral-teardown path), and every claw's ZeroClaw config, .brain files,
and DB rows. Observed live with 0 missions in the DB: 174 orphaned gateway
claw configs, 7 orphaned teams, 31 agents, 39 .brain files, 6 workspace
dirs, a 4-day-old orphaned container, and 123 detached topology_runs.

delete() now calls reap_mission_resources() before the row delete:
- resolve the mission's teams (mission_teams) → claws (team_members)
- per claw: deprovision_claw (gateway) + rm .brain files + hard_purge (DB),
  reusing the manual agent-reap pattern in routes/claws.rs
- delete the permanent-lifecycle teams (team_members cascades)
- delete the mission's topology_runs (else they linger with mission_id
  nulled by the cascade and accumulate)
- teardown_container(), now extended to also rm the /mission/repo workspace
  dir and tolerate an already-gone container (idempotent for the sweeper +
  delete paths)

Runtime-side steps are best-effort (Postgres authoritative; fleet sweeper
reconciles daemon config); DB purges are logged on failure but never block.

Co-Authored-By: Claude Opus 4.8 <[email protected]>
This commit is contained in:
Omar Sobh
2026-07-28 09:28:56 +02:00
co-authored by Claude Opus 4.8
parent 11e1379c5f
commit bf4ef4c4bf
2 changed files with 118 additions and 5 deletions
+21 -3
View File
@@ -482,10 +482,14 @@ fn extract_pairing_code_from_json(body: &str) -> Option<String> {
} }
impl MissionRuntimeProvisioner { impl MissionRuntimeProvisioner {
/// Force-remove the mission's runtime container. Idempotent. /// Force-remove the mission's runtime container AND its host workspace
/// dir (the `/mission/repo` checkout bind source). Idempotent: a missing
/// container or dir is not an error — this is called both by the terminal
/// sweeper and by mission delete, where the container may already be gone.
pub async fn teardown_container(&self, mission_id: Uuid) -> Result<(), String> { pub async fn teardown_container(&self, mission_id: Uuid) -> Result<(), String> {
let name = container_name(mission_id); let name = container_name(mission_id);
self.docker if let Err(e) = self
.docker
.remove_container( .remove_container(
&name, &name,
Some(RemoveContainerOptions { Some(RemoveContainerOptions {
@@ -494,7 +498,21 @@ impl MissionRuntimeProvisioner {
}), }),
) )
.await .await
.map_err(|e| format!("remove mission runtime container: {e}"))?; {
// 404 (already gone) is fine; anything else is worth surfacing.
let msg = e.to_string();
if !msg.contains("No such container") && !msg.contains("404") {
return Err(format!("remove mission runtime container: {e}"));
}
}
// Remove the per-mission workspace dir (repo checkout + scratch). This
// path is bind-mounted into cm-api, so we can reap it directly.
let mission_dir = format!("{MISSIONS_HOST_ROOT}/{mission_id}");
if let Err(e) = tokio::fs::remove_dir_all(&mission_dir).await {
if e.kind() != std::io::ErrorKind::NotFound {
eprintln!("mission_runtime: rm workspace dir {mission_dir}: {e}");
}
}
Ok(()) Ok(())
} }
} }
+97 -2
View File
@@ -353,14 +353,109 @@ pub async fn delete(
Authed(user): Authed, Authed(user): Authed,
Path(id): Path<Uuid>, Path(id): Path<Uuid>,
) -> Result<Json<serde_json::Value>, ApiError> { ) -> Result<Json<serde_json::Value>, ApiError> {
let deleted = let ws = user.workspace_id.as_uuid();
cm_db::repo::missions::delete(&state.pool, id, user.workspace_id.as_uuid()).await?; // Verify the mission exists in this workspace before we start reaping.
let exists: Option<Uuid> =
sqlx::query_scalar("SELECT id FROM missions WHERE id = $1 AND workspace_id = $2")
.bind(id)
.bind(ws)
.fetch_optional(&state.pool)
.await
.map_err(|_| ApiError::Internal)?;
if exists.is_none() {
return Err(ApiError::NotFound);
}
// Reap every resource the mission provisioned BEFORE the DB delete, so
// nothing is left hanging. Runtime-side steps are best-effort (Postgres
// is authoritative; the daemon config is a cache the fleet sweeper can
// reconcile) — a failure logs and continues rather than blocking delete.
reap_mission_resources(&state, id).await;
let deleted = cm_db::repo::missions::delete(&state.pool, id, ws).await?;
if deleted == 0 { if deleted == 0 {
return Err(ApiError::NotFound); return Err(ApiError::NotFound);
} }
Ok(Json(serde_json::json!({ "deleted": true }))) Ok(Json(serde_json::json!({ "deleted": true })))
} }
/// Tear down all resources a mission created: its per-mission runtime
/// container + workspace dir, every claw (ZeroClaw config, `.brain` files,
/// and all DB rows via `hard_purge`), the (permanent-lifecycle) teams, and
/// its topology runs. Called before the `missions` row is deleted so the
/// `mission_teams` junction is still resolvable. Best-effort throughout.
async fn reap_mission_resources(state: &AppState, mission_id: Uuid) {
// 1. Resolve the mission's teams, then their claws.
let team_ids: Vec<Uuid> =
sqlx::query_scalar("SELECT team_id FROM mission_teams WHERE mission_id = $1")
.bind(mission_id)
.fetch_all(&state.pool)
.await
.unwrap_or_default();
let claw_ids: Vec<Uuid> = if team_ids.is_empty() {
Vec::new()
} else {
sqlx::query_scalar(
"SELECT DISTINCT claw_id FROM team_members WHERE team_id = ANY($1)",
)
.bind(&team_ids)
.fetch_all(&state.pool)
.await
.unwrap_or_default()
};
// 2. Reap each claw: ZeroClaw config → .brain files → all DB rows.
let provisioner = crate::runtime_provision::RuntimeProvisioner::from_env();
for cid in &claw_ids {
if let Some(p) = &provisioner {
let _ = p.deprovision_claw(*cid).await;
}
let brain = crate::routes::claws::brain_dir();
let _ = std::fs::remove_file(brain.join(format!("claw_{cid}.h5")));
let _ = std::fs::remove_file(brain.join(format!("claw_{cid}.h5.onion")));
if let Err(e) =
cm_db::repo::agents::hard_purge(&state.pool, cm_domain::AgentId::from(*cid)).await
{
eprintln!("missions::delete: hard_purge claw {cid} failed (continuing): {e}");
}
}
// 3. Delete the (permanent-lifecycle) teams — no mission FK cascades them.
// team_members cascades from teams.
if !team_ids.is_empty() {
if let Err(e) = sqlx::query("DELETE FROM teams WHERE id = ANY($1)")
.bind(&team_ids)
.execute(&state.pool)
.await
{
eprintln!("missions::delete: delete teams for {mission_id} failed (continuing): {e}");
}
}
// 4. Delete this mission's topology runs (else they linger with
// mission_id nulled by the cascade and accumulate forever).
if let Err(e) = sqlx::query("DELETE FROM topology_runs WHERE mission_id = $1")
.bind(mission_id)
.execute(&state.pool)
.await
{
eprintln!("missions::delete: delete topology_runs for {mission_id} failed (continuing): {e}");
}
// 5. Tear down the per-mission runtime container + its workspace dir.
if let Some(mp) = crate::mission_runtime::MissionRuntimeProvisioner::from_env() {
if let Err(e) = mp.teardown_container(mission_id).await {
eprintln!("missions::delete: teardown container for {mission_id} failed (continuing): {e}");
}
}
eprintln!(
"missions::delete: reaped {} claw(s), {} team(s) for mission {mission_id}",
claw_ids.len(),
team_ids.len()
);
}
#[derive(Debug, Deserialize)] #[derive(Debug, Deserialize)]
pub struct HerdrDispatchRequest { pub struct HerdrDispatchRequest {
pub cli: String, pub cli: String,