fix(missions): reap all mission resources on delete (no hanging claws/files)
DELETE /api/missions/{id} was a bare `DELETE FROM missions` relying on FK
cascades that only cover mission-owned tables. Everything the mission
provisioned leaked: per-mission runtime container, host workspace dir,
teams (created lifecycle=permanent, so no cascade + skipped by the
ephemeral-teardown path), and every claw's ZeroClaw config, .brain files,
and DB rows. Observed live with 0 missions in the DB: 174 orphaned gateway
claw configs, 7 orphaned teams, 31 agents, 39 .brain files, 6 workspace
dirs, a 4-day-old orphaned container, and 123 detached topology_runs.
delete() now calls reap_mission_resources() before the row delete:
- resolve the mission's teams (mission_teams) → claws (team_members)
- per claw: deprovision_claw (gateway) + rm .brain files + hard_purge (DB),
reusing the manual agent-reap pattern in routes/claws.rs
- delete the permanent-lifecycle teams (team_members cascades)
- delete the mission's topology_runs (else they linger with mission_id
nulled by the cascade and accumulate)
- teardown_container(), now extended to also rm the /mission/repo workspace
dir and tolerate an already-gone container (idempotent for the sweeper +
delete paths)
Runtime-side steps are best-effort (Postgres authoritative; fleet sweeper
reconciles daemon config); DB purges are logged on failure but never block.
Co-Authored-By: Claude Opus 4.8 <[email protected]>
This commit is contained in:
co-authored by
Claude Opus 4.8
parent
11e1379c5f
commit
bf4ef4c4bf
@@ -482,10 +482,14 @@ fn extract_pairing_code_from_json(body: &str) -> Option<String> {
|
|||||||
}
|
}
|
||||||
|
|
||||||
impl MissionRuntimeProvisioner {
|
impl MissionRuntimeProvisioner {
|
||||||
/// Force-remove the mission's runtime container. Idempotent.
|
/// Force-remove the mission's runtime container AND its host workspace
|
||||||
|
/// dir (the `/mission/repo` checkout bind source). Idempotent: a missing
|
||||||
|
/// container or dir is not an error — this is called both by the terminal
|
||||||
|
/// sweeper and by mission delete, where the container may already be gone.
|
||||||
pub async fn teardown_container(&self, mission_id: Uuid) -> Result<(), String> {
|
pub async fn teardown_container(&self, mission_id: Uuid) -> Result<(), String> {
|
||||||
let name = container_name(mission_id);
|
let name = container_name(mission_id);
|
||||||
self.docker
|
if let Err(e) = self
|
||||||
|
.docker
|
||||||
.remove_container(
|
.remove_container(
|
||||||
&name,
|
&name,
|
||||||
Some(RemoveContainerOptions {
|
Some(RemoveContainerOptions {
|
||||||
@@ -494,7 +498,21 @@ impl MissionRuntimeProvisioner {
|
|||||||
}),
|
}),
|
||||||
)
|
)
|
||||||
.await
|
.await
|
||||||
.map_err(|e| format!("remove mission runtime container: {e}"))?;
|
{
|
||||||
|
// 404 (already gone) is fine; anything else is worth surfacing.
|
||||||
|
let msg = e.to_string();
|
||||||
|
if !msg.contains("No such container") && !msg.contains("404") {
|
||||||
|
return Err(format!("remove mission runtime container: {e}"));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
// Remove the per-mission workspace dir (repo checkout + scratch). This
|
||||||
|
// path is bind-mounted into cm-api, so we can reap it directly.
|
||||||
|
let mission_dir = format!("{MISSIONS_HOST_ROOT}/{mission_id}");
|
||||||
|
if let Err(e) = tokio::fs::remove_dir_all(&mission_dir).await {
|
||||||
|
if e.kind() != std::io::ErrorKind::NotFound {
|
||||||
|
eprintln!("mission_runtime: rm workspace dir {mission_dir}: {e}");
|
||||||
|
}
|
||||||
|
}
|
||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -353,14 +353,109 @@ pub async fn delete(
|
|||||||
Authed(user): Authed,
|
Authed(user): Authed,
|
||||||
Path(id): Path<Uuid>,
|
Path(id): Path<Uuid>,
|
||||||
) -> Result<Json<serde_json::Value>, ApiError> {
|
) -> Result<Json<serde_json::Value>, ApiError> {
|
||||||
let deleted =
|
let ws = user.workspace_id.as_uuid();
|
||||||
cm_db::repo::missions::delete(&state.pool, id, user.workspace_id.as_uuid()).await?;
|
// Verify the mission exists in this workspace before we start reaping.
|
||||||
|
let exists: Option<Uuid> =
|
||||||
|
sqlx::query_scalar("SELECT id FROM missions WHERE id = $1 AND workspace_id = $2")
|
||||||
|
.bind(id)
|
||||||
|
.bind(ws)
|
||||||
|
.fetch_optional(&state.pool)
|
||||||
|
.await
|
||||||
|
.map_err(|_| ApiError::Internal)?;
|
||||||
|
if exists.is_none() {
|
||||||
|
return Err(ApiError::NotFound);
|
||||||
|
}
|
||||||
|
|
||||||
|
// Reap every resource the mission provisioned BEFORE the DB delete, so
|
||||||
|
// nothing is left hanging. Runtime-side steps are best-effort (Postgres
|
||||||
|
// is authoritative; the daemon config is a cache the fleet sweeper can
|
||||||
|
// reconcile) — a failure logs and continues rather than blocking delete.
|
||||||
|
reap_mission_resources(&state, id).await;
|
||||||
|
|
||||||
|
let deleted = cm_db::repo::missions::delete(&state.pool, id, ws).await?;
|
||||||
if deleted == 0 {
|
if deleted == 0 {
|
||||||
return Err(ApiError::NotFound);
|
return Err(ApiError::NotFound);
|
||||||
}
|
}
|
||||||
Ok(Json(serde_json::json!({ "deleted": true })))
|
Ok(Json(serde_json::json!({ "deleted": true })))
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Tear down all resources a mission created: its per-mission runtime
|
||||||
|
/// container + workspace dir, every claw (ZeroClaw config, `.brain` files,
|
||||||
|
/// and all DB rows via `hard_purge`), the (permanent-lifecycle) teams, and
|
||||||
|
/// its topology runs. Called before the `missions` row is deleted so the
|
||||||
|
/// `mission_teams` junction is still resolvable. Best-effort throughout.
|
||||||
|
async fn reap_mission_resources(state: &AppState, mission_id: Uuid) {
|
||||||
|
// 1. Resolve the mission's teams, then their claws.
|
||||||
|
let team_ids: Vec<Uuid> =
|
||||||
|
sqlx::query_scalar("SELECT team_id FROM mission_teams WHERE mission_id = $1")
|
||||||
|
.bind(mission_id)
|
||||||
|
.fetch_all(&state.pool)
|
||||||
|
.await
|
||||||
|
.unwrap_or_default();
|
||||||
|
let claw_ids: Vec<Uuid> = if team_ids.is_empty() {
|
||||||
|
Vec::new()
|
||||||
|
} else {
|
||||||
|
sqlx::query_scalar(
|
||||||
|
"SELECT DISTINCT claw_id FROM team_members WHERE team_id = ANY($1)",
|
||||||
|
)
|
||||||
|
.bind(&team_ids)
|
||||||
|
.fetch_all(&state.pool)
|
||||||
|
.await
|
||||||
|
.unwrap_or_default()
|
||||||
|
};
|
||||||
|
|
||||||
|
// 2. Reap each claw: ZeroClaw config → .brain files → all DB rows.
|
||||||
|
let provisioner = crate::runtime_provision::RuntimeProvisioner::from_env();
|
||||||
|
for cid in &claw_ids {
|
||||||
|
if let Some(p) = &provisioner {
|
||||||
|
let _ = p.deprovision_claw(*cid).await;
|
||||||
|
}
|
||||||
|
let brain = crate::routes::claws::brain_dir();
|
||||||
|
let _ = std::fs::remove_file(brain.join(format!("claw_{cid}.h5")));
|
||||||
|
let _ = std::fs::remove_file(brain.join(format!("claw_{cid}.h5.onion")));
|
||||||
|
if let Err(e) =
|
||||||
|
cm_db::repo::agents::hard_purge(&state.pool, cm_domain::AgentId::from(*cid)).await
|
||||||
|
{
|
||||||
|
eprintln!("missions::delete: hard_purge claw {cid} failed (continuing): {e}");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// 3. Delete the (permanent-lifecycle) teams — no mission FK cascades them.
|
||||||
|
// team_members cascades from teams.
|
||||||
|
if !team_ids.is_empty() {
|
||||||
|
if let Err(e) = sqlx::query("DELETE FROM teams WHERE id = ANY($1)")
|
||||||
|
.bind(&team_ids)
|
||||||
|
.execute(&state.pool)
|
||||||
|
.await
|
||||||
|
{
|
||||||
|
eprintln!("missions::delete: delete teams for {mission_id} failed (continuing): {e}");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// 4. Delete this mission's topology runs (else they linger with
|
||||||
|
// mission_id nulled by the cascade and accumulate forever).
|
||||||
|
if let Err(e) = sqlx::query("DELETE FROM topology_runs WHERE mission_id = $1")
|
||||||
|
.bind(mission_id)
|
||||||
|
.execute(&state.pool)
|
||||||
|
.await
|
||||||
|
{
|
||||||
|
eprintln!("missions::delete: delete topology_runs for {mission_id} failed (continuing): {e}");
|
||||||
|
}
|
||||||
|
|
||||||
|
// 5. Tear down the per-mission runtime container + its workspace dir.
|
||||||
|
if let Some(mp) = crate::mission_runtime::MissionRuntimeProvisioner::from_env() {
|
||||||
|
if let Err(e) = mp.teardown_container(mission_id).await {
|
||||||
|
eprintln!("missions::delete: teardown container for {mission_id} failed (continuing): {e}");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
eprintln!(
|
||||||
|
"missions::delete: reaped {} claw(s), {} team(s) for mission {mission_id}",
|
||||||
|
claw_ids.len(),
|
||||||
|
team_ids.len()
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
#[derive(Debug, Deserialize)]
|
#[derive(Debug, Deserialize)]
|
||||||
pub struct HerdrDispatchRequest {
|
pub struct HerdrDispatchRequest {
|
||||||
pub cli: String,
|
pub cli: String,
|
||||||
|
|||||||
Reference in New Issue
Block a user