Reaping: sandbox orphan reaper + DB expiry/retention sweeper + volume-init retry

Closes the cleanup gaps found in review (latent today; bites under load/crashes).

Sandbox containers (cm-sandbox / cm-runtime):
- label every sandbox `clawmates.sandbox={agent|browser}` at create
- SandboxDriver::list_managed(kind) (Docker label filter + K8s label selector)
- SandboxManager::reconcile_orphans(ttl) + spawn_reaper: removes engine
  containers no live handle owns (ZERO = all)
- boot reconciliation (every pre-existing sandbox is an orphan from a dead
  process) + periodic reaper (5m interval / 10m TTL)
- SIGTERM graceful drain: serve().with_graceful_shutdown → shutdown() both
  managers so a redeploy can't leak; destroy errors now logged not swallowed

DB expiry/retention (cm-db cleanup.rs + cm-api cleanup_sweeper, hourly):
- expire auth_sessions + oauth_states past expires_at (security)
- prune sent outbox(7d), run_events(14d), routine_runs(30d), terminal
  topology_runs(90d), consumed execution_grants(7d)

Compose: volume-init restart "no" → on-failure:5 (retry instead of wedging boot).

Co-Authored-By: Claude Opus 4.8 <[email protected]>

crates/bins/clawmates-server/src/main.rs

@@ -155,18 +155,31 @@ async fn run() -> Result<(), String> {
                     driver.clone(),
                     &config.sandbox.image,
                 ));
+                let browser = std::sync::Arc::new(
+                    cm_runtime::SandboxManager::new(driver, &config.sandbox.browser_image)
+                        .with_egress(),
+                );
+                // Boot reconciliation: any sandbox the engine still holds is an
+                // orphan from a dead process (we track none yet) — remove them
+                // before warming so a crash/redeploy can't leak containers.
+                agents.reconcile_orphans(std::time::Duration::ZERO).await;
+                browser.reconcile_orphans(std::time::Duration::ZERO).await;
                 let agents = if config.sandbox.warm_pool > 0 {
                     agents.warm(config.sandbox.warm_pool)
                 } else {
                     agents
                 };
-                (
-                    Some(agents),
-                    Some(std::sync::Arc::new(
-                        cm_runtime::SandboxManager::new(driver, &config.sandbox.browser_image)
-                            .with_egress(),
-                    )),
-                )
+                // Periodic reaper: catch leaks while we run (idle > 10 min and
+                // owned by no live handle), checked every 5 min.
+                agents.clone().spawn_reaper(
+                    std::time::Duration::from_secs(300),
+                    std::time::Duration::from_secs(600),
+                );
+                browser.clone().spawn_reaper(
+                    std::time::Duration::from_secs(300),
+                    std::time::Duration::from_secs(600),
+                );
+                (Some(agents), Some(browser))
             }
             Err(error) => {
                 eprintln!("clawmates-server: sandbox engine unavailable: {error}");
@@ -176,6 +189,10 @@ async fn run() -> Result<(), String> {
     } else {
         (None, None)
     };
+    // Keep handles for the graceful-shutdown drain (SIGTERM) — the managers
+    // themselves are moved into the runtime config below.
+    let drain_agents = sandboxes.clone();
+    let drain_browser = browser.clone();
     let runtime = Runtime::with_blob_store(
         pool.clone(),
         provider,
@@ -203,6 +220,9 @@ async fn run() -> Result<(), String> {
     // Outbound-email delivery: drains the §15-gated `outbox` over SMTP. Inert
     // until CLAWMATES_SMTP_* is set, so it ships safely before credentials exist.
     cm_runtime::spawn_drainer(pool.clone(), std::time::Duration::from_secs(10));
+    // Expiry/retention sweep: expires stale auth/oauth rows and prunes old
+    // journal/audit rows hourly so unbounded tables don't accumulate.
+    cm_api::cleanup_sweeper::spawn(pool.clone(), std::time::Duration::from_secs(3600));
 
     // Hosted identity (Clerk / OIDC): pin the issuer and load its JWKS.
     let auth_verifier = match config.auth.mode {
@@ -235,7 +255,27 @@ async fn run() -> Result<(), String> {
         .await
         .map_err(|e| format!("bind {} failed: {e}", config.listen_addr))?;
     println!("clawmates-server listening on {}", config.listen_addr);
+    // Graceful shutdown: on SIGTERM/Ctrl-C, stop accepting, finish in-flight
+    // requests, then DRAIN the sandbox managers so no container is left running.
+    let shutdown = async move {
+        let mut term = tokio::signal::unix::signal(tokio::signal::unix::SignalKind::terminate())
+            .expect("install SIGTERM handler");
+        tokio::select! {
+            _ = term.recv() => {}
+            _ = tokio::signal::ctrl_c() => {}
+        }
+        eprintln!("clawmates-server: shutdown signal received");
+    };
     axum::serve(listener, app)
+        .with_graceful_shutdown(shutdown)
         .await
-        .map_err(|e| format!("server error: {e}"))
+        .map_err(|e| format!("server error: {e}"))?;
+    eprintln!("clawmates-server: draining sandboxes…");
+    if let Some(agents) = drain_agents {
+        agents.shutdown().await;
+    }
+    if let Some(browser) = drain_browser {
+        browser.shutdown().await;
+    }
+    Ok(())
 }