Durable topology jobs (3+4/4): background worker + async run API

POST /api/topologies/run now ENQUEUES a durable job and returns 202
{run_id, status:queued} instead of executing inside the HTTP request — the
prerequisite for long-horizon runs (no client/proxy/LB timeout, survives
restarts).

topology_worker: a spawned loop that requeues stale running jobs, claims the
next queued one (CAS via FOR UPDATE SKIP LOCKED), drives it through
execute_resumable, and checkpoints RunProgress after every step; on crash the
stale sweep requeues it and the next claim resumes from the last checkpoint.
Wired into server startup beside the scheduler + resume sweeper.

GET /api/topology-runs/{id} now reports lifecycle status/kind/error/checkpoint
+ the result blob (kept the `comparison` field name for back-compat with the
compare UI; null until completed). list_runs includes status + kind.

Tests: durable lifecycle (enqueue→claim→checkpoint→complete) + stale-requeue
resume, both green; p0 endpoints (compare path) unchanged. 13 + 2 tests pass,
clippy clean.

Co-Authored-By: Claude Opus 4.8 <[email protected]>

crates/cm-api/src/routes/topology.rs

@@ -4,8 +4,9 @@
 //! provider-backed endpoint.
 
 use axum::extract::{Path, State};
+use axum::http::StatusCode;
 use axum::Json;
-use cm_orchestrator::{compare, execute, Comparison, JudgeScorer, ProviderExecutor, RunRecord};
+use cm_orchestrator::{compare, Comparison, JudgeScorer, ProviderExecutor};
 use cm_topology::{build, classify, heuristics, Classification, TopologyGraph, TopologyKind};
 use serde::{Deserialize, Serialize};
 use time::format_description::well_known::Rfc3339;
@@ -128,44 +129,50 @@ pub struct RunRequest {
     pub graph: TopologyGraph,
 }
 
-/// `POST /api/topologies/run` — execute one topology by driving real ZeroClaw
-/// role-agents (in a container) for each turn, returning the run journal. The
-/// orchestrator owns the graph; agents are tool-free behind the Clawmates MCP
-/// door, so §15 holds by construction. Result is persisted best-effort to the
-/// existing `topology_runs` table.
+/// Response to an accepted async run job.
+#[derive(Serialize)]
+pub struct RunAccepted {
+    pub run_id: String,
+    pub status: String,
+}
+
+/// `POST /api/topologies/run` — ENQUEUE a durable single-topology run and return
+/// its id immediately (202). The background worker (`topology_worker`) claims it,
+/// drives the ZeroClaw role-agents turn-by-turn (orchestrator owns the graph;
+/// agents are tool-free behind the §15 MCP door), and checkpoints per step so a
+/// long-horizon run survives restarts. Poll `GET /api/topology-runs/{id}` for
+/// status and the final result. This async model is what makes minutes-to-hours
+/// runs viable — the work no longer lives inside the HTTP request.
 pub async fn run_topology(
     State(state): State<AppState>,
     Authed(user): Authed,
     Json(req): Json<RunRequest>,
-) -> Result<Json<RunRecord>, ApiError> {
-    let executor =
-        crate::topology_exec::ZeroClawDriveExecutor::from_env().map_err(|_| ApiError::Internal)?;
-    let record = execute(&req.graph, &req.task, &executor)
-        .await
-        .map_err(|_| ApiError::Internal)?;
-
-    if let Ok(value) = serde_json::to_value(&record) {
-        let _ = cm_db::repo::topology_runs::insert(
-            &state.pool,
-            Uuid::now_v7(),
-            user.workspace_id,
-            &req.task,
-            &value,
-        )
-        .await;
-    }
-    Ok(Json(record))
+) -> Result<(StatusCode, Json<RunAccepted>), ApiError> {
+    let graph = serde_json::to_value(&req.graph).map_err(|_| ApiError::Internal)?;
+    let id = Uuid::now_v7();
+    cm_db::repo::topology_runs::enqueue_run(&state.pool, id, user.workspace_id, &req.task, &graph)
+        .await?;
+    Ok((
+        StatusCode::ACCEPTED,
+        Json(RunAccepted {
+            run_id: id.to_string(),
+            status: "queued".into(),
+        }),
+    ))
 }
 
-/// A saved comparison run, summarized.
+/// A saved/queued run, summarized (now includes lifecycle status + kind).
 #[derive(Serialize)]
 pub struct RunSummary {
     pub id: String,
     pub task: String,
+    pub status: String,
+    pub kind: String,
     pub created_at: String,
 }
 
-/// `GET /api/topology-runs` — recent saved comparison runs for the workspace.
+/// `GET /api/topology-runs` — recent runs for the workspace (compares + durable
+/// run jobs), newest first.
 pub async fn list_runs(
     State(state): State<AppState>,
     Authed(user): Authed,
@@ -176,32 +183,47 @@ pub async fn list_runs(
         .map(|r| RunSummary {
             id: r.id.to_string(),
             task: r.task,
+            status: r.status,
+            kind: r.kind,
             created_at: r.created_at.format(&Rfc3339).unwrap_or_default(),
         })
         .collect();
     Ok(Json(out))
 }
 
-/// A full saved comparison run.
+/// A single run with lifecycle status + progress. `comparison` is the result
+/// blob (a `Comparison` for compares, a `RunRecord` for run jobs) and is `null`
+/// until the job completes; pollers watch `status` and read `comparison` when it
+/// flips to `completed`. `checkpoint` exposes mid-run progress for live views.
 #[derive(Serialize)]
 pub struct RunDetail {
     pub id: String,
     pub task: String,
+    pub kind: String,
+    pub status: String,
+    pub error: Option<String>,
     pub created_at: String,
+    pub updated_at: String,
     pub comparison: serde_json::Value,
+    pub checkpoint: Option<serde_json::Value>,
 }
 
-/// `GET /api/topology-runs/{id}` — a single saved comparison run.
+/// `GET /api/topology-runs/{id}` — a single run with status + result.
 pub async fn get_run(
     State(state): State<AppState>,
     Authed(user): Authed,
     Path(id): Path<Uuid>,
 ) -> Result<Json<RunDetail>, ApiError> {
-    let run = cm_db::repo::topology_runs::get(&state.pool, id, user.workspace_id).await?;
+    let run = cm_db::repo::topology_runs::status(&state.pool, id, user.workspace_id).await?;
     Ok(Json(RunDetail {
         id: run.id.to_string(),
         task: run.task,
+        kind: run.kind,
+        status: run.status,
+        error: run.error,
         created_at: run.created_at.format(&Rfc3339).unwrap_or_default(),
-        comparison: run.comparison,
+        updated_at: run.updated_at.format(&Rfc3339).unwrap_or_default(),
+        comparison: run.result.unwrap_or(serde_json::Value::Null),
+        checkpoint: run.checkpoint,
     }))
 }