Durable topology jobs (1/4): job-table schema + repo
Evolve topology_runs into a durable-job table (migration 0009): status state machine (queued/running/completed/failed/cancelled), kind, input graph, per-step checkpoint, error, last_event_id, timestamps. comparison becomes nullable (the result blob, absent until completion). Back-compat: existing rows default to completed/compare. cm-db repo gains the durable-job ops: enqueue_run, claim_next_queued (CAS via FOR UPDATE SKIP LOCKED), checkpoint, complete, fail, requeue_stale (resume sweep), and status(). Regenerated .sqlx cache. Also fix two pre-existing test RuntimeConfig literals missing the providers field (from the registry work). Co-Authored-By: Claude Opus 4.8 <[email protected]>
This commit is contained in:
Omar Sobh
co-authored by
Claude Opus 4.8
Claude Opus 4.8
parent
b1991cee3b
commit
8dee01c77b
@@ -1,4 +1,6 @@
|
||||
//! Persistence for saved multi-topology comparison runs.
|
||||
//! Persistence for topology runs — both saved "compare" results and durable
|
||||
//! single-topology "run" jobs (queued → running → completed/failed), the latter
|
||||
//! checkpointed per step for crash-resumable long-horizon execution.
|
||||
|
||||
use cm_domain::WorkspaceId;
|
||||
use serde_json::Value;
|
||||
@@ -12,6 +14,8 @@ use crate::DbError;
|
||||
pub struct TopologyRunSummary {
|
||||
pub id: Uuid,
|
||||
pub task: String,
|
||||
pub status: String,
|
||||
pub kind: String,
|
||||
pub created_at: OffsetDateTime,
|
||||
}
|
||||
|
||||
@@ -23,7 +27,35 @@ pub struct TopologyRun {
|
||||
pub created_at: OffsetDateTime,
|
||||
}
|
||||
|
||||
/// Save a comparison run for a workspace.
|
||||
/// A durable topology job claimed by the background worker (queued or resumed).
|
||||
pub struct ClaimedTopologyRun {
|
||||
pub id: Uuid,
|
||||
pub workspace_id: Uuid,
|
||||
pub task: String,
|
||||
/// Input graph (present for `kind = 'run'` jobs).
|
||||
pub graph: Option<Value>,
|
||||
/// Completed-step outputs persisted by the last checkpoint, for resume.
|
||||
pub checkpoint: Option<Value>,
|
||||
/// Event-journal offset reached so far.
|
||||
pub last_event_id: i64,
|
||||
}
|
||||
|
||||
/// Lifecycle status + progress for a durable run (status endpoint).
|
||||
pub struct TopologyRunStatus {
|
||||
pub id: Uuid,
|
||||
pub task: String,
|
||||
pub kind: String,
|
||||
pub status: String,
|
||||
pub error: Option<String>,
|
||||
pub checkpoint: Option<Value>,
|
||||
/// The final result blob (the `RunRecord`/`Comparison`) once completed.
|
||||
pub result: Option<Value>,
|
||||
pub last_event_id: i64,
|
||||
pub created_at: OffsetDateTime,
|
||||
pub updated_at: OffsetDateTime,
|
||||
}
|
||||
|
||||
/// Save a finished comparison run for a workspace (synchronous path).
|
||||
pub async fn insert(
|
||||
pool: &PgPool,
|
||||
id: Uuid,
|
||||
@@ -31,6 +63,7 @@ pub async fn insert(
|
||||
task: &str,
|
||||
comparison: &Value,
|
||||
) -> Result<(), DbError> {
|
||||
// status/kind default to 'completed'/'compare' for the synchronous path.
|
||||
sqlx::query!(
|
||||
"INSERT INTO topology_runs (id, workspace_id, task, comparison)
|
||||
VALUES ($1, $2, $3, $4)",
|
||||
@@ -44,6 +77,120 @@ pub async fn insert(
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Enqueue a durable single-topology run job (`kind = 'run'`, `status = 'queued'`).
|
||||
/// The background worker claims and executes it; the result lands in `comparison`.
|
||||
pub async fn enqueue_run(
|
||||
pool: &PgPool,
|
||||
id: Uuid,
|
||||
workspace_id: WorkspaceId,
|
||||
task: &str,
|
||||
graph: &Value,
|
||||
) -> Result<(), DbError> {
|
||||
sqlx::query!(
|
||||
"INSERT INTO topology_runs (id, workspace_id, task, kind, status, graph)
|
||||
VALUES ($1, $2, $3, 'run', 'queued', $4)",
|
||||
id,
|
||||
workspace_id.as_uuid(),
|
||||
task,
|
||||
graph,
|
||||
)
|
||||
.execute(pool)
|
||||
.await?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Atomically claim the oldest queued job, flipping it to `running`. Uses
|
||||
/// `FOR UPDATE SKIP LOCKED` so multiple workers never claim the same job.
|
||||
/// Returns `None` when the queue is empty.
|
||||
pub async fn claim_next_queued(pool: &PgPool) -> Result<Option<ClaimedTopologyRun>, DbError> {
|
||||
let row = sqlx::query!(
|
||||
"UPDATE topology_runs
|
||||
SET status = 'running', started_at = COALESCE(started_at, now()), updated_at = now()
|
||||
WHERE id = (
|
||||
SELECT id FROM topology_runs
|
||||
WHERE status = 'queued'
|
||||
ORDER BY created_at
|
||||
FOR UPDATE SKIP LOCKED
|
||||
LIMIT 1
|
||||
)
|
||||
RETURNING id, workspace_id, task, graph, checkpoint, last_event_id",
|
||||
)
|
||||
.fetch_optional(pool)
|
||||
.await?;
|
||||
Ok(row.map(|r| ClaimedTopologyRun {
|
||||
id: r.id,
|
||||
workspace_id: r.workspace_id,
|
||||
task: r.task,
|
||||
graph: r.graph,
|
||||
checkpoint: r.checkpoint,
|
||||
last_event_id: r.last_event_id,
|
||||
}))
|
||||
}
|
||||
|
||||
/// Persist mid-run progress: the completed-step checkpoint + journal offset.
|
||||
/// Touches `updated_at` so the stale-run sweeper treats the job as alive.
|
||||
pub async fn checkpoint(
|
||||
pool: &PgPool,
|
||||
id: Uuid,
|
||||
checkpoint: &Value,
|
||||
last_event_id: i64,
|
||||
) -> Result<(), DbError> {
|
||||
sqlx::query!(
|
||||
"UPDATE topology_runs
|
||||
SET checkpoint = $2, last_event_id = $3, updated_at = now()
|
||||
WHERE id = $1",
|
||||
id,
|
||||
checkpoint,
|
||||
last_event_id,
|
||||
)
|
||||
.execute(pool)
|
||||
.await?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Mark a job completed and store its final result blob.
|
||||
pub async fn complete(pool: &PgPool, id: Uuid, result: &Value) -> Result<(), DbError> {
|
||||
sqlx::query!(
|
||||
"UPDATE topology_runs
|
||||
SET status = 'completed', comparison = $2, finished_at = now(), updated_at = now()
|
||||
WHERE id = $1",
|
||||
id,
|
||||
result,
|
||||
)
|
||||
.execute(pool)
|
||||
.await?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Mark a job failed with an error message.
|
||||
pub async fn fail(pool: &PgPool, id: Uuid, error: &str) -> Result<(), DbError> {
|
||||
sqlx::query!(
|
||||
"UPDATE topology_runs
|
||||
SET status = 'failed', error = $2, finished_at = now(), updated_at = now()
|
||||
WHERE id = $1",
|
||||
id,
|
||||
error,
|
||||
)
|
||||
.execute(pool)
|
||||
.await?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Resume sweep: requeue `running` jobs whose worker went silent (no checkpoint
|
||||
/// touch within `older_than_secs`). The next claim resumes them from checkpoint.
|
||||
/// Returns how many were requeued.
|
||||
pub async fn requeue_stale(pool: &PgPool, older_than_secs: f64) -> Result<u64, DbError> {
|
||||
let result = sqlx::query!(
|
||||
"UPDATE topology_runs
|
||||
SET status = 'queued', updated_at = now()
|
||||
WHERE status = 'running' AND updated_at < now() - make_interval(secs => $1)",
|
||||
older_than_secs,
|
||||
)
|
||||
.execute(pool)
|
||||
.await?;
|
||||
Ok(result.rows_affected())
|
||||
}
|
||||
|
||||
/// The most recent runs for a workspace, newest first.
|
||||
pub async fn list_recent(
|
||||
pool: &PgPool,
|
||||
@@ -51,7 +198,7 @@ pub async fn list_recent(
|
||||
limit: i64,
|
||||
) -> Result<Vec<TopologyRunSummary>, DbError> {
|
||||
let rows = sqlx::query!(
|
||||
"SELECT id, task, created_at FROM topology_runs
|
||||
"SELECT id, task, status, kind, created_at FROM topology_runs
|
||||
WHERE workspace_id = $1 ORDER BY created_at DESC LIMIT $2",
|
||||
workspace_id.as_uuid(),
|
||||
limit,
|
||||
@@ -63,12 +210,14 @@ pub async fn list_recent(
|
||||
.map(|r| TopologyRunSummary {
|
||||
id: r.id,
|
||||
task: r.task,
|
||||
status: r.status,
|
||||
kind: r.kind,
|
||||
created_at: r.created_at,
|
||||
})
|
||||
.collect())
|
||||
}
|
||||
|
||||
/// A single saved run, scoped to its workspace.
|
||||
/// A single saved comparison/run result, scoped to its workspace.
|
||||
pub async fn get(
|
||||
pool: &PgPool,
|
||||
id: Uuid,
|
||||
@@ -86,7 +235,38 @@ pub async fn get(
|
||||
Ok(TopologyRun {
|
||||
id: row.id,
|
||||
task: row.task,
|
||||
comparison: row.comparison,
|
||||
// Nullable since the durable path: a queued/running run has no result yet.
|
||||
comparison: row.comparison.unwrap_or(Value::Null),
|
||||
created_at: row.created_at,
|
||||
})
|
||||
}
|
||||
|
||||
/// Lifecycle status + progress for a durable run (status endpoint), workspace-scoped.
|
||||
pub async fn status(
|
||||
pool: &PgPool,
|
||||
id: Uuid,
|
||||
workspace_id: WorkspaceId,
|
||||
) -> Result<TopologyRunStatus, DbError> {
|
||||
let row = sqlx::query!(
|
||||
"SELECT id, task, kind, status, error, checkpoint, comparison, last_event_id,
|
||||
created_at, updated_at
|
||||
FROM topology_runs WHERE id = $1 AND workspace_id = $2",
|
||||
id,
|
||||
workspace_id.as_uuid(),
|
||||
)
|
||||
.fetch_optional(pool)
|
||||
.await?
|
||||
.ok_or(DbError::NotFound)?;
|
||||
Ok(TopologyRunStatus {
|
||||
id: row.id,
|
||||
task: row.task,
|
||||
kind: row.kind,
|
||||
status: row.status,
|
||||
error: row.error,
|
||||
checkpoint: row.checkpoint,
|
||||
result: row.comparison,
|
||||
last_event_id: row.last_event_id,
|
||||
created_at: row.created_at,
|
||||
updated_at: row.updated_at,
|
||||
})
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user