loops: kind column + initial_burst + on_artifact_update trigger fan-out
Foundation for folding research into loops as a first-class kind.
This commit ships the plumbing; the research-kind dispatch itself
lands next. Behavior for existing exec-kind loops is unchanged unless
they opt into the new trigger fields.
Migration 0044:
- kind TEXT NOT NULL DEFAULT 'exec' CHECK ('exec' | 'research'). New
research-kind will run the research pipeline each iteration (next
commit); 'exec' preserves today's behavior.
- initial_burst_remaining INT NOT NULL DEFAULT 0 — countdown for the
triggers.initial_burst quota. Decremented CAS-safely on each
completion until it hits 0.
- Two partial indexes: (kind, source_research_topic_id) for kind-
aware lookups, and (source_research_topic_id) filtered on
on_artifact_update=true + enabled=true for the fan-out hook.
Trigger schema extended with two optional fields:
- initial_burst: N — fire N iterations back-to-back at create time.
create_loop enqueues the first iteration inline (subject to
empty-roster gate), sets remaining=N-1, and the completion hook
continues the chain until exhausted.
- on_artifact_update: true — when a bound research_outcomes row is
inserted for the source topic, wake up one iteration of this loop.
Coalesced against has_active_run so a burst of rapid revisions
doesn't queue duplicates.
Backend:
- cm_db::repo::loops helpers (all dynamic sqlx, no offline cache
regen needed):
- set_initial_burst_remaining
- take_initial_burst_slot (CAS UPDATE returning prev value; 0 on
exhausted or race loss)
- loops_awaiting_topic (fan-out query: kind=exec + enabled +
on_artifact_update=true bound to the given topic)
- has_active_run (queued|running iteration existence check)
- get_any_workspace (bypasses the workspace scope guard; used by
the completion hook where the run row is authoritative)
- routes/loops::compose_iteration_task made pub so the completion
hook can build the same enriched task string as run_now.
- topology_worker::freeze_research_outcome now fans out to awakened
loops after the outcome insert, using compose_iteration_task and
coalescing on has_active_run.
- topology_worker::continue_initial_burst runs on every completion:
· take_initial_burst_slot (CAS) — no-op if already exhausted
· has_active_run coalesce guard
· re-fetches the loop via get_any_workspace + compose_iteration_task
· enqueues via loops::enqueue_iteration with parent_run_id set
Follow-ups already queued:
- kind='research' dispatch in run_job — build the research
coordinator task from the topic config, run the research pipeline
each iteration. Requires factoring start_topic's task-build.
- ResearchWizard "When should this run?" step (Just once / Nightly /
Manual) creating the topic + paired research-kind loop.
- LoopsWizard trigger UI matching the design proposal (burst count,
cron, on-artifact checkbox).
This commit is contained in:
@@ -171,6 +171,7 @@ async fn run_job(
|
||||
}
|
||||
freeze_research_outcome(pool, id, &record.final_output).await;
|
||||
advance_loop_after_completion(pool, id, &record.final_output).await;
|
||||
continue_initial_burst(pool, id).await;
|
||||
}
|
||||
Err(e) => {
|
||||
// Don't clobber a cancellation (or any already-terminal state) with `failed`.
|
||||
@@ -207,6 +208,94 @@ async fn freeze_research_outcome(pool: &PgPool, run_id: Uuid, final_output: &str
|
||||
cm_db::repo::research_outcomes::insert(pool, topic_id, final_output, Some(run_id)).await
|
||||
{
|
||||
eprintln!("topology_worker: research_outcomes::insert({run_id}) failed: {e}");
|
||||
return;
|
||||
}
|
||||
// Fan-out: any exec-kind loop bound to this topic with the
|
||||
// on_artifact_update trigger enabled wakes up now. Coalesced —
|
||||
// if a loop already has a queued/running run we skip (D3 fallback:
|
||||
// the coordinator sees the fresh artifact on its next iteration
|
||||
// anyway). Best-effort per loop; one loop's Docker/DB hiccup
|
||||
// doesn't affect the others.
|
||||
let awakened = cm_db::repo::loops::loops_awaiting_topic(pool, topic_id)
|
||||
.await
|
||||
.unwrap_or_default();
|
||||
for (loop_id, workspace_id, task_template, graph) in awakened {
|
||||
if cm_db::repo::loops::has_active_run(pool, loop_id)
|
||||
.await
|
||||
.unwrap_or(false)
|
||||
{
|
||||
continue; // Coalesce.
|
||||
}
|
||||
let iter = cm_db::repo::loops::next_iteration(pool, loop_id)
|
||||
.await
|
||||
.unwrap_or(0);
|
||||
// Build the enriched task with the freshly-inserted artifact
|
||||
// (compose_iteration_task reads latest, which is what we just
|
||||
// wrote).
|
||||
let task =
|
||||
crate::routes::loops::compose_iteration_task(pool, loop_id, &task_template).await;
|
||||
if let Err(e) = cm_db::repo::loops::enqueue_iteration(
|
||||
pool,
|
||||
loop_id,
|
||||
workspace_id,
|
||||
&task,
|
||||
&graph,
|
||||
iter,
|
||||
Some(run_id),
|
||||
)
|
||||
.await
|
||||
{
|
||||
eprintln!("topology_worker: on_artifact_update enqueue({loop_id}) failed: {e}");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// If the just-completed run was a loop iteration with
|
||||
/// initial_burst_remaining > 0, enqueue the next iteration and
|
||||
/// decrement the counter (CAS-safe via take_initial_burst_slot).
|
||||
/// No-op for non-loop runs and for loops whose burst is exhausted.
|
||||
async fn continue_initial_burst(pool: &PgPool, run_id: Uuid) {
|
||||
let loop_id = match cm_db::repo::topology_runs::loop_id_for_run(pool, run_id).await {
|
||||
Ok(Some(id)) => id,
|
||||
_ => return,
|
||||
};
|
||||
// If another worker races us, only ONE gets the slot; the other
|
||||
// sees 0 (no-op).
|
||||
let prev = cm_db::repo::loops::take_initial_burst_slot(pool, loop_id)
|
||||
.await
|
||||
.unwrap_or(0);
|
||||
if prev == 0 {
|
||||
return;
|
||||
}
|
||||
// Coalesce with a concurrently-in-flight iteration (a webhook
|
||||
// arriving during a burst, say).
|
||||
if cm_db::repo::loops::has_active_run(pool, loop_id)
|
||||
.await
|
||||
.unwrap_or(false)
|
||||
{
|
||||
return;
|
||||
}
|
||||
// Fetch the loop so we have the workspace + graph + task_template
|
||||
// to enqueue the next iteration.
|
||||
let Ok(Some(l)) = cm_db::repo::loops::get_any_workspace(pool, loop_id).await else {
|
||||
return;
|
||||
};
|
||||
let iter = cm_db::repo::loops::next_iteration(pool, loop_id)
|
||||
.await
|
||||
.unwrap_or(0);
|
||||
let task = crate::routes::loops::compose_iteration_task(pool, loop_id, &l.task_template).await;
|
||||
if let Err(e) = cm_db::repo::loops::enqueue_iteration(
|
||||
pool,
|
||||
loop_id,
|
||||
l.workspace_id,
|
||||
&task,
|
||||
&l.graph,
|
||||
iter,
|
||||
Some(run_id),
|
||||
)
|
||||
.await
|
||||
{
|
||||
eprintln!("topology_worker: continue_initial_burst enqueue failed: {e}");
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user