loops: backend routes + repo + cron scheduler + HMAC webhook
Third commit of the Research + Loops arc. Lights up loops as durable
recurring topology executions:
GET /api/loops list workspace's loops
POST /api/loops create — returns webhook_token +
signing_key ONCE when webhook trigger
is enabled; never exposed again
GET /api/loops/:id detail
PATCH /api/loops/:id update definition
DELETE /api/loops/:id delete
POST /api/loops/:id/run trigger one iteration NOW
POST /api/loops/:id/enable set enabled=true
POST /api/loops/:id/disable set enabled=false
POST /webhooks/loops/:token public; HMAC-SHA256-verified
Scheduler (cm_runtime::spawn_loop_scheduler) wakes every 10s, queries the
partial index on (next_fire_at) for due loops, enqueues one topology_runs
row per fire with loop_id + iteration + parent_run_id chained back to the
previous iteration. Uses croner via the existing scheduling::next_occurrence
helper. Missed windows fire ONCE and skip the backlog — next_fire_at is
always computed strictly AFTER now(), so a late scheduler doesn't drain a
buildup.
Webhook signatures follow the same pattern as the Stripe billing webhook
(HMAC-SHA256 with constant-time hex compare). Token + signing key are
24-byte OS-RNG values; the URL uses base64-url for the token, and the
signing key is base64-std. Both surface exactly once at create time.
All three fire paths (scheduler, immediate-run, webhook) funnel through
`cm_db::repo::loops::enqueue_iteration` so the invariants stay in one
place. `iters` repeat policy is enforced by the scheduler tick; `until`
and `on_completion` land with the orchestrator hook in commit 4.
Adds cm-llm as a direct cm-api dep, getrandom for the webhook material
generator, and wires the scheduler spawn into the server binary alongside
the resume sweeper and outbox drainer.
This commit is contained in:
@@ -0,0 +1,141 @@
|
||||
//! Loop scheduler: periodically wakes, finds loops whose `next_fire_at`
|
||||
//! has arrived, enqueues one topology_runs row per fire (with loop_id +
|
||||
//! iteration + parent_run_id set), and computes the next fire time from
|
||||
//! the cron trigger.
|
||||
//!
|
||||
//! Missed-window rule: `next_fire_at` is always computed strictly AFTER
|
||||
//! `now()`, so a scheduler that woke up late (restart, long stall) fires
|
||||
//! ONCE and skips whatever windows were in the backlog. This matches the
|
||||
//! "fire once and move on" behavior we chose in the spec.
|
||||
|
||||
use std::time::Duration;
|
||||
|
||||
use serde::Deserialize;
|
||||
use time::OffsetDateTime;
|
||||
use tokio::time::interval;
|
||||
|
||||
use crate::scheduling::next_occurrence;
|
||||
|
||||
/// The subset of `triggers` JSONB the scheduler needs to make decisions.
|
||||
#[derive(Debug, Default, Deserialize)]
|
||||
struct Triggers {
|
||||
/// Cron pattern (5-field). None => scheduler doesn't participate.
|
||||
#[serde(default)]
|
||||
cron: Option<String>,
|
||||
/// Enqueue next iteration when the previous one hits `run_completed`.
|
||||
/// Handled by the run driver on completion (see runtime::spawn_drive);
|
||||
/// the scheduler doesn't own this branch, we surface it here just so
|
||||
/// mark_fired() below knows whether to null out next_fire_at.
|
||||
#[serde(default)]
|
||||
on_completion: bool,
|
||||
#[serde(default)]
|
||||
webhook_enabled: bool,
|
||||
}
|
||||
|
||||
/// The subset of `repeat_policy` JSONB the scheduler needs.
|
||||
#[derive(Debug, Default, Deserialize)]
|
||||
struct RepeatPolicy {
|
||||
/// `infinite` | `iters` | `until`. Anything unknown = `infinite`.
|
||||
#[serde(default = "default_kind")]
|
||||
kind: String,
|
||||
/// `iters.n` stopping condition.
|
||||
#[serde(default)]
|
||||
n: Option<i32>,
|
||||
}
|
||||
fn default_kind() -> String {
|
||||
"infinite".to_string()
|
||||
}
|
||||
|
||||
/// Runs the tick every `interval_duration` until the process exits.
|
||||
pub fn spawn_loop_scheduler(pool: sqlx::PgPool, interval_duration: Duration) {
|
||||
tokio::spawn(async move {
|
||||
let mut tick = interval(interval_duration);
|
||||
// First tick fires immediately; second waits the full interval. That's
|
||||
// fine — the query is a bounded partial-index scan.
|
||||
loop {
|
||||
tick.tick().await;
|
||||
if let Err(e) = fire_due(&pool).await {
|
||||
eprintln!("loop scheduler tick failed: {e}");
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
/// One tick: find due loops, fire each. Errors from one loop don't stop the
|
||||
/// others.
|
||||
async fn fire_due(pool: &sqlx::PgPool) -> Result<(), sqlx::Error> {
|
||||
let due = cm_db::repo::loops::due(pool).await.map_err(sqlx_err)?;
|
||||
for l in due {
|
||||
if let Err(e) = fire_one(pool, &l).await {
|
||||
eprintln!("loop {} fire failed: {e}", l.id);
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn fire_one(pool: &sqlx::PgPool, l: &cm_db::repo::loops::DueLoop) -> Result<(), sqlx::Error> {
|
||||
let triggers: Triggers = serde_json::from_value(l.triggers.clone()).unwrap_or_default();
|
||||
let policy: RepeatPolicy = serde_json::from_value(l.repeat_policy.clone()).unwrap_or_default();
|
||||
|
||||
let iter = cm_db::repo::loops::next_iteration(pool, l.id)
|
||||
.await
|
||||
.map_err(sqlx_err)?;
|
||||
|
||||
// Repeat cap. `iters` stops after N total iterations; `infinite` and
|
||||
// `until` don't check here (until is enforced by the on-completion path
|
||||
// which inspects the run's terminal event; scope for the scheduler stops
|
||||
// at cron time-based firing).
|
||||
if policy.kind == "iters" {
|
||||
if let Some(cap) = policy.n {
|
||||
if iter > cap {
|
||||
// Silently disable the loop so we don't tick it forever.
|
||||
let _ = cm_db::repo::loops::set_enabled(pool, l.id, l.workspace_id, false).await;
|
||||
return Ok(());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
let run_id = cm_db::repo::loops::enqueue_iteration(
|
||||
pool,
|
||||
l.id,
|
||||
l.workspace_id,
|
||||
&l.task_template,
|
||||
&l.graph,
|
||||
iter,
|
||||
l.last_run_id,
|
||||
)
|
||||
.await
|
||||
.map_err(sqlx_err)?;
|
||||
|
||||
// Advance next_fire_at strictly AFTER now(). If there's no cron trigger
|
||||
// (e.g. webhook-only or on-completion-only), null it out so the partial
|
||||
// index stops matching this loop for the scheduler.
|
||||
let next = match triggers.cron.as_deref() {
|
||||
Some(pattern) if !pattern.is_empty() => {
|
||||
match next_occurrence(pattern, OffsetDateTime::now_utc()) {
|
||||
Ok(t) => Some(t),
|
||||
Err(e) => {
|
||||
eprintln!("loop {} invalid cron '{}': {e}", l.id, pattern);
|
||||
None
|
||||
}
|
||||
}
|
||||
}
|
||||
_ => None,
|
||||
};
|
||||
// Also null it out when the cron trigger vanished but on_completion or
|
||||
// webhook_enabled is still on — those paths will re-fire independently.
|
||||
let _ = (triggers.on_completion, triggers.webhook_enabled);
|
||||
|
||||
cm_db::repo::loops::mark_fired(pool, l.id, run_id, next)
|
||||
.await
|
||||
.map_err(sqlx_err)?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn sqlx_err(e: cm_db::DbError) -> sqlx::Error {
|
||||
match e {
|
||||
cm_db::DbError::Other(e) => e,
|
||||
cm_db::DbError::NotFound => sqlx::Error::RowNotFound,
|
||||
cm_db::DbError::Conflict(_) => sqlx::Error::PoolTimedOut,
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user