loops: backend routes + repo + cron scheduler + HMAC webhook
ci / gates (push) Successful in 5s
ci / frontend (push) Successful in 24s
ci / rust (push) Successful in 3m43s
ci / publish (push) Successful in 2m15s
ci / e2e (push) Failing after 29m57s

Third commit of the Research + Loops arc. Lights up loops as durable
recurring topology executions:

  GET    /api/loops                 list workspace's loops
  POST   /api/loops                 create — returns webhook_token +
                                    signing_key ONCE when webhook trigger
                                    is enabled; never exposed again
  GET    /api/loops/:id             detail
  PATCH  /api/loops/:id             update definition
  DELETE /api/loops/:id             delete
  POST   /api/loops/:id/run         trigger one iteration NOW
  POST   /api/loops/:id/enable      set enabled=true
  POST   /api/loops/:id/disable     set enabled=false
  POST   /webhooks/loops/:token     public; HMAC-SHA256-verified

Scheduler (cm_runtime::spawn_loop_scheduler) wakes every 10s, queries the
partial index on (next_fire_at) for due loops, enqueues one topology_runs
row per fire with loop_id + iteration + parent_run_id chained back to the
previous iteration. Uses croner via the existing scheduling::next_occurrence
helper. Missed windows fire ONCE and skip the backlog — next_fire_at is
always computed strictly AFTER now(), so a late scheduler doesn't drain a
buildup.

Webhook signatures follow the same pattern as the Stripe billing webhook
(HMAC-SHA256 with constant-time hex compare). Token + signing key are
24-byte OS-RNG values; the URL uses base64-url for the token, and the
signing key is base64-std. Both surface exactly once at create time.

All three fire paths (scheduler, immediate-run, webhook) funnel through
`cm_db::repo::loops::enqueue_iteration` so the invariants stay in one
place. `iters` repeat policy is enforced by the scheduler tick; `until`
and `on_completion` land with the orchestrator hook in commit 4.

Adds cm-llm as a direct cm-api dep, getrandom for the webhook material
generator, and wires the scheduler spawn into the server binary alongside
the resume sweeper and outbox drainer.
This commit is contained in:
Omar Sobh
2026-07-06 04:39:27 -07:00
parent 4ffcb3d652
commit 4b48c521eb
20 changed files with 1297 additions and 0 deletions
@@ -0,0 +1,22 @@
{
"db_name": "PostgreSQL",
"query": "SELECT MAX(iteration) FROM topology_runs WHERE loop_id = $1",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "max",
"type_info": "Int4"
}
],
"parameters": {
"Left": [
"Uuid"
]
},
"nullable": [
null
]
},
"hash": "00106727aa8f52160c50750997e79463328846a119d27c67cba0efb5ffcfef2d"
}
@@ -0,0 +1,20 @@
{
"db_name": "PostgreSQL",
"query": "INSERT INTO topology_runs\n (id, workspace_id, task, kind, status, graph, tier,\n loop_id, iteration, parent_run_id)\n VALUES ($1, $2, $3, 'run', 'queued', $4, 'team', $5, $6, $7)",
"describe": {
"columns": [],
"parameters": {
"Left": [
"Uuid",
"Uuid",
"Text",
"Jsonb",
"Uuid",
"Int4",
"Uuid"
]
},
"nullable": []
},
"hash": "03c6baa216872a93409211f09f24a07d81a433d1a94e4f0a9fd2666cec997169"
}
@@ -0,0 +1,26 @@
{
"db_name": "PostgreSQL",
"query": "INSERT INTO loops\n (id, workspace_id, title, description, graph, task_template,\n triggers, repeat_policy, enabled, next_fire_at,\n webhook_token, webhook_signing_key, created_by)\n VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13)",
"describe": {
"columns": [],
"parameters": {
"Left": [
"Uuid",
"Uuid",
"Text",
"Text",
"Jsonb",
"Text",
"Jsonb",
"Jsonb",
"Bool",
"Timestamptz",
"Text",
"Text",
"Uuid"
]
},
"nullable": []
},
"hash": "18f21e18db478667a6edf42b3e38100574fe82c91df2a19b6d3747c09a7c574d"
}
@@ -0,0 +1,16 @@
{
"db_name": "PostgreSQL",
"query": "UPDATE loops SET enabled = $3, updated_at = now()\n WHERE id = $1 AND workspace_id = $2",
"describe": {
"columns": [],
"parameters": {
"Left": [
"Uuid",
"Uuid",
"Bool"
]
},
"nullable": []
},
"hash": "1b2dc407fe913e1bfa516a7f6c88bd6a3805281dbeefbae485241a23e8d16ee2"
}
@@ -0,0 +1,56 @@
{
"db_name": "PostgreSQL",
"query": "SELECT id, workspace_id, graph, task_template, triggers, repeat_policy,\n last_run_id\n FROM loops\n WHERE enabled AND next_fire_at IS NOT NULL AND next_fire_at <= now()",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "id",
"type_info": "Uuid"
},
{
"ordinal": 1,
"name": "workspace_id",
"type_info": "Uuid"
},
{
"ordinal": 2,
"name": "graph",
"type_info": "Jsonb"
},
{
"ordinal": 3,
"name": "task_template",
"type_info": "Text"
},
{
"ordinal": 4,
"name": "triggers",
"type_info": "Jsonb"
},
{
"ordinal": 5,
"name": "repeat_policy",
"type_info": "Jsonb"
},
{
"ordinal": 6,
"name": "last_run_id",
"type_info": "Uuid"
}
],
"parameters": {
"Left": []
},
"nullable": [
false,
false,
false,
false,
false,
false,
true
]
},
"hash": "2165f2d82b83fc827f133c150c310ea7609eb26b2513e4438ffca30391d55321"
}
@@ -0,0 +1,15 @@
{
"db_name": "PostgreSQL",
"query": "DELETE FROM loops WHERE id = $1 AND workspace_id = $2",
"describe": {
"columns": [],
"parameters": {
"Left": [
"Uuid",
"Uuid"
]
},
"nullable": []
},
"hash": "3c3a6e0584505808f3b41d11133f0a15f11dbd44087b9de01096018f1619051a"
}
@@ -0,0 +1,16 @@
{
"db_name": "PostgreSQL",
"query": "UPDATE loops\n SET last_run_id = $2, next_fire_at = $3, updated_at = now()\n WHERE id = $1",
"describe": {
"columns": [],
"parameters": {
"Left": [
"Uuid",
"Uuid",
"Timestamptz"
]
},
"nullable": []
},
"hash": "7b22d878f560708e4e9723d71423e0c5800c1f714b5afc693aea49d352003f1f"
}
@@ -0,0 +1,112 @@
{
"db_name": "PostgreSQL",
"query": "SELECT id, workspace_id, title, description, graph, task_template,\n triggers, repeat_policy, enabled, next_fire_at, last_run_id,\n webhook_token, webhook_signing_key, created_by, created_at, updated_at\n FROM loops\n WHERE workspace_id = $1\n ORDER BY updated_at DESC",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "id",
"type_info": "Uuid"
},
{
"ordinal": 1,
"name": "workspace_id",
"type_info": "Uuid"
},
{
"ordinal": 2,
"name": "title",
"type_info": "Text"
},
{
"ordinal": 3,
"name": "description",
"type_info": "Text"
},
{
"ordinal": 4,
"name": "graph",
"type_info": "Jsonb"
},
{
"ordinal": 5,
"name": "task_template",
"type_info": "Text"
},
{
"ordinal": 6,
"name": "triggers",
"type_info": "Jsonb"
},
{
"ordinal": 7,
"name": "repeat_policy",
"type_info": "Jsonb"
},
{
"ordinal": 8,
"name": "enabled",
"type_info": "Bool"
},
{
"ordinal": 9,
"name": "next_fire_at",
"type_info": "Timestamptz"
},
{
"ordinal": 10,
"name": "last_run_id",
"type_info": "Uuid"
},
{
"ordinal": 11,
"name": "webhook_token",
"type_info": "Text"
},
{
"ordinal": 12,
"name": "webhook_signing_key",
"type_info": "Text"
},
{
"ordinal": 13,
"name": "created_by",
"type_info": "Uuid"
},
{
"ordinal": 14,
"name": "created_at",
"type_info": "Timestamptz"
},
{
"ordinal": 15,
"name": "updated_at",
"type_info": "Timestamptz"
}
],
"parameters": {
"Left": [
"Uuid"
]
},
"nullable": [
false,
false,
false,
false,
false,
false,
false,
false,
false,
true,
true,
true,
true,
false,
false,
false
]
},
"hash": "911a088dac3b2464d817d76831731a65d0b3dbb9264f538fcde60f48808ab4bd"
}
@@ -0,0 +1,113 @@
{
"db_name": "PostgreSQL",
"query": "SELECT id, workspace_id, title, description, graph, task_template,\n triggers, repeat_policy, enabled, next_fire_at, last_run_id,\n webhook_token, webhook_signing_key, created_by, created_at, updated_at\n FROM loops\n WHERE id = $1 AND workspace_id = $2",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "id",
"type_info": "Uuid"
},
{
"ordinal": 1,
"name": "workspace_id",
"type_info": "Uuid"
},
{
"ordinal": 2,
"name": "title",
"type_info": "Text"
},
{
"ordinal": 3,
"name": "description",
"type_info": "Text"
},
{
"ordinal": 4,
"name": "graph",
"type_info": "Jsonb"
},
{
"ordinal": 5,
"name": "task_template",
"type_info": "Text"
},
{
"ordinal": 6,
"name": "triggers",
"type_info": "Jsonb"
},
{
"ordinal": 7,
"name": "repeat_policy",
"type_info": "Jsonb"
},
{
"ordinal": 8,
"name": "enabled",
"type_info": "Bool"
},
{
"ordinal": 9,
"name": "next_fire_at",
"type_info": "Timestamptz"
},
{
"ordinal": 10,
"name": "last_run_id",
"type_info": "Uuid"
},
{
"ordinal": 11,
"name": "webhook_token",
"type_info": "Text"
},
{
"ordinal": 12,
"name": "webhook_signing_key",
"type_info": "Text"
},
{
"ordinal": 13,
"name": "created_by",
"type_info": "Uuid"
},
{
"ordinal": 14,
"name": "created_at",
"type_info": "Timestamptz"
},
{
"ordinal": 15,
"name": "updated_at",
"type_info": "Timestamptz"
}
],
"parameters": {
"Left": [
"Uuid",
"Uuid"
]
},
"nullable": [
false,
false,
false,
false,
false,
false,
false,
false,
false,
true,
true,
true,
true,
false,
false,
false
]
},
"hash": "a91674c6d98c029b2e5eb4ead0ab7b974ac839c7d8d44446ff1e864b070174f2"
}
@@ -0,0 +1,22 @@
{
"db_name": "PostgreSQL",
"query": "UPDATE loops\n SET title = $3, description = $4, graph = $5, task_template = $6,\n triggers = $7, repeat_policy = $8, next_fire_at = $9,\n updated_at = now()\n WHERE id = $1 AND workspace_id = $2",
"describe": {
"columns": [],
"parameters": {
"Left": [
"Uuid",
"Uuid",
"Text",
"Text",
"Jsonb",
"Text",
"Jsonb",
"Jsonb",
"Timestamptz"
]
},
"nullable": []
},
"hash": "ae226c3156f612252d07fd17aacff0a3c9de6ff6bb25d043914fa996b9b38270"
}
@@ -0,0 +1,64 @@
{
"db_name": "PostgreSQL",
"query": "SELECT id, workspace_id, graph, task_template, triggers, repeat_policy,\n last_run_id, webhook_signing_key\n FROM loops\n WHERE webhook_token = $1 AND enabled",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "id",
"type_info": "Uuid"
},
{
"ordinal": 1,
"name": "workspace_id",
"type_info": "Uuid"
},
{
"ordinal": 2,
"name": "graph",
"type_info": "Jsonb"
},
{
"ordinal": 3,
"name": "task_template",
"type_info": "Text"
},
{
"ordinal": 4,
"name": "triggers",
"type_info": "Jsonb"
},
{
"ordinal": 5,
"name": "repeat_policy",
"type_info": "Jsonb"
},
{
"ordinal": 6,
"name": "last_run_id",
"type_info": "Uuid"
},
{
"ordinal": 7,
"name": "webhook_signing_key",
"type_info": "Text"
}
],
"parameters": {
"Left": [
"Text"
]
},
"nullable": [
false,
false,
false,
false,
false,
false,
true,
true
]
},
"hash": "be08a19d993480156d961cb170633e0133d802db02e1d8a10fa1f8ccac819d18"
}
+3
View File
@@ -274,6 +274,9 @@ async fn run() -> Result<(), String> {
// Outbound-email delivery: drains the §15-gated `outbox` over SMTP. Inert // Outbound-email delivery: drains the §15-gated `outbox` over SMTP. Inert
// until CLAWMATES_SMTP_* is set, so it ships safely before credentials exist. // until CLAWMATES_SMTP_* is set, so it ships safely before credentials exist.
cm_runtime::spawn_drainer(pool.clone(), std::time::Duration::from_secs(10)); cm_runtime::spawn_drainer(pool.clone(), std::time::Duration::from_secs(10));
// Loop scheduler: fires cron-triggered loop iterations. Missed windows
// fire ONCE and skip the backlog (see cm_runtime::loops for details).
cm_runtime::spawn_loop_scheduler(pool.clone(), std::time::Duration::from_secs(10));
// Expiry/retention sweep: expires stale auth/oauth rows and prunes old // Expiry/retention sweep: expires stale auth/oauth rows and prunes old
// journal/audit rows hourly so unbounded tables don't accumulate. // journal/audit rows hourly so unbounded tables don't accumulate.
cm_api::cleanup_sweeper::spawn(pool.clone(), std::time::Duration::from_secs(3600)); cm_api::cleanup_sweeper::spawn(pool.clone(), std::time::Duration::from_secs(3600));
+1
View File
@@ -7,6 +7,7 @@ license.workspace = true
publish.workspace = true publish.workspace = true
[dependencies] [dependencies]
getrandom = "0.2"
hex = "0.4" hex = "0.4"
hmac = "0.12" hmac = "0.12"
sha2 = "0.10" sha2 = "0.10"
+17
View File
@@ -403,6 +403,23 @@ pub fn router(state: AppState) -> Router {
"/api/research/wizard/refine", "/api/research/wizard/refine",
post(routes::research::refine_wizard), post(routes::research::refine_wizard),
) )
.route(
"/api/loops",
get(routes::loops::list_loops).post(routes::loops::create_loop),
)
.route(
"/api/loops/{id}",
get(routes::loops::get_loop)
.patch(routes::loops::patch_loop)
.delete(routes::loops::delete_loop),
)
.route("/api/loops/{id}/run", post(routes::loops::run_now))
.route("/api/loops/{id}/enable", post(routes::loops::enable_loop))
.route("/api/loops/{id}/disable", post(routes::loops::disable_loop))
.route(
"/webhooks/loops/{token}",
post(routes::loops::webhook_receive),
)
.route("/api/structure/stats", get(routes::structure::stats)) .route("/api/structure/stats", get(routes::structure::stats))
.route("/api/structure/{level}/{id}", get(routes::structure::node)) .route("/api/structure/{level}/{id}", get(routes::structure::node))
.route("/api/topology-runs", get(routes::topology::list_runs)) .route("/api/topology-runs", get(routes::topology::list_runs))
+336
View File
@@ -0,0 +1,336 @@
//! Loop endpoints — CRUD, enable/disable, immediate-run, and the public
//! webhook receiver.
//!
//! GET /api/loops list workspace's loops
//! POST /api/loops create
//! GET /api/loops/:id detail
//! PATCH /api/loops/:id update definition
//! DELETE /api/loops/:id delete
//! POST /api/loops/:id/run trigger one iteration NOW (bypass schedule)
//! POST /api/loops/:id/enable set enabled=true; recomputes next_fire_at
//! POST /api/loops/:id/disable set enabled=false
//! POST /webhooks/loops/:token public; HMAC-SHA256-verified via
//! X-Loop-Signature: sha256=<hex>
use axum::body::Bytes;
use axum::extract::{Path, State};
use axum::http::{HeaderMap, StatusCode};
use axum::Json;
use base64::Engine;
use cm_runtime::scheduling::next_occurrence;
use hmac::{Hmac, Mac};
use serde::{Deserialize, Serialize};
use serde_json::Value;
use time::OffsetDateTime;
use uuid::Uuid;
use crate::{ApiError, AppState, Authed};
#[derive(Deserialize)]
pub struct CreateLoopRequest {
pub title: String,
pub description: String,
pub graph: Value,
pub task_template: String,
/// {cron?: '0 */6 * * *', on_completion?: bool, webhook_enabled?: bool}
#[serde(default)]
pub triggers: Value,
/// {kind: 'infinite' | 'iters' | 'until', n?: int}
#[serde(default = "default_repeat")]
pub repeat_policy: Value,
}
fn default_repeat() -> Value {
serde_json::json!({"kind": "infinite"})
}
#[derive(Serialize)]
pub struct LoopCreated {
pub id: Uuid,
/// Set when `triggers.webhook_enabled == true`. The full URL is
/// `<origin>/webhooks/loops/<webhook_token>`; the signing key is
/// returned exactly once at creation and never surfaced again.
#[serde(skip_serializing_if = "Option::is_none")]
pub webhook_token: Option<String>,
#[serde(skip_serializing_if = "Option::is_none")]
pub webhook_signing_key: Option<String>,
}
fn parse_triggers(v: &Value) -> Option<Triggers> {
serde_json::from_value(v.clone()).ok()
}
#[derive(Deserialize)]
struct Triggers {
#[serde(default)]
cron: Option<String>,
#[serde(default)]
#[allow(dead_code)]
on_completion: bool,
#[serde(default)]
webhook_enabled: bool,
}
fn make_webhook_material() -> (String, String) {
// 24 bytes ≈ 192 bits of entropy each; URL-safe base64 for the token,
// standard base64 for the signing key.
let mut token_buf = [0u8; 24];
let mut key_buf = [0u8; 24];
// getrandom is already in the dep tree via base64/hmac/etc; failure
// (broken kernel RNG) is fatal enough that unwrapping is fine here.
getrandom::getrandom(&mut token_buf).expect("OS RNG");
getrandom::getrandom(&mut key_buf).expect("OS RNG");
let token = base64::engine::general_purpose::URL_SAFE_NO_PAD.encode(token_buf);
let key = base64::engine::general_purpose::STANDARD_NO_PAD.encode(key_buf);
(token, key)
}
fn compute_next_fire(triggers: &Value) -> Option<OffsetDateTime> {
let t = parse_triggers(triggers)?;
let pattern = t.cron?;
if pattern.trim().is_empty() {
return None;
}
next_occurrence(pattern.trim(), OffsetDateTime::now_utc()).ok()
}
pub async fn create_loop(
State(state): State<AppState>,
Authed(user): Authed,
Json(body): Json<CreateLoopRequest>,
) -> Result<(StatusCode, Json<LoopCreated>), ApiError> {
if body.title.trim().is_empty() || body.task_template.trim().is_empty() {
return Err(ApiError::BadRequest);
}
let webhook_enabled = parse_triggers(&body.triggers)
.map(|t| t.webhook_enabled)
.unwrap_or(false);
let (webhook_token, webhook_signing_key) = if webhook_enabled {
let (t, k) = make_webhook_material();
(Some(t), Some(k))
} else {
(None, None)
};
let next_fire_at = compute_next_fire(&body.triggers);
let id = cm_db::repo::loops::create(
&state.pool,
cm_db::repo::loops::NewLoop {
workspace_id: user.workspace_id.as_uuid(),
title: body.title.trim(),
description: body.description.trim(),
graph: &body.graph,
task_template: body.task_template.trim(),
triggers: &body.triggers,
repeat_policy: &body.repeat_policy,
enabled: true,
next_fire_at,
webhook_token: webhook_token.as_deref(),
webhook_signing_key: webhook_signing_key.as_deref(),
created_by: user.user_id.as_uuid(),
},
)
.await?;
Ok((
StatusCode::CREATED,
Json(LoopCreated {
id,
webhook_token,
webhook_signing_key,
}),
))
}
pub async fn list_loops(
State(state): State<AppState>,
Authed(user): Authed,
) -> Result<Json<Vec<cm_db::repo::loops::Loop>>, ApiError> {
Ok(Json(
cm_db::repo::loops::list(&state.pool, user.workspace_id.as_uuid()).await?,
))
}
pub async fn get_loop(
State(state): State<AppState>,
Authed(user): Authed,
Path(id): Path<Uuid>,
) -> Result<Json<cm_db::repo::loops::Loop>, ApiError> {
cm_db::repo::loops::get(&state.pool, id, user.workspace_id.as_uuid())
.await?
.map(Json)
.ok_or(ApiError::NotFound)
}
#[derive(Deserialize)]
pub struct UpdateLoopRequest {
pub title: String,
pub description: String,
pub graph: Value,
pub task_template: String,
pub triggers: Value,
pub repeat_policy: Value,
}
pub async fn patch_loop(
State(state): State<AppState>,
Authed(user): Authed,
Path(id): Path<Uuid>,
Json(body): Json<UpdateLoopRequest>,
) -> Result<StatusCode, ApiError> {
if body.title.trim().is_empty() || body.task_template.trim().is_empty() {
return Err(ApiError::BadRequest);
}
cm_db::repo::loops::get(&state.pool, id, user.workspace_id.as_uuid())
.await?
.ok_or(ApiError::NotFound)?;
let next_fire_at = compute_next_fire(&body.triggers);
cm_db::repo::loops::update(
&state.pool,
id,
user.workspace_id.as_uuid(),
cm_db::repo::loops::UpdateLoop {
title: body.title.trim(),
description: body.description.trim(),
graph: &body.graph,
task_template: body.task_template.trim(),
triggers: &body.triggers,
repeat_policy: &body.repeat_policy,
next_fire_at,
},
)
.await?;
Ok(StatusCode::NO_CONTENT)
}
pub async fn delete_loop(
State(state): State<AppState>,
Authed(user): Authed,
Path(id): Path<Uuid>,
) -> Result<StatusCode, ApiError> {
cm_db::repo::loops::delete(&state.pool, id, user.workspace_id.as_uuid()).await?;
Ok(StatusCode::NO_CONTENT)
}
pub async fn enable_loop(
State(state): State<AppState>,
Authed(user): Authed,
Path(id): Path<Uuid>,
) -> Result<StatusCode, ApiError> {
cm_db::repo::loops::set_enabled(&state.pool, id, user.workspace_id.as_uuid(), true).await?;
Ok(StatusCode::NO_CONTENT)
}
pub async fn disable_loop(
State(state): State<AppState>,
Authed(user): Authed,
Path(id): Path<Uuid>,
) -> Result<StatusCode, ApiError> {
cm_db::repo::loops::set_enabled(&state.pool, id, user.workspace_id.as_uuid(), false).await?;
Ok(StatusCode::NO_CONTENT)
}
#[derive(Serialize)]
pub struct RunTriggered {
pub run_id: Uuid,
pub iteration: i32,
}
/// `POST /api/loops/:id/run` — enqueue one iteration NOW, bypassing the
/// scheduler and any trigger config. Iteration counter continues from
/// wherever it was; parent_run_id chains to whatever last_run_id points at.
pub async fn run_now(
State(state): State<AppState>,
Authed(user): Authed,
Path(id): Path<Uuid>,
) -> Result<Json<RunTriggered>, ApiError> {
let l = cm_db::repo::loops::get(&state.pool, id, user.workspace_id.as_uuid())
.await?
.ok_or(ApiError::NotFound)?;
let iter = cm_db::repo::loops::next_iteration(&state.pool, l.id).await?;
let run_id = cm_db::repo::loops::enqueue_iteration(
&state.pool,
l.id,
l.workspace_id,
&l.task_template,
&l.graph,
iter,
l.last_run_id,
)
.await?;
cm_db::repo::loops::mark_fired(&state.pool, l.id, run_id, l.next_fire_at).await?;
Ok(Json(RunTriggered {
run_id,
iteration: iter,
}))
}
/// `POST /webhooks/loops/:token` — public, HMAC-verified. Enqueues one
/// iteration on the loop that owns `token`. Returns 202 + `{run_id}` on
/// success, 401 on missing/bad signature, 404 on unknown token.
pub async fn webhook_receive(
State(state): State<AppState>,
Path(token): Path<String>,
headers: HeaderMap,
body: Bytes,
) -> (StatusCode, Json<Value>) {
let Ok(Some((_id, _ws, key, l))) =
cm_db::repo::loops::get_by_webhook_token(&state.pool, &token).await
else {
return (StatusCode::NOT_FOUND, Json(Value::Null));
};
let Some(sig_header) = headers
.get("X-Loop-Signature")
.and_then(|v| v.to_str().ok())
else {
return (StatusCode::UNAUTHORIZED, Json(Value::Null));
};
let Some(provided) = sig_header.strip_prefix("sha256=") else {
return (StatusCode::UNAUTHORIZED, Json(Value::Null));
};
if !verify_hmac(&key, &body, provided) {
return (StatusCode::UNAUTHORIZED, Json(Value::Null));
}
let iter = match cm_db::repo::loops::next_iteration(&state.pool, l.id).await {
Ok(n) => n,
Err(_) => return (StatusCode::INTERNAL_SERVER_ERROR, Json(Value::Null)),
};
let run_id = match cm_db::repo::loops::enqueue_iteration(
&state.pool,
l.id,
l.workspace_id,
&l.task_template,
&l.graph,
iter,
l.last_run_id,
)
.await
{
Ok(r) => r,
Err(_) => return (StatusCode::INTERNAL_SERVER_ERROR, Json(Value::Null)),
};
let _ = cm_db::repo::loops::mark_fired(&state.pool, l.id, run_id, None).await;
(
StatusCode::ACCEPTED,
Json(serde_json::json!({"run_id": run_id, "iteration": iter})),
)
}
fn verify_hmac(key: &str, body: &[u8], provided_hex: &str) -> bool {
let Ok(mut mac) = Hmac::<sha2::Sha256>::new_from_slice(key.as_bytes()) else {
return false;
};
mac.update(body);
let expected = hex::encode(mac.finalize().into_bytes());
if expected.len() != provided_hex.len() {
return false;
}
// Constant-time compare.
expected
.bytes()
.zip(provided_hex.bytes())
.fold(0u8, |acc, (a, b)| acc | (a ^ b))
== 0
}
+1
View File
@@ -12,6 +12,7 @@ pub mod files;
pub mod gateway; pub mod gateway;
pub mod health; pub mod health;
pub mod identity; pub mod identity;
pub mod loops;
pub mod nodes; pub mod nodes;
pub mod oauth; pub mod oauth;
pub mod orgs; pub mod orgs;
+313
View File
@@ -0,0 +1,313 @@
//! Loops — durable recurring topology executions. The row holds the
//! definition (graph + task_template + triggers + repeat_policy) and a small
//! amount of scheduler state (enabled, next_fire_at, last_run_id).
//! Each fire produces a normal `topology_runs` row with loop_id + iteration
//! + parent_run_id set, so the run driver picks it up like any other job.
//!
//! See 0031 migration header for the state semantics and missed-window rule.
use serde::{Deserialize, Serialize};
use serde_json::Value;
use sqlx::PgPool;
use time::OffsetDateTime;
use uuid::Uuid;
use crate::DbError;
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct Loop {
pub id: Uuid,
pub workspace_id: Uuid,
pub title: String,
pub description: String,
pub graph: Value,
pub task_template: String,
pub triggers: Value,
pub repeat_policy: Value,
pub enabled: bool,
pub next_fire_at: Option<OffsetDateTime>,
pub last_run_id: Option<Uuid>,
pub webhook_token: Option<String>,
pub webhook_signing_key: Option<String>,
pub created_by: Uuid,
pub created_at: OffsetDateTime,
pub updated_at: OffsetDateTime,
}
/// Minimal fields the scheduler needs when it wakes up.
#[derive(Debug, Clone)]
pub struct DueLoop {
pub id: Uuid,
pub workspace_id: Uuid,
pub graph: Value,
pub task_template: String,
pub triggers: Value,
pub repeat_policy: Value,
pub last_run_id: Option<Uuid>,
}
pub struct NewLoop<'a> {
pub workspace_id: Uuid,
pub title: &'a str,
pub description: &'a str,
pub graph: &'a Value,
pub task_template: &'a str,
pub triggers: &'a Value,
pub repeat_policy: &'a Value,
pub enabled: bool,
pub next_fire_at: Option<OffsetDateTime>,
pub webhook_token: Option<&'a str>,
pub webhook_signing_key: Option<&'a str>,
pub created_by: Uuid,
}
pub async fn create(pool: &PgPool, input: NewLoop<'_>) -> Result<Uuid, DbError> {
let id = Uuid::now_v7();
sqlx::query!(
"INSERT INTO loops
(id, workspace_id, title, description, graph, task_template,
triggers, repeat_policy, enabled, next_fire_at,
webhook_token, webhook_signing_key, created_by)
VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13)",
id,
input.workspace_id,
input.title,
input.description,
input.graph,
input.task_template,
input.triggers,
input.repeat_policy,
input.enabled,
input.next_fire_at,
input.webhook_token,
input.webhook_signing_key,
input.created_by,
)
.execute(pool)
.await?;
Ok(id)
}
pub async fn list(pool: &PgPool, workspace_id: Uuid) -> Result<Vec<Loop>, DbError> {
let rows = sqlx::query_as!(
Loop,
"SELECT id, workspace_id, title, description, graph, task_template,
triggers, repeat_policy, enabled, next_fire_at, last_run_id,
webhook_token, webhook_signing_key, created_by, created_at, updated_at
FROM loops
WHERE workspace_id = $1
ORDER BY updated_at DESC",
workspace_id,
)
.fetch_all(pool)
.await?;
Ok(rows)
}
pub async fn get(pool: &PgPool, id: Uuid, workspace_id: Uuid) -> Result<Option<Loop>, DbError> {
let row = sqlx::query_as!(
Loop,
"SELECT id, workspace_id, title, description, graph, task_template,
triggers, repeat_policy, enabled, next_fire_at, last_run_id,
webhook_token, webhook_signing_key, created_by, created_at, updated_at
FROM loops
WHERE id = $1 AND workspace_id = $2",
id,
workspace_id,
)
.fetch_optional(pool)
.await?;
Ok(row)
}
/// Look up a loop by its webhook token — used only by the webhook receiver,
/// which has no session context. Returns the minimal shape needed to enqueue
/// an iteration and verify the HMAC signature.
pub async fn get_by_webhook_token(
pool: &PgPool,
token: &str,
) -> Result<Option<(Uuid, Uuid, String, DueLoop)>, DbError> {
let row = sqlx::query!(
"SELECT id, workspace_id, graph, task_template, triggers, repeat_policy,
last_run_id, webhook_signing_key
FROM loops
WHERE webhook_token = $1 AND enabled",
token,
)
.fetch_optional(pool)
.await?;
Ok(row.and_then(|r| {
let key = r.webhook_signing_key?;
Some((
r.id,
r.workspace_id,
key,
DueLoop {
id: r.id,
workspace_id: r.workspace_id,
graph: r.graph,
task_template: r.task_template,
triggers: r.triggers,
repeat_policy: r.repeat_policy,
last_run_id: r.last_run_id,
},
))
}))
}
pub struct UpdateLoop<'a> {
pub title: &'a str,
pub description: &'a str,
pub graph: &'a Value,
pub task_template: &'a str,
pub triggers: &'a Value,
pub repeat_policy: &'a Value,
pub next_fire_at: Option<OffsetDateTime>,
}
pub async fn update(
pool: &PgPool,
id: Uuid,
workspace_id: Uuid,
input: UpdateLoop<'_>,
) -> Result<(), DbError> {
sqlx::query!(
"UPDATE loops
SET title = $3, description = $4, graph = $5, task_template = $6,
triggers = $7, repeat_policy = $8, next_fire_at = $9,
updated_at = now()
WHERE id = $1 AND workspace_id = $2",
id,
workspace_id,
input.title,
input.description,
input.graph,
input.task_template,
input.triggers,
input.repeat_policy,
input.next_fire_at,
)
.execute(pool)
.await?;
Ok(())
}
pub async fn set_enabled(
pool: &PgPool,
id: Uuid,
workspace_id: Uuid,
enabled: bool,
) -> Result<(), DbError> {
sqlx::query!(
"UPDATE loops SET enabled = $3, updated_at = now()
WHERE id = $1 AND workspace_id = $2",
id,
workspace_id,
enabled,
)
.execute(pool)
.await?;
Ok(())
}
pub async fn delete(pool: &PgPool, id: Uuid, workspace_id: Uuid) -> Result<(), DbError> {
sqlx::query!(
"DELETE FROM loops WHERE id = $1 AND workspace_id = $2",
id,
workspace_id,
)
.execute(pool)
.await?;
Ok(())
}
/// Loops the scheduler tick should fire NOW. Only reads what the enqueue
/// path needs, so the tick stays cheap even when the workspace has hundreds
/// of loops.
pub async fn due(pool: &PgPool) -> Result<Vec<DueLoop>, DbError> {
let rows = sqlx::query!(
"SELECT id, workspace_id, graph, task_template, triggers, repeat_policy,
last_run_id
FROM loops
WHERE enabled AND next_fire_at IS NOT NULL AND next_fire_at <= now()",
)
.fetch_all(pool)
.await?;
Ok(rows
.into_iter()
.map(|r| DueLoop {
id: r.id,
workspace_id: r.workspace_id,
graph: r.graph,
task_template: r.task_template,
triggers: r.triggers,
repeat_policy: r.repeat_policy,
last_run_id: r.last_run_id,
})
.collect())
}
/// Post-fire bookkeeping: bump last_run_id + advance next_fire_at (NULL when
/// the loop has no cron trigger). Called by the scheduler after a successful
/// enqueue_iteration.
pub async fn mark_fired(
pool: &PgPool,
id: Uuid,
run_id: Uuid,
next_fire_at: Option<OffsetDateTime>,
) -> Result<(), DbError> {
sqlx::query!(
"UPDATE loops
SET last_run_id = $2, next_fire_at = $3, updated_at = now()
WHERE id = $1",
id,
run_id,
next_fire_at,
)
.execute(pool)
.await?;
Ok(())
}
/// Next iteration number for a loop (1 if it has never fired).
pub async fn next_iteration(pool: &PgPool, loop_id: Uuid) -> Result<i32, DbError> {
let n: Option<i32> = sqlx::query_scalar!(
"SELECT MAX(iteration) FROM topology_runs WHERE loop_id = $1",
loop_id,
)
.fetch_one(pool)
.await?;
Ok(n.unwrap_or(0) + 1)
}
/// Enqueue an iteration as a normal `topology_runs` row. The scheduler,
/// on-completion hook, and webhook receiver all funnel through here so the
/// invariants (loop_id + iteration + parent_run_id all set together) stay
/// in one place.
pub async fn enqueue_iteration(
pool: &PgPool,
loop_id: Uuid,
workspace_id: Uuid,
task: &str,
graph: &Value,
iteration: i32,
parent_run_id: Option<Uuid>,
) -> Result<Uuid, DbError> {
let run_id = Uuid::now_v7();
sqlx::query!(
"INSERT INTO topology_runs
(id, workspace_id, task, kind, status, graph, tier,
loop_id, iteration, parent_run_id)
VALUES ($1, $2, $3, 'run', 'queued', $4, 'team', $5, $6, $7)",
run_id,
workspace_id,
task,
graph,
loop_id,
iteration,
parent_run_id,
)
.execute(pool)
.await?;
Ok(run_id)
}
+1
View File
@@ -8,6 +8,7 @@ pub mod credits;
pub mod files; pub mod files;
pub mod fleet_beszel; pub mod fleet_beszel;
pub mod fleet_tailscale; pub mod fleet_tailscale;
pub mod loops;
pub mod messages; pub mod messages;
pub mod node_metrics; pub mod node_metrics;
pub mod node_rules; pub mod node_rules;
+2
View File
@@ -4,6 +4,7 @@
mod brain; mod brain;
mod events; mod events;
pub mod loops;
pub mod outbox; pub mod outbox;
mod runtime; mod runtime;
mod sandboxes; mod sandboxes;
@@ -12,6 +13,7 @@ mod terminals;
mod tools; mod tools;
pub use events::{RunEventBody, RunEventEnvelope}; pub use events::{RunEventBody, RunEventEnvelope};
pub use loops::spawn_loop_scheduler;
pub use outbox::{drain_once, spawn_drainer, EmailSender, LettreSender, SmtpConfig}; pub use outbox::{drain_once, spawn_drainer, EmailSender, LettreSender, SmtpConfig};
pub use runtime::{ pub use runtime::{
judge_model, ProviderRegistry, Runtime, RuntimeConfig, RuntimeError, StartedRun, judge_model, ProviderRegistry, Runtime, RuntimeConfig, RuntimeError, StartedRun,
+141
View File
@@ -0,0 +1,141 @@
//! Loop scheduler: periodically wakes, finds loops whose `next_fire_at`
//! has arrived, enqueues one topology_runs row per fire (with loop_id +
//! iteration + parent_run_id set), and computes the next fire time from
//! the cron trigger.
//!
//! Missed-window rule: `next_fire_at` is always computed strictly AFTER
//! `now()`, so a scheduler that woke up late (restart, long stall) fires
//! ONCE and skips whatever windows were in the backlog. This matches the
//! "fire once and move on" behavior we chose in the spec.
use std::time::Duration;
use serde::Deserialize;
use time::OffsetDateTime;
use tokio::time::interval;
use crate::scheduling::next_occurrence;
/// The subset of `triggers` JSONB the scheduler needs to make decisions.
#[derive(Debug, Default, Deserialize)]
struct Triggers {
/// Cron pattern (5-field). None => scheduler doesn't participate.
#[serde(default)]
cron: Option<String>,
/// Enqueue next iteration when the previous one hits `run_completed`.
/// Handled by the run driver on completion (see runtime::spawn_drive);
/// the scheduler doesn't own this branch, we surface it here just so
/// mark_fired() below knows whether to null out next_fire_at.
#[serde(default)]
on_completion: bool,
#[serde(default)]
webhook_enabled: bool,
}
/// The subset of `repeat_policy` JSONB the scheduler needs.
#[derive(Debug, Default, Deserialize)]
struct RepeatPolicy {
/// `infinite` | `iters` | `until`. Anything unknown = `infinite`.
#[serde(default = "default_kind")]
kind: String,
/// `iters.n` stopping condition.
#[serde(default)]
n: Option<i32>,
}
fn default_kind() -> String {
"infinite".to_string()
}
/// Runs the tick every `interval_duration` until the process exits.
pub fn spawn_loop_scheduler(pool: sqlx::PgPool, interval_duration: Duration) {
tokio::spawn(async move {
let mut tick = interval(interval_duration);
// First tick fires immediately; second waits the full interval. That's
// fine — the query is a bounded partial-index scan.
loop {
tick.tick().await;
if let Err(e) = fire_due(&pool).await {
eprintln!("loop scheduler tick failed: {e}");
}
}
});
}
/// One tick: find due loops, fire each. Errors from one loop don't stop the
/// others.
async fn fire_due(pool: &sqlx::PgPool) -> Result<(), sqlx::Error> {
let due = cm_db::repo::loops::due(pool).await.map_err(sqlx_err)?;
for l in due {
if let Err(e) = fire_one(pool, &l).await {
eprintln!("loop {} fire failed: {e}", l.id);
}
}
Ok(())
}
async fn fire_one(pool: &sqlx::PgPool, l: &cm_db::repo::loops::DueLoop) -> Result<(), sqlx::Error> {
let triggers: Triggers = serde_json::from_value(l.triggers.clone()).unwrap_or_default();
let policy: RepeatPolicy = serde_json::from_value(l.repeat_policy.clone()).unwrap_or_default();
let iter = cm_db::repo::loops::next_iteration(pool, l.id)
.await
.map_err(sqlx_err)?;
// Repeat cap. `iters` stops after N total iterations; `infinite` and
// `until` don't check here (until is enforced by the on-completion path
// which inspects the run's terminal event; scope for the scheduler stops
// at cron time-based firing).
if policy.kind == "iters" {
if let Some(cap) = policy.n {
if iter > cap {
// Silently disable the loop so we don't tick it forever.
let _ = cm_db::repo::loops::set_enabled(pool, l.id, l.workspace_id, false).await;
return Ok(());
}
}
}
let run_id = cm_db::repo::loops::enqueue_iteration(
pool,
l.id,
l.workspace_id,
&l.task_template,
&l.graph,
iter,
l.last_run_id,
)
.await
.map_err(sqlx_err)?;
// Advance next_fire_at strictly AFTER now(). If there's no cron trigger
// (e.g. webhook-only or on-completion-only), null it out so the partial
// index stops matching this loop for the scheduler.
let next = match triggers.cron.as_deref() {
Some(pattern) if !pattern.is_empty() => {
match next_occurrence(pattern, OffsetDateTime::now_utc()) {
Ok(t) => Some(t),
Err(e) => {
eprintln!("loop {} invalid cron '{}': {e}", l.id, pattern);
None
}
}
}
_ => None,
};
// Also null it out when the cron trigger vanished but on_completion or
// webhook_enabled is still on — those paths will re-fire independently.
let _ = (triggers.on_completion, triggers.webhook_enabled);
cm_db::repo::loops::mark_fired(pool, l.id, run_id, next)
.await
.map_err(sqlx_err)?;
Ok(())
}
fn sqlx_err(e: cm_db::DbError) -> sqlx::Error {
match e {
cm_db::DbError::Other(e) => e,
cm_db::DbError::NotFound => sqlx::Error::RowNotFound,
cm_db::DbError::Conflict(_) => sqlx::Error::PoolTimedOut,
}
}