Turn the Beszel-tapped metrics into a self-managing loop. - migration node_rules (workspace/node-scoped: metric op threshold, for_seconds, action JSONB, last_fired). - cm-db: repo/node_rules.rs (CRUD + list_enabled); node_metrics::eval_all merges Beszel + heartbeat scalars per node + a headroom() heuristic; nodes::status_of; heartbeat now PRESERVES a `draining` status across heartbeats (so a cordon sticks). - cm-api: node_rules.rs evaluator (spawn_evaluator, 20s) — when a metric condition holds for the rule's window it fires drain / undrain / alert (in-memory sustained + cooldown tracking, modeled on the node sweeper); routes/beszel.rs rules CRUD (GET/POST/PATCH/DELETE /api/fleet/rules); spawned in clawmates-server. - cm-runtime: placement_node() is metrics-aware — a `draining` node stops receiving new agent sandboxes (falls back to local), so the drain rule is actionable. - frontend: FleetRules section in the Local view — build rules (node · metric · op · threshold · duration → action), toggle/delete, with fired-history. The loop: hot/overloaded node → rule drains it → placement avoids it → recovers → undrain rule brings it back. Deployed; node_rules migration applied. Co-Authored-By: Claude Opus 4.8 (1M context) <[email protected]>
128 lines
3.6 KiB
Rust
128 lines
3.6 KiB
Rust
//! Fleet automation rules: metric thresholds → actions (drain/undrain/alert),
|
|
//! evaluated by the background rules engine.
|
|
|
|
use cm_domain::{NodeId, WorkspaceId};
|
|
use serde_json::Value;
|
|
use sqlx::{PgPool, Row};
|
|
use time::OffsetDateTime;
|
|
use uuid::Uuid;
|
|
|
|
use crate::DbError;
|
|
|
|
#[derive(Debug, Clone)]
|
|
pub struct NodeRule {
|
|
pub id: Uuid,
|
|
pub workspace_id: WorkspaceId,
|
|
pub node_id: Option<NodeId>,
|
|
pub name: String,
|
|
pub enabled: bool,
|
|
pub metric: String,
|
|
pub op: String,
|
|
pub threshold: f64,
|
|
pub for_seconds: i32,
|
|
pub action: Value,
|
|
pub last_fired_at: Option<OffsetDateTime>,
|
|
}
|
|
|
|
fn map_rule(r: sqlx::postgres::PgRow) -> NodeRule {
|
|
NodeRule {
|
|
id: r.get("id"),
|
|
workspace_id: WorkspaceId::from(r.get::<Uuid, _>("workspace_id")),
|
|
node_id: r.get::<Option<Uuid>, _>("node_id").map(NodeId::from),
|
|
name: r.get("name"),
|
|
enabled: r.get("enabled"),
|
|
metric: r.get("metric"),
|
|
op: r.get("op"),
|
|
threshold: r.get("threshold"),
|
|
for_seconds: r.get("for_seconds"),
|
|
action: r.get("action"),
|
|
last_fired_at: r.get("last_fired_at"),
|
|
}
|
|
}
|
|
|
|
const COLS: &str =
|
|
"id, workspace_id, node_id, name, enabled, metric, op, threshold, for_seconds, action, last_fired_at";
|
|
|
|
#[allow(clippy::too_many_arguments)]
|
|
pub async fn create(
|
|
pool: &PgPool,
|
|
workspace_id: WorkspaceId,
|
|
node_id: Option<NodeId>,
|
|
name: &str,
|
|
metric: &str,
|
|
op: &str,
|
|
threshold: f64,
|
|
for_seconds: i32,
|
|
action: &Value,
|
|
) -> Result<Uuid, DbError> {
|
|
let id = Uuid::now_v7();
|
|
sqlx::query(
|
|
"INSERT INTO node_rules (id, workspace_id, node_id, name, metric, op, threshold, for_seconds, action)
|
|
VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9)",
|
|
)
|
|
.bind(id)
|
|
.bind(workspace_id.as_uuid())
|
|
.bind(node_id.map(|n| n.as_uuid()))
|
|
.bind(name)
|
|
.bind(metric)
|
|
.bind(op)
|
|
.bind(threshold)
|
|
.bind(for_seconds)
|
|
.bind(action)
|
|
.execute(pool)
|
|
.await?;
|
|
Ok(id)
|
|
}
|
|
|
|
/// List a workspace's rules (newest last).
|
|
pub async fn list(pool: &PgPool, workspace_id: WorkspaceId) -> Result<Vec<NodeRule>, DbError> {
|
|
let rows = sqlx::query(&format!(
|
|
"SELECT {COLS} FROM node_rules WHERE workspace_id = $1 ORDER BY created_at"
|
|
))
|
|
.bind(workspace_id.as_uuid())
|
|
.fetch_all(pool)
|
|
.await?;
|
|
Ok(rows.into_iter().map(map_rule).collect())
|
|
}
|
|
|
|
/// All enabled rules across workspaces (for the evaluator).
|
|
pub async fn list_enabled(pool: &PgPool) -> Result<Vec<NodeRule>, DbError> {
|
|
let rows = sqlx::query(&format!("SELECT {COLS} FROM node_rules WHERE enabled"))
|
|
.fetch_all(pool)
|
|
.await?;
|
|
Ok(rows.into_iter().map(map_rule).collect())
|
|
}
|
|
|
|
pub async fn set_enabled(
|
|
pool: &PgPool,
|
|
id: Uuid,
|
|
workspace_id: WorkspaceId,
|
|
enabled: bool,
|
|
) -> Result<(), DbError> {
|
|
sqlx::query("UPDATE node_rules SET enabled = $3 WHERE id = $1 AND workspace_id = $2")
|
|
.bind(id)
|
|
.bind(workspace_id.as_uuid())
|
|
.bind(enabled)
|
|
.execute(pool)
|
|
.await?;
|
|
Ok(())
|
|
}
|
|
|
|
pub async fn delete(pool: &PgPool, id: Uuid, workspace_id: WorkspaceId) -> Result<(), DbError> {
|
|
sqlx::query("DELETE FROM node_rules WHERE id = $1 AND workspace_id = $2")
|
|
.bind(id)
|
|
.bind(workspace_id.as_uuid())
|
|
.execute(pool)
|
|
.await?;
|
|
Ok(())
|
|
}
|
|
|
|
/// Stamp a rule as just-fired (debounce + UI).
|
|
pub async fn mark_fired(pool: &PgPool, id: Uuid) -> Result<(), DbError> {
|
|
sqlx::query("UPDATE node_rules SET last_fired_at = now() WHERE id = $1")
|
|
.bind(id)
|
|
.execute(pool)
|
|
.await?;
|
|
Ok(())
|
|
}
|