Fleet: actionable executions — rules engine + metrics-aware placement (Phase 2)

Turn the Beszel-tapped metrics into a self-managing loop.

- migration node_rules (workspace/node-scoped: metric op threshold, for_seconds,
  action JSONB, last_fired).
- cm-db: repo/node_rules.rs (CRUD + list_enabled); node_metrics::eval_all merges
  Beszel + heartbeat scalars per node + a headroom() heuristic; nodes::status_of;
  heartbeat now PRESERVES a `draining` status across heartbeats (so a cordon sticks).
- cm-api: node_rules.rs evaluator (spawn_evaluator, 20s) — when a metric condition
  holds for the rule's window it fires drain / undrain / alert (in-memory sustained
  + cooldown tracking, modeled on the node sweeper); routes/beszel.rs rules CRUD
  (GET/POST/PATCH/DELETE /api/fleet/rules); spawned in clawmates-server.
- cm-runtime: placement_node() is metrics-aware — a `draining` node stops receiving
  new agent sandboxes (falls back to local), so the drain rule is actionable.
- frontend: FleetRules section in the Local view — build rules (node · metric · op ·
  threshold · duration → action), toggle/delete, with fired-history.

The loop: hot/overloaded node → rule drains it → placement avoids it → recovers →
undrain rule brings it back. Deployed; node_rules migration applied.

Co-Authored-By: Claude Opus 4.8 (1M context) <[email protected]>

crates/cm-db/src/repo/node_rules.rs

@@ -0,0 +1,127 @@
+//! Fleet automation rules: metric thresholds → actions (drain/undrain/alert),
+//! evaluated by the background rules engine.
+
+use cm_domain::{NodeId, WorkspaceId};
+use serde_json::Value;
+use sqlx::{PgPool, Row};
+use time::OffsetDateTime;
+use uuid::Uuid;
+
+use crate::DbError;
+
+#[derive(Debug, Clone)]
+pub struct NodeRule {
+    pub id: Uuid,
+    pub workspace_id: WorkspaceId,
+    pub node_id: Option<NodeId>,
+    pub name: String,
+    pub enabled: bool,
+    pub metric: String,
+    pub op: String,
+    pub threshold: f64,
+    pub for_seconds: i32,
+    pub action: Value,
+    pub last_fired_at: Option<OffsetDateTime>,
+}
+
+fn map_rule(r: sqlx::postgres::PgRow) -> NodeRule {
+    NodeRule {
+        id: r.get("id"),
+        workspace_id: WorkspaceId::from(r.get::<Uuid, _>("workspace_id")),
+        node_id: r.get::<Option<Uuid>, _>("node_id").map(NodeId::from),
+        name: r.get("name"),
+        enabled: r.get("enabled"),
+        metric: r.get("metric"),
+        op: r.get("op"),
+        threshold: r.get("threshold"),
+        for_seconds: r.get("for_seconds"),
+        action: r.get("action"),
+        last_fired_at: r.get("last_fired_at"),
+    }
+}
+
+const COLS: &str =
+    "id, workspace_id, node_id, name, enabled, metric, op, threshold, for_seconds, action, last_fired_at";
+
+#[allow(clippy::too_many_arguments)]
+pub async fn create(
+    pool: &PgPool,
+    workspace_id: WorkspaceId,
+    node_id: Option<NodeId>,
+    name: &str,
+    metric: &str,
+    op: &str,
+    threshold: f64,
+    for_seconds: i32,
+    action: &Value,
+) -> Result<Uuid, DbError> {
+    let id = Uuid::now_v7();
+    sqlx::query(
+        "INSERT INTO node_rules (id, workspace_id, node_id, name, metric, op, threshold, for_seconds, action)
+         VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9)",
+    )
+    .bind(id)
+    .bind(workspace_id.as_uuid())
+    .bind(node_id.map(|n| n.as_uuid()))
+    .bind(name)
+    .bind(metric)
+    .bind(op)
+    .bind(threshold)
+    .bind(for_seconds)
+    .bind(action)
+    .execute(pool)
+    .await?;
+    Ok(id)
+}
+
+/// List a workspace's rules (newest last).
+pub async fn list(pool: &PgPool, workspace_id: WorkspaceId) -> Result<Vec<NodeRule>, DbError> {
+    let rows = sqlx::query(&format!(
+        "SELECT {COLS} FROM node_rules WHERE workspace_id = $1 ORDER BY created_at"
+    ))
+    .bind(workspace_id.as_uuid())
+    .fetch_all(pool)
+    .await?;
+    Ok(rows.into_iter().map(map_rule).collect())
+}
+
+/// All enabled rules across workspaces (for the evaluator).
+pub async fn list_enabled(pool: &PgPool) -> Result<Vec<NodeRule>, DbError> {
+    let rows = sqlx::query(&format!("SELECT {COLS} FROM node_rules WHERE enabled"))
+        .fetch_all(pool)
+        .await?;
+    Ok(rows.into_iter().map(map_rule).collect())
+}
+
+pub async fn set_enabled(
+    pool: &PgPool,
+    id: Uuid,
+    workspace_id: WorkspaceId,
+    enabled: bool,
+) -> Result<(), DbError> {
+    sqlx::query("UPDATE node_rules SET enabled = $3 WHERE id = $1 AND workspace_id = $2")
+        .bind(id)
+        .bind(workspace_id.as_uuid())
+        .bind(enabled)
+        .execute(pool)
+        .await?;
+    Ok(())
+}
+
+pub async fn delete(pool: &PgPool, id: Uuid, workspace_id: WorkspaceId) -> Result<(), DbError> {
+    sqlx::query("DELETE FROM node_rules WHERE id = $1 AND workspace_id = $2")
+        .bind(id)
+        .bind(workspace_id.as_uuid())
+        .execute(pool)
+        .await?;
+    Ok(())
+}
+
+/// Stamp a rule as just-fired (debounce + UI).
+pub async fn mark_fired(pool: &PgPool, id: Uuid) -> Result<(), DbError> {
+    sqlx::query("UPDATE node_rules SET last_fired_at = now() WHERE id = $1")
+        .bind(id)
+        .execute(pool)
+        .await?;
+    Ok(())
+}