Fleet: actionable executions — rules engine + metrics-aware placement (Phase 2)

Turn the Beszel-tapped metrics into a self-managing loop.

- migration node_rules (workspace/node-scoped: metric op threshold, for_seconds,
  action JSONB, last_fired).
- cm-db: repo/node_rules.rs (CRUD + list_enabled); node_metrics::eval_all merges
  Beszel + heartbeat scalars per node + a headroom() heuristic; nodes::status_of;
  heartbeat now PRESERVES a `draining` status across heartbeats (so a cordon sticks).
- cm-api: node_rules.rs evaluator (spawn_evaluator, 20s) — when a metric condition
  holds for the rule's window it fires drain / undrain / alert (in-memory sustained
  + cooldown tracking, modeled on the node sweeper); routes/beszel.rs rules CRUD
  (GET/POST/PATCH/DELETE /api/fleet/rules); spawned in clawmates-server.
- cm-runtime: placement_node() is metrics-aware — a `draining` node stops receiving
  new agent sandboxes (falls back to local), so the drain rule is actionable.
- frontend: FleetRules section in the Local view — build rules (node · metric · op ·
  threshold · duration → action), toggle/delete, with fired-history.

The loop: hot/overloaded node → rule drains it → placement avoids it → recovers →
undrain rule brings it back. Deployed; node_rules migration applied.

Co-Authored-By: Claude Opus 4.8 (1M context) <[email protected]>

crates/cm-api/src/routes/beszel.rs

@@ -5,7 +5,7 @@
 use axum::extract::{Path, State};
 use axum::Json;
 use cm_db::repo::fleet_beszel::BeszelConn;
-use cm_db::repo::{fleet_beszel, node_metrics, nodes};
+use cm_db::repo::{fleet_beszel, node_metrics, node_rules, nodes};
 use cm_domain::NodeId;
 use serde::Deserialize;
 use serde_json::{json, Value};
@@ -88,3 +88,105 @@ pub async fn node_metrics_get(
     }
     Ok(Json(json!({ "latest": latest, "history": history })))
 }
+
+// ── Fleet automation rules ──────────────────────────────────────────────────
+
+const METRICS: [&str; 6] = ["cpu_pct", "mem_pct", "disk_pct", "gpu_pct", "temp_max", "load1"];
+const OPS: [&str; 4] = [">", "<", ">=", "<="];
+
+fn rule_json(r: &node_rules::NodeRule) -> Value {
+    json!({
+        "id": r.id,
+        "nodeId": r.node_id.map(|n| n.as_uuid()),
+        "name": r.name,
+        "enabled": r.enabled,
+        "metric": r.metric,
+        "op": r.op,
+        "threshold": r.threshold,
+        "forSeconds": r.for_seconds,
+        "action": r.action,
+        "lastFired": r.last_fired_at.map(|t| t.unix_timestamp()),
+    })
+}
+
+/// `GET /api/fleet/rules` — the workspace's automation rules.
+pub async fn rules_list(
+    State(state): State<AppState>,
+    Authed(user): Authed,
+) -> Result<Json<Value>, ApiError> {
+    let rules = node_rules::list(&state.pool, user.workspace_id).await?;
+    Ok(Json(json!({ "rules": rules.iter().map(rule_json).collect::<Vec<_>>() })))
+}
+
+#[derive(Deserialize)]
+pub struct RuleReq {
+    pub name: String,
+    #[serde(rename = "nodeId")]
+    pub node_id: Option<String>,
+    pub metric: String,
+    pub op: String,
+    pub threshold: f64,
+    #[serde(rename = "forSeconds")]
+    pub for_seconds: Option<i32>,
+    pub action: Value,
+}
+
+/// `POST /api/fleet/rules` — create an automation rule.
+pub async fn rules_create(
+    State(state): State<AppState>,
+    Authed(user): Authed,
+    Json(req): Json<RuleReq>,
+) -> Result<Json<Value>, ApiError> {
+    if !METRICS.contains(&req.metric.as_str()) || !OPS.contains(&req.op.as_str()) {
+        return Ok(Json(json!({ "ok": false, "error": "invalid metric or operator" })));
+    }
+    let node_id = match req.node_id.as_deref() {
+        Some(s) if !s.is_empty() && s != "all" => {
+            let nid = NodeId::from(s.parse::<Uuid>().map_err(|_| ApiError::NotFound)?);
+            nodes::get(&state.pool, nid, user.workspace_id)
+                .await?
+                .ok_or(ApiError::NotFound)?;
+            Some(nid)
+        }
+        _ => None,
+    };
+    let id = node_rules::create(
+        &state.pool,
+        user.workspace_id,
+        node_id,
+        req.name.trim(),
+        &req.metric,
+        &req.op,
+        req.threshold,
+        req.for_seconds.unwrap_or(60).max(0),
+        &req.action,
+    )
+    .await?;
+    Ok(Json(json!({ "ok": true, "id": id })))
+}
+
+#[derive(Deserialize)]
+pub struct RulePatch {
+    pub enabled: bool,
+}
+
+/// `PATCH /api/fleet/rules/{id}` — enable/disable a rule.
+pub async fn rules_patch(
+    State(state): State<AppState>,
+    Authed(user): Authed,
+    Path(id): Path<Uuid>,
+    Json(req): Json<RulePatch>,
+) -> Result<Json<Value>, ApiError> {
+    node_rules::set_enabled(&state.pool, id, user.workspace_id, req.enabled).await?;
+    Ok(Json(json!({ "ok": true })))
+}
+
+/// `DELETE /api/fleet/rules/{id}` — delete a rule.
+pub async fn rules_delete(
+    State(state): State<AppState>,
+    Authed(user): Authed,
+    Path(id): Path<Uuid>,
+) -> Result<Json<Value>, ApiError> {
+    node_rules::delete(&state.pool, id, user.workspace_id).await?;
+    Ok(Json(json!({ "ok": true })))
+}