Fleet: actionable executions — rules engine + metrics-aware placement (Phase 2)

Turn the Beszel-tapped metrics into a self-managing loop.

- migration node_rules (workspace/node-scoped: metric op threshold, for_seconds,
  action JSONB, last_fired).
- cm-db: repo/node_rules.rs (CRUD + list_enabled); node_metrics::eval_all merges
  Beszel + heartbeat scalars per node + a headroom() heuristic; nodes::status_of;
  heartbeat now PRESERVES a `draining` status across heartbeats (so a cordon sticks).
- cm-api: node_rules.rs evaluator (spawn_evaluator, 20s) — when a metric condition
  holds for the rule's window it fires drain / undrain / alert (in-memory sustained
  + cooldown tracking, modeled on the node sweeper); routes/beszel.rs rules CRUD
  (GET/POST/PATCH/DELETE /api/fleet/rules); spawned in clawmates-server.
- cm-runtime: placement_node() is metrics-aware — a `draining` node stops receiving
  new agent sandboxes (falls back to local), so the drain rule is actionable.
- frontend: FleetRules section in the Local view — build rules (node · metric · op ·
  threshold · duration → action), toggle/delete, with fired-history.

The loop: hot/overloaded node → rule drains it → placement avoids it → recovers →
undrain rule brings it back. Deployed; node_rules migration applied.

Co-Authored-By: Claude Opus 4.8 (1M context) <[email protected]>

crates/bins/clawmates-server/src/main.rs

@@ -271,6 +271,8 @@ async fn run() -> Result<(), String> {
     cm_api::fleet::spawn_node_sweeper(pool.clone(), std::time::Duration::from_secs(8), 20);
     // Beszel: poll each workspace's monitoring hub for rich per-node metrics.
     cm_api::beszel::spawn_poller(pool.clone(), std::time::Duration::from_secs(15));
+    // Fleet automation: evaluate metric-threshold rules → drain/undrain/alert.
+    cm_api::node_rules::spawn_evaluator(pool.clone(), std::time::Duration::from_secs(20));
 
     // Hosted identity (Clerk / OIDC): pin the issuer and load its JWKS.
     let auth_verifier = match config.auth.mode {

crates/cm-api/src/lib.rs

@@ -2,6 +2,7 @@
 
 pub mod beszel;
 pub mod cleanup_sweeper;
+pub mod node_rules;
 mod error;
 mod extract;
 pub mod fleet;
@@ -132,6 +133,14 @@ pub fn router(state: AppState) -> Router {
                 .post(routes::beszel::connect)
                 .delete(routes::beszel::disconnect),
         )
+        .route(
+            "/api/fleet/rules",
+            get(routes::beszel::rules_list).post(routes::beszel::rules_create),
+        )
+        .route(
+            "/api/fleet/rules/{id}",
+            patch(routes::beszel::rules_patch).delete(routes::beszel::rules_delete),
+        )
         .route(
             "/api/fleet/tailscale",
             get(routes::tailscale::status)

crates/cm-api/src/node_rules.rs

@@ -0,0 +1,95 @@
+//! Fleet rules engine: evaluate metric-threshold rules against live node metrics
+//! and fire sustained-condition actions (drain / undrain / alert). Modeled on the
+//! node sweeper. Paired with metrics-aware placement (draining nodes stop taking
+//! new agent workloads).
+
+use std::collections::HashMap;
+use std::time::{Duration, Instant};
+
+use cm_db::repo::node_metrics::EvalRow;
+use cm_db::repo::{node_metrics, node_rules, nodes};
+use cm_domain::NodeId;
+use serde_json::Value;
+use sqlx::PgPool;
+use uuid::Uuid;
+
+fn cmp(value: f64, op: &str, threshold: f64) -> bool {
+    match op {
+        ">" => value > threshold,
+        "<" => value < threshold,
+        ">=" => value >= threshold,
+        "<=" => value <= threshold,
+        _ => false,
+    }
+}
+
+/// Run a rule's action against a node.
+async fn fire(pool: &PgPool, action: &Value, node: &EvalRow) {
+    match action.get("type").and_then(Value::as_str).unwrap_or("") {
+        "drain" => {
+            if node.status != "draining" {
+                let _ = nodes::set_status(pool, node.node_id, "draining").await;
+                eprintln!("[rules] drain → node {}", node.node_id.as_uuid());
+            }
+        }
+        "undrain" => {
+            if node.status == "draining" {
+                let _ = nodes::set_status(pool, node.node_id, "online").await;
+                eprintln!("[rules] undrain → node {}", node.node_id.as_uuid());
+            }
+        }
+        "alert" => eprintln!("[rules] ALERT → node {}", node.node_id.as_uuid()),
+        _ => {}
+    }
+}
+
+/// Spawn the rules evaluator: every `interval`, evaluate enabled rules against the
+/// latest metrics; fire when a condition has held for the rule's window.
+pub fn spawn_evaluator(pool: PgPool, interval: Duration) {
+    tokio::spawn(async move {
+        // (rule, node) → when the condition first held; and last-fired (cooldown).
+        let mut true_since: HashMap<(Uuid, NodeId), Instant> = HashMap::new();
+        let mut last_fired: HashMap<(Uuid, NodeId), Instant> = HashMap::new();
+        let mut tick = tokio::time::interval(interval);
+        loop {
+            tick.tick().await;
+            let (rules, evals) = match (
+                node_rules::list_enabled(&pool).await,
+                node_metrics::eval_all(&pool).await,
+            ) {
+                (Ok(r), Ok(e)) => (r, e),
+                _ => continue,
+            };
+            let live: std::collections::HashSet<NodeId> = evals.iter().map(|e| e.node_id).collect();
+            let now = Instant::now();
+            for rule in &rules {
+                for node in evals.iter().filter(|e| {
+                    e.workspace_id == rule.workspace_id
+                        && rule.node_id.map_or(true, |n| n == e.node_id)
+                }) {
+                    let key = (rule.id, node.node_id);
+                    let held = node
+                        .metric(&rule.metric)
+                        .is_some_and(|v| cmp(v, &rule.op, rule.threshold));
+                    if !held {
+                        true_since.remove(&key);
+                        continue;
+                    }
+                    let since = *true_since.entry(key).or_insert(now);
+                    let sustained =
+                        now.duration_since(since) >= Duration::from_secs(rule.for_seconds.max(0) as u64);
+                    let cooled = last_fired.get(&key).is_none_or(|&t| {
+                        now.duration_since(t) >= Duration::from_secs(rule.for_seconds.max(30) as u64)
+                    });
+                    if sustained && cooled {
+                        fire(&pool, &rule.action, node).await;
+                        last_fired.insert(key, now);
+                        let _ = node_rules::mark_fired(&pool, rule.id).await;
+                    }
+                }
+            }
+            true_since.retain(|(_, n), _| live.contains(n));
+            last_fired.retain(|(_, n), _| live.contains(n));
+        }
+    });
+}

crates/cm-api/src/routes/beszel.rs

@@ -5,7 +5,7 @@
 use axum::extract::{Path, State};
 use axum::Json;
 use cm_db::repo::fleet_beszel::BeszelConn;
-use cm_db::repo::{fleet_beszel, node_metrics, nodes};
+use cm_db::repo::{fleet_beszel, node_metrics, node_rules, nodes};
 use cm_domain::NodeId;
 use serde::Deserialize;
 use serde_json::{json, Value};
@@ -88,3 +88,105 @@ pub async fn node_metrics_get(
     }
     Ok(Json(json!({ "latest": latest, "history": history })))
 }
+
+// ── Fleet automation rules ──────────────────────────────────────────────────
+
+const METRICS: [&str; 6] = ["cpu_pct", "mem_pct", "disk_pct", "gpu_pct", "temp_max", "load1"];
+const OPS: [&str; 4] = [">", "<", ">=", "<="];
+
+fn rule_json(r: &node_rules::NodeRule) -> Value {
+    json!({
+        "id": r.id,
+        "nodeId": r.node_id.map(|n| n.as_uuid()),
+        "name": r.name,
+        "enabled": r.enabled,
+        "metric": r.metric,
+        "op": r.op,
+        "threshold": r.threshold,
+        "forSeconds": r.for_seconds,
+        "action": r.action,
+        "lastFired": r.last_fired_at.map(|t| t.unix_timestamp()),
+    })
+}
+
+/// `GET /api/fleet/rules` — the workspace's automation rules.
+pub async fn rules_list(
+    State(state): State<AppState>,
+    Authed(user): Authed,
+) -> Result<Json<Value>, ApiError> {
+    let rules = node_rules::list(&state.pool, user.workspace_id).await?;
+    Ok(Json(json!({ "rules": rules.iter().map(rule_json).collect::<Vec<_>>() })))
+}
+
+#[derive(Deserialize)]
+pub struct RuleReq {
+    pub name: String,
+    #[serde(rename = "nodeId")]
+    pub node_id: Option<String>,
+    pub metric: String,
+    pub op: String,
+    pub threshold: f64,
+    #[serde(rename = "forSeconds")]
+    pub for_seconds: Option<i32>,
+    pub action: Value,
+}
+
+/// `POST /api/fleet/rules` — create an automation rule.
+pub async fn rules_create(
+    State(state): State<AppState>,
+    Authed(user): Authed,
+    Json(req): Json<RuleReq>,
+) -> Result<Json<Value>, ApiError> {
+    if !METRICS.contains(&req.metric.as_str()) || !OPS.contains(&req.op.as_str()) {
+        return Ok(Json(json!({ "ok": false, "error": "invalid metric or operator" })));
+    }
+    let node_id = match req.node_id.as_deref() {
+        Some(s) if !s.is_empty() && s != "all" => {
+            let nid = NodeId::from(s.parse::<Uuid>().map_err(|_| ApiError::NotFound)?);
+            nodes::get(&state.pool, nid, user.workspace_id)
+                .await?
+                .ok_or(ApiError::NotFound)?;
+            Some(nid)
+        }
+        _ => None,
+    };
+    let id = node_rules::create(
+        &state.pool,
+        user.workspace_id,
+        node_id,
+        req.name.trim(),
+        &req.metric,
+        &req.op,
+        req.threshold,
+        req.for_seconds.unwrap_or(60).max(0),
+        &req.action,
+    )
+    .await?;
+    Ok(Json(json!({ "ok": true, "id": id })))
+}
+
+#[derive(Deserialize)]
+pub struct RulePatch {
+    pub enabled: bool,
+}
+
+/// `PATCH /api/fleet/rules/{id}` — enable/disable a rule.
+pub async fn rules_patch(
+    State(state): State<AppState>,
+    Authed(user): Authed,
+    Path(id): Path<Uuid>,
+    Json(req): Json<RulePatch>,
+) -> Result<Json<Value>, ApiError> {
+    node_rules::set_enabled(&state.pool, id, user.workspace_id, req.enabled).await?;
+    Ok(Json(json!({ "ok": true })))
+}
+
+/// `DELETE /api/fleet/rules/{id}` — delete a rule.
+pub async fn rules_delete(
+    State(state): State<AppState>,
+    Authed(user): Authed,
+    Path(id): Path<Uuid>,
+) -> Result<Json<Value>, ApiError> {
+    node_rules::delete(&state.pool, id, user.workspace_id).await?;
+    Ok(Json(json!({ "ok": true })))
+}

crates/cm-db/src/repo/mod.rs

@@ -10,6 +10,7 @@ pub mod fleet_beszel;
 pub mod fleet_tailscale;
 pub mod messages;
 pub mod node_metrics;
+pub mod node_rules;
 pub mod nodes;
 pub mod orgs;
 pub mod outbox;

crates/cm-db/src/repo/node_metrics.rs

@@ -2,9 +2,10 @@
 //! per node: scalar columns for the fleet cards + rules engine, plus a JSONB blob
 //! for the per-node monitor page.
 
-use cm_domain::NodeId;
+use cm_domain::{NodeId, WorkspaceId};
 use serde_json::Value;
 use sqlx::{PgPool, Row};
+use uuid::Uuid;
 
 use crate::DbError;
 
@@ -57,6 +58,72 @@ pub async fn upsert(pool: &PgPool, node_id: NodeId, m: &NodeMetrics) -> Result<(
     Ok(())
 }
 
+/// A node's current evaluatable scalars (Beszel-tapped, falling back to the
+/// heartbeat health), for the rules engine + metrics-aware placement.
+#[derive(Debug, Clone)]
+pub struct EvalRow {
+    pub node_id: NodeId,
+    pub workspace_id: WorkspaceId,
+    pub status: String,
+    pub cpu_pct: Option<f64>,
+    pub mem_pct: Option<f64>,
+    pub disk_pct: Option<f64>,
+    pub gpu_pct: Option<f64>,
+    pub temp_max: Option<f64>,
+    pub load1: Option<f64>,
+}
+
+impl EvalRow {
+    /// Look up a metric by rule name.
+    pub fn metric(&self, name: &str) -> Option<f64> {
+        match name {
+            "cpu_pct" => self.cpu_pct,
+            "mem_pct" => self.mem_pct,
+            "disk_pct" => self.disk_pct,
+            "gpu_pct" => self.gpu_pct,
+            "temp_max" => self.temp_max,
+            "load1" => self.load1,
+            _ => None,
+        }
+    }
+    /// Free headroom heuristic (higher = more capacity) for placement ranking.
+    pub fn headroom(&self) -> f64 {
+        let used = self.cpu_pct.unwrap_or(0.0).max(self.mem_pct.unwrap_or(0.0));
+        100.0 - used
+    }
+}
+
+/// Every node's current metric scalars (merged Beszel + heartbeat health).
+pub async fn eval_all(pool: &PgPool) -> Result<Vec<EvalRow>, DbError> {
+    let rows = sqlx::query(
+        "SELECT n.id, n.workspace_id, n.status,
+            COALESCE(m.cpu_pct, h.cpu_pct) AS cpu_pct,
+            COALESCE(m.mem_pct, CASE WHEN h.mem_total > 0 THEN h.mem_used::float8 / h.mem_total * 100 END) AS mem_pct,
+            COALESCE(m.disk_pct, CASE WHEN h.disk_total > 0 THEN (h.disk_total - h.disk_free)::float8 / h.disk_total * 100 END) AS disk_pct,
+            m.gpu_pct, m.temp_max,
+            COALESCE(m.load1, h.load1) AS load1
+         FROM nodes n
+         LEFT JOIN node_health h ON h.node_id = n.id
+         LEFT JOIN node_metrics m ON m.node_id = n.id",
+    )
+    .fetch_all(pool)
+    .await?;
+    Ok(rows
+        .into_iter()
+        .map(|r| EvalRow {
+            node_id: NodeId::from(r.get::<Uuid, _>("id")),
+            workspace_id: WorkspaceId::from(r.get::<Uuid, _>("workspace_id")),
+            status: r.get("status"),
+            cpu_pct: r.get("cpu_pct"),
+            mem_pct: r.get("mem_pct"),
+            disk_pct: r.get("disk_pct"),
+            gpu_pct: r.get("gpu_pct"),
+            temp_max: r.get("temp_max"),
+            load1: r.get("load1"),
+        })
+        .collect())
+}
+
 /// The latest metrics blob for a node (the full snapshot for the monitor page).
 pub async fn latest(pool: &PgPool, node_id: NodeId) -> Result<Option<Value>, DbError> {
     let row = sqlx::query(

crates/cm-db/src/repo/node_rules.rs

@@ -0,0 +1,127 @@
+//! Fleet automation rules: metric thresholds → actions (drain/undrain/alert),
+//! evaluated by the background rules engine.
+
+use cm_domain::{NodeId, WorkspaceId};
+use serde_json::Value;
+use sqlx::{PgPool, Row};
+use time::OffsetDateTime;
+use uuid::Uuid;
+
+use crate::DbError;
+
+#[derive(Debug, Clone)]
+pub struct NodeRule {
+    pub id: Uuid,
+    pub workspace_id: WorkspaceId,
+    pub node_id: Option<NodeId>,
+    pub name: String,
+    pub enabled: bool,
+    pub metric: String,
+    pub op: String,
+    pub threshold: f64,
+    pub for_seconds: i32,
+    pub action: Value,
+    pub last_fired_at: Option<OffsetDateTime>,
+}
+
+fn map_rule(r: sqlx::postgres::PgRow) -> NodeRule {
+    NodeRule {
+        id: r.get("id"),
+        workspace_id: WorkspaceId::from(r.get::<Uuid, _>("workspace_id")),
+        node_id: r.get::<Option<Uuid>, _>("node_id").map(NodeId::from),
+        name: r.get("name"),
+        enabled: r.get("enabled"),
+        metric: r.get("metric"),
+        op: r.get("op"),
+        threshold: r.get("threshold"),
+        for_seconds: r.get("for_seconds"),
+        action: r.get("action"),
+        last_fired_at: r.get("last_fired_at"),
+    }
+}
+
+const COLS: &str =
+    "id, workspace_id, node_id, name, enabled, metric, op, threshold, for_seconds, action, last_fired_at";
+
+#[allow(clippy::too_many_arguments)]
+pub async fn create(
+    pool: &PgPool,
+    workspace_id: WorkspaceId,
+    node_id: Option<NodeId>,
+    name: &str,
+    metric: &str,
+    op: &str,
+    threshold: f64,
+    for_seconds: i32,
+    action: &Value,
+) -> Result<Uuid, DbError> {
+    let id = Uuid::now_v7();
+    sqlx::query(
+        "INSERT INTO node_rules (id, workspace_id, node_id, name, metric, op, threshold, for_seconds, action)
+         VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9)",
+    )
+    .bind(id)
+    .bind(workspace_id.as_uuid())
+    .bind(node_id.map(|n| n.as_uuid()))
+    .bind(name)
+    .bind(metric)
+    .bind(op)
+    .bind(threshold)
+    .bind(for_seconds)
+    .bind(action)
+    .execute(pool)
+    .await?;
+    Ok(id)
+}
+
+/// List a workspace's rules (newest last).
+pub async fn list(pool: &PgPool, workspace_id: WorkspaceId) -> Result<Vec<NodeRule>, DbError> {
+    let rows = sqlx::query(&format!(
+        "SELECT {COLS} FROM node_rules WHERE workspace_id = $1 ORDER BY created_at"
+    ))
+    .bind(workspace_id.as_uuid())
+    .fetch_all(pool)
+    .await?;
+    Ok(rows.into_iter().map(map_rule).collect())
+}
+
+/// All enabled rules across workspaces (for the evaluator).
+pub async fn list_enabled(pool: &PgPool) -> Result<Vec<NodeRule>, DbError> {
+    let rows = sqlx::query(&format!("SELECT {COLS} FROM node_rules WHERE enabled"))
+        .fetch_all(pool)
+        .await?;
+    Ok(rows.into_iter().map(map_rule).collect())
+}
+
+pub async fn set_enabled(
+    pool: &PgPool,
+    id: Uuid,
+    workspace_id: WorkspaceId,
+    enabled: bool,
+) -> Result<(), DbError> {
+    sqlx::query("UPDATE node_rules SET enabled = $3 WHERE id = $1 AND workspace_id = $2")
+        .bind(id)
+        .bind(workspace_id.as_uuid())
+        .bind(enabled)
+        .execute(pool)
+        .await?;
+    Ok(())
+}
+
+pub async fn delete(pool: &PgPool, id: Uuid, workspace_id: WorkspaceId) -> Result<(), DbError> {
+    sqlx::query("DELETE FROM node_rules WHERE id = $1 AND workspace_id = $2")
+        .bind(id)
+        .bind(workspace_id.as_uuid())
+        .execute(pool)
+        .await?;
+    Ok(())
+}
+
+/// Stamp a rule as just-fired (debounce + UI).
+pub async fn mark_fired(pool: &PgPool, id: Uuid) -> Result<(), DbError> {
+    sqlx::query("UPDATE node_rules SET last_fired_at = now() WHERE id = $1")
+        .bind(id)
+        .execute(pool)
+        .await?;
+    Ok(())
+}

crates/cm-db/src/repo/nodes.rs

@@ -121,7 +121,10 @@ pub async fn heartbeat(
     h: &NodeHealth,
 ) -> Result<(), DbError> {
     sqlx::query(
-        "UPDATE nodes SET status = 'online', last_seen = now(),
+        // Preserve a rule-/operator-set `draining` state across heartbeats; a node
+        // is only un-drained by an explicit set_status.
+        "UPDATE nodes SET status = CASE WHEN status = 'draining' THEN 'draining' ELSE 'online' END,
+            last_seen = now(),
             agent_version = COALESCE($2, agent_version),
             tailscale_ip = COALESCE($3, tailscale_ip),
             hostname = COALESCE($4, hostname),
@@ -198,6 +201,16 @@ pub async fn delete(pool: &PgPool, id: NodeId, workspace_id: WorkspaceId) -> Res
     Ok(())
 }
 
+/// A node's current status (for metrics-aware placement: skip 'draining').
+pub async fn status_of(pool: &PgPool, id: NodeId) -> Result<Option<String>, DbError> {
+    Ok(
+        sqlx::query_scalar::<_, String>("SELECT status FROM nodes WHERE id = $1")
+            .bind(id.as_uuid())
+            .fetch_optional(pool)
+            .await?,
+    )
+}
+
 fn map_node(r: sqlx::postgres::PgRow) -> NodeRow {
     let health = r
         .get::<Option<uuid::Uuid>, _>("health_node")

crates/cm-runtime/src/sandboxes.rs

@@ -86,6 +86,16 @@ impl SandboxManager {
     async fn placement_node(&self, agent_id: AgentId) -> String {
         match cm_db::repo::workspace_placement::for_agent(&self.db, agent_id).await {
             Ok(Some(node)) if node != self.node_id => {
+                // Metrics-aware: a node the rules engine cordoned (`draining`)
+                // stops receiving new workloads — fall back to local.
+                if let Ok(nid) = node.parse::<uuid::Uuid>() {
+                    if matches!(
+                        cm_db::repo::nodes::status_of(&self.db, cm_domain::NodeId::from(nid)).await,
+                        Ok(Some(ref s)) if s == "draining"
+                    ) {
+                        return self.node_id.clone();
+                    }
+                }
                 if self.node_provider.as_ref().and_then(|p| p.driver(&node)).is_some() {
                     return node;
                 }

frontend/src/components/dashboard/fleet/FleetConsole.tsx

@@ -268,6 +268,102 @@ function BeszelSection() {
   );
 }
 
+// ── Automation rules ─────────────────────────────────────────────────────────
+interface Rule {
+  id: string;
+  nodeId: string | null;
+  name: string;
+  enabled: boolean;
+  metric: string;
+  op: string;
+  threshold: number;
+  forSeconds: number;
+  action: { type: string };
+  lastFired: number | null;
+}
+const RULE_METRICS: string[][] = [["cpu_pct", "CPU %"], ["mem_pct", "Memory %"], ["disk_pct", "Disk %"], ["gpu_pct", "GPU %"], ["temp_max", "Temp °C"], ["load1", "Load"]];
+const RULE_ACTIONS: string[][] = [["drain", "drain node"], ["undrain", "un-drain node"], ["alert", "alert"]];
+function timeAgo(unixSec: number): string {
+  const s = Math.max(0, Math.floor(Date.now() / 1000 - unixSec));
+  if (s < 60) return `${s}s ago`;
+  if (s < 3600) return `${Math.floor(s / 60)}m ago`;
+  return `${Math.floor(s / 3600)}h ago`;
+}
+
+/** Metric-threshold automation rules: when a node crosses a threshold for a
+ *  window, drain/undrain/alert. Paired with metrics-aware placement. */
+function FleetRules({ nodes }: { nodes: FleetNode[] }) {
+  const { data, refresh } = useFetchJson<{ rules: Rule[] }>("/api/fleet/rules");
+  const rules = data?.rules ?? [];
+  const [show, setShow] = useState(false);
+  const [node, setNode] = useState("all");
+  const [metric, setMetric] = useState("cpu_pct");
+  const [op, setOp] = useState(">");
+  const [threshold, setThreshold] = useState("90");
+  const [forSeconds, setForSeconds] = useState("120");
+  const [action, setAction] = useState("drain");
+
+  const create = () => {
+    fetch("/api/fleet/rules", { method: "POST", headers: { "Content-Type": "application/json" }, body: JSON.stringify({ name: `${metric} ${op} ${threshold}`, nodeId: node, metric, op, threshold: parseFloat(threshold) || 0, forSeconds: parseInt(forSeconds, 10) || 60, action: { type: action } }) })
+      .then((r) => r.json())
+      .then(() => { setShow(false); refresh(); });
+  };
+  const toggle = (id: string, enabled: boolean) => fetch(`/api/fleet/rules/${id}`, { method: "PATCH", headers: { "Content-Type": "application/json" }, body: JSON.stringify({ enabled }) }).then(refresh);
+  const del = (id: string) => fetch(`/api/fleet/rules/${id}`, { method: "DELETE" }).then(refresh);
+
+  const sel = (val: string, set: (v: string) => void, opts: string[][]) => (
+    <select value={val} onChange={(e) => set(e.target.value)} style={{ height: 30, borderRadius: 7, border: "1px solid rgba(255,255,255,.1)", background: "#0d0d10", color: "#cfcfd5", padding: "0 8px", fontFamily: mono, fontSize: 11 }}>
+      {opts.map(([v, l]) => <option key={v} value={v}>{l}</option>)}
+    </select>
+  );
+  const metricLabel = (m: string) => RULE_METRICS.find(([v]) => v === m)?.[1] ?? m;
+  const nodeName = (id: string | null) => (id == null ? "all nodes" : nodes.find((n) => n.id === id)?.hostname ?? nodes.find((n) => n.id === id)?.name ?? id.slice(0, 8));
+
+  return (
+    <div style={{ marginTop: 26 }}>
+      <div style={{ display: "flex", alignItems: "center", gap: 10, marginBottom: 12 }}>
+        <span style={{ fontFamily: mono, fontSize: 10, letterSpacing: ".12em", color: "#5a5a62" }}>AUTOMATION RULES</span>
+        <span style={{ flex: 1, height: 1, background: "rgba(255,255,255,.06)" }} />
+        <button type="button" onClick={() => setShow((v) => !v)} style={{ fontFamily: mono, fontSize: 10, color: "#5ec8d8", background: "transparent", border: 0, cursor: "pointer" }}>{show ? "cancel" : "+ rule"}</button>
+      </div>
+
+      {show ? (
+        <div style={{ borderRadius: 12, border: "1px solid rgba(255,255,255,.08)", background: "#0c0c0f", padding: 14, marginBottom: 12, display: "flex", flexWrap: "wrap", gap: 8, alignItems: "center" }}>
+          <span style={{ fontFamily: mono, fontSize: 11, color: "#8a8a92" }}>when</span>
+          {sel(node, setNode, [["all", "all nodes"], ...nodes.map((n) => [n.id, n.hostname ?? n.name] as string[])])}
+          {sel(metric, setMetric, RULE_METRICS)}
+          {sel(op, setOp, [[">", ">"], ["<", "<"], [">=", "≥"], ["<=", "≤"]])}
+          <input value={threshold} onChange={(e) => setThreshold(e.target.value)} style={{ width: 60, height: 30, borderRadius: 7, border: "1px solid rgba(255,255,255,.1)", background: "#0d0d10", color: "#cfcfd5", padding: "0 8px", fontFamily: mono, fontSize: 11 }} />
+          <span style={{ fontFamily: mono, fontSize: 11, color: "#8a8a92" }}>for</span>
+          <input value={forSeconds} onChange={(e) => setForSeconds(e.target.value)} style={{ width: 56, height: 30, borderRadius: 7, border: "1px solid rgba(255,255,255,.1)", background: "#0d0d10", color: "#cfcfd5", padding: "0 8px", fontFamily: mono, fontSize: 11 }} />
+          <span style={{ fontFamily: mono, fontSize: 11, color: "#8a8a92" }}>s →</span>
+          {sel(action, setAction, RULE_ACTIONS)}
+          <button type="button" onClick={create} style={{ height: 30, padding: "0 14px", borderRadius: 7, border: 0, background: "linear-gradient(135deg,#7fe0a0,#5fd08a)", color: "#06281a", fontSize: 12, fontWeight: 700, cursor: "pointer" }}>add</button>
+        </div>
+      ) : null}
+
+      {rules.length === 0 ? (
+        <div style={{ fontFamily: mono, fontSize: 11, color: "#6a6a72" }}>No rules yet — e.g. “cpu_pct &gt; 90 for 120s → drain”. Drained nodes stop receiving new agents (metrics-aware placement).</div>
+      ) : (
+        <div style={{ display: "flex", flexDirection: "column", gap: 6 }}>
+          {rules.map((r) => (
+            <div key={r.id} style={{ display: "flex", alignItems: "center", gap: 10, padding: "9px 13px", borderRadius: 9, border: "1px solid rgba(255,255,255,.07)", background: "#0d0d10" }}>
+              <button type="button" onClick={() => toggle(r.id, !r.enabled)} title={r.enabled ? "enabled" : "disabled"} aria-label="toggle rule" style={{ width: 30, height: 18, borderRadius: 9, border: 0, background: r.enabled ? "#5fd08a" : "rgba(255,255,255,.12)", position: "relative", cursor: "pointer", flex: "none" }}>
+                <span style={{ position: "absolute", top: 2, left: r.enabled ? 14 : 2, width: 14, height: 14, borderRadius: "50%", background: "#0a0a0c", transition: "left .15s" }} />
+              </button>
+              <span style={{ fontFamily: mono, fontSize: 12, color: "#cfcfd5", flex: 1, opacity: r.enabled ? 1 : 0.5 }}>
+                <span style={{ color: "#9a9aa2" }}>{nodeName(r.nodeId)}</span> · {metricLabel(r.metric)} {r.op} {r.threshold} for {r.forSeconds}s → <span style={{ color: r.action.type === "drain" ? "#ff8a7a" : "#5fd08a" }}>{r.action.type}</span>
+              </span>
+              {r.lastFired ? <span style={{ fontFamily: mono, fontSize: 9, color: "#e8b465" }}>fired {timeAgo(r.lastFired)}</span> : null}
+              <button type="button" onClick={() => del(r.id)} aria-label="delete rule" style={{ fontFamily: mono, fontSize: 11, color: "#ff8a7a", background: "transparent", border: 0, cursor: "pointer" }}>✕</button>
+            </div>
+          ))}
+        </div>
+      )}
+    </div>
+  );
+}
+
 export function FleetConsole({ view, onConnectHost, onMonitor }: { view: string; onConnectHost: () => void; onMonitor: (id: string, name: string) => void }) {
   const { nodes, refresh } = useNodes();
   const { data: ts } = useFetchJson<{ connected: boolean; tailnet: string | null; devices: TsDevice[] }>("/api/fleet/tailscale/devices");
@@ -351,6 +447,8 @@ export function FleetConsole({ view, onConnectHost, onMonitor }: { view: string;
           <span style={{ fontFamily: mono, fontSize: 9, color: "#6a6a72" }}>Mac · Linux · edge device</span>
         </button>
       </div>
+
+      <FleetRules nodes={nodes} />
     </div>
   );
 }

migrations/0024_node_rules.sql

@@ -0,0 +1,19 @@
+-- Fleet automation: metric-driven rules. When a node's metric crosses a threshold
+-- for a sustained window, fire an action (drain a hot node, bring one back, alert).
+-- A null node_id applies the rule to every node in the workspace. Paired with the
+-- metrics-aware placement (draining nodes stop receiving new agent workloads).
+CREATE TABLE node_rules (
+    id            UUID PRIMARY KEY,
+    workspace_id  UUID NOT NULL REFERENCES workspaces (id) ON DELETE CASCADE,
+    node_id       UUID REFERENCES nodes (id) ON DELETE CASCADE,
+    name          TEXT NOT NULL,
+    enabled       BOOLEAN NOT NULL DEFAULT true,
+    metric        TEXT NOT NULL,             -- cpu_pct | mem_pct | disk_pct | gpu_pct | temp_max | load1
+    op            TEXT NOT NULL,             -- '>' | '<' | '>=' | '<='
+    threshold     DOUBLE PRECISION NOT NULL,
+    for_seconds   INTEGER NOT NULL DEFAULT 60,
+    action        JSONB NOT NULL,            -- {"type":"drain"|"undrain"|"alert"}
+    last_fired_at TIMESTAMPTZ,
+    created_at    TIMESTAMPTZ NOT NULL DEFAULT now()
+);
+CREATE INDEX node_rules_ws ON node_rules (workspace_id);