Turn the Beszel-tapped metrics into a self-managing loop. - migration node_rules (workspace/node-scoped: metric op threshold, for_seconds, action JSONB, last_fired). - cm-db: repo/node_rules.rs (CRUD + list_enabled); node_metrics::eval_all merges Beszel + heartbeat scalars per node + a headroom() heuristic; nodes::status_of; heartbeat now PRESERVES a `draining` status across heartbeats (so a cordon sticks). - cm-api: node_rules.rs evaluator (spawn_evaluator, 20s) — when a metric condition holds for the rule's window it fires drain / undrain / alert (in-memory sustained + cooldown tracking, modeled on the node sweeper); routes/beszel.rs rules CRUD (GET/POST/PATCH/DELETE /api/fleet/rules); spawned in clawmates-server. - cm-runtime: placement_node() is metrics-aware — a `draining` node stops receiving new agent sandboxes (falls back to local), so the drain rule is actionable. - frontend: FleetRules section in the Local view — build rules (node · metric · op · threshold · duration → action), toggle/delete, with fired-history. The loop: hot/overloaded node → rule drains it → placement avoids it → recovers → undrain rule brings it back. Deployed; node_rules migration applied. Co-Authored-By: Claude Opus 4.8 (1M context) <[email protected]>
245 lines
8.1 KiB
Rust
245 lines
8.1 KiB
Rust
//! Fleet node registry: user-connected local-hardware hosts that run the
|
|
//! clawmates-node daemon. The daemon authenticates its control channel with the
|
|
//! node `token`, then upserts its host-health snapshot on every heartbeat.
|
|
|
|
use cm_domain::{NodeId, WorkspaceId};
|
|
use sqlx::{PgPool, Row};
|
|
use time::OffsetDateTime;
|
|
|
|
use crate::DbError;
|
|
|
|
/// Latest host-health snapshot for a node.
|
|
#[derive(Debug, Clone)]
|
|
pub struct NodeHealth {
|
|
pub cpu_pct: f64,
|
|
pub mem_total: i64,
|
|
pub mem_used: i64,
|
|
pub mem_pressure: f64,
|
|
pub swap_used: i64,
|
|
pub disk_total: i64,
|
|
pub disk_free: i64,
|
|
pub load1: f64,
|
|
pub load5: f64,
|
|
pub load15: f64,
|
|
pub container_count: i32,
|
|
}
|
|
|
|
/// A registered node plus its latest health (if it has reported one).
|
|
#[derive(Debug, Clone)]
|
|
pub struct NodeRow {
|
|
pub id: NodeId,
|
|
pub name: String,
|
|
pub hostname: Option<String>,
|
|
pub local_ip: Option<String>,
|
|
pub status: String,
|
|
pub agent_version: Option<String>,
|
|
pub tailscale_ip: Option<String>,
|
|
pub last_seen: Option<OffsetDateTime>,
|
|
pub created_at: OffsetDateTime,
|
|
pub health: Option<NodeHealth>,
|
|
/// Latest Beszel metrics scalars (the rich bits beyond basic health).
|
|
pub gpu_pct: Option<f64>,
|
|
pub temp_max: Option<f64>,
|
|
}
|
|
|
|
/// Register a new (pending) node with its control-channel token.
|
|
pub async fn create(
|
|
pool: &PgPool,
|
|
workspace_id: WorkspaceId,
|
|
name: &str,
|
|
token: &str,
|
|
) -> Result<NodeId, DbError> {
|
|
let id = NodeId::new();
|
|
sqlx::query("INSERT INTO nodes (id, workspace_id, name, status, token) VALUES ($1, $2, $3, 'pending', $4)")
|
|
.bind(id.as_uuid())
|
|
.bind(workspace_id.as_uuid())
|
|
.bind(name)
|
|
.bind(token)
|
|
.execute(pool)
|
|
.await?;
|
|
Ok(id)
|
|
}
|
|
|
|
/// Resolve a control-channel token to its node + workspace (daemon auth).
|
|
pub async fn auth(pool: &PgPool, token: &str) -> Result<Option<(NodeId, WorkspaceId)>, DbError> {
|
|
let row = sqlx::query("SELECT id, workspace_id FROM nodes WHERE token = $1")
|
|
.bind(token)
|
|
.fetch_optional(pool)
|
|
.await?;
|
|
Ok(row.map(|r| {
|
|
(
|
|
NodeId::from(r.get::<uuid::Uuid, _>("id")),
|
|
WorkspaceId::from(r.get::<uuid::Uuid, _>("workspace_id")),
|
|
)
|
|
}))
|
|
}
|
|
|
|
const SELECT_WITH_HEALTH: &str = "SELECT n.id, n.name, n.hostname, n.local_ip, n.status, n.agent_version, n.tailscale_ip, n.last_seen, n.created_at,
|
|
h.node_id AS health_node, h.cpu_pct, h.mem_total, h.mem_used, h.mem_pressure, h.swap_used,
|
|
h.disk_total, h.disk_free, h.load1, h.load5, h.load15, h.container_count,
|
|
m.gpu_pct AS m_gpu_pct, m.temp_max AS m_temp_max
|
|
FROM nodes n LEFT JOIN node_health h ON h.node_id = n.id
|
|
LEFT JOIN node_metrics m ON m.node_id = n.id";
|
|
|
|
/// List a workspace's nodes (oldest first) with their latest health.
|
|
pub async fn list(pool: &PgPool, workspace_id: WorkspaceId) -> Result<Vec<NodeRow>, DbError> {
|
|
let rows = sqlx::query(&format!(
|
|
"{SELECT_WITH_HEALTH} WHERE n.workspace_id = $1 ORDER BY n.created_at"
|
|
))
|
|
.bind(workspace_id.as_uuid())
|
|
.fetch_all(pool)
|
|
.await?;
|
|
Ok(rows.into_iter().map(map_node).collect())
|
|
}
|
|
|
|
/// Fetch one node (workspace-scoped) with its latest health.
|
|
pub async fn get(
|
|
pool: &PgPool,
|
|
id: NodeId,
|
|
workspace_id: WorkspaceId,
|
|
) -> Result<Option<NodeRow>, DbError> {
|
|
let row = sqlx::query(&format!(
|
|
"{SELECT_WITH_HEALTH} WHERE n.id = $1 AND n.workspace_id = $2"
|
|
))
|
|
.bind(id.as_uuid())
|
|
.bind(workspace_id.as_uuid())
|
|
.fetch_optional(pool)
|
|
.await?;
|
|
Ok(row.map(map_node))
|
|
}
|
|
|
|
/// Record a heartbeat: mark the node online + refresh its version/tailscale IP,
|
|
/// and upsert its latest host-health snapshot.
|
|
#[allow(clippy::too_many_arguments)]
|
|
pub async fn heartbeat(
|
|
pool: &PgPool,
|
|
id: NodeId,
|
|
agent_version: Option<&str>,
|
|
tailscale_ip: Option<&str>,
|
|
hostname: Option<&str>,
|
|
local_ip: Option<&str>,
|
|
h: &NodeHealth,
|
|
) -> Result<(), DbError> {
|
|
sqlx::query(
|
|
// Preserve a rule-/operator-set `draining` state across heartbeats; a node
|
|
// is only un-drained by an explicit set_status.
|
|
"UPDATE nodes SET status = CASE WHEN status = 'draining' THEN 'draining' ELSE 'online' END,
|
|
last_seen = now(),
|
|
agent_version = COALESCE($2, agent_version),
|
|
tailscale_ip = COALESCE($3, tailscale_ip),
|
|
hostname = COALESCE($4, hostname),
|
|
local_ip = COALESCE($5, local_ip)
|
|
WHERE id = $1",
|
|
)
|
|
.bind(id.as_uuid())
|
|
.bind(agent_version)
|
|
.bind(tailscale_ip)
|
|
.bind(hostname)
|
|
.bind(local_ip)
|
|
.execute(pool)
|
|
.await?;
|
|
sqlx::query(
|
|
"INSERT INTO node_health
|
|
(node_id, captured_at, cpu_pct, mem_total, mem_used, mem_pressure, swap_used,
|
|
disk_total, disk_free, load1, load5, load15, container_count)
|
|
VALUES ($1, now(), $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12)
|
|
ON CONFLICT (node_id) DO UPDATE SET
|
|
captured_at = now(), cpu_pct = excluded.cpu_pct, mem_total = excluded.mem_total,
|
|
mem_used = excluded.mem_used, mem_pressure = excluded.mem_pressure,
|
|
swap_used = excluded.swap_used, disk_total = excluded.disk_total,
|
|
disk_free = excluded.disk_free, load1 = excluded.load1, load5 = excluded.load5,
|
|
load15 = excluded.load15, container_count = excluded.container_count",
|
|
)
|
|
.bind(id.as_uuid())
|
|
.bind(h.cpu_pct)
|
|
.bind(h.mem_total)
|
|
.bind(h.mem_used)
|
|
.bind(h.mem_pressure)
|
|
.bind(h.swap_used)
|
|
.bind(h.disk_total)
|
|
.bind(h.disk_free)
|
|
.bind(h.load1)
|
|
.bind(h.load5)
|
|
.bind(h.load15)
|
|
.bind(h.container_count)
|
|
.execute(pool)
|
|
.await?;
|
|
Ok(())
|
|
}
|
|
|
|
/// Set a node's status (e.g. 'offline' when its channel drops, 'draining' on
|
|
/// deregister).
|
|
pub async fn set_status(pool: &PgPool, id: NodeId, status: &str) -> Result<(), DbError> {
|
|
sqlx::query("UPDATE nodes SET status = $2 WHERE id = $1")
|
|
.bind(id.as_uuid())
|
|
.bind(status)
|
|
.execute(pool)
|
|
.await?;
|
|
Ok(())
|
|
}
|
|
|
|
/// Mark online nodes whose last heartbeat is older than `secs` as offline.
|
|
pub async fn mark_stale_offline(pool: &PgPool, secs: i64) -> Result<(), DbError> {
|
|
sqlx::query(
|
|
"UPDATE nodes SET status = 'offline'
|
|
WHERE status = 'online'
|
|
AND (last_seen IS NULL OR last_seen < now() - ($1 * interval '1 second'))",
|
|
)
|
|
.bind(secs)
|
|
.execute(pool)
|
|
.await?;
|
|
Ok(())
|
|
}
|
|
|
|
/// Remove a node from the registry (workspace-scoped).
|
|
pub async fn delete(pool: &PgPool, id: NodeId, workspace_id: WorkspaceId) -> Result<(), DbError> {
|
|
sqlx::query("DELETE FROM nodes WHERE id = $1 AND workspace_id = $2")
|
|
.bind(id.as_uuid())
|
|
.bind(workspace_id.as_uuid())
|
|
.execute(pool)
|
|
.await?;
|
|
Ok(())
|
|
}
|
|
|
|
/// A node's current status (for metrics-aware placement: skip 'draining').
|
|
pub async fn status_of(pool: &PgPool, id: NodeId) -> Result<Option<String>, DbError> {
|
|
Ok(
|
|
sqlx::query_scalar::<_, String>("SELECT status FROM nodes WHERE id = $1")
|
|
.bind(id.as_uuid())
|
|
.fetch_optional(pool)
|
|
.await?,
|
|
)
|
|
}
|
|
|
|
fn map_node(r: sqlx::postgres::PgRow) -> NodeRow {
|
|
let health = r
|
|
.get::<Option<uuid::Uuid>, _>("health_node")
|
|
.map(|_| NodeHealth {
|
|
cpu_pct: r.get("cpu_pct"),
|
|
mem_total: r.get("mem_total"),
|
|
mem_used: r.get("mem_used"),
|
|
mem_pressure: r.get("mem_pressure"),
|
|
swap_used: r.get("swap_used"),
|
|
disk_total: r.get("disk_total"),
|
|
disk_free: r.get("disk_free"),
|
|
load1: r.get("load1"),
|
|
load5: r.get("load5"),
|
|
load15: r.get("load15"),
|
|
container_count: r.get("container_count"),
|
|
});
|
|
NodeRow {
|
|
id: NodeId::from(r.get::<uuid::Uuid, _>("id")),
|
|
name: r.get("name"),
|
|
hostname: r.get("hostname"),
|
|
local_ip: r.get("local_ip"),
|
|
status: r.get("status"),
|
|
agent_version: r.get("agent_version"),
|
|
tailscale_ip: r.get("tailscale_ip"),
|
|
last_seen: r.get("last_seen"),
|
|
created_at: r.get("created_at"),
|
|
health,
|
|
gpu_pct: r.get("m_gpu_pct"),
|
|
temp_max: r.get("m_temp_max"),
|
|
}
|
|
}
|