Fleet P0: node registry + daemon + health + connect-host wizard
ci / gates (push) Failing after 5s
ci / rust (push) Has been skipped
ci / sandbox-k8s (push) Has been skipped
ci / frontend (push) Has been skipped
ci / e2e (push) Has been skipped

Users can connect their own local-hardware nodes into a fleet. Each node runs a
new Rust daemon that dials home over an outbound WebSocket, reports host health,
and runs commands we send.

Backend:
- migrations/0018_fleet_nodes.sql: nodes + node_health tables + agent_containers
  (node_id, workspace_id) index. cm-domain NodeId.
- cm-db repo/nodes.rs: create/auth/list+health/get/heartbeat/set_status/delete
  (unchecked sqlx, no .sqlx regen).
- cm-api fleet.rs NodeHub: live daemon channels (node_id→sender) + the WS channel
  runner (heartbeat→DB upsert, exec request/response framing). routes/nodes.rs:
  POST /pair, GET /nodes, SSE /nodes/live, POST /{id}/exec-test, DELETE /{id},
  WS /nodes/agent (token-auth). Wired into AppState + router.

Daemon (new crate crates/bins/clawmates-node):
- sysinfo host metrics (cpu/mem/pressure/swap/disk/load/containers), outbound WSS
  dial + reconnect, heartbeat loop, exec command handling, tailscale-ip probe.
  install.sh convenience installer.

Frontend:
- Fleet sidebar item + FleetOverview + LocalHardware node-health cards (live via
  /api/nodes, 3s poll) + ConnectHostWizard (install → verify connection →
  exec-test). InfraStage dispatches fleet/local; default selection = fleet.

Deferred: P1 (BYO Tailscale + network metrics), P2 (RemoteDriver + placement so
agents actually run on connected nodes).

Co-Authored-By: Claude Opus 4.8 (1M context) <[email protected]>
This commit is contained in:
Omar Sobh
2026-06-24 08:09:14 -07:00
co-authored by Claude Opus 4.8
parent b853aab6fd
commit 2bdd0a23e8
18 changed files with 1343 additions and 9 deletions
+1
View File
@@ -7,6 +7,7 @@ pub mod connections;
pub mod credits;
pub mod files;
pub mod messages;
pub mod nodes;
pub mod orgs;
pub mod outbox;
pub mod routine_runs;
+213
View File
@@ -0,0 +1,213 @@
//! Fleet node registry: user-connected local-hardware hosts that run the
//! clawmates-node daemon. The daemon authenticates its control channel with the
//! node `token`, then upserts its host-health snapshot on every heartbeat.
use cm_domain::{NodeId, WorkspaceId};
use sqlx::{PgPool, Row};
use time::OffsetDateTime;
use crate::DbError;
/// Latest host-health snapshot for a node.
#[derive(Debug, Clone)]
pub struct NodeHealth {
pub cpu_pct: f64,
pub mem_total: i64,
pub mem_used: i64,
pub mem_pressure: f64,
pub swap_used: i64,
pub disk_total: i64,
pub disk_free: i64,
pub load1: f64,
pub load5: f64,
pub load15: f64,
pub container_count: i32,
}
/// A registered node plus its latest health (if it has reported one).
#[derive(Debug, Clone)]
pub struct NodeRow {
pub id: NodeId,
pub name: String,
pub status: String,
pub agent_version: Option<String>,
pub tailscale_ip: Option<String>,
pub last_seen: Option<OffsetDateTime>,
pub created_at: OffsetDateTime,
pub health: Option<NodeHealth>,
}
/// Register a new (pending) node with its control-channel token.
pub async fn create(
pool: &PgPool,
workspace_id: WorkspaceId,
name: &str,
token: &str,
) -> Result<NodeId, DbError> {
let id = NodeId::new();
sqlx::query("INSERT INTO nodes (id, workspace_id, name, status, token) VALUES ($1, $2, $3, 'pending', $4)")
.bind(id.as_uuid())
.bind(workspace_id.as_uuid())
.bind(name)
.bind(token)
.execute(pool)
.await?;
Ok(id)
}
/// Resolve a control-channel token to its node + workspace (daemon auth).
pub async fn auth(pool: &PgPool, token: &str) -> Result<Option<(NodeId, WorkspaceId)>, DbError> {
let row = sqlx::query("SELECT id, workspace_id FROM nodes WHERE token = $1")
.bind(token)
.fetch_optional(pool)
.await?;
Ok(row.map(|r| {
(
NodeId::from(r.get::<uuid::Uuid, _>("id")),
WorkspaceId::from(r.get::<uuid::Uuid, _>("workspace_id")),
)
}))
}
const SELECT_WITH_HEALTH: &str = "SELECT n.id, n.name, n.status, n.agent_version, n.tailscale_ip, n.last_seen, n.created_at,
h.node_id AS health_node, h.cpu_pct, h.mem_total, h.mem_used, h.mem_pressure, h.swap_used,
h.disk_total, h.disk_free, h.load1, h.load5, h.load15, h.container_count
FROM nodes n LEFT JOIN node_health h ON h.node_id = n.id";
/// List a workspace's nodes (oldest first) with their latest health.
pub async fn list(pool: &PgPool, workspace_id: WorkspaceId) -> Result<Vec<NodeRow>, DbError> {
let rows = sqlx::query(&format!(
"{SELECT_WITH_HEALTH} WHERE n.workspace_id = $1 ORDER BY n.created_at"
))
.bind(workspace_id.as_uuid())
.fetch_all(pool)
.await?;
Ok(rows.into_iter().map(map_node).collect())
}
/// Fetch one node (workspace-scoped) with its latest health.
pub async fn get(
pool: &PgPool,
id: NodeId,
workspace_id: WorkspaceId,
) -> Result<Option<NodeRow>, DbError> {
let row = sqlx::query(&format!(
"{SELECT_WITH_HEALTH} WHERE n.id = $1 AND n.workspace_id = $2"
))
.bind(id.as_uuid())
.bind(workspace_id.as_uuid())
.fetch_optional(pool)
.await?;
Ok(row.map(map_node))
}
/// Record a heartbeat: mark the node online + refresh its version/tailscale IP,
/// and upsert its latest host-health snapshot.
pub async fn heartbeat(
pool: &PgPool,
id: NodeId,
agent_version: Option<&str>,
tailscale_ip: Option<&str>,
h: &NodeHealth,
) -> Result<(), DbError> {
sqlx::query(
"UPDATE nodes SET status = 'online', last_seen = now(),
agent_version = COALESCE($2, agent_version),
tailscale_ip = COALESCE($3, tailscale_ip)
WHERE id = $1",
)
.bind(id.as_uuid())
.bind(agent_version)
.bind(tailscale_ip)
.execute(pool)
.await?;
sqlx::query(
"INSERT INTO node_health
(node_id, captured_at, cpu_pct, mem_total, mem_used, mem_pressure, swap_used,
disk_total, disk_free, load1, load5, load15, container_count)
VALUES ($1, now(), $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12)
ON CONFLICT (node_id) DO UPDATE SET
captured_at = now(), cpu_pct = excluded.cpu_pct, mem_total = excluded.mem_total,
mem_used = excluded.mem_used, mem_pressure = excluded.mem_pressure,
swap_used = excluded.swap_used, disk_total = excluded.disk_total,
disk_free = excluded.disk_free, load1 = excluded.load1, load5 = excluded.load5,
load15 = excluded.load15, container_count = excluded.container_count",
)
.bind(id.as_uuid())
.bind(h.cpu_pct)
.bind(h.mem_total)
.bind(h.mem_used)
.bind(h.mem_pressure)
.bind(h.swap_used)
.bind(h.disk_total)
.bind(h.disk_free)
.bind(h.load1)
.bind(h.load5)
.bind(h.load15)
.bind(h.container_count)
.execute(pool)
.await?;
Ok(())
}
/// Set a node's status (e.g. 'offline' when its channel drops, 'draining' on
/// deregister).
pub async fn set_status(pool: &PgPool, id: NodeId, status: &str) -> Result<(), DbError> {
sqlx::query("UPDATE nodes SET status = $2 WHERE id = $1")
.bind(id.as_uuid())
.bind(status)
.execute(pool)
.await?;
Ok(())
}
/// Mark online nodes whose last heartbeat is older than `secs` as offline.
pub async fn mark_stale_offline(pool: &PgPool, secs: i64) -> Result<(), DbError> {
sqlx::query(
"UPDATE nodes SET status = 'offline'
WHERE status = 'online'
AND (last_seen IS NULL OR last_seen < now() - ($1 * interval '1 second'))",
)
.bind(secs)
.execute(pool)
.await?;
Ok(())
}
/// Remove a node from the registry (workspace-scoped).
pub async fn delete(pool: &PgPool, id: NodeId, workspace_id: WorkspaceId) -> Result<(), DbError> {
sqlx::query("DELETE FROM nodes WHERE id = $1 AND workspace_id = $2")
.bind(id.as_uuid())
.bind(workspace_id.as_uuid())
.execute(pool)
.await?;
Ok(())
}
fn map_node(r: sqlx::postgres::PgRow) -> NodeRow {
let health = r
.get::<Option<uuid::Uuid>, _>("health_node")
.map(|_| NodeHealth {
cpu_pct: r.get("cpu_pct"),
mem_total: r.get("mem_total"),
mem_used: r.get("mem_used"),
mem_pressure: r.get("mem_pressure"),
swap_used: r.get("swap_used"),
disk_total: r.get("disk_total"),
disk_free: r.get("disk_free"),
load1: r.get("load1"),
load5: r.get("load5"),
load15: r.get("load15"),
container_count: r.get("container_count"),
});
NodeRow {
id: NodeId::from(r.get::<uuid::Uuid, _>("id")),
name: r.get("name"),
status: r.get("status"),
agent_version: r.get("agent_version"),
tailscale_ip: r.get("tailscale_ip"),
last_seen: r.get("last_seen"),
created_at: r.get("created_at"),
health,
}
}