Fleet P0: node registry + daemon + health + connect-host wizard
Users can connect their own local-hardware nodes into a fleet. Each node runs a
new Rust daemon that dials home over an outbound WebSocket, reports host health,
and runs commands we send.
Backend:
- migrations/0018_fleet_nodes.sql: nodes + node_health tables + agent_containers
(node_id, workspace_id) index. cm-domain NodeId.
- cm-db repo/nodes.rs: create/auth/list+health/get/heartbeat/set_status/delete
(unchecked sqlx, no .sqlx regen).
- cm-api fleet.rs NodeHub: live daemon channels (node_id→sender) + the WS channel
runner (heartbeat→DB upsert, exec request/response framing). routes/nodes.rs:
POST /pair, GET /nodes, SSE /nodes/live, POST /{id}/exec-test, DELETE /{id},
WS /nodes/agent (token-auth). Wired into AppState + router.
Daemon (new crate crates/bins/clawmates-node):
- sysinfo host metrics (cpu/mem/pressure/swap/disk/load/containers), outbound WSS
dial + reconnect, heartbeat loop, exec command handling, tailscale-ip probe.
install.sh convenience installer.
Frontend:
- Fleet sidebar item + FleetOverview + LocalHardware node-health cards (live via
/api/nodes, 3s poll) + ConnectHostWizard (install → verify connection →
exec-test). InfraStage dispatches fleet/local; default selection = fleet.
Deferred: P1 (BYO Tailscale + network metrics), P2 (RemoteDriver + placement so
agents actually run on connected nodes).
Co-Authored-By: Claude Opus 4.8 (1M context) <[email protected]>
This commit is contained in:
co-authored by
Claude Opus 4.8
parent
b853aab6fd
commit
2bdd0a23e8
@@ -7,6 +7,7 @@ pub mod connections;
|
||||
pub mod credits;
|
||||
pub mod files;
|
||||
pub mod messages;
|
||||
pub mod nodes;
|
||||
pub mod orgs;
|
||||
pub mod outbox;
|
||||
pub mod routine_runs;
|
||||
|
||||
@@ -0,0 +1,213 @@
|
||||
//! Fleet node registry: user-connected local-hardware hosts that run the
|
||||
//! clawmates-node daemon. The daemon authenticates its control channel with the
|
||||
//! node `token`, then upserts its host-health snapshot on every heartbeat.
|
||||
|
||||
use cm_domain::{NodeId, WorkspaceId};
|
||||
use sqlx::{PgPool, Row};
|
||||
use time::OffsetDateTime;
|
||||
|
||||
use crate::DbError;
|
||||
|
||||
/// Latest host-health snapshot for a node.
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct NodeHealth {
|
||||
pub cpu_pct: f64,
|
||||
pub mem_total: i64,
|
||||
pub mem_used: i64,
|
||||
pub mem_pressure: f64,
|
||||
pub swap_used: i64,
|
||||
pub disk_total: i64,
|
||||
pub disk_free: i64,
|
||||
pub load1: f64,
|
||||
pub load5: f64,
|
||||
pub load15: f64,
|
||||
pub container_count: i32,
|
||||
}
|
||||
|
||||
/// A registered node plus its latest health (if it has reported one).
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct NodeRow {
|
||||
pub id: NodeId,
|
||||
pub name: String,
|
||||
pub status: String,
|
||||
pub agent_version: Option<String>,
|
||||
pub tailscale_ip: Option<String>,
|
||||
pub last_seen: Option<OffsetDateTime>,
|
||||
pub created_at: OffsetDateTime,
|
||||
pub health: Option<NodeHealth>,
|
||||
}
|
||||
|
||||
/// Register a new (pending) node with its control-channel token.
|
||||
pub async fn create(
|
||||
pool: &PgPool,
|
||||
workspace_id: WorkspaceId,
|
||||
name: &str,
|
||||
token: &str,
|
||||
) -> Result<NodeId, DbError> {
|
||||
let id = NodeId::new();
|
||||
sqlx::query("INSERT INTO nodes (id, workspace_id, name, status, token) VALUES ($1, $2, $3, 'pending', $4)")
|
||||
.bind(id.as_uuid())
|
||||
.bind(workspace_id.as_uuid())
|
||||
.bind(name)
|
||||
.bind(token)
|
||||
.execute(pool)
|
||||
.await?;
|
||||
Ok(id)
|
||||
}
|
||||
|
||||
/// Resolve a control-channel token to its node + workspace (daemon auth).
|
||||
pub async fn auth(pool: &PgPool, token: &str) -> Result<Option<(NodeId, WorkspaceId)>, DbError> {
|
||||
let row = sqlx::query("SELECT id, workspace_id FROM nodes WHERE token = $1")
|
||||
.bind(token)
|
||||
.fetch_optional(pool)
|
||||
.await?;
|
||||
Ok(row.map(|r| {
|
||||
(
|
||||
NodeId::from(r.get::<uuid::Uuid, _>("id")),
|
||||
WorkspaceId::from(r.get::<uuid::Uuid, _>("workspace_id")),
|
||||
)
|
||||
}))
|
||||
}
|
||||
|
||||
const SELECT_WITH_HEALTH: &str = "SELECT n.id, n.name, n.status, n.agent_version, n.tailscale_ip, n.last_seen, n.created_at,
|
||||
h.node_id AS health_node, h.cpu_pct, h.mem_total, h.mem_used, h.mem_pressure, h.swap_used,
|
||||
h.disk_total, h.disk_free, h.load1, h.load5, h.load15, h.container_count
|
||||
FROM nodes n LEFT JOIN node_health h ON h.node_id = n.id";
|
||||
|
||||
/// List a workspace's nodes (oldest first) with their latest health.
|
||||
pub async fn list(pool: &PgPool, workspace_id: WorkspaceId) -> Result<Vec<NodeRow>, DbError> {
|
||||
let rows = sqlx::query(&format!(
|
||||
"{SELECT_WITH_HEALTH} WHERE n.workspace_id = $1 ORDER BY n.created_at"
|
||||
))
|
||||
.bind(workspace_id.as_uuid())
|
||||
.fetch_all(pool)
|
||||
.await?;
|
||||
Ok(rows.into_iter().map(map_node).collect())
|
||||
}
|
||||
|
||||
/// Fetch one node (workspace-scoped) with its latest health.
|
||||
pub async fn get(
|
||||
pool: &PgPool,
|
||||
id: NodeId,
|
||||
workspace_id: WorkspaceId,
|
||||
) -> Result<Option<NodeRow>, DbError> {
|
||||
let row = sqlx::query(&format!(
|
||||
"{SELECT_WITH_HEALTH} WHERE n.id = $1 AND n.workspace_id = $2"
|
||||
))
|
||||
.bind(id.as_uuid())
|
||||
.bind(workspace_id.as_uuid())
|
||||
.fetch_optional(pool)
|
||||
.await?;
|
||||
Ok(row.map(map_node))
|
||||
}
|
||||
|
||||
/// Record a heartbeat: mark the node online + refresh its version/tailscale IP,
|
||||
/// and upsert its latest host-health snapshot.
|
||||
pub async fn heartbeat(
|
||||
pool: &PgPool,
|
||||
id: NodeId,
|
||||
agent_version: Option<&str>,
|
||||
tailscale_ip: Option<&str>,
|
||||
h: &NodeHealth,
|
||||
) -> Result<(), DbError> {
|
||||
sqlx::query(
|
||||
"UPDATE nodes SET status = 'online', last_seen = now(),
|
||||
agent_version = COALESCE($2, agent_version),
|
||||
tailscale_ip = COALESCE($3, tailscale_ip)
|
||||
WHERE id = $1",
|
||||
)
|
||||
.bind(id.as_uuid())
|
||||
.bind(agent_version)
|
||||
.bind(tailscale_ip)
|
||||
.execute(pool)
|
||||
.await?;
|
||||
sqlx::query(
|
||||
"INSERT INTO node_health
|
||||
(node_id, captured_at, cpu_pct, mem_total, mem_used, mem_pressure, swap_used,
|
||||
disk_total, disk_free, load1, load5, load15, container_count)
|
||||
VALUES ($1, now(), $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12)
|
||||
ON CONFLICT (node_id) DO UPDATE SET
|
||||
captured_at = now(), cpu_pct = excluded.cpu_pct, mem_total = excluded.mem_total,
|
||||
mem_used = excluded.mem_used, mem_pressure = excluded.mem_pressure,
|
||||
swap_used = excluded.swap_used, disk_total = excluded.disk_total,
|
||||
disk_free = excluded.disk_free, load1 = excluded.load1, load5 = excluded.load5,
|
||||
load15 = excluded.load15, container_count = excluded.container_count",
|
||||
)
|
||||
.bind(id.as_uuid())
|
||||
.bind(h.cpu_pct)
|
||||
.bind(h.mem_total)
|
||||
.bind(h.mem_used)
|
||||
.bind(h.mem_pressure)
|
||||
.bind(h.swap_used)
|
||||
.bind(h.disk_total)
|
||||
.bind(h.disk_free)
|
||||
.bind(h.load1)
|
||||
.bind(h.load5)
|
||||
.bind(h.load15)
|
||||
.bind(h.container_count)
|
||||
.execute(pool)
|
||||
.await?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Set a node's status (e.g. 'offline' when its channel drops, 'draining' on
|
||||
/// deregister).
|
||||
pub async fn set_status(pool: &PgPool, id: NodeId, status: &str) -> Result<(), DbError> {
|
||||
sqlx::query("UPDATE nodes SET status = $2 WHERE id = $1")
|
||||
.bind(id.as_uuid())
|
||||
.bind(status)
|
||||
.execute(pool)
|
||||
.await?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Mark online nodes whose last heartbeat is older than `secs` as offline.
|
||||
pub async fn mark_stale_offline(pool: &PgPool, secs: i64) -> Result<(), DbError> {
|
||||
sqlx::query(
|
||||
"UPDATE nodes SET status = 'offline'
|
||||
WHERE status = 'online'
|
||||
AND (last_seen IS NULL OR last_seen < now() - ($1 * interval '1 second'))",
|
||||
)
|
||||
.bind(secs)
|
||||
.execute(pool)
|
||||
.await?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Remove a node from the registry (workspace-scoped).
|
||||
pub async fn delete(pool: &PgPool, id: NodeId, workspace_id: WorkspaceId) -> Result<(), DbError> {
|
||||
sqlx::query("DELETE FROM nodes WHERE id = $1 AND workspace_id = $2")
|
||||
.bind(id.as_uuid())
|
||||
.bind(workspace_id.as_uuid())
|
||||
.execute(pool)
|
||||
.await?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn map_node(r: sqlx::postgres::PgRow) -> NodeRow {
|
||||
let health = r
|
||||
.get::<Option<uuid::Uuid>, _>("health_node")
|
||||
.map(|_| NodeHealth {
|
||||
cpu_pct: r.get("cpu_pct"),
|
||||
mem_total: r.get("mem_total"),
|
||||
mem_used: r.get("mem_used"),
|
||||
mem_pressure: r.get("mem_pressure"),
|
||||
swap_used: r.get("swap_used"),
|
||||
disk_total: r.get("disk_total"),
|
||||
disk_free: r.get("disk_free"),
|
||||
load1: r.get("load1"),
|
||||
load5: r.get("load5"),
|
||||
load15: r.get("load15"),
|
||||
container_count: r.get("container_count"),
|
||||
});
|
||||
NodeRow {
|
||||
id: NodeId::from(r.get::<uuid::Uuid, _>("id")),
|
||||
name: r.get("name"),
|
||||
status: r.get("status"),
|
||||
agent_version: r.get("agent_version"),
|
||||
tailscale_ip: r.get("tailscale_ip"),
|
||||
last_seen: r.get("last_seen"),
|
||||
created_at: r.get("created_at"),
|
||||
health,
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user