Fleet: Beszel hub integration — rich per-node metrics + per-node monitor (Phase 1)
Tap each node's Beszel metrics (GPU/temps/disk-IO/network/per-container — beyond
our basic heartbeat) by reading the workspace's Beszel hub. The agents run in
WS-only mode with no locally-readable socket, so (per the de-risk) the server taps
the hub's PocketBase API instead of the daemon reading agents — no daemon changes.
- migrations: workspace_beszel (BYO hub URL + login, server-side only, mirrors the
Tailscale BYO pattern) + node_metrics (latest scalar columns + JSONB blob).
- cm-db: repo/fleet_beszel.rs, repo/node_metrics.rs; nodes SELECT joins node_metrics
(gpu_pct/temp_max surfaced on node_json for the live cards).
- cm-api: beszel.rs client (auth-with-password, poll `systems`, map to nodes by
hostname, upsert metrics) + a 15s spawn_poller; routes/beszel.rs (connect/status/
disconnect + GET /api/nodes/{id}/metrics with history proxied live from the hub).
- frontend: HostCard gains a GPU/temp readout + a Monitor button; NodeMonitor is a
full-width per-node page (current panel + CPU/mem/GPU/temp/net/disk charts from the
hub's 1m history); a "Beszel monitoring" connect form in the Local view.
Reachability confirmed: gw-04 → the hub over the tailnet (100.123.224.84:8090). Needs
the user to connect their hub login to activate the poller.
Co-Authored-By: Claude Opus 4.8 (1M context) <[email protected]>
This commit is contained in:
co-authored by
Claude Opus 4.8
parent
4de2f31b50
commit
36a227566b
Generated
+1
-1
@@ -835,7 +835,7 @@ dependencies = [
|
|||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "clawmates-node"
|
name = "clawmates-node"
|
||||||
version = "0.3.0"
|
version = "0.4.0"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"base64",
|
"base64",
|
||||||
"bytes",
|
"bytes",
|
||||||
|
|||||||
@@ -269,6 +269,8 @@ async fn run() -> Result<(), String> {
|
|||||||
// Fleet backstop: a node whose heartbeats stop (without a clean channel
|
// Fleet backstop: a node whose heartbeats stop (without a clean channel
|
||||||
// close) goes offline within ~28s even if its control channel hangs.
|
// close) goes offline within ~28s even if its control channel hangs.
|
||||||
cm_api::fleet::spawn_node_sweeper(pool.clone(), std::time::Duration::from_secs(8), 20);
|
cm_api::fleet::spawn_node_sweeper(pool.clone(), std::time::Duration::from_secs(8), 20);
|
||||||
|
// Beszel: poll each workspace's monitoring hub for rich per-node metrics.
|
||||||
|
cm_api::beszel::spawn_poller(pool.clone(), std::time::Duration::from_secs(15));
|
||||||
|
|
||||||
// Hosted identity (Clerk / OIDC): pin the issuer and load its JWKS.
|
// Hosted identity (Clerk / OIDC): pin the issuer and load its JWKS.
|
||||||
let auth_verifier = match config.auth.mode {
|
let auth_verifier = match config.auth.mode {
|
||||||
|
|||||||
@@ -0,0 +1,180 @@
|
|||||||
|
//! Bring-your-own Beszel hub client: authenticate to a workspace's Beszel hub
|
||||||
|
//! (PocketBase) and poll rich per-node metrics, mapping its `systems` to our fleet
|
||||||
|
//! nodes by hostname and upserting `node_metrics`. Server-side only.
|
||||||
|
|
||||||
|
use std::collections::HashMap;
|
||||||
|
use std::time::Duration;
|
||||||
|
|
||||||
|
use cm_db::repo::fleet_beszel::BeszelConn;
|
||||||
|
use cm_db::repo::node_metrics::NodeMetrics;
|
||||||
|
use cm_db::repo::{fleet_beszel, node_metrics, nodes};
|
||||||
|
use cm_domain::{NodeId, WorkspaceId};
|
||||||
|
use serde_json::{json, Value};
|
||||||
|
use sqlx::PgPool;
|
||||||
|
|
||||||
|
/// Authenticate to a Beszel hub → a PocketBase token. Tries the regular users
|
||||||
|
/// collection first, then superusers.
|
||||||
|
pub async fn authenticate(client: &reqwest::Client, conn: &BeszelConn) -> Result<String, String> {
|
||||||
|
let base = conn.hub_url.trim_end_matches('/');
|
||||||
|
for coll in ["users", "_superusers"] {
|
||||||
|
let url = format!("{base}/api/collections/{coll}/auth-with-password");
|
||||||
|
let resp = client
|
||||||
|
.post(&url)
|
||||||
|
.json(&json!({ "identity": conn.username, "password": conn.password }))
|
||||||
|
.send()
|
||||||
|
.await
|
||||||
|
.map_err(|e| e.to_string())?;
|
||||||
|
if resp.status().is_success() {
|
||||||
|
let body: Value = resp.json().await.map_err(|e| e.to_string())?;
|
||||||
|
if let Some(tok) = body.get("token").and_then(Value::as_str) {
|
||||||
|
return Ok(tok.to_owned());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
Err("auth failed (check hub URL + credentials)".into())
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Fetch all `systems` records (current status + the `info` snapshot).
|
||||||
|
pub async fn fetch_systems(
|
||||||
|
client: &reqwest::Client,
|
||||||
|
conn: &BeszelConn,
|
||||||
|
token: &str,
|
||||||
|
) -> Result<Vec<Value>, String> {
|
||||||
|
let base = conn.hub_url.trim_end_matches('/');
|
||||||
|
let resp = client
|
||||||
|
.get(format!("{base}/api/collections/systems/records"))
|
||||||
|
.query(&[("perPage", "500")])
|
||||||
|
.header("Authorization", token)
|
||||||
|
.send()
|
||||||
|
.await
|
||||||
|
.map_err(|e| e.to_string())?;
|
||||||
|
if !resp.status().is_success() {
|
||||||
|
return Err(format!("systems list HTTP {}", resp.status()));
|
||||||
|
}
|
||||||
|
let body: Value = resp.json().await.map_err(|e| e.to_string())?;
|
||||||
|
Ok(body
|
||||||
|
.get("items")
|
||||||
|
.and_then(Value::as_array)
|
||||||
|
.cloned()
|
||||||
|
.unwrap_or_default())
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Proxy a system's recent 1m time-series (for the monitor-page charts).
|
||||||
|
pub async fn fetch_history(
|
||||||
|
client: &reqwest::Client,
|
||||||
|
conn: &BeszelConn,
|
||||||
|
token: &str,
|
||||||
|
system_id: &str,
|
||||||
|
) -> Result<Value, String> {
|
||||||
|
let base = conn.hub_url.trim_end_matches('/');
|
||||||
|
let filter = format!("system='{system_id}' && type='1m'");
|
||||||
|
let resp = client
|
||||||
|
.get(format!("{base}/api/collections/system_stats/records"))
|
||||||
|
.query(&[
|
||||||
|
("perPage", "120"),
|
||||||
|
("sort", "-created"),
|
||||||
|
("filter", filter.as_str()),
|
||||||
|
])
|
||||||
|
.header("Authorization", token)
|
||||||
|
.send()
|
||||||
|
.await
|
||||||
|
.map_err(|e| e.to_string())?;
|
||||||
|
if !resp.status().is_success() {
|
||||||
|
return Err(format!("system_stats HTTP {}", resp.status()));
|
||||||
|
}
|
||||||
|
resp.json().await.map_err(|e| e.to_string())
|
||||||
|
}
|
||||||
|
|
||||||
|
fn f(v: &Value, k: &str) -> Option<f64> {
|
||||||
|
v.get(k).and_then(Value::as_f64)
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Map a Beszel `systems` record (its `info` snapshot) into our NodeMetrics.
|
||||||
|
fn metrics_from_system(system: &Value) -> NodeMetrics {
|
||||||
|
let info = system.get("info").cloned().unwrap_or_else(|| json!({}));
|
||||||
|
let load1 = info
|
||||||
|
.get("la")
|
||||||
|
.and_then(Value::as_array)
|
||||||
|
.and_then(|a| a.first())
|
||||||
|
.and_then(Value::as_f64);
|
||||||
|
NodeMetrics {
|
||||||
|
cpu_pct: f(&info, "cpu"),
|
||||||
|
mem_pct: f(&info, "mp"),
|
||||||
|
disk_pct: f(&info, "dp"),
|
||||||
|
gpu_pct: f(&info, "g"),
|
||||||
|
temp_max: f(&info, "dt"),
|
||||||
|
net_sent_ps: None,
|
||||||
|
net_recv_ps: None,
|
||||||
|
disk_read_ps: None,
|
||||||
|
disk_write_ps: None,
|
||||||
|
load1,
|
||||||
|
container_count: None,
|
||||||
|
data: json!({
|
||||||
|
"beszelSystemId": system.get("id").and_then(Value::as_str),
|
||||||
|
"status": system.get("status").and_then(Value::as_str),
|
||||||
|
"name": system.get("name").and_then(Value::as_str),
|
||||||
|
"host": system.get("host").and_then(Value::as_str),
|
||||||
|
"info": info,
|
||||||
|
}),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Poll one workspace's hub: auth, fetch systems, match to nodes by hostname,
|
||||||
|
/// upsert metrics. Returns how many nodes were updated.
|
||||||
|
pub async fn poll_workspace(
|
||||||
|
client: &reqwest::Client,
|
||||||
|
pool: &PgPool,
|
||||||
|
ws: WorkspaceId,
|
||||||
|
conn: &BeszelConn,
|
||||||
|
) -> Result<usize, String> {
|
||||||
|
let token = authenticate(client, conn).await?;
|
||||||
|
let systems = fetch_systems(client, conn, &token).await?;
|
||||||
|
let node_rows = nodes::list(pool, ws).await.map_err(|e| e.to_string())?;
|
||||||
|
// hostname/name (lowercased) → node id.
|
||||||
|
let mut by_host: HashMap<String, NodeId> = HashMap::new();
|
||||||
|
for n in &node_rows {
|
||||||
|
if let Some(h) = n.hostname.as_deref() {
|
||||||
|
by_host.insert(h.to_lowercase(), n.id);
|
||||||
|
}
|
||||||
|
by_host.entry(n.name.to_lowercase()).or_insert(n.id);
|
||||||
|
}
|
||||||
|
let mut updated = 0;
|
||||||
|
for sys in &systems {
|
||||||
|
let key = sys
|
||||||
|
.get("name")
|
||||||
|
.and_then(Value::as_str)
|
||||||
|
.or_else(|| sys.get("host").and_then(Value::as_str))
|
||||||
|
.map(str::to_lowercase);
|
||||||
|
let Some(node_id) = key.as_deref().and_then(|k| by_host.get(k).copied()) else {
|
||||||
|
continue;
|
||||||
|
};
|
||||||
|
if node_metrics::upsert(pool, node_id, &metrics_from_system(sys))
|
||||||
|
.await
|
||||||
|
.is_ok()
|
||||||
|
{
|
||||||
|
updated += 1;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
Ok(updated)
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Spawn the background poller: every `interval`, refresh node_metrics for each
|
||||||
|
/// workspace with a Beszel hub connected.
|
||||||
|
pub fn spawn_poller(pool: PgPool, interval: Duration) {
|
||||||
|
tokio::spawn(async move {
|
||||||
|
let client = reqwest::Client::new();
|
||||||
|
let mut tick = tokio::time::interval(interval);
|
||||||
|
loop {
|
||||||
|
tick.tick().await;
|
||||||
|
let workspaces = match fleet_beszel::all(&pool).await {
|
||||||
|
Ok(w) => w,
|
||||||
|
Err(_) => continue,
|
||||||
|
};
|
||||||
|
for (ws, conn) in workspaces {
|
||||||
|
if let Err(e) = poll_workspace(&client, &pool, ws, &conn).await {
|
||||||
|
eprintln!("beszel poll (ws {}): {e}", ws.as_uuid());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
});
|
||||||
|
}
|
||||||
@@ -1,5 +1,6 @@
|
|||||||
//! REST API for Clawmates (spec §13). One route resource per module.
|
//! REST API for Clawmates (spec §13). One route resource per module.
|
||||||
|
|
||||||
|
pub mod beszel;
|
||||||
pub mod cleanup_sweeper;
|
pub mod cleanup_sweeper;
|
||||||
mod error;
|
mod error;
|
||||||
mod extract;
|
mod extract;
|
||||||
@@ -123,7 +124,14 @@ pub fn router(state: AppState) -> Router {
|
|||||||
.route("/api/nodes/{id}/sandbox-check", post(routes::nodes::sandbox_check))
|
.route("/api/nodes/{id}/sandbox-check", post(routes::nodes::sandbox_check))
|
||||||
.route("/api/nodes/{id}/terminal/ticket", post(routes::nodes::terminal_ticket))
|
.route("/api/nodes/{id}/terminal/ticket", post(routes::nodes::terminal_ticket))
|
||||||
.route("/api/nodes/{id}/terminal/ws", get(routes::nodes::terminal_ws))
|
.route("/api/nodes/{id}/terminal/ws", get(routes::nodes::terminal_ws))
|
||||||
|
.route("/api/nodes/{id}/metrics", get(routes::beszel::node_metrics_get))
|
||||||
.route("/api/nodes/{id}", delete(routes::nodes::remove))
|
.route("/api/nodes/{id}", delete(routes::nodes::remove))
|
||||||
|
.route(
|
||||||
|
"/api/fleet/beszel",
|
||||||
|
get(routes::beszel::status)
|
||||||
|
.post(routes::beszel::connect)
|
||||||
|
.delete(routes::beszel::disconnect),
|
||||||
|
)
|
||||||
.route(
|
.route(
|
||||||
"/api/fleet/tailscale",
|
"/api/fleet/tailscale",
|
||||||
get(routes::tailscale::status)
|
get(routes::tailscale::status)
|
||||||
|
|||||||
@@ -0,0 +1,90 @@
|
|||||||
|
//! Bring-your-own Beszel hub: connect a workspace's Beszel monitoring hub (store +
|
||||||
|
//! verify its login) and serve per-node metrics (latest snapshot + history proxied
|
||||||
|
//! live from the hub). Credentials are used server-side only.
|
||||||
|
|
||||||
|
use axum::extract::{Path, State};
|
||||||
|
use axum::Json;
|
||||||
|
use cm_db::repo::fleet_beszel::BeszelConn;
|
||||||
|
use cm_db::repo::{fleet_beszel, node_metrics, nodes};
|
||||||
|
use cm_domain::NodeId;
|
||||||
|
use serde::Deserialize;
|
||||||
|
use serde_json::{json, Value};
|
||||||
|
use uuid::Uuid;
|
||||||
|
|
||||||
|
use crate::{beszel, ApiError, AppState, Authed};
|
||||||
|
|
||||||
|
#[derive(Deserialize)]
|
||||||
|
pub struct ConnectReq {
|
||||||
|
#[serde(rename = "hubUrl")]
|
||||||
|
pub hub_url: String,
|
||||||
|
pub username: String,
|
||||||
|
pub password: String,
|
||||||
|
}
|
||||||
|
|
||||||
|
/// `POST /api/fleet/beszel` — store + verify the workspace's Beszel hub login.
|
||||||
|
pub async fn connect(
|
||||||
|
State(state): State<AppState>,
|
||||||
|
Authed(user): Authed,
|
||||||
|
Json(req): Json<ConnectReq>,
|
||||||
|
) -> Result<Json<Value>, ApiError> {
|
||||||
|
let hub = req.hub_url.trim().trim_end_matches('/').to_owned();
|
||||||
|
let username = req.username.trim().to_owned();
|
||||||
|
if hub.is_empty() || username.is_empty() || req.password.is_empty() {
|
||||||
|
return Ok(Json(
|
||||||
|
json!({ "ok": false, "error": "hubUrl, username and password are required" }),
|
||||||
|
));
|
||||||
|
}
|
||||||
|
let conn = BeszelConn { hub_url: hub.clone(), username: username.clone(), password: req.password.clone() };
|
||||||
|
// Verify the credentials authenticate before persisting.
|
||||||
|
if let Err(e) = beszel::authenticate(&reqwest::Client::new(), &conn).await {
|
||||||
|
return Ok(Json(json!({ "ok": false, "error": e })));
|
||||||
|
}
|
||||||
|
fleet_beszel::set(&state.pool, user.workspace_id, &hub, &username, &req.password).await?;
|
||||||
|
Ok(Json(json!({ "ok": true, "hubUrl": hub })))
|
||||||
|
}
|
||||||
|
|
||||||
|
/// `GET /api/fleet/beszel` — connection status.
|
||||||
|
pub async fn status(
|
||||||
|
State(state): State<AppState>,
|
||||||
|
Authed(user): Authed,
|
||||||
|
) -> Result<Json<Value>, ApiError> {
|
||||||
|
let conn = fleet_beszel::get(&state.pool, user.workspace_id).await?;
|
||||||
|
Ok(Json(json!({ "connected": conn.is_some(), "hubUrl": conn.map(|c| c.hub_url) })))
|
||||||
|
}
|
||||||
|
|
||||||
|
/// `DELETE /api/fleet/beszel` — disconnect.
|
||||||
|
pub async fn disconnect(
|
||||||
|
State(state): State<AppState>,
|
||||||
|
Authed(user): Authed,
|
||||||
|
) -> Result<Json<Value>, ApiError> {
|
||||||
|
fleet_beszel::delete(&state.pool, user.workspace_id).await?;
|
||||||
|
Ok(Json(json!({ "ok": true })))
|
||||||
|
}
|
||||||
|
|
||||||
|
/// `GET /api/nodes/{id}/metrics` — latest snapshot + recent 1m history (proxied
|
||||||
|
/// live from the hub) for the per-node monitor page.
|
||||||
|
pub async fn node_metrics_get(
|
||||||
|
State(state): State<AppState>,
|
||||||
|
Authed(user): Authed,
|
||||||
|
Path(id): Path<Uuid>,
|
||||||
|
) -> Result<Json<Value>, ApiError> {
|
||||||
|
let node_id = NodeId::from(id);
|
||||||
|
nodes::get(&state.pool, node_id, user.workspace_id)
|
||||||
|
.await?
|
||||||
|
.ok_or(ApiError::NotFound)?;
|
||||||
|
let latest = node_metrics::latest(&state.pool, node_id).await?;
|
||||||
|
let mut history = json!([]);
|
||||||
|
if let (Some(latest_v), Some(conn)) =
|
||||||
|
(&latest, fleet_beszel::get(&state.pool, user.workspace_id).await?)
|
||||||
|
{
|
||||||
|
if let Some(sysid) = latest_v.get("beszelSystemId").and_then(Value::as_str) {
|
||||||
|
let client = reqwest::Client::new();
|
||||||
|
if let Ok(token) = beszel::authenticate(&client, &conn).await {
|
||||||
|
if let Ok(h) = beszel::fetch_history(&client, &conn, &token, sysid).await {
|
||||||
|
history = h.get("items").cloned().unwrap_or_else(|| json!([]));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
Ok(Json(json!({ "latest": latest, "history": history })))
|
||||||
|
}
|
||||||
@@ -1,6 +1,7 @@
|
|||||||
pub mod approvals;
|
pub mod approvals;
|
||||||
pub mod apps;
|
pub mod apps;
|
||||||
pub mod auth;
|
pub mod auth;
|
||||||
|
pub mod beszel;
|
||||||
pub mod billing;
|
pub mod billing;
|
||||||
pub mod browser;
|
pub mod browser;
|
||||||
pub mod claw_chat;
|
pub mod claw_chat;
|
||||||
|
|||||||
@@ -37,6 +37,8 @@ fn node_json(n: &nodes::NodeRow) -> Value {
|
|||||||
"tailscaleIp": n.tailscale_ip,
|
"tailscaleIp": n.tailscale_ip,
|
||||||
"lastSeen": n.last_seen.map(|t| t.unix_timestamp()),
|
"lastSeen": n.last_seen.map(|t| t.unix_timestamp()),
|
||||||
"createdAt": n.created_at.unix_timestamp(),
|
"createdAt": n.created_at.unix_timestamp(),
|
||||||
|
"gpuPct": n.gpu_pct,
|
||||||
|
"tempMax": n.temp_max,
|
||||||
"health": n.health.as_ref().map(|h| json!({
|
"health": n.health.as_ref().map(|h| json!({
|
||||||
"cpuPct": h.cpu_pct,
|
"cpuPct": h.cpu_pct,
|
||||||
"memTotal": h.mem_total,
|
"memTotal": h.mem_total,
|
||||||
|
|||||||
@@ -0,0 +1,85 @@
|
|||||||
|
//! Per-workspace Beszel hub connection (BYO monitoring): the hub URL + a login we
|
||||||
|
//! use to read rich per-node metrics via the hub's PocketBase API. Server-side only.
|
||||||
|
|
||||||
|
use cm_domain::WorkspaceId;
|
||||||
|
use sqlx::{PgPool, Row};
|
||||||
|
|
||||||
|
use crate::DbError;
|
||||||
|
|
||||||
|
/// A stored Beszel hub connection.
|
||||||
|
#[derive(Debug, Clone)]
|
||||||
|
pub struct BeszelConn {
|
||||||
|
pub hub_url: String,
|
||||||
|
pub username: String,
|
||||||
|
pub password: String,
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Store (or replace) a workspace's Beszel hub URL + credentials.
|
||||||
|
pub async fn set(
|
||||||
|
pool: &PgPool,
|
||||||
|
workspace_id: WorkspaceId,
|
||||||
|
hub_url: &str,
|
||||||
|
username: &str,
|
||||||
|
password: &str,
|
||||||
|
) -> Result<(), DbError> {
|
||||||
|
sqlx::query(
|
||||||
|
"INSERT INTO workspace_beszel (workspace_id, hub_url, username, password, connected_at)
|
||||||
|
VALUES ($1, $2, $3, $4, now())
|
||||||
|
ON CONFLICT (workspace_id) DO UPDATE SET
|
||||||
|
hub_url = excluded.hub_url, username = excluded.username,
|
||||||
|
password = excluded.password, connected_at = now()",
|
||||||
|
)
|
||||||
|
.bind(workspace_id.as_uuid())
|
||||||
|
.bind(hub_url)
|
||||||
|
.bind(username)
|
||||||
|
.bind(password)
|
||||||
|
.execute(pool)
|
||||||
|
.await?;
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Get a workspace's stored Beszel connection, if any.
|
||||||
|
pub async fn get(
|
||||||
|
pool: &PgPool,
|
||||||
|
workspace_id: WorkspaceId,
|
||||||
|
) -> Result<Option<BeszelConn>, DbError> {
|
||||||
|
let row =
|
||||||
|
sqlx::query("SELECT hub_url, username, password FROM workspace_beszel WHERE workspace_id = $1")
|
||||||
|
.bind(workspace_id.as_uuid())
|
||||||
|
.fetch_optional(pool)
|
||||||
|
.await?;
|
||||||
|
Ok(row.map(|r| BeszelConn {
|
||||||
|
hub_url: r.get("hub_url"),
|
||||||
|
username: r.get("username"),
|
||||||
|
password: r.get("password"),
|
||||||
|
}))
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Every workspace with a Beszel hub connected (for the background poll task).
|
||||||
|
pub async fn all(pool: &PgPool) -> Result<Vec<(WorkspaceId, BeszelConn)>, DbError> {
|
||||||
|
let rows = sqlx::query("SELECT workspace_id, hub_url, username, password FROM workspace_beszel")
|
||||||
|
.fetch_all(pool)
|
||||||
|
.await?;
|
||||||
|
Ok(rows
|
||||||
|
.into_iter()
|
||||||
|
.map(|r| {
|
||||||
|
(
|
||||||
|
WorkspaceId::from(r.get::<uuid::Uuid, _>("workspace_id")),
|
||||||
|
BeszelConn {
|
||||||
|
hub_url: r.get("hub_url"),
|
||||||
|
username: r.get("username"),
|
||||||
|
password: r.get("password"),
|
||||||
|
},
|
||||||
|
)
|
||||||
|
})
|
||||||
|
.collect())
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Disconnect a workspace's Beszel hub.
|
||||||
|
pub async fn delete(pool: &PgPool, workspace_id: WorkspaceId) -> Result<(), DbError> {
|
||||||
|
sqlx::query("DELETE FROM workspace_beszel WHERE workspace_id = $1")
|
||||||
|
.bind(workspace_id.as_uuid())
|
||||||
|
.execute(pool)
|
||||||
|
.await?;
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
@@ -6,8 +6,10 @@ pub mod companies;
|
|||||||
pub mod connections;
|
pub mod connections;
|
||||||
pub mod credits;
|
pub mod credits;
|
||||||
pub mod files;
|
pub mod files;
|
||||||
|
pub mod fleet_beszel;
|
||||||
pub mod fleet_tailscale;
|
pub mod fleet_tailscale;
|
||||||
pub mod messages;
|
pub mod messages;
|
||||||
|
pub mod node_metrics;
|
||||||
pub mod nodes;
|
pub mod nodes;
|
||||||
pub mod orgs;
|
pub mod orgs;
|
||||||
pub mod outbox;
|
pub mod outbox;
|
||||||
|
|||||||
@@ -0,0 +1,75 @@
|
|||||||
|
//! Rich per-node metrics (from the workspace's Beszel hub). One latest snapshot
|
||||||
|
//! per node: scalar columns for the fleet cards + rules engine, plus a JSONB blob
|
||||||
|
//! for the per-node monitor page.
|
||||||
|
|
||||||
|
use cm_domain::NodeId;
|
||||||
|
use serde_json::Value;
|
||||||
|
use sqlx::{PgPool, Row};
|
||||||
|
|
||||||
|
use crate::DbError;
|
||||||
|
|
||||||
|
/// The latest metric snapshot for a node (nullable scalars + the full blob).
|
||||||
|
#[derive(Debug, Clone, Default)]
|
||||||
|
pub struct NodeMetrics {
|
||||||
|
pub cpu_pct: Option<f64>,
|
||||||
|
pub mem_pct: Option<f64>,
|
||||||
|
pub disk_pct: Option<f64>,
|
||||||
|
pub gpu_pct: Option<f64>,
|
||||||
|
pub temp_max: Option<f64>,
|
||||||
|
pub net_sent_ps: Option<i64>,
|
||||||
|
pub net_recv_ps: Option<i64>,
|
||||||
|
pub disk_read_ps: Option<i64>,
|
||||||
|
pub disk_write_ps: Option<i64>,
|
||||||
|
pub load1: Option<f64>,
|
||||||
|
pub container_count: Option<i32>,
|
||||||
|
pub data: Value,
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Upsert a node's latest metrics snapshot.
|
||||||
|
pub async fn upsert(pool: &PgPool, node_id: NodeId, m: &NodeMetrics) -> Result<(), DbError> {
|
||||||
|
sqlx::query(
|
||||||
|
"INSERT INTO node_metrics
|
||||||
|
(node_id, updated_at, cpu_pct, mem_pct, disk_pct, gpu_pct, temp_max,
|
||||||
|
net_sent_ps, net_recv_ps, disk_read_ps, disk_write_ps, load1, container_count, data)
|
||||||
|
VALUES ($1, now(), $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13)
|
||||||
|
ON CONFLICT (node_id) DO UPDATE SET
|
||||||
|
updated_at = now(), cpu_pct = excluded.cpu_pct, mem_pct = excluded.mem_pct,
|
||||||
|
disk_pct = excluded.disk_pct, gpu_pct = excluded.gpu_pct, temp_max = excluded.temp_max,
|
||||||
|
net_sent_ps = excluded.net_sent_ps, net_recv_ps = excluded.net_recv_ps,
|
||||||
|
disk_read_ps = excluded.disk_read_ps, disk_write_ps = excluded.disk_write_ps,
|
||||||
|
load1 = excluded.load1, container_count = excluded.container_count, data = excluded.data",
|
||||||
|
)
|
||||||
|
.bind(node_id.as_uuid())
|
||||||
|
.bind(m.cpu_pct)
|
||||||
|
.bind(m.mem_pct)
|
||||||
|
.bind(m.disk_pct)
|
||||||
|
.bind(m.gpu_pct)
|
||||||
|
.bind(m.temp_max)
|
||||||
|
.bind(m.net_sent_ps)
|
||||||
|
.bind(m.net_recv_ps)
|
||||||
|
.bind(m.disk_read_ps)
|
||||||
|
.bind(m.disk_write_ps)
|
||||||
|
.bind(m.load1)
|
||||||
|
.bind(m.container_count)
|
||||||
|
.bind(&m.data)
|
||||||
|
.execute(pool)
|
||||||
|
.await?;
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
/// The latest metrics blob for a node (the full snapshot for the monitor page).
|
||||||
|
pub async fn latest(pool: &PgPool, node_id: NodeId) -> Result<Option<Value>, DbError> {
|
||||||
|
let row = sqlx::query(
|
||||||
|
"SELECT data, extract(epoch from updated_at)::bigint AS updated FROM node_metrics WHERE node_id = $1",
|
||||||
|
)
|
||||||
|
.bind(node_id.as_uuid())
|
||||||
|
.fetch_optional(pool)
|
||||||
|
.await?;
|
||||||
|
Ok(row.map(|r| {
|
||||||
|
let mut data: Value = r.get("data");
|
||||||
|
if let Some(obj) = data.as_object_mut() {
|
||||||
|
obj.insert("updatedAt".into(), serde_json::json!(r.get::<i64, _>("updated")));
|
||||||
|
}
|
||||||
|
data
|
||||||
|
}))
|
||||||
|
}
|
||||||
@@ -37,6 +37,9 @@ pub struct NodeRow {
|
|||||||
pub last_seen: Option<OffsetDateTime>,
|
pub last_seen: Option<OffsetDateTime>,
|
||||||
pub created_at: OffsetDateTime,
|
pub created_at: OffsetDateTime,
|
||||||
pub health: Option<NodeHealth>,
|
pub health: Option<NodeHealth>,
|
||||||
|
/// Latest Beszel metrics scalars (the rich bits beyond basic health).
|
||||||
|
pub gpu_pct: Option<f64>,
|
||||||
|
pub temp_max: Option<f64>,
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Register a new (pending) node with its control-channel token.
|
/// Register a new (pending) node with its control-channel token.
|
||||||
@@ -73,8 +76,10 @@ pub async fn auth(pool: &PgPool, token: &str) -> Result<Option<(NodeId, Workspac
|
|||||||
|
|
||||||
const SELECT_WITH_HEALTH: &str = "SELECT n.id, n.name, n.hostname, n.local_ip, n.status, n.agent_version, n.tailscale_ip, n.last_seen, n.created_at,
|
const SELECT_WITH_HEALTH: &str = "SELECT n.id, n.name, n.hostname, n.local_ip, n.status, n.agent_version, n.tailscale_ip, n.last_seen, n.created_at,
|
||||||
h.node_id AS health_node, h.cpu_pct, h.mem_total, h.mem_used, h.mem_pressure, h.swap_used,
|
h.node_id AS health_node, h.cpu_pct, h.mem_total, h.mem_used, h.mem_pressure, h.swap_used,
|
||||||
h.disk_total, h.disk_free, h.load1, h.load5, h.load15, h.container_count
|
h.disk_total, h.disk_free, h.load1, h.load5, h.load15, h.container_count,
|
||||||
FROM nodes n LEFT JOIN node_health h ON h.node_id = n.id";
|
m.gpu_pct AS m_gpu_pct, m.temp_max AS m_temp_max
|
||||||
|
FROM nodes n LEFT JOIN node_health h ON h.node_id = n.id
|
||||||
|
LEFT JOIN node_metrics m ON m.node_id = n.id";
|
||||||
|
|
||||||
/// List a workspace's nodes (oldest first) with their latest health.
|
/// List a workspace's nodes (oldest first) with their latest health.
|
||||||
pub async fn list(pool: &PgPool, workspace_id: WorkspaceId) -> Result<Vec<NodeRow>, DbError> {
|
pub async fn list(pool: &PgPool, workspace_id: WorkspaceId) -> Result<Vec<NodeRow>, DbError> {
|
||||||
@@ -220,5 +225,7 @@ fn map_node(r: sqlx::postgres::PgRow) -> NodeRow {
|
|||||||
last_seen: r.get("last_seen"),
|
last_seen: r.get("last_seen"),
|
||||||
created_at: r.get("created_at"),
|
created_at: r.get("created_at"),
|
||||||
health,
|
health,
|
||||||
|
gpu_pct: r.get("m_gpu_pct"),
|
||||||
|
temp_max: r.get("m_temp_max"),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -26,6 +26,7 @@ import { StructureTree, orgNode, clawNode, type TreeItem } from "./StructureTree
|
|||||||
import { UserMenu } from "./UserMenu";
|
import { UserMenu } from "./UserMenu";
|
||||||
import { ToolPanel, type ToolKey } from "./ToolPanel";
|
import { ToolPanel, type ToolKey } from "./ToolPanel";
|
||||||
import { InfraNav, FleetConsole, FleetStatusBar, FleetPill } from "./fleet/FleetConsole";
|
import { InfraNav, FleetConsole, FleetStatusBar, FleetPill } from "./fleet/FleetConsole";
|
||||||
|
import { NodeMonitor } from "./fleet/NodeMonitor";
|
||||||
import { ConnectHostWizard } from "./ConnectHostWizard";
|
import { ConnectHostWizard } from "./ConnectHostWizard";
|
||||||
import { INFRA_CATALOG } from "@/components/computer/catalogs/infra";
|
import { INFRA_CATALOG } from "@/components/computer/catalogs/infra";
|
||||||
import { ClawChatSection } from "./ClawChatSection";
|
import { ClawChatSection } from "./ClawChatSection";
|
||||||
@@ -195,6 +196,8 @@ export function Dashboard({ user, orgs, claws }: { user?: { display_name?: strin
|
|||||||
// Infrastructure tier: which nav view (local / cloud) + the connect-host wizard.
|
// Infrastructure tier: which nav view (local / cloud) + the connect-host wizard.
|
||||||
const [infraSel, setInfraSel] = useState<string | null>("local");
|
const [infraSel, setInfraSel] = useState<string | null>("local");
|
||||||
const [infraConnectOpen, setInfraConnectOpen] = useState(false);
|
const [infraConnectOpen, setInfraConnectOpen] = useState(false);
|
||||||
|
// When set, the infra center shows the full-page monitor for this node.
|
||||||
|
const [monitorNode, setMonitorNode] = useState<{ id: string; name: string } | null>(null);
|
||||||
|
|
||||||
const org: DemoOrg = findOrg(orgId) ?? fallbackOrg;
|
const org: DemoOrg = findOrg(orgId) ?? fallbackOrg;
|
||||||
const company: DemoCompany = findCompany(companyId) ?? org.companies[0] ?? EMPTY_ORG.companies[0];
|
const company: DemoCompany = findCompany(companyId) ?? org.companies[0] ?? EMPTY_ORG.companies[0];
|
||||||
@@ -662,7 +665,11 @@ export function Dashboard({ user, orgs, claws }: { user?: { display_name?: strin
|
|||||||
{/* Center: the operator console (stats → Tailscale → host cards) +
|
{/* Center: the operator console (stats → Tailscale → host cards) +
|
||||||
a thin status bar, condensed by the computer pull-out's width. */}
|
a thin status bar, condensed by the computer pull-out's width. */}
|
||||||
<div style={{ position: "absolute", top: 0, bottom: 0, left: 0, right: "var(--computer-width)", display: "flex", flexDirection: "column", transition: "right var(--duration-normal) var(--ease-app)" }}>
|
<div style={{ position: "absolute", top: 0, bottom: 0, left: 0, right: "var(--computer-width)", display: "flex", flexDirection: "column", transition: "right var(--duration-normal) var(--ease-app)" }}>
|
||||||
<FleetConsole view={infraSel ?? "local"} onConnectHost={() => setInfraConnectOpen(true)} />
|
{monitorNode ? (
|
||||||
|
<NodeMonitor nodeId={monitorNode.id} name={monitorNode.name} onBack={() => setMonitorNode(null)} />
|
||||||
|
) : (
|
||||||
|
<FleetConsole view={infraSel ?? "local"} onConnectHost={() => setInfraConnectOpen(true)} onMonitor={(id, name) => setMonitorNode({ id, name })} />
|
||||||
|
)}
|
||||||
<FleetStatusBar />
|
<FleetStatusBar />
|
||||||
</div>
|
</div>
|
||||||
{/* Right: the IDENTICAL computer chrome, with cloud-infra apps. */}
|
{/* Right: the IDENTICAL computer chrome, with cloud-infra apps. */}
|
||||||
|
|||||||
@@ -7,7 +7,7 @@
|
|||||||
// the connect-host wizard is a modal; both open via onConnectHost.
|
// the connect-host wizard is a modal; both open via onConnectHost.
|
||||||
|
|
||||||
import { useCallback, useState } from "react";
|
import { useCallback, useState } from "react";
|
||||||
import { Cloud, Server, Terminal } from "lucide-react";
|
import { Activity, Cloud, Server, Terminal } from "lucide-react";
|
||||||
import { useQueryStates } from "nuqs";
|
import { useQueryStates } from "nuqs";
|
||||||
|
|
||||||
import { useFetchJson } from "@/lib/api/use-fetch";
|
import { useFetchJson } from "@/lib/api/use-fetch";
|
||||||
@@ -122,7 +122,7 @@ function Mini({ label, value }: { label: string; value: string }) {
|
|||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
function HostCard({ node, onRemoved }: { node: FleetNode; onRemoved: () => void }) {
|
function HostCard({ node, onRemoved, onMonitor }: { node: FleetNode; onRemoved: () => void; onMonitor: (id: string, name: string) => void }) {
|
||||||
const [, setParams] = useQueryStates(panelParsers, { shallow: true });
|
const [, setParams] = useQueryStates(panelParsers, { shallow: true });
|
||||||
const [copied, setCopied] = useState(false);
|
const [copied, setCopied] = useState(false);
|
||||||
const h = node.health;
|
const h = node.health;
|
||||||
@@ -150,6 +150,9 @@ function HostCard({ node, onRemoved }: { node: FleetNode; onRemoved: () => void
|
|||||||
<span style={{ width: 9, height: 9, borderRadius: "50%", background: STATUS_COLOR[st], animation: st === "online" ? "cm-blink 1.6s infinite" : "none" }} />
|
<span style={{ width: 9, height: 9, borderRadius: "50%", background: STATUS_COLOR[st], animation: st === "online" ? "cm-blink 1.6s infinite" : "none" }} />
|
||||||
<span style={{ fontSize: 14, fontWeight: 700, whiteSpace: "nowrap", overflow: "hidden", textOverflow: "ellipsis" }}>{node.hostname ?? node.name}</span>
|
<span style={{ fontSize: 14, fontWeight: 700, whiteSpace: "nowrap", overflow: "hidden", textOverflow: "ellipsis" }}>{node.hostname ?? node.name}</span>
|
||||||
<span style={{ flex: 1 }} />
|
<span style={{ flex: 1 }} />
|
||||||
|
{st === "online" ? (
|
||||||
|
<button type="button" onClick={() => onMonitor(node.id, node.hostname ?? node.name)} title="Monitor" aria-label="Monitor node" style={{ width: 26, height: 26, borderRadius: 7, border: "1px solid rgba(95,208,138,.3)", background: "rgba(95,208,138,.08)", color: "#5fd08a", cursor: "pointer", display: "flex", alignItems: "center", justifyContent: "center" }}><Activity size={13} /></button>
|
||||||
|
) : null}
|
||||||
{st === "online" ? (
|
{st === "online" ? (
|
||||||
<button type="button" onClick={() => setParams({ app: "terminal", node: node.id, device: "phone" })} title="Open terminal" aria-label="Open terminal" style={{ width: 26, height: 26, borderRadius: 7, border: "1px solid rgba(94,200,216,.3)", background: "rgba(94,200,216,.08)", color: "#5ec8d8", cursor: "pointer", display: "flex", alignItems: "center", justifyContent: "center" }}><Terminal size={13} /></button>
|
<button type="button" onClick={() => setParams({ app: "terminal", node: node.id, device: "phone" })} title="Open terminal" aria-label="Open terminal" style={{ width: 26, height: 26, borderRadius: 7, border: "1px solid rgba(94,200,216,.3)", background: "rgba(94,200,216,.08)", color: "#5ec8d8", cursor: "pointer", display: "flex", alignItems: "center", justifyContent: "center" }}><Terminal size={13} /></button>
|
||||||
) : null}
|
) : null}
|
||||||
@@ -173,6 +176,14 @@ function HostCard({ node, onRemoved }: { node: FleetNode; onRemoved: () => void
|
|||||||
</>
|
</>
|
||||||
) : null}
|
) : null}
|
||||||
|
|
||||||
|
{node.gpuPct != null || node.tempMax != null ? (
|
||||||
|
<div style={{ display: "flex", gap: 6, marginBottom: 13 }}>
|
||||||
|
{node.gpuPct != null ? <Mini label="gpu" value={`${node.gpuPct.toFixed(0)}%`} /> : null}
|
||||||
|
{node.tempMax != null ? <Mini label="temp" value={`${node.tempMax.toFixed(0)}°C`} /> : null}
|
||||||
|
<button type="button" onClick={() => onMonitor(node.id, node.hostname ?? node.name)} style={{ flex: 1, textAlign: "center", padding: "7px 0", borderRadius: 8, background: "rgba(95,208,138,.06)", border: "1px solid rgba(95,208,138,.18)", color: "#5fd08a", cursor: "pointer", fontFamily: mono, fontSize: 10 }}>monitor →</button>
|
||||||
|
</div>
|
||||||
|
) : null}
|
||||||
|
|
||||||
{st === "online" && sshHost ? (
|
{st === "online" && sshHost ? (
|
||||||
<button type="button" onClick={copySsh} style={{ display: "flex", alignItems: "center", gap: 8, padding: "8px 11px", borderRadius: 8, background: "#070708", border: "1px solid rgba(255,255,255,.07)", cursor: "pointer", textAlign: "left" }}>
|
<button type="button" onClick={copySsh} style={{ display: "flex", alignItems: "center", gap: 8, padding: "8px 11px", borderRadius: 8, background: "#070708", border: "1px solid rgba(255,255,255,.07)", cursor: "pointer", textAlign: "left" }}>
|
||||||
<Server size={13} style={{ color: "#5ec8d8", flex: "none" }} />
|
<Server size={13} style={{ color: "#5ec8d8", flex: "none" }} />
|
||||||
@@ -199,7 +210,65 @@ function Stat({ label, value, tint }: { label: string; value: string; tint?: str
|
|||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
export function FleetConsole({ view, onConnectHost }: { view: string; onConnectHost: () => void }) {
|
/** Connect a workspace's Beszel hub (BYO monitoring) for rich per-node metrics. */
|
||||||
|
function BeszelSection() {
|
||||||
|
const { data, refresh } = useFetchJson<{ connected: boolean; hubUrl: string | null }>("/api/fleet/beszel");
|
||||||
|
const [hubUrl, setHubUrl] = useState("");
|
||||||
|
const [username, setUsername] = useState("");
|
||||||
|
const [password, setPassword] = useState("");
|
||||||
|
const [busy, setBusy] = useState(false);
|
||||||
|
const [err, setErr] = useState<string | null>(null);
|
||||||
|
|
||||||
|
if (data?.connected) {
|
||||||
|
return (
|
||||||
|
<div style={{ borderRadius: 14, border: "1px solid rgba(95,208,138,.18)", background: "#0c0c0f", marginBottom: 22, padding: "13px 16px", display: "flex", alignItems: "center", gap: 11 }}>
|
||||||
|
<Activity size={16} style={{ color: "#5fd08a" }} />
|
||||||
|
<div style={{ flex: 1 }}>
|
||||||
|
<div style={{ fontSize: 14, fontWeight: 700 }}>Beszel monitoring</div>
|
||||||
|
<div style={{ fontFamily: mono, fontSize: 10, color: "#6a6a72" }}>connected · {data.hubUrl}</div>
|
||||||
|
</div>
|
||||||
|
<button type="button" onClick={() => fetch("/api/fleet/beszel", { method: "DELETE" }).then(refresh)} style={{ fontFamily: mono, fontSize: 10, color: "#ff8a7a", background: "transparent", border: "1px solid rgba(255,138,122,.3)", borderRadius: 7, padding: "5px 10px", cursor: "pointer" }}>disconnect</button>
|
||||||
|
</div>
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
const connect = () => {
|
||||||
|
setBusy(true);
|
||||||
|
setErr(null);
|
||||||
|
fetch("/api/fleet/beszel", { method: "POST", headers: { "Content-Type": "application/json" }, body: JSON.stringify({ hubUrl, username, password }) })
|
||||||
|
.then((r) => r.json())
|
||||||
|
.then((d: { ok?: boolean; error?: string }) => {
|
||||||
|
if (d.ok) refresh();
|
||||||
|
else setErr(d.error ?? "failed");
|
||||||
|
})
|
||||||
|
.catch((e: Error) => setErr(e.message))
|
||||||
|
.finally(() => setBusy(false));
|
||||||
|
};
|
||||||
|
const field = (ph: string, val: string, set: (v: string) => void, type = "text") => (
|
||||||
|
<input value={val} onChange={(e) => set(e.target.value)} placeholder={ph} type={type} style={{ flex: 1, minWidth: 130, height: 32, borderRadius: 8, border: "1px solid rgba(255,255,255,.1)", background: "#0d0d10", color: "#cfcfd5", padding: "0 11px", fontFamily: mono, fontSize: 12 }} />
|
||||||
|
);
|
||||||
|
|
||||||
|
return (
|
||||||
|
<div style={{ borderRadius: 14, border: "1px solid rgba(255,255,255,.08)", background: "#0c0c0f", marginBottom: 22, padding: "13px 16px" }}>
|
||||||
|
<div style={{ display: "flex", alignItems: "center", gap: 11, marginBottom: 11 }}>
|
||||||
|
<Activity size={16} style={{ color: "#5fd08a" }} />
|
||||||
|
<div style={{ flex: 1 }}>
|
||||||
|
<div style={{ fontSize: 14, fontWeight: 700 }}>Beszel monitoring</div>
|
||||||
|
<div style={{ fontFamily: mono, fontSize: 10, color: "#6a6a72" }}>connect your hub for rich GPU / temp / disk-IO / network / container metrics</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
<div style={{ display: "flex", flexWrap: "wrap", gap: 8 }}>
|
||||||
|
{field("hub URL (http://100.x.x.x:8090)", hubUrl, setHubUrl)}
|
||||||
|
{field("username / email", username, setUsername)}
|
||||||
|
{field("password", password, setPassword, "password")}
|
||||||
|
<button type="button" onClick={connect} disabled={busy || !hubUrl || !username || !password} style={{ height: 32, padding: "0 16px", borderRadius: 8, border: 0, background: "linear-gradient(135deg,#7fe0a0,#5fd08a)", color: "#06281a", fontSize: 12.5, fontWeight: 700, cursor: busy ? "default" : "pointer", opacity: busy || !hubUrl || !username || !password ? 0.6 : 1 }}>{busy ? "connecting…" : "connect"}</button>
|
||||||
|
</div>
|
||||||
|
{err ? <div style={{ marginTop: 8, fontFamily: mono, fontSize: 11, color: "#ff8a7a" }}>{err}</div> : null}
|
||||||
|
</div>
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
export function FleetConsole({ view, onConnectHost, onMonitor }: { view: string; onConnectHost: () => void; onMonitor: (id: string, name: string) => void }) {
|
||||||
const { nodes, refresh } = useNodes();
|
const { nodes, refresh } = useNodes();
|
||||||
const { data: ts } = useFetchJson<{ connected: boolean; tailnet: string | null; devices: TsDevice[] }>("/api/fleet/tailscale/devices");
|
const { data: ts } = useFetchJson<{ connected: boolean; tailnet: string | null; devices: TsDevice[] }>("/api/fleet/tailscale/devices");
|
||||||
const online = nodes.filter(isLive);
|
const online = nodes.filter(isLive);
|
||||||
@@ -265,6 +334,8 @@ export function FleetConsole({ view, onConnectHost }: { view: string; onConnectH
|
|||||||
)}
|
)}
|
||||||
</div>
|
</div>
|
||||||
|
|
||||||
|
<BeszelSection />
|
||||||
|
|
||||||
<div style={{ display: "flex", alignItems: "center", gap: 10, marginBottom: 12 }}>
|
<div style={{ display: "flex", alignItems: "center", gap: 10, marginBottom: 12 }}>
|
||||||
<span style={{ fontFamily: mono, fontSize: 10, letterSpacing: ".12em", color: "#5a5a62" }}>LOCAL HOSTS</span>
|
<span style={{ fontFamily: mono, fontSize: 10, letterSpacing: ".12em", color: "#5a5a62" }}>LOCAL HOSTS</span>
|
||||||
<span style={{ flex: 1, height: 1, background: "rgba(255,255,255,.06)" }} />
|
<span style={{ flex: 1, height: 1, background: "rgba(255,255,255,.06)" }} />
|
||||||
@@ -272,7 +343,7 @@ export function FleetConsole({ view, onConnectHost }: { view: string; onConnectH
|
|||||||
</div>
|
</div>
|
||||||
<div style={{ display: "grid", gridTemplateColumns: "repeat(auto-fill, minmax(290px, 1fr))", gap: 14 }}>
|
<div style={{ display: "grid", gridTemplateColumns: "repeat(auto-fill, minmax(290px, 1fr))", gap: 14 }}>
|
||||||
{nodes.map((n) => (
|
{nodes.map((n) => (
|
||||||
<HostCard key={n.id} node={n} onRemoved={refresh} />
|
<HostCard key={n.id} node={n} onRemoved={refresh} onMonitor={onMonitor} />
|
||||||
))}
|
))}
|
||||||
<button type="button" onClick={onConnectHost} style={{ borderRadius: 13, border: "1.5px dashed rgba(255,255,255,.14)", background: "transparent", padding: 15, display: "flex", flexDirection: "column", alignItems: "center", justifyContent: "center", gap: 8, minHeight: 200, cursor: "pointer" }}>
|
<button type="button" onClick={onConnectHost} style={{ borderRadius: 13, border: "1.5px dashed rgba(255,255,255,.14)", background: "transparent", padding: 15, display: "flex", flexDirection: "column", alignItems: "center", justifyContent: "center", gap: 8, minHeight: 200, cursor: "pointer" }}>
|
||||||
<span style={{ width: 38, height: 38, borderRadius: 11, background: "rgba(255,111,97,.12)", display: "flex", alignItems: "center", justifyContent: "center", color: "#ff6f61", fontSize: 22, fontWeight: 300 }}>+</span>
|
<span style={{ width: 38, height: 38, borderRadius: 11, background: "rgba(255,111,97,.12)", display: "flex", alignItems: "center", justifyContent: "center", color: "#ff6f61", fontSize: 22, fontWeight: 300 }}>+</span>
|
||||||
|
|||||||
@@ -39,6 +39,9 @@ export interface FleetNode {
|
|||||||
lastSeen: number | null;
|
lastSeen: number | null;
|
||||||
createdAt: number;
|
createdAt: number;
|
||||||
health: NodeHealth | null;
|
health: NodeHealth | null;
|
||||||
|
/** Latest Beszel metrics (rich bits beyond basic health). Null until tapped. */
|
||||||
|
gpuPct: number | null;
|
||||||
|
tempMax: number | null;
|
||||||
}
|
}
|
||||||
|
|
||||||
const STATUS_COLOR: Record<FleetNode["status"], string> = {
|
const STATUS_COLOR: Record<FleetNode["status"], string> = {
|
||||||
|
|||||||
@@ -0,0 +1,135 @@
|
|||||||
|
"use client";
|
||||||
|
|
||||||
|
// Full-page per-node monitor: current metrics + recent 1-minute history charts,
|
||||||
|
// tapped from the workspace's Beszel hub (CPU, memory, disk, network, temps, GPU,
|
||||||
|
// per-container). Lives full-width in the infra center.
|
||||||
|
|
||||||
|
import { ArrowLeft, Cpu, HardDrive, MemoryStick, Network, Thermometer, Zap } from "lucide-react";
|
||||||
|
import { type ReactNode, useCallback, useEffect, useState } from "react";
|
||||||
|
|
||||||
|
const mono = "'JetBrains Mono', ui-monospace, monospace";
|
||||||
|
|
||||||
|
interface StatsRow {
|
||||||
|
created: string;
|
||||||
|
stats: Record<string, unknown>;
|
||||||
|
}
|
||||||
|
interface MetricsResp {
|
||||||
|
latest: { name?: string; status?: string; info?: Record<string, unknown>; updatedAt?: number } | null;
|
||||||
|
history: StatsRow[];
|
||||||
|
}
|
||||||
|
|
||||||
|
const num = (v: unknown): number | null => (typeof v === "number" && Number.isFinite(v) ? v : null);
|
||||||
|
/** GPU usage = max `u` across the per-gpu map; temps = max across the temp map. */
|
||||||
|
function mapMax(m: unknown, key?: string): number | null {
|
||||||
|
if (!m || typeof m !== "object") return null;
|
||||||
|
let max: number | null = null;
|
||||||
|
for (const v of Object.values(m as Record<string, unknown>)) {
|
||||||
|
const n = key ? num((v as Record<string, unknown>)?.[key]) : num(v);
|
||||||
|
if (n != null) max = max == null ? n : Math.max(max, n);
|
||||||
|
}
|
||||||
|
return max;
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Compact inline line chart (SVG) over a numeric series, oldest→newest. */
|
||||||
|
function Line({ series, color, unit = "%", max }: { series: (number | null)[]; color: string; unit?: string; max?: number }) {
|
||||||
|
const vals = series.filter((v): v is number => v != null);
|
||||||
|
const hi = max ?? (vals.length ? Math.max(...vals, 1) : 1);
|
||||||
|
const w = 100;
|
||||||
|
const h = 34;
|
||||||
|
const pts = series
|
||||||
|
.map((v, i) => (v == null ? null : `${(i / Math.max(1, series.length - 1)) * w},${h - (v / hi) * h}`))
|
||||||
|
.filter(Boolean)
|
||||||
|
.join(" ");
|
||||||
|
const last = vals.length ? vals[vals.length - 1] : null;
|
||||||
|
return (
|
||||||
|
<div style={{ position: "relative" }}>
|
||||||
|
<svg viewBox={`0 0 ${w} ${h}`} preserveAspectRatio="none" style={{ width: "100%", height: 40, display: "block" }}>
|
||||||
|
<polyline points={pts} fill="none" stroke={color} strokeWidth={1.4} vectorEffect="non-scaling-stroke" />
|
||||||
|
</svg>
|
||||||
|
<span style={{ position: "absolute", top: -2, right: 0, fontFamily: mono, fontSize: 11, fontWeight: 700, color }}>
|
||||||
|
{last == null ? "—" : `${last.toFixed(unit === "%" ? 0 : 1)}${unit}`}
|
||||||
|
</span>
|
||||||
|
</div>
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
function Chart({ icon, label, series, color, unit, max }: { icon: ReactNode; label: string; series: (number | null)[]; color: string; unit?: string; max?: number }) {
|
||||||
|
return (
|
||||||
|
<div style={{ borderRadius: 12, border: "1px solid rgba(255,255,255,.07)", background: "#0d0d10", padding: "12px 14px" }}>
|
||||||
|
<div style={{ display: "flex", alignItems: "center", gap: 7, marginBottom: 8, fontFamily: mono, fontSize: 10, letterSpacing: ".1em", color: "#8a8a92" }}>
|
||||||
|
<span style={{ color }}>{icon}</span>
|
||||||
|
{label}
|
||||||
|
</div>
|
||||||
|
<Line series={series} color={color} unit={unit} max={max} />
|
||||||
|
</div>
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
export function NodeMonitor({ nodeId, name, onBack }: { nodeId: string; name: string; onBack: () => void }) {
|
||||||
|
const [data, setData] = useState<MetricsResp | null>(null);
|
||||||
|
const [err, setErr] = useState<string | null>(null);
|
||||||
|
const load = useCallback(() => {
|
||||||
|
fetch(`/api/nodes/${nodeId}/metrics`)
|
||||||
|
.then((r) => (r.ok ? r.json() : Promise.reject(new Error(`${r.status}`))))
|
||||||
|
.then((d: MetricsResp) => { setData(d); setErr(null); })
|
||||||
|
.catch((e: Error) => setErr(e.message));
|
||||||
|
}, [nodeId]);
|
||||||
|
useEffect(() => {
|
||||||
|
load();
|
||||||
|
const t = setInterval(load, 15000);
|
||||||
|
return () => clearInterval(t);
|
||||||
|
}, [load]);
|
||||||
|
|
||||||
|
// history is newest-first; reverse for left→right time.
|
||||||
|
const hist = [...(data?.history ?? [])].reverse();
|
||||||
|
const ser = (pick: (s: Record<string, unknown>) => number | null) => hist.map((r) => pick(r.stats));
|
||||||
|
const info = data?.latest?.info ?? {};
|
||||||
|
const connected = !!data?.latest;
|
||||||
|
|
||||||
|
const big = (label: string, v: number | null, unit: string, color: string) => (
|
||||||
|
<div style={{ borderRadius: 12, border: `1px solid ${color}30`, background: "#0d0d10", padding: "13px 16px", minWidth: 110 }}>
|
||||||
|
<div style={{ fontFamily: mono, fontSize: 9, letterSpacing: ".1em", color: "#6a6a72" }}>{label}</div>
|
||||||
|
<div style={{ fontSize: 24, fontWeight: 700, marginTop: 3, color }}>{v == null ? "—" : `${v.toFixed(0)}${unit}`}</div>
|
||||||
|
</div>
|
||||||
|
);
|
||||||
|
|
||||||
|
return (
|
||||||
|
<div style={{ flex: 1, minHeight: 0, overflowY: "auto", padding: "20px 26px 30px" }}>
|
||||||
|
<div style={{ display: "flex", alignItems: "center", gap: 12, marginBottom: 18 }}>
|
||||||
|
<button type="button" onClick={onBack} style={{ display: "inline-flex", alignItems: "center", gap: 6, fontFamily: mono, fontSize: 11, color: "#9a9aa2", background: "transparent", border: "1px solid rgba(255,255,255,.12)", borderRadius: 8, padding: "6px 11px", cursor: "pointer" }}>
|
||||||
|
<ArrowLeft size={13} /> Fleet
|
||||||
|
</button>
|
||||||
|
<div>
|
||||||
|
<div style={{ fontFamily: mono, fontSize: 10, letterSpacing: ".14em", color: "#5a5a62" }}>MONITORING</div>
|
||||||
|
<div style={{ fontSize: 22, fontWeight: 700, letterSpacing: "-.02em" }}>{name}</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
{!connected ? (
|
||||||
|
<div style={{ borderRadius: 12, border: "1px solid rgba(232,196,106,.2)", background: "rgba(232,196,106,.05)", padding: 16, fontSize: 13, color: "#e8b465" }}>
|
||||||
|
{err ? `Could not load metrics (${err}).` : "No metrics yet — connect your Beszel hub under Local & Tailscale, and confirm this node's name matches its Beszel system."}
|
||||||
|
</div>
|
||||||
|
) : (
|
||||||
|
<>
|
||||||
|
<div style={{ display: "flex", flexWrap: "wrap", gap: 10, marginBottom: 22 }}>
|
||||||
|
{big("CPU", num(info.cpu), "%", "#5ec8d8")}
|
||||||
|
{big("MEMORY", num(info.mp), "%", "#7c9cff")}
|
||||||
|
{big("DISK", num(info.dp), "%", "#b07cff")}
|
||||||
|
{info.g != null ? big("GPU", num(info.g), "%", "#5fd08a") : null}
|
||||||
|
{info.dt != null ? big("TEMP", num(info.dt), "°C", "#e8b465") : null}
|
||||||
|
{Array.isArray(info.la) ? big("LOAD", num((info.la as number[])[0]), "", "#ff8a7a") : null}
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<div style={{ display: "grid", gridTemplateColumns: "repeat(auto-fill, minmax(240px, 1fr))", gap: 12 }}>
|
||||||
|
<Chart icon={<Cpu size={13} />} label="CPU %" color="#5ec8d8" series={ser((s) => num(s.cpu))} max={100} />
|
||||||
|
<Chart icon={<MemoryStick size={13} />} label="MEMORY %" color="#7c9cff" series={ser((s) => num(s.mp))} max={100} />
|
||||||
|
<Chart icon={<Zap size={13} />} label="GPU %" color="#5fd08a" series={ser((s) => mapMax(s.g, "u"))} max={100} />
|
||||||
|
<Chart icon={<Thermometer size={13} />} label="TEMP °C" color="#e8b465" unit="°" series={ser((s) => mapMax(s.t))} />
|
||||||
|
<Chart icon={<Network size={13} />} label="NET ↑ MB/s" color="#5fd08a" unit="" series={ser((s) => { const v = num(s.ns); return v == null ? null : v / 1e6; })} />
|
||||||
|
<Chart icon={<HardDrive size={13} />} label="DISK R MB/s" color="#b07cff" unit="" series={ser((s) => { const v = num(s.dr); return v == null ? null : v / 1e6; })} />
|
||||||
|
</div>
|
||||||
|
</>
|
||||||
|
)}
|
||||||
|
</div>
|
||||||
|
);
|
||||||
|
}
|
||||||
@@ -0,0 +1,11 @@
|
|||||||
|
-- Bring-your-own Beszel hub: a workspace connects its Beszel monitoring hub so we
|
||||||
|
-- read rich per-node metrics (GPU / temps / disk-IO / network / per-container) via
|
||||||
|
-- its PocketBase API. Credentials are used server-side only (never returned to the
|
||||||
|
-- client). TODO: move credentials into the cm-secrets broker store (like Tailscale).
|
||||||
|
CREATE TABLE workspace_beszel (
|
||||||
|
workspace_id UUID PRIMARY KEY REFERENCES workspaces (id) ON DELETE CASCADE,
|
||||||
|
hub_url TEXT NOT NULL,
|
||||||
|
username TEXT NOT NULL,
|
||||||
|
password TEXT NOT NULL,
|
||||||
|
connected_at TIMESTAMPTZ NOT NULL DEFAULT now()
|
||||||
|
);
|
||||||
@@ -0,0 +1,21 @@
|
|||||||
|
-- Rich per-node metrics tapped from the workspace's Beszel hub (beyond the basic
|
||||||
|
-- heartbeat health). One latest snapshot per node: queryable scalar columns drive
|
||||||
|
-- the fleet card widgets + the rules engine; the JSONB `data` holds the full
|
||||||
|
-- snapshot (per-core CPU, per-GPU, per-interface network, per-container) for the
|
||||||
|
-- per-node monitor page. History for charts is proxied live from the hub.
|
||||||
|
CREATE TABLE node_metrics (
|
||||||
|
node_id UUID PRIMARY KEY REFERENCES nodes (id) ON DELETE CASCADE,
|
||||||
|
updated_at TIMESTAMPTZ NOT NULL DEFAULT now(),
|
||||||
|
cpu_pct DOUBLE PRECISION,
|
||||||
|
mem_pct DOUBLE PRECISION,
|
||||||
|
disk_pct DOUBLE PRECISION,
|
||||||
|
gpu_pct DOUBLE PRECISION,
|
||||||
|
temp_max DOUBLE PRECISION,
|
||||||
|
net_sent_ps BIGINT,
|
||||||
|
net_recv_ps BIGINT,
|
||||||
|
disk_read_ps BIGINT,
|
||||||
|
disk_write_ps BIGINT,
|
||||||
|
load1 DOUBLE PRECISION,
|
||||||
|
container_count INTEGER,
|
||||||
|
data JSONB NOT NULL DEFAULT '{}'::jsonb
|
||||||
|
);
|
||||||
Reference in New Issue
Block a user