feat(T3.3): cpu load readout + lifeguard health score

rpc.rs: Add read_cpu_load_1m() (/proc/loadavg), compute_health_score()
(hot>90% +1, fs>90% +1, load>4 +1, failed timers +1), expose both in
DashboardStatusReply. NodeStatusV2 and api.ts propagated.

NodeCard: cpu load row in facts grid (amber when >4), healthOf() now
driven by server-side health_score (>= 3 → err, 1-2 → warn) with
client-side fs/timer fallback for older daemons.

Co-Authored-By: Claude Sonnet 4.6 <[email protected]>
This commit is contained in:
Omar Sobh
2026-07-23 17:45:25 +00:00
co-authored by Claude Sonnet 4.6
parent 70ad863006
commit 01ba85b2dd
4 changed files with 88 additions and 0 deletions
+59
View File
@@ -370,6 +370,14 @@ pub struct DashboardStatusReply {
/// reset — proxy for when the daemon last started/restarted. /// reset — proxy for when the daemon last started/restarted.
#[serde(default)] #[serde(default)]
pub daemon_started_unix: Option<u64>, pub daemon_started_unix: Option<u64>,
/// 1-minute load average from /proc/loadavg. None on non-Linux.
#[serde(default)]
pub cpu_load_1m: Option<f32>,
/// Composite Lifeguard-style health score (T3.3).
/// 0 = healthy; 1-2 = degraded; 3+ = stressed.
/// Increments for: hot >90%, fs >90%, load >4.0, failed timers.
#[serde(default)]
pub health_score: u8,
} }
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] #[derive(Debug, Clone, Serialize, Deserialize, PartialEq)]
@@ -715,6 +723,31 @@ fn timer_status(unit: &str) -> TimerStatus {
} }
} }
/// 1-minute load average from /proc/loadavg. None on non-Linux or read error.
fn read_cpu_load_1m() -> Option<f32> {
std::fs::read_to_string("/proc/loadavg")
.ok()
.and_then(|s| s.split_whitespace().next()?.parse::<f32>().ok())
}
/// Composite Lifeguard-style health score: 0 = healthy, higher = stressed.
/// Increments for: hot tier >90%, filesystem >90%, cpu load >4.0, failed timers.
fn compute_health_score(
hot_pct: f64,
fs_pct: f64,
cpu_load: Option<f32>,
failed_timers: usize,
) -> u8 {
let mut score: u8 = 0;
if hot_pct > 0.90 { score = score.saturating_add(1); }
if fs_pct > 0.90 { score = score.saturating_add(1); }
if cpu_load.map(|l| l > 4.0).unwrap_or(false) {
score = score.saturating_add(1);
}
if failed_timers > 0 { score = score.saturating_add(1); }
score
}
/// Is `path` currently a mount point? Cheap Linux check: read /// Is `path` currently a mount point? Cheap Linux check: read
/// /proc/mounts. On macOS returns None (aggregator doesn't run /// /proc/mounts. On macOS returns None (aggregator doesn't run
/// mounts). /// mounts).
@@ -1047,6 +1080,30 @@ impl RpcRouter {
timer_status("clawstor-ref-sweep.timer"), timer_status("clawstor-ref-sweep.timer"),
timer_status("clawstor-snapshot-rotate.timer"), timer_status("clawstor-snapshot-rotate.timer"),
]; ];
// Lifeguard-style health score (T3.3).
let cpu_load_1m = read_cpu_load_1m();
let hot_pct = if hot_max > 0 {
hot_used as f64 / hot_max as f64
} else {
0.0
};
let fs_pct = filesystem.as_ref().map(|f| {
if f.total_bytes > 0 {
f.used_bytes as f64 / f.total_bytes as f64
} else {
0.0
}
}).unwrap_or(0.0);
let failed_timers = timers
.iter()
.filter(|t| {
t.last_result
.as_deref()
.map(|r| r != "success")
.unwrap_or(false)
})
.count();
let health_score = compute_health_score(hot_pct, fs_pct, cpu_load_1m, failed_timers);
let reply = DashboardStatusReply { let reply = DashboardStatusReply {
node_name: self.local_name.clone(), node_name: self.local_name.clone(),
zone: self.local_zone.clone(), zone: self.local_zone.clone(),
@@ -1064,6 +1121,8 @@ impl RpcRouter {
cache, cache,
timers, timers,
daemon_started_unix, daemon_started_unix,
cpu_load_1m,
health_score,
}; };
let json = serde_json::to_vec(&reply) let json = serde_json::to_vec(&reply)
.context("encoding DashboardStatusReply as JSON")?; .context("encoding DashboardStatusReply as JSON")?;
+10
View File
@@ -368,6 +368,12 @@ pub struct NodeStatusV2 {
/// Unix timestamp (seconds) when the peer's daemon last started. /// Unix timestamp (seconds) when the peer's daemon last started.
#[serde(default)] #[serde(default)]
pub daemon_started_unix: Option<u64>, pub daemon_started_unix: Option<u64>,
/// 1-minute load average. None on non-Linux or when the peer is offline.
#[serde(default)]
pub cpu_load_1m: Option<f32>,
/// Lifeguard-style health score: 0 = healthy, higher = stressed (T3.3).
#[serde(default)]
pub health_score: u8,
/// `true` when the aggregator successfully talked to the peer; /// `true` when the aggregator successfully talked to the peer;
/// `false` when the RPC failed. Frontend uses this to badge the /// `false` when the RPC failed. Frontend uses this to badge the
/// card as offline. /// card as offline.
@@ -395,6 +401,8 @@ impl NodeStatusV2 {
cache: r.cache, cache: r.cache,
timers: r.timers, timers: r.timers,
daemon_started_unix: r.daemon_started_unix, daemon_started_unix: r.daemon_started_unix,
cpu_load_1m: r.cpu_load_1m,
health_score: r.health_score,
online: true, online: true,
error: None, error: None,
}; };
@@ -421,6 +429,8 @@ impl NodeStatusV2 {
cache: None, cache: None,
timers: Vec::new(), timers: Vec::new(),
daemon_started_unix: None, daemon_started_unix: None,
cpu_load_1m: None,
health_score: 0,
online: false, online: false,
error: Some(e), error: Some(e),
} }
+15
View File
@@ -118,6 +118,17 @@ export function NodeCard({ node, anomalyLevel }: Props) {
</span> </span>
</> </>
)} )}
{node.cpu_load_1m != null && (
<>
<span className="text-slate-500">cpu load</span>
<span className={[
'text-right font-mono text-xs',
node.cpu_load_1m > 4 ? 'text-amber-300' : 'text-slate-300',
].join(' ')}>
{node.cpu_load_1m.toFixed(2)}
</span>
</>
)}
<span className="text-slate-500">next scheduled job</span> <span className="text-slate-500">next scheduled job</span>
<span className="text-right font-mono text-xs"> <span className="text-right font-mono text-xs">
{nextTimer(node)} {nextTimer(node)}
@@ -165,6 +176,10 @@ function failedTimers(n: NodeStatusV2) {
function healthOf(n: NodeStatusV2): 'ok' | 'warn' | 'err' | 'idle' { function healthOf(n: NodeStatusV2): 'ok' | 'warn' | 'err' | 'idle' {
if (!n.online) return 'err'; if (!n.online) return 'err';
// Server-side health_score already accounts for hot/fs/cpu/timers.
if (n.health_score >= 3) return 'err';
if (n.health_score >= 1) return 'warn';
// Client-side fallback for older daemons that don't send health_score.
const fsPct = n.filesystem const fsPct = n.filesystem
? n.filesystem.used_bytes / Math.max(n.filesystem.total_bytes, 1) ? n.filesystem.used_bytes / Math.max(n.filesystem.total_bytes, 1)
: 0; : 0;
+4
View File
@@ -59,6 +59,10 @@ export interface NodeStatusV2 {
timers: TimerStatus[]; timers: TimerStatus[];
/** Unix timestamp (seconds) when the daemon last started. */ /** Unix timestamp (seconds) when the daemon last started. */
daemon_started_unix: number | null; daemon_started_unix: number | null;
/** 1-minute load average from /proc/loadavg. Null on non-Linux. */
cpu_load_1m: number | null;
/** Lifeguard health score: 0=healthy, 1-2=degraded, 3+=stressed. */
health_score: number;
online: boolean; online: boolean;
error: string | null; error: string | null;
} }