feat(T3.3): cpu load readout + lifeguard health score
rpc.rs: Add read_cpu_load_1m() (/proc/loadavg), compute_health_score() (hot>90% +1, fs>90% +1, load>4 +1, failed timers +1), expose both in DashboardStatusReply. NodeStatusV2 and api.ts propagated. NodeCard: cpu load row in facts grid (amber when >4), healthOf() now driven by server-side health_score (>= 3 → err, 1-2 → warn) with client-side fs/timer fallback for older daemons. Co-Authored-By: Claude Sonnet 4.6 <[email protected]>
This commit is contained in:
co-authored by
Claude Sonnet 4.6
parent
70ad863006
commit
01ba85b2dd
@@ -370,6 +370,14 @@ pub struct DashboardStatusReply {
|
|||||||
/// reset — proxy for when the daemon last started/restarted.
|
/// reset — proxy for when the daemon last started/restarted.
|
||||||
#[serde(default)]
|
#[serde(default)]
|
||||||
pub daemon_started_unix: Option<u64>,
|
pub daemon_started_unix: Option<u64>,
|
||||||
|
/// 1-minute load average from /proc/loadavg. None on non-Linux.
|
||||||
|
#[serde(default)]
|
||||||
|
pub cpu_load_1m: Option<f32>,
|
||||||
|
/// Composite Lifeguard-style health score (T3.3).
|
||||||
|
/// 0 = healthy; 1-2 = degraded; 3+ = stressed.
|
||||||
|
/// Increments for: hot >90%, fs >90%, load >4.0, failed timers.
|
||||||
|
#[serde(default)]
|
||||||
|
pub health_score: u8,
|
||||||
}
|
}
|
||||||
|
|
||||||
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)]
|
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)]
|
||||||
@@ -715,6 +723,31 @@ fn timer_status(unit: &str) -> TimerStatus {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// 1-minute load average from /proc/loadavg. None on non-Linux or read error.
|
||||||
|
fn read_cpu_load_1m() -> Option<f32> {
|
||||||
|
std::fs::read_to_string("/proc/loadavg")
|
||||||
|
.ok()
|
||||||
|
.and_then(|s| s.split_whitespace().next()?.parse::<f32>().ok())
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Composite Lifeguard-style health score: 0 = healthy, higher = stressed.
|
||||||
|
/// Increments for: hot tier >90%, filesystem >90%, cpu load >4.0, failed timers.
|
||||||
|
fn compute_health_score(
|
||||||
|
hot_pct: f64,
|
||||||
|
fs_pct: f64,
|
||||||
|
cpu_load: Option<f32>,
|
||||||
|
failed_timers: usize,
|
||||||
|
) -> u8 {
|
||||||
|
let mut score: u8 = 0;
|
||||||
|
if hot_pct > 0.90 { score = score.saturating_add(1); }
|
||||||
|
if fs_pct > 0.90 { score = score.saturating_add(1); }
|
||||||
|
if cpu_load.map(|l| l > 4.0).unwrap_or(false) {
|
||||||
|
score = score.saturating_add(1);
|
||||||
|
}
|
||||||
|
if failed_timers > 0 { score = score.saturating_add(1); }
|
||||||
|
score
|
||||||
|
}
|
||||||
|
|
||||||
/// Is `path` currently a mount point? Cheap Linux check: read
|
/// Is `path` currently a mount point? Cheap Linux check: read
|
||||||
/// /proc/mounts. On macOS returns None (aggregator doesn't run
|
/// /proc/mounts. On macOS returns None (aggregator doesn't run
|
||||||
/// mounts).
|
/// mounts).
|
||||||
@@ -1047,6 +1080,30 @@ impl RpcRouter {
|
|||||||
timer_status("clawstor-ref-sweep.timer"),
|
timer_status("clawstor-ref-sweep.timer"),
|
||||||
timer_status("clawstor-snapshot-rotate.timer"),
|
timer_status("clawstor-snapshot-rotate.timer"),
|
||||||
];
|
];
|
||||||
|
// Lifeguard-style health score (T3.3).
|
||||||
|
let cpu_load_1m = read_cpu_load_1m();
|
||||||
|
let hot_pct = if hot_max > 0 {
|
||||||
|
hot_used as f64 / hot_max as f64
|
||||||
|
} else {
|
||||||
|
0.0
|
||||||
|
};
|
||||||
|
let fs_pct = filesystem.as_ref().map(|f| {
|
||||||
|
if f.total_bytes > 0 {
|
||||||
|
f.used_bytes as f64 / f.total_bytes as f64
|
||||||
|
} else {
|
||||||
|
0.0
|
||||||
|
}
|
||||||
|
}).unwrap_or(0.0);
|
||||||
|
let failed_timers = timers
|
||||||
|
.iter()
|
||||||
|
.filter(|t| {
|
||||||
|
t.last_result
|
||||||
|
.as_deref()
|
||||||
|
.map(|r| r != "success")
|
||||||
|
.unwrap_or(false)
|
||||||
|
})
|
||||||
|
.count();
|
||||||
|
let health_score = compute_health_score(hot_pct, fs_pct, cpu_load_1m, failed_timers);
|
||||||
let reply = DashboardStatusReply {
|
let reply = DashboardStatusReply {
|
||||||
node_name: self.local_name.clone(),
|
node_name: self.local_name.clone(),
|
||||||
zone: self.local_zone.clone(),
|
zone: self.local_zone.clone(),
|
||||||
@@ -1064,6 +1121,8 @@ impl RpcRouter {
|
|||||||
cache,
|
cache,
|
||||||
timers,
|
timers,
|
||||||
daemon_started_unix,
|
daemon_started_unix,
|
||||||
|
cpu_load_1m,
|
||||||
|
health_score,
|
||||||
};
|
};
|
||||||
let json = serde_json::to_vec(&reply)
|
let json = serde_json::to_vec(&reply)
|
||||||
.context("encoding DashboardStatusReply as JSON")?;
|
.context("encoding DashboardStatusReply as JSON")?;
|
||||||
|
|||||||
@@ -368,6 +368,12 @@ pub struct NodeStatusV2 {
|
|||||||
/// Unix timestamp (seconds) when the peer's daemon last started.
|
/// Unix timestamp (seconds) when the peer's daemon last started.
|
||||||
#[serde(default)]
|
#[serde(default)]
|
||||||
pub daemon_started_unix: Option<u64>,
|
pub daemon_started_unix: Option<u64>,
|
||||||
|
/// 1-minute load average. None on non-Linux or when the peer is offline.
|
||||||
|
#[serde(default)]
|
||||||
|
pub cpu_load_1m: Option<f32>,
|
||||||
|
/// Lifeguard-style health score: 0 = healthy, higher = stressed (T3.3).
|
||||||
|
#[serde(default)]
|
||||||
|
pub health_score: u8,
|
||||||
/// `true` when the aggregator successfully talked to the peer;
|
/// `true` when the aggregator successfully talked to the peer;
|
||||||
/// `false` when the RPC failed. Frontend uses this to badge the
|
/// `false` when the RPC failed. Frontend uses this to badge the
|
||||||
/// card as offline.
|
/// card as offline.
|
||||||
@@ -395,6 +401,8 @@ impl NodeStatusV2 {
|
|||||||
cache: r.cache,
|
cache: r.cache,
|
||||||
timers: r.timers,
|
timers: r.timers,
|
||||||
daemon_started_unix: r.daemon_started_unix,
|
daemon_started_unix: r.daemon_started_unix,
|
||||||
|
cpu_load_1m: r.cpu_load_1m,
|
||||||
|
health_score: r.health_score,
|
||||||
online: true,
|
online: true,
|
||||||
error: None,
|
error: None,
|
||||||
};
|
};
|
||||||
@@ -421,6 +429,8 @@ impl NodeStatusV2 {
|
|||||||
cache: None,
|
cache: None,
|
||||||
timers: Vec::new(),
|
timers: Vec::new(),
|
||||||
daemon_started_unix: None,
|
daemon_started_unix: None,
|
||||||
|
cpu_load_1m: None,
|
||||||
|
health_score: 0,
|
||||||
online: false,
|
online: false,
|
||||||
error: Some(e),
|
error: Some(e),
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -118,6 +118,17 @@ export function NodeCard({ node, anomalyLevel }: Props) {
|
|||||||
</span>
|
</span>
|
||||||
</>
|
</>
|
||||||
)}
|
)}
|
||||||
|
{node.cpu_load_1m != null && (
|
||||||
|
<>
|
||||||
|
<span className="text-slate-500">cpu load</span>
|
||||||
|
<span className={[
|
||||||
|
'text-right font-mono text-xs',
|
||||||
|
node.cpu_load_1m > 4 ? 'text-amber-300' : 'text-slate-300',
|
||||||
|
].join(' ')}>
|
||||||
|
{node.cpu_load_1m.toFixed(2)}
|
||||||
|
</span>
|
||||||
|
</>
|
||||||
|
)}
|
||||||
<span className="text-slate-500">next scheduled job</span>
|
<span className="text-slate-500">next scheduled job</span>
|
||||||
<span className="text-right font-mono text-xs">
|
<span className="text-right font-mono text-xs">
|
||||||
{nextTimer(node)}
|
{nextTimer(node)}
|
||||||
@@ -165,6 +176,10 @@ function failedTimers(n: NodeStatusV2) {
|
|||||||
|
|
||||||
function healthOf(n: NodeStatusV2): 'ok' | 'warn' | 'err' | 'idle' {
|
function healthOf(n: NodeStatusV2): 'ok' | 'warn' | 'err' | 'idle' {
|
||||||
if (!n.online) return 'err';
|
if (!n.online) return 'err';
|
||||||
|
// Server-side health_score already accounts for hot/fs/cpu/timers.
|
||||||
|
if (n.health_score >= 3) return 'err';
|
||||||
|
if (n.health_score >= 1) return 'warn';
|
||||||
|
// Client-side fallback for older daemons that don't send health_score.
|
||||||
const fsPct = n.filesystem
|
const fsPct = n.filesystem
|
||||||
? n.filesystem.used_bytes / Math.max(n.filesystem.total_bytes, 1)
|
? n.filesystem.used_bytes / Math.max(n.filesystem.total_bytes, 1)
|
||||||
: 0;
|
: 0;
|
||||||
|
|||||||
@@ -59,6 +59,10 @@ export interface NodeStatusV2 {
|
|||||||
timers: TimerStatus[];
|
timers: TimerStatus[];
|
||||||
/** Unix timestamp (seconds) when the daemon last started. */
|
/** Unix timestamp (seconds) when the daemon last started. */
|
||||||
daemon_started_unix: number | null;
|
daemon_started_unix: number | null;
|
||||||
|
/** 1-minute load average from /proc/loadavg. Null on non-Linux. */
|
||||||
|
cpu_load_1m: number | null;
|
||||||
|
/** Lifeguard health score: 0=healthy, 1-2=degraded, 3+=stressed. */
|
||||||
|
health_score: number;
|
||||||
online: boolean;
|
online: boolean;
|
||||||
error: string | null;
|
error: string | null;
|
||||||
}
|
}
|
||||||
|
|||||||
Reference in New Issue
Block a user