Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
425ce58127 | ||
|
|
5e7f7dfc83 | ||
|
|
5001ac233c | ||
|
|
a8f0aa911c | ||
|
|
5a44bfb443 | ||
|
|
8a70588fee | ||
|
|
01ba85b2dd | ||
|
|
70ad863006 | ||
|
|
3fdf46d416 | ||
|
|
a3efdbfc04 |
@@ -703,6 +703,90 @@ impl BlobStore {
|
|||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// T3.4: pollution-score eviction.
|
||||||
|
///
|
||||||
|
/// Evicts blobs using `score = ln(chunk_count+1) * ln(age_secs+1) / (hit_count+1)`.
|
||||||
|
/// Highest-scoring (large, old, cold) blobs are evicted first. Falls back to
|
||||||
|
/// LRU ordering for blobs not in `hit_map` (treated as hit_count=0).
|
||||||
|
pub async fn evict_to_size_cap_with_scores(
|
||||||
|
&self,
|
||||||
|
max_bytes: u64,
|
||||||
|
pinned_blobs: &std::collections::HashSet<BlobId>,
|
||||||
|
hit_map: &std::collections::HashMap<BlobId, u64>,
|
||||||
|
) -> Result<GcReport> {
|
||||||
|
let manifest_summaries = self.collect_manifest_summaries().await?;
|
||||||
|
let mut referenced: std::collections::HashMap<ChunkHash, u32> =
|
||||||
|
std::collections::HashMap::new();
|
||||||
|
for summary in &manifest_summaries {
|
||||||
|
for hash in &summary.chunks {
|
||||||
|
*referenced.entry(*hash).or_insert(0) += 1;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
let mut current_size: u64 = 0;
|
||||||
|
for hash in referenced.keys() {
|
||||||
|
if let Ok(meta) = tokio::fs::metadata(&self.chunk_path(hash)).await {
|
||||||
|
current_size = current_size.saturating_add(meta.len());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
let now_unix = std::time::SystemTime::now()
|
||||||
|
.duration_since(std::time::UNIX_EPOCH)
|
||||||
|
.map(|d| d.as_secs())
|
||||||
|
.unwrap_or(0);
|
||||||
|
|
||||||
|
let mut summaries = manifest_summaries;
|
||||||
|
// Sort by pollution eviction score descending (evict highest first).
|
||||||
|
summaries.sort_by(|a, b| {
|
||||||
|
let score_of = |s: &ManifestSummary| {
|
||||||
|
let hit_count = hit_map.get(&s.blob_id).copied().unwrap_or(0);
|
||||||
|
let age = now_unix.saturating_sub(s.manifest_mtime);
|
||||||
|
let size_f = (s.chunks.len() as f64 + 1.0).ln();
|
||||||
|
let age_f = (age as f64 + 1.0).ln();
|
||||||
|
size_f * age_f / (hit_count as f64 + 1.0)
|
||||||
|
};
|
||||||
|
score_of(b)
|
||||||
|
.partial_cmp(&score_of(a))
|
||||||
|
.unwrap_or(std::cmp::Ordering::Equal)
|
||||||
|
});
|
||||||
|
|
||||||
|
let mut chunks_removed = 0usize;
|
||||||
|
let mut bytes_reclaimed = 0u64;
|
||||||
|
|
||||||
|
for summary in summaries {
|
||||||
|
if current_size <= max_bytes {
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
if pinned_blobs.contains(&summary.blob_id) {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
self.delete_manifest(&summary.blob_id).await?;
|
||||||
|
for hash in &summary.chunks {
|
||||||
|
let entry = referenced.entry(*hash).or_insert(0);
|
||||||
|
if *entry > 0 {
|
||||||
|
*entry -= 1;
|
||||||
|
}
|
||||||
|
if *entry == 0 {
|
||||||
|
let path = self.chunk_path(hash);
|
||||||
|
if let Ok(meta) = tokio::fs::metadata(&path).await {
|
||||||
|
let sz = meta.len();
|
||||||
|
if tokio::fs::remove_file(&path).await.is_ok() {
|
||||||
|
chunks_removed += 1;
|
||||||
|
bytes_reclaimed = bytes_reclaimed.saturating_add(sz);
|
||||||
|
current_size = current_size.saturating_sub(sz);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
referenced.remove(hash);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
Ok(GcReport {
|
||||||
|
chunks_scanned: 0,
|
||||||
|
chunks_removed,
|
||||||
|
bytes_reclaimed,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
/// Enumerate every on-disk manifest with the info eviction needs:
|
/// Enumerate every on-disk manifest with the info eviction needs:
|
||||||
/// blob_id, chunk set, and mtime for LRU ordering. Bounded by the
|
/// blob_id, chunk set, and mtime for LRU ordering. Bounded by the
|
||||||
/// number of manifests (small — one per cached target dir).
|
/// number of manifests (small — one per cached target dir).
|
||||||
|
|||||||
@@ -145,13 +145,15 @@ impl FingerprintInputs {
|
|||||||
/// inputs, order-independent for features (already sorted), and
|
/// inputs, order-independent for features (already sorted), and
|
||||||
/// null-byte-separated so `["ab","c"]` hashes differently from
|
/// null-byte-separated so `["ab","c"]` hashes differently from
|
||||||
/// `["a","bc"]`.
|
/// `["a","bc"]`.
|
||||||
|
///
|
||||||
|
/// v2: Cargo.lock is canonicalized before hashing — comment-only
|
||||||
|
/// changes and whitespace drift in the lock file no longer cause
|
||||||
|
/// spurious cache misses.
|
||||||
pub fn compute(&self) -> Fingerprint {
|
pub fn compute(&self) -> Fingerprint {
|
||||||
let mut h = Hasher::new();
|
let mut h = Hasher::new();
|
||||||
// Domain-separated by field with a fixed sentinel — different
|
h.update(b"clawstor.fingerprint.v2\0");
|
||||||
// versions of this struct produce different hashes without a
|
let canonical_lock = canonicalize_lock(&self.cargo_lock);
|
||||||
// manual version tag.
|
update_field(&mut h, b"cargo_lock", canonical_lock.as_bytes());
|
||||||
h.update(b"clawstor.fingerprint.v1\0");
|
|
||||||
update_field(&mut h, b"cargo_lock", self.cargo_lock.as_bytes());
|
|
||||||
update_field(
|
update_field(
|
||||||
&mut h,
|
&mut h,
|
||||||
b"rustc_version_verbose",
|
b"rustc_version_verbose",
|
||||||
@@ -192,6 +194,41 @@ fn update_field(h: &mut Hasher, name: &[u8], value: &[u8]) {
|
|||||||
h.update(b"\0");
|
h.update(b"\0");
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Produce a canonical string from a Cargo.lock that is stable across
|
||||||
|
/// comment-only edits and whitespace drift.
|
||||||
|
///
|
||||||
|
/// Parses the TOML, extracts each `[[package]]` entry's (name, version,
|
||||||
|
/// checksum) tuple, sorts them, and emits one line per package:
|
||||||
|
/// `<name>@<version>#<checksum>\n`. If the lock is empty or cannot be
|
||||||
|
/// parsed as valid TOML the raw text is returned unchanged so we never
|
||||||
|
/// silently produce a wrong hash.
|
||||||
|
fn canonicalize_lock(raw: &str) -> String {
|
||||||
|
if raw.is_empty() {
|
||||||
|
return raw.to_string();
|
||||||
|
}
|
||||||
|
let Ok(doc) = raw.parse::<toml::Value>() else {
|
||||||
|
return raw.to_string();
|
||||||
|
};
|
||||||
|
let Some(packages) = doc.get("package").and_then(|v| v.as_array()) else {
|
||||||
|
return raw.to_string();
|
||||||
|
};
|
||||||
|
let mut entries: Vec<String> = packages
|
||||||
|
.iter()
|
||||||
|
.filter_map(|pkg| {
|
||||||
|
let name = pkg.get("name")?.as_str()?;
|
||||||
|
let version = pkg.get("version")?.as_str()?;
|
||||||
|
let checksum = pkg
|
||||||
|
.get("checksum")
|
||||||
|
.and_then(|v| v.as_str())
|
||||||
|
.unwrap_or("-");
|
||||||
|
Some(format!("{name}@{version}#{checksum}"))
|
||||||
|
})
|
||||||
|
.collect();
|
||||||
|
entries.sort();
|
||||||
|
entries.push(String::new());
|
||||||
|
entries.join("\n")
|
||||||
|
}
|
||||||
|
|
||||||
/// 32-byte BLAKE3 fingerprint over build inputs. Shape parallels
|
/// 32-byte BLAKE3 fingerprint over build inputs. Shape parallels
|
||||||
/// [`crate::cluster::blob::BlobId`] — same content-addressing spirit,
|
/// [`crate::cluster::blob::BlobId`] — same content-addressing spirit,
|
||||||
/// but keyed to inputs rather than outputs. A fingerprint identifies
|
/// but keyed to inputs rather than outputs. A fingerprint identifies
|
||||||
@@ -484,13 +521,27 @@ mod tests {
|
|||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn fingerprint_changes_when_cargo_lock_changes() {
|
fn fingerprint_changes_when_cargo_lock_package_changes() {
|
||||||
let base = baseline_inputs().compute();
|
let base = baseline_inputs().compute();
|
||||||
let mut mutated = baseline_inputs();
|
let mut mutated = baseline_inputs();
|
||||||
mutated.cargo_lock.push_str("# extra line\n");
|
// A version bump changes the fingerprint.
|
||||||
|
mutated.cargo_lock = mutated.cargo_lock.replace("0.1.0", "0.2.0");
|
||||||
assert_ne!(base, mutated.compute());
|
assert_ne!(base, mutated.compute());
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn fingerprint_stable_across_cargo_lock_comment_changes() {
|
||||||
|
let base = baseline_inputs().compute();
|
||||||
|
let mut mutated = baseline_inputs();
|
||||||
|
// Comments and extra blank lines must NOT change the fingerprint.
|
||||||
|
mutated.cargo_lock.push_str("# extra line\n\n");
|
||||||
|
assert_eq!(
|
||||||
|
base,
|
||||||
|
mutated.compute(),
|
||||||
|
"comment-only lock change should not alter fingerprint"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn fingerprint_changes_when_profile_changes() {
|
fn fingerprint_changes_when_profile_changes() {
|
||||||
let base = baseline_inputs().compute();
|
let base = baseline_inputs().compute();
|
||||||
@@ -545,6 +596,33 @@ mod tests {
|
|||||||
assert_eq!(recomputed.to_hex(), fp.to_hex());
|
assert_eq!(recomputed.to_hex(), fp.to_hex());
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn canonicalize_lock_is_stable_across_comments() {
|
||||||
|
let with_comment =
|
||||||
|
"# generated\n[[package]]\nname = \"foo\"\nversion = \"1.0.0\"\nchecksum = \"abc\"\n";
|
||||||
|
let without_comment =
|
||||||
|
"[[package]]\nname = \"foo\"\nversion = \"1.0.0\"\nchecksum = \"abc\"\n";
|
||||||
|
assert_eq!(
|
||||||
|
canonicalize_lock(with_comment),
|
||||||
|
canonicalize_lock(without_comment)
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn canonicalize_lock_sorts_packages() {
|
||||||
|
let a_first =
|
||||||
|
"[[package]]\nname = \"aaa\"\nversion = \"1.0\"\n\n[[package]]\nname = \"zzz\"\nversion = \"1.0\"\n";
|
||||||
|
let z_first =
|
||||||
|
"[[package]]\nname = \"zzz\"\nversion = \"1.0\"\n\n[[package]]\nname = \"aaa\"\nversion = \"1.0\"\n";
|
||||||
|
assert_eq!(canonicalize_lock(a_first), canonicalize_lock(z_first));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn canonicalize_lock_fallback_on_invalid_toml() {
|
||||||
|
let garbage = "not valid toml [[[";
|
||||||
|
assert_eq!(canonicalize_lock(garbage), garbage);
|
||||||
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn read_optional_returns_empty_for_missing() {
|
fn read_optional_returns_empty_for_missing() {
|
||||||
let tmp = tempfile::TempDir::new().unwrap();
|
let tmp = tempfile::TempDir::new().unwrap();
|
||||||
|
|||||||
@@ -67,6 +67,10 @@ pub mod keys {
|
|||||||
/// that two nodes will silo their caches. Peer-visible via
|
/// that two nodes will silo their caches. Peer-visible via
|
||||||
/// `PeerView.rustc_release` and `cluster-peer-status`.
|
/// `PeerView.rustc_release` and `cluster-peer-status`.
|
||||||
pub const RUSTC_RELEASE: &str = "clawstor.rustc.release";
|
pub const RUSTC_RELEASE: &str = "clawstor.rustc.release";
|
||||||
|
/// T3.3 / T3.6: Lifeguard-style composite health score (u8 as string).
|
||||||
|
/// 0 = healthy, 1-2 = degraded (hot >90% or load >4), 3+ = stressed.
|
||||||
|
/// Peers use this to apply leniency before marking a stressed node dead.
|
||||||
|
pub const HEALTH_SCORE: &str = "clawstor.health_score";
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Cluster identifier — every node in the same fleet must agree on this
|
/// Cluster identifier — every node in the same fleet must agree on this
|
||||||
@@ -121,6 +125,10 @@ pub struct PeerView {
|
|||||||
/// rustc. Used to surface toolchain drift that would otherwise
|
/// rustc. Used to surface toolchain drift that would otherwise
|
||||||
/// silently silo caches.
|
/// silently silo caches.
|
||||||
pub rustc_release: Option<String>,
|
pub rustc_release: Option<String>,
|
||||||
|
/// T3.3: Lifeguard health score advertised by the peer.
|
||||||
|
/// 0 = healthy, 1-2 = degraded, 3+ = stressed. `None` until the peer
|
||||||
|
/// has published the score (older daemons or very early boot).
|
||||||
|
pub health_score: Option<u8>,
|
||||||
}
|
}
|
||||||
|
|
||||||
impl PeerView {
|
impl PeerView {
|
||||||
@@ -276,6 +284,12 @@ impl ClusterGossip {
|
|||||||
/// rate locally via [`PeerView::cache_get_ref_hit_rate`]. Called
|
/// rate locally via [`PeerView::cache_get_ref_hit_rate`]. Called
|
||||||
/// periodically by [`crate::cluster::services::ClusterServices`]
|
/// periodically by [`crate::cluster::services::ClusterServices`]
|
||||||
/// once RPC has come up.
|
/// once RPC has come up.
|
||||||
|
/// Publish the Lifeguard health score so peers can observe our stress
|
||||||
|
/// without making an RPC round-trip. Called from the cache-metric tick.
|
||||||
|
pub async fn set_health_score(&self, score: u8) {
|
||||||
|
self.set(keys::HEALTH_SCORE, score.to_string()).await;
|
||||||
|
}
|
||||||
|
|
||||||
pub async fn set_cache_metrics(&self, snap: &MetricsReply) {
|
pub async fn set_cache_metrics(&self, snap: &MetricsReply) {
|
||||||
// Batch all four writes under one lock — chitchat serialises
|
// Batch all four writes under one lock — chitchat serialises
|
||||||
// per-node kv writes, so a single locked block avoids an
|
// per-node kv writes, so a single locked block avoids an
|
||||||
@@ -414,6 +428,7 @@ fn peer_view_from_state(id: &ChitchatId, state: &chitchat::NodeState, alive: boo
|
|||||||
cache_blob_get_bytes: get_u64(state, keys::CACHE_BLOB_GET_BYTES),
|
cache_blob_get_bytes: get_u64(state, keys::CACHE_BLOB_GET_BYTES),
|
||||||
cache_blob_put_bytes: get_u64(state, keys::CACHE_BLOB_PUT_BYTES),
|
cache_blob_put_bytes: get_u64(state, keys::CACHE_BLOB_PUT_BYTES),
|
||||||
rustc_release: get_str(state, keys::RUSTC_RELEASE),
|
rustc_release: get_str(state, keys::RUSTC_RELEASE),
|
||||||
|
health_score: get_u64(state, keys::HEALTH_SCORE).map(|v| v as u8),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -746,6 +761,7 @@ mod tests {
|
|||||||
cache_blob_get_bytes: None,
|
cache_blob_get_bytes: None,
|
||||||
cache_blob_put_bytes: None,
|
cache_blob_put_bytes: None,
|
||||||
rustc_release: None,
|
rustc_release: None,
|
||||||
|
health_score: None,
|
||||||
};
|
};
|
||||||
assert_eq!(base.cache_get_ref_hit_rate(), None, "no counters → None");
|
assert_eq!(base.cache_get_ref_hit_rate(), None, "no counters → None");
|
||||||
|
|
||||||
@@ -792,6 +808,7 @@ mod tests {
|
|||||||
cache_blob_get_bytes: None,
|
cache_blob_get_bytes: None,
|
||||||
cache_blob_put_bytes: None,
|
cache_blob_put_bytes: None,
|
||||||
rustc_release: None,
|
rustc_release: None,
|
||||||
|
health_score: None,
|
||||||
};
|
};
|
||||||
assert_eq!(base.hot_fill_ratio(), None, "no used → None");
|
assert_eq!(base.hot_fill_ratio(), None, "no used → None");
|
||||||
|
|
||||||
|
|||||||
@@ -366,6 +366,18 @@ pub struct DashboardStatusReply {
|
|||||||
pub cache: Option<CacheSummary>,
|
pub cache: Option<CacheSummary>,
|
||||||
#[serde(default)]
|
#[serde(default)]
|
||||||
pub timers: Vec<TimerStatus>,
|
pub timers: Vec<TimerStatus>,
|
||||||
|
/// Unix timestamp (seconds) when this daemon's metrics counters were
|
||||||
|
/// reset — proxy for when the daemon last started/restarted.
|
||||||
|
#[serde(default)]
|
||||||
|
pub daemon_started_unix: Option<u64>,
|
||||||
|
/// 1-minute load average from /proc/loadavg. None on non-Linux.
|
||||||
|
#[serde(default)]
|
||||||
|
pub cpu_load_1m: Option<f32>,
|
||||||
|
/// Composite Lifeguard-style health score (T3.3).
|
||||||
|
/// 0 = healthy; 1-2 = degraded; 3+ = stressed.
|
||||||
|
/// Increments for: hot >90%, fs >90%, load >4.0, failed timers.
|
||||||
|
#[serde(default)]
|
||||||
|
pub health_score: u8,
|
||||||
}
|
}
|
||||||
|
|
||||||
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)]
|
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)]
|
||||||
@@ -395,11 +407,27 @@ pub struct MountStatus {
|
|||||||
|
|
||||||
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)]
|
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)]
|
||||||
pub struct CacheSummary {
|
pub struct CacheSummary {
|
||||||
|
/// Composite totals across ref + tag + chunk lookups (for hit_rate).
|
||||||
pub hits: u64,
|
pub hits: u64,
|
||||||
pub misses: u64,
|
pub misses: u64,
|
||||||
pub bytes_served: u64,
|
pub bytes_served: u64,
|
||||||
pub bytes_ingested: u64,
|
pub bytes_ingested: u64,
|
||||||
pub hit_rate: f64,
|
pub hit_rate: f64,
|
||||||
|
/// Per-type breakdown so the dashboard can show ref vs tag vs chunk rates.
|
||||||
|
#[serde(default)]
|
||||||
|
pub get_ref_hits: u64,
|
||||||
|
#[serde(default)]
|
||||||
|
pub get_ref_misses: u64,
|
||||||
|
#[serde(default)]
|
||||||
|
pub get_tag_hits: u64,
|
||||||
|
#[serde(default)]
|
||||||
|
pub get_tag_misses: u64,
|
||||||
|
/// HasChunk probes — measures dedup efficiency (how many chunk uploads
|
||||||
|
/// were skipped because the receiver already had the chunk).
|
||||||
|
#[serde(default)]
|
||||||
|
pub has_chunk_hits: u64,
|
||||||
|
#[serde(default)]
|
||||||
|
pub has_chunk_misses: u64,
|
||||||
}
|
}
|
||||||
|
|
||||||
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)]
|
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)]
|
||||||
@@ -448,6 +476,27 @@ pub struct DashboardStorageReply {
|
|||||||
/// Sorted by last_seen_unix descending — hottest first.
|
/// Sorted by last_seen_unix descending — hottest first.
|
||||||
#[serde(default)]
|
#[serde(default)]
|
||||||
pub projects: Vec<DashboardProject>,
|
pub projects: Vec<DashboardProject>,
|
||||||
|
/// T2.7/T2.8: top-20 fingerprints by GetRef hit count since
|
||||||
|
/// daemon start. Used for cache-warming recommendations and
|
||||||
|
/// pollution detection. Empty until the first GetRef hit.
|
||||||
|
#[serde(default)]
|
||||||
|
pub hot_refs: Vec<HotRef>,
|
||||||
|
/// T1.2: recent GC/scrub/repair events on this node, newest first.
|
||||||
|
/// Capped at 100 entries; resets on daemon restart.
|
||||||
|
#[serde(default)]
|
||||||
|
pub maintenance_events: Vec<MaintenanceEvent>,
|
||||||
|
}
|
||||||
|
|
||||||
|
/// One entry from the per-fingerprint access log.
|
||||||
|
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||||
|
pub struct HotRef {
|
||||||
|
pub fingerprint_hex: String,
|
||||||
|
/// Total successful GetRef lookups since daemon start.
|
||||||
|
pub hit_count: u64,
|
||||||
|
/// Unix timestamp of the most recent hit.
|
||||||
|
pub last_hit_unix: u64,
|
||||||
|
/// Unix timestamp of the first hit.
|
||||||
|
pub first_hit_unix: u64,
|
||||||
}
|
}
|
||||||
|
|
||||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||||
@@ -695,6 +744,31 @@ fn timer_status(unit: &str) -> TimerStatus {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// 1-minute load average from /proc/loadavg. None on non-Linux or read error.
|
||||||
|
pub(crate) fn read_cpu_load_1m() -> Option<f32> {
|
||||||
|
std::fs::read_to_string("/proc/loadavg")
|
||||||
|
.ok()
|
||||||
|
.and_then(|s| s.split_whitespace().next()?.parse::<f32>().ok())
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Composite Lifeguard-style health score: 0 = healthy, higher = stressed.
|
||||||
|
/// Increments for: hot tier >90%, filesystem >90%, cpu load >4.0, failed timers.
|
||||||
|
fn compute_health_score(
|
||||||
|
hot_pct: f64,
|
||||||
|
fs_pct: f64,
|
||||||
|
cpu_load: Option<f32>,
|
||||||
|
failed_timers: usize,
|
||||||
|
) -> u8 {
|
||||||
|
let mut score: u8 = 0;
|
||||||
|
if hot_pct > 0.90 { score = score.saturating_add(1); }
|
||||||
|
if fs_pct > 0.90 { score = score.saturating_add(1); }
|
||||||
|
if cpu_load.map(|l| l > 4.0).unwrap_or(false) {
|
||||||
|
score = score.saturating_add(1);
|
||||||
|
}
|
||||||
|
if failed_timers > 0 { score = score.saturating_add(1); }
|
||||||
|
score
|
||||||
|
}
|
||||||
|
|
||||||
/// Is `path` currently a mount point? Cheap Linux check: read
|
/// Is `path` currently a mount point? Cheap Linux check: read
|
||||||
/// /proc/mounts. On macOS returns None (aggregator doesn't run
|
/// /proc/mounts. On macOS returns None (aggregator doesn't run
|
||||||
/// mounts).
|
/// mounts).
|
||||||
@@ -744,12 +818,131 @@ fn dir_size_bytes(root: &std::path::Path) -> u64 {
|
|||||||
total
|
total
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Maximum number of fingerprints tracked in the access log before the
|
||||||
|
/// lowest-count entry is evicted. 2000 covers ~3 months of daily unique
|
||||||
|
/// fingerprints at typical CI cadences.
|
||||||
|
const ACCESS_LOG_MAX: usize = 2000;
|
||||||
|
|
||||||
|
/// Per-fingerprint access record. Counts how many times a fingerprint
|
||||||
|
/// has been successfully served via `GetRef` or `GetRefVersioned`.
|
||||||
|
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||||
|
pub struct AccessRecord {
|
||||||
|
/// Total successful lookups since daemon start.
|
||||||
|
pub count: u64,
|
||||||
|
/// Unix timestamp of the very first hit.
|
||||||
|
pub first_unix: u64,
|
||||||
|
/// Unix timestamp of the most recent hit.
|
||||||
|
pub last_unix: u64,
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Bounded in-process log of per-fingerprint hit counts. Enables T2.7
|
||||||
|
/// (cache warming) and T2.8 (pollution detection) by recording which
|
||||||
|
/// fingerprints are hot, warm, or cold since the daemon started.
|
||||||
|
pub struct AccessLog {
|
||||||
|
entries: std::collections::HashMap<[u8; 32], AccessRecord>,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl Default for AccessLog {
|
||||||
|
fn default() -> Self {
|
||||||
|
Self { entries: std::collections::HashMap::new() }
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl AccessLog {
|
||||||
|
/// Record one successful lookup for `key` at wall-clock `now_unix`.
|
||||||
|
/// On overflow (> ACCESS_LOG_MAX), the lowest-count entry is evicted.
|
||||||
|
pub fn record(&mut self, key: &[u8; 32], now_unix: u64) {
|
||||||
|
if let Some(rec) = self.entries.get_mut(key) {
|
||||||
|
rec.count += 1;
|
||||||
|
rec.last_unix = now_unix;
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
if self.entries.len() >= ACCESS_LOG_MAX {
|
||||||
|
// Evict the least-accessed entry so hot fingerprints survive.
|
||||||
|
if let Some(&evict_key) = self
|
||||||
|
.entries
|
||||||
|
.iter()
|
||||||
|
.min_by_key(|(_, v)| v.count)
|
||||||
|
.map(|(k, _)| k)
|
||||||
|
{
|
||||||
|
self.entries.remove(&evict_key);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
self.entries.insert(*key, AccessRecord { count: 1, first_unix: now_unix, last_unix: now_unix });
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Return the top `n` entries sorted by count descending.
|
||||||
|
pub fn top(&self, n: usize) -> Vec<([u8; 32], AccessRecord)> {
|
||||||
|
let mut v: Vec<_> = self
|
||||||
|
.entries
|
||||||
|
.iter()
|
||||||
|
.map(|(k, v)| (*k, v.clone()))
|
||||||
|
.collect();
|
||||||
|
v.sort_by(|a, b| b.1.count.cmp(&a.1.count).then(b.1.last_unix.cmp(&a.1.last_unix)));
|
||||||
|
v.truncate(n);
|
||||||
|
v
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Total number of fingerprints currently tracked.
|
||||||
|
pub fn len(&self) -> usize {
|
||||||
|
self.entries.len()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// T1.2: one entry in the per-node maintenance event log.
|
||||||
|
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||||
|
pub struct MaintenanceEvent {
|
||||||
|
/// Logical kind: "gc_orphan", "gc_eviction_scored", "scrub", "repair".
|
||||||
|
pub kind: String,
|
||||||
|
/// Wall-clock unix timestamp when the event completed.
|
||||||
|
pub unix_ts: u64,
|
||||||
|
pub chunks_scanned: usize,
|
||||||
|
pub chunks_removed: usize,
|
||||||
|
pub bytes_reclaimed: u64,
|
||||||
|
/// Non-empty when the operation encountered an error.
|
||||||
|
#[serde(default)]
|
||||||
|
pub error: Option<String>,
|
||||||
|
}
|
||||||
|
|
||||||
|
const MAINTENANCE_LOG_MAX: usize = 100;
|
||||||
|
|
||||||
|
/// Bounded ring of recent maintenance events. Oldest entry is dropped
|
||||||
|
/// when the cap is reached.
|
||||||
|
pub struct MaintenanceLog {
|
||||||
|
entries: std::collections::VecDeque<MaintenanceEvent>,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl Default for MaintenanceLog {
|
||||||
|
fn default() -> Self {
|
||||||
|
Self { entries: std::collections::VecDeque::new() }
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl MaintenanceLog {
|
||||||
|
pub fn push(&mut self, ev: MaintenanceEvent) {
|
||||||
|
if self.entries.len() >= MAINTENANCE_LOG_MAX {
|
||||||
|
self.entries.pop_front();
|
||||||
|
}
|
||||||
|
self.entries.push_back(ev);
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Return all events, most-recent first.
|
||||||
|
pub fn recent(&self) -> Vec<MaintenanceEvent> {
|
||||||
|
self.entries.iter().cloned().rev().collect()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
pub struct RpcRouter {
|
pub struct RpcRouter {
|
||||||
gossip: Arc<ClusterGossip>,
|
gossip: Arc<ClusterGossip>,
|
||||||
blob_store: Option<Arc<BlobStore>>,
|
blob_store: Option<Arc<BlobStore>>,
|
||||||
ref_store: Option<Arc<RefStore>>,
|
ref_store: Option<Arc<RefStore>>,
|
||||||
tag_store: Option<Arc<TagStore>>,
|
tag_store: Option<Arc<TagStore>>,
|
||||||
metrics: Arc<CacheMetrics>,
|
metrics: Arc<CacheMetrics>,
|
||||||
|
/// T2.7/T2.8: per-fingerprint hit counter. Mutex because it's
|
||||||
|
/// written on every GetRef hit and read by the dashboard poller.
|
||||||
|
access_log: std::sync::Arc<std::sync::Mutex<AccessLog>>,
|
||||||
|
/// T1.2: recent GC/scrub/repair events for the maintenance panel.
|
||||||
|
maintenance_log: std::sync::Arc<std::sync::Mutex<MaintenanceLog>>,
|
||||||
local_name: String,
|
local_name: String,
|
||||||
local_zone: String,
|
local_zone: String,
|
||||||
/// Ref-forwarding (2026-07-13): when set, `GetRef` misses fan out
|
/// Ref-forwarding (2026-07-13): when set, `GetRef` misses fan out
|
||||||
@@ -784,6 +977,8 @@ impl RpcRouter {
|
|||||||
ref_store: None,
|
ref_store: None,
|
||||||
tag_store: None,
|
tag_store: None,
|
||||||
metrics: Arc::new(CacheMetrics::new()),
|
metrics: Arc::new(CacheMetrics::new()),
|
||||||
|
access_log: std::sync::Arc::new(std::sync::Mutex::new(AccessLog::default())),
|
||||||
|
maintenance_log: std::sync::Arc::new(std::sync::Mutex::new(MaintenanceLog::default())),
|
||||||
local_name,
|
local_name,
|
||||||
local_zone,
|
local_zone,
|
||||||
outbound_client: None,
|
outbound_client: None,
|
||||||
@@ -816,6 +1011,44 @@ impl RpcRouter {
|
|||||||
&self.metrics
|
&self.metrics
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Read-only access to the outbound QUIC client. Used by T3.5
|
||||||
|
/// prefetch task in `ClusterServices` to open peer connections.
|
||||||
|
pub fn outbound_client(
|
||||||
|
&self,
|
||||||
|
) -> Option<Arc<crate::cluster::transport::QuicClient>> {
|
||||||
|
self.outbound_client.clone()
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Shared handle to the per-fingerprint access log (T2.7/T2.8).
|
||||||
|
/// The dashboard poller reads this to identify hot/cold refs.
|
||||||
|
pub fn access_log(&self) -> std::sync::Arc<std::sync::Mutex<AccessLog>> {
|
||||||
|
self.access_log.clone()
|
||||||
|
}
|
||||||
|
|
||||||
|
/// T1.2: shared handle to the maintenance event log.
|
||||||
|
pub fn maintenance_log(&self) -> std::sync::Arc<std::sync::Mutex<MaintenanceLog>> {
|
||||||
|
self.maintenance_log.clone()
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Append a maintenance event. Called from services.rs gc_task after
|
||||||
|
/// each GC/scrub run so the dashboard can show recent activity.
|
||||||
|
pub fn push_maintenance_event(&self, ev: MaintenanceEvent) {
|
||||||
|
if let Ok(mut log) = self.maintenance_log.lock() {
|
||||||
|
log.push(ev);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Record one successful GetRef hit in the access log.
|
||||||
|
fn record_access(&self, key: &[u8; 32]) {
|
||||||
|
let now = std::time::SystemTime::now()
|
||||||
|
.duration_since(std::time::UNIX_EPOCH)
|
||||||
|
.map(|d| d.as_secs())
|
||||||
|
.unwrap_or(0);
|
||||||
|
if let Ok(mut log) = self.access_log.lock() {
|
||||||
|
log.record(key, now);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
/// Attach a local blob store. Enables the `Blob*` methods; nodes
|
/// Attach a local blob store. Enables the `Blob*` methods; nodes
|
||||||
/// without a store return [`ErrorCode::NotConfigured`] for those.
|
/// without a store return [`ErrorCode::NotConfigured`] for those.
|
||||||
pub fn with_blob_store(mut self, store: Arc<BlobStore>) -> Self {
|
pub fn with_blob_store(mut self, store: Arc<BlobStore>) -> Self {
|
||||||
@@ -981,7 +1214,7 @@ impl RpcRouter {
|
|||||||
// Cache metrics — the router already tracks these
|
// Cache metrics — the router already tracks these
|
||||||
// in-memory. Compute hit-rate here so the
|
// in-memory. Compute hit-rate here so the
|
||||||
// dashboard doesn't need to divide.
|
// dashboard doesn't need to divide.
|
||||||
let cache = {
|
let (cache, daemon_started_unix) = {
|
||||||
let snap = self.metrics.snapshot();
|
let snap = self.metrics.snapshot();
|
||||||
// The dashboard cares about "did the peer find
|
// The dashboard cares about "did the peer find
|
||||||
// what someone asked for". Sum the get_ref /
|
// what someone asked for". Sum the get_ref /
|
||||||
@@ -999,13 +1232,25 @@ impl RpcRouter {
|
|||||||
} else {
|
} else {
|
||||||
0.0
|
0.0
|
||||||
};
|
};
|
||||||
Some(CacheSummary {
|
let summary = Some(CacheSummary {
|
||||||
hits,
|
hits,
|
||||||
misses,
|
misses,
|
||||||
bytes_served: snap.blob_get_bytes,
|
bytes_served: snap.blob_get_bytes,
|
||||||
bytes_ingested: snap.blob_put_bytes,
|
bytes_ingested: snap.blob_put_bytes,
|
||||||
hit_rate: rate,
|
hit_rate: rate,
|
||||||
})
|
get_ref_hits: snap.get_ref_hits,
|
||||||
|
get_ref_misses: snap.get_ref_misses,
|
||||||
|
get_tag_hits: snap.get_tag_hits,
|
||||||
|
get_tag_misses: snap.get_tag_misses,
|
||||||
|
has_chunk_hits: snap.has_chunk_hits,
|
||||||
|
has_chunk_misses: snap.has_chunk_misses,
|
||||||
|
});
|
||||||
|
let started = if snap.started_unix > 0 {
|
||||||
|
Some(snap.started_unix)
|
||||||
|
} else {
|
||||||
|
None
|
||||||
|
};
|
||||||
|
(summary, started)
|
||||||
};
|
};
|
||||||
// Well-known timer set. Missing timers just get
|
// Well-known timer set. Missing timers just get
|
||||||
// next_fire_unix=None / last_result=None.
|
// next_fire_unix=None / last_result=None.
|
||||||
@@ -1015,6 +1260,30 @@ impl RpcRouter {
|
|||||||
timer_status("clawstor-ref-sweep.timer"),
|
timer_status("clawstor-ref-sweep.timer"),
|
||||||
timer_status("clawstor-snapshot-rotate.timer"),
|
timer_status("clawstor-snapshot-rotate.timer"),
|
||||||
];
|
];
|
||||||
|
// Lifeguard-style health score (T3.3).
|
||||||
|
let cpu_load_1m = read_cpu_load_1m();
|
||||||
|
let hot_pct = if hot_max > 0 {
|
||||||
|
hot_used as f64 / hot_max as f64
|
||||||
|
} else {
|
||||||
|
0.0
|
||||||
|
};
|
||||||
|
let fs_pct = filesystem.as_ref().map(|f| {
|
||||||
|
if f.total_bytes > 0 {
|
||||||
|
f.used_bytes as f64 / f.total_bytes as f64
|
||||||
|
} else {
|
||||||
|
0.0
|
||||||
|
}
|
||||||
|
}).unwrap_or(0.0);
|
||||||
|
let failed_timers = timers
|
||||||
|
.iter()
|
||||||
|
.filter(|t| {
|
||||||
|
t.last_result
|
||||||
|
.as_deref()
|
||||||
|
.map(|r| r != "success")
|
||||||
|
.unwrap_or(false)
|
||||||
|
})
|
||||||
|
.count();
|
||||||
|
let health_score = compute_health_score(hot_pct, fs_pct, cpu_load_1m, failed_timers);
|
||||||
let reply = DashboardStatusReply {
|
let reply = DashboardStatusReply {
|
||||||
node_name: self.local_name.clone(),
|
node_name: self.local_name.clone(),
|
||||||
zone: self.local_zone.clone(),
|
zone: self.local_zone.clone(),
|
||||||
@@ -1031,6 +1300,9 @@ impl RpcRouter {
|
|||||||
mount,
|
mount,
|
||||||
cache,
|
cache,
|
||||||
timers,
|
timers,
|
||||||
|
daemon_started_unix,
|
||||||
|
cpu_load_1m,
|
||||||
|
health_score,
|
||||||
};
|
};
|
||||||
let json = serde_json::to_vec(&reply)
|
let json = serde_json::to_vec(&reply)
|
||||||
.context("encoding DashboardStatusReply as JSON")?;
|
.context("encoding DashboardStatusReply as JSON")?;
|
||||||
@@ -1124,6 +1396,35 @@ impl RpcRouter {
|
|||||||
)
|
)
|
||||||
.await;
|
.await;
|
||||||
|
|
||||||
|
// T2.7/T2.8: embed top-20 hot fingerprints from the
|
||||||
|
// access log so peers can recommend cross-node warming.
|
||||||
|
let hot_refs = self
|
||||||
|
.access_log
|
||||||
|
.lock()
|
||||||
|
.map(|log| {
|
||||||
|
log.top(20)
|
||||||
|
.into_iter()
|
||||||
|
.map(|(k, r)| {
|
||||||
|
let mut hex = String::with_capacity(64);
|
||||||
|
for b in &k { hex.push_str(&format!("{b:02x}")); }
|
||||||
|
HotRef {
|
||||||
|
fingerprint_hex: hex,
|
||||||
|
hit_count: r.count,
|
||||||
|
last_hit_unix: r.last_unix,
|
||||||
|
first_hit_unix: r.first_unix,
|
||||||
|
}
|
||||||
|
})
|
||||||
|
.collect()
|
||||||
|
})
|
||||||
|
.unwrap_or_default();
|
||||||
|
|
||||||
|
// T1.2: recent GC/scrub/repair events for the maintenance panel.
|
||||||
|
let maintenance_events = self
|
||||||
|
.maintenance_log
|
||||||
|
.lock()
|
||||||
|
.map(|log| log.recent())
|
||||||
|
.unwrap_or_default();
|
||||||
|
|
||||||
let reply = DashboardStorageReply {
|
let reply = DashboardStorageReply {
|
||||||
node_name: self.local_name.clone(),
|
node_name: self.local_name.clone(),
|
||||||
tags,
|
tags,
|
||||||
@@ -1134,6 +1435,8 @@ impl RpcRouter {
|
|||||||
refs_sample_capped_at: SAMPLE_CAP,
|
refs_sample_capped_at: SAMPLE_CAP,
|
||||||
refs_sample,
|
refs_sample,
|
||||||
projects,
|
projects,
|
||||||
|
hot_refs,
|
||||||
|
maintenance_events,
|
||||||
};
|
};
|
||||||
let json = serde_json::to_vec(&reply)
|
let json = serde_json::to_vec(&reply)
|
||||||
.context("encoding DashboardStorageReply as JSON")?;
|
.context("encoding DashboardStorageReply as JSON")?;
|
||||||
@@ -1353,6 +1656,7 @@ impl RpcRouter {
|
|||||||
// Local first.
|
// Local first.
|
||||||
if let Some(value) = store.get(&key).await? {
|
if let Some(value) = store.get(&key).await? {
|
||||||
self.metrics.record_get_ref_hit();
|
self.metrics.record_get_ref_hit();
|
||||||
|
self.record_access(&key);
|
||||||
return Ok(HandlerOutcome::Reply(value.to_vec()));
|
return Ok(HandlerOutcome::Reply(value.to_vec()));
|
||||||
}
|
}
|
||||||
// Ref-forwarding: try peers via gossip. First hit wins
|
// Ref-forwarding: try peers via gossip. First hit wins
|
||||||
@@ -1360,6 +1664,7 @@ impl RpcRouter {
|
|||||||
// (and reads) are all local.
|
// (and reads) are all local.
|
||||||
if let Some(value) = self.forward_get_ref(&key).await {
|
if let Some(value) = self.forward_get_ref(&key).await {
|
||||||
self.metrics.record_get_ref_hit();
|
self.metrics.record_get_ref_hit();
|
||||||
|
self.record_access(&key);
|
||||||
return Ok(HandlerOutcome::Reply(value.to_vec()));
|
return Ok(HandlerOutcome::Reply(value.to_vec()));
|
||||||
}
|
}
|
||||||
self.metrics.record_get_ref_miss();
|
self.metrics.record_get_ref_miss();
|
||||||
@@ -1430,6 +1735,7 @@ impl RpcRouter {
|
|||||||
// Local first.
|
// Local first.
|
||||||
if let Some(s) = store.get_stamped(&key).await? {
|
if let Some(s) = store.get_stamped(&key).await? {
|
||||||
self.metrics.record_get_ref_hit();
|
self.metrics.record_get_ref_hit();
|
||||||
|
self.record_access(&key);
|
||||||
return Ok(HandlerOutcome::Reply(s.to_bytes().to_vec()));
|
return Ok(HandlerOutcome::Reply(s.to_bytes().to_vec()));
|
||||||
}
|
}
|
||||||
// Phase 3b: cross-runner sharing for stamped refs.
|
// Phase 3b: cross-runner sharing for stamped refs.
|
||||||
@@ -1438,6 +1744,7 @@ impl RpcRouter {
|
|||||||
// pure local hits (same semantics as GetRef path).
|
// pure local hits (same semantics as GetRef path).
|
||||||
if let Some(s) = self.forward_get_ref_versioned(&key).await {
|
if let Some(s) = self.forward_get_ref_versioned(&key).await {
|
||||||
self.metrics.record_get_ref_hit();
|
self.metrics.record_get_ref_hit();
|
||||||
|
self.record_access(&key);
|
||||||
return Ok(HandlerOutcome::Reply(s.to_bytes().to_vec()));
|
return Ok(HandlerOutcome::Reply(s.to_bytes().to_vec()));
|
||||||
}
|
}
|
||||||
self.metrics.record_get_ref_miss();
|
self.metrics.record_get_ref_miss();
|
||||||
@@ -1747,7 +2054,7 @@ impl RpcRouter {
|
|||||||
/// missing chunks from `conn` into the local `BlobStore`. Same shape
|
/// missing chunks from `conn` into the local `BlobStore`. Same shape
|
||||||
/// as `prewarm_missing_chunks_between_parallel` but the downstream is
|
/// as `prewarm_missing_chunks_between_parallel` but the downstream is
|
||||||
/// in-process rather than another peer.
|
/// in-process rather than another peer.
|
||||||
async fn pull_blob_locally(
|
pub(crate) async fn pull_blob_locally(
|
||||||
conn: &Connection,
|
conn: &Connection,
|
||||||
local: &BlobStore,
|
local: &BlobStore,
|
||||||
id: &crate::cluster::blob::BlobId,
|
id: &crate::cluster::blob::BlobId,
|
||||||
|
|||||||
@@ -38,6 +38,20 @@ const HOT_METRIC_INTERVAL: Duration = Duration::from_secs(30);
|
|||||||
/// cadence still sees at-worst-60s-stale counters.
|
/// cadence still sees at-worst-60s-stale counters.
|
||||||
const CACHE_METRIC_INTERVAL: Duration = Duration::from_secs(60);
|
const CACHE_METRIC_INTERVAL: Duration = Duration::from_secs(60);
|
||||||
|
|
||||||
|
/// T3.5: how often the idle prefetch task wakes and checks peers.
|
||||||
|
/// 10-minute cadence keeps cross-node traffic minimal; each tick pulls
|
||||||
|
/// at most `PREFETCH_MAX_PER_TICK` blobs per peer.
|
||||||
|
const PREFETCH_INTERVAL: Duration = Duration::from_secs(600);
|
||||||
|
|
||||||
|
/// T3.5: maximum blobs pulled from any single peer per prefetch tick.
|
||||||
|
/// Caps burst bandwidth to ≈ N × blob_size during each idle window.
|
||||||
|
const PREFETCH_MAX_PER_TICK: usize = 5;
|
||||||
|
|
||||||
|
/// T3.5: hot-tier fill fraction below which the node is considered idle
|
||||||
|
/// enough to speculate-pull blobs from peers. Above this threshold we
|
||||||
|
/// don't want to grow disk usage further.
|
||||||
|
const PREFETCH_IDLE_THRESHOLD: f64 = 0.60;
|
||||||
|
|
||||||
/// Live cluster services attached to a running daemon.
|
/// Live cluster services attached to a running daemon.
|
||||||
///
|
///
|
||||||
/// Drop shuts down all background tasks. Ownership is single: the
|
/// Drop shuts down all background tasks. Ownership is single: the
|
||||||
@@ -84,6 +98,9 @@ pub struct ClusterServices {
|
|||||||
/// `cluster.gc_interval_hours` is unset or the daemon has no blob
|
/// `cluster.gc_interval_hours` is unset or the daemon has no blob
|
||||||
/// store (nothing to sweep).
|
/// store (nothing to sweep).
|
||||||
gc_task: Option<JoinHandle<()>>,
|
gc_task: Option<JoinHandle<()>>,
|
||||||
|
/// T3.5: speculative blob prefetch from peers during idle periods.
|
||||||
|
/// Active only when the daemon has both a blob store and a ref store.
|
||||||
|
prefetch_task: Option<JoinHandle<()>>,
|
||||||
}
|
}
|
||||||
|
|
||||||
impl std::fmt::Debug for ClusterServices {
|
impl std::fmt::Debug for ClusterServices {
|
||||||
@@ -306,6 +323,7 @@ impl ClusterServices {
|
|||||||
let gossip = gossip.clone();
|
let gossip = gossip.clone();
|
||||||
gossip.set_cache_metrics(&router.metrics().snapshot()).await;
|
gossip.set_cache_metrics(&router.metrics().snapshot()).await;
|
||||||
Some(tokio::spawn(async move {
|
Some(tokio::spawn(async move {
|
||||||
|
use crate::cluster::gossip::keys;
|
||||||
let mut ticker = tokio::time::interval(CACHE_METRIC_INTERVAL);
|
let mut ticker = tokio::time::interval(CACHE_METRIC_INTERVAL);
|
||||||
// First tick fires immediately (tokio interval default),
|
// First tick fires immediately (tokio interval default),
|
||||||
// covered by the initial publish above — skip it here.
|
// covered by the initial publish above — skip it here.
|
||||||
@@ -314,6 +332,28 @@ impl ClusterServices {
|
|||||||
ticker.tick().await;
|
ticker.tick().await;
|
||||||
let snap = router.metrics().snapshot();
|
let snap = router.metrics().snapshot();
|
||||||
gossip.set_cache_metrics(&snap).await;
|
gossip.set_cache_metrics(&snap).await;
|
||||||
|
// T3.3/T3.6: publish Lifeguard health score so peers
|
||||||
|
// can observe our stress level via gossip, without RPC.
|
||||||
|
// Simplified: hot-tier pressure + cpu load only (no
|
||||||
|
// filesystem or timer checks — those require blocking I/O
|
||||||
|
// that doesn't belong on the gossip tick path).
|
||||||
|
let hot_used = gossip
|
||||||
|
.self_kv(keys::HOT_USED_BYTES)
|
||||||
|
.await
|
||||||
|
.and_then(|s| s.parse::<u64>().ok())
|
||||||
|
.unwrap_or(0);
|
||||||
|
let hot_max = gossip
|
||||||
|
.self_kv(keys::HOT_MAX_BYTES)
|
||||||
|
.await
|
||||||
|
.and_then(|s| s.parse::<u64>().ok())
|
||||||
|
.unwrap_or(1);
|
||||||
|
let hot_pct =
|
||||||
|
if hot_max > 0 { hot_used as f64 / hot_max as f64 } else { 0.0 };
|
||||||
|
let cpu_load = crate::cluster::rpc::read_cpu_load_1m();
|
||||||
|
let score: u8 =
|
||||||
|
(if hot_pct > 0.90 { 1u8 } else { 0 })
|
||||||
|
.saturating_add(if cpu_load.map(|l| l > 4.0).unwrap_or(false) { 1 } else { 0 });
|
||||||
|
gossip.set_health_score(score).await;
|
||||||
}
|
}
|
||||||
}))
|
}))
|
||||||
} else {
|
} else {
|
||||||
@@ -349,6 +389,8 @@ impl ClusterServices {
|
|||||||
(Some(store), Some(hours)) if hours > 0 => {
|
(Some(store), Some(hours)) if hours > 0 => {
|
||||||
let store = store.clone();
|
let store = store.clone();
|
||||||
let tag_store_for_gc = tag_store.clone();
|
let tag_store_for_gc = tag_store.clone();
|
||||||
|
let ref_store_for_gc = ref_store.clone();
|
||||||
|
let router_for_gc = router.clone();
|
||||||
// Phase 7d follow-on: snapshot store is under the
|
// Phase 7d follow-on: snapshot store is under the
|
||||||
// same root as the blob store. Open once here so the
|
// same root as the blob store. Open once here so the
|
||||||
// ticker doesn't pay the fs setup cost every tick.
|
// ticker doesn't pay the fs setup cost every tick.
|
||||||
@@ -366,15 +408,45 @@ impl ClusterServices {
|
|||||||
ticker.tick().await;
|
ticker.tick().await;
|
||||||
loop {
|
loop {
|
||||||
ticker.tick().await;
|
ticker.tick().await;
|
||||||
|
let now_unix = std::time::SystemTime::now()
|
||||||
|
.duration_since(std::time::UNIX_EPOCH)
|
||||||
|
.map(|d| d.as_secs())
|
||||||
|
.unwrap_or(0);
|
||||||
match store.gc_orphan_chunks().await {
|
match store.gc_orphan_chunks().await {
|
||||||
Ok(r) => tracing::info!(
|
Ok(r) => {
|
||||||
|
tracing::info!(
|
||||||
chunks_scanned = r.chunks_scanned,
|
chunks_scanned = r.chunks_scanned,
|
||||||
chunks_removed = r.chunks_removed,
|
chunks_removed = r.chunks_removed,
|
||||||
bytes_reclaimed = r.bytes_reclaimed,
|
bytes_reclaimed = r.bytes_reclaimed,
|
||||||
"auto-GC swept orphan chunks"
|
"auto-GC swept orphan chunks"
|
||||||
),
|
);
|
||||||
|
if let Some(router) = &router_for_gc {
|
||||||
|
router.push_maintenance_event(
|
||||||
|
crate::cluster::rpc::MaintenanceEvent {
|
||||||
|
kind: "gc_orphan".into(),
|
||||||
|
unix_ts: now_unix,
|
||||||
|
chunks_scanned: r.chunks_scanned,
|
||||||
|
chunks_removed: r.chunks_removed,
|
||||||
|
bytes_reclaimed: r.bytes_reclaimed,
|
||||||
|
error: None,
|
||||||
|
},
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
Err(e) => {
|
Err(e) => {
|
||||||
tracing::warn!(error = %e, "auto-GC failed; will retry next tick")
|
tracing::warn!(error = %e, "auto-GC failed; will retry next tick");
|
||||||
|
if let Some(router) = &router_for_gc {
|
||||||
|
router.push_maintenance_event(
|
||||||
|
crate::cluster::rpc::MaintenanceEvent {
|
||||||
|
kind: "gc_orphan".into(),
|
||||||
|
unix_ts: now_unix,
|
||||||
|
chunks_scanned: 0,
|
||||||
|
chunks_removed: 0,
|
||||||
|
bytes_reclaimed: 0,
|
||||||
|
error: Some(e.to_string()),
|
||||||
|
},
|
||||||
|
);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
// Field finding 2026-07-12: if configured with a
|
// Field finding 2026-07-12: if configured with a
|
||||||
@@ -423,23 +495,186 @@ impl ClusterServices {
|
|||||||
pinned.extend(snaps);
|
pinned.extend(snaps);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
// T3.4: build blob→hit_count map for
|
||||||
|
// pollution-score eviction (replaces pure LRU).
|
||||||
|
let hit_map = build_blob_hit_map(
|
||||||
|
&router_for_gc,
|
||||||
|
&ref_store_for_gc,
|
||||||
|
).await;
|
||||||
|
let hit_count = hit_map.values().sum::<u64>();
|
||||||
|
let eviction_now = std::time::SystemTime::now()
|
||||||
|
.duration_since(std::time::UNIX_EPOCH)
|
||||||
|
.map(|d| d.as_secs())
|
||||||
|
.unwrap_or(0);
|
||||||
match store
|
match store
|
||||||
.evict_to_size_cap_with_pins(cap, &pinned)
|
.evict_to_size_cap_with_scores(cap, &pinned, &hit_map)
|
||||||
.await
|
.await
|
||||||
{
|
{
|
||||||
Ok(r) if r.chunks_removed > 0 => tracing::info!(
|
Ok(r) if r.chunks_removed > 0 => {
|
||||||
|
tracing::info!(
|
||||||
chunks_removed = r.chunks_removed,
|
chunks_removed = r.chunks_removed,
|
||||||
bytes_reclaimed = r.bytes_reclaimed,
|
bytes_reclaimed = r.bytes_reclaimed,
|
||||||
max_gb = gb,
|
max_gb = gb,
|
||||||
pinned_blobs = pinned.len(),
|
pinned_blobs = pinned.len(),
|
||||||
snapshot_pins = snapshot_pin_count,
|
snapshot_pins = snapshot_pin_count,
|
||||||
"auto-GC evicted LRU blobs to hit size cap"
|
scored_blobs = hit_count,
|
||||||
),
|
"auto-GC evicted pollution-scored blobs to hit size cap"
|
||||||
|
);
|
||||||
|
if let Some(router) = &router_for_gc {
|
||||||
|
router.push_maintenance_event(
|
||||||
|
crate::cluster::rpc::MaintenanceEvent {
|
||||||
|
kind: "gc_eviction_scored".into(),
|
||||||
|
unix_ts: eviction_now,
|
||||||
|
chunks_scanned: 0,
|
||||||
|
chunks_removed: r.chunks_removed,
|
||||||
|
bytes_reclaimed: r.bytes_reclaimed,
|
||||||
|
error: None,
|
||||||
|
},
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
Ok(_) => {} // under cap already; keep quiet
|
Ok(_) => {} // under cap already; keep quiet
|
||||||
Err(e) => tracing::warn!(
|
Err(e) => {
|
||||||
|
tracing::warn!(error = %e, "auto-GC eviction failed; will retry next tick");
|
||||||
|
if let Some(router) = &router_for_gc {
|
||||||
|
router.push_maintenance_event(
|
||||||
|
crate::cluster::rpc::MaintenanceEvent {
|
||||||
|
kind: "gc_eviction_scored".into(),
|
||||||
|
unix_ts: eviction_now,
|
||||||
|
chunks_scanned: 0,
|
||||||
|
chunks_removed: 0,
|
||||||
|
bytes_reclaimed: 0,
|
||||||
|
error: Some(e.to_string()),
|
||||||
|
},
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}))
|
||||||
|
}
|
||||||
|
_ => None,
|
||||||
|
};
|
||||||
|
|
||||||
|
// T3.5: speculative blob prefetch during idle periods. Active only
|
||||||
|
// when we have an outbound RPC client (blob store + ref store +
|
||||||
|
// outbound_client all wired). The task wakes every PREFETCH_INTERVAL,
|
||||||
|
// checks if the node is idle (hot-tier < PREFETCH_IDLE_THRESHOLD),
|
||||||
|
// and pulls up to PREFETCH_MAX_PER_TICK blobs from each live peer.
|
||||||
|
let prefetch_task = match (&blob_store, &ref_store, &router) {
|
||||||
|
(Some(blob_store), Some(ref_store), Some(router))
|
||||||
|
if router.outbound_client().is_some() =>
|
||||||
|
{
|
||||||
|
let blob_store = blob_store.clone();
|
||||||
|
let ref_store = ref_store.clone();
|
||||||
|
let gossip = gossip.clone();
|
||||||
|
let router = router.clone();
|
||||||
|
Some(tokio::spawn(async move {
|
||||||
|
use crate::cluster::gossip::keys;
|
||||||
|
use crate::cluster::rpc::{call_dashboard_storage, pull_blob_locally};
|
||||||
|
use crate::cluster::blob::BlobId;
|
||||||
|
let mut ticker = tokio::time::interval(PREFETCH_INTERVAL);
|
||||||
|
ticker.tick().await; // skip immediate first tick
|
||||||
|
loop {
|
||||||
|
ticker.tick().await;
|
||||||
|
// Idle check: hot-tier fill below threshold.
|
||||||
|
let hot_used = gossip
|
||||||
|
.self_kv(keys::HOT_USED_BYTES)
|
||||||
|
.await
|
||||||
|
.and_then(|s| s.parse::<u64>().ok())
|
||||||
|
.unwrap_or(0);
|
||||||
|
let hot_max = gossip
|
||||||
|
.self_kv(keys::HOT_MAX_BYTES)
|
||||||
|
.await
|
||||||
|
.and_then(|s| s.parse::<u64>().ok())
|
||||||
|
.unwrap_or(1);
|
||||||
|
let fill = if hot_max > 0 {
|
||||||
|
hot_used as f64 / hot_max as f64
|
||||||
|
} else {
|
||||||
|
1.0
|
||||||
|
};
|
||||||
|
if fill >= PREFETCH_IDLE_THRESHOLD {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
// Prefetch from each live peer.
|
||||||
|
let peers = gossip
|
||||||
|
.peers()
|
||||||
|
.await
|
||||||
|
.into_iter()
|
||||||
|
.filter(|p| p.alive && p.rpc_lan.or(p.rpc_tailscale).is_some())
|
||||||
|
.collect::<Vec<_>>();
|
||||||
|
for peer in peers {
|
||||||
|
let Some(client) = router.outbound_client() else { break };
|
||||||
|
let addr = match peer.rpc_lan.or(peer.rpc_tailscale) {
|
||||||
|
Some(a) => a,
|
||||||
|
None => continue,
|
||||||
|
};
|
||||||
|
let conn = match tokio::time::timeout(
|
||||||
|
Duration::from_secs(5),
|
||||||
|
client.connect(addr, &peer.name),
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
{
|
||||||
|
Ok(Ok(c)) => c,
|
||||||
|
_ => continue,
|
||||||
|
};
|
||||||
|
let storage = match call_dashboard_storage(&conn).await {
|
||||||
|
Ok(s) => s,
|
||||||
|
Err(e) => {
|
||||||
|
tracing::debug!(
|
||||||
|
peer = %peer.name,
|
||||||
error = %e,
|
error = %e,
|
||||||
"auto-GC eviction failed; will retry next tick"
|
"prefetch: DashboardStorage failed"
|
||||||
),
|
);
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
};
|
||||||
|
let mut pulled = 0usize;
|
||||||
|
for r in &storage.refs_sample {
|
||||||
|
if pulled >= PREFETCH_MAX_PER_TICK {
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
let fp_id = match BlobId::from_hex(&r.fingerprint_hex) {
|
||||||
|
Ok(id) => id,
|
||||||
|
Err(_) => continue,
|
||||||
|
};
|
||||||
|
let fp: [u8; 32] = *fp_id.as_bytes();
|
||||||
|
// Skip refs we already have locally.
|
||||||
|
if ref_store.contains(&fp).await.unwrap_or(true) {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
let blob_id = match BlobId::from_hex(&r.blob_id_hex) {
|
||||||
|
Ok(id) => id,
|
||||||
|
Err(_) => continue,
|
||||||
|
};
|
||||||
|
// Pull chunks + manifest from peer.
|
||||||
|
match pull_blob_locally(&conn, &blob_store, &blob_id).await {
|
||||||
|
Ok(()) => {
|
||||||
|
let _ = ref_store.put(&fp, blob_id.as_bytes()).await;
|
||||||
|
pulled += 1;
|
||||||
|
tracing::debug!(
|
||||||
|
peer = %peer.name,
|
||||||
|
blob = %r.blob_id_hex,
|
||||||
|
"prefetch: pulled blob"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
Err(e) => {
|
||||||
|
tracing::debug!(
|
||||||
|
peer = %peer.name,
|
||||||
|
blob = %r.blob_id_hex,
|
||||||
|
error = %e,
|
||||||
|
"prefetch: pull failed"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if pulled > 0 {
|
||||||
|
tracing::info!(
|
||||||
|
peer = %peer.name,
|
||||||
|
count = pulled,
|
||||||
|
"prefetch: idle-pulled blobs from peer"
|
||||||
|
);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -459,6 +694,7 @@ impl ClusterServices {
|
|||||||
cache_metric_task,
|
cache_metric_task,
|
||||||
prom_server,
|
prom_server,
|
||||||
gc_task,
|
gc_task,
|
||||||
|
prefetch_task,
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -501,7 +737,47 @@ impl ClusterServices {
|
|||||||
if let Some(task) = self.gc_task {
|
if let Some(task) = self.gc_task {
|
||||||
task.abort();
|
task.abort();
|
||||||
}
|
}
|
||||||
|
if let Some(task) = self.prefetch_task {
|
||||||
|
task.abort();
|
||||||
}
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// T3.4: build a blob_id → total_hit_count map from the access log + ref store.
|
||||||
|
///
|
||||||
|
/// The AccessLog tracks fingerprint→hits; the ref store maps fingerprint→blob_id.
|
||||||
|
/// Joining them gives blob_id→hits which the pollution-score eviction needs.
|
||||||
|
/// Returns an empty map when either input is unavailable (GC falls back to mtime ordering).
|
||||||
|
async fn build_blob_hit_map(
|
||||||
|
router: &Option<Arc<crate::cluster::rpc::RpcRouter>>,
|
||||||
|
ref_store: &Option<Arc<crate::cluster::refs::RefStore>>,
|
||||||
|
) -> std::collections::HashMap<crate::cluster::blob::BlobId, u64> {
|
||||||
|
use crate::cluster::blob::BlobId;
|
||||||
|
let mut map = std::collections::HashMap::new();
|
||||||
|
let (Some(router), Some(rs)) = (router, ref_store) else {
|
||||||
|
return map;
|
||||||
|
};
|
||||||
|
// fingerprint_bytes → hit_count from the in-memory access log
|
||||||
|
let fp_hits: std::collections::HashMap<[u8; 32], u64> =
|
||||||
|
match router.access_log().lock() {
|
||||||
|
Ok(log) => log.top(usize::MAX).into_iter().map(|(fp, rec)| (fp, rec.count)).collect(),
|
||||||
|
Err(_) => return map,
|
||||||
|
};
|
||||||
|
if fp_hits.is_empty() {
|
||||||
|
return map;
|
||||||
|
}
|
||||||
|
// fingerprint → blob_id from the ref store
|
||||||
|
let pairs = match rs.list().await {
|
||||||
|
Ok(p) => p,
|
||||||
|
Err(_) => return map,
|
||||||
|
};
|
||||||
|
for (fp_key, blob_id_bytes) in pairs {
|
||||||
|
if let Some(&hits) = fp_hits.get(&fp_key) {
|
||||||
|
let bid = BlobId::from_bytes(blob_id_bytes);
|
||||||
|
*map.entry(bid).or_insert(0u64) += hits;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
map
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Loop accepting incoming QUIC connections and dispatching each to a
|
/// Loop accepting incoming QUIC connections and dispatching each to a
|
||||||
|
|||||||
@@ -188,10 +188,14 @@ pub async fn run(cfg: Config, mut manifest: Manifest) -> Result<()> {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
_ = snap_tick.tick() => {
|
_ = snap_tick.tick() => {
|
||||||
|
let dataset = &cfg.warm.zfs_dataset;
|
||||||
|
if dataset.is_empty() || dataset == "none" {
|
||||||
|
tracing::debug!("zfs_dataset=none — skipping snapshot");
|
||||||
|
} else {
|
||||||
let ts = chrono::Utc::now().format("%Y-%m-%d-%H%M").to_string();
|
let ts = chrono::Utc::now().format("%Y-%m-%d-%H%M").to_string();
|
||||||
tracing::info!("taking snapshot {}", ts);
|
tracing::info!("taking snapshot {}", ts);
|
||||||
if let Err(e) = snapshot::run_snapshot_cycle(
|
if let Err(e) = snapshot::run_snapshot_cycle(
|
||||||
&zfs, &cfg.warm.zfs_dataset, &ts,
|
&zfs, dataset, &ts,
|
||||||
cfg.warm.snapshot_retain_hours as usize,
|
cfg.warm.snapshot_retain_hours as usize,
|
||||||
cfg.warm.snapshot_retain_days as usize,
|
cfg.warm.snapshot_retain_days as usize,
|
||||||
cfg.warm.snapshot_retain_weeks as usize,
|
cfg.warm.snapshot_retain_weeks as usize,
|
||||||
@@ -199,14 +203,17 @@ pub async fn run(cfg: Config, mut manifest: Manifest) -> Result<()> {
|
|||||||
tracing::error!("snapshot failed: {:#}", e);
|
tracing::error!("snapshot failed: {:#}", e);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
}
|
||||||
_ = repl_tick.tick() => {
|
_ = repl_tick.tick() => {
|
||||||
|
let dataset = &cfg.warm.zfs_dataset;
|
||||||
|
if !dataset.is_empty() && dataset != "none" {
|
||||||
if let Some(rep) = &cfg.replication {
|
if let Some(rep) = &cfg.replication {
|
||||||
if let (Some(host), Some(user), Some(dest)) = (
|
if let (Some(host), Some(user), Some(dest)) = (
|
||||||
&rep.send_to_host, &rep.send_to_user, &rep.cold_dataset_on_peer
|
&rep.send_to_host, &rep.send_to_user, &rep.cold_dataset_on_peer
|
||||||
) {
|
) {
|
||||||
tracing::info!("replicating warm → cold on {}", host);
|
tracing::info!("replicating warm → cold on {}", host);
|
||||||
if let Err(e) = snapshot::replicate_to_cold(
|
if let Err(e) = snapshot::replicate_to_cold(
|
||||||
&zfs, &cfg.warm.zfs_dataset, user, host, dest
|
&zfs, dataset, user, host, dest
|
||||||
) {
|
) {
|
||||||
tracing::error!("replication failed: {:#}", e);
|
tracing::error!("replication failed: {:#}", e);
|
||||||
}
|
}
|
||||||
@@ -215,6 +222,7 @@ pub async fn run(cfg: Config, mut manifest: Manifest) -> Result<()> {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
}
|
||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
+27
-10
@@ -55,25 +55,42 @@ pub fn gc_stale_targets(manifest: &Manifest, stale_hours: u64) -> Result<Vec<Str
|
|||||||
Ok(evicted)
|
Ok(evicted)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Eviction priority score: larger and older = higher = evict first.
|
||||||
|
/// log-scaled so a 10GB project doesn't completely dominate a 1GB project
|
||||||
|
/// and a week-old entry doesn't mask a 3-day-old one of similar value.
|
||||||
|
fn eviction_score(size_bytes: u64, secs_since_active: u64) -> f64 {
|
||||||
|
let size_factor = (size_bytes as f64 + 1.0).ln();
|
||||||
|
let age_factor = (secs_since_active as f64 + 1.0).ln();
|
||||||
|
size_factor * age_factor
|
||||||
|
}
|
||||||
|
|
||||||
pub fn gc_by_space(manifest: &mut Manifest, max_gb: f64) -> Result<Vec<String>> {
|
pub fn gc_by_space(manifest: &mut Manifest, max_gb: f64) -> Result<Vec<String>> {
|
||||||
|
let now = Utc::now();
|
||||||
let mut evicted = Vec::new();
|
let mut evicted = Vec::new();
|
||||||
loop {
|
loop {
|
||||||
let used = total_used_gb(manifest)?;
|
let used = total_used_gb(manifest)?;
|
||||||
if used <= max_gb {
|
if used <= max_gb {
|
||||||
break;
|
break;
|
||||||
}
|
}
|
||||||
// LRU eviction skips pinned projects — they may NEVER be evicted
|
// Score-based eviction: prefer removing large, stale projects over
|
||||||
// for space pressure. The trade-off: if every non-pinned project
|
// arbitrarily picking the oldest-active one. Pinned projects are
|
||||||
// is gone and we're still over `max_gb`, we stop and log; better
|
// never evicted regardless of score — operator intent is absolute.
|
||||||
// to over-allocate hot than to violate operator intent.
|
let best = manifest
|
||||||
let lru_name = manifest
|
|
||||||
.projects
|
.projects
|
||||||
.iter()
|
.iter()
|
||||||
.filter(|p| !p.pinned)
|
.filter(|p| !p.pinned && p.hot_target_path.exists())
|
||||||
.filter(|p| p.hot_target_path.exists())
|
.map(|p| {
|
||||||
.min_by_key(|p| p.last_active)
|
let size = target_size_bytes(&p.hot_target_path).unwrap_or(0);
|
||||||
.map(|p| p.name.clone());
|
let age_secs = p
|
||||||
match lru_name {
|
.last_active
|
||||||
|
.map(|t| (now - t).num_seconds().max(0) as u64)
|
||||||
|
.unwrap_or(0);
|
||||||
|
(p.name.clone(), eviction_score(size, age_secs))
|
||||||
|
})
|
||||||
|
.max_by(|(_, a), (_, b)| a.partial_cmp(b).unwrap_or(std::cmp::Ordering::Equal))
|
||||||
|
.map(|(name, _)| name);
|
||||||
|
|
||||||
|
match best {
|
||||||
None => {
|
None => {
|
||||||
tracing::warn!(
|
tracing::warn!(
|
||||||
used_gb = used,
|
used_gb = used,
|
||||||
|
|||||||
+17
-6
@@ -719,12 +719,23 @@ pub fn build_app_with_v2(
|
|||||||
v2_router = v2_router.merge(crate::serve_v2::build(v2s));
|
v2_router = v2_router.merge(crate::serve_v2::build(v2s));
|
||||||
}
|
}
|
||||||
if let Some(dir) = v2_static_dir {
|
if let Some(dir) = v2_static_dir {
|
||||||
v2_router = v2_router.nest_service(
|
// Redirect bare paths (no trailing slash) to their canonical
|
||||||
"/v2",
|
// slash-terminated form. Without this, browsers treat the path
|
||||||
tower_http::services::ServeDir::new(&dir).fallback(
|
// segment as a file and resolve relative asset paths one level
|
||||||
tower_http::services::ServeFile::new(dir.join("index.html")),
|
// too high, producing 404s for `./assets/…`.
|
||||||
),
|
let redirect_v2 = get(|| async {
|
||||||
);
|
axum::response::Redirect::permanent("/v2/")
|
||||||
|
});
|
||||||
|
let redirect_clawstor = get(|| async {
|
||||||
|
axum::response::Redirect::permanent("/clawstor/")
|
||||||
|
});
|
||||||
|
let spa = tower_http::services::ServeDir::new(&dir)
|
||||||
|
.fallback(tower_http::services::ServeFile::new(dir.join("index.html")));
|
||||||
|
v2_router = v2_router
|
||||||
|
.route("/v2", redirect_v2)
|
||||||
|
.route("/clawstor", redirect_clawstor)
|
||||||
|
.nest_service("/v2/", spa.clone())
|
||||||
|
.nest_service("/clawstor/", spa);
|
||||||
}
|
}
|
||||||
|
|
||||||
Router::new()
|
Router::new()
|
||||||
|
|||||||
+475
-3
@@ -13,12 +13,13 @@
|
|||||||
//! Design doc: `docs/dashboard-v2.md`.
|
//! Design doc: `docs/dashboard-v2.md`.
|
||||||
|
|
||||||
use axum::{
|
use axum::{
|
||||||
extract::{Path, State},
|
extract::{Path, Query, State},
|
||||||
http::StatusCode,
|
http::StatusCode,
|
||||||
routing::{get, post},
|
routing::{get, post},
|
||||||
Json, Router,
|
Json, Router,
|
||||||
};
|
};
|
||||||
use serde::{Deserialize, Serialize};
|
use serde::{Deserialize, Serialize};
|
||||||
|
use std::collections::{HashMap, VecDeque};
|
||||||
use std::net::SocketAddr;
|
use std::net::SocketAddr;
|
||||||
use std::sync::Arc;
|
use std::sync::Arc;
|
||||||
use std::time::Duration;
|
use std::time::Duration;
|
||||||
@@ -33,6 +34,197 @@ use crate::cluster::transport::{NodeIdentity, QuicClient};
|
|||||||
use crate::config::{Config, PeerEntry, TokenEntry};
|
use crate::config::{Config, PeerEntry, TokenEntry};
|
||||||
use crate::sessions::{LeasedTag, Session, SessionStore};
|
use crate::sessions::{LeasedTag, Session, SessionStore};
|
||||||
|
|
||||||
|
// ── metrics history ring buffer + drift-adaptive anomaly detection ──
|
||||||
|
|
||||||
|
const HISTORY_MAX_SAMPLES: usize = 1440; // 24h at 1-min resolution
|
||||||
|
/// Require at least this many samples before reporting an anomaly score.
|
||||||
|
/// Protects against false positives on a freshly-started daemon.
|
||||||
|
const ANOMALY_MIN_SAMPLES: u64 = 10;
|
||||||
|
/// Sum-of-z² above this value triggers a "warn" level anomaly.
|
||||||
|
/// Interpretation: average ~1.5σ deviation across 4 tracked metrics.
|
||||||
|
const ANOMALY_WARN_THRESHOLD: f64 = 9.0;
|
||||||
|
/// Sum-of-z² above this value triggers "alert".
|
||||||
|
const ANOMALY_ALERT_THRESHOLD: f64 = 16.0;
|
||||||
|
|
||||||
|
/// Welford's online algorithm for a running mean + variance.
|
||||||
|
/// Numerically stable, O(1) per update.
|
||||||
|
#[derive(Default, Clone)]
|
||||||
|
struct Welford {
|
||||||
|
n: u64,
|
||||||
|
mean: f64,
|
||||||
|
m2: f64,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl Welford {
|
||||||
|
fn update(&mut self, x: f64) {
|
||||||
|
self.n += 1;
|
||||||
|
let delta = x - self.mean;
|
||||||
|
self.mean += delta / self.n as f64;
|
||||||
|
self.m2 += delta * (x - self.mean);
|
||||||
|
}
|
||||||
|
|
||||||
|
fn stddev(&self) -> f64 {
|
||||||
|
if self.n < 2 { return 1.0; }
|
||||||
|
(self.m2 / (self.n - 1) as f64).sqrt()
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Squared z-score for a new observation (non-destructive).
|
||||||
|
///
|
||||||
|
/// Returns 0 when:
|
||||||
|
/// * fewer than ANOMALY_MIN_SAMPLES have been seen, OR
|
||||||
|
/// * the metric has been flat (stddev < 1% of its mean) — a near-zero
|
||||||
|
/// stddev means the baseline is perfectly stable, so the floor for
|
||||||
|
/// "meaningful deviation" is the metric's own magnitude, not 1e-9.
|
||||||
|
/// Without this guard, a metric stuck at 0.0 for the first N samples
|
||||||
|
/// then spiking to any non-zero value produces z² → ∞.
|
||||||
|
fn z_sq(&self, x: f64) -> f64 {
|
||||||
|
if self.n < ANOMALY_MIN_SAMPLES { return 0.0; }
|
||||||
|
let sd = self.stddev();
|
||||||
|
// Effective floor: 1% of |mean| when the signal is near-zero
|
||||||
|
// noise-free, otherwise the raw stddev.
|
||||||
|
let effective_sd = sd.max(self.mean.abs() * 0.01).max(1e-4);
|
||||||
|
let z = (x - self.mean) / effective_sd;
|
||||||
|
z * z
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Per-node rolling stats for drift-adaptive anomaly detection.
|
||||||
|
#[derive(Default, Clone)]
|
||||||
|
struct NodeAnomalyStats {
|
||||||
|
hot_pct: Welford,
|
||||||
|
hit_rate: Welford,
|
||||||
|
fs_pct: Welford,
|
||||||
|
chunk_miss_rate: Welford,
|
||||||
|
/// Most recent anomaly score (sum of z²); 0 until ANOMALY_MIN_SAMPLES.
|
||||||
|
pub last_score: f64,
|
||||||
|
/// Total samples seen (mirrors hot_pct.n for convenience).
|
||||||
|
pub n: u64,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl NodeAnomalyStats {
|
||||||
|
fn update(&mut self, s: &MetricSample) {
|
||||||
|
let hot_pct = if s.hot_max_bytes > 0 {
|
||||||
|
s.hot_used_bytes as f64 / s.hot_max_bytes as f64
|
||||||
|
} else {
|
||||||
|
0.0
|
||||||
|
};
|
||||||
|
let fs_pct = if s.fs_total_bytes > 0 {
|
||||||
|
s.fs_used_bytes as f64 / s.fs_total_bytes as f64
|
||||||
|
} else {
|
||||||
|
0.0
|
||||||
|
};
|
||||||
|
let total_chunks = s.has_chunk_hits + s.has_chunk_misses;
|
||||||
|
let chunk_miss_rate = if total_chunks > 0 {
|
||||||
|
s.has_chunk_misses as f64 / total_chunks as f64
|
||||||
|
} else {
|
||||||
|
0.0
|
||||||
|
};
|
||||||
|
|
||||||
|
self.last_score = self.hot_pct.z_sq(hot_pct)
|
||||||
|
+ self.hit_rate.z_sq(s.cache_hit_rate)
|
||||||
|
+ self.fs_pct.z_sq(fs_pct)
|
||||||
|
+ self.chunk_miss_rate.z_sq(chunk_miss_rate);
|
||||||
|
|
||||||
|
self.hot_pct.update(hot_pct);
|
||||||
|
self.hit_rate.update(s.cache_hit_rate);
|
||||||
|
self.fs_pct.update(fs_pct);
|
||||||
|
self.chunk_miss_rate.update(chunk_miss_rate);
|
||||||
|
self.n = self.hot_pct.n;
|
||||||
|
}
|
||||||
|
|
||||||
|
fn level(&self) -> &'static str {
|
||||||
|
if self.n < ANOMALY_MIN_SAMPLES { return "ok"; }
|
||||||
|
if self.last_score >= ANOMALY_ALERT_THRESHOLD { "alert" }
|
||||||
|
else if self.last_score >= ANOMALY_WARN_THRESHOLD { "warn" }
|
||||||
|
else { "ok" }
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// One time-series sample snapshotted from a peer's DashboardStatus RPC.
|
||||||
|
#[derive(Debug, Clone, Serialize)]
|
||||||
|
pub struct MetricSample {
|
||||||
|
pub unix_ts: u64,
|
||||||
|
pub hot_used_bytes: u64,
|
||||||
|
pub hot_max_bytes: u64,
|
||||||
|
pub cache_hit_rate: f64,
|
||||||
|
pub cache_hits: u64,
|
||||||
|
pub cache_misses: u64,
|
||||||
|
pub has_chunk_hits: u64,
|
||||||
|
pub has_chunk_misses: u64,
|
||||||
|
pub fs_used_bytes: u64,
|
||||||
|
pub fs_total_bytes: u64,
|
||||||
|
pub fs_available_bytes: u64,
|
||||||
|
/// Drift-adaptive anomaly score (sum of z² across 4 metrics).
|
||||||
|
/// `None` until the node has collected ANOMALY_MIN_SAMPLES.
|
||||||
|
#[serde(skip_serializing_if = "Option::is_none")]
|
||||||
|
pub anomaly_score: Option<f64>,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Serialize)]
|
||||||
|
pub struct AnomalyStatus {
|
||||||
|
pub node: String,
|
||||||
|
pub score: f64,
|
||||||
|
pub level: &'static str,
|
||||||
|
pub samples_used: u64,
|
||||||
|
}
|
||||||
|
|
||||||
|
pub struct MetricsHistory {
|
||||||
|
samples: HashMap<String, VecDeque<MetricSample>>,
|
||||||
|
anomaly: HashMap<String, NodeAnomalyStats>,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl MetricsHistory {
|
||||||
|
fn new() -> Self {
|
||||||
|
Self {
|
||||||
|
samples: HashMap::new(),
|
||||||
|
anomaly: HashMap::new(),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn push(&mut self, node: &str, mut sample: MetricSample) {
|
||||||
|
// Update Welford stats and embed anomaly score in the sample.
|
||||||
|
let stats = self.anomaly.entry(node.to_string()).or_default();
|
||||||
|
stats.update(&sample);
|
||||||
|
if stats.n >= ANOMALY_MIN_SAMPLES {
|
||||||
|
sample.anomaly_score = Some(stats.last_score);
|
||||||
|
}
|
||||||
|
|
||||||
|
let deque = self.samples.entry(node.to_string()).or_default();
|
||||||
|
deque.push_back(sample);
|
||||||
|
while deque.len() > HISTORY_MAX_SAMPLES {
|
||||||
|
deque.pop_front();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn get_last(&self, node: &str, limit: usize) -> Vec<MetricSample> {
|
||||||
|
let limit = limit.min(HISTORY_MAX_SAMPLES);
|
||||||
|
self.samples
|
||||||
|
.get(node)
|
||||||
|
.map(|d| {
|
||||||
|
let skip = d.len().saturating_sub(limit);
|
||||||
|
d.iter().skip(skip).cloned().collect()
|
||||||
|
})
|
||||||
|
.unwrap_or_default()
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn anomaly_statuses(&self) -> Vec<AnomalyStatus> {
|
||||||
|
let mut out: Vec<AnomalyStatus> = self
|
||||||
|
.anomaly
|
||||||
|
.iter()
|
||||||
|
.map(|(node, s)| AnomalyStatus {
|
||||||
|
node: node.clone(),
|
||||||
|
score: s.last_score,
|
||||||
|
level: s.level(),
|
||||||
|
samples_used: s.n,
|
||||||
|
})
|
||||||
|
.collect();
|
||||||
|
out.sort_by(|a, b| a.node.cmp(&b.node));
|
||||||
|
out
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// ─────────────────────────────────────────────────────────────────
|
||||||
|
|
||||||
/// Aggregator runtime: one QuicClient, one peer list, one identity.
|
/// Aggregator runtime: one QuicClient, one peer list, one identity.
|
||||||
///
|
///
|
||||||
/// The client is reused across every RPC (quinn holds one UDP
|
/// The client is reused across every RPC (quinn holds one UDP
|
||||||
@@ -68,6 +260,9 @@ pub struct V2State {
|
|||||||
/// leases and are reaped by a background sweeper when their
|
/// leases and are reaped by a background sweeper when their
|
||||||
/// `expires_at_unix` passes without a `renew` or `commit`.
|
/// `expires_at_unix` passes without a `renew` or `commit`.
|
||||||
pub sessions: SessionStore,
|
pub sessions: SessionStore,
|
||||||
|
/// Ring buffer of per-node metric samples (Phase B). 1440 entries
|
||||||
|
/// = 24h at 1-min resolution. Written by `metrics_poller`.
|
||||||
|
pub history: Arc<tokio::sync::Mutex<MetricsHistory>>,
|
||||||
}
|
}
|
||||||
|
|
||||||
impl V2State {
|
impl V2State {
|
||||||
@@ -91,9 +286,26 @@ impl V2State {
|
|||||||
.join("aggregator-sessions.json");
|
.join("aggregator-sessions.json");
|
||||||
let sessions = SessionStore::load(sessions_path)
|
let sessions = SessionStore::load(sessions_path)
|
||||||
.map_err(|e| anyhow::anyhow!("loading session store: {e}"))?;
|
.map_err(|e| anyhow::anyhow!("loading session store: {e}"))?;
|
||||||
|
// Include the serving node itself as a peer so it appears in
|
||||||
|
// fleet/storage/aggregated views. Skip if already listed.
|
||||||
|
let self_name = cfg.node.name.clone();
|
||||||
|
let mut peers = cluster.peers.clone();
|
||||||
|
if !peers.iter().any(|p| p.name == self_name) {
|
||||||
|
if cluster.bind_lan.is_some() || cluster.bind_tailscale.is_some() {
|
||||||
|
peers.insert(
|
||||||
|
0,
|
||||||
|
crate::config::PeerEntry {
|
||||||
|
name: self_name.clone(),
|
||||||
|
zone: cluster.zone.clone(),
|
||||||
|
lan_addr: cluster.bind_lan,
|
||||||
|
tailscale_addr: cluster.bind_tailscale,
|
||||||
|
},
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
Ok(Self {
|
Ok(Self {
|
||||||
aggregator_name: cfg.node.name.clone(),
|
aggregator_name: self_name,
|
||||||
peers: cluster.peers.clone(),
|
peers,
|
||||||
client: std::sync::Arc::new(client),
|
client: std::sync::Arc::new(client),
|
||||||
default_rpc_port_offset: 1,
|
default_rpc_port_offset: 1,
|
||||||
api_token: cfg.api_token.clone(),
|
api_token: cfg.api_token.clone(),
|
||||||
@@ -103,6 +315,7 @@ impl V2State {
|
|||||||
.map(|a| a.tokens.clone())
|
.map(|a| a.tokens.clone())
|
||||||
.unwrap_or_default(),
|
.unwrap_or_default(),
|
||||||
sessions,
|
sessions,
|
||||||
|
history: Arc::new(tokio::sync::Mutex::new(MetricsHistory::new())),
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -181,6 +394,15 @@ pub struct NodeStatusV2 {
|
|||||||
pub cache: Option<CacheSummary>,
|
pub cache: Option<CacheSummary>,
|
||||||
#[serde(default)]
|
#[serde(default)]
|
||||||
pub timers: Vec<TimerStatus>,
|
pub timers: Vec<TimerStatus>,
|
||||||
|
/// Unix timestamp (seconds) when the peer's daemon last started.
|
||||||
|
#[serde(default)]
|
||||||
|
pub daemon_started_unix: Option<u64>,
|
||||||
|
/// 1-minute load average. None on non-Linux or when the peer is offline.
|
||||||
|
#[serde(default)]
|
||||||
|
pub cpu_load_1m: Option<f32>,
|
||||||
|
/// Lifeguard-style health score: 0 = healthy, higher = stressed (T3.3).
|
||||||
|
#[serde(default)]
|
||||||
|
pub health_score: u8,
|
||||||
/// `true` when the aggregator successfully talked to the peer;
|
/// `true` when the aggregator successfully talked to the peer;
|
||||||
/// `false` when the RPC failed. Frontend uses this to badge the
|
/// `false` when the RPC failed. Frontend uses this to badge the
|
||||||
/// card as offline.
|
/// card as offline.
|
||||||
@@ -207,6 +429,9 @@ impl NodeStatusV2 {
|
|||||||
mount: r.mount,
|
mount: r.mount,
|
||||||
cache: r.cache,
|
cache: r.cache,
|
||||||
timers: r.timers,
|
timers: r.timers,
|
||||||
|
daemon_started_unix: r.daemon_started_unix,
|
||||||
|
cpu_load_1m: r.cpu_load_1m,
|
||||||
|
health_score: r.health_score,
|
||||||
online: true,
|
online: true,
|
||||||
error: None,
|
error: None,
|
||||||
};
|
};
|
||||||
@@ -232,6 +457,9 @@ impl NodeStatusV2 {
|
|||||||
mount: None,
|
mount: None,
|
||||||
cache: None,
|
cache: None,
|
||||||
timers: Vec::new(),
|
timers: Vec::new(),
|
||||||
|
daemon_started_unix: None,
|
||||||
|
cpu_load_1m: None,
|
||||||
|
health_score: 0,
|
||||||
online: false,
|
online: false,
|
||||||
error: Some(e),
|
error: Some(e),
|
||||||
}
|
}
|
||||||
@@ -509,6 +737,7 @@ impl V2State {
|
|||||||
api_token: self.api_token.clone(),
|
api_token: self.api_token.clone(),
|
||||||
token_entries: self.token_entries.clone(),
|
token_entries: self.token_entries.clone(),
|
||||||
sessions: self.sessions.clone(),
|
sessions: self.sessions.clone(),
|
||||||
|
history: self.history.clone(),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -1284,6 +1513,242 @@ async fn reap_expired(state: Arc<V2State>, sess: Session) {
|
|||||||
///
|
///
|
||||||
/// Also spawns the background TTL sweeper (Phase 9 S1). The task is
|
/// Also spawns the background TTL sweeper (Phase 9 S1). The task is
|
||||||
/// detached — its lifetime is the process lifetime.
|
/// detached — its lifetime is the process lifetime.
|
||||||
|
// ── metrics poller (Phase B) ─────────────────────────────────────
|
||||||
|
|
||||||
|
/// Background task: poll every peer every 60s, append a `MetricSample`
|
||||||
|
/// to the ring buffer. Runs indefinitely — dropped only on daemon exit.
|
||||||
|
async fn metrics_poller(state: Arc<V2State>) {
|
||||||
|
let mut interval = tokio::time::interval(Duration::from_secs(60));
|
||||||
|
interval.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip);
|
||||||
|
loop {
|
||||||
|
interval.tick().await;
|
||||||
|
for peer in &state.peers {
|
||||||
|
let peer_name = peer.name.clone();
|
||||||
|
match state.fetch_node(peer).await {
|
||||||
|
Ok(r) => {
|
||||||
|
let now = std::time::SystemTime::now()
|
||||||
|
.duration_since(std::time::UNIX_EPOCH)
|
||||||
|
.unwrap_or_default()
|
||||||
|
.as_secs();
|
||||||
|
let sample = MetricSample {
|
||||||
|
unix_ts: now,
|
||||||
|
hot_used_bytes: r.hot.as_ref().map(|h| h.used_bytes).unwrap_or(0),
|
||||||
|
hot_max_bytes: r.hot.as_ref().map(|h| h.max_bytes).unwrap_or(0),
|
||||||
|
cache_hit_rate: r.cache.as_ref().map(|c| c.hit_rate).unwrap_or(0.0),
|
||||||
|
cache_hits: r.cache.as_ref().map(|c| c.hits).unwrap_or(0),
|
||||||
|
cache_misses: r.cache.as_ref().map(|c| c.misses).unwrap_or(0),
|
||||||
|
has_chunk_hits: r.cache.as_ref().map(|c| c.has_chunk_hits).unwrap_or(0),
|
||||||
|
has_chunk_misses: r.cache.as_ref().map(|c| c.has_chunk_misses).unwrap_or(0),
|
||||||
|
fs_used_bytes: r.filesystem.as_ref().map(|f| f.used_bytes).unwrap_or(0),
|
||||||
|
fs_total_bytes: r.filesystem.as_ref().map(|f| f.total_bytes).unwrap_or(0),
|
||||||
|
fs_available_bytes: r.filesystem.as_ref().map(|f| f.available_bytes).unwrap_or(0),
|
||||||
|
// Filled in by MetricsHistory::push after Welford update.
|
||||||
|
anomaly_score: None,
|
||||||
|
};
|
||||||
|
let mut hist = state.history.lock().await;
|
||||||
|
hist.push(&peer_name, sample);
|
||||||
|
}
|
||||||
|
Err(e) => {
|
||||||
|
tracing::debug!(peer = %peer_name, error = %e, "metrics poll skipped");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Deserialize)]
|
||||||
|
struct MetricsHistoryQuery {
|
||||||
|
limit: Option<usize>,
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn handle_metrics_history(
|
||||||
|
Path(name): Path<String>,
|
||||||
|
Query(q): Query<MetricsHistoryQuery>,
|
||||||
|
State(s): State<Arc<V2State>>,
|
||||||
|
) -> Json<Vec<MetricSample>> {
|
||||||
|
let limit = q.limit.unwrap_or(60).min(1440);
|
||||||
|
let hist = s.history.lock().await;
|
||||||
|
Json(hist.get_last(&name, limit))
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn handle_anomalies(State(s): State<Arc<V2State>>) -> Json<Vec<AnomalyStatus>> {
|
||||||
|
let hist = s.history.lock().await;
|
||||||
|
Json(hist.anomaly_statuses())
|
||||||
|
}
|
||||||
|
|
||||||
|
/// T2.7: cross-node cache warming candidates.
|
||||||
|
///
|
||||||
|
/// A warming candidate is a fingerprint that is frequently accessed on
|
||||||
|
/// at least one node but absent (never hit) on at least one other node.
|
||||||
|
/// Pushing the blob to the missing node turns the next GetRef there
|
||||||
|
/// into a local hit instead of a cross-node forwarded fetch.
|
||||||
|
#[derive(serde::Serialize)]
|
||||||
|
struct WarmingCandidate {
|
||||||
|
fingerprint_hex: String,
|
||||||
|
/// Nodes where this fingerprint has been accessed. Sorted.
|
||||||
|
hot_on: Vec<String>,
|
||||||
|
/// Nodes where the fingerprint has zero recorded hits. Sorted.
|
||||||
|
missing_on: Vec<String>,
|
||||||
|
/// Highest hit count seen across all hot nodes.
|
||||||
|
max_hit_count: u64,
|
||||||
|
/// Most recent access timestamp across all hot nodes.
|
||||||
|
last_hit_unix: u64,
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn handle_hot_refs(State(s): State<Arc<V2State>>) -> Json<Vec<WarmingCandidate>> {
|
||||||
|
// Collect each node's hot refs via the existing gather_storage fan-out.
|
||||||
|
let storage = gather_storage(&s).await;
|
||||||
|
let node_names: Vec<String> = storage.iter().map(|(n, _)| n.clone()).collect();
|
||||||
|
|
||||||
|
// Build a map: fingerprint_hex → { node → (hit_count, last_hit_unix) }
|
||||||
|
let mut fp_map: std::collections::HashMap<
|
||||||
|
String,
|
||||||
|
std::collections::HashMap<String, (u64, u64)>,
|
||||||
|
> = std::collections::HashMap::new();
|
||||||
|
|
||||||
|
for (node, reply) in &storage {
|
||||||
|
for hr in &reply.hot_refs {
|
||||||
|
fp_map
|
||||||
|
.entry(hr.fingerprint_hex.clone())
|
||||||
|
.or_default()
|
||||||
|
.insert(node.clone(), (hr.hit_count, hr.last_hit_unix));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// A candidate is hot on ≥1 node and missing on ≥1 other node.
|
||||||
|
let mut candidates: Vec<WarmingCandidate> = fp_map
|
||||||
|
.into_iter()
|
||||||
|
.filter_map(|(fp, node_hits)| {
|
||||||
|
let missing_on: Vec<String> = node_names
|
||||||
|
.iter()
|
||||||
|
.filter(|n| !node_hits.contains_key(*n))
|
||||||
|
.cloned()
|
||||||
|
.collect();
|
||||||
|
if missing_on.is_empty() {
|
||||||
|
return None; // present (or hit-tracked) on all nodes
|
||||||
|
}
|
||||||
|
let mut hot_on: Vec<String> = node_hits.keys().cloned().collect();
|
||||||
|
hot_on.sort();
|
||||||
|
let max_hit_count = node_hits.values().map(|&(c, _)| c).max().unwrap_or(0);
|
||||||
|
let last_hit_unix = node_hits.values().map(|&(_, t)| t).max().unwrap_or(0);
|
||||||
|
Some(WarmingCandidate {
|
||||||
|
fingerprint_hex: fp,
|
||||||
|
hot_on,
|
||||||
|
missing_on: { let mut v = missing_on; v.sort(); v },
|
||||||
|
max_hit_count,
|
||||||
|
last_hit_unix,
|
||||||
|
})
|
||||||
|
})
|
||||||
|
.collect();
|
||||||
|
|
||||||
|
// Hottest candidates first.
|
||||||
|
candidates.sort_by(|a, b| b.max_hit_count.cmp(&a.max_hit_count));
|
||||||
|
candidates.truncate(50);
|
||||||
|
Json(candidates)
|
||||||
|
}
|
||||||
|
|
||||||
|
/// T1.2: fleet-wide maintenance event log.
|
||||||
|
///
|
||||||
|
/// Aggregates GC/scrub/repair events from every peer and returns them
|
||||||
|
/// sorted newest-first, so the operator sees a single unified timeline
|
||||||
|
/// rather than having to check each node individually.
|
||||||
|
#[derive(serde::Serialize)]
|
||||||
|
struct MaintenanceRow {
|
||||||
|
node: String,
|
||||||
|
kind: String,
|
||||||
|
unix_ts: u64,
|
||||||
|
chunks_scanned: usize,
|
||||||
|
chunks_removed: usize,
|
||||||
|
bytes_reclaimed: u64,
|
||||||
|
error: Option<String>,
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn handle_maintenance(State(s): State<Arc<V2State>>) -> Json<Vec<MaintenanceRow>> {
|
||||||
|
let storage = gather_storage(&s).await;
|
||||||
|
let mut rows: Vec<MaintenanceRow> = storage
|
||||||
|
.iter()
|
||||||
|
.flat_map(|(node, reply)| {
|
||||||
|
reply.maintenance_events.iter().map(move |ev| MaintenanceRow {
|
||||||
|
node: node.clone(),
|
||||||
|
kind: ev.kind.clone(),
|
||||||
|
unix_ts: ev.unix_ts,
|
||||||
|
chunks_scanned: ev.chunks_scanned,
|
||||||
|
chunks_removed: ev.chunks_removed,
|
||||||
|
bytes_reclaimed: ev.bytes_reclaimed,
|
||||||
|
error: ev.error.clone(),
|
||||||
|
})
|
||||||
|
})
|
||||||
|
.collect();
|
||||||
|
rows.sort_by(|a, b| b.unix_ts.cmp(&a.unix_ts));
|
||||||
|
Json(rows)
|
||||||
|
}
|
||||||
|
|
||||||
|
/// T2.8: cache pollution candidates (ACPC-inspired).
|
||||||
|
///
|
||||||
|
/// A blob is "polluting" the hot tier when it is large relative to how often
|
||||||
|
/// it is actually accessed. Pollution score = size_bytes / (hit_count + 1).
|
||||||
|
/// Zero-hit blobs score highest; large blobs that are hit frequently score low.
|
||||||
|
#[derive(serde::Serialize)]
|
||||||
|
struct PollutionCandidate {
|
||||||
|
node: String,
|
||||||
|
fingerprint_hex: String,
|
||||||
|
blob_id_hex: String,
|
||||||
|
size_bytes: u64,
|
||||||
|
/// GetRef hits since last daemon restart.
|
||||||
|
hit_count: u64,
|
||||||
|
/// size_bytes / (hit_count + 1) — higher = more polluting.
|
||||||
|
pollution_score: f64,
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn handle_pollution(State(s): State<Arc<V2State>>) -> Json<Vec<PollutionCandidate>> {
|
||||||
|
let storage = gather_storage(&s).await;
|
||||||
|
let mut candidates = Vec::new();
|
||||||
|
|
||||||
|
for (node, reply) in &storage {
|
||||||
|
// fingerprint → hit_count
|
||||||
|
let hot_map: std::collections::HashMap<&str, u64> = reply
|
||||||
|
.hot_refs
|
||||||
|
.iter()
|
||||||
|
.map(|hr| (hr.fingerprint_hex.as_str(), hr.hit_count))
|
||||||
|
.collect();
|
||||||
|
|
||||||
|
// blob_id → size_bytes
|
||||||
|
let blob_map: std::collections::HashMap<&str, u64> = reply
|
||||||
|
.blobs_sample
|
||||||
|
.iter()
|
||||||
|
.map(|b| (b.blob_id_hex.as_str(), b.size_bytes))
|
||||||
|
.collect();
|
||||||
|
|
||||||
|
for dr in &reply.refs_sample {
|
||||||
|
let size_bytes = match blob_map.get(dr.blob_id_hex.as_str()) {
|
||||||
|
Some(&s) if s > 0 => s,
|
||||||
|
_ => continue, // skip unknown-size entries
|
||||||
|
};
|
||||||
|
let hit_count = *hot_map.get(dr.fingerprint_hex.as_str()).unwrap_or(&0);
|
||||||
|
let pollution_score = size_bytes as f64 / (hit_count as f64 + 1.0);
|
||||||
|
candidates.push(PollutionCandidate {
|
||||||
|
node: node.clone(),
|
||||||
|
fingerprint_hex: dr.fingerprint_hex.clone(),
|
||||||
|
blob_id_hex: dr.blob_id_hex.clone(),
|
||||||
|
size_bytes,
|
||||||
|
hit_count,
|
||||||
|
pollution_score,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Highest pollution score first (largest cold blobs at the top).
|
||||||
|
candidates.sort_by(|a, b| {
|
||||||
|
b.pollution_score
|
||||||
|
.partial_cmp(&a.pollution_score)
|
||||||
|
.unwrap_or(std::cmp::Ordering::Equal)
|
||||||
|
});
|
||||||
|
candidates.truncate(50);
|
||||||
|
Json(candidates)
|
||||||
|
}
|
||||||
|
|
||||||
|
// ─────────────────────────────────────────────────────────────────
|
||||||
|
|
||||||
pub fn build(state: Arc<V2State>) -> Router {
|
pub fn build(state: Arc<V2State>) -> Router {
|
||||||
// Spawn the sweeper. 15s tick is a reasonable balance: quick
|
// Spawn the sweeper. 15s tick is a reasonable balance: quick
|
||||||
// enough that a mid-wizard-close cleanup feels prompt, slow
|
// enough that a mid-wizard-close cleanup feels prompt, slow
|
||||||
@@ -1300,9 +1765,16 @@ pub fn build(state: Arc<V2State>) -> Router {
|
|||||||
},
|
},
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
// Spawn the metrics ring-buffer poller (Phase B).
|
||||||
|
tokio::spawn(metrics_poller(state.clone()));
|
||||||
Router::new()
|
Router::new()
|
||||||
.route("/api/v2/fleet", get(handle_fleet))
|
.route("/api/v2/fleet", get(handle_fleet))
|
||||||
.route("/api/v2/node/:name/status", get(handle_node_status))
|
.route("/api/v2/node/:name/status", get(handle_node_status))
|
||||||
|
.route("/api/v2/node/:name/metrics-history", get(handle_metrics_history))
|
||||||
|
.route("/api/v2/anomalies", get(handle_anomalies))
|
||||||
|
.route("/api/v2/hot-refs", get(handle_hot_refs))
|
||||||
|
.route("/api/v2/pollution", get(handle_pollution))
|
||||||
|
.route("/api/v2/maintenance", get(handle_maintenance))
|
||||||
.route("/api/v2/storage/blobs", get(handle_blobs))
|
.route("/api/v2/storage/blobs", get(handle_blobs))
|
||||||
.route("/api/v2/storage/tags", get(handle_tags))
|
.route("/api/v2/storage/tags", get(handle_tags))
|
||||||
.route("/api/v2/storage/refs", get(handle_refs))
|
.route("/api/v2/storage/refs", get(handle_refs))
|
||||||
|
|||||||
Generated
+345
-32
@@ -10,6 +10,7 @@
|
|||||||
"dependencies": {
|
"dependencies": {
|
||||||
"react": "^19.2.6",
|
"react": "^19.2.6",
|
||||||
"react-dom": "^19.2.6",
|
"react-dom": "^19.2.6",
|
||||||
|
"recharts": "^3.10.0",
|
||||||
"wouter": "^3.7.1"
|
"wouter": "^3.7.1"
|
||||||
},
|
},
|
||||||
"devDependencies": {
|
"devDependencies": {
|
||||||
@@ -176,6 +177,31 @@
|
|||||||
"url": "https://github.com/sponsors/Boshen"
|
"url": "https://github.com/sponsors/Boshen"
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
|
"node_modules/@reduxjs/toolkit": {
|
||||||
|
"version": "2.12.0",
|
||||||
|
"resolved": "https://registry.npmjs.org/@reduxjs/toolkit/-/toolkit-2.12.0.tgz",
|
||||||
|
"integrity": "sha512-KiT+RzZbp6mQET+Mg+h2c97+9j1sNflUxQkIHI7Yuzf6Peu+OYpmkn6nbHWmLLWj+1ZODUJFwGZ7gx3L9R9EOw==",
|
||||||
|
"dependencies": {
|
||||||
|
"@standard-schema/spec": "^1.0.0",
|
||||||
|
"@standard-schema/utils": "^0.3.0",
|
||||||
|
"immer": "^11.0.0",
|
||||||
|
"redux": "^5.0.1",
|
||||||
|
"redux-thunk": "^3.1.0",
|
||||||
|
"reselect": "^5.1.0"
|
||||||
|
},
|
||||||
|
"peerDependencies": {
|
||||||
|
"react": "^16.9.0 || ^17.0.0 || ^18 || ^19",
|
||||||
|
"react-redux": "^7.2.1 || ^8.1.3 || ^9.0.0"
|
||||||
|
},
|
||||||
|
"peerDependenciesMeta": {
|
||||||
|
"react": {
|
||||||
|
"optional": true
|
||||||
|
},
|
||||||
|
"react-redux": {
|
||||||
|
"optional": true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
"node_modules/@rolldown/binding-android-arm64": {
|
"node_modules/@rolldown/binding-android-arm64": {
|
||||||
"version": "1.1.5",
|
"version": "1.1.5",
|
||||||
"resolved": "https://registry.npmjs.org/@rolldown/binding-android-arm64/-/binding-android-arm64-1.1.5.tgz",
|
"resolved": "https://registry.npmjs.org/@rolldown/binding-android-arm64/-/binding-android-arm64-1.1.5.tgz",
|
||||||
@@ -269,9 +295,6 @@
|
|||||||
"arm64"
|
"arm64"
|
||||||
],
|
],
|
||||||
"dev": true,
|
"dev": true,
|
||||||
"libc": [
|
|
||||||
"glibc"
|
|
||||||
],
|
|
||||||
"license": "MIT",
|
"license": "MIT",
|
||||||
"optional": true,
|
"optional": true,
|
||||||
"os": [
|
"os": [
|
||||||
@@ -289,9 +312,6 @@
|
|||||||
"arm64"
|
"arm64"
|
||||||
],
|
],
|
||||||
"dev": true,
|
"dev": true,
|
||||||
"libc": [
|
|
||||||
"musl"
|
|
||||||
],
|
|
||||||
"license": "MIT",
|
"license": "MIT",
|
||||||
"optional": true,
|
"optional": true,
|
||||||
"os": [
|
"os": [
|
||||||
@@ -309,9 +329,6 @@
|
|||||||
"ppc64"
|
"ppc64"
|
||||||
],
|
],
|
||||||
"dev": true,
|
"dev": true,
|
||||||
"libc": [
|
|
||||||
"glibc"
|
|
||||||
],
|
|
||||||
"license": "MIT",
|
"license": "MIT",
|
||||||
"optional": true,
|
"optional": true,
|
||||||
"os": [
|
"os": [
|
||||||
@@ -329,9 +346,6 @@
|
|||||||
"s390x"
|
"s390x"
|
||||||
],
|
],
|
||||||
"dev": true,
|
"dev": true,
|
||||||
"libc": [
|
|
||||||
"glibc"
|
|
||||||
],
|
|
||||||
"license": "MIT",
|
"license": "MIT",
|
||||||
"optional": true,
|
"optional": true,
|
||||||
"os": [
|
"os": [
|
||||||
@@ -349,9 +363,6 @@
|
|||||||
"x64"
|
"x64"
|
||||||
],
|
],
|
||||||
"dev": true,
|
"dev": true,
|
||||||
"libc": [
|
|
||||||
"glibc"
|
|
||||||
],
|
|
||||||
"license": "MIT",
|
"license": "MIT",
|
||||||
"optional": true,
|
"optional": true,
|
||||||
"os": [
|
"os": [
|
||||||
@@ -369,9 +380,6 @@
|
|||||||
"x64"
|
"x64"
|
||||||
],
|
],
|
||||||
"dev": true,
|
"dev": true,
|
||||||
"libc": [
|
|
||||||
"musl"
|
|
||||||
],
|
|
||||||
"license": "MIT",
|
"license": "MIT",
|
||||||
"optional": true,
|
"optional": true,
|
||||||
"os": [
|
"os": [
|
||||||
@@ -458,6 +466,16 @@
|
|||||||
"dev": true,
|
"dev": true,
|
||||||
"license": "MIT"
|
"license": "MIT"
|
||||||
},
|
},
|
||||||
|
"node_modules/@standard-schema/spec": {
|
||||||
|
"version": "1.1.0",
|
||||||
|
"resolved": "https://registry.npmjs.org/@standard-schema/spec/-/spec-1.1.0.tgz",
|
||||||
|
"integrity": "sha512-l2aFy5jALhniG5HgqrD6jXLi/rUWrKvqN/qJx6yoJsgKhblVd+iqqU4RCXavm/jPityDo5TCvKMnpjKnOriy0w=="
|
||||||
|
},
|
||||||
|
"node_modules/@standard-schema/utils": {
|
||||||
|
"version": "0.3.0",
|
||||||
|
"resolved": "https://registry.npmjs.org/@standard-schema/utils/-/utils-0.3.0.tgz",
|
||||||
|
"integrity": "sha512-e7Mew686owMaPJVNNLs55PUvgz371nKgwsc4vxE49zsODpJEnxgxRo2y/OKrqueavXgZNMDVj3DdHFlaSAeU8g=="
|
||||||
|
},
|
||||||
"node_modules/@tybys/wasm-util": {
|
"node_modules/@tybys/wasm-util": {
|
||||||
"version": "0.10.3",
|
"version": "0.10.3",
|
||||||
"resolved": "https://registry.npmjs.org/@tybys/wasm-util/-/wasm-util-0.10.3.tgz",
|
"resolved": "https://registry.npmjs.org/@tybys/wasm-util/-/wasm-util-0.10.3.tgz",
|
||||||
@@ -469,11 +487,65 @@
|
|||||||
"tslib": "^2.4.0"
|
"tslib": "^2.4.0"
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
|
"node_modules/@types/d3-array": {
|
||||||
|
"version": "3.2.2",
|
||||||
|
"resolved": "https://registry.npmjs.org/@types/d3-array/-/d3-array-3.2.2.tgz",
|
||||||
|
"integrity": "sha512-hOLWVbm7uRza0BYXpIIW5pxfrKe0W+D5lrFiAEYR+pb6w3N2SwSMaJbXdUfSEv+dT4MfHBLtn5js0LAWaO6otw=="
|
||||||
|
},
|
||||||
|
"node_modules/@types/d3-color": {
|
||||||
|
"version": "3.1.3",
|
||||||
|
"resolved": "https://registry.npmjs.org/@types/d3-color/-/d3-color-3.1.3.tgz",
|
||||||
|
"integrity": "sha512-iO90scth9WAbmgv7ogoq57O9YpKmFBbmoEoCHDB2xMBY0+/KVrqAaCDyCE16dUspeOvIxFFRI+0sEtqDqy2b4A=="
|
||||||
|
},
|
||||||
|
"node_modules/@types/d3-ease": {
|
||||||
|
"version": "3.0.2",
|
||||||
|
"resolved": "https://registry.npmjs.org/@types/d3-ease/-/d3-ease-3.0.2.tgz",
|
||||||
|
"integrity": "sha512-NcV1JjO5oDzoK26oMzbILE6HW7uVXOHLQvHshBUW4UMdZGfiY6v5BeQwh9a9tCzv+CeefZQHJt5SRgK154RtiA=="
|
||||||
|
},
|
||||||
|
"node_modules/@types/d3-interpolate": {
|
||||||
|
"version": "3.0.4",
|
||||||
|
"resolved": "https://registry.npmjs.org/@types/d3-interpolate/-/d3-interpolate-3.0.4.tgz",
|
||||||
|
"integrity": "sha512-mgLPETlrpVV1YRJIglr4Ez47g7Yxjl1lj7YKsiMCb27VJH9W8NVM6Bb9d8kkpG/uAQS5AmbA48q2IAolKKo1MA==",
|
||||||
|
"dependencies": {
|
||||||
|
"@types/d3-color": "*"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"node_modules/@types/d3-path": {
|
||||||
|
"version": "3.1.1",
|
||||||
|
"resolved": "https://registry.npmjs.org/@types/d3-path/-/d3-path-3.1.1.tgz",
|
||||||
|
"integrity": "sha512-VMZBYyQvbGmWyWVea0EHs/BwLgxc+MKi1zLDCONksozI4YJMcTt8ZEuIR4Sb1MMTE8MMW49v0IwI5+b7RmfWlg=="
|
||||||
|
},
|
||||||
|
"node_modules/@types/d3-scale": {
|
||||||
|
"version": "4.0.9",
|
||||||
|
"resolved": "https://registry.npmjs.org/@types/d3-scale/-/d3-scale-4.0.9.tgz",
|
||||||
|
"integrity": "sha512-dLmtwB8zkAeO/juAMfnV+sItKjlsw2lKdZVVy6LRr0cBmegxSABiLEpGVmSJJ8O08i4+sGR6qQtb6WtuwJdvVw==",
|
||||||
|
"dependencies": {
|
||||||
|
"@types/d3-time": "*"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"node_modules/@types/d3-shape": {
|
||||||
|
"version": "3.1.8",
|
||||||
|
"resolved": "https://registry.npmjs.org/@types/d3-shape/-/d3-shape-3.1.8.tgz",
|
||||||
|
"integrity": "sha512-lae0iWfcDeR7qt7rA88BNiqdvPS5pFVPpo5OfjElwNaT2yyekbM0C9vK+yqBqEmHr6lDkRnYNoTBYlAgJa7a4w==",
|
||||||
|
"dependencies": {
|
||||||
|
"@types/d3-path": "*"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"node_modules/@types/d3-time": {
|
||||||
|
"version": "3.0.4",
|
||||||
|
"resolved": "https://registry.npmjs.org/@types/d3-time/-/d3-time-3.0.4.tgz",
|
||||||
|
"integrity": "sha512-yuzZug1nkAAaBlBBikKZTgzCeA+k1uy4ZFwWANOfKw5z5LRhV0gNA7gNkKm7HoK+HRN0wX3EkxGk0fpbWhmB7g=="
|
||||||
|
},
|
||||||
|
"node_modules/@types/d3-timer": {
|
||||||
|
"version": "3.0.2",
|
||||||
|
"resolved": "https://registry.npmjs.org/@types/d3-timer/-/d3-timer-3.0.2.tgz",
|
||||||
|
"integrity": "sha512-Ps3T8E8dZDam6fUyNiMkekK3XUsaUEik+idO9/YjPtfj2qruF8tFBXS7XhtE4iIXBLxhmLjP3SXpLhVf21I9Lw=="
|
||||||
|
},
|
||||||
"node_modules/@types/react": {
|
"node_modules/@types/react": {
|
||||||
"version": "19.2.17",
|
"version": "19.2.17",
|
||||||
"resolved": "https://registry.npmjs.org/@types/react/-/react-19.2.17.tgz",
|
"resolved": "https://registry.npmjs.org/@types/react/-/react-19.2.17.tgz",
|
||||||
"integrity": "sha512-MXfmqaVPEVgkBT/aY0aGCkRWWtByiYQXo3xdQ8r5RzuFrPiRn8Gar2tQdXSUQ2GKV3bkXckek89V8wQBY2Q/Aw==",
|
"integrity": "sha512-MXfmqaVPEVgkBT/aY0aGCkRWWtByiYQXo3xdQ8r5RzuFrPiRn8Gar2tQdXSUQ2GKV3bkXckek89V8wQBY2Q/Aw==",
|
||||||
"dev": true,
|
"devOptional": true,
|
||||||
"license": "MIT",
|
"license": "MIT",
|
||||||
"dependencies": {
|
"dependencies": {
|
||||||
"csstype": "^3.2.2"
|
"csstype": "^3.2.2"
|
||||||
@@ -489,6 +561,11 @@
|
|||||||
"@types/react": "^19.2.0"
|
"@types/react": "^19.2.0"
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
|
"node_modules/@types/use-sync-external-store": {
|
||||||
|
"version": "0.0.6",
|
||||||
|
"resolved": "https://registry.npmjs.org/@types/use-sync-external-store/-/use-sync-external-store-0.0.6.tgz",
|
||||||
|
"integrity": "sha512-zFDAD+tlpf2r4asuHEj0XH6pY6i0g5NeAHPn+15wk3BV6JA69eERFXC1gyGThDkVa1zCyKr5jox1+2LbV/AMLg=="
|
||||||
|
},
|
||||||
"node_modules/@vitejs/plugin-react": {
|
"node_modules/@vitejs/plugin-react": {
|
||||||
"version": "6.0.3",
|
"version": "6.0.3",
|
||||||
"resolved": "https://registry.npmjs.org/@vitejs/plugin-react/-/plugin-react-6.0.3.tgz",
|
"resolved": "https://registry.npmjs.org/@vitejs/plugin-react/-/plugin-react-6.0.3.tgz",
|
||||||
@@ -722,6 +799,14 @@
|
|||||||
"node": ">= 6"
|
"node": ">= 6"
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
|
"node_modules/clsx": {
|
||||||
|
"version": "2.1.1",
|
||||||
|
"resolved": "https://registry.npmjs.org/clsx/-/clsx-2.1.1.tgz",
|
||||||
|
"integrity": "sha512-eYm0QWBtUrBWZWG0d386OGAw16Z995PiOVo2B7bjWSbHedGl5e0ZWaq65kOGgUSNesEIDkB9ISbTg/JK9dhCZA==",
|
||||||
|
"engines": {
|
||||||
|
"node": ">=6"
|
||||||
|
}
|
||||||
|
},
|
||||||
"node_modules/commander": {
|
"node_modules/commander": {
|
||||||
"version": "4.1.1",
|
"version": "4.1.1",
|
||||||
"resolved": "https://registry.npmjs.org/commander/-/commander-4.1.1.tgz",
|
"resolved": "https://registry.npmjs.org/commander/-/commander-4.1.1.tgz",
|
||||||
@@ -749,9 +834,124 @@
|
|||||||
"version": "3.2.3",
|
"version": "3.2.3",
|
||||||
"resolved": "https://registry.npmjs.org/csstype/-/csstype-3.2.3.tgz",
|
"resolved": "https://registry.npmjs.org/csstype/-/csstype-3.2.3.tgz",
|
||||||
"integrity": "sha512-z1HGKcYy2xA8AGQfwrn0PAy+PB7X/GSj3UVJW9qKyn43xWa+gl5nXmU4qqLMRzWVLFC8KusUX8T/0kCiOYpAIQ==",
|
"integrity": "sha512-z1HGKcYy2xA8AGQfwrn0PAy+PB7X/GSj3UVJW9qKyn43xWa+gl5nXmU4qqLMRzWVLFC8KusUX8T/0kCiOYpAIQ==",
|
||||||
"dev": true,
|
"devOptional": true,
|
||||||
"license": "MIT"
|
"license": "MIT"
|
||||||
},
|
},
|
||||||
|
"node_modules/d3-array": {
|
||||||
|
"version": "3.2.4",
|
||||||
|
"resolved": "https://registry.npmjs.org/d3-array/-/d3-array-3.2.4.tgz",
|
||||||
|
"integrity": "sha512-tdQAmyA18i4J7wprpYq8ClcxZy3SC31QMeByyCFyRt7BVHdREQZ5lpzoe5mFEYZUWe+oq8HBvk9JjpibyEV4Jg==",
|
||||||
|
"dependencies": {
|
||||||
|
"internmap": "1 - 2"
|
||||||
|
},
|
||||||
|
"engines": {
|
||||||
|
"node": ">=12"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"node_modules/d3-color": {
|
||||||
|
"version": "3.1.0",
|
||||||
|
"resolved": "https://registry.npmjs.org/d3-color/-/d3-color-3.1.0.tgz",
|
||||||
|
"integrity": "sha512-zg/chbXyeBtMQ1LbD/WSoW2DpC3I0mpmPdW+ynRTj/x2DAWYrIY7qeZIHidozwV24m4iavr15lNwIwLxRmOxhA==",
|
||||||
|
"engines": {
|
||||||
|
"node": ">=12"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"node_modules/d3-ease": {
|
||||||
|
"version": "3.0.1",
|
||||||
|
"resolved": "https://registry.npmjs.org/d3-ease/-/d3-ease-3.0.1.tgz",
|
||||||
|
"integrity": "sha512-wR/XK3D3XcLIZwpbvQwQ5fK+8Ykds1ip7A2Txe0yxncXSdq1L9skcG7blcedkOX+ZcgxGAmLX1FrRGbADwzi0w==",
|
||||||
|
"engines": {
|
||||||
|
"node": ">=12"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"node_modules/d3-format": {
|
||||||
|
"version": "3.1.2",
|
||||||
|
"resolved": "https://registry.npmjs.org/d3-format/-/d3-format-3.1.2.tgz",
|
||||||
|
"integrity": "sha512-AJDdYOdnyRDV5b6ArilzCPPwc1ejkHcoyFarqlPqT7zRYjhavcT3uSrqcMvsgh2CgoPbK3RCwyHaVyxYcP2Arg==",
|
||||||
|
"engines": {
|
||||||
|
"node": ">=12"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"node_modules/d3-interpolate": {
|
||||||
|
"version": "3.0.1",
|
||||||
|
"resolved": "https://registry.npmjs.org/d3-interpolate/-/d3-interpolate-3.0.1.tgz",
|
||||||
|
"integrity": "sha512-3bYs1rOD33uo8aqJfKP3JWPAibgw8Zm2+L9vBKEHJ2Rg+viTR7o5Mmv5mZcieN+FRYaAOWX5SJATX6k1PWz72g==",
|
||||||
|
"dependencies": {
|
||||||
|
"d3-color": "1 - 3"
|
||||||
|
},
|
||||||
|
"engines": {
|
||||||
|
"node": ">=12"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"node_modules/d3-path": {
|
||||||
|
"version": "3.1.0",
|
||||||
|
"resolved": "https://registry.npmjs.org/d3-path/-/d3-path-3.1.0.tgz",
|
||||||
|
"integrity": "sha512-p3KP5HCf/bvjBSSKuXid6Zqijx7wIfNW+J/maPs+iwR35at5JCbLUT0LzF1cnjbCHWhqzQTIN2Jpe8pRebIEFQ==",
|
||||||
|
"engines": {
|
||||||
|
"node": ">=12"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"node_modules/d3-scale": {
|
||||||
|
"version": "4.0.2",
|
||||||
|
"resolved": "https://registry.npmjs.org/d3-scale/-/d3-scale-4.0.2.tgz",
|
||||||
|
"integrity": "sha512-GZW464g1SH7ag3Y7hXjf8RoUuAFIqklOAq3MRl4OaWabTFJY9PN/E1YklhXLh+OQ3fM9yS2nOkCoS+WLZ6kvxQ==",
|
||||||
|
"dependencies": {
|
||||||
|
"d3-array": "2.10.0 - 3",
|
||||||
|
"d3-format": "1 - 3",
|
||||||
|
"d3-interpolate": "1.2.0 - 3",
|
||||||
|
"d3-time": "2.1.1 - 3",
|
||||||
|
"d3-time-format": "2 - 4"
|
||||||
|
},
|
||||||
|
"engines": {
|
||||||
|
"node": ">=12"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"node_modules/d3-shape": {
|
||||||
|
"version": "3.2.0",
|
||||||
|
"resolved": "https://registry.npmjs.org/d3-shape/-/d3-shape-3.2.0.tgz",
|
||||||
|
"integrity": "sha512-SaLBuwGm3MOViRq2ABk3eLoxwZELpH6zhl3FbAoJ7Vm1gofKx6El1Ib5z23NUEhF9AsGl7y+dzLe5Cw2AArGTA==",
|
||||||
|
"dependencies": {
|
||||||
|
"d3-path": "^3.1.0"
|
||||||
|
},
|
||||||
|
"engines": {
|
||||||
|
"node": ">=12"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"node_modules/d3-time": {
|
||||||
|
"version": "3.1.0",
|
||||||
|
"resolved": "https://registry.npmjs.org/d3-time/-/d3-time-3.1.0.tgz",
|
||||||
|
"integrity": "sha512-VqKjzBLejbSMT4IgbmVgDjpkYrNWUYJnbCGo874u7MMKIWsILRX+OpX/gTk8MqjpT1A/c6HY2dCA77ZN0lkQ2Q==",
|
||||||
|
"dependencies": {
|
||||||
|
"d3-array": "2 - 3"
|
||||||
|
},
|
||||||
|
"engines": {
|
||||||
|
"node": ">=12"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"node_modules/d3-time-format": {
|
||||||
|
"version": "4.1.0",
|
||||||
|
"resolved": "https://registry.npmjs.org/d3-time-format/-/d3-time-format-4.1.0.tgz",
|
||||||
|
"integrity": "sha512-dJxPBlzC7NugB2PDLwo9Q8JiTR3M3e4/XANkreKSUxF8vvXKqm1Yfq4Q5dl8budlunRVlUUaDUgFt7eA8D6NLg==",
|
||||||
|
"dependencies": {
|
||||||
|
"d3-time": "1 - 3"
|
||||||
|
},
|
||||||
|
"engines": {
|
||||||
|
"node": ">=12"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"node_modules/d3-timer": {
|
||||||
|
"version": "3.0.1",
|
||||||
|
"resolved": "https://registry.npmjs.org/d3-timer/-/d3-timer-3.0.1.tgz",
|
||||||
|
"integrity": "sha512-ndfJ/JxxMd3nw31uyKoY2naivF+r29V+Lc0svZxe1JvvIRmi8hUsrMvdOwgS1o6uBHmiz91geQ0ylPP0aj1VUA==",
|
||||||
|
"engines": {
|
||||||
|
"node": ">=12"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"node_modules/decimal.js-light": {
|
||||||
|
"version": "2.5.1",
|
||||||
|
"resolved": "https://registry.npmjs.org/decimal.js-light/-/decimal.js-light-2.5.1.tgz",
|
||||||
|
"integrity": "sha512-qIMFpTMZmny+MMIitAB6D7iVPEorVw6YQRWkvarTkT4tBeSLLiHzcwj6q0MmYSFCiVpiqPJTJEYIrpcPzVEIvg=="
|
||||||
|
},
|
||||||
"node_modules/detect-libc": {
|
"node_modules/detect-libc": {
|
||||||
"version": "2.1.2",
|
"version": "2.1.2",
|
||||||
"resolved": "https://registry.npmjs.org/detect-libc/-/detect-libc-2.1.2.tgz",
|
"resolved": "https://registry.npmjs.org/detect-libc/-/detect-libc-2.1.2.tgz",
|
||||||
@@ -793,6 +993,11 @@
|
|||||||
"node": ">= 0.4"
|
"node": ">= 0.4"
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
|
"node_modules/es-toolkit": {
|
||||||
|
"version": "1.49.0",
|
||||||
|
"resolved": "https://registry.npmjs.org/es-toolkit/-/es-toolkit-1.49.0.tgz",
|
||||||
|
"integrity": "sha512-G5iZ6Pc/FNRY/soKZHC+TxGDD83rHUDXxzaWhGCX44vAv/tMs56WMusnm/KMNK+luUPsgA9U28cGr4RDlSzL2g=="
|
||||||
|
},
|
||||||
"node_modules/escalade": {
|
"node_modules/escalade": {
|
||||||
"version": "3.2.0",
|
"version": "3.2.0",
|
||||||
"resolved": "https://registry.npmjs.org/escalade/-/escalade-3.2.0.tgz",
|
"resolved": "https://registry.npmjs.org/escalade/-/escalade-3.2.0.tgz",
|
||||||
@@ -803,6 +1008,11 @@
|
|||||||
"node": ">=6"
|
"node": ">=6"
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
|
"node_modules/eventemitter3": {
|
||||||
|
"version": "5.0.4",
|
||||||
|
"resolved": "https://registry.npmjs.org/eventemitter3/-/eventemitter3-5.0.4.tgz",
|
||||||
|
"integrity": "sha512-mlsTRyGaPBjPedk6Bvw+aqbsXDtoAyAzm5MO7JgU+yVRyMQ5O8bD4Kcci7BS85f93veegeCPkL8R4GLClnjLFw=="
|
||||||
|
},
|
||||||
"node_modules/fast-glob": {
|
"node_modules/fast-glob": {
|
||||||
"version": "3.3.3",
|
"version": "3.3.3",
|
||||||
"resolved": "https://registry.npmjs.org/fast-glob/-/fast-glob-3.3.3.tgz",
|
"resolved": "https://registry.npmjs.org/fast-glob/-/fast-glob-3.3.3.tgz",
|
||||||
@@ -921,6 +1131,23 @@
|
|||||||
"node": ">= 0.4"
|
"node": ">= 0.4"
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
|
"node_modules/immer": {
|
||||||
|
"version": "11.1.15",
|
||||||
|
"resolved": "https://registry.npmjs.org/immer/-/immer-11.1.15.tgz",
|
||||||
|
"integrity": "sha512-VrNANlmnWQnh5COXIIOQXM9oOJw7naGKlBT74ZOOR6lpVXc3gFEu9FJLDFcpCJ2j+NWr8TIwtWD//T6ZX6TKiQ==",
|
||||||
|
"funding": {
|
||||||
|
"type": "opencollective",
|
||||||
|
"url": "https://opencollective.com/immer"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"node_modules/internmap": {
|
||||||
|
"version": "2.0.3",
|
||||||
|
"resolved": "https://registry.npmjs.org/internmap/-/internmap-2.0.3.tgz",
|
||||||
|
"integrity": "sha512-5Hh7Y1wQbvY5ooGgPbDaL5iYLAPzMTUrjMulskHLH6wnv/A+1q5rgEaiuqEjB+oxGXIVZs1FF+R/KPN3ZSQYYg==",
|
||||||
|
"engines": {
|
||||||
|
"node": ">=12"
|
||||||
|
}
|
||||||
|
},
|
||||||
"node_modules/is-binary-path": {
|
"node_modules/is-binary-path": {
|
||||||
"version": "2.1.0",
|
"version": "2.1.0",
|
||||||
"resolved": "https://registry.npmjs.org/is-binary-path/-/is-binary-path-2.1.0.tgz",
|
"resolved": "https://registry.npmjs.org/is-binary-path/-/is-binary-path-2.1.0.tgz",
|
||||||
@@ -1136,9 +1363,6 @@
|
|||||||
"arm64"
|
"arm64"
|
||||||
],
|
],
|
||||||
"dev": true,
|
"dev": true,
|
||||||
"libc": [
|
|
||||||
"glibc"
|
|
||||||
],
|
|
||||||
"license": "MPL-2.0",
|
"license": "MPL-2.0",
|
||||||
"optional": true,
|
"optional": true,
|
||||||
"os": [
|
"os": [
|
||||||
@@ -1160,9 +1384,6 @@
|
|||||||
"arm64"
|
"arm64"
|
||||||
],
|
],
|
||||||
"dev": true,
|
"dev": true,
|
||||||
"libc": [
|
|
||||||
"musl"
|
|
||||||
],
|
|
||||||
"license": "MPL-2.0",
|
"license": "MPL-2.0",
|
||||||
"optional": true,
|
"optional": true,
|
||||||
"os": [
|
"os": [
|
||||||
@@ -1184,9 +1405,6 @@
|
|||||||
"x64"
|
"x64"
|
||||||
],
|
],
|
||||||
"dev": true,
|
"dev": true,
|
||||||
"libc": [
|
|
||||||
"glibc"
|
|
||||||
],
|
|
||||||
"license": "MPL-2.0",
|
"license": "MPL-2.0",
|
||||||
"optional": true,
|
"optional": true,
|
||||||
"os": [
|
"os": [
|
||||||
@@ -1208,9 +1426,6 @@
|
|||||||
"x64"
|
"x64"
|
||||||
],
|
],
|
||||||
"dev": true,
|
"dev": true,
|
||||||
"libc": [
|
|
||||||
"musl"
|
|
||||||
],
|
|
||||||
"license": "MPL-2.0",
|
"license": "MPL-2.0",
|
||||||
"optional": true,
|
"optional": true,
|
||||||
"os": [
|
"os": [
|
||||||
@@ -1639,6 +1854,34 @@
|
|||||||
"react": "^19.2.7"
|
"react": "^19.2.7"
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
|
"node_modules/react-is": {
|
||||||
|
"version": "19.2.8",
|
||||||
|
"resolved": "https://registry.npmjs.org/react-is/-/react-is-19.2.8.tgz",
|
||||||
|
"integrity": "sha512-s5un28nYxKJw5gvUHyW5PCC28CvBqLu9r3cWgzHT4Vo/5fqqkFcdRYsGcKf50WMPpjjFZS5d76fn3YCo2njKwQ==",
|
||||||
|
"peer": true
|
||||||
|
},
|
||||||
|
"node_modules/react-redux": {
|
||||||
|
"version": "9.3.0",
|
||||||
|
"resolved": "https://registry.npmjs.org/react-redux/-/react-redux-9.3.0.tgz",
|
||||||
|
"integrity": "sha512-KQopgqFo/p/fgmAs5qz6p5RWaNAzq40WAu7fJIXnQpYxFPbJYtsJPWvGeF2rOBaY/kEuV77AVsX8TsQzKm+A/g==",
|
||||||
|
"dependencies": {
|
||||||
|
"@types/use-sync-external-store": "^0.0.6",
|
||||||
|
"use-sync-external-store": "^1.4.0"
|
||||||
|
},
|
||||||
|
"peerDependencies": {
|
||||||
|
"@types/react": "^18.2.25 || ^19",
|
||||||
|
"react": "^18.0 || ^19",
|
||||||
|
"redux": "^5.0.0"
|
||||||
|
},
|
||||||
|
"peerDependenciesMeta": {
|
||||||
|
"@types/react": {
|
||||||
|
"optional": true
|
||||||
|
},
|
||||||
|
"redux": {
|
||||||
|
"optional": true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
"node_modules/read-cache": {
|
"node_modules/read-cache": {
|
||||||
"version": "1.0.0",
|
"version": "1.0.0",
|
||||||
"resolved": "https://registry.npmjs.org/read-cache/-/read-cache-1.0.0.tgz",
|
"resolved": "https://registry.npmjs.org/read-cache/-/read-cache-1.0.0.tgz",
|
||||||
@@ -1662,6 +1905,45 @@
|
|||||||
"node": ">=8.10.0"
|
"node": ">=8.10.0"
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
|
"node_modules/recharts": {
|
||||||
|
"version": "3.10.0",
|
||||||
|
"resolved": "https://registry.npmjs.org/recharts/-/recharts-3.10.0.tgz",
|
||||||
|
"integrity": "sha512-wulMvfncpIlmu2uFtRU/mE5/+NiVtASXkw2KdwJTdHs3WsASX0WxZlX+rpKgyn5BDbIhkPtCpUKkB9XNK5KE0w==",
|
||||||
|
"dependencies": {
|
||||||
|
"@reduxjs/toolkit": "^1.9.0 || 2.x.x",
|
||||||
|
"clsx": "^2.1.1",
|
||||||
|
"decimal.js-light": "^2.5.1",
|
||||||
|
"es-toolkit": "^1.39.3",
|
||||||
|
"eventemitter3": "^5.0.1",
|
||||||
|
"immer": "^11.1.8",
|
||||||
|
"react-redux": "8.x.x || 9.x.x",
|
||||||
|
"reselect": "5.2.0",
|
||||||
|
"tiny-invariant": "^1.3.3",
|
||||||
|
"use-sync-external-store": "^1.2.2",
|
||||||
|
"victory-vendor": "^37.0.2"
|
||||||
|
},
|
||||||
|
"engines": {
|
||||||
|
"node": ">=18"
|
||||||
|
},
|
||||||
|
"peerDependencies": {
|
||||||
|
"react": "^16.8.0 || ^17.0.0 || ^18.0.0 || ^19.0.0",
|
||||||
|
"react-dom": "^16.0.0 || ^17.0.0 || ^18.0.0 || ^19.0.0",
|
||||||
|
"react-is": "^16.8.0 || ^17.0.0 || ^18.0.0 || ^19.0.0"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"node_modules/redux": {
|
||||||
|
"version": "5.0.1",
|
||||||
|
"resolved": "https://registry.npmjs.org/redux/-/redux-5.0.1.tgz",
|
||||||
|
"integrity": "sha512-M9/ELqF6fy8FwmkpnF0S3YKOqMyoWJ4+CS5Efg2ct3oY9daQvd/Pc71FpGZsVsbl3Cpb+IIcjBDUnnyBdQbq4w=="
|
||||||
|
},
|
||||||
|
"node_modules/redux-thunk": {
|
||||||
|
"version": "3.1.0",
|
||||||
|
"resolved": "https://registry.npmjs.org/redux-thunk/-/redux-thunk-3.1.0.tgz",
|
||||||
|
"integrity": "sha512-NW2r5T6ksUKXCabzhL9z+h206HQw/NJkcLm1GPImRQ8IzfXwRGqjVhKJGauHirT0DAuyy6hjdnMZaRoAcy0Klw==",
|
||||||
|
"peerDependencies": {
|
||||||
|
"redux": "^5.0.0"
|
||||||
|
}
|
||||||
|
},
|
||||||
"node_modules/regexparam": {
|
"node_modules/regexparam": {
|
||||||
"version": "3.0.0",
|
"version": "3.0.0",
|
||||||
"resolved": "https://registry.npmjs.org/regexparam/-/regexparam-3.0.0.tgz",
|
"resolved": "https://registry.npmjs.org/regexparam/-/regexparam-3.0.0.tgz",
|
||||||
@@ -1671,6 +1953,11 @@
|
|||||||
"node": ">=8"
|
"node": ">=8"
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
|
"node_modules/reselect": {
|
||||||
|
"version": "5.2.0",
|
||||||
|
"resolved": "https://registry.npmjs.org/reselect/-/reselect-5.2.0.tgz",
|
||||||
|
"integrity": "sha512-AgZ3UOZm3YndfrJ4OYjgrT7bmCm/1iqkjvEfH/oYjzh6PD2qw4QuT3jjnXIrpdt4MTpMXclMT3lXbmRY+XRakw=="
|
||||||
|
},
|
||||||
"node_modules/resolve": {
|
"node_modules/resolve": {
|
||||||
"version": "1.22.12",
|
"version": "1.22.12",
|
||||||
"resolved": "https://registry.npmjs.org/resolve/-/resolve-1.22.12.tgz",
|
"resolved": "https://registry.npmjs.org/resolve/-/resolve-1.22.12.tgz",
|
||||||
@@ -1875,6 +2162,11 @@
|
|||||||
"node": ">=0.8"
|
"node": ">=0.8"
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
|
"node_modules/tiny-invariant": {
|
||||||
|
"version": "1.3.3",
|
||||||
|
"resolved": "https://registry.npmjs.org/tiny-invariant/-/tiny-invariant-1.3.3.tgz",
|
||||||
|
"integrity": "sha512-+FbBPE1o9QAYvviau/qC5SE3caw21q3xkvWKBtja5vgqOWIHHJ3ioaq1VPfn/Szqctz2bU/oYeKd9/z5BL+PVg=="
|
||||||
|
},
|
||||||
"node_modules/tinyglobby": {
|
"node_modules/tinyglobby": {
|
||||||
"version": "0.2.17",
|
"version": "0.2.17",
|
||||||
"resolved": "https://registry.npmjs.org/tinyglobby/-/tinyglobby-0.2.17.tgz",
|
"resolved": "https://registry.npmjs.org/tinyglobby/-/tinyglobby-0.2.17.tgz",
|
||||||
@@ -2012,6 +2304,27 @@
|
|||||||
"dev": true,
|
"dev": true,
|
||||||
"license": "MIT"
|
"license": "MIT"
|
||||||
},
|
},
|
||||||
|
"node_modules/victory-vendor": {
|
||||||
|
"version": "37.3.6",
|
||||||
|
"resolved": "https://registry.npmjs.org/victory-vendor/-/victory-vendor-37.3.6.tgz",
|
||||||
|
"integrity": "sha512-SbPDPdDBYp+5MJHhBCAyI7wKM3d5ivekigc2Dk2s7pgbZ9wIgIBYGVw4zGHBml/qTFbexrofXW6Gu4noGxrOwQ==",
|
||||||
|
"dependencies": {
|
||||||
|
"@types/d3-array": "^3.0.3",
|
||||||
|
"@types/d3-ease": "^3.0.0",
|
||||||
|
"@types/d3-interpolate": "^3.0.1",
|
||||||
|
"@types/d3-scale": "^4.0.2",
|
||||||
|
"@types/d3-shape": "^3.1.0",
|
||||||
|
"@types/d3-time": "^3.0.0",
|
||||||
|
"@types/d3-timer": "^3.0.0",
|
||||||
|
"d3-array": "^3.1.6",
|
||||||
|
"d3-ease": "^3.0.1",
|
||||||
|
"d3-interpolate": "^3.0.1",
|
||||||
|
"d3-scale": "^4.0.2",
|
||||||
|
"d3-shape": "^3.1.0",
|
||||||
|
"d3-time": "^3.0.0",
|
||||||
|
"d3-timer": "^3.0.1"
|
||||||
|
}
|
||||||
|
},
|
||||||
"node_modules/vite": {
|
"node_modules/vite": {
|
||||||
"version": "8.1.4",
|
"version": "8.1.4",
|
||||||
"resolved": "https://registry.npmjs.org/vite/-/vite-8.1.4.tgz",
|
"resolved": "https://registry.npmjs.org/vite/-/vite-8.1.4.tgz",
|
||||||
|
|||||||
@@ -11,6 +11,7 @@
|
|||||||
"dependencies": {
|
"dependencies": {
|
||||||
"react": "^19.2.6",
|
"react": "^19.2.6",
|
||||||
"react-dom": "^19.2.6",
|
"react-dom": "^19.2.6",
|
||||||
|
"recharts": "^3.10.0",
|
||||||
"wouter": "^3.7.1"
|
"wouter": "^3.7.1"
|
||||||
},
|
},
|
||||||
"devDependencies": {
|
"devDependencies": {
|
||||||
|
|||||||
@@ -0,0 +1,36 @@
|
|||||||
|
import { jsx as _jsx, jsxs as _jsxs } from "react/jsx-runtime";
|
||||||
|
import { useEffect, useState } from 'react';
|
||||||
|
import { api, fmtBytes, fmtAge } from '../lib/api';
|
||||||
|
function kindLabel(kind) {
|
||||||
|
if (kind === 'gc_orphan')
|
||||||
|
return 'Orphan GC';
|
||||||
|
if (kind === 'gc_eviction_scored')
|
||||||
|
return 'Scored Eviction';
|
||||||
|
if (kind === 'scrub')
|
||||||
|
return 'Scrub';
|
||||||
|
return kind;
|
||||||
|
}
|
||||||
|
function kindColor(kind) {
|
||||||
|
if (kind === 'gc_eviction_scored')
|
||||||
|
return 'text-amber-300';
|
||||||
|
if (kind === 'scrub')
|
||||||
|
return 'text-blue-400';
|
||||||
|
return 'text-slate-300';
|
||||||
|
}
|
||||||
|
export function MaintenancePanel() {
|
||||||
|
const [rows, setRows] = useState([]);
|
||||||
|
const [err, setErr] = useState(null);
|
||||||
|
const [loading, setLoading] = useState(true);
|
||||||
|
useEffect(() => {
|
||||||
|
api
|
||||||
|
.maintenance()
|
||||||
|
.then((r) => { setRows(r); setErr(null); })
|
||||||
|
.catch((e) => setErr(String(e)))
|
||||||
|
.finally(() => setLoading(false));
|
||||||
|
}, []);
|
||||||
|
return (_jsxs("div", { className: "rounded border border-slate-800 bg-slate-900/40 p-4", children: [_jsx("h2", { className: "text-sm font-semibold text-slate-300 uppercase tracking-wider mb-3", children: "GC / Maintenance Log" }), loading && _jsx("div", { className: "text-slate-500 text-sm", children: "Loading\u2026" }), err && _jsx("div", { className: "text-amber-400 text-sm", children: err }), !loading && !err && rows.length === 0 && (_jsx("div", { className: "text-slate-500 text-sm italic", children: "No maintenance events recorded yet." })), rows.length > 0 && (_jsx("div", { className: "overflow-x-auto", children: _jsxs("table", { className: "w-full text-sm border-collapse", children: [_jsx("thead", { children: _jsxs("tr", { className: "text-left text-xs text-slate-500 uppercase tracking-wider border-b border-slate-800", children: [_jsx("th", { className: "pr-3 pb-2", children: "Age" }), _jsx("th", { className: "pr-3 pb-2", children: "Node" }), _jsx("th", { className: "pr-3 pb-2", children: "Type" }), _jsx("th", { className: "pr-3 pb-2 text-right", children: "Scanned" }), _jsx("th", { className: "pr-3 pb-2 text-right", children: "Removed" }), _jsx("th", { className: "pr-3 pb-2 text-right", children: "Reclaimed" }), _jsx("th", { className: "pb-2", children: "Status" })] }) }), _jsx("tbody", { children: rows.map((row, i) => (_jsxs("tr", { className: "border-b border-slate-800/50 hover:bg-slate-800/20", children: [_jsx("td", { className: "pr-3 py-1.5 text-slate-400 whitespace-nowrap", children: fmtAge(row.unix_ts) }), _jsx("td", { className: "pr-3 py-1.5 font-mono text-xs text-slate-300", children: row.node }), _jsx("td", { className: `pr-3 py-1.5 font-medium ${kindColor(row.kind)}`, children: kindLabel(row.kind) }), _jsx("td", { className: "pr-3 py-1.5 text-right text-slate-300", children: row.chunks_scanned.toLocaleString() }), _jsx("td", { className: "pr-3 py-1.5 text-right text-slate-300", children: row.chunks_removed > 0
|
||||||
|
? _jsx("span", { className: "text-amber-300", children: row.chunks_removed.toLocaleString() })
|
||||||
|
: _jsx("span", { className: "text-slate-600", children: "0" }) }), _jsx("td", { className: "pr-3 py-1.5 text-right text-slate-300", children: row.bytes_reclaimed > 0 ? fmtBytes(row.bytes_reclaimed) : '—' }), _jsx("td", { className: "py-1.5", children: row.error
|
||||||
|
? _jsx("span", { className: "text-red-400 text-xs", children: row.error })
|
||||||
|
: _jsx("span", { className: "text-emerald-500 text-xs", children: "ok" }) })] }, i))) })] }) }))] }));
|
||||||
|
}
|
||||||
@@ -0,0 +1,93 @@
|
|||||||
|
import { useEffect, useState } from 'react';
|
||||||
|
import { api, MaintenanceRow, fmtBytes, fmtAge } from '../lib/api';
|
||||||
|
|
||||||
|
function kindLabel(kind: string): string {
|
||||||
|
if (kind === 'gc_orphan') return 'Orphan GC';
|
||||||
|
if (kind === 'gc_eviction_scored') return 'Scored Eviction';
|
||||||
|
if (kind === 'scrub') return 'Scrub';
|
||||||
|
return kind;
|
||||||
|
}
|
||||||
|
|
||||||
|
function kindColor(kind: string): string {
|
||||||
|
if (kind === 'gc_eviction_scored') return 'text-amber-300';
|
||||||
|
if (kind === 'scrub') return 'text-blue-400';
|
||||||
|
return 'text-slate-300';
|
||||||
|
}
|
||||||
|
|
||||||
|
export function MaintenancePanel() {
|
||||||
|
const [rows, setRows] = useState<MaintenanceRow[]>([]);
|
||||||
|
const [err, setErr] = useState<string | null>(null);
|
||||||
|
const [loading, setLoading] = useState(true);
|
||||||
|
|
||||||
|
useEffect(() => {
|
||||||
|
api
|
||||||
|
.maintenance()
|
||||||
|
.then((r) => { setRows(r); setErr(null); })
|
||||||
|
.catch((e) => setErr(String(e)))
|
||||||
|
.finally(() => setLoading(false));
|
||||||
|
}, []);
|
||||||
|
|
||||||
|
return (
|
||||||
|
<div className="rounded border border-slate-800 bg-slate-900/40 p-4">
|
||||||
|
<h2 className="text-sm font-semibold text-slate-300 uppercase tracking-wider mb-3">
|
||||||
|
GC / Maintenance Log
|
||||||
|
</h2>
|
||||||
|
|
||||||
|
{loading && <div className="text-slate-500 text-sm">Loading…</div>}
|
||||||
|
{err && <div className="text-amber-400 text-sm">{err}</div>}
|
||||||
|
|
||||||
|
{!loading && !err && rows.length === 0 && (
|
||||||
|
<div className="text-slate-500 text-sm italic">No maintenance events recorded yet.</div>
|
||||||
|
)}
|
||||||
|
|
||||||
|
{rows.length > 0 && (
|
||||||
|
<div className="overflow-x-auto">
|
||||||
|
<table className="w-full text-sm border-collapse">
|
||||||
|
<thead>
|
||||||
|
<tr className="text-left text-xs text-slate-500 uppercase tracking-wider border-b border-slate-800">
|
||||||
|
<th className="pr-3 pb-2">Age</th>
|
||||||
|
<th className="pr-3 pb-2">Node</th>
|
||||||
|
<th className="pr-3 pb-2">Type</th>
|
||||||
|
<th className="pr-3 pb-2 text-right">Scanned</th>
|
||||||
|
<th className="pr-3 pb-2 text-right">Removed</th>
|
||||||
|
<th className="pr-3 pb-2 text-right">Reclaimed</th>
|
||||||
|
<th className="pb-2">Status</th>
|
||||||
|
</tr>
|
||||||
|
</thead>
|
||||||
|
<tbody>
|
||||||
|
{rows.map((row, i) => (
|
||||||
|
<tr key={i} className="border-b border-slate-800/50 hover:bg-slate-800/20">
|
||||||
|
<td className="pr-3 py-1.5 text-slate-400 whitespace-nowrap">
|
||||||
|
{fmtAge(row.unix_ts)}
|
||||||
|
</td>
|
||||||
|
<td className="pr-3 py-1.5 font-mono text-xs text-slate-300">
|
||||||
|
{row.node}
|
||||||
|
</td>
|
||||||
|
<td className={`pr-3 py-1.5 font-medium ${kindColor(row.kind)}`}>
|
||||||
|
{kindLabel(row.kind)}
|
||||||
|
</td>
|
||||||
|
<td className="pr-3 py-1.5 text-right text-slate-300">
|
||||||
|
{row.chunks_scanned.toLocaleString()}
|
||||||
|
</td>
|
||||||
|
<td className="pr-3 py-1.5 text-right text-slate-300">
|
||||||
|
{row.chunks_removed > 0
|
||||||
|
? <span className="text-amber-300">{row.chunks_removed.toLocaleString()}</span>
|
||||||
|
: <span className="text-slate-600">0</span>}
|
||||||
|
</td>
|
||||||
|
<td className="pr-3 py-1.5 text-right text-slate-300">
|
||||||
|
{row.bytes_reclaimed > 0 ? fmtBytes(row.bytes_reclaimed) : '—'}
|
||||||
|
</td>
|
||||||
|
<td className="py-1.5">
|
||||||
|
{row.error
|
||||||
|
? <span className="text-red-400 text-xs">{row.error}</span>
|
||||||
|
: <span className="text-emerald-500 text-xs">ok</span>}
|
||||||
|
</td>
|
||||||
|
</tr>
|
||||||
|
))}
|
||||||
|
</tbody>
|
||||||
|
</table>
|
||||||
|
</div>
|
||||||
|
)}
|
||||||
|
</div>
|
||||||
|
);
|
||||||
|
}
|
||||||
@@ -1,10 +1,12 @@
|
|||||||
import { jsx as _jsx, jsxs as _jsxs, Fragment as _Fragment } from "react/jsx-runtime";
|
import { jsx as _jsx, jsxs as _jsxs, Fragment as _Fragment } from "react/jsx-runtime";
|
||||||
import { Link } from 'wouter';
|
import { Link } from 'wouter';
|
||||||
|
import { fmtBytes, fmtUptime } from '../lib/api';
|
||||||
import { StorageBar } from './StorageBar';
|
import { StorageBar } from './StorageBar';
|
||||||
|
import { NodeHistorySparklines } from './NodeHistorySparklines';
|
||||||
/// Human-oriented node card for the FleetHealth landing.
|
/// Human-oriented node card for the FleetHealth landing.
|
||||||
/// Shows: overall health traffic-light, storage bars, mount state,
|
/// Shows: overall health traffic-light, storage bars, mount state,
|
||||||
/// cache hit rate, next scheduled job. No hex, no primitives.
|
/// cache hit rate, next scheduled job. No hex, no primitives.
|
||||||
export function NodeCard({ node }) {
|
export function NodeCard({ node, anomalyLevel }) {
|
||||||
const health = healthOf(node);
|
const health = healthOf(node);
|
||||||
const border = {
|
const border = {
|
||||||
ok: 'border-emerald-700 hover:border-emerald-500',
|
ok: 'border-emerald-700 hover:border-emerald-500',
|
||||||
@@ -22,13 +24,40 @@ export function NodeCard({ node }) {
|
|||||||
'block rounded-lg bg-slate-900 border transition-colors',
|
'block rounded-lg bg-slate-900 border transition-colors',
|
||||||
'p-5 space-y-4',
|
'p-5 space-y-4',
|
||||||
border,
|
border,
|
||||||
].join(' '), children: [_jsxs("div", { className: "flex items-center justify-between", children: [_jsxs("div", { className: "flex items-center gap-2", children: [_jsx("span", { className: `inline-block w-2.5 h-2.5 rounded-full ${dot}` }), _jsx("span", { className: "text-lg font-semibold text-slate-100", children: node.node_name })] }), _jsx("span", { className: "text-xs text-slate-500 font-mono", children: node.zone || '—' })] }), !node.online && (_jsx("div", { className: "text-sm text-red-400 break-words", children: node.error ?? 'offline' })), node.online && (_jsxs(_Fragment, { children: [node.filesystem && (_jsx(StorageBar, { label: "disk", used: node.filesystem.used_bytes, total: node.filesystem.total_bytes })), node.hot && node.hot.max_bytes > 0 && (_jsx(StorageBar, { label: "hot tier", used: node.hot.used_bytes, total: node.hot.max_bytes, pinned: node.hot.pinned_bytes ?? undefined })), _jsxs("div", { className: "grid grid-cols-2 gap-y-1 text-sm", children: [_jsx("span", { className: "text-slate-500", children: "mount" }), _jsx("span", { className: "text-right font-mono text-xs", children: node.mount?.active ? (_jsx("span", { className: "text-emerald-300", children: "\u2713 mounted" })) : (_jsx("span", { className: "text-slate-500", children: "not mounted" })) }), _jsx("span", { className: "text-slate-500", children: "cache hit rate" }), _jsx("span", { className: "text-right font-mono text-xs", children: node.cache && node.cache.hits + node.cache.misses > 0
|
].join(' '), children: [_jsxs("div", { className: "flex items-center justify-between", children: [_jsxs("div", { className: "flex items-center gap-2", children: [_jsx("span", { className: `inline-block w-2.5 h-2.5 rounded-full ${dot}` }), _jsx("span", { className: "text-lg font-semibold text-slate-100", children: node.node_name }), anomalyLevel === 'alert' && (_jsx("span", { className: "text-xs font-mono px-1.5 py-0.5 rounded bg-red-900/60 text-red-300 border border-red-800", children: "anomaly" })), anomalyLevel === 'warn' && (_jsx("span", { className: "text-xs font-mono px-1.5 py-0.5 rounded bg-amber-900/60 text-amber-300 border border-amber-800", children: "drift" }))] }), _jsx("span", { className: "text-xs text-slate-500 font-mono", children: node.zone || '—' })] }), !node.online && (_jsx("div", { className: "text-sm text-red-400 break-words", children: node.error ?? 'offline' })), node.online && (_jsxs(_Fragment, { children: [node.filesystem && (_jsx(StorageBar, { label: node.filesystem.available_bytes != null
|
||||||
|
? `disk — ${fmtBytes(node.filesystem.available_bytes)} free`
|
||||||
|
: 'disk', used: node.filesystem.used_bytes, total: node.filesystem.total_bytes })), node.hot && node.hot.max_bytes > 0 && (_jsx(StorageBar, { label: "hot tier", used: node.hot.used_bytes, total: node.hot.max_bytes, pinned: node.hot.pinned_bytes ?? undefined })), _jsxs("div", { className: "grid grid-cols-2 gap-y-1 text-sm", children: [_jsx("span", { className: "text-slate-500", children: "uptime" }), _jsx("span", { className: "text-right font-mono text-xs", children: node.daemon_started_unix
|
||||||
|
? _jsx("span", { className: "text-slate-300", children: fmtUptime(node.daemon_started_unix) })
|
||||||
|
: _jsx("span", { className: "text-slate-500", children: "\u2014" }) }), _jsx("span", { className: "text-slate-500", children: "mount" }), _jsx("span", { className: "text-right font-mono text-xs", children: node.mount?.active ? (_jsx("span", { className: "text-emerald-300", children: "\u2713 mounted" })) : (_jsx("span", { className: "text-slate-500", children: "not mounted" })) }), _jsx("span", { className: "text-slate-500", children: "cache hit rate" }), _jsx("span", { className: "text-right font-mono text-xs", children: node.cache && node.cache.hits + node.cache.misses > 0
|
||||||
? `${Math.round(node.cache.hit_rate * 100)}%`
|
? `${Math.round(node.cache.hit_rate * 100)}%`
|
||||||
: _jsx("span", { className: "text-slate-500", children: "idle" }) }), _jsx("span", { className: "text-slate-500", children: "next scheduled job" }), _jsx("span", { className: "text-right font-mono text-xs", children: nextTimer(node) })] })] }))] }) }));
|
: _jsx("span", { className: "text-slate-500", children: "idle" }) }), dedupRate(node) !== null && (_jsxs(_Fragment, { children: [_jsx("span", { className: "text-slate-500", children: "dedup efficiency" }), _jsxs("span", { className: "text-right font-mono text-xs text-slate-300", children: [dedupRate(node), "% chunks skipped"] })] })), node.cpu_load_1m != null && (_jsxs(_Fragment, { children: [_jsx("span", { className: "text-slate-500", children: "cpu load" }), _jsx("span", { className: [
|
||||||
|
'text-right font-mono text-xs',
|
||||||
|
node.cpu_load_1m > 4 ? 'text-amber-300' : 'text-slate-300',
|
||||||
|
].join(' '), children: node.cpu_load_1m.toFixed(2) })] })), _jsx("span", { className: "text-slate-500", children: "next scheduled job" }), _jsx("span", { className: "text-right font-mono text-xs", children: nextTimer(node) })] }), failedTimers(node).length > 0 && (_jsx("div", { className: "space-y-0.5", children: failedTimers(node).map((t) => (_jsxs("div", { className: "text-xs text-amber-400 font-mono truncate", children: ["\u26A0 ", t.unit.replace(/^clawstor-/, '').replace(/\.timer$/, ''), ": ", t.last_result] }, t.unit))) })), _jsx(NodeHistorySparklines, { nodeName: node.node_name, hotMaxBytes: node.hot?.max_bytes ?? 0 })] }))] }) }));
|
||||||
|
}
|
||||||
|
/** HasChunk dedup efficiency as an integer percent, or null if no data. */
|
||||||
|
function dedupRate(n) {
|
||||||
|
const c = n.cache;
|
||||||
|
if (!c)
|
||||||
|
return null;
|
||||||
|
const total = (c.has_chunk_hits ?? 0) + (c.has_chunk_misses ?? 0);
|
||||||
|
if (total === 0)
|
||||||
|
return null;
|
||||||
|
return Math.round(((c.has_chunk_hits ?? 0) / total) * 100);
|
||||||
|
}
|
||||||
|
/** Timers whose last_result is a non-success string. */
|
||||||
|
function failedTimers(n) {
|
||||||
|
return n.timers.filter((t) => t.last_result && t.last_result !== 'success');
|
||||||
}
|
}
|
||||||
function healthOf(n) {
|
function healthOf(n) {
|
||||||
if (!n.online)
|
if (!n.online)
|
||||||
return 'err';
|
return 'err';
|
||||||
|
// Server-side health_score already accounts for hot/fs/cpu/timers.
|
||||||
|
if (n.health_score >= 3)
|
||||||
|
return 'err';
|
||||||
|
if (n.health_score >= 1)
|
||||||
|
return 'warn';
|
||||||
|
// Client-side fallback for older daemons that don't send health_score.
|
||||||
const fsPct = n.filesystem
|
const fsPct = n.filesystem
|
||||||
? n.filesystem.used_bytes / Math.max(n.filesystem.total_bytes, 1)
|
? n.filesystem.used_bytes / Math.max(n.filesystem.total_bytes, 1)
|
||||||
: 0;
|
: 0;
|
||||||
|
|||||||
@@ -1,15 +1,17 @@
|
|||||||
import { Link } from 'wouter';
|
import { Link } from 'wouter';
|
||||||
import { NodeStatusV2 } from '../lib/api';
|
import { NodeStatusV2, fmtBytes, fmtUptime } from '../lib/api';
|
||||||
import { StorageBar } from './StorageBar';
|
import { StorageBar } from './StorageBar';
|
||||||
|
import { NodeHistorySparklines } from './NodeHistorySparklines';
|
||||||
|
|
||||||
interface Props {
|
interface Props {
|
||||||
node: NodeStatusV2;
|
node: NodeStatusV2;
|
||||||
|
anomalyLevel?: 'ok' | 'warn' | 'alert';
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Human-oriented node card for the FleetHealth landing.
|
/// Human-oriented node card for the FleetHealth landing.
|
||||||
/// Shows: overall health traffic-light, storage bars, mount state,
|
/// Shows: overall health traffic-light, storage bars, mount state,
|
||||||
/// cache hit rate, next scheduled job. No hex, no primitives.
|
/// cache hit rate, next scheduled job. No hex, no primitives.
|
||||||
export function NodeCard({ node }: Props) {
|
export function NodeCard({ node, anomalyLevel }: Props) {
|
||||||
const health = healthOf(node);
|
const health = healthOf(node);
|
||||||
const border = {
|
const border = {
|
||||||
ok: 'border-emerald-700 hover:border-emerald-500',
|
ok: 'border-emerald-700 hover:border-emerald-500',
|
||||||
@@ -40,6 +42,16 @@ export function NodeCard({ node }: Props) {
|
|||||||
<span className="text-lg font-semibold text-slate-100">
|
<span className="text-lg font-semibold text-slate-100">
|
||||||
{node.node_name}
|
{node.node_name}
|
||||||
</span>
|
</span>
|
||||||
|
{anomalyLevel === 'alert' && (
|
||||||
|
<span className="text-xs font-mono px-1.5 py-0.5 rounded bg-red-900/60 text-red-300 border border-red-800">
|
||||||
|
anomaly
|
||||||
|
</span>
|
||||||
|
)}
|
||||||
|
{anomalyLevel === 'warn' && (
|
||||||
|
<span className="text-xs font-mono px-1.5 py-0.5 rounded bg-amber-900/60 text-amber-300 border border-amber-800">
|
||||||
|
drift
|
||||||
|
</span>
|
||||||
|
)}
|
||||||
</div>
|
</div>
|
||||||
<span className="text-xs text-slate-500 font-mono">
|
<span className="text-xs text-slate-500 font-mono">
|
||||||
{node.zone || '—'}
|
{node.zone || '—'}
|
||||||
@@ -58,7 +70,11 @@ export function NodeCard({ node }: Props) {
|
|||||||
{/* Storage bars */}
|
{/* Storage bars */}
|
||||||
{node.filesystem && (
|
{node.filesystem && (
|
||||||
<StorageBar
|
<StorageBar
|
||||||
label="disk"
|
label={
|
||||||
|
node.filesystem.available_bytes != null
|
||||||
|
? `disk — ${fmtBytes(node.filesystem.available_bytes)} free`
|
||||||
|
: 'disk'
|
||||||
|
}
|
||||||
used={node.filesystem.used_bytes}
|
used={node.filesystem.used_bytes}
|
||||||
total={node.filesystem.total_bytes}
|
total={node.filesystem.total_bytes}
|
||||||
/>
|
/>
|
||||||
@@ -74,6 +90,12 @@ export function NodeCard({ node }: Props) {
|
|||||||
|
|
||||||
{/* One-liner facts */}
|
{/* One-liner facts */}
|
||||||
<div className="grid grid-cols-2 gap-y-1 text-sm">
|
<div className="grid grid-cols-2 gap-y-1 text-sm">
|
||||||
|
<span className="text-slate-500">uptime</span>
|
||||||
|
<span className="text-right font-mono text-xs">
|
||||||
|
{node.daemon_started_unix
|
||||||
|
? <span className="text-slate-300">{fmtUptime(node.daemon_started_unix)}</span>
|
||||||
|
: <span className="text-slate-500">—</span>}
|
||||||
|
</span>
|
||||||
<span className="text-slate-500">mount</span>
|
<span className="text-slate-500">mount</span>
|
||||||
<span className="text-right font-mono text-xs">
|
<span className="text-right font-mono text-xs">
|
||||||
{node.mount?.active ? (
|
{node.mount?.active ? (
|
||||||
@@ -88,11 +110,47 @@ export function NodeCard({ node }: Props) {
|
|||||||
? `${Math.round(node.cache.hit_rate * 100)}%`
|
? `${Math.round(node.cache.hit_rate * 100)}%`
|
||||||
: <span className="text-slate-500">idle</span>}
|
: <span className="text-slate-500">idle</span>}
|
||||||
</span>
|
</span>
|
||||||
|
{dedupRate(node) !== null && (
|
||||||
|
<>
|
||||||
|
<span className="text-slate-500">dedup efficiency</span>
|
||||||
|
<span className="text-right font-mono text-xs text-slate-300">
|
||||||
|
{dedupRate(node)}% chunks skipped
|
||||||
|
</span>
|
||||||
|
</>
|
||||||
|
)}
|
||||||
|
{node.cpu_load_1m != null && (
|
||||||
|
<>
|
||||||
|
<span className="text-slate-500">cpu load</span>
|
||||||
|
<span className={[
|
||||||
|
'text-right font-mono text-xs',
|
||||||
|
node.cpu_load_1m > 4 ? 'text-amber-300' : 'text-slate-300',
|
||||||
|
].join(' ')}>
|
||||||
|
{node.cpu_load_1m.toFixed(2)}
|
||||||
|
</span>
|
||||||
|
</>
|
||||||
|
)}
|
||||||
<span className="text-slate-500">next scheduled job</span>
|
<span className="text-slate-500">next scheduled job</span>
|
||||||
<span className="text-right font-mono text-xs">
|
<span className="text-right font-mono text-xs">
|
||||||
{nextTimer(node)}
|
{nextTimer(node)}
|
||||||
</span>
|
</span>
|
||||||
</div>
|
</div>
|
||||||
|
|
||||||
|
{/* Failed timer callout (T1.3) */}
|
||||||
|
{failedTimers(node).length > 0 && (
|
||||||
|
<div className="space-y-0.5">
|
||||||
|
{failedTimers(node).map((t) => (
|
||||||
|
<div key={t.unit} className="text-xs text-amber-400 font-mono truncate">
|
||||||
|
⚠ {t.unit.replace(/^clawstor-/, '').replace(/\.timer$/, '')}: {t.last_result}
|
||||||
|
</div>
|
||||||
|
))}
|
||||||
|
</div>
|
||||||
|
)}
|
||||||
|
|
||||||
|
{/* Sparklines (Phase B) — fetches its own 1h history */}
|
||||||
|
<NodeHistorySparklines
|
||||||
|
nodeName={node.node_name}
|
||||||
|
hotMaxBytes={node.hot?.max_bytes ?? 0}
|
||||||
|
/>
|
||||||
</>
|
</>
|
||||||
)}
|
)}
|
||||||
</a>
|
</a>
|
||||||
@@ -100,8 +158,28 @@ export function NodeCard({ node }: Props) {
|
|||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/** HasChunk dedup efficiency as an integer percent, or null if no data. */
|
||||||
|
function dedupRate(n: NodeStatusV2): number | null {
|
||||||
|
const c = n.cache;
|
||||||
|
if (!c) return null;
|
||||||
|
const total = (c.has_chunk_hits ?? 0) + (c.has_chunk_misses ?? 0);
|
||||||
|
if (total === 0) return null;
|
||||||
|
return Math.round(((c.has_chunk_hits ?? 0) / total) * 100);
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Timers whose last_result is a non-success string. */
|
||||||
|
function failedTimers(n: NodeStatusV2) {
|
||||||
|
return n.timers.filter(
|
||||||
|
(t) => t.last_result && t.last_result !== 'success'
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
function healthOf(n: NodeStatusV2): 'ok' | 'warn' | 'err' | 'idle' {
|
function healthOf(n: NodeStatusV2): 'ok' | 'warn' | 'err' | 'idle' {
|
||||||
if (!n.online) return 'err';
|
if (!n.online) return 'err';
|
||||||
|
// Server-side health_score already accounts for hot/fs/cpu/timers.
|
||||||
|
if (n.health_score >= 3) return 'err';
|
||||||
|
if (n.health_score >= 1) return 'warn';
|
||||||
|
// Client-side fallback for older daemons that don't send health_score.
|
||||||
const fsPct = n.filesystem
|
const fsPct = n.filesystem
|
||||||
? n.filesystem.used_bytes / Math.max(n.filesystem.total_bytes, 1)
|
? n.filesystem.used_bytes / Math.max(n.filesystem.total_bytes, 1)
|
||||||
: 0;
|
: 0;
|
||||||
|
|||||||
@@ -0,0 +1,37 @@
|
|||||||
|
import { jsx as _jsx, jsxs as _jsxs } from "react/jsx-runtime";
|
||||||
|
import { useEffect, useState } from 'react';
|
||||||
|
import { AreaChart, Area, LineChart, Line, ResponsiveContainer, Tooltip, ReferenceLine, } from 'recharts';
|
||||||
|
import { api, fmtBytes } from '../lib/api';
|
||||||
|
// Compact tooltip for sparklines — shows a single value line.
|
||||||
|
function SparkTooltip({ active, payload, label: _label, formatter, }) {
|
||||||
|
if (!active || !payload?.length)
|
||||||
|
return null;
|
||||||
|
return (_jsx("div", { className: "rounded bg-slate-800 border border-slate-700 px-2 py-1 text-xs text-slate-200 shadow-lg", children: formatter(payload[0].value) }));
|
||||||
|
}
|
||||||
|
export function NodeHistorySparklines({ nodeName, hotMaxBytes }) {
|
||||||
|
const [samples, setSamples] = useState([]);
|
||||||
|
useEffect(() => {
|
||||||
|
let cancelled = false;
|
||||||
|
api
|
||||||
|
.metricsHistory(nodeName, 60)
|
||||||
|
.then((data) => {
|
||||||
|
if (!cancelled)
|
||||||
|
setSamples(data);
|
||||||
|
})
|
||||||
|
.catch(() => { });
|
||||||
|
return () => {
|
||||||
|
cancelled = true;
|
||||||
|
};
|
||||||
|
}, [nodeName]);
|
||||||
|
if (samples.length < 2)
|
||||||
|
return null;
|
||||||
|
// Recharts data arrays
|
||||||
|
const hotData = samples.map((s) => ({ t: s.unix_ts, v: s.hot_used_bytes }));
|
||||||
|
const hitData = samples.map((s) => ({
|
||||||
|
t: s.unix_ts,
|
||||||
|
v: Math.round(s.cache_hit_rate * 100),
|
||||||
|
}));
|
||||||
|
// Hot-tier ceiling reference line (90% of max = gc threshold)
|
||||||
|
const gcLine = hotMaxBytes > 0 ? hotMaxBytes * 0.9 : null;
|
||||||
|
return (_jsxs("div", { className: "space-y-3", children: [hotMaxBytes > 0 && (_jsxs("div", { children: [_jsxs("div", { className: "flex justify-between text-xs text-slate-500 mb-0.5", children: [_jsx("span", { children: "hot tier \u2014 1h history" }), _jsx("span", { children: fmtBytes(hotData[hotData.length - 1].v) })] }), _jsx(ResponsiveContainer, { width: "100%", height: 40, children: _jsxs(AreaChart, { data: hotData, margin: { top: 2, right: 0, bottom: 2, left: 0 }, children: [_jsx("defs", { children: _jsxs("linearGradient", { id: `hg-${nodeName}`, x1: "0", y1: "0", x2: "0", y2: "1", children: [_jsx("stop", { offset: "5%", stopColor: "#3b82f6", stopOpacity: 0.4 }), _jsx("stop", { offset: "95%", stopColor: "#3b82f6", stopOpacity: 0 })] }) }), gcLine && (_jsx(ReferenceLine, { y: gcLine, stroke: "#ef4444", strokeDasharray: "3 3", strokeWidth: 1 })), _jsx(Area, { type: "monotone", dataKey: "v", stroke: "#3b82f6", strokeWidth: 1.5, fill: `url(#hg-${nodeName})`, dot: false, isAnimationActive: false }), _jsx(Tooltip, { content: _jsx(SparkTooltip, { formatter: (v) => fmtBytes(v) }) })] }) })] })), _jsxs("div", { children: [_jsxs("div", { className: "flex justify-between text-xs text-slate-500 mb-0.5", children: [_jsx("span", { children: "cache hit rate \u2014 1h history" }), _jsxs("span", { children: [hitData[hitData.length - 1].v, "%"] })] }), _jsx(ResponsiveContainer, { width: "100%", height: 40, children: _jsxs(LineChart, { data: hitData, margin: { top: 2, right: 0, bottom: 2, left: 0 }, children: [_jsx(Line, { type: "monotone", dataKey: "v", stroke: "#10b981", strokeWidth: 1.5, dot: false, isAnimationActive: false }), _jsx(Tooltip, { content: _jsx(SparkTooltip, { formatter: (v) => `${v}%` }) })] }) })] })] }));
|
||||||
|
}
|
||||||
@@ -0,0 +1,136 @@
|
|||||||
|
import { useEffect, useState } from 'react';
|
||||||
|
import {
|
||||||
|
AreaChart,
|
||||||
|
Area,
|
||||||
|
LineChart,
|
||||||
|
Line,
|
||||||
|
ResponsiveContainer,
|
||||||
|
Tooltip,
|
||||||
|
ReferenceLine,
|
||||||
|
} from 'recharts';
|
||||||
|
import { api, MetricSample, fmtBytes } from '../lib/api';
|
||||||
|
|
||||||
|
interface Props {
|
||||||
|
nodeName: string;
|
||||||
|
hotMaxBytes: number;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Compact tooltip for sparklines — shows a single value line.
|
||||||
|
function SparkTooltip({
|
||||||
|
active,
|
||||||
|
payload,
|
||||||
|
label: _label,
|
||||||
|
formatter,
|
||||||
|
}: {
|
||||||
|
active?: boolean;
|
||||||
|
payload?: { value: number }[];
|
||||||
|
label?: unknown;
|
||||||
|
formatter: (v: number) => string;
|
||||||
|
}) {
|
||||||
|
if (!active || !payload?.length) return null;
|
||||||
|
return (
|
||||||
|
<div className="rounded bg-slate-800 border border-slate-700 px-2 py-1 text-xs text-slate-200 shadow-lg">
|
||||||
|
{formatter(payload[0].value)}
|
||||||
|
</div>
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
export function NodeHistorySparklines({ nodeName, hotMaxBytes }: Props) {
|
||||||
|
const [samples, setSamples] = useState<MetricSample[]>([]);
|
||||||
|
|
||||||
|
useEffect(() => {
|
||||||
|
let cancelled = false;
|
||||||
|
api
|
||||||
|
.metricsHistory(nodeName, 60)
|
||||||
|
.then((data) => {
|
||||||
|
if (!cancelled) setSamples(data);
|
||||||
|
})
|
||||||
|
.catch(() => {});
|
||||||
|
return () => {
|
||||||
|
cancelled = true;
|
||||||
|
};
|
||||||
|
}, [nodeName]);
|
||||||
|
|
||||||
|
if (samples.length < 2) return null;
|
||||||
|
|
||||||
|
// Recharts data arrays
|
||||||
|
const hotData = samples.map((s) => ({ t: s.unix_ts, v: s.hot_used_bytes }));
|
||||||
|
const hitData = samples.map((s) => ({
|
||||||
|
t: s.unix_ts,
|
||||||
|
v: Math.round(s.cache_hit_rate * 100),
|
||||||
|
}));
|
||||||
|
|
||||||
|
// Hot-tier ceiling reference line (90% of max = gc threshold)
|
||||||
|
const gcLine = hotMaxBytes > 0 ? hotMaxBytes * 0.9 : null;
|
||||||
|
|
||||||
|
return (
|
||||||
|
<div className="space-y-3">
|
||||||
|
{/* Hot-tier usage sparkline */}
|
||||||
|
{hotMaxBytes > 0 && (
|
||||||
|
<div>
|
||||||
|
<div className="flex justify-between text-xs text-slate-500 mb-0.5">
|
||||||
|
<span>hot tier — 1h history</span>
|
||||||
|
<span>{fmtBytes(hotData[hotData.length - 1].v)}</span>
|
||||||
|
</div>
|
||||||
|
<ResponsiveContainer width="100%" height={40}>
|
||||||
|
<AreaChart data={hotData} margin={{ top: 2, right: 0, bottom: 2, left: 0 }}>
|
||||||
|
<defs>
|
||||||
|
<linearGradient id={`hg-${nodeName}`} x1="0" y1="0" x2="0" y2="1">
|
||||||
|
<stop offset="5%" stopColor="#3b82f6" stopOpacity={0.4} />
|
||||||
|
<stop offset="95%" stopColor="#3b82f6" stopOpacity={0} />
|
||||||
|
</linearGradient>
|
||||||
|
</defs>
|
||||||
|
{gcLine && (
|
||||||
|
<ReferenceLine
|
||||||
|
y={gcLine}
|
||||||
|
stroke="#ef4444"
|
||||||
|
strokeDasharray="3 3"
|
||||||
|
strokeWidth={1}
|
||||||
|
/>
|
||||||
|
)}
|
||||||
|
<Area
|
||||||
|
type="monotone"
|
||||||
|
dataKey="v"
|
||||||
|
stroke="#3b82f6"
|
||||||
|
strokeWidth={1.5}
|
||||||
|
fill={`url(#hg-${nodeName})`}
|
||||||
|
dot={false}
|
||||||
|
isAnimationActive={false}
|
||||||
|
/>
|
||||||
|
<Tooltip
|
||||||
|
content={
|
||||||
|
<SparkTooltip formatter={(v) => fmtBytes(v)} />
|
||||||
|
}
|
||||||
|
/>
|
||||||
|
</AreaChart>
|
||||||
|
</ResponsiveContainer>
|
||||||
|
</div>
|
||||||
|
)}
|
||||||
|
|
||||||
|
{/* Cache hit-rate sparkline */}
|
||||||
|
<div>
|
||||||
|
<div className="flex justify-between text-xs text-slate-500 mb-0.5">
|
||||||
|
<span>cache hit rate — 1h history</span>
|
||||||
|
<span>{hitData[hitData.length - 1].v}%</span>
|
||||||
|
</div>
|
||||||
|
<ResponsiveContainer width="100%" height={40}>
|
||||||
|
<LineChart data={hitData} margin={{ top: 2, right: 0, bottom: 2, left: 0 }}>
|
||||||
|
<Line
|
||||||
|
type="monotone"
|
||||||
|
dataKey="v"
|
||||||
|
stroke="#10b981"
|
||||||
|
strokeWidth={1.5}
|
||||||
|
dot={false}
|
||||||
|
isAnimationActive={false}
|
||||||
|
/>
|
||||||
|
<Tooltip
|
||||||
|
content={
|
||||||
|
<SparkTooltip formatter={(v) => `${v}%`} />
|
||||||
|
}
|
||||||
|
/>
|
||||||
|
</LineChart>
|
||||||
|
</ResponsiveContainer>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
);
|
||||||
|
}
|
||||||
@@ -0,0 +1,40 @@
|
|||||||
|
import { jsx as _jsx, jsxs as _jsxs } from "react/jsx-runtime";
|
||||||
|
import { useEffect, useState } from 'react';
|
||||||
|
import { api, fmtBytes } from '../lib/api';
|
||||||
|
function ScoreBar({ score, max }) {
|
||||||
|
const pct = max > 0 ? Math.min(100, (score / max) * 100) : 0;
|
||||||
|
const color = pct > 66
|
||||||
|
? 'bg-red-500'
|
||||||
|
: pct > 33
|
||||||
|
? 'bg-amber-500'
|
||||||
|
: 'bg-emerald-500';
|
||||||
|
return (_jsx("div", { className: "w-20 h-1.5 bg-zinc-700 rounded-full overflow-hidden", children: _jsx("div", { className: `h-full rounded-full ${color}`, style: { width: `${pct}%` } }) }));
|
||||||
|
}
|
||||||
|
export function PollutionPanel() {
|
||||||
|
const [candidates, setCandidates] = useState([]);
|
||||||
|
const [loading, setLoading] = useState(true);
|
||||||
|
const [error, setError] = useState(null);
|
||||||
|
useEffect(() => {
|
||||||
|
api
|
||||||
|
.pollution()
|
||||||
|
.then((rows) => {
|
||||||
|
setCandidates(rows);
|
||||||
|
setLoading(false);
|
||||||
|
})
|
||||||
|
.catch((e) => {
|
||||||
|
setError(String(e));
|
||||||
|
setLoading(false);
|
||||||
|
});
|
||||||
|
}, []);
|
||||||
|
if (loading) {
|
||||||
|
return (_jsx("div", { className: "rounded-xl border border-zinc-700/50 bg-zinc-800/40 p-4 text-zinc-400 text-sm", children: "Loading pollution candidates\u2026" }));
|
||||||
|
}
|
||||||
|
if (error) {
|
||||||
|
return (_jsx("div", { className: "rounded-xl border border-red-700/40 bg-zinc-800/40 p-4 text-red-400 text-sm", children: error }));
|
||||||
|
}
|
||||||
|
if (candidates.length === 0) {
|
||||||
|
return (_jsxs("div", { className: "rounded-xl border border-zinc-700/50 bg-zinc-800/40 p-4", children: [_jsx("h2", { className: "text-sm font-semibold text-zinc-300 mb-1", children: "Cache Pollution" }), _jsx("p", { className: "text-zinc-500 text-sm", children: "No blobs tracked yet \u2014 pollution scores accumulate as builds run." })] }));
|
||||||
|
}
|
||||||
|
const maxScore = candidates[0]?.pollution_score ?? 1;
|
||||||
|
return (_jsxs("div", { className: "rounded-xl border border-zinc-700/50 bg-zinc-800/40 p-4", children: [_jsxs("div", { className: "flex items-baseline justify-between mb-3", children: [_jsxs("h2", { className: "text-sm font-semibold text-zinc-300", children: ["Cache Pollution", _jsx("span", { className: "ml-2 text-xs text-zinc-500 font-normal", children: "T2.8" })] }), _jsxs("span", { className: "text-xs text-zinc-500", children: [candidates.length, " entries"] })] }), _jsx("p", { className: "text-xs text-zinc-500 mb-3", children: "Large blobs with few recorded accesses since last restart. High-score entries consume hot-tier space disproportionate to their build value." }), _jsx("div", { className: "overflow-x-auto", children: _jsxs("table", { className: "w-full text-xs text-left", children: [_jsx("thead", { children: _jsxs("tr", { className: "text-zinc-500 border-b border-zinc-700/50", children: [_jsx("th", { className: "pb-2 pr-4 font-medium", children: "Node" }), _jsx("th", { className: "pb-2 pr-4 font-medium", children: "Fingerprint" }), _jsx("th", { className: "pb-2 pr-4 font-medium text-right", children: "Size" }), _jsx("th", { className: "pb-2 pr-4 font-medium text-right", children: "Hits" }), _jsx("th", { className: "pb-2 font-medium", children: "Score" })] }) }), _jsx("tbody", { children: candidates.map((c) => (_jsxs("tr", { className: "border-b border-zinc-700/30 last:border-0 hover:bg-zinc-700/20", children: [_jsx("td", { className: "py-2 pr-4", children: _jsx("span", { className: "text-zinc-400 font-mono", children: c.node }) }), _jsx("td", { className: "py-2 pr-4", children: _jsxs("code", { className: "font-mono text-xs bg-zinc-700 px-1 py-0.5 rounded", children: [c.fingerprint_hex.slice(0, 12), "\u2026"] }) }), _jsx("td", { className: "py-2 pr-4 text-right text-zinc-300 font-mono", children: fmtBytes(c.size_bytes) }), _jsx("td", { className: "py-2 pr-4 text-right", children: _jsx("span", { className: c.hit_count === 0 ? 'text-red-400' : 'text-zinc-400', children: c.hit_count === 0 ? '0 ✗' : c.hit_count.toLocaleString() }) }), _jsx("td", { className: "py-2", children: _jsx(ScoreBar, { score: c.pollution_score, max: maxScore }) })] }, `${c.node}-${c.fingerprint_hex}`))) })] }) })] }));
|
||||||
|
}
|
||||||
@@ -0,0 +1,124 @@
|
|||||||
|
import { useEffect, useState } from 'react';
|
||||||
|
import { api, PollutionCandidate, fmtBytes } from '../lib/api';
|
||||||
|
|
||||||
|
function ScoreBar({ score, max }: { score: number; max: number }) {
|
||||||
|
const pct = max > 0 ? Math.min(100, (score / max) * 100) : 0;
|
||||||
|
const color =
|
||||||
|
pct > 66
|
||||||
|
? 'bg-red-500'
|
||||||
|
: pct > 33
|
||||||
|
? 'bg-amber-500'
|
||||||
|
: 'bg-emerald-500';
|
||||||
|
return (
|
||||||
|
<div className="w-20 h-1.5 bg-zinc-700 rounded-full overflow-hidden">
|
||||||
|
<div className={`h-full rounded-full ${color}`} style={{ width: `${pct}%` }} />
|
||||||
|
</div>
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
export function PollutionPanel() {
|
||||||
|
const [candidates, setCandidates] = useState<PollutionCandidate[]>([]);
|
||||||
|
const [loading, setLoading] = useState(true);
|
||||||
|
const [error, setError] = useState<string | null>(null);
|
||||||
|
|
||||||
|
useEffect(() => {
|
||||||
|
api
|
||||||
|
.pollution()
|
||||||
|
.then((rows) => {
|
||||||
|
setCandidates(rows);
|
||||||
|
setLoading(false);
|
||||||
|
})
|
||||||
|
.catch((e) => {
|
||||||
|
setError(String(e));
|
||||||
|
setLoading(false);
|
||||||
|
});
|
||||||
|
}, []);
|
||||||
|
|
||||||
|
if (loading) {
|
||||||
|
return (
|
||||||
|
<div className="rounded-xl border border-zinc-700/50 bg-zinc-800/40 p-4 text-zinc-400 text-sm">
|
||||||
|
Loading pollution candidates…
|
||||||
|
</div>
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
if (error) {
|
||||||
|
return (
|
||||||
|
<div className="rounded-xl border border-red-700/40 bg-zinc-800/40 p-4 text-red-400 text-sm">
|
||||||
|
{error}
|
||||||
|
</div>
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
if (candidates.length === 0) {
|
||||||
|
return (
|
||||||
|
<div className="rounded-xl border border-zinc-700/50 bg-zinc-800/40 p-4">
|
||||||
|
<h2 className="text-sm font-semibold text-zinc-300 mb-1">Cache Pollution</h2>
|
||||||
|
<p className="text-zinc-500 text-sm">
|
||||||
|
No blobs tracked yet — pollution scores accumulate as builds run.
|
||||||
|
</p>
|
||||||
|
</div>
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
const maxScore = candidates[0]?.pollution_score ?? 1;
|
||||||
|
|
||||||
|
return (
|
||||||
|
<div className="rounded-xl border border-zinc-700/50 bg-zinc-800/40 p-4">
|
||||||
|
<div className="flex items-baseline justify-between mb-3">
|
||||||
|
<h2 className="text-sm font-semibold text-zinc-300">
|
||||||
|
Cache Pollution
|
||||||
|
<span className="ml-2 text-xs text-zinc-500 font-normal">T2.8</span>
|
||||||
|
</h2>
|
||||||
|
<span className="text-xs text-zinc-500">{candidates.length} entries</span>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<p className="text-xs text-zinc-500 mb-3">
|
||||||
|
Large blobs with few recorded accesses since last restart. High-score entries
|
||||||
|
consume hot-tier space disproportionate to their build value.
|
||||||
|
</p>
|
||||||
|
|
||||||
|
<div className="overflow-x-auto">
|
||||||
|
<table className="w-full text-xs text-left">
|
||||||
|
<thead>
|
||||||
|
<tr className="text-zinc-500 border-b border-zinc-700/50">
|
||||||
|
<th className="pb-2 pr-4 font-medium">Node</th>
|
||||||
|
<th className="pb-2 pr-4 font-medium">Fingerprint</th>
|
||||||
|
<th className="pb-2 pr-4 font-medium text-right">Size</th>
|
||||||
|
<th className="pb-2 pr-4 font-medium text-right">Hits</th>
|
||||||
|
<th className="pb-2 font-medium">Score</th>
|
||||||
|
</tr>
|
||||||
|
</thead>
|
||||||
|
<tbody>
|
||||||
|
{candidates.map((c) => (
|
||||||
|
<tr
|
||||||
|
key={`${c.node}-${c.fingerprint_hex}`}
|
||||||
|
className="border-b border-zinc-700/30 last:border-0 hover:bg-zinc-700/20"
|
||||||
|
>
|
||||||
|
<td className="py-2 pr-4">
|
||||||
|
<span className="text-zinc-400 font-mono">{c.node}</span>
|
||||||
|
</td>
|
||||||
|
<td className="py-2 pr-4">
|
||||||
|
<code className="font-mono text-xs bg-zinc-700 px-1 py-0.5 rounded">
|
||||||
|
{c.fingerprint_hex.slice(0, 12)}…
|
||||||
|
</code>
|
||||||
|
</td>
|
||||||
|
<td className="py-2 pr-4 text-right text-zinc-300 font-mono">
|
||||||
|
{fmtBytes(c.size_bytes)}
|
||||||
|
</td>
|
||||||
|
<td className="py-2 pr-4 text-right">
|
||||||
|
<span className={c.hit_count === 0 ? 'text-red-400' : 'text-zinc-400'}>
|
||||||
|
{c.hit_count === 0 ? '0 ✗' : c.hit_count.toLocaleString()}
|
||||||
|
</span>
|
||||||
|
</td>
|
||||||
|
<td className="py-2">
|
||||||
|
<ScoreBar score={c.pollution_score} max={maxScore} />
|
||||||
|
</td>
|
||||||
|
</tr>
|
||||||
|
))}
|
||||||
|
</tbody>
|
||||||
|
</table>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
);
|
||||||
|
}
|
||||||
@@ -0,0 +1,39 @@
|
|||||||
|
import { jsxs as _jsxs, jsx as _jsx } from "react/jsx-runtime";
|
||||||
|
import { useEffect, useState } from 'react';
|
||||||
|
import { api, fmtAge } from '../lib/api';
|
||||||
|
function FpBadge({ hex }) {
|
||||||
|
return (_jsxs("code", { className: "font-mono text-xs bg-zinc-700 px-1 py-0.5 rounded", children: [hex.slice(0, 12), "\u2026"] }));
|
||||||
|
}
|
||||||
|
function NodePill({ name, variant }) {
|
||||||
|
const colors = variant === 'hot'
|
||||||
|
? 'bg-orange-900/50 text-orange-300 border border-orange-700/40'
|
||||||
|
: 'bg-zinc-700/60 text-zinc-400 border border-zinc-600/40';
|
||||||
|
return (_jsx("span", { className: `inline-block px-1.5 py-0.5 rounded text-xs mr-1 ${colors}`, children: name }));
|
||||||
|
}
|
||||||
|
export function WarmingCandidatesPanel() {
|
||||||
|
const [candidates, setCandidates] = useState([]);
|
||||||
|
const [loading, setLoading] = useState(true);
|
||||||
|
const [error, setError] = useState(null);
|
||||||
|
useEffect(() => {
|
||||||
|
api
|
||||||
|
.hotRefs()
|
||||||
|
.then((rows) => {
|
||||||
|
setCandidates(rows);
|
||||||
|
setLoading(false);
|
||||||
|
})
|
||||||
|
.catch((e) => {
|
||||||
|
setError(String(e));
|
||||||
|
setLoading(false);
|
||||||
|
});
|
||||||
|
}, []);
|
||||||
|
if (loading) {
|
||||||
|
return (_jsx("div", { className: "rounded-xl border border-zinc-700/50 bg-zinc-800/40 p-4 text-zinc-400 text-sm", children: "Loading warming candidates\u2026" }));
|
||||||
|
}
|
||||||
|
if (error) {
|
||||||
|
return (_jsx("div", { className: "rounded-xl border border-red-700/40 bg-zinc-800/40 p-4 text-red-400 text-sm", children: error }));
|
||||||
|
}
|
||||||
|
if (candidates.length === 0) {
|
||||||
|
return (_jsxs("div", { className: "rounded-xl border border-zinc-700/50 bg-zinc-800/40 p-4", children: [_jsx("h2", { className: "text-sm font-semibold text-zinc-300 mb-1", children: "Warming Candidates" }), _jsx("p", { className: "text-zinc-500 text-sm", children: "No asymmetric cache entries yet \u2014 all hot fingerprints are present on every node, or no access data has been collected." })] }));
|
||||||
|
}
|
||||||
|
return (_jsxs("div", { className: "rounded-xl border border-zinc-700/50 bg-zinc-800/40 p-4", children: [_jsxs("div", { className: "flex items-baseline justify-between mb-3", children: [_jsxs("h2", { className: "text-sm font-semibold text-zinc-300", children: ["Warming Candidates", _jsx("span", { className: "ml-2 text-xs text-zinc-500 font-normal", children: "T2.7" })] }), _jsxs("span", { className: "text-xs text-zinc-500", children: [candidates.length, " fingerprints"] })] }), _jsx("p", { className: "text-xs text-zinc-500 mb-3", children: "Fingerprints frequently accessed on some nodes but absent on others. Pre-positioning these blobs eliminates cross-node forwarding on the next build." }), _jsx("div", { className: "overflow-x-auto", children: _jsxs("table", { className: "w-full text-xs text-left", children: [_jsx("thead", { children: _jsxs("tr", { className: "text-zinc-500 border-b border-zinc-700/50", children: [_jsx("th", { className: "pb-2 pr-4 font-medium", children: "Fingerprint" }), _jsx("th", { className: "pb-2 pr-4 font-medium", children: "Hot on" }), _jsx("th", { className: "pb-2 pr-4 font-medium", children: "Missing on" }), _jsx("th", { className: "pb-2 pr-4 font-medium text-right", children: "Hits" }), _jsx("th", { className: "pb-2 font-medium text-right", children: "Last seen" })] }) }), _jsx("tbody", { children: candidates.map((c) => (_jsxs("tr", { className: "border-b border-zinc-700/30 last:border-0 hover:bg-zinc-700/20", children: [_jsx("td", { className: "py-2 pr-4", children: _jsx(FpBadge, { hex: c.fingerprint_hex }) }), _jsx("td", { className: "py-2 pr-4", children: c.hot_on.map((n) => (_jsx(NodePill, { name: n, variant: "hot" }, n))) }), _jsx("td", { className: "py-2 pr-4", children: c.missing_on.map((n) => (_jsx(NodePill, { name: n, variant: "missing" }, n))) }), _jsx("td", { className: "py-2 pr-4 text-right text-zinc-300 font-mono", children: c.max_hit_count.toLocaleString() }), _jsx("td", { className: "py-2 text-right text-zinc-400", children: c.last_hit_unix > 0 ? fmtAge(c.last_hit_unix) : '—' })] }, c.fingerprint_hex))) })] }) })] }));
|
||||||
|
}
|
||||||
@@ -0,0 +1,128 @@
|
|||||||
|
import { useEffect, useState } from 'react';
|
||||||
|
import { api, WarmingCandidate, fmtAge } from '../lib/api';
|
||||||
|
|
||||||
|
function FpBadge({ hex }: { hex: string }) {
|
||||||
|
return (
|
||||||
|
<code className="font-mono text-xs bg-zinc-700 px-1 py-0.5 rounded">
|
||||||
|
{hex.slice(0, 12)}…
|
||||||
|
</code>
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
function NodePill({ name, variant }: { name: string; variant: 'hot' | 'missing' }) {
|
||||||
|
const colors =
|
||||||
|
variant === 'hot'
|
||||||
|
? 'bg-orange-900/50 text-orange-300 border border-orange-700/40'
|
||||||
|
: 'bg-zinc-700/60 text-zinc-400 border border-zinc-600/40';
|
||||||
|
return (
|
||||||
|
<span className={`inline-block px-1.5 py-0.5 rounded text-xs mr-1 ${colors}`}>
|
||||||
|
{name}
|
||||||
|
</span>
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
export function WarmingCandidatesPanel() {
|
||||||
|
const [candidates, setCandidates] = useState<WarmingCandidate[]>([]);
|
||||||
|
const [loading, setLoading] = useState(true);
|
||||||
|
const [error, setError] = useState<string | null>(null);
|
||||||
|
|
||||||
|
useEffect(() => {
|
||||||
|
api
|
||||||
|
.hotRefs()
|
||||||
|
.then((rows) => {
|
||||||
|
setCandidates(rows);
|
||||||
|
setLoading(false);
|
||||||
|
})
|
||||||
|
.catch((e) => {
|
||||||
|
setError(String(e));
|
||||||
|
setLoading(false);
|
||||||
|
});
|
||||||
|
}, []);
|
||||||
|
|
||||||
|
if (loading) {
|
||||||
|
return (
|
||||||
|
<div className="rounded-xl border border-zinc-700/50 bg-zinc-800/40 p-4 text-zinc-400 text-sm">
|
||||||
|
Loading warming candidates…
|
||||||
|
</div>
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
if (error) {
|
||||||
|
return (
|
||||||
|
<div className="rounded-xl border border-red-700/40 bg-zinc-800/40 p-4 text-red-400 text-sm">
|
||||||
|
{error}
|
||||||
|
</div>
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
if (candidates.length === 0) {
|
||||||
|
return (
|
||||||
|
<div className="rounded-xl border border-zinc-700/50 bg-zinc-800/40 p-4">
|
||||||
|
<h2 className="text-sm font-semibold text-zinc-300 mb-1">Warming Candidates</h2>
|
||||||
|
<p className="text-zinc-500 text-sm">
|
||||||
|
No asymmetric cache entries yet — all hot fingerprints are present on every node, or
|
||||||
|
no access data has been collected.
|
||||||
|
</p>
|
||||||
|
</div>
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
return (
|
||||||
|
<div className="rounded-xl border border-zinc-700/50 bg-zinc-800/40 p-4">
|
||||||
|
<div className="flex items-baseline justify-between mb-3">
|
||||||
|
<h2 className="text-sm font-semibold text-zinc-300">
|
||||||
|
Warming Candidates
|
||||||
|
<span className="ml-2 text-xs text-zinc-500 font-normal">T2.7</span>
|
||||||
|
</h2>
|
||||||
|
<span className="text-xs text-zinc-500">{candidates.length} fingerprints</span>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<p className="text-xs text-zinc-500 mb-3">
|
||||||
|
Fingerprints frequently accessed on some nodes but absent on others. Pre-positioning these
|
||||||
|
blobs eliminates cross-node forwarding on the next build.
|
||||||
|
</p>
|
||||||
|
|
||||||
|
<div className="overflow-x-auto">
|
||||||
|
<table className="w-full text-xs text-left">
|
||||||
|
<thead>
|
||||||
|
<tr className="text-zinc-500 border-b border-zinc-700/50">
|
||||||
|
<th className="pb-2 pr-4 font-medium">Fingerprint</th>
|
||||||
|
<th className="pb-2 pr-4 font-medium">Hot on</th>
|
||||||
|
<th className="pb-2 pr-4 font-medium">Missing on</th>
|
||||||
|
<th className="pb-2 pr-4 font-medium text-right">Hits</th>
|
||||||
|
<th className="pb-2 font-medium text-right">Last seen</th>
|
||||||
|
</tr>
|
||||||
|
</thead>
|
||||||
|
<tbody>
|
||||||
|
{candidates.map((c) => (
|
||||||
|
<tr
|
||||||
|
key={c.fingerprint_hex}
|
||||||
|
className="border-b border-zinc-700/30 last:border-0 hover:bg-zinc-700/20"
|
||||||
|
>
|
||||||
|
<td className="py-2 pr-4">
|
||||||
|
<FpBadge hex={c.fingerprint_hex} />
|
||||||
|
</td>
|
||||||
|
<td className="py-2 pr-4">
|
||||||
|
{c.hot_on.map((n) => (
|
||||||
|
<NodePill key={n} name={n} variant="hot" />
|
||||||
|
))}
|
||||||
|
</td>
|
||||||
|
<td className="py-2 pr-4">
|
||||||
|
{c.missing_on.map((n) => (
|
||||||
|
<NodePill key={n} name={n} variant="missing" />
|
||||||
|
))}
|
||||||
|
</td>
|
||||||
|
<td className="py-2 pr-4 text-right text-zinc-300 font-mono">
|
||||||
|
{c.max_hit_count.toLocaleString()}
|
||||||
|
</td>
|
||||||
|
<td className="py-2 text-right text-zinc-400">
|
||||||
|
{c.last_hit_unix > 0 ? fmtAge(c.last_hit_unix) : '—'}
|
||||||
|
</td>
|
||||||
|
</tr>
|
||||||
|
))}
|
||||||
|
</tbody>
|
||||||
|
</table>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
);
|
||||||
|
}
|
||||||
@@ -27,6 +27,11 @@ export const api = {
|
|||||||
fleet: () => get('/v2/fleet'),
|
fleet: () => get('/v2/fleet'),
|
||||||
projects: () => get('/v2/projects'),
|
projects: () => get('/v2/projects'),
|
||||||
nodeStatus: (name) => get(`/v2/node/${name}/status`),
|
nodeStatus: (name) => get(`/v2/node/${name}/status`),
|
||||||
|
metricsHistory: (name, limit = 60) => get(`/v2/node/${name}/metrics-history?limit=${limit}`),
|
||||||
|
anomalies: () => get('/v2/anomalies'),
|
||||||
|
hotRefs: () => get('/v2/hot-refs'),
|
||||||
|
pollution: () => get('/v2/pollution'),
|
||||||
|
maintenance: () => get('/v2/maintenance'),
|
||||||
blobs: (limit = 200, offset = 0) => get(`/v2/storage/blobs?limit=${limit}&offset=${offset}`),
|
blobs: (limit = 200, offset = 0) => get(`/v2/storage/blobs?limit=${limit}&offset=${offset}`),
|
||||||
tags: (prefix = '') => get(`/v2/storage/tags?prefix=${encodeURIComponent(prefix)}`),
|
tags: (prefix = '') => get(`/v2/storage/tags?prefix=${encodeURIComponent(prefix)}`),
|
||||||
refs: (limit = 200, offset = 0) => get(`/v2/storage/refs?limit=${limit}&offset=${offset}`),
|
refs: (limit = 200, offset = 0) => get(`/v2/storage/refs?limit=${limit}&offset=${offset}`),
|
||||||
@@ -57,3 +62,16 @@ export function fmtAge(unix) {
|
|||||||
return `${Math.floor(diff / 3600)}h ago`;
|
return `${Math.floor(diff / 3600)}h ago`;
|
||||||
return `${Math.floor(diff / 86400)}d ago`;
|
return `${Math.floor(diff / 86400)}d ago`;
|
||||||
}
|
}
|
||||||
|
/** "up 3d 14h" from a daemon start unix timestamp. */
|
||||||
|
export function fmtUptime(startedUnix) {
|
||||||
|
const now = Math.floor(Date.now() / 1000);
|
||||||
|
const secs = Math.max(0, now - startedUnix);
|
||||||
|
const days = Math.floor(secs / 86400);
|
||||||
|
const hours = Math.floor((secs % 86400) / 3600);
|
||||||
|
const mins = Math.floor((secs % 3600) / 60);
|
||||||
|
if (days > 0)
|
||||||
|
return `up ${days}d ${hours}h`;
|
||||||
|
if (hours > 0)
|
||||||
|
return `up ${hours}h ${mins}m`;
|
||||||
|
return `up ${mins}m`;
|
||||||
|
}
|
||||||
|
|||||||
@@ -25,6 +25,14 @@ export interface CacheSummary {
|
|||||||
bytes_served: number;
|
bytes_served: number;
|
||||||
bytes_ingested: number;
|
bytes_ingested: number;
|
||||||
hit_rate: number;
|
hit_rate: number;
|
||||||
|
// Per-type breakdown (available when backend >= Phase A).
|
||||||
|
get_ref_hits: number;
|
||||||
|
get_ref_misses: number;
|
||||||
|
get_tag_hits: number;
|
||||||
|
get_tag_misses: number;
|
||||||
|
/** HasChunk probes — non-zero means partial-sync dedup is active. */
|
||||||
|
has_chunk_hits: number;
|
||||||
|
has_chunk_misses: number;
|
||||||
}
|
}
|
||||||
|
|
||||||
export interface TimerStatus {
|
export interface TimerStatus {
|
||||||
@@ -49,6 +57,12 @@ export interface NodeStatusV2 {
|
|||||||
mount: MountStatus | null;
|
mount: MountStatus | null;
|
||||||
cache: CacheSummary | null;
|
cache: CacheSummary | null;
|
||||||
timers: TimerStatus[];
|
timers: TimerStatus[];
|
||||||
|
/** Unix timestamp (seconds) when the daemon last started. */
|
||||||
|
daemon_started_unix: number | null;
|
||||||
|
/** 1-minute load average from /proc/loadavg. Null on non-Linux. */
|
||||||
|
cpu_load_1m: number | null;
|
||||||
|
/** Lifeguard health score: 0=healthy, 1-2=degraded, 3+=stressed. */
|
||||||
|
health_score: number;
|
||||||
online: boolean;
|
online: boolean;
|
||||||
error: string | null;
|
error: string | null;
|
||||||
}
|
}
|
||||||
@@ -132,10 +146,67 @@ export interface ProjectRow {
|
|||||||
tier: 'active' | 'recent' | 'idle' | string;
|
tier: 'active' | 'recent' | 'idle' | string;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/** One 1-minute snapshot from the server-side metrics ring buffer. */
|
||||||
|
export interface MetricSample {
|
||||||
|
unix_ts: number;
|
||||||
|
hot_used_bytes: number;
|
||||||
|
hot_max_bytes: number;
|
||||||
|
cache_hit_rate: number;
|
||||||
|
cache_hits: number;
|
||||||
|
cache_misses: number;
|
||||||
|
has_chunk_hits: number;
|
||||||
|
has_chunk_misses: number;
|
||||||
|
fs_used_bytes: number;
|
||||||
|
fs_total_bytes: number;
|
||||||
|
fs_available_bytes: number;
|
||||||
|
/** Sum of z² across 4 drift-adapted metrics. Absent until 10 samples collected. */
|
||||||
|
anomaly_score?: number;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface AnomalyStatus {
|
||||||
|
node: string;
|
||||||
|
score: number;
|
||||||
|
level: 'ok' | 'warn' | 'alert';
|
||||||
|
samples_used: number;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface WarmingCandidate {
|
||||||
|
fingerprint_hex: string;
|
||||||
|
hot_on: string[];
|
||||||
|
missing_on: string[];
|
||||||
|
max_hit_count: number;
|
||||||
|
last_hit_unix: number;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface PollutionCandidate {
|
||||||
|
node: string;
|
||||||
|
fingerprint_hex: string;
|
||||||
|
blob_id_hex: string;
|
||||||
|
size_bytes: number;
|
||||||
|
hit_count: number;
|
||||||
|
pollution_score: number;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface MaintenanceRow {
|
||||||
|
node: string;
|
||||||
|
kind: string;
|
||||||
|
unix_ts: number;
|
||||||
|
chunks_scanned: number;
|
||||||
|
chunks_removed: number;
|
||||||
|
bytes_reclaimed: number;
|
||||||
|
error: string | null;
|
||||||
|
}
|
||||||
|
|
||||||
export const api = {
|
export const api = {
|
||||||
fleet: () => get<FleetSnapshot>('/v2/fleet'),
|
fleet: () => get<FleetSnapshot>('/v2/fleet'),
|
||||||
projects: () => get<ProjectRow[]>('/v2/projects'),
|
projects: () => get<ProjectRow[]>('/v2/projects'),
|
||||||
nodeStatus: (name: string) => get<NodeStatusV2>(`/v2/node/${name}/status`),
|
nodeStatus: (name: string) => get<NodeStatusV2>(`/v2/node/${name}/status`),
|
||||||
|
metricsHistory: (name: string, limit = 60) =>
|
||||||
|
get<MetricSample[]>(`/v2/node/${name}/metrics-history?limit=${limit}`),
|
||||||
|
anomalies: () => get<AnomalyStatus[]>('/v2/anomalies'),
|
||||||
|
hotRefs: () => get<WarmingCandidate[]>('/v2/hot-refs'),
|
||||||
|
pollution: () => get<PollutionCandidate[]>('/v2/pollution'),
|
||||||
|
maintenance: () => get<MaintenanceRow[]>('/v2/maintenance'),
|
||||||
blobs: (limit = 200, offset = 0) =>
|
blobs: (limit = 200, offset = 0) =>
|
||||||
get<BlobSummary[]>(`/v2/storage/blobs?limit=${limit}&offset=${offset}`),
|
get<BlobSummary[]>(`/v2/storage/blobs?limit=${limit}&offset=${offset}`),
|
||||||
tags: (prefix = '') =>
|
tags: (prefix = '') =>
|
||||||
@@ -165,3 +236,15 @@ export function fmtAge(unix: number): string {
|
|||||||
if (diff < 86400) return `${Math.floor(diff / 3600)}h ago`;
|
if (diff < 86400) return `${Math.floor(diff / 3600)}h ago`;
|
||||||
return `${Math.floor(diff / 86400)}d ago`;
|
return `${Math.floor(diff / 86400)}d ago`;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/** "up 3d 14h" from a daemon start unix timestamp. */
|
||||||
|
export function fmtUptime(startedUnix: number): string {
|
||||||
|
const now = Math.floor(Date.now() / 1000);
|
||||||
|
const secs = Math.max(0, now - startedUnix);
|
||||||
|
const days = Math.floor(secs / 86400);
|
||||||
|
const hours = Math.floor((secs % 86400) / 3600);
|
||||||
|
const mins = Math.floor((secs % 3600) / 60);
|
||||||
|
if (days > 0) return `up ${days}d ${hours}h`;
|
||||||
|
if (hours > 0) return `up ${hours}h ${mins}m`;
|
||||||
|
return `up ${mins}m`;
|
||||||
|
}
|
||||||
|
|||||||
@@ -3,10 +3,13 @@ import { useEffect, useState } from 'react';
|
|||||||
import { api, fmtBytes, fmtAge } from '../lib/api';
|
import { api, fmtBytes, fmtAge } from '../lib/api';
|
||||||
import { NodeCard } from '../components/NodeCard';
|
import { NodeCard } from '../components/NodeCard';
|
||||||
import { ProjectsPanel } from '../components/ProjectsPanel';
|
import { ProjectsPanel } from '../components/ProjectsPanel';
|
||||||
|
import { WarmingCandidatesPanel } from '../components/WarmingCandidatesPanel';
|
||||||
|
import { PollutionPanel } from '../components/PollutionPanel';
|
||||||
// FleetHealth landing — human-oriented single-pane-of-glass.
|
// FleetHealth landing — human-oriented single-pane-of-glass.
|
||||||
// Polls the aggregator's /api/v2/fleet every 10 s.
|
// Polls the aggregator's /api/v2/fleet every 10 s.
|
||||||
export function CommandCenter() {
|
export function CommandCenter() {
|
||||||
const [fleet, setFleet] = useState(null);
|
const [fleet, setFleet] = useState(null);
|
||||||
|
const [anomalies, setAnomalies] = useState([]);
|
||||||
const [err, setErr] = useState(null);
|
const [err, setErr] = useState(null);
|
||||||
const [tick, setTick] = useState(0);
|
const [tick, setTick] = useState(0);
|
||||||
useEffect(() => {
|
useEffect(() => {
|
||||||
@@ -17,6 +20,7 @@ export function CommandCenter() {
|
|||||||
setErr(null);
|
setErr(null);
|
||||||
})
|
})
|
||||||
.catch((e) => setErr(String(e)));
|
.catch((e) => setErr(String(e)));
|
||||||
|
api.anomalies().then(setAnomalies).catch(() => { });
|
||||||
}, [tick]);
|
}, [tick]);
|
||||||
useEffect(() => {
|
useEffect(() => {
|
||||||
const id = setInterval(() => setTick((t) => t + 1), 10_000);
|
const id = setInterval(() => setTick((t) => t + 1), 10_000);
|
||||||
@@ -32,6 +36,10 @@ export function CommandCenter() {
|
|||||||
mounted: a.mounted + (n.mount?.active ? 1 : 0),
|
mounted: a.mounted + (n.mount?.active ? 1 : 0),
|
||||||
}), { diskUsed: 0, diskTotal: 0, hotUsed: 0, hotMax: 0, online: 0, mounted: 0 })
|
}), { diskUsed: 0, diskTotal: 0, hotUsed: 0, hotMax: 0, online: 0, mounted: 0 })
|
||||||
: null;
|
: null;
|
||||||
return (_jsxs("div", { className: "space-y-6", children: [_jsxs("div", { children: [_jsx("h1", { className: "text-2xl font-semibold text-slate-100", children: "Fleet health" }), _jsx("div", { className: "text-sm text-slate-500 mt-1", children: fleet && (_jsxs(_Fragment, { children: [_jsx("span", { className: "text-emerald-300", children: totals?.online }), "/", fleet.nodes.length, " nodes online", ' · ', totals?.mounted, "/", fleet.nodes.length, " mounted", ' · ', "updated ", _jsx("span", { className: "font-mono", children: fmtAge(fleet.fetched_at_unix) }), ' · ', "hosted by ", _jsx("span", { className: "font-mono text-slate-300", children: fleet.aggregator_name })] })) })] }), err && (_jsx("div", { className: "rounded border border-red-800 bg-red-950/40 p-3 text-red-300 text-sm", children: err })), totals && totals.diskTotal > 0 && (_jsxs("div", { className: "rounded-lg border border-slate-800 bg-slate-900/60 p-4 flex items-center justify-between", children: [_jsxs("div", { children: [_jsx("div", { className: "text-xs uppercase tracking-wider text-slate-500", children: "Fleet-wide storage" }), _jsxs("div", { className: "text-2xl font-semibold font-mono mt-1", children: [fmtBytes(totals.diskUsed), ' ', _jsxs("span", { className: "text-slate-500 text-lg", children: ["/ ", fmtBytes(totals.diskTotal)] })] })] }), _jsxs("div", { className: "text-right text-slate-400 text-sm", children: [_jsxs("div", { children: [Math.round((totals.diskUsed / totals.diskTotal) * 100), "% used"] }), _jsxs("div", { className: "text-xs text-slate-500 mt-1", children: ["hot tier: ", fmtBytes(totals.hotUsed), " / ", fmtBytes(totals.hotMax)] })] })] })), _jsxs("section", { children: [_jsx("h2", { className: "text-lg font-semibold text-slate-100 mb-3", children: "Nodes" }), _jsxs("div", { className: "grid grid-cols-1 sm:grid-cols-2 lg:grid-cols-3 gap-4", children: [fleet?.nodes.map((n) => (_jsx(NodeCard, { node: n }, n.node_name))), !fleet &&
|
// Build a node→level lookup for NodeCard props.
|
||||||
[1, 2, 3].map((i) => (_jsx("div", { className: "rounded-lg border border-slate-800 bg-slate-900/40 p-5 h-64 animate-pulse" }, i)))] })] }), _jsx(ProjectsPanel, {})] }));
|
const anomalyMap = Object.fromEntries(anomalies.map((a) => [a.node, a.level]));
|
||||||
|
const alertNodes = anomalies.filter((a) => a.level === 'alert');
|
||||||
|
const warnNodes = anomalies.filter((a) => a.level === 'warn');
|
||||||
|
return (_jsxs("div", { className: "space-y-6", children: [_jsxs("div", { children: [_jsx("h1", { className: "text-2xl font-semibold text-slate-100", children: "Fleet health" }), _jsx("div", { className: "text-sm text-slate-500 mt-1", children: fleet && (_jsxs(_Fragment, { children: [_jsx("span", { className: "text-emerald-300", children: totals?.online }), "/", fleet.nodes.length, " nodes online", ' · ', totals?.mounted, "/", fleet.nodes.length, " mounted", ' · ', "updated ", _jsx("span", { className: "font-mono", children: fmtAge(fleet.fetched_at_unix) }), ' · ', "hosted by ", _jsx("span", { className: "font-mono text-slate-300", children: fleet.aggregator_name })] })) })] }), err && (_jsx("div", { className: "rounded border border-red-800 bg-red-950/40 p-3 text-red-300 text-sm", children: err })), alertNodes.length > 0 && (_jsxs("div", { className: "rounded border border-red-800 bg-red-950/40 px-4 py-3 text-sm flex items-start gap-3", children: [_jsx("span", { className: "text-red-400 font-bold mt-0.5", children: "\u25CF" }), _jsxs("div", { children: [_jsx("span", { className: "text-red-300 font-semibold", children: "Metric anomaly detected \u2014 " }), _jsxs("span", { className: "text-red-200", children: [alertNodes.map((a) => a.node).join(', '), " deviating >2\u03C3 from baseline", alertNodes.length === 1 && ` (score ${alertNodes[0].score.toFixed(1)})`] })] })] })), alertNodes.length === 0 && warnNodes.length > 0 && (_jsxs("div", { className: "rounded border border-amber-800 bg-amber-950/40 px-4 py-3 text-sm flex items-start gap-3", children: [_jsx("span", { className: "text-amber-400 font-bold mt-0.5", children: "\u25CF" }), _jsxs("div", { children: [_jsx("span", { className: "text-amber-300 font-semibold", children: "Metric drift \u2014 " }), _jsxs("span", { className: "text-amber-200", children: [warnNodes.map((a) => a.node).join(', '), " showing unusual patterns"] })] })] })), totals && totals.diskTotal > 0 && (_jsxs("div", { className: "rounded-lg border border-slate-800 bg-slate-900/60 p-4 flex items-center justify-between", children: [_jsxs("div", { children: [_jsx("div", { className: "text-xs uppercase tracking-wider text-slate-500", children: "Fleet-wide storage" }), _jsxs("div", { className: "text-2xl font-semibold font-mono mt-1", children: [fmtBytes(totals.diskUsed), ' ', _jsxs("span", { className: "text-slate-500 text-lg", children: ["/ ", fmtBytes(totals.diskTotal)] })] })] }), _jsxs("div", { className: "text-right text-slate-400 text-sm", children: [_jsxs("div", { children: [Math.round((totals.diskUsed / totals.diskTotal) * 100), "% used"] }), _jsxs("div", { className: "text-xs text-slate-500 mt-1", children: ["hot tier: ", fmtBytes(totals.hotUsed), " / ", fmtBytes(totals.hotMax)] })] })] })), _jsxs("section", { children: [_jsx("h2", { className: "text-lg font-semibold text-slate-100 mb-3", children: "Nodes" }), _jsxs("div", { className: "grid grid-cols-1 sm:grid-cols-2 lg:grid-cols-3 gap-4", children: [fleet?.nodes.map((n) => (_jsx(NodeCard, { node: n, anomalyLevel: anomalyMap[n.node_name] }, n.node_name))), !fleet &&
|
||||||
|
[1, 2, 3].map((i) => (_jsx("div", { className: "rounded-lg border border-slate-800 bg-slate-900/40 p-5 h-64 animate-pulse" }, i)))] })] }), _jsx(ProjectsPanel, {}), _jsx(WarmingCandidatesPanel, {}), _jsx(PollutionPanel, {})] }));
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1,12 +1,15 @@
|
|||||||
import { useEffect, useState } from 'react';
|
import { useEffect, useState } from 'react';
|
||||||
import { api, FleetSnapshot, fmtBytes, fmtAge } from '../lib/api';
|
import { api, FleetSnapshot, AnomalyStatus, fmtBytes, fmtAge } from '../lib/api';
|
||||||
import { NodeCard } from '../components/NodeCard';
|
import { NodeCard } from '../components/NodeCard';
|
||||||
import { ProjectsPanel } from '../components/ProjectsPanel';
|
import { ProjectsPanel } from '../components/ProjectsPanel';
|
||||||
|
import { WarmingCandidatesPanel } from '../components/WarmingCandidatesPanel';
|
||||||
|
import { PollutionPanel } from '../components/PollutionPanel';
|
||||||
|
|
||||||
// FleetHealth landing — human-oriented single-pane-of-glass.
|
// FleetHealth landing — human-oriented single-pane-of-glass.
|
||||||
// Polls the aggregator's /api/v2/fleet every 10 s.
|
// Polls the aggregator's /api/v2/fleet every 10 s.
|
||||||
export function CommandCenter() {
|
export function CommandCenter() {
|
||||||
const [fleet, setFleet] = useState<FleetSnapshot | null>(null);
|
const [fleet, setFleet] = useState<FleetSnapshot | null>(null);
|
||||||
|
const [anomalies, setAnomalies] = useState<AnomalyStatus[]>([]);
|
||||||
const [err, setErr] = useState<string | null>(null);
|
const [err, setErr] = useState<string | null>(null);
|
||||||
const [tick, setTick] = useState(0);
|
const [tick, setTick] = useState(0);
|
||||||
|
|
||||||
@@ -18,6 +21,7 @@ export function CommandCenter() {
|
|||||||
setErr(null);
|
setErr(null);
|
||||||
})
|
})
|
||||||
.catch((e) => setErr(String(e)));
|
.catch((e) => setErr(String(e)));
|
||||||
|
api.anomalies().then(setAnomalies).catch(() => {});
|
||||||
}, [tick]);
|
}, [tick]);
|
||||||
|
|
||||||
useEffect(() => {
|
useEffect(() => {
|
||||||
@@ -39,6 +43,13 @@ export function CommandCenter() {
|
|||||||
)
|
)
|
||||||
: null;
|
: null;
|
||||||
|
|
||||||
|
// Build a node→level lookup for NodeCard props.
|
||||||
|
const anomalyMap = Object.fromEntries(
|
||||||
|
anomalies.map((a) => [a.node, a.level] as const)
|
||||||
|
);
|
||||||
|
const alertNodes = anomalies.filter((a) => a.level === 'alert');
|
||||||
|
const warnNodes = anomalies.filter((a) => a.level === 'warn');
|
||||||
|
|
||||||
return (
|
return (
|
||||||
<div className="space-y-6">
|
<div className="space-y-6">
|
||||||
<div>
|
<div>
|
||||||
@@ -65,6 +76,31 @@ export function CommandCenter() {
|
|||||||
</div>
|
</div>
|
||||||
)}
|
)}
|
||||||
|
|
||||||
|
{/* Anomaly banners (only shown once the ring buffer has 10+ samples) */}
|
||||||
|
{alertNodes.length > 0 && (
|
||||||
|
<div className="rounded border border-red-800 bg-red-950/40 px-4 py-3 text-sm flex items-start gap-3">
|
||||||
|
<span className="text-red-400 font-bold mt-0.5">●</span>
|
||||||
|
<div>
|
||||||
|
<span className="text-red-300 font-semibold">Metric anomaly detected — </span>
|
||||||
|
<span className="text-red-200">
|
||||||
|
{alertNodes.map((a) => a.node).join(', ')} deviating >2σ from baseline
|
||||||
|
{alertNodes.length === 1 && ` (score ${alertNodes[0].score.toFixed(1)})`}
|
||||||
|
</span>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
)}
|
||||||
|
{alertNodes.length === 0 && warnNodes.length > 0 && (
|
||||||
|
<div className="rounded border border-amber-800 bg-amber-950/40 px-4 py-3 text-sm flex items-start gap-3">
|
||||||
|
<span className="text-amber-400 font-bold mt-0.5">●</span>
|
||||||
|
<div>
|
||||||
|
<span className="text-amber-300 font-semibold">Metric drift — </span>
|
||||||
|
<span className="text-amber-200">
|
||||||
|
{warnNodes.map((a) => a.node).join(', ')} showing unusual patterns
|
||||||
|
</span>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
)}
|
||||||
|
|
||||||
{totals && totals.diskTotal > 0 && (
|
{totals && totals.diskTotal > 0 && (
|
||||||
<div className="rounded-lg border border-slate-800 bg-slate-900/60 p-4 flex items-center justify-between">
|
<div className="rounded-lg border border-slate-800 bg-slate-900/60 p-4 flex items-center justify-between">
|
||||||
<div>
|
<div>
|
||||||
@@ -89,7 +125,11 @@ export function CommandCenter() {
|
|||||||
<h2 className="text-lg font-semibold text-slate-100 mb-3">Nodes</h2>
|
<h2 className="text-lg font-semibold text-slate-100 mb-3">Nodes</h2>
|
||||||
<div className="grid grid-cols-1 sm:grid-cols-2 lg:grid-cols-3 gap-4">
|
<div className="grid grid-cols-1 sm:grid-cols-2 lg:grid-cols-3 gap-4">
|
||||||
{fleet?.nodes.map((n) => (
|
{fleet?.nodes.map((n) => (
|
||||||
<NodeCard key={n.node_name} node={n} />
|
<NodeCard
|
||||||
|
key={n.node_name}
|
||||||
|
node={n}
|
||||||
|
anomalyLevel={anomalyMap[n.node_name]}
|
||||||
|
/>
|
||||||
))}
|
))}
|
||||||
{!fleet &&
|
{!fleet &&
|
||||||
[1, 2, 3].map((i) => (
|
[1, 2, 3].map((i) => (
|
||||||
@@ -102,6 +142,10 @@ export function CommandCenter() {
|
|||||||
</section>
|
</section>
|
||||||
|
|
||||||
<ProjectsPanel />
|
<ProjectsPanel />
|
||||||
|
|
||||||
|
<WarmingCandidatesPanel />
|
||||||
|
|
||||||
|
<PollutionPanel />
|
||||||
</div>
|
</div>
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -3,6 +3,7 @@ import { useEffect, useState } from 'react';
|
|||||||
import { Link } from 'wouter';
|
import { Link } from 'wouter';
|
||||||
import { api, fmtBytes } from '../lib/api';
|
import { api, fmtBytes } from '../lib/api';
|
||||||
import { StatTile } from '../components/StatTile';
|
import { StatTile } from '../components/StatTile';
|
||||||
|
import { MaintenancePanel } from '../components/MaintenancePanel';
|
||||||
export function NodeDetail({ name }) {
|
export function NodeDetail({ name }) {
|
||||||
const [status, setStatus] = useState(null);
|
const [status, setStatus] = useState(null);
|
||||||
const [err, setErr] = useState(null);
|
const [err, setErr] = useState(null);
|
||||||
@@ -15,5 +16,5 @@ export function NodeDetail({ name }) {
|
|||||||
})
|
})
|
||||||
.catch((e) => setErr(String(e)));
|
.catch((e) => setErr(String(e)));
|
||||||
}, [name]);
|
}, [name]);
|
||||||
return (_jsxs("div", { className: "space-y-6", children: [_jsxs("div", { children: [_jsx(Link, { href: "/", children: _jsx("a", { className: "text-sm text-slate-500 hover:text-slate-300", children: "\u2190 fleet" }) }), _jsx("h1", { className: "text-2xl font-semibold text-slate-100 mt-2", children: name })] }), err && (_jsxs("div", { className: "rounded border border-amber-800 bg-amber-950/40 p-3 text-amber-300 text-sm", children: [err, _jsxs("div", { className: "mt-2 text-xs text-slate-400", children: ["Cross-node lookup lands in a follow-on PR. Until then this page shows detail only when you're already viewing the dashboard hosted by ", name, ". Try opening", ' ', _jsxs("span", { className: "font-mono", children: ["http://", name, ":7700/v2/#/nodes/", name] }), ' ', "directly."] })] })), status && (_jsxs(_Fragment, { children: [_jsxs("div", { className: "grid grid-cols-2 sm:grid-cols-3 lg:grid-cols-6 gap-3", children: [_jsx(StatTile, { label: "blobs", value: status.blob_count.toLocaleString(), color: "ok" }), _jsx(StatTile, { label: "tags", value: status.tag_count, color: "ok" }), _jsx(StatTile, { label: "refs", value: status.ref_count, color: "ok" }), _jsx(StatTile, { label: "snapshots", value: status.snapshot_count, color: "ok" }), _jsx(StatTile, { label: "ref-tracking", value: status.ref_tracking_count, color: "ok" }), _jsx(StatTile, { label: "store size", value: fmtBytes(status.blob_store_bytes), color: "ok" })] }), _jsxs("div", { className: "rounded border border-slate-800 bg-slate-900/40 p-4 text-sm", children: [_jsx("div", { className: "text-slate-500 uppercase text-xs tracking-wider", children: "blob store root" }), _jsx("div", { className: "font-mono mt-1", children: status.blob_store_root ?? '—' })] })] }))] }));
|
return (_jsxs("div", { className: "space-y-6", children: [_jsxs("div", { children: [_jsx(Link, { href: "/", children: _jsx("a", { className: "text-sm text-slate-500 hover:text-slate-300", children: "\u2190 fleet" }) }), _jsx("h1", { className: "text-2xl font-semibold text-slate-100 mt-2", children: name })] }), err && (_jsxs("div", { className: "rounded border border-amber-800 bg-amber-950/40 p-3 text-amber-300 text-sm", children: [err, _jsxs("div", { className: "mt-2 text-xs text-slate-400", children: ["Cross-node lookup lands in a follow-on PR. Until then this page shows detail only when you're already viewing the dashboard hosted by ", name, ". Try opening", ' ', _jsxs("span", { className: "font-mono", children: ["http://", name, ":7700/v2/#/nodes/", name] }), ' ', "directly."] })] })), status && (_jsxs(_Fragment, { children: [_jsxs("div", { className: "grid grid-cols-2 sm:grid-cols-3 lg:grid-cols-6 gap-3", children: [_jsx(StatTile, { label: "blobs", value: status.blob_count.toLocaleString(), color: "ok" }), _jsx(StatTile, { label: "tags", value: status.tag_count, color: "ok" }), _jsx(StatTile, { label: "refs", value: status.ref_count, color: "ok" }), _jsx(StatTile, { label: "snapshots", value: status.snapshot_count, color: "ok" }), _jsx(StatTile, { label: "ref-tracking", value: status.ref_tracking_count, color: "ok" }), _jsx(StatTile, { label: "store size", value: fmtBytes(status.blob_store_bytes), color: "ok" })] }), _jsxs("div", { className: "rounded border border-slate-800 bg-slate-900/40 p-4 text-sm", children: [_jsx("div", { className: "text-slate-500 uppercase text-xs tracking-wider", children: "blob store root" }), _jsx("div", { className: "font-mono mt-1", children: status.blob_store_root ?? '—' })] })] })), _jsx(MaintenancePanel, {})] }));
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -2,6 +2,7 @@ import { useEffect, useState } from 'react';
|
|||||||
import { Link } from 'wouter';
|
import { Link } from 'wouter';
|
||||||
import { api, NodeStatusV2, fmtBytes } from '../lib/api';
|
import { api, NodeStatusV2, fmtBytes } from '../lib/api';
|
||||||
import { StatTile } from '../components/StatTile';
|
import { StatTile } from '../components/StatTile';
|
||||||
|
import { MaintenancePanel } from '../components/MaintenancePanel';
|
||||||
|
|
||||||
interface Props {
|
interface Props {
|
||||||
name: string;
|
name: string;
|
||||||
@@ -69,6 +70,8 @@ export function NodeDetail({ name }: Props) {
|
|||||||
</div>
|
</div>
|
||||||
</>
|
</>
|
||||||
)}
|
)}
|
||||||
|
|
||||||
|
<MaintenancePanel />
|
||||||
</div>
|
</div>
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1 +1 @@
|
|||||||
{"root":["./src/app.tsx","./src/main.tsx","./src/components/nodecard.tsx","./src/components/projectspanel.tsx","./src/components/stattile.tsx","./src/components/storagebar.tsx","./src/lib/api.ts","./src/pages/commandcenter.tsx","./src/pages/nodedetail.tsx","./src/pages/reftrackingpage.tsx","./src/pages/storagebrowser.tsx"],"version":"6.0.3"}
|
{"root":["./src/App.tsx","./src/main.tsx","./src/components/MaintenancePanel.tsx","./src/components/NodeCard.tsx","./src/components/NodeHistorySparklines.tsx","./src/components/PollutionPanel.tsx","./src/components/ProjectsPanel.tsx","./src/components/StatTile.tsx","./src/components/StorageBar.tsx","./src/components/WarmingCandidatesPanel.tsx","./src/lib/api.ts","./src/pages/CommandCenter.tsx","./src/pages/NodeDetail.tsx","./src/pages/RefTrackingPage.tsx","./src/pages/StorageBrowser.tsx"],"version":"6.0.3"}
|
||||||
@@ -6,13 +6,12 @@ import react from '@vitejs/plugin-react';
|
|||||||
// During local dev the daemon proxies /api/v2/* on :7700 so
|
// During local dev the daemon proxies /api/v2/* on :7700 so
|
||||||
// `vite dev` on :5173 can hit it via server.proxy.
|
// `vite dev` on :5173 can hit it via server.proxy.
|
||||||
export default defineConfig({
|
export default defineConfig({
|
||||||
// Absolute base tied to the deploy path. Prior try was `./`
|
// Relative base so the SPA works under any mount point (/v2/,
|
||||||
// (fully relative) which broke when the user hit `/clawstor`
|
// /clawstor/, or bare /). The no-trailing-slash edge case
|
||||||
// without trailing slash — browser resolves `./assets/…`
|
// (browser treats the path as a file and strips the last segment)
|
||||||
// against `/clawstor` treated as a file, gives `/assets/…`,
|
// is handled server-side: serve.rs redirects /v2 → /v2/ and
|
||||||
// 404 from Tailscale. Absolute `/clawstor/` sidesteps the
|
// /clawstor → /clawstor/ before serving the SPA.
|
||||||
// slash / no-slash ambiguity.
|
base: './',
|
||||||
base: '/clawstor/',
|
|
||||||
plugins: [react()],
|
plugins: [react()],
|
||||||
server: {
|
server: {
|
||||||
port: 5173,
|
port: 5173,
|
||||||
|
|||||||
Reference in New Issue
Block a user