Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
f38efc7096 | ||
|
|
2d0c225f98 | ||
|
|
4ea1cbed2e | ||
|
|
6a4bc09cbb | ||
|
|
fe815db981 | ||
|
|
4132937021 | ||
|
|
334cd068d2 | ||
|
|
f30ea04ab6 | ||
|
|
bb24c77676 | ||
|
|
75d822d0f2 | ||
|
|
dd2b90872a | ||
|
|
5c9bc7eb9c | ||
|
|
5c1d962bf2 | ||
|
|
a3fe1d147c |
@@ -26,6 +26,7 @@ pub mod refs;
|
|||||||
pub mod repo_ensure;
|
pub mod repo_ensure;
|
||||||
pub mod rpc;
|
pub mod rpc;
|
||||||
pub mod services;
|
pub mod services;
|
||||||
|
pub mod shutdown_prep;
|
||||||
pub mod snapshot;
|
pub mod snapshot;
|
||||||
pub mod tags;
|
pub mod tags;
|
||||||
pub mod tailscale;
|
pub mod tailscale;
|
||||||
|
|||||||
@@ -703,90 +703,6 @@ impl BlobStore {
|
|||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
/// T3.4: pollution-score eviction.
|
|
||||||
///
|
|
||||||
/// Evicts blobs using `score = ln(chunk_count+1) * ln(age_secs+1) / (hit_count+1)`.
|
|
||||||
/// Highest-scoring (large, old, cold) blobs are evicted first. Falls back to
|
|
||||||
/// LRU ordering for blobs not in `hit_map` (treated as hit_count=0).
|
|
||||||
pub async fn evict_to_size_cap_with_scores(
|
|
||||||
&self,
|
|
||||||
max_bytes: u64,
|
|
||||||
pinned_blobs: &std::collections::HashSet<BlobId>,
|
|
||||||
hit_map: &std::collections::HashMap<BlobId, u64>,
|
|
||||||
) -> Result<GcReport> {
|
|
||||||
let manifest_summaries = self.collect_manifest_summaries().await?;
|
|
||||||
let mut referenced: std::collections::HashMap<ChunkHash, u32> =
|
|
||||||
std::collections::HashMap::new();
|
|
||||||
for summary in &manifest_summaries {
|
|
||||||
for hash in &summary.chunks {
|
|
||||||
*referenced.entry(*hash).or_insert(0) += 1;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
let mut current_size: u64 = 0;
|
|
||||||
for hash in referenced.keys() {
|
|
||||||
if let Ok(meta) = tokio::fs::metadata(&self.chunk_path(hash)).await {
|
|
||||||
current_size = current_size.saturating_add(meta.len());
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
let now_unix = std::time::SystemTime::now()
|
|
||||||
.duration_since(std::time::UNIX_EPOCH)
|
|
||||||
.map(|d| d.as_secs())
|
|
||||||
.unwrap_or(0);
|
|
||||||
|
|
||||||
let mut summaries = manifest_summaries;
|
|
||||||
// Sort by pollution eviction score descending (evict highest first).
|
|
||||||
summaries.sort_by(|a, b| {
|
|
||||||
let score_of = |s: &ManifestSummary| {
|
|
||||||
let hit_count = hit_map.get(&s.blob_id).copied().unwrap_or(0);
|
|
||||||
let age = now_unix.saturating_sub(s.manifest_mtime);
|
|
||||||
let size_f = (s.chunks.len() as f64 + 1.0).ln();
|
|
||||||
let age_f = (age as f64 + 1.0).ln();
|
|
||||||
size_f * age_f / (hit_count as f64 + 1.0)
|
|
||||||
};
|
|
||||||
score_of(b)
|
|
||||||
.partial_cmp(&score_of(a))
|
|
||||||
.unwrap_or(std::cmp::Ordering::Equal)
|
|
||||||
});
|
|
||||||
|
|
||||||
let mut chunks_removed = 0usize;
|
|
||||||
let mut bytes_reclaimed = 0u64;
|
|
||||||
|
|
||||||
for summary in summaries {
|
|
||||||
if current_size <= max_bytes {
|
|
||||||
break;
|
|
||||||
}
|
|
||||||
if pinned_blobs.contains(&summary.blob_id) {
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
self.delete_manifest(&summary.blob_id).await?;
|
|
||||||
for hash in &summary.chunks {
|
|
||||||
let entry = referenced.entry(*hash).or_insert(0);
|
|
||||||
if *entry > 0 {
|
|
||||||
*entry -= 1;
|
|
||||||
}
|
|
||||||
if *entry == 0 {
|
|
||||||
let path = self.chunk_path(hash);
|
|
||||||
if let Ok(meta) = tokio::fs::metadata(&path).await {
|
|
||||||
let sz = meta.len();
|
|
||||||
if tokio::fs::remove_file(&path).await.is_ok() {
|
|
||||||
chunks_removed += 1;
|
|
||||||
bytes_reclaimed = bytes_reclaimed.saturating_add(sz);
|
|
||||||
current_size = current_size.saturating_sub(sz);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
referenced.remove(hash);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
Ok(GcReport {
|
|
||||||
chunks_scanned: 0,
|
|
||||||
chunks_removed,
|
|
||||||
bytes_reclaimed,
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Enumerate every on-disk manifest with the info eviction needs:
|
/// Enumerate every on-disk manifest with the info eviction needs:
|
||||||
/// blob_id, chunk set, and mtime for LRU ordering. Bounded by the
|
/// blob_id, chunk set, and mtime for LRU ordering. Bounded by the
|
||||||
/// number of manifests (small — one per cached target dir).
|
/// number of manifests (small — one per cached target dir).
|
||||||
|
|||||||
@@ -145,15 +145,13 @@ impl FingerprintInputs {
|
|||||||
/// inputs, order-independent for features (already sorted), and
|
/// inputs, order-independent for features (already sorted), and
|
||||||
/// null-byte-separated so `["ab","c"]` hashes differently from
|
/// null-byte-separated so `["ab","c"]` hashes differently from
|
||||||
/// `["a","bc"]`.
|
/// `["a","bc"]`.
|
||||||
///
|
|
||||||
/// v2: Cargo.lock is canonicalized before hashing — comment-only
|
|
||||||
/// changes and whitespace drift in the lock file no longer cause
|
|
||||||
/// spurious cache misses.
|
|
||||||
pub fn compute(&self) -> Fingerprint {
|
pub fn compute(&self) -> Fingerprint {
|
||||||
let mut h = Hasher::new();
|
let mut h = Hasher::new();
|
||||||
h.update(b"clawstor.fingerprint.v2\0");
|
// Domain-separated by field with a fixed sentinel — different
|
||||||
let canonical_lock = canonicalize_lock(&self.cargo_lock);
|
// versions of this struct produce different hashes without a
|
||||||
update_field(&mut h, b"cargo_lock", canonical_lock.as_bytes());
|
// manual version tag.
|
||||||
|
h.update(b"clawstor.fingerprint.v1\0");
|
||||||
|
update_field(&mut h, b"cargo_lock", self.cargo_lock.as_bytes());
|
||||||
update_field(
|
update_field(
|
||||||
&mut h,
|
&mut h,
|
||||||
b"rustc_version_verbose",
|
b"rustc_version_verbose",
|
||||||
@@ -194,41 +192,6 @@ fn update_field(h: &mut Hasher, name: &[u8], value: &[u8]) {
|
|||||||
h.update(b"\0");
|
h.update(b"\0");
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Produce a canonical string from a Cargo.lock that is stable across
|
|
||||||
/// comment-only edits and whitespace drift.
|
|
||||||
///
|
|
||||||
/// Parses the TOML, extracts each `[[package]]` entry's (name, version,
|
|
||||||
/// checksum) tuple, sorts them, and emits one line per package:
|
|
||||||
/// `<name>@<version>#<checksum>\n`. If the lock is empty or cannot be
|
|
||||||
/// parsed as valid TOML the raw text is returned unchanged so we never
|
|
||||||
/// silently produce a wrong hash.
|
|
||||||
fn canonicalize_lock(raw: &str) -> String {
|
|
||||||
if raw.is_empty() {
|
|
||||||
return raw.to_string();
|
|
||||||
}
|
|
||||||
let Ok(doc) = raw.parse::<toml::Value>() else {
|
|
||||||
return raw.to_string();
|
|
||||||
};
|
|
||||||
let Some(packages) = doc.get("package").and_then(|v| v.as_array()) else {
|
|
||||||
return raw.to_string();
|
|
||||||
};
|
|
||||||
let mut entries: Vec<String> = packages
|
|
||||||
.iter()
|
|
||||||
.filter_map(|pkg| {
|
|
||||||
let name = pkg.get("name")?.as_str()?;
|
|
||||||
let version = pkg.get("version")?.as_str()?;
|
|
||||||
let checksum = pkg
|
|
||||||
.get("checksum")
|
|
||||||
.and_then(|v| v.as_str())
|
|
||||||
.unwrap_or("-");
|
|
||||||
Some(format!("{name}@{version}#{checksum}"))
|
|
||||||
})
|
|
||||||
.collect();
|
|
||||||
entries.sort();
|
|
||||||
entries.push(String::new());
|
|
||||||
entries.join("\n")
|
|
||||||
}
|
|
||||||
|
|
||||||
/// 32-byte BLAKE3 fingerprint over build inputs. Shape parallels
|
/// 32-byte BLAKE3 fingerprint over build inputs. Shape parallels
|
||||||
/// [`crate::cluster::blob::BlobId`] — same content-addressing spirit,
|
/// [`crate::cluster::blob::BlobId`] — same content-addressing spirit,
|
||||||
/// but keyed to inputs rather than outputs. A fingerprint identifies
|
/// but keyed to inputs rather than outputs. A fingerprint identifies
|
||||||
@@ -521,27 +484,13 @@ mod tests {
|
|||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn fingerprint_changes_when_cargo_lock_package_changes() {
|
fn fingerprint_changes_when_cargo_lock_changes() {
|
||||||
let base = baseline_inputs().compute();
|
let base = baseline_inputs().compute();
|
||||||
let mut mutated = baseline_inputs();
|
let mut mutated = baseline_inputs();
|
||||||
// A version bump changes the fingerprint.
|
mutated.cargo_lock.push_str("# extra line\n");
|
||||||
mutated.cargo_lock = mutated.cargo_lock.replace("0.1.0", "0.2.0");
|
|
||||||
assert_ne!(base, mutated.compute());
|
assert_ne!(base, mutated.compute());
|
||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
|
||||||
fn fingerprint_stable_across_cargo_lock_comment_changes() {
|
|
||||||
let base = baseline_inputs().compute();
|
|
||||||
let mut mutated = baseline_inputs();
|
|
||||||
// Comments and extra blank lines must NOT change the fingerprint.
|
|
||||||
mutated.cargo_lock.push_str("# extra line\n\n");
|
|
||||||
assert_eq!(
|
|
||||||
base,
|
|
||||||
mutated.compute(),
|
|
||||||
"comment-only lock change should not alter fingerprint"
|
|
||||||
);
|
|
||||||
}
|
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn fingerprint_changes_when_profile_changes() {
|
fn fingerprint_changes_when_profile_changes() {
|
||||||
let base = baseline_inputs().compute();
|
let base = baseline_inputs().compute();
|
||||||
@@ -596,33 +545,6 @@ mod tests {
|
|||||||
assert_eq!(recomputed.to_hex(), fp.to_hex());
|
assert_eq!(recomputed.to_hex(), fp.to_hex());
|
||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
|
||||||
fn canonicalize_lock_is_stable_across_comments() {
|
|
||||||
let with_comment =
|
|
||||||
"# generated\n[[package]]\nname = \"foo\"\nversion = \"1.0.0\"\nchecksum = \"abc\"\n";
|
|
||||||
let without_comment =
|
|
||||||
"[[package]]\nname = \"foo\"\nversion = \"1.0.0\"\nchecksum = \"abc\"\n";
|
|
||||||
assert_eq!(
|
|
||||||
canonicalize_lock(with_comment),
|
|
||||||
canonicalize_lock(without_comment)
|
|
||||||
);
|
|
||||||
}
|
|
||||||
|
|
||||||
#[test]
|
|
||||||
fn canonicalize_lock_sorts_packages() {
|
|
||||||
let a_first =
|
|
||||||
"[[package]]\nname = \"aaa\"\nversion = \"1.0\"\n\n[[package]]\nname = \"zzz\"\nversion = \"1.0\"\n";
|
|
||||||
let z_first =
|
|
||||||
"[[package]]\nname = \"zzz\"\nversion = \"1.0\"\n\n[[package]]\nname = \"aaa\"\nversion = \"1.0\"\n";
|
|
||||||
assert_eq!(canonicalize_lock(a_first), canonicalize_lock(z_first));
|
|
||||||
}
|
|
||||||
|
|
||||||
#[test]
|
|
||||||
fn canonicalize_lock_fallback_on_invalid_toml() {
|
|
||||||
let garbage = "not valid toml [[[";
|
|
||||||
assert_eq!(canonicalize_lock(garbage), garbage);
|
|
||||||
}
|
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn read_optional_returns_empty_for_missing() {
|
fn read_optional_returns_empty_for_missing() {
|
||||||
let tmp = tempfile::TempDir::new().unwrap();
|
let tmp = tempfile::TempDir::new().unwrap();
|
||||||
|
|||||||
@@ -67,10 +67,6 @@ pub mod keys {
|
|||||||
/// that two nodes will silo their caches. Peer-visible via
|
/// that two nodes will silo their caches. Peer-visible via
|
||||||
/// `PeerView.rustc_release` and `cluster-peer-status`.
|
/// `PeerView.rustc_release` and `cluster-peer-status`.
|
||||||
pub const RUSTC_RELEASE: &str = "clawstor.rustc.release";
|
pub const RUSTC_RELEASE: &str = "clawstor.rustc.release";
|
||||||
/// T3.3 / T3.6: Lifeguard-style composite health score (u8 as string).
|
|
||||||
/// 0 = healthy, 1-2 = degraded (hot >90% or load >4), 3+ = stressed.
|
|
||||||
/// Peers use this to apply leniency before marking a stressed node dead.
|
|
||||||
pub const HEALTH_SCORE: &str = "clawstor.health_score";
|
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Cluster identifier — every node in the same fleet must agree on this
|
/// Cluster identifier — every node in the same fleet must agree on this
|
||||||
@@ -125,10 +121,6 @@ pub struct PeerView {
|
|||||||
/// rustc. Used to surface toolchain drift that would otherwise
|
/// rustc. Used to surface toolchain drift that would otherwise
|
||||||
/// silently silo caches.
|
/// silently silo caches.
|
||||||
pub rustc_release: Option<String>,
|
pub rustc_release: Option<String>,
|
||||||
/// T3.3: Lifeguard health score advertised by the peer.
|
|
||||||
/// 0 = healthy, 1-2 = degraded, 3+ = stressed. `None` until the peer
|
|
||||||
/// has published the score (older daemons or very early boot).
|
|
||||||
pub health_score: Option<u8>,
|
|
||||||
}
|
}
|
||||||
|
|
||||||
impl PeerView {
|
impl PeerView {
|
||||||
@@ -284,12 +276,6 @@ impl ClusterGossip {
|
|||||||
/// rate locally via [`PeerView::cache_get_ref_hit_rate`]. Called
|
/// rate locally via [`PeerView::cache_get_ref_hit_rate`]. Called
|
||||||
/// periodically by [`crate::cluster::services::ClusterServices`]
|
/// periodically by [`crate::cluster::services::ClusterServices`]
|
||||||
/// once RPC has come up.
|
/// once RPC has come up.
|
||||||
/// Publish the Lifeguard health score so peers can observe our stress
|
|
||||||
/// without making an RPC round-trip. Called from the cache-metric tick.
|
|
||||||
pub async fn set_health_score(&self, score: u8) {
|
|
||||||
self.set(keys::HEALTH_SCORE, score.to_string()).await;
|
|
||||||
}
|
|
||||||
|
|
||||||
pub async fn set_cache_metrics(&self, snap: &MetricsReply) {
|
pub async fn set_cache_metrics(&self, snap: &MetricsReply) {
|
||||||
// Batch all four writes under one lock — chitchat serialises
|
// Batch all four writes under one lock — chitchat serialises
|
||||||
// per-node kv writes, so a single locked block avoids an
|
// per-node kv writes, so a single locked block avoids an
|
||||||
@@ -428,7 +414,6 @@ fn peer_view_from_state(id: &ChitchatId, state: &chitchat::NodeState, alive: boo
|
|||||||
cache_blob_get_bytes: get_u64(state, keys::CACHE_BLOB_GET_BYTES),
|
cache_blob_get_bytes: get_u64(state, keys::CACHE_BLOB_GET_BYTES),
|
||||||
cache_blob_put_bytes: get_u64(state, keys::CACHE_BLOB_PUT_BYTES),
|
cache_blob_put_bytes: get_u64(state, keys::CACHE_BLOB_PUT_BYTES),
|
||||||
rustc_release: get_str(state, keys::RUSTC_RELEASE),
|
rustc_release: get_str(state, keys::RUSTC_RELEASE),
|
||||||
health_score: get_u64(state, keys::HEALTH_SCORE).map(|v| v as u8),
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -761,7 +746,6 @@ mod tests {
|
|||||||
cache_blob_get_bytes: None,
|
cache_blob_get_bytes: None,
|
||||||
cache_blob_put_bytes: None,
|
cache_blob_put_bytes: None,
|
||||||
rustc_release: None,
|
rustc_release: None,
|
||||||
health_score: None,
|
|
||||||
};
|
};
|
||||||
assert_eq!(base.cache_get_ref_hit_rate(), None, "no counters → None");
|
assert_eq!(base.cache_get_ref_hit_rate(), None, "no counters → None");
|
||||||
|
|
||||||
@@ -808,7 +792,6 @@ mod tests {
|
|||||||
cache_blob_get_bytes: None,
|
cache_blob_get_bytes: None,
|
||||||
cache_blob_put_bytes: None,
|
cache_blob_put_bytes: None,
|
||||||
rustc_release: None,
|
rustc_release: None,
|
||||||
health_score: None,
|
|
||||||
};
|
};
|
||||||
assert_eq!(base.hot_fill_ratio(), None, "no used → None");
|
assert_eq!(base.hot_fill_ratio(), None, "no used → None");
|
||||||
|
|
||||||
|
|||||||
+42
-311
@@ -217,6 +217,22 @@ pub enum Method {
|
|||||||
/// `payload`: JSON [`crate::cluster::repo_ensure::RepoReleaseRequest`].
|
/// `payload`: JSON [`crate::cluster::repo_ensure::RepoReleaseRequest`].
|
||||||
/// Reply: JSON [`crate::cluster::repo_ensure::RepoReleaseReply`].
|
/// Reply: JSON [`crate::cluster::repo_ensure::RepoReleaseReply`].
|
||||||
RepoRelease = 0x1f,
|
RepoRelease = 0x1f,
|
||||||
|
/// Runs `safe-shutdown-prep.sh --dry-run` to completion on this
|
||||||
|
/// node and returns the full report. Never stops anything —
|
||||||
|
/// dry-run only, safe to call repeatedly.
|
||||||
|
///
|
||||||
|
/// `payload`: JSON [`crate::cluster::shutdown_prep::ShutdownPrepCheckRequest`].
|
||||||
|
/// Reply: JSON [`crate::cluster::shutdown_prep::ShutdownPrepCheckReply`].
|
||||||
|
ShutdownPrepCheck = 0x20,
|
||||||
|
/// Starts the real `safe-shutdown-prep.sh` run in a detached
|
||||||
|
/// systemd scope and returns immediately — the script's own step
|
||||||
|
/// stops `claw-store.service`, so this RPC connection cannot
|
||||||
|
/// outlive full completion. See
|
||||||
|
/// [`crate::cluster::shutdown_prep`] module docs.
|
||||||
|
///
|
||||||
|
/// `payload`: JSON [`crate::cluster::shutdown_prep::ShutdownPrepExecuteRequest`].
|
||||||
|
/// Reply: JSON [`crate::cluster::shutdown_prep::ShutdownPrepExecuteReply`].
|
||||||
|
ShutdownPrepExecute = 0x21,
|
||||||
}
|
}
|
||||||
|
|
||||||
impl Method {
|
impl Method {
|
||||||
@@ -255,6 +271,8 @@ impl Method {
|
|||||||
0x1d => Some(Method::DashboardStorage),
|
0x1d => Some(Method::DashboardStorage),
|
||||||
0x1e => Some(Method::RepoEnsure),
|
0x1e => Some(Method::RepoEnsure),
|
||||||
0x1f => Some(Method::RepoRelease),
|
0x1f => Some(Method::RepoRelease),
|
||||||
|
0x20 => Some(Method::ShutdownPrepCheck),
|
||||||
|
0x21 => Some(Method::ShutdownPrepExecute),
|
||||||
_ => None,
|
_ => None,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -366,18 +384,6 @@ pub struct DashboardStatusReply {
|
|||||||
pub cache: Option<CacheSummary>,
|
pub cache: Option<CacheSummary>,
|
||||||
#[serde(default)]
|
#[serde(default)]
|
||||||
pub timers: Vec<TimerStatus>,
|
pub timers: Vec<TimerStatus>,
|
||||||
/// Unix timestamp (seconds) when this daemon's metrics counters were
|
|
||||||
/// reset — proxy for when the daemon last started/restarted.
|
|
||||||
#[serde(default)]
|
|
||||||
pub daemon_started_unix: Option<u64>,
|
|
||||||
/// 1-minute load average from /proc/loadavg. None on non-Linux.
|
|
||||||
#[serde(default)]
|
|
||||||
pub cpu_load_1m: Option<f32>,
|
|
||||||
/// Composite Lifeguard-style health score (T3.3).
|
|
||||||
/// 0 = healthy; 1-2 = degraded; 3+ = stressed.
|
|
||||||
/// Increments for: hot >90%, fs >90%, load >4.0, failed timers.
|
|
||||||
#[serde(default)]
|
|
||||||
pub health_score: u8,
|
|
||||||
}
|
}
|
||||||
|
|
||||||
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)]
|
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)]
|
||||||
@@ -407,27 +413,11 @@ pub struct MountStatus {
|
|||||||
|
|
||||||
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)]
|
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)]
|
||||||
pub struct CacheSummary {
|
pub struct CacheSummary {
|
||||||
/// Composite totals across ref + tag + chunk lookups (for hit_rate).
|
|
||||||
pub hits: u64,
|
pub hits: u64,
|
||||||
pub misses: u64,
|
pub misses: u64,
|
||||||
pub bytes_served: u64,
|
pub bytes_served: u64,
|
||||||
pub bytes_ingested: u64,
|
pub bytes_ingested: u64,
|
||||||
pub hit_rate: f64,
|
pub hit_rate: f64,
|
||||||
/// Per-type breakdown so the dashboard can show ref vs tag vs chunk rates.
|
|
||||||
#[serde(default)]
|
|
||||||
pub get_ref_hits: u64,
|
|
||||||
#[serde(default)]
|
|
||||||
pub get_ref_misses: u64,
|
|
||||||
#[serde(default)]
|
|
||||||
pub get_tag_hits: u64,
|
|
||||||
#[serde(default)]
|
|
||||||
pub get_tag_misses: u64,
|
|
||||||
/// HasChunk probes — measures dedup efficiency (how many chunk uploads
|
|
||||||
/// were skipped because the receiver already had the chunk).
|
|
||||||
#[serde(default)]
|
|
||||||
pub has_chunk_hits: u64,
|
|
||||||
#[serde(default)]
|
|
||||||
pub has_chunk_misses: u64,
|
|
||||||
}
|
}
|
||||||
|
|
||||||
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)]
|
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)]
|
||||||
@@ -476,27 +466,6 @@ pub struct DashboardStorageReply {
|
|||||||
/// Sorted by last_seen_unix descending — hottest first.
|
/// Sorted by last_seen_unix descending — hottest first.
|
||||||
#[serde(default)]
|
#[serde(default)]
|
||||||
pub projects: Vec<DashboardProject>,
|
pub projects: Vec<DashboardProject>,
|
||||||
/// T2.7/T2.8: top-20 fingerprints by GetRef hit count since
|
|
||||||
/// daemon start. Used for cache-warming recommendations and
|
|
||||||
/// pollution detection. Empty until the first GetRef hit.
|
|
||||||
#[serde(default)]
|
|
||||||
pub hot_refs: Vec<HotRef>,
|
|
||||||
/// T1.2: recent GC/scrub/repair events on this node, newest first.
|
|
||||||
/// Capped at 100 entries; resets on daemon restart.
|
|
||||||
#[serde(default)]
|
|
||||||
pub maintenance_events: Vec<MaintenanceEvent>,
|
|
||||||
}
|
|
||||||
|
|
||||||
/// One entry from the per-fingerprint access log.
|
|
||||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
|
||||||
pub struct HotRef {
|
|
||||||
pub fingerprint_hex: String,
|
|
||||||
/// Total successful GetRef lookups since daemon start.
|
|
||||||
pub hit_count: u64,
|
|
||||||
/// Unix timestamp of the most recent hit.
|
|
||||||
pub last_hit_unix: u64,
|
|
||||||
/// Unix timestamp of the first hit.
|
|
||||||
pub first_hit_unix: u64,
|
|
||||||
}
|
}
|
||||||
|
|
||||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||||
@@ -744,31 +713,6 @@ fn timer_status(unit: &str) -> TimerStatus {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/// 1-minute load average from /proc/loadavg. None on non-Linux or read error.
|
|
||||||
pub(crate) fn read_cpu_load_1m() -> Option<f32> {
|
|
||||||
std::fs::read_to_string("/proc/loadavg")
|
|
||||||
.ok()
|
|
||||||
.and_then(|s| s.split_whitespace().next()?.parse::<f32>().ok())
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Composite Lifeguard-style health score: 0 = healthy, higher = stressed.
|
|
||||||
/// Increments for: hot tier >90%, filesystem >90%, cpu load >4.0, failed timers.
|
|
||||||
fn compute_health_score(
|
|
||||||
hot_pct: f64,
|
|
||||||
fs_pct: f64,
|
|
||||||
cpu_load: Option<f32>,
|
|
||||||
failed_timers: usize,
|
|
||||||
) -> u8 {
|
|
||||||
let mut score: u8 = 0;
|
|
||||||
if hot_pct > 0.90 { score = score.saturating_add(1); }
|
|
||||||
if fs_pct > 0.90 { score = score.saturating_add(1); }
|
|
||||||
if cpu_load.map(|l| l > 4.0).unwrap_or(false) {
|
|
||||||
score = score.saturating_add(1);
|
|
||||||
}
|
|
||||||
if failed_timers > 0 { score = score.saturating_add(1); }
|
|
||||||
score
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Is `path` currently a mount point? Cheap Linux check: read
|
/// Is `path` currently a mount point? Cheap Linux check: read
|
||||||
/// /proc/mounts. On macOS returns None (aggregator doesn't run
|
/// /proc/mounts. On macOS returns None (aggregator doesn't run
|
||||||
/// mounts).
|
/// mounts).
|
||||||
@@ -818,131 +762,12 @@ fn dir_size_bytes(root: &std::path::Path) -> u64 {
|
|||||||
total
|
total
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Maximum number of fingerprints tracked in the access log before the
|
|
||||||
/// lowest-count entry is evicted. 2000 covers ~3 months of daily unique
|
|
||||||
/// fingerprints at typical CI cadences.
|
|
||||||
const ACCESS_LOG_MAX: usize = 2000;
|
|
||||||
|
|
||||||
/// Per-fingerprint access record. Counts how many times a fingerprint
|
|
||||||
/// has been successfully served via `GetRef` or `GetRefVersioned`.
|
|
||||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
|
||||||
pub struct AccessRecord {
|
|
||||||
/// Total successful lookups since daemon start.
|
|
||||||
pub count: u64,
|
|
||||||
/// Unix timestamp of the very first hit.
|
|
||||||
pub first_unix: u64,
|
|
||||||
/// Unix timestamp of the most recent hit.
|
|
||||||
pub last_unix: u64,
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Bounded in-process log of per-fingerprint hit counts. Enables T2.7
|
|
||||||
/// (cache warming) and T2.8 (pollution detection) by recording which
|
|
||||||
/// fingerprints are hot, warm, or cold since the daemon started.
|
|
||||||
pub struct AccessLog {
|
|
||||||
entries: std::collections::HashMap<[u8; 32], AccessRecord>,
|
|
||||||
}
|
|
||||||
|
|
||||||
impl Default for AccessLog {
|
|
||||||
fn default() -> Self {
|
|
||||||
Self { entries: std::collections::HashMap::new() }
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
impl AccessLog {
|
|
||||||
/// Record one successful lookup for `key` at wall-clock `now_unix`.
|
|
||||||
/// On overflow (> ACCESS_LOG_MAX), the lowest-count entry is evicted.
|
|
||||||
pub fn record(&mut self, key: &[u8; 32], now_unix: u64) {
|
|
||||||
if let Some(rec) = self.entries.get_mut(key) {
|
|
||||||
rec.count += 1;
|
|
||||||
rec.last_unix = now_unix;
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
if self.entries.len() >= ACCESS_LOG_MAX {
|
|
||||||
// Evict the least-accessed entry so hot fingerprints survive.
|
|
||||||
if let Some(&evict_key) = self
|
|
||||||
.entries
|
|
||||||
.iter()
|
|
||||||
.min_by_key(|(_, v)| v.count)
|
|
||||||
.map(|(k, _)| k)
|
|
||||||
{
|
|
||||||
self.entries.remove(&evict_key);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
self.entries.insert(*key, AccessRecord { count: 1, first_unix: now_unix, last_unix: now_unix });
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Return the top `n` entries sorted by count descending.
|
|
||||||
pub fn top(&self, n: usize) -> Vec<([u8; 32], AccessRecord)> {
|
|
||||||
let mut v: Vec<_> = self
|
|
||||||
.entries
|
|
||||||
.iter()
|
|
||||||
.map(|(k, v)| (*k, v.clone()))
|
|
||||||
.collect();
|
|
||||||
v.sort_by(|a, b| b.1.count.cmp(&a.1.count).then(b.1.last_unix.cmp(&a.1.last_unix)));
|
|
||||||
v.truncate(n);
|
|
||||||
v
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Total number of fingerprints currently tracked.
|
|
||||||
pub fn len(&self) -> usize {
|
|
||||||
self.entries.len()
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/// T1.2: one entry in the per-node maintenance event log.
|
|
||||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
|
||||||
pub struct MaintenanceEvent {
|
|
||||||
/// Logical kind: "gc_orphan", "gc_eviction_scored", "scrub", "repair".
|
|
||||||
pub kind: String,
|
|
||||||
/// Wall-clock unix timestamp when the event completed.
|
|
||||||
pub unix_ts: u64,
|
|
||||||
pub chunks_scanned: usize,
|
|
||||||
pub chunks_removed: usize,
|
|
||||||
pub bytes_reclaimed: u64,
|
|
||||||
/// Non-empty when the operation encountered an error.
|
|
||||||
#[serde(default)]
|
|
||||||
pub error: Option<String>,
|
|
||||||
}
|
|
||||||
|
|
||||||
const MAINTENANCE_LOG_MAX: usize = 100;
|
|
||||||
|
|
||||||
/// Bounded ring of recent maintenance events. Oldest entry is dropped
|
|
||||||
/// when the cap is reached.
|
|
||||||
pub struct MaintenanceLog {
|
|
||||||
entries: std::collections::VecDeque<MaintenanceEvent>,
|
|
||||||
}
|
|
||||||
|
|
||||||
impl Default for MaintenanceLog {
|
|
||||||
fn default() -> Self {
|
|
||||||
Self { entries: std::collections::VecDeque::new() }
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
impl MaintenanceLog {
|
|
||||||
pub fn push(&mut self, ev: MaintenanceEvent) {
|
|
||||||
if self.entries.len() >= MAINTENANCE_LOG_MAX {
|
|
||||||
self.entries.pop_front();
|
|
||||||
}
|
|
||||||
self.entries.push_back(ev);
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Return all events, most-recent first.
|
|
||||||
pub fn recent(&self) -> Vec<MaintenanceEvent> {
|
|
||||||
self.entries.iter().cloned().rev().collect()
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
pub struct RpcRouter {
|
pub struct RpcRouter {
|
||||||
gossip: Arc<ClusterGossip>,
|
gossip: Arc<ClusterGossip>,
|
||||||
blob_store: Option<Arc<BlobStore>>,
|
blob_store: Option<Arc<BlobStore>>,
|
||||||
ref_store: Option<Arc<RefStore>>,
|
ref_store: Option<Arc<RefStore>>,
|
||||||
tag_store: Option<Arc<TagStore>>,
|
tag_store: Option<Arc<TagStore>>,
|
||||||
metrics: Arc<CacheMetrics>,
|
metrics: Arc<CacheMetrics>,
|
||||||
/// T2.7/T2.8: per-fingerprint hit counter. Mutex because it's
|
|
||||||
/// written on every GetRef hit and read by the dashboard poller.
|
|
||||||
access_log: std::sync::Arc<std::sync::Mutex<AccessLog>>,
|
|
||||||
/// T1.2: recent GC/scrub/repair events for the maintenance panel.
|
|
||||||
maintenance_log: std::sync::Arc<std::sync::Mutex<MaintenanceLog>>,
|
|
||||||
local_name: String,
|
local_name: String,
|
||||||
local_zone: String,
|
local_zone: String,
|
||||||
/// Ref-forwarding (2026-07-13): when set, `GetRef` misses fan out
|
/// Ref-forwarding (2026-07-13): when set, `GetRef` misses fan out
|
||||||
@@ -977,8 +802,6 @@ impl RpcRouter {
|
|||||||
ref_store: None,
|
ref_store: None,
|
||||||
tag_store: None,
|
tag_store: None,
|
||||||
metrics: Arc::new(CacheMetrics::new()),
|
metrics: Arc::new(CacheMetrics::new()),
|
||||||
access_log: std::sync::Arc::new(std::sync::Mutex::new(AccessLog::default())),
|
|
||||||
maintenance_log: std::sync::Arc::new(std::sync::Mutex::new(MaintenanceLog::default())),
|
|
||||||
local_name,
|
local_name,
|
||||||
local_zone,
|
local_zone,
|
||||||
outbound_client: None,
|
outbound_client: None,
|
||||||
@@ -1011,44 +834,6 @@ impl RpcRouter {
|
|||||||
&self.metrics
|
&self.metrics
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Read-only access to the outbound QUIC client. Used by T3.5
|
|
||||||
/// prefetch task in `ClusterServices` to open peer connections.
|
|
||||||
pub fn outbound_client(
|
|
||||||
&self,
|
|
||||||
) -> Option<Arc<crate::cluster::transport::QuicClient>> {
|
|
||||||
self.outbound_client.clone()
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Shared handle to the per-fingerprint access log (T2.7/T2.8).
|
|
||||||
/// The dashboard poller reads this to identify hot/cold refs.
|
|
||||||
pub fn access_log(&self) -> std::sync::Arc<std::sync::Mutex<AccessLog>> {
|
|
||||||
self.access_log.clone()
|
|
||||||
}
|
|
||||||
|
|
||||||
/// T1.2: shared handle to the maintenance event log.
|
|
||||||
pub fn maintenance_log(&self) -> std::sync::Arc<std::sync::Mutex<MaintenanceLog>> {
|
|
||||||
self.maintenance_log.clone()
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Append a maintenance event. Called from services.rs gc_task after
|
|
||||||
/// each GC/scrub run so the dashboard can show recent activity.
|
|
||||||
pub fn push_maintenance_event(&self, ev: MaintenanceEvent) {
|
|
||||||
if let Ok(mut log) = self.maintenance_log.lock() {
|
|
||||||
log.push(ev);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Record one successful GetRef hit in the access log.
|
|
||||||
fn record_access(&self, key: &[u8; 32]) {
|
|
||||||
let now = std::time::SystemTime::now()
|
|
||||||
.duration_since(std::time::UNIX_EPOCH)
|
|
||||||
.map(|d| d.as_secs())
|
|
||||||
.unwrap_or(0);
|
|
||||||
if let Ok(mut log) = self.access_log.lock() {
|
|
||||||
log.record(key, now);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Attach a local blob store. Enables the `Blob*` methods; nodes
|
/// Attach a local blob store. Enables the `Blob*` methods; nodes
|
||||||
/// without a store return [`ErrorCode::NotConfigured`] for those.
|
/// without a store return [`ErrorCode::NotConfigured`] for those.
|
||||||
pub fn with_blob_store(mut self, store: Arc<BlobStore>) -> Self {
|
pub fn with_blob_store(mut self, store: Arc<BlobStore>) -> Self {
|
||||||
@@ -1214,7 +999,7 @@ impl RpcRouter {
|
|||||||
// Cache metrics — the router already tracks these
|
// Cache metrics — the router already tracks these
|
||||||
// in-memory. Compute hit-rate here so the
|
// in-memory. Compute hit-rate here so the
|
||||||
// dashboard doesn't need to divide.
|
// dashboard doesn't need to divide.
|
||||||
let (cache, daemon_started_unix) = {
|
let cache = {
|
||||||
let snap = self.metrics.snapshot();
|
let snap = self.metrics.snapshot();
|
||||||
// The dashboard cares about "did the peer find
|
// The dashboard cares about "did the peer find
|
||||||
// what someone asked for". Sum the get_ref /
|
// what someone asked for". Sum the get_ref /
|
||||||
@@ -1232,25 +1017,13 @@ impl RpcRouter {
|
|||||||
} else {
|
} else {
|
||||||
0.0
|
0.0
|
||||||
};
|
};
|
||||||
let summary = Some(CacheSummary {
|
Some(CacheSummary {
|
||||||
hits,
|
hits,
|
||||||
misses,
|
misses,
|
||||||
bytes_served: snap.blob_get_bytes,
|
bytes_served: snap.blob_get_bytes,
|
||||||
bytes_ingested: snap.blob_put_bytes,
|
bytes_ingested: snap.blob_put_bytes,
|
||||||
hit_rate: rate,
|
hit_rate: rate,
|
||||||
get_ref_hits: snap.get_ref_hits,
|
})
|
||||||
get_ref_misses: snap.get_ref_misses,
|
|
||||||
get_tag_hits: snap.get_tag_hits,
|
|
||||||
get_tag_misses: snap.get_tag_misses,
|
|
||||||
has_chunk_hits: snap.has_chunk_hits,
|
|
||||||
has_chunk_misses: snap.has_chunk_misses,
|
|
||||||
});
|
|
||||||
let started = if snap.started_unix > 0 {
|
|
||||||
Some(snap.started_unix)
|
|
||||||
} else {
|
|
||||||
None
|
|
||||||
};
|
|
||||||
(summary, started)
|
|
||||||
};
|
};
|
||||||
// Well-known timer set. Missing timers just get
|
// Well-known timer set. Missing timers just get
|
||||||
// next_fire_unix=None / last_result=None.
|
// next_fire_unix=None / last_result=None.
|
||||||
@@ -1260,30 +1033,6 @@ impl RpcRouter {
|
|||||||
timer_status("clawstor-ref-sweep.timer"),
|
timer_status("clawstor-ref-sweep.timer"),
|
||||||
timer_status("clawstor-snapshot-rotate.timer"),
|
timer_status("clawstor-snapshot-rotate.timer"),
|
||||||
];
|
];
|
||||||
// Lifeguard-style health score (T3.3).
|
|
||||||
let cpu_load_1m = read_cpu_load_1m();
|
|
||||||
let hot_pct = if hot_max > 0 {
|
|
||||||
hot_used as f64 / hot_max as f64
|
|
||||||
} else {
|
|
||||||
0.0
|
|
||||||
};
|
|
||||||
let fs_pct = filesystem.as_ref().map(|f| {
|
|
||||||
if f.total_bytes > 0 {
|
|
||||||
f.used_bytes as f64 / f.total_bytes as f64
|
|
||||||
} else {
|
|
||||||
0.0
|
|
||||||
}
|
|
||||||
}).unwrap_or(0.0);
|
|
||||||
let failed_timers = timers
|
|
||||||
.iter()
|
|
||||||
.filter(|t| {
|
|
||||||
t.last_result
|
|
||||||
.as_deref()
|
|
||||||
.map(|r| r != "success")
|
|
||||||
.unwrap_or(false)
|
|
||||||
})
|
|
||||||
.count();
|
|
||||||
let health_score = compute_health_score(hot_pct, fs_pct, cpu_load_1m, failed_timers);
|
|
||||||
let reply = DashboardStatusReply {
|
let reply = DashboardStatusReply {
|
||||||
node_name: self.local_name.clone(),
|
node_name: self.local_name.clone(),
|
||||||
zone: self.local_zone.clone(),
|
zone: self.local_zone.clone(),
|
||||||
@@ -1300,9 +1049,6 @@ impl RpcRouter {
|
|||||||
mount,
|
mount,
|
||||||
cache,
|
cache,
|
||||||
timers,
|
timers,
|
||||||
daemon_started_unix,
|
|
||||||
cpu_load_1m,
|
|
||||||
health_score,
|
|
||||||
};
|
};
|
||||||
let json = serde_json::to_vec(&reply)
|
let json = serde_json::to_vec(&reply)
|
||||||
.context("encoding DashboardStatusReply as JSON")?;
|
.context("encoding DashboardStatusReply as JSON")?;
|
||||||
@@ -1396,35 +1142,6 @@ impl RpcRouter {
|
|||||||
)
|
)
|
||||||
.await;
|
.await;
|
||||||
|
|
||||||
// T2.7/T2.8: embed top-20 hot fingerprints from the
|
|
||||||
// access log so peers can recommend cross-node warming.
|
|
||||||
let hot_refs = self
|
|
||||||
.access_log
|
|
||||||
.lock()
|
|
||||||
.map(|log| {
|
|
||||||
log.top(20)
|
|
||||||
.into_iter()
|
|
||||||
.map(|(k, r)| {
|
|
||||||
let mut hex = String::with_capacity(64);
|
|
||||||
for b in &k { hex.push_str(&format!("{b:02x}")); }
|
|
||||||
HotRef {
|
|
||||||
fingerprint_hex: hex,
|
|
||||||
hit_count: r.count,
|
|
||||||
last_hit_unix: r.last_unix,
|
|
||||||
first_hit_unix: r.first_unix,
|
|
||||||
}
|
|
||||||
})
|
|
||||||
.collect()
|
|
||||||
})
|
|
||||||
.unwrap_or_default();
|
|
||||||
|
|
||||||
// T1.2: recent GC/scrub/repair events for the maintenance panel.
|
|
||||||
let maintenance_events = self
|
|
||||||
.maintenance_log
|
|
||||||
.lock()
|
|
||||||
.map(|log| log.recent())
|
|
||||||
.unwrap_or_default();
|
|
||||||
|
|
||||||
let reply = DashboardStorageReply {
|
let reply = DashboardStorageReply {
|
||||||
node_name: self.local_name.clone(),
|
node_name: self.local_name.clone(),
|
||||||
tags,
|
tags,
|
||||||
@@ -1435,8 +1152,6 @@ impl RpcRouter {
|
|||||||
refs_sample_capped_at: SAMPLE_CAP,
|
refs_sample_capped_at: SAMPLE_CAP,
|
||||||
refs_sample,
|
refs_sample,
|
||||||
projects,
|
projects,
|
||||||
hot_refs,
|
|
||||||
maintenance_events,
|
|
||||||
};
|
};
|
||||||
let json = serde_json::to_vec(&reply)
|
let json = serde_json::to_vec(&reply)
|
||||||
.context("encoding DashboardStorageReply as JSON")?;
|
.context("encoding DashboardStorageReply as JSON")?;
|
||||||
@@ -1479,6 +1194,26 @@ impl RpcRouter {
|
|||||||
.context("encoding RepoReleaseReply as JSON")?;
|
.context("encoding RepoReleaseReply as JSON")?;
|
||||||
Ok(HandlerOutcome::Reply(json))
|
Ok(HandlerOutcome::Reply(json))
|
||||||
}
|
}
|
||||||
|
Method::ShutdownPrepCheck => {
|
||||||
|
let reply = crate::cluster::shutdown_prep::check().await?;
|
||||||
|
let json = serde_json::to_vec(&reply)
|
||||||
|
.context("encoding ShutdownPrepCheckReply as JSON")?;
|
||||||
|
Ok(HandlerOutcome::Reply(json))
|
||||||
|
}
|
||||||
|
Method::ShutdownPrepExecute => {
|
||||||
|
let req: crate::cluster::shutdown_prep::ShutdownPrepExecuteRequest =
|
||||||
|
match serde_json::from_slice(payload) {
|
||||||
|
Ok(r) => r,
|
||||||
|
Err(_) => {
|
||||||
|
return Ok(HandlerOutcome::Error(ErrorCode::InvalidRequest))
|
||||||
|
}
|
||||||
|
};
|
||||||
|
let reply =
|
||||||
|
crate::cluster::shutdown_prep::execute(&self.local_name, &req).await?;
|
||||||
|
let json = serde_json::to_vec(&reply)
|
||||||
|
.context("encoding ShutdownPrepExecuteReply as JSON")?;
|
||||||
|
Ok(HandlerOutcome::Reply(json))
|
||||||
|
}
|
||||||
Method::BlobStat => {
|
Method::BlobStat => {
|
||||||
let store = match &self.blob_store {
|
let store = match &self.blob_store {
|
||||||
Some(s) => s,
|
Some(s) => s,
|
||||||
@@ -1656,7 +1391,6 @@ impl RpcRouter {
|
|||||||
// Local first.
|
// Local first.
|
||||||
if let Some(value) = store.get(&key).await? {
|
if let Some(value) = store.get(&key).await? {
|
||||||
self.metrics.record_get_ref_hit();
|
self.metrics.record_get_ref_hit();
|
||||||
self.record_access(&key);
|
|
||||||
return Ok(HandlerOutcome::Reply(value.to_vec()));
|
return Ok(HandlerOutcome::Reply(value.to_vec()));
|
||||||
}
|
}
|
||||||
// Ref-forwarding: try peers via gossip. First hit wins
|
// Ref-forwarding: try peers via gossip. First hit wins
|
||||||
@@ -1664,7 +1398,6 @@ impl RpcRouter {
|
|||||||
// (and reads) are all local.
|
// (and reads) are all local.
|
||||||
if let Some(value) = self.forward_get_ref(&key).await {
|
if let Some(value) = self.forward_get_ref(&key).await {
|
||||||
self.metrics.record_get_ref_hit();
|
self.metrics.record_get_ref_hit();
|
||||||
self.record_access(&key);
|
|
||||||
return Ok(HandlerOutcome::Reply(value.to_vec()));
|
return Ok(HandlerOutcome::Reply(value.to_vec()));
|
||||||
}
|
}
|
||||||
self.metrics.record_get_ref_miss();
|
self.metrics.record_get_ref_miss();
|
||||||
@@ -1735,7 +1468,6 @@ impl RpcRouter {
|
|||||||
// Local first.
|
// Local first.
|
||||||
if let Some(s) = store.get_stamped(&key).await? {
|
if let Some(s) = store.get_stamped(&key).await? {
|
||||||
self.metrics.record_get_ref_hit();
|
self.metrics.record_get_ref_hit();
|
||||||
self.record_access(&key);
|
|
||||||
return Ok(HandlerOutcome::Reply(s.to_bytes().to_vec()));
|
return Ok(HandlerOutcome::Reply(s.to_bytes().to_vec()));
|
||||||
}
|
}
|
||||||
// Phase 3b: cross-runner sharing for stamped refs.
|
// Phase 3b: cross-runner sharing for stamped refs.
|
||||||
@@ -1744,7 +1476,6 @@ impl RpcRouter {
|
|||||||
// pure local hits (same semantics as GetRef path).
|
// pure local hits (same semantics as GetRef path).
|
||||||
if let Some(s) = self.forward_get_ref_versioned(&key).await {
|
if let Some(s) = self.forward_get_ref_versioned(&key).await {
|
||||||
self.metrics.record_get_ref_hit();
|
self.metrics.record_get_ref_hit();
|
||||||
self.record_access(&key);
|
|
||||||
return Ok(HandlerOutcome::Reply(s.to_bytes().to_vec()));
|
return Ok(HandlerOutcome::Reply(s.to_bytes().to_vec()));
|
||||||
}
|
}
|
||||||
self.metrics.record_get_ref_miss();
|
self.metrics.record_get_ref_miss();
|
||||||
@@ -2054,7 +1785,7 @@ impl RpcRouter {
|
|||||||
/// missing chunks from `conn` into the local `BlobStore`. Same shape
|
/// missing chunks from `conn` into the local `BlobStore`. Same shape
|
||||||
/// as `prewarm_missing_chunks_between_parallel` but the downstream is
|
/// as `prewarm_missing_chunks_between_parallel` but the downstream is
|
||||||
/// in-process rather than another peer.
|
/// in-process rather than another peer.
|
||||||
pub(crate) async fn pull_blob_locally(
|
async fn pull_blob_locally(
|
||||||
conn: &Connection,
|
conn: &Connection,
|
||||||
local: &BlobStore,
|
local: &BlobStore,
|
||||||
id: &crate::cluster::blob::BlobId,
|
id: &crate::cluster::blob::BlobId,
|
||||||
|
|||||||
@@ -229,6 +229,45 @@ pub async fn call_repo_release(
|
|||||||
serde_json::from_slice(&reply).context("decoding RepoReleaseReply JSON")
|
serde_json::from_slice(&reply).context("decoding RepoReleaseReply JSON")
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Convenience wrapper for [`Method::ShutdownPrepCheck`]. Runs
|
||||||
|
/// `safe-shutdown-prep.sh --dry-run` on the connected peer and waits
|
||||||
|
/// for the full report. Never stops anything on the peer.
|
||||||
|
pub async fn call_shutdown_prep_check(
|
||||||
|
conn: &Connection,
|
||||||
|
) -> Result<crate::cluster::shutdown_prep::ShutdownPrepCheckReply> {
|
||||||
|
let req = crate::cluster::shutdown_prep::ShutdownPrepCheckRequest {};
|
||||||
|
let payload = serde_json::to_vec(&req).context("encoding ShutdownPrepCheckRequest")?;
|
||||||
|
let reply = rpc_call(conn, Method::ShutdownPrepCheck, &payload).await?;
|
||||||
|
if reply.len() == 1 {
|
||||||
|
if let Some(code) = decode_error(reply[0]) {
|
||||||
|
bail!("peer replied with error: {}", code.describe());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
serde_json::from_slice(&reply).context("decoding ShutdownPrepCheckReply JSON")
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Convenience wrapper for [`Method::ShutdownPrepExecute`]. Starts the
|
||||||
|
/// real shutdown-prep run on the connected peer (detached — this call
|
||||||
|
/// returns as soon as the peer confirms it started, not when it
|
||||||
|
/// finishes, since the peer's own daemon stops itself partway
|
||||||
|
/// through).
|
||||||
|
pub async fn call_shutdown_prep_execute(
|
||||||
|
conn: &Connection,
|
||||||
|
confirm_node_name: &str,
|
||||||
|
) -> Result<crate::cluster::shutdown_prep::ShutdownPrepExecuteReply> {
|
||||||
|
let req = crate::cluster::shutdown_prep::ShutdownPrepExecuteRequest {
|
||||||
|
confirm_node_name: confirm_node_name.to_string(),
|
||||||
|
};
|
||||||
|
let payload = serde_json::to_vec(&req).context("encoding ShutdownPrepExecuteRequest")?;
|
||||||
|
let reply = rpc_call(conn, Method::ShutdownPrepExecute, &payload).await?;
|
||||||
|
if reply.len() == 1 {
|
||||||
|
if let Some(code) = decode_error(reply[0]) {
|
||||||
|
bail!("peer replied with error: {}", code.describe());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
serde_json::from_slice(&reply).context("decoding ShutdownPrepExecuteReply JSON")
|
||||||
|
}
|
||||||
|
|
||||||
/// Recognise a single-byte reply as one of our error codes. Returns
|
/// Recognise a single-byte reply as one of our error codes. Returns
|
||||||
/// `None` for any other single-byte value (which is a valid reply,
|
/// `None` for any other single-byte value (which is a valid reply,
|
||||||
/// just an unusually short one).
|
/// just an unusually short one).
|
||||||
|
|||||||
@@ -38,20 +38,6 @@ const HOT_METRIC_INTERVAL: Duration = Duration::from_secs(30);
|
|||||||
/// cadence still sees at-worst-60s-stale counters.
|
/// cadence still sees at-worst-60s-stale counters.
|
||||||
const CACHE_METRIC_INTERVAL: Duration = Duration::from_secs(60);
|
const CACHE_METRIC_INTERVAL: Duration = Duration::from_secs(60);
|
||||||
|
|
||||||
/// T3.5: how often the idle prefetch task wakes and checks peers.
|
|
||||||
/// 10-minute cadence keeps cross-node traffic minimal; each tick pulls
|
|
||||||
/// at most `PREFETCH_MAX_PER_TICK` blobs per peer.
|
|
||||||
const PREFETCH_INTERVAL: Duration = Duration::from_secs(600);
|
|
||||||
|
|
||||||
/// T3.5: maximum blobs pulled from any single peer per prefetch tick.
|
|
||||||
/// Caps burst bandwidth to ≈ N × blob_size during each idle window.
|
|
||||||
const PREFETCH_MAX_PER_TICK: usize = 5;
|
|
||||||
|
|
||||||
/// T3.5: hot-tier fill fraction below which the node is considered idle
|
|
||||||
/// enough to speculate-pull blobs from peers. Above this threshold we
|
|
||||||
/// don't want to grow disk usage further.
|
|
||||||
const PREFETCH_IDLE_THRESHOLD: f64 = 0.60;
|
|
||||||
|
|
||||||
/// Live cluster services attached to a running daemon.
|
/// Live cluster services attached to a running daemon.
|
||||||
///
|
///
|
||||||
/// Drop shuts down all background tasks. Ownership is single: the
|
/// Drop shuts down all background tasks. Ownership is single: the
|
||||||
@@ -98,9 +84,6 @@ pub struct ClusterServices {
|
|||||||
/// `cluster.gc_interval_hours` is unset or the daemon has no blob
|
/// `cluster.gc_interval_hours` is unset or the daemon has no blob
|
||||||
/// store (nothing to sweep).
|
/// store (nothing to sweep).
|
||||||
gc_task: Option<JoinHandle<()>>,
|
gc_task: Option<JoinHandle<()>>,
|
||||||
/// T3.5: speculative blob prefetch from peers during idle periods.
|
|
||||||
/// Active only when the daemon has both a blob store and a ref store.
|
|
||||||
prefetch_task: Option<JoinHandle<()>>,
|
|
||||||
}
|
}
|
||||||
|
|
||||||
impl std::fmt::Debug for ClusterServices {
|
impl std::fmt::Debug for ClusterServices {
|
||||||
@@ -323,7 +306,6 @@ impl ClusterServices {
|
|||||||
let gossip = gossip.clone();
|
let gossip = gossip.clone();
|
||||||
gossip.set_cache_metrics(&router.metrics().snapshot()).await;
|
gossip.set_cache_metrics(&router.metrics().snapshot()).await;
|
||||||
Some(tokio::spawn(async move {
|
Some(tokio::spawn(async move {
|
||||||
use crate::cluster::gossip::keys;
|
|
||||||
let mut ticker = tokio::time::interval(CACHE_METRIC_INTERVAL);
|
let mut ticker = tokio::time::interval(CACHE_METRIC_INTERVAL);
|
||||||
// First tick fires immediately (tokio interval default),
|
// First tick fires immediately (tokio interval default),
|
||||||
// covered by the initial publish above — skip it here.
|
// covered by the initial publish above — skip it here.
|
||||||
@@ -332,28 +314,6 @@ impl ClusterServices {
|
|||||||
ticker.tick().await;
|
ticker.tick().await;
|
||||||
let snap = router.metrics().snapshot();
|
let snap = router.metrics().snapshot();
|
||||||
gossip.set_cache_metrics(&snap).await;
|
gossip.set_cache_metrics(&snap).await;
|
||||||
// T3.3/T3.6: publish Lifeguard health score so peers
|
|
||||||
// can observe our stress level via gossip, without RPC.
|
|
||||||
// Simplified: hot-tier pressure + cpu load only (no
|
|
||||||
// filesystem or timer checks — those require blocking I/O
|
|
||||||
// that doesn't belong on the gossip tick path).
|
|
||||||
let hot_used = gossip
|
|
||||||
.self_kv(keys::HOT_USED_BYTES)
|
|
||||||
.await
|
|
||||||
.and_then(|s| s.parse::<u64>().ok())
|
|
||||||
.unwrap_or(0);
|
|
||||||
let hot_max = gossip
|
|
||||||
.self_kv(keys::HOT_MAX_BYTES)
|
|
||||||
.await
|
|
||||||
.and_then(|s| s.parse::<u64>().ok())
|
|
||||||
.unwrap_or(1);
|
|
||||||
let hot_pct =
|
|
||||||
if hot_max > 0 { hot_used as f64 / hot_max as f64 } else { 0.0 };
|
|
||||||
let cpu_load = crate::cluster::rpc::read_cpu_load_1m();
|
|
||||||
let score: u8 =
|
|
||||||
(if hot_pct > 0.90 { 1u8 } else { 0 })
|
|
||||||
.saturating_add(if cpu_load.map(|l| l > 4.0).unwrap_or(false) { 1 } else { 0 });
|
|
||||||
gossip.set_health_score(score).await;
|
|
||||||
}
|
}
|
||||||
}))
|
}))
|
||||||
} else {
|
} else {
|
||||||
@@ -389,8 +349,6 @@ impl ClusterServices {
|
|||||||
(Some(store), Some(hours)) if hours > 0 => {
|
(Some(store), Some(hours)) if hours > 0 => {
|
||||||
let store = store.clone();
|
let store = store.clone();
|
||||||
let tag_store_for_gc = tag_store.clone();
|
let tag_store_for_gc = tag_store.clone();
|
||||||
let ref_store_for_gc = ref_store.clone();
|
|
||||||
let router_for_gc = router.clone();
|
|
||||||
// Phase 7d follow-on: snapshot store is under the
|
// Phase 7d follow-on: snapshot store is under the
|
||||||
// same root as the blob store. Open once here so the
|
// same root as the blob store. Open once here so the
|
||||||
// ticker doesn't pay the fs setup cost every tick.
|
// ticker doesn't pay the fs setup cost every tick.
|
||||||
@@ -408,45 +366,15 @@ impl ClusterServices {
|
|||||||
ticker.tick().await;
|
ticker.tick().await;
|
||||||
loop {
|
loop {
|
||||||
ticker.tick().await;
|
ticker.tick().await;
|
||||||
let now_unix = std::time::SystemTime::now()
|
|
||||||
.duration_since(std::time::UNIX_EPOCH)
|
|
||||||
.map(|d| d.as_secs())
|
|
||||||
.unwrap_or(0);
|
|
||||||
match store.gc_orphan_chunks().await {
|
match store.gc_orphan_chunks().await {
|
||||||
Ok(r) => {
|
Ok(r) => tracing::info!(
|
||||||
tracing::info!(
|
|
||||||
chunks_scanned = r.chunks_scanned,
|
chunks_scanned = r.chunks_scanned,
|
||||||
chunks_removed = r.chunks_removed,
|
chunks_removed = r.chunks_removed,
|
||||||
bytes_reclaimed = r.bytes_reclaimed,
|
bytes_reclaimed = r.bytes_reclaimed,
|
||||||
"auto-GC swept orphan chunks"
|
"auto-GC swept orphan chunks"
|
||||||
);
|
),
|
||||||
if let Some(router) = &router_for_gc {
|
|
||||||
router.push_maintenance_event(
|
|
||||||
crate::cluster::rpc::MaintenanceEvent {
|
|
||||||
kind: "gc_orphan".into(),
|
|
||||||
unix_ts: now_unix,
|
|
||||||
chunks_scanned: r.chunks_scanned,
|
|
||||||
chunks_removed: r.chunks_removed,
|
|
||||||
bytes_reclaimed: r.bytes_reclaimed,
|
|
||||||
error: None,
|
|
||||||
},
|
|
||||||
);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
Err(e) => {
|
Err(e) => {
|
||||||
tracing::warn!(error = %e, "auto-GC failed; will retry next tick");
|
tracing::warn!(error = %e, "auto-GC failed; will retry next tick")
|
||||||
if let Some(router) = &router_for_gc {
|
|
||||||
router.push_maintenance_event(
|
|
||||||
crate::cluster::rpc::MaintenanceEvent {
|
|
||||||
kind: "gc_orphan".into(),
|
|
||||||
unix_ts: now_unix,
|
|
||||||
chunks_scanned: 0,
|
|
||||||
chunks_removed: 0,
|
|
||||||
bytes_reclaimed: 0,
|
|
||||||
error: Some(e.to_string()),
|
|
||||||
},
|
|
||||||
);
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
// Field finding 2026-07-12: if configured with a
|
// Field finding 2026-07-12: if configured with a
|
||||||
@@ -495,186 +423,23 @@ impl ClusterServices {
|
|||||||
pinned.extend(snaps);
|
pinned.extend(snaps);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
// T3.4: build blob→hit_count map for
|
|
||||||
// pollution-score eviction (replaces pure LRU).
|
|
||||||
let hit_map = build_blob_hit_map(
|
|
||||||
&router_for_gc,
|
|
||||||
&ref_store_for_gc,
|
|
||||||
).await;
|
|
||||||
let hit_count = hit_map.values().sum::<u64>();
|
|
||||||
let eviction_now = std::time::SystemTime::now()
|
|
||||||
.duration_since(std::time::UNIX_EPOCH)
|
|
||||||
.map(|d| d.as_secs())
|
|
||||||
.unwrap_or(0);
|
|
||||||
match store
|
match store
|
||||||
.evict_to_size_cap_with_scores(cap, &pinned, &hit_map)
|
.evict_to_size_cap_with_pins(cap, &pinned)
|
||||||
.await
|
.await
|
||||||
{
|
{
|
||||||
Ok(r) if r.chunks_removed > 0 => {
|
Ok(r) if r.chunks_removed > 0 => tracing::info!(
|
||||||
tracing::info!(
|
|
||||||
chunks_removed = r.chunks_removed,
|
chunks_removed = r.chunks_removed,
|
||||||
bytes_reclaimed = r.bytes_reclaimed,
|
bytes_reclaimed = r.bytes_reclaimed,
|
||||||
max_gb = gb,
|
max_gb = gb,
|
||||||
pinned_blobs = pinned.len(),
|
pinned_blobs = pinned.len(),
|
||||||
snapshot_pins = snapshot_pin_count,
|
snapshot_pins = snapshot_pin_count,
|
||||||
scored_blobs = hit_count,
|
"auto-GC evicted LRU blobs to hit size cap"
|
||||||
"auto-GC evicted pollution-scored blobs to hit size cap"
|
),
|
||||||
);
|
|
||||||
if let Some(router) = &router_for_gc {
|
|
||||||
router.push_maintenance_event(
|
|
||||||
crate::cluster::rpc::MaintenanceEvent {
|
|
||||||
kind: "gc_eviction_scored".into(),
|
|
||||||
unix_ts: eviction_now,
|
|
||||||
chunks_scanned: 0,
|
|
||||||
chunks_removed: r.chunks_removed,
|
|
||||||
bytes_reclaimed: r.bytes_reclaimed,
|
|
||||||
error: None,
|
|
||||||
},
|
|
||||||
);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
Ok(_) => {} // under cap already; keep quiet
|
Ok(_) => {} // under cap already; keep quiet
|
||||||
Err(e) => {
|
Err(e) => tracing::warn!(
|
||||||
tracing::warn!(error = %e, "auto-GC eviction failed; will retry next tick");
|
|
||||||
if let Some(router) = &router_for_gc {
|
|
||||||
router.push_maintenance_event(
|
|
||||||
crate::cluster::rpc::MaintenanceEvent {
|
|
||||||
kind: "gc_eviction_scored".into(),
|
|
||||||
unix_ts: eviction_now,
|
|
||||||
chunks_scanned: 0,
|
|
||||||
chunks_removed: 0,
|
|
||||||
bytes_reclaimed: 0,
|
|
||||||
error: Some(e.to_string()),
|
|
||||||
},
|
|
||||||
);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}))
|
|
||||||
}
|
|
||||||
_ => None,
|
|
||||||
};
|
|
||||||
|
|
||||||
// T3.5: speculative blob prefetch during idle periods. Active only
|
|
||||||
// when we have an outbound RPC client (blob store + ref store +
|
|
||||||
// outbound_client all wired). The task wakes every PREFETCH_INTERVAL,
|
|
||||||
// checks if the node is idle (hot-tier < PREFETCH_IDLE_THRESHOLD),
|
|
||||||
// and pulls up to PREFETCH_MAX_PER_TICK blobs from each live peer.
|
|
||||||
let prefetch_task = match (&blob_store, &ref_store, &router) {
|
|
||||||
(Some(blob_store), Some(ref_store), Some(router))
|
|
||||||
if router.outbound_client().is_some() =>
|
|
||||||
{
|
|
||||||
let blob_store = blob_store.clone();
|
|
||||||
let ref_store = ref_store.clone();
|
|
||||||
let gossip = gossip.clone();
|
|
||||||
let router = router.clone();
|
|
||||||
Some(tokio::spawn(async move {
|
|
||||||
use crate::cluster::gossip::keys;
|
|
||||||
use crate::cluster::rpc::{call_dashboard_storage, pull_blob_locally};
|
|
||||||
use crate::cluster::blob::BlobId;
|
|
||||||
let mut ticker = tokio::time::interval(PREFETCH_INTERVAL);
|
|
||||||
ticker.tick().await; // skip immediate first tick
|
|
||||||
loop {
|
|
||||||
ticker.tick().await;
|
|
||||||
// Idle check: hot-tier fill below threshold.
|
|
||||||
let hot_used = gossip
|
|
||||||
.self_kv(keys::HOT_USED_BYTES)
|
|
||||||
.await
|
|
||||||
.and_then(|s| s.parse::<u64>().ok())
|
|
||||||
.unwrap_or(0);
|
|
||||||
let hot_max = gossip
|
|
||||||
.self_kv(keys::HOT_MAX_BYTES)
|
|
||||||
.await
|
|
||||||
.and_then(|s| s.parse::<u64>().ok())
|
|
||||||
.unwrap_or(1);
|
|
||||||
let fill = if hot_max > 0 {
|
|
||||||
hot_used as f64 / hot_max as f64
|
|
||||||
} else {
|
|
||||||
1.0
|
|
||||||
};
|
|
||||||
if fill >= PREFETCH_IDLE_THRESHOLD {
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
// Prefetch from each live peer.
|
|
||||||
let peers = gossip
|
|
||||||
.peers()
|
|
||||||
.await
|
|
||||||
.into_iter()
|
|
||||||
.filter(|p| p.alive && p.rpc_lan.or(p.rpc_tailscale).is_some())
|
|
||||||
.collect::<Vec<_>>();
|
|
||||||
for peer in peers {
|
|
||||||
let Some(client) = router.outbound_client() else { break };
|
|
||||||
let addr = match peer.rpc_lan.or(peer.rpc_tailscale) {
|
|
||||||
Some(a) => a,
|
|
||||||
None => continue,
|
|
||||||
};
|
|
||||||
let conn = match tokio::time::timeout(
|
|
||||||
Duration::from_secs(5),
|
|
||||||
client.connect(addr, &peer.name),
|
|
||||||
)
|
|
||||||
.await
|
|
||||||
{
|
|
||||||
Ok(Ok(c)) => c,
|
|
||||||
_ => continue,
|
|
||||||
};
|
|
||||||
let storage = match call_dashboard_storage(&conn).await {
|
|
||||||
Ok(s) => s,
|
|
||||||
Err(e) => {
|
|
||||||
tracing::debug!(
|
|
||||||
peer = %peer.name,
|
|
||||||
error = %e,
|
error = %e,
|
||||||
"prefetch: DashboardStorage failed"
|
"auto-GC eviction failed; will retry next tick"
|
||||||
);
|
),
|
||||||
continue;
|
|
||||||
}
|
|
||||||
};
|
|
||||||
let mut pulled = 0usize;
|
|
||||||
for r in &storage.refs_sample {
|
|
||||||
if pulled >= PREFETCH_MAX_PER_TICK {
|
|
||||||
break;
|
|
||||||
}
|
|
||||||
let fp_id = match BlobId::from_hex(&r.fingerprint_hex) {
|
|
||||||
Ok(id) => id,
|
|
||||||
Err(_) => continue,
|
|
||||||
};
|
|
||||||
let fp: [u8; 32] = *fp_id.as_bytes();
|
|
||||||
// Skip refs we already have locally.
|
|
||||||
if ref_store.contains(&fp).await.unwrap_or(true) {
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
let blob_id = match BlobId::from_hex(&r.blob_id_hex) {
|
|
||||||
Ok(id) => id,
|
|
||||||
Err(_) => continue,
|
|
||||||
};
|
|
||||||
// Pull chunks + manifest from peer.
|
|
||||||
match pull_blob_locally(&conn, &blob_store, &blob_id).await {
|
|
||||||
Ok(()) => {
|
|
||||||
let _ = ref_store.put(&fp, blob_id.as_bytes()).await;
|
|
||||||
pulled += 1;
|
|
||||||
tracing::debug!(
|
|
||||||
peer = %peer.name,
|
|
||||||
blob = %r.blob_id_hex,
|
|
||||||
"prefetch: pulled blob"
|
|
||||||
);
|
|
||||||
}
|
|
||||||
Err(e) => {
|
|
||||||
tracing::debug!(
|
|
||||||
peer = %peer.name,
|
|
||||||
blob = %r.blob_id_hex,
|
|
||||||
error = %e,
|
|
||||||
"prefetch: pull failed"
|
|
||||||
);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if pulled > 0 {
|
|
||||||
tracing::info!(
|
|
||||||
peer = %peer.name,
|
|
||||||
count = pulled,
|
|
||||||
"prefetch: idle-pulled blobs from peer"
|
|
||||||
);
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -694,7 +459,6 @@ impl ClusterServices {
|
|||||||
cache_metric_task,
|
cache_metric_task,
|
||||||
prom_server,
|
prom_server,
|
||||||
gc_task,
|
gc_task,
|
||||||
prefetch_task,
|
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -737,47 +501,7 @@ impl ClusterServices {
|
|||||||
if let Some(task) = self.gc_task {
|
if let Some(task) = self.gc_task {
|
||||||
task.abort();
|
task.abort();
|
||||||
}
|
}
|
||||||
if let Some(task) = self.prefetch_task {
|
|
||||||
task.abort();
|
|
||||||
}
|
}
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/// T3.4: build a blob_id → total_hit_count map from the access log + ref store.
|
|
||||||
///
|
|
||||||
/// The AccessLog tracks fingerprint→hits; the ref store maps fingerprint→blob_id.
|
|
||||||
/// Joining them gives blob_id→hits which the pollution-score eviction needs.
|
|
||||||
/// Returns an empty map when either input is unavailable (GC falls back to mtime ordering).
|
|
||||||
async fn build_blob_hit_map(
|
|
||||||
router: &Option<Arc<crate::cluster::rpc::RpcRouter>>,
|
|
||||||
ref_store: &Option<Arc<crate::cluster::refs::RefStore>>,
|
|
||||||
) -> std::collections::HashMap<crate::cluster::blob::BlobId, u64> {
|
|
||||||
use crate::cluster::blob::BlobId;
|
|
||||||
let mut map = std::collections::HashMap::new();
|
|
||||||
let (Some(router), Some(rs)) = (router, ref_store) else {
|
|
||||||
return map;
|
|
||||||
};
|
|
||||||
// fingerprint_bytes → hit_count from the in-memory access log
|
|
||||||
let fp_hits: std::collections::HashMap<[u8; 32], u64> =
|
|
||||||
match router.access_log().lock() {
|
|
||||||
Ok(log) => log.top(usize::MAX).into_iter().map(|(fp, rec)| (fp, rec.count)).collect(),
|
|
||||||
Err(_) => return map,
|
|
||||||
};
|
|
||||||
if fp_hits.is_empty() {
|
|
||||||
return map;
|
|
||||||
}
|
|
||||||
// fingerprint → blob_id from the ref store
|
|
||||||
let pairs = match rs.list().await {
|
|
||||||
Ok(p) => p,
|
|
||||||
Err(_) => return map,
|
|
||||||
};
|
|
||||||
for (fp_key, blob_id_bytes) in pairs {
|
|
||||||
if let Some(&hits) = fp_hits.get(&fp_key) {
|
|
||||||
let bid = BlobId::from_bytes(blob_id_bytes);
|
|
||||||
*map.entry(bid).or_insert(0u64) += hits;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
map
|
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Loop accepting incoming QUIC connections and dispatching each to a
|
/// Loop accepting incoming QUIC connections and dispatching each to a
|
||||||
|
|||||||
@@ -0,0 +1,166 @@
|
|||||||
|
//! Peer-side wiring for `deploy/scripts/safe-shutdown-prep.sh`,
|
||||||
|
//! surfaced through the RPC layer so the dashboard-v2 aggregator can
|
||||||
|
//! offer a "prepare this node for shutdown" action.
|
||||||
|
//!
|
||||||
|
//! Split into two RPCs deliberately:
|
||||||
|
//!
|
||||||
|
//! - [`Method::ShutdownPrepCheck`] runs the script's `--dry-run` mode
|
||||||
|
//! and waits for it to finish. Dry-run never stops this node's own
|
||||||
|
//! daemon, so the RPC connection survives to deliver the full
|
||||||
|
//! report — this is the part a browser can meaningfully show.
|
||||||
|
//! - [`Method::ShutdownPrepExecute`] runs the real script, which (by
|
||||||
|
//! design) stops `claw-store.service` — i.e. the very process
|
||||||
|
//! handling this RPC. There is no way to stream a live result past
|
||||||
|
//! that point, so this RPC detaches the script into its own
|
||||||
|
//! transient systemd scope (outside this daemon's service cgroup,
|
||||||
|
//! so `systemctl stop claw-store.service` doesn't take the script
|
||||||
|
//! down with it) and returns immediately. The full report lands in
|
||||||
|
//! `SHUTDOWN_PREP_LOG` for whoever is physically at the machine (or
|
||||||
|
//! over SSH) to read once the node has gone dark.
|
||||||
|
//!
|
||||||
|
//! [`Method::ShutdownPrepCheck`]: crate::cluster::rpc::Method::ShutdownPrepCheck
|
||||||
|
//! [`Method::ShutdownPrepExecute`]: crate::cluster::rpc::Method::ShutdownPrepExecute
|
||||||
|
|
||||||
|
use anyhow::{bail, Context, Result};
|
||||||
|
use serde::{Deserialize, Serialize};
|
||||||
|
use std::path::PathBuf;
|
||||||
|
use std::time::Duration;
|
||||||
|
use tokio::process::Command;
|
||||||
|
use tokio::time::timeout;
|
||||||
|
|
||||||
|
/// Dry-run does a real snapshot + replicate, which can legitimately
|
||||||
|
/// take a while on a large delta. Generous but bounded so a stuck
|
||||||
|
/// peer connection doesn't hang the RPC forever.
|
||||||
|
const CHECK_TIMEOUT: Duration = Duration::from_secs(300);
|
||||||
|
|
||||||
|
/// Where the real run's output lands once this node's daemon (and
|
||||||
|
/// therefore this RPC connection) is gone.
|
||||||
|
pub const SHUTDOWN_PREP_LOG: &str = "/var/lib/claw-store/shutdown-prep.log";
|
||||||
|
|
||||||
|
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
|
||||||
|
pub struct ShutdownPrepCheckRequest {}
|
||||||
|
|
||||||
|
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
|
||||||
|
pub struct ShutdownPrepCheckReply {
|
||||||
|
/// True iff the script exited 0 (every guard passed, "SAFE TO
|
||||||
|
/// POWER OFF" printed for the checked steps).
|
||||||
|
pub ready: bool,
|
||||||
|
/// Full combined stdout+stderr from `--dry-run`.
|
||||||
|
pub output: String,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
|
||||||
|
pub struct ShutdownPrepExecuteRequest {
|
||||||
|
/// Defense in depth beyond RPC targeting: the caller must name
|
||||||
|
/// the exact node it thinks it's shutting down. Checked against
|
||||||
|
/// this node's own configured name before anything runs.
|
||||||
|
pub confirm_node_name: String,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
|
||||||
|
pub struct ShutdownPrepExecuteReply {
|
||||||
|
pub started: bool,
|
||||||
|
pub message: String,
|
||||||
|
pub log_path: String,
|
||||||
|
}
|
||||||
|
|
||||||
|
fn script_path() -> PathBuf {
|
||||||
|
if let Ok(p) = std::env::var("CLAWSTOR_SHUTDOWN_SCRIPT") {
|
||||||
|
if !p.is_empty() {
|
||||||
|
return PathBuf::from(p);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if let Ok(home) = std::env::var("HOME") {
|
||||||
|
if !home.is_empty() {
|
||||||
|
return PathBuf::from(home)
|
||||||
|
.join("clawstor-deploy/scripts/safe-shutdown-prep.sh");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
PathBuf::from("/usr/local/share/claw-store/safe-shutdown-prep.sh")
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Run `safe-shutdown-prep.sh --dry-run` to completion and report the
|
||||||
|
/// full output. Never stops anything on this node — safe to call any
|
||||||
|
/// time, including repeatedly.
|
||||||
|
pub async fn check() -> Result<ShutdownPrepCheckReply> {
|
||||||
|
let script = script_path();
|
||||||
|
if !script.exists() {
|
||||||
|
bail!("shutdown-prep script not found at {}", script.display());
|
||||||
|
}
|
||||||
|
let run = Command::new("bash")
|
||||||
|
.arg(&script)
|
||||||
|
.arg("--dry-run")
|
||||||
|
.output();
|
||||||
|
let output = timeout(CHECK_TIMEOUT, run)
|
||||||
|
.await
|
||||||
|
.context("shutdown-prep --dry-run timed out")?
|
||||||
|
.context("spawning shutdown-prep --dry-run")?;
|
||||||
|
let mut combined = String::from_utf8_lossy(&output.stdout).into_owned();
|
||||||
|
combined.push_str(&String::from_utf8_lossy(&output.stderr));
|
||||||
|
Ok(ShutdownPrepCheckReply {
|
||||||
|
ready: output.status.success(),
|
||||||
|
output: combined,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Kick off the real (non-dry-run) script in a transient systemd
|
||||||
|
/// scope detached from this daemon's own service cgroup, then return
|
||||||
|
/// immediately without waiting for it. The script's own step 6 stops
|
||||||
|
/// `claw-store.service` — waiting for it to exit here would mean
|
||||||
|
/// waiting for our own process to be killed.
|
||||||
|
pub async fn execute(local_node_name: &str, req: &ShutdownPrepExecuteRequest) -> Result<ShutdownPrepExecuteReply> {
|
||||||
|
if req.confirm_node_name != local_node_name {
|
||||||
|
bail!(
|
||||||
|
"confirm_node_name '{}' does not match this node ('{}') — refusing",
|
||||||
|
req.confirm_node_name,
|
||||||
|
local_node_name
|
||||||
|
);
|
||||||
|
}
|
||||||
|
let script = script_path();
|
||||||
|
if !script.exists() {
|
||||||
|
bail!("shutdown-prep script not found at {}", script.display());
|
||||||
|
}
|
||||||
|
let unit = format!(
|
||||||
|
"clawstor-shutdown-prep-{}",
|
||||||
|
std::time::SystemTime::now()
|
||||||
|
.duration_since(std::time::UNIX_EPOCH)
|
||||||
|
.map(|d| d.as_secs())
|
||||||
|
.unwrap_or(0)
|
||||||
|
);
|
||||||
|
// `--user --scope` places this under the user session's cgroup
|
||||||
|
// tree (/user.slice/...), a sibling of — not a descendant of —
|
||||||
|
// /system.slice/claw-store.service. `systemctl stop
|
||||||
|
// claw-store.service` only tears down its own cgroup, so this
|
||||||
|
// keeps running (and completes step 6, which stops that very
|
||||||
|
// service) unaffected.
|
||||||
|
// Deliberately no --force: the real run re-checks active builds
|
||||||
|
// and the sync queue itself, even though `check()` may have run
|
||||||
|
// moments ago — state can change between the two clicks, and
|
||||||
|
// re-validating is cheap.
|
||||||
|
let cmd = format!(
|
||||||
|
"{} >> {} 2>&1",
|
||||||
|
script.display(),
|
||||||
|
SHUTDOWN_PREP_LOG
|
||||||
|
);
|
||||||
|
let spawn = Command::new("systemd-run")
|
||||||
|
.arg("--user")
|
||||||
|
.arg("--scope")
|
||||||
|
.arg("--collect")
|
||||||
|
.arg(format!("--unit={unit}"))
|
||||||
|
.arg("bash")
|
||||||
|
.arg("-c")
|
||||||
|
.arg(&cmd)
|
||||||
|
.spawn();
|
||||||
|
match spawn {
|
||||||
|
Ok(_child) => Ok(ShutdownPrepExecuteReply {
|
||||||
|
started: true,
|
||||||
|
message: format!(
|
||||||
|
"shutdown-prep started on {local_node_name} as transient unit {unit}. \
|
||||||
|
This node's daemon (and dashboard) will go offline as part of the \
|
||||||
|
process — that is expected. Full output: {SHUTDOWN_PREP_LOG}."
|
||||||
|
),
|
||||||
|
log_path: SHUTDOWN_PREP_LOG.to_string(),
|
||||||
|
}),
|
||||||
|
Err(e) => Err(e).context("spawning systemd-run for shutdown-prep"),
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -27,12 +27,26 @@ pub struct HotConfig {
|
|||||||
#[derive(Debug, Clone, Deserialize, Serialize)]
|
#[derive(Debug, Clone, Deserialize, Serialize)]
|
||||||
pub struct WarmConfig {
|
pub struct WarmConfig {
|
||||||
pub projects_path: PathBuf,
|
pub projects_path: PathBuf,
|
||||||
|
/// Dataset name for `zfs`/`zpool` operations against the warm
|
||||||
|
/// tier. The literal value `"none"` means this node's warm tier
|
||||||
|
/// is a plain directory, not ZFS-backed (e.g. a build node with
|
||||||
|
/// no ZFS pool) — snapshot/replicate become no-ops instead of
|
||||||
|
/// erroring on a missing `zfs`/`zpool` binary. See
|
||||||
|
/// [`WarmConfig::zfs_enabled`].
|
||||||
pub zfs_dataset: String,
|
pub zfs_dataset: String,
|
||||||
pub snapshot_retain_hours: u64,
|
pub snapshot_retain_hours: u64,
|
||||||
pub snapshot_retain_days: u64,
|
pub snapshot_retain_days: u64,
|
||||||
pub snapshot_retain_weeks: u64,
|
pub snapshot_retain_weeks: u64,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
impl WarmConfig {
|
||||||
|
/// `false` when `zfs_dataset = "none"` — this node's warm tier
|
||||||
|
/// has no ZFS pool underneath it.
|
||||||
|
pub fn zfs_enabled(&self) -> bool {
|
||||||
|
self.zfs_dataset != "none"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
#[derive(Debug, Clone, Deserialize, Serialize)]
|
#[derive(Debug, Clone, Deserialize, Serialize)]
|
||||||
pub struct ColdConfig {
|
pub struct ColdConfig {
|
||||||
pub archive_path: PathBuf,
|
pub archive_path: PathBuf,
|
||||||
|
|||||||
+17
-19
@@ -7,7 +7,7 @@ use crate::manifest::Manifest;
|
|||||||
use crate::snapshot;
|
use crate::snapshot;
|
||||||
use crate::sync::{SyncQueue, drain_sync_queue};
|
use crate::sync::{SyncQueue, drain_sync_queue};
|
||||||
use crate::zfs::SystemZfs;
|
use crate::zfs::SystemZfs;
|
||||||
use anyhow::Result;
|
use anyhow::{Context, Result};
|
||||||
use chrono::Utc;
|
use chrono::Utc;
|
||||||
use sysinfo::{ProcessRefreshKind, RefreshKind, System};
|
use sysinfo::{ProcessRefreshKind, RefreshKind, System};
|
||||||
use tokio::time::{interval, Duration};
|
use tokio::time::{interval, Duration};
|
||||||
@@ -36,7 +36,14 @@ pub async fn run(cfg: Config, mut manifest: Manifest) -> Result<()> {
|
|||||||
let hot_dir = cfg.hot.path.clone();
|
let hot_dir = cfg.hot.path.clone();
|
||||||
let hot_max_bytes = cfg.hot.max_gb.saturating_mul(1024 * 1024 * 1024);
|
let hot_max_bytes = cfg.hot.max_gb.saturating_mul(1024 * 1024 * 1024);
|
||||||
let blob_root = cluster_cfg.blob_store_root.clone();
|
let blob_root = cluster_cfg.blob_store_root.clone();
|
||||||
match ClusterServices::start(
|
// Fail fast rather than degrade silently: a bind failure here is
|
||||||
|
// almost always a boot-time race against DHCP/network-online
|
||||||
|
// (the bind address isn't assigned to the interface yet). The
|
||||||
|
// systemd unit has `Restart=on-failure`; exiting lets it retry
|
||||||
|
// a few seconds later once the network is actually up, instead
|
||||||
|
// of leaving the daemon running indefinitely with no gossip,
|
||||||
|
// RPC, or Prometheus endpoint and no visible failure state.
|
||||||
|
let svc = ClusterServices::start(
|
||||||
cluster_cfg,
|
cluster_cfg,
|
||||||
cfg.node.name.clone(),
|
cfg.node.name.clone(),
|
||||||
hot_dir,
|
hot_dir,
|
||||||
@@ -44,8 +51,7 @@ pub async fn run(cfg: Config, mut manifest: Manifest) -> Result<()> {
|
|||||||
blob_root,
|
blob_root,
|
||||||
)
|
)
|
||||||
.await
|
.await
|
||||||
{
|
.context("starting cluster services")?;
|
||||||
Ok(svc) => {
|
|
||||||
tracing::info!(
|
tracing::info!(
|
||||||
rpc_enabled = svc.rpc_enabled(),
|
rpc_enabled = svc.rpc_enabled(),
|
||||||
blob_store_enabled = svc.blob_store_enabled(),
|
blob_store_enabled = svc.blob_store_enabled(),
|
||||||
@@ -54,12 +60,6 @@ pub async fn run(cfg: Config, mut manifest: Manifest) -> Result<()> {
|
|||||||
);
|
);
|
||||||
Some(svc)
|
Some(svc)
|
||||||
}
|
}
|
||||||
Err(e) => {
|
|
||||||
tracing::error!(error = %e, "cluster services failed to start; continuing without cluster");
|
|
||||||
None
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
None => {
|
None => {
|
||||||
tracing::info!("no [cluster] section in config; running standalone");
|
tracing::info!("no [cluster] section in config; running standalone");
|
||||||
None
|
None
|
||||||
@@ -188,14 +188,13 @@ pub async fn run(cfg: Config, mut manifest: Manifest) -> Result<()> {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
_ = snap_tick.tick() => {
|
_ = snap_tick.tick() => {
|
||||||
let dataset = &cfg.warm.zfs_dataset;
|
if !cfg.warm.zfs_enabled() {
|
||||||
if dataset.is_empty() || dataset == "none" {
|
tracing::debug!("skipping snapshot tick — zfs_dataset = \"none\" on this node");
|
||||||
tracing::debug!("zfs_dataset=none — skipping snapshot");
|
|
||||||
} else {
|
} else {
|
||||||
let ts = chrono::Utc::now().format("%Y-%m-%d-%H%M").to_string();
|
let ts = chrono::Utc::now().format("%Y-%m-%d-%H%M").to_string();
|
||||||
tracing::info!("taking snapshot {}", ts);
|
tracing::info!("taking snapshot {}", ts);
|
||||||
if let Err(e) = snapshot::run_snapshot_cycle(
|
if let Err(e) = snapshot::run_snapshot_cycle(
|
||||||
&zfs, dataset, &ts,
|
&zfs, &cfg.warm.zfs_dataset, &ts,
|
||||||
cfg.warm.snapshot_retain_hours as usize,
|
cfg.warm.snapshot_retain_hours as usize,
|
||||||
cfg.warm.snapshot_retain_days as usize,
|
cfg.warm.snapshot_retain_days as usize,
|
||||||
cfg.warm.snapshot_retain_weeks as usize,
|
cfg.warm.snapshot_retain_weeks as usize,
|
||||||
@@ -205,15 +204,15 @@ pub async fn run(cfg: Config, mut manifest: Manifest) -> Result<()> {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
_ = repl_tick.tick() => {
|
_ = repl_tick.tick() => {
|
||||||
let dataset = &cfg.warm.zfs_dataset;
|
if !cfg.warm.zfs_enabled() {
|
||||||
if !dataset.is_empty() && dataset != "none" {
|
tracing::debug!("skipping replication tick — zfs_dataset = \"none\" on this node");
|
||||||
if let Some(rep) = &cfg.replication {
|
} else if let Some(rep) = &cfg.replication {
|
||||||
if let (Some(host), Some(user), Some(dest)) = (
|
if let (Some(host), Some(user), Some(dest)) = (
|
||||||
&rep.send_to_host, &rep.send_to_user, &rep.cold_dataset_on_peer
|
&rep.send_to_host, &rep.send_to_user, &rep.cold_dataset_on_peer
|
||||||
) {
|
) {
|
||||||
tracing::info!("replicating warm → cold on {}", host);
|
tracing::info!("replicating warm → cold on {}", host);
|
||||||
if let Err(e) = snapshot::replicate_to_cold(
|
if let Err(e) = snapshot::replicate_to_cold(
|
||||||
&zfs, dataset, user, host, dest
|
&zfs, &cfg.warm.zfs_dataset, user, host, dest
|
||||||
) {
|
) {
|
||||||
tracing::error!("replication failed: {:#}", e);
|
tracing::error!("replication failed: {:#}", e);
|
||||||
}
|
}
|
||||||
@@ -222,7 +221,6 @@ pub async fn run(cfg: Config, mut manifest: Manifest) -> Result<()> {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
|
||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
+10
-27
@@ -55,42 +55,25 @@ pub fn gc_stale_targets(manifest: &Manifest, stale_hours: u64) -> Result<Vec<Str
|
|||||||
Ok(evicted)
|
Ok(evicted)
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Eviction priority score: larger and older = higher = evict first.
|
|
||||||
/// log-scaled so a 10GB project doesn't completely dominate a 1GB project
|
|
||||||
/// and a week-old entry doesn't mask a 3-day-old one of similar value.
|
|
||||||
fn eviction_score(size_bytes: u64, secs_since_active: u64) -> f64 {
|
|
||||||
let size_factor = (size_bytes as f64 + 1.0).ln();
|
|
||||||
let age_factor = (secs_since_active as f64 + 1.0).ln();
|
|
||||||
size_factor * age_factor
|
|
||||||
}
|
|
||||||
|
|
||||||
pub fn gc_by_space(manifest: &mut Manifest, max_gb: f64) -> Result<Vec<String>> {
|
pub fn gc_by_space(manifest: &mut Manifest, max_gb: f64) -> Result<Vec<String>> {
|
||||||
let now = Utc::now();
|
|
||||||
let mut evicted = Vec::new();
|
let mut evicted = Vec::new();
|
||||||
loop {
|
loop {
|
||||||
let used = total_used_gb(manifest)?;
|
let used = total_used_gb(manifest)?;
|
||||||
if used <= max_gb {
|
if used <= max_gb {
|
||||||
break;
|
break;
|
||||||
}
|
}
|
||||||
// Score-based eviction: prefer removing large, stale projects over
|
// LRU eviction skips pinned projects — they may NEVER be evicted
|
||||||
// arbitrarily picking the oldest-active one. Pinned projects are
|
// for space pressure. The trade-off: if every non-pinned project
|
||||||
// never evicted regardless of score — operator intent is absolute.
|
// is gone and we're still over `max_gb`, we stop and log; better
|
||||||
let best = manifest
|
// to over-allocate hot than to violate operator intent.
|
||||||
|
let lru_name = manifest
|
||||||
.projects
|
.projects
|
||||||
.iter()
|
.iter()
|
||||||
.filter(|p| !p.pinned && p.hot_target_path.exists())
|
.filter(|p| !p.pinned)
|
||||||
.map(|p| {
|
.filter(|p| p.hot_target_path.exists())
|
||||||
let size = target_size_bytes(&p.hot_target_path).unwrap_or(0);
|
.min_by_key(|p| p.last_active)
|
||||||
let age_secs = p
|
.map(|p| p.name.clone());
|
||||||
.last_active
|
match lru_name {
|
||||||
.map(|t| (now - t).num_seconds().max(0) as u64)
|
|
||||||
.unwrap_or(0);
|
|
||||||
(p.name.clone(), eviction_score(size, age_secs))
|
|
||||||
})
|
|
||||||
.max_by(|(_, a), (_, b)| a.partial_cmp(b).unwrap_or(std::cmp::Ordering::Equal))
|
|
||||||
.map(|(name, _)| name);
|
|
||||||
|
|
||||||
match best {
|
|
||||||
None => {
|
None => {
|
||||||
tracing::warn!(
|
tracing::warn!(
|
||||||
used_gb = used,
|
used_gb = used,
|
||||||
|
|||||||
@@ -1502,6 +1502,10 @@ fn cmd_gc(cfg: &Config, manifest: &mut Manifest) -> Result<()> {
|
|||||||
}
|
}
|
||||||
|
|
||||||
fn cmd_snapshot(cfg: &Config, zfs: &SystemZfs) -> Result<()> {
|
fn cmd_snapshot(cfg: &Config, zfs: &SystemZfs) -> Result<()> {
|
||||||
|
if !cfg.warm.zfs_enabled() {
|
||||||
|
println!("Warm tier is not ZFS-backed on this node (zfs_dataset = \"none\") — nothing to snapshot.");
|
||||||
|
return Ok(());
|
||||||
|
}
|
||||||
let ts = chrono::Utc::now().format("%Y-%m-%d-%H%M").to_string();
|
let ts = chrono::Utc::now().format("%Y-%m-%d-%H%M").to_string();
|
||||||
snapshot::run_snapshot_cycle(
|
snapshot::run_snapshot_cycle(
|
||||||
zfs, &cfg.warm.zfs_dataset, &ts,
|
zfs, &cfg.warm.zfs_dataset, &ts,
|
||||||
@@ -1514,6 +1518,10 @@ fn cmd_snapshot(cfg: &Config, zfs: &SystemZfs) -> Result<()> {
|
|||||||
}
|
}
|
||||||
|
|
||||||
fn cmd_list_snapshots(cfg: &Config, zfs: &SystemZfs, _project: &str) -> Result<()> {
|
fn cmd_list_snapshots(cfg: &Config, zfs: &SystemZfs, _project: &str) -> Result<()> {
|
||||||
|
if !cfg.warm.zfs_enabled() {
|
||||||
|
println!("Warm tier is not ZFS-backed on this node (zfs_dataset = \"none\").");
|
||||||
|
return Ok(());
|
||||||
|
}
|
||||||
let snaps = zfs.list_snapshots(&cfg.warm.zfs_dataset)?;
|
let snaps = zfs.list_snapshots(&cfg.warm.zfs_dataset)?;
|
||||||
if snaps.is_empty() { println!("No snapshots found."); return Ok(()); }
|
if snaps.is_empty() { println!("No snapshots found."); return Ok(()); }
|
||||||
for s in &snaps { println!(" {}", s); }
|
for s in &snaps { println!(" {}", s); }
|
||||||
@@ -1531,6 +1539,10 @@ fn cmd_restore(cfg: &Config, zfs: &SystemZfs, project: &str, snap: &str) -> Resu
|
|||||||
}
|
}
|
||||||
|
|
||||||
fn cmd_replicate(cfg: &Config, zfs: &SystemZfs) -> Result<()> {
|
fn cmd_replicate(cfg: &Config, zfs: &SystemZfs) -> Result<()> {
|
||||||
|
if !cfg.warm.zfs_enabled() {
|
||||||
|
println!("Warm tier is not ZFS-backed on this node (zfs_dataset = \"none\") — nothing to replicate.");
|
||||||
|
return Ok(());
|
||||||
|
}
|
||||||
let rep = cfg.replication.as_ref()
|
let rep = cfg.replication.as_ref()
|
||||||
.context("no replication config — this node does not replicate")?;
|
.context("no replication config — this node does not replicate")?;
|
||||||
let host = rep.send_to_host.as_ref().context("send_to_host not set")?;
|
let host = rep.send_to_host.as_ref().context("send_to_host not set")?;
|
||||||
|
|||||||
+6
-17
@@ -719,23 +719,12 @@ pub fn build_app_with_v2(
|
|||||||
v2_router = v2_router.merge(crate::serve_v2::build(v2s));
|
v2_router = v2_router.merge(crate::serve_v2::build(v2s));
|
||||||
}
|
}
|
||||||
if let Some(dir) = v2_static_dir {
|
if let Some(dir) = v2_static_dir {
|
||||||
// Redirect bare paths (no trailing slash) to their canonical
|
v2_router = v2_router.nest_service(
|
||||||
// slash-terminated form. Without this, browsers treat the path
|
"/v2",
|
||||||
// segment as a file and resolve relative asset paths one level
|
tower_http::services::ServeDir::new(&dir).fallback(
|
||||||
// too high, producing 404s for `./assets/…`.
|
tower_http::services::ServeFile::new(dir.join("index.html")),
|
||||||
let redirect_v2 = get(|| async {
|
),
|
||||||
axum::response::Redirect::permanent("/v2/")
|
);
|
||||||
});
|
|
||||||
let redirect_clawstor = get(|| async {
|
|
||||||
axum::response::Redirect::permanent("/clawstor/")
|
|
||||||
});
|
|
||||||
let spa = tower_http::services::ServeDir::new(&dir)
|
|
||||||
.fallback(tower_http::services::ServeFile::new(dir.join("index.html")));
|
|
||||||
v2_router = v2_router
|
|
||||||
.route("/v2", redirect_v2)
|
|
||||||
.route("/clawstor", redirect_clawstor)
|
|
||||||
.nest_service("/v2/", spa.clone())
|
|
||||||
.nest_service("/clawstor/", spa);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
Router::new()
|
Router::new()
|
||||||
|
|||||||
+128
-471
@@ -13,13 +13,12 @@
|
|||||||
//! Design doc: `docs/dashboard-v2.md`.
|
//! Design doc: `docs/dashboard-v2.md`.
|
||||||
|
|
||||||
use axum::{
|
use axum::{
|
||||||
extract::{Path, Query, State},
|
extract::{Path, State},
|
||||||
http::StatusCode,
|
http::StatusCode,
|
||||||
routing::{get, post},
|
routing::{get, post},
|
||||||
Json, Router,
|
Json, Router,
|
||||||
};
|
};
|
||||||
use serde::{Deserialize, Serialize};
|
use serde::{Deserialize, Serialize};
|
||||||
use std::collections::{HashMap, VecDeque};
|
|
||||||
use std::net::SocketAddr;
|
use std::net::SocketAddr;
|
||||||
use std::sync::Arc;
|
use std::sync::Arc;
|
||||||
use std::time::Duration;
|
use std::time::Duration;
|
||||||
@@ -34,197 +33,6 @@ use crate::cluster::transport::{NodeIdentity, QuicClient};
|
|||||||
use crate::config::{Config, PeerEntry, TokenEntry};
|
use crate::config::{Config, PeerEntry, TokenEntry};
|
||||||
use crate::sessions::{LeasedTag, Session, SessionStore};
|
use crate::sessions::{LeasedTag, Session, SessionStore};
|
||||||
|
|
||||||
// ── metrics history ring buffer + drift-adaptive anomaly detection ──
|
|
||||||
|
|
||||||
const HISTORY_MAX_SAMPLES: usize = 1440; // 24h at 1-min resolution
|
|
||||||
/// Require at least this many samples before reporting an anomaly score.
|
|
||||||
/// Protects against false positives on a freshly-started daemon.
|
|
||||||
const ANOMALY_MIN_SAMPLES: u64 = 10;
|
|
||||||
/// Sum-of-z² above this value triggers a "warn" level anomaly.
|
|
||||||
/// Interpretation: average ~1.5σ deviation across 4 tracked metrics.
|
|
||||||
const ANOMALY_WARN_THRESHOLD: f64 = 9.0;
|
|
||||||
/// Sum-of-z² above this value triggers "alert".
|
|
||||||
const ANOMALY_ALERT_THRESHOLD: f64 = 16.0;
|
|
||||||
|
|
||||||
/// Welford's online algorithm for a running mean + variance.
|
|
||||||
/// Numerically stable, O(1) per update.
|
|
||||||
#[derive(Default, Clone)]
|
|
||||||
struct Welford {
|
|
||||||
n: u64,
|
|
||||||
mean: f64,
|
|
||||||
m2: f64,
|
|
||||||
}
|
|
||||||
|
|
||||||
impl Welford {
|
|
||||||
fn update(&mut self, x: f64) {
|
|
||||||
self.n += 1;
|
|
||||||
let delta = x - self.mean;
|
|
||||||
self.mean += delta / self.n as f64;
|
|
||||||
self.m2 += delta * (x - self.mean);
|
|
||||||
}
|
|
||||||
|
|
||||||
fn stddev(&self) -> f64 {
|
|
||||||
if self.n < 2 { return 1.0; }
|
|
||||||
(self.m2 / (self.n - 1) as f64).sqrt()
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Squared z-score for a new observation (non-destructive).
|
|
||||||
///
|
|
||||||
/// Returns 0 when:
|
|
||||||
/// * fewer than ANOMALY_MIN_SAMPLES have been seen, OR
|
|
||||||
/// * the metric has been flat (stddev < 1% of its mean) — a near-zero
|
|
||||||
/// stddev means the baseline is perfectly stable, so the floor for
|
|
||||||
/// "meaningful deviation" is the metric's own magnitude, not 1e-9.
|
|
||||||
/// Without this guard, a metric stuck at 0.0 for the first N samples
|
|
||||||
/// then spiking to any non-zero value produces z² → ∞.
|
|
||||||
fn z_sq(&self, x: f64) -> f64 {
|
|
||||||
if self.n < ANOMALY_MIN_SAMPLES { return 0.0; }
|
|
||||||
let sd = self.stddev();
|
|
||||||
// Effective floor: 1% of |mean| when the signal is near-zero
|
|
||||||
// noise-free, otherwise the raw stddev.
|
|
||||||
let effective_sd = sd.max(self.mean.abs() * 0.01).max(1e-4);
|
|
||||||
let z = (x - self.mean) / effective_sd;
|
|
||||||
z * z
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Per-node rolling stats for drift-adaptive anomaly detection.
|
|
||||||
#[derive(Default, Clone)]
|
|
||||||
struct NodeAnomalyStats {
|
|
||||||
hot_pct: Welford,
|
|
||||||
hit_rate: Welford,
|
|
||||||
fs_pct: Welford,
|
|
||||||
chunk_miss_rate: Welford,
|
|
||||||
/// Most recent anomaly score (sum of z²); 0 until ANOMALY_MIN_SAMPLES.
|
|
||||||
pub last_score: f64,
|
|
||||||
/// Total samples seen (mirrors hot_pct.n for convenience).
|
|
||||||
pub n: u64,
|
|
||||||
}
|
|
||||||
|
|
||||||
impl NodeAnomalyStats {
|
|
||||||
fn update(&mut self, s: &MetricSample) {
|
|
||||||
let hot_pct = if s.hot_max_bytes > 0 {
|
|
||||||
s.hot_used_bytes as f64 / s.hot_max_bytes as f64
|
|
||||||
} else {
|
|
||||||
0.0
|
|
||||||
};
|
|
||||||
let fs_pct = if s.fs_total_bytes > 0 {
|
|
||||||
s.fs_used_bytes as f64 / s.fs_total_bytes as f64
|
|
||||||
} else {
|
|
||||||
0.0
|
|
||||||
};
|
|
||||||
let total_chunks = s.has_chunk_hits + s.has_chunk_misses;
|
|
||||||
let chunk_miss_rate = if total_chunks > 0 {
|
|
||||||
s.has_chunk_misses as f64 / total_chunks as f64
|
|
||||||
} else {
|
|
||||||
0.0
|
|
||||||
};
|
|
||||||
|
|
||||||
self.last_score = self.hot_pct.z_sq(hot_pct)
|
|
||||||
+ self.hit_rate.z_sq(s.cache_hit_rate)
|
|
||||||
+ self.fs_pct.z_sq(fs_pct)
|
|
||||||
+ self.chunk_miss_rate.z_sq(chunk_miss_rate);
|
|
||||||
|
|
||||||
self.hot_pct.update(hot_pct);
|
|
||||||
self.hit_rate.update(s.cache_hit_rate);
|
|
||||||
self.fs_pct.update(fs_pct);
|
|
||||||
self.chunk_miss_rate.update(chunk_miss_rate);
|
|
||||||
self.n = self.hot_pct.n;
|
|
||||||
}
|
|
||||||
|
|
||||||
fn level(&self) -> &'static str {
|
|
||||||
if self.n < ANOMALY_MIN_SAMPLES { return "ok"; }
|
|
||||||
if self.last_score >= ANOMALY_ALERT_THRESHOLD { "alert" }
|
|
||||||
else if self.last_score >= ANOMALY_WARN_THRESHOLD { "warn" }
|
|
||||||
else { "ok" }
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/// One time-series sample snapshotted from a peer's DashboardStatus RPC.
|
|
||||||
#[derive(Debug, Clone, Serialize)]
|
|
||||||
pub struct MetricSample {
|
|
||||||
pub unix_ts: u64,
|
|
||||||
pub hot_used_bytes: u64,
|
|
||||||
pub hot_max_bytes: u64,
|
|
||||||
pub cache_hit_rate: f64,
|
|
||||||
pub cache_hits: u64,
|
|
||||||
pub cache_misses: u64,
|
|
||||||
pub has_chunk_hits: u64,
|
|
||||||
pub has_chunk_misses: u64,
|
|
||||||
pub fs_used_bytes: u64,
|
|
||||||
pub fs_total_bytes: u64,
|
|
||||||
pub fs_available_bytes: u64,
|
|
||||||
/// Drift-adaptive anomaly score (sum of z² across 4 metrics).
|
|
||||||
/// `None` until the node has collected ANOMALY_MIN_SAMPLES.
|
|
||||||
#[serde(skip_serializing_if = "Option::is_none")]
|
|
||||||
pub anomaly_score: Option<f64>,
|
|
||||||
}
|
|
||||||
|
|
||||||
#[derive(Serialize)]
|
|
||||||
pub struct AnomalyStatus {
|
|
||||||
pub node: String,
|
|
||||||
pub score: f64,
|
|
||||||
pub level: &'static str,
|
|
||||||
pub samples_used: u64,
|
|
||||||
}
|
|
||||||
|
|
||||||
pub struct MetricsHistory {
|
|
||||||
samples: HashMap<String, VecDeque<MetricSample>>,
|
|
||||||
anomaly: HashMap<String, NodeAnomalyStats>,
|
|
||||||
}
|
|
||||||
|
|
||||||
impl MetricsHistory {
|
|
||||||
fn new() -> Self {
|
|
||||||
Self {
|
|
||||||
samples: HashMap::new(),
|
|
||||||
anomaly: HashMap::new(),
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
fn push(&mut self, node: &str, mut sample: MetricSample) {
|
|
||||||
// Update Welford stats and embed anomaly score in the sample.
|
|
||||||
let stats = self.anomaly.entry(node.to_string()).or_default();
|
|
||||||
stats.update(&sample);
|
|
||||||
if stats.n >= ANOMALY_MIN_SAMPLES {
|
|
||||||
sample.anomaly_score = Some(stats.last_score);
|
|
||||||
}
|
|
||||||
|
|
||||||
let deque = self.samples.entry(node.to_string()).or_default();
|
|
||||||
deque.push_back(sample);
|
|
||||||
while deque.len() > HISTORY_MAX_SAMPLES {
|
|
||||||
deque.pop_front();
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
fn get_last(&self, node: &str, limit: usize) -> Vec<MetricSample> {
|
|
||||||
let limit = limit.min(HISTORY_MAX_SAMPLES);
|
|
||||||
self.samples
|
|
||||||
.get(node)
|
|
||||||
.map(|d| {
|
|
||||||
let skip = d.len().saturating_sub(limit);
|
|
||||||
d.iter().skip(skip).cloned().collect()
|
|
||||||
})
|
|
||||||
.unwrap_or_default()
|
|
||||||
}
|
|
||||||
|
|
||||||
pub fn anomaly_statuses(&self) -> Vec<AnomalyStatus> {
|
|
||||||
let mut out: Vec<AnomalyStatus> = self
|
|
||||||
.anomaly
|
|
||||||
.iter()
|
|
||||||
.map(|(node, s)| AnomalyStatus {
|
|
||||||
node: node.clone(),
|
|
||||||
score: s.last_score,
|
|
||||||
level: s.level(),
|
|
||||||
samples_used: s.n,
|
|
||||||
})
|
|
||||||
.collect();
|
|
||||||
out.sort_by(|a, b| a.node.cmp(&b.node));
|
|
||||||
out
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// ─────────────────────────────────────────────────────────────────
|
|
||||||
|
|
||||||
/// Aggregator runtime: one QuicClient, one peer list, one identity.
|
/// Aggregator runtime: one QuicClient, one peer list, one identity.
|
||||||
///
|
///
|
||||||
/// The client is reused across every RPC (quinn holds one UDP
|
/// The client is reused across every RPC (quinn holds one UDP
|
||||||
@@ -260,9 +68,6 @@ pub struct V2State {
|
|||||||
/// leases and are reaped by a background sweeper when their
|
/// leases and are reaped by a background sweeper when their
|
||||||
/// `expires_at_unix` passes without a `renew` or `commit`.
|
/// `expires_at_unix` passes without a `renew` or `commit`.
|
||||||
pub sessions: SessionStore,
|
pub sessions: SessionStore,
|
||||||
/// Ring buffer of per-node metric samples (Phase B). 1440 entries
|
|
||||||
/// = 24h at 1-min resolution. Written by `metrics_poller`.
|
|
||||||
pub history: Arc<tokio::sync::Mutex<MetricsHistory>>,
|
|
||||||
}
|
}
|
||||||
|
|
||||||
impl V2State {
|
impl V2State {
|
||||||
@@ -286,25 +91,29 @@ impl V2State {
|
|||||||
.join("aggregator-sessions.json");
|
.join("aggregator-sessions.json");
|
||||||
let sessions = SessionStore::load(sessions_path)
|
let sessions = SessionStore::load(sessions_path)
|
||||||
.map_err(|e| anyhow::anyhow!("loading session store: {e}"))?;
|
.map_err(|e| anyhow::anyhow!("loading session store: {e}"))?;
|
||||||
// Include the serving node itself as a peer so it appears in
|
// Bug fix 2026-07-31: the fleet view previously never included
|
||||||
// fleet/storage/aggregated views. Skip if already listed.
|
// the node actually serving the dashboard — `cluster.peers` is
|
||||||
let self_name = cfg.node.name.clone();
|
// by definition every *other* node, so hitting a given node's
|
||||||
let mut peers = cluster.peers.clone();
|
// `/api/v2/fleet` directly silently dropped that node from its
|
||||||
if !peers.iter().any(|p| p.name == self_name) {
|
// own view (looked like "node X is missing" from the UI, even
|
||||||
if cluster.bind_lan.is_some() || cluster.bind_tailscale.is_some() {
|
// though X was perfectly healthy — it just never queried
|
||||||
peers.insert(
|
// itself). Fix: synthesize a self `PeerEntry` from our own
|
||||||
0,
|
// gossip bind address and include it in the fan-out list, same
|
||||||
crate::config::PeerEntry {
|
// as any other peer. `peer_rpc_addr` derives the RPC port from
|
||||||
name: self_name.clone(),
|
// `lan_addr`/`tailscale_addr` via the fleet's +1 convention, so
|
||||||
|
// this resolves to the same `bind_rpc_lan`/`bind_rpc_tailscale`
|
||||||
|
// the daemon actually listens on.
|
||||||
|
let self_peer = PeerEntry {
|
||||||
|
name: cfg.node.name.clone(),
|
||||||
zone: cluster.zone.clone(),
|
zone: cluster.zone.clone(),
|
||||||
lan_addr: cluster.bind_lan,
|
lan_addr: cluster.bind_lan,
|
||||||
tailscale_addr: cluster.bind_tailscale,
|
tailscale_addr: cluster.bind_tailscale,
|
||||||
},
|
};
|
||||||
);
|
let mut peers = cluster.peers.clone();
|
||||||
}
|
peers.push(self_peer);
|
||||||
}
|
|
||||||
Ok(Self {
|
Ok(Self {
|
||||||
aggregator_name: self_name,
|
aggregator_name: cfg.node.name.clone(),
|
||||||
peers,
|
peers,
|
||||||
client: std::sync::Arc::new(client),
|
client: std::sync::Arc::new(client),
|
||||||
default_rpc_port_offset: 1,
|
default_rpc_port_offset: 1,
|
||||||
@@ -315,7 +124,6 @@ impl V2State {
|
|||||||
.map(|a| a.tokens.clone())
|
.map(|a| a.tokens.clone())
|
||||||
.unwrap_or_default(),
|
.unwrap_or_default(),
|
||||||
sessions,
|
sessions,
|
||||||
history: Arc::new(tokio::sync::Mutex::new(MetricsHistory::new())),
|
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -394,15 +202,6 @@ pub struct NodeStatusV2 {
|
|||||||
pub cache: Option<CacheSummary>,
|
pub cache: Option<CacheSummary>,
|
||||||
#[serde(default)]
|
#[serde(default)]
|
||||||
pub timers: Vec<TimerStatus>,
|
pub timers: Vec<TimerStatus>,
|
||||||
/// Unix timestamp (seconds) when the peer's daemon last started.
|
|
||||||
#[serde(default)]
|
|
||||||
pub daemon_started_unix: Option<u64>,
|
|
||||||
/// 1-minute load average. None on non-Linux or when the peer is offline.
|
|
||||||
#[serde(default)]
|
|
||||||
pub cpu_load_1m: Option<f32>,
|
|
||||||
/// Lifeguard-style health score: 0 = healthy, higher = stressed (T3.3).
|
|
||||||
#[serde(default)]
|
|
||||||
pub health_score: u8,
|
|
||||||
/// `true` when the aggregator successfully talked to the peer;
|
/// `true` when the aggregator successfully talked to the peer;
|
||||||
/// `false` when the RPC failed. Frontend uses this to badge the
|
/// `false` when the RPC failed. Frontend uses this to badge the
|
||||||
/// card as offline.
|
/// card as offline.
|
||||||
@@ -429,9 +228,6 @@ impl NodeStatusV2 {
|
|||||||
mount: r.mount,
|
mount: r.mount,
|
||||||
cache: r.cache,
|
cache: r.cache,
|
||||||
timers: r.timers,
|
timers: r.timers,
|
||||||
daemon_started_unix: r.daemon_started_unix,
|
|
||||||
cpu_load_1m: r.cpu_load_1m,
|
|
||||||
health_score: r.health_score,
|
|
||||||
online: true,
|
online: true,
|
||||||
error: None,
|
error: None,
|
||||||
};
|
};
|
||||||
@@ -457,9 +253,6 @@ impl NodeStatusV2 {
|
|||||||
mount: None,
|
mount: None,
|
||||||
cache: None,
|
cache: None,
|
||||||
timers: Vec::new(),
|
timers: Vec::new(),
|
||||||
daemon_started_unix: None,
|
|
||||||
cpu_load_1m: None,
|
|
||||||
health_score: 0,
|
|
||||||
online: false,
|
online: false,
|
||||||
error: Some(e),
|
error: Some(e),
|
||||||
}
|
}
|
||||||
@@ -737,7 +530,6 @@ impl V2State {
|
|||||||
api_token: self.api_token.clone(),
|
api_token: self.api_token.clone(),
|
||||||
token_entries: self.token_entries.clone(),
|
token_entries: self.token_entries.clone(),
|
||||||
sessions: self.sessions.clone(),
|
sessions: self.sessions.clone(),
|
||||||
history: self.history.clone(),
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -778,6 +570,19 @@ impl AuthedCaller {
|
|||||||
/// in the request body. Explicit user-supplied workspace is only
|
/// in the request body. Explicit user-supplied workspace is only
|
||||||
/// honored for admin/open — a namespaced caller supplying a
|
/// honored for admin/open — a namespaced caller supplying a
|
||||||
/// mismatched workspace is a forbidden write.
|
/// mismatched workspace is a forbidden write.
|
||||||
|
/// Gate for fleet-infrastructure actions (shutdown-prep) that
|
||||||
|
/// have nothing to do with a tag/repo namespace — a namespaced
|
||||||
|
/// per-app token has no business stopping a node's services.
|
||||||
|
pub fn require_admin(&self) -> Result<(), (StatusCode, String)> {
|
||||||
|
match self {
|
||||||
|
AuthedCaller::Admin | AuthedCaller::Open => Ok(()),
|
||||||
|
AuthedCaller::Namespaced { .. } => Err((
|
||||||
|
StatusCode::FORBIDDEN,
|
||||||
|
"this action requires an admin token".to_string(),
|
||||||
|
)),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
pub fn resolve_workspace(&self, requested: Option<&str>) -> Result<String, (StatusCode, String)> {
|
pub fn resolve_workspace(&self, requested: Option<&str>) -> Result<String, (StatusCode, String)> {
|
||||||
match self {
|
match self {
|
||||||
AuthedCaller::Admin | AuthedCaller::Open => match requested {
|
AuthedCaller::Admin | AuthedCaller::Open => match requested {
|
||||||
@@ -1074,6 +879,93 @@ async fn handle_repos_release(
|
|||||||
}))
|
}))
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// ── shutdown-prep (targets one specific node, not a fan-out) ──────
|
||||||
|
|
||||||
|
#[derive(Serialize)]
|
||||||
|
pub struct ShutdownPrepCheckResponse {
|
||||||
|
pub node: String,
|
||||||
|
pub ready: bool,
|
||||||
|
pub output: String,
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn handle_shutdown_prep_check(
|
||||||
|
State(s): State<Arc<V2State>>,
|
||||||
|
Path(name): Path<String>,
|
||||||
|
axum::extract::Extension(caller): axum::extract::Extension<AuthedCaller>,
|
||||||
|
) -> Result<Json<ShutdownPrepCheckResponse>, (StatusCode, String)> {
|
||||||
|
caller.require_admin()?;
|
||||||
|
let peer = s
|
||||||
|
.peers
|
||||||
|
.iter()
|
||||||
|
.find(|p| p.name == name)
|
||||||
|
.cloned()
|
||||||
|
.ok_or((StatusCode::NOT_FOUND, format!("no peer named {name}")))?;
|
||||||
|
let conn = s
|
||||||
|
.dial(&peer)
|
||||||
|
.await
|
||||||
|
.map_err(|e| (StatusCode::BAD_GATEWAY, format!("connecting to {name}: {e:#}")))?;
|
||||||
|
let reply = crate::cluster::rpc::call_shutdown_prep_check(&conn).await;
|
||||||
|
conn.close(quinn::VarInt::from_u32(0), b"done");
|
||||||
|
let reply = reply.map_err(|e| (StatusCode::BAD_GATEWAY, format!("{e:#}")))?;
|
||||||
|
Ok(Json(ShutdownPrepCheckResponse {
|
||||||
|
node: name,
|
||||||
|
ready: reply.ready,
|
||||||
|
output: reply.output,
|
||||||
|
}))
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Deserialize)]
|
||||||
|
pub struct ShutdownPrepExecuteBody {
|
||||||
|
/// Must equal the target node's own name — a second, server-side
|
||||||
|
/// confirmation beyond "the operator clicked the right button in
|
||||||
|
/// the UI". Checked again on the peer itself in
|
||||||
|
/// `shutdown_prep::execute`.
|
||||||
|
pub confirm_node_name: String,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Serialize)]
|
||||||
|
pub struct ShutdownPrepExecuteResponse {
|
||||||
|
pub node: String,
|
||||||
|
pub started: bool,
|
||||||
|
pub message: String,
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn handle_shutdown_prep_execute(
|
||||||
|
State(s): State<Arc<V2State>>,
|
||||||
|
Path(name): Path<String>,
|
||||||
|
axum::extract::Extension(caller): axum::extract::Extension<AuthedCaller>,
|
||||||
|
Json(body): Json<ShutdownPrepExecuteBody>,
|
||||||
|
) -> Result<Json<ShutdownPrepExecuteResponse>, (StatusCode, String)> {
|
||||||
|
caller.require_admin()?;
|
||||||
|
if body.confirm_node_name != name {
|
||||||
|
return Err((
|
||||||
|
StatusCode::BAD_REQUEST,
|
||||||
|
format!(
|
||||||
|
"confirm_node_name '{}' does not match target node '{name}'",
|
||||||
|
body.confirm_node_name
|
||||||
|
),
|
||||||
|
));
|
||||||
|
}
|
||||||
|
let peer = s
|
||||||
|
.peers
|
||||||
|
.iter()
|
||||||
|
.find(|p| p.name == name)
|
||||||
|
.cloned()
|
||||||
|
.ok_or((StatusCode::NOT_FOUND, format!("no peer named {name}")))?;
|
||||||
|
let conn = s
|
||||||
|
.dial(&peer)
|
||||||
|
.await
|
||||||
|
.map_err(|e| (StatusCode::BAD_GATEWAY, format!("connecting to {name}: {e:#}")))?;
|
||||||
|
let reply = crate::cluster::rpc::call_shutdown_prep_execute(&conn, &name).await;
|
||||||
|
conn.close(quinn::VarInt::from_u32(0), b"done");
|
||||||
|
let reply = reply.map_err(|e| (StatusCode::BAD_GATEWAY, format!("{e:#}")))?;
|
||||||
|
Ok(Json(ShutdownPrepExecuteResponse {
|
||||||
|
node: name,
|
||||||
|
started: reply.started,
|
||||||
|
message: reply.message,
|
||||||
|
}))
|
||||||
|
}
|
||||||
|
|
||||||
async fn fanout_repo_ensure(
|
async fn fanout_repo_ensure(
|
||||||
s: &V2State,
|
s: &V2State,
|
||||||
req: &crate::cluster::repo_ensure::RepoEnsureRequest,
|
req: &crate::cluster::repo_ensure::RepoEnsureRequest,
|
||||||
@@ -1513,242 +1405,6 @@ async fn reap_expired(state: Arc<V2State>, sess: Session) {
|
|||||||
///
|
///
|
||||||
/// Also spawns the background TTL sweeper (Phase 9 S1). The task is
|
/// Also spawns the background TTL sweeper (Phase 9 S1). The task is
|
||||||
/// detached — its lifetime is the process lifetime.
|
/// detached — its lifetime is the process lifetime.
|
||||||
// ── metrics poller (Phase B) ─────────────────────────────────────
|
|
||||||
|
|
||||||
/// Background task: poll every peer every 60s, append a `MetricSample`
|
|
||||||
/// to the ring buffer. Runs indefinitely — dropped only on daemon exit.
|
|
||||||
async fn metrics_poller(state: Arc<V2State>) {
|
|
||||||
let mut interval = tokio::time::interval(Duration::from_secs(60));
|
|
||||||
interval.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip);
|
|
||||||
loop {
|
|
||||||
interval.tick().await;
|
|
||||||
for peer in &state.peers {
|
|
||||||
let peer_name = peer.name.clone();
|
|
||||||
match state.fetch_node(peer).await {
|
|
||||||
Ok(r) => {
|
|
||||||
let now = std::time::SystemTime::now()
|
|
||||||
.duration_since(std::time::UNIX_EPOCH)
|
|
||||||
.unwrap_or_default()
|
|
||||||
.as_secs();
|
|
||||||
let sample = MetricSample {
|
|
||||||
unix_ts: now,
|
|
||||||
hot_used_bytes: r.hot.as_ref().map(|h| h.used_bytes).unwrap_or(0),
|
|
||||||
hot_max_bytes: r.hot.as_ref().map(|h| h.max_bytes).unwrap_or(0),
|
|
||||||
cache_hit_rate: r.cache.as_ref().map(|c| c.hit_rate).unwrap_or(0.0),
|
|
||||||
cache_hits: r.cache.as_ref().map(|c| c.hits).unwrap_or(0),
|
|
||||||
cache_misses: r.cache.as_ref().map(|c| c.misses).unwrap_or(0),
|
|
||||||
has_chunk_hits: r.cache.as_ref().map(|c| c.has_chunk_hits).unwrap_or(0),
|
|
||||||
has_chunk_misses: r.cache.as_ref().map(|c| c.has_chunk_misses).unwrap_or(0),
|
|
||||||
fs_used_bytes: r.filesystem.as_ref().map(|f| f.used_bytes).unwrap_or(0),
|
|
||||||
fs_total_bytes: r.filesystem.as_ref().map(|f| f.total_bytes).unwrap_or(0),
|
|
||||||
fs_available_bytes: r.filesystem.as_ref().map(|f| f.available_bytes).unwrap_or(0),
|
|
||||||
// Filled in by MetricsHistory::push after Welford update.
|
|
||||||
anomaly_score: None,
|
|
||||||
};
|
|
||||||
let mut hist = state.history.lock().await;
|
|
||||||
hist.push(&peer_name, sample);
|
|
||||||
}
|
|
||||||
Err(e) => {
|
|
||||||
tracing::debug!(peer = %peer_name, error = %e, "metrics poll skipped");
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
#[derive(Deserialize)]
|
|
||||||
struct MetricsHistoryQuery {
|
|
||||||
limit: Option<usize>,
|
|
||||||
}
|
|
||||||
|
|
||||||
async fn handle_metrics_history(
|
|
||||||
Path(name): Path<String>,
|
|
||||||
Query(q): Query<MetricsHistoryQuery>,
|
|
||||||
State(s): State<Arc<V2State>>,
|
|
||||||
) -> Json<Vec<MetricSample>> {
|
|
||||||
let limit = q.limit.unwrap_or(60).min(1440);
|
|
||||||
let hist = s.history.lock().await;
|
|
||||||
Json(hist.get_last(&name, limit))
|
|
||||||
}
|
|
||||||
|
|
||||||
async fn handle_anomalies(State(s): State<Arc<V2State>>) -> Json<Vec<AnomalyStatus>> {
|
|
||||||
let hist = s.history.lock().await;
|
|
||||||
Json(hist.anomaly_statuses())
|
|
||||||
}
|
|
||||||
|
|
||||||
/// T2.7: cross-node cache warming candidates.
|
|
||||||
///
|
|
||||||
/// A warming candidate is a fingerprint that is frequently accessed on
|
|
||||||
/// at least one node but absent (never hit) on at least one other node.
|
|
||||||
/// Pushing the blob to the missing node turns the next GetRef there
|
|
||||||
/// into a local hit instead of a cross-node forwarded fetch.
|
|
||||||
#[derive(serde::Serialize)]
|
|
||||||
struct WarmingCandidate {
|
|
||||||
fingerprint_hex: String,
|
|
||||||
/// Nodes where this fingerprint has been accessed. Sorted.
|
|
||||||
hot_on: Vec<String>,
|
|
||||||
/// Nodes where the fingerprint has zero recorded hits. Sorted.
|
|
||||||
missing_on: Vec<String>,
|
|
||||||
/// Highest hit count seen across all hot nodes.
|
|
||||||
max_hit_count: u64,
|
|
||||||
/// Most recent access timestamp across all hot nodes.
|
|
||||||
last_hit_unix: u64,
|
|
||||||
}
|
|
||||||
|
|
||||||
async fn handle_hot_refs(State(s): State<Arc<V2State>>) -> Json<Vec<WarmingCandidate>> {
|
|
||||||
// Collect each node's hot refs via the existing gather_storage fan-out.
|
|
||||||
let storage = gather_storage(&s).await;
|
|
||||||
let node_names: Vec<String> = storage.iter().map(|(n, _)| n.clone()).collect();
|
|
||||||
|
|
||||||
// Build a map: fingerprint_hex → { node → (hit_count, last_hit_unix) }
|
|
||||||
let mut fp_map: std::collections::HashMap<
|
|
||||||
String,
|
|
||||||
std::collections::HashMap<String, (u64, u64)>,
|
|
||||||
> = std::collections::HashMap::new();
|
|
||||||
|
|
||||||
for (node, reply) in &storage {
|
|
||||||
for hr in &reply.hot_refs {
|
|
||||||
fp_map
|
|
||||||
.entry(hr.fingerprint_hex.clone())
|
|
||||||
.or_default()
|
|
||||||
.insert(node.clone(), (hr.hit_count, hr.last_hit_unix));
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// A candidate is hot on ≥1 node and missing on ≥1 other node.
|
|
||||||
let mut candidates: Vec<WarmingCandidate> = fp_map
|
|
||||||
.into_iter()
|
|
||||||
.filter_map(|(fp, node_hits)| {
|
|
||||||
let missing_on: Vec<String> = node_names
|
|
||||||
.iter()
|
|
||||||
.filter(|n| !node_hits.contains_key(*n))
|
|
||||||
.cloned()
|
|
||||||
.collect();
|
|
||||||
if missing_on.is_empty() {
|
|
||||||
return None; // present (or hit-tracked) on all nodes
|
|
||||||
}
|
|
||||||
let mut hot_on: Vec<String> = node_hits.keys().cloned().collect();
|
|
||||||
hot_on.sort();
|
|
||||||
let max_hit_count = node_hits.values().map(|&(c, _)| c).max().unwrap_or(0);
|
|
||||||
let last_hit_unix = node_hits.values().map(|&(_, t)| t).max().unwrap_or(0);
|
|
||||||
Some(WarmingCandidate {
|
|
||||||
fingerprint_hex: fp,
|
|
||||||
hot_on,
|
|
||||||
missing_on: { let mut v = missing_on; v.sort(); v },
|
|
||||||
max_hit_count,
|
|
||||||
last_hit_unix,
|
|
||||||
})
|
|
||||||
})
|
|
||||||
.collect();
|
|
||||||
|
|
||||||
// Hottest candidates first.
|
|
||||||
candidates.sort_by(|a, b| b.max_hit_count.cmp(&a.max_hit_count));
|
|
||||||
candidates.truncate(50);
|
|
||||||
Json(candidates)
|
|
||||||
}
|
|
||||||
|
|
||||||
/// T1.2: fleet-wide maintenance event log.
|
|
||||||
///
|
|
||||||
/// Aggregates GC/scrub/repair events from every peer and returns them
|
|
||||||
/// sorted newest-first, so the operator sees a single unified timeline
|
|
||||||
/// rather than having to check each node individually.
|
|
||||||
#[derive(serde::Serialize)]
|
|
||||||
struct MaintenanceRow {
|
|
||||||
node: String,
|
|
||||||
kind: String,
|
|
||||||
unix_ts: u64,
|
|
||||||
chunks_scanned: usize,
|
|
||||||
chunks_removed: usize,
|
|
||||||
bytes_reclaimed: u64,
|
|
||||||
error: Option<String>,
|
|
||||||
}
|
|
||||||
|
|
||||||
async fn handle_maintenance(State(s): State<Arc<V2State>>) -> Json<Vec<MaintenanceRow>> {
|
|
||||||
let storage = gather_storage(&s).await;
|
|
||||||
let mut rows: Vec<MaintenanceRow> = storage
|
|
||||||
.iter()
|
|
||||||
.flat_map(|(node, reply)| {
|
|
||||||
reply.maintenance_events.iter().map(move |ev| MaintenanceRow {
|
|
||||||
node: node.clone(),
|
|
||||||
kind: ev.kind.clone(),
|
|
||||||
unix_ts: ev.unix_ts,
|
|
||||||
chunks_scanned: ev.chunks_scanned,
|
|
||||||
chunks_removed: ev.chunks_removed,
|
|
||||||
bytes_reclaimed: ev.bytes_reclaimed,
|
|
||||||
error: ev.error.clone(),
|
|
||||||
})
|
|
||||||
})
|
|
||||||
.collect();
|
|
||||||
rows.sort_by(|a, b| b.unix_ts.cmp(&a.unix_ts));
|
|
||||||
Json(rows)
|
|
||||||
}
|
|
||||||
|
|
||||||
/// T2.8: cache pollution candidates (ACPC-inspired).
|
|
||||||
///
|
|
||||||
/// A blob is "polluting" the hot tier when it is large relative to how often
|
|
||||||
/// it is actually accessed. Pollution score = size_bytes / (hit_count + 1).
|
|
||||||
/// Zero-hit blobs score highest; large blobs that are hit frequently score low.
|
|
||||||
#[derive(serde::Serialize)]
|
|
||||||
struct PollutionCandidate {
|
|
||||||
node: String,
|
|
||||||
fingerprint_hex: String,
|
|
||||||
blob_id_hex: String,
|
|
||||||
size_bytes: u64,
|
|
||||||
/// GetRef hits since last daemon restart.
|
|
||||||
hit_count: u64,
|
|
||||||
/// size_bytes / (hit_count + 1) — higher = more polluting.
|
|
||||||
pollution_score: f64,
|
|
||||||
}
|
|
||||||
|
|
||||||
async fn handle_pollution(State(s): State<Arc<V2State>>) -> Json<Vec<PollutionCandidate>> {
|
|
||||||
let storage = gather_storage(&s).await;
|
|
||||||
let mut candidates = Vec::new();
|
|
||||||
|
|
||||||
for (node, reply) in &storage {
|
|
||||||
// fingerprint → hit_count
|
|
||||||
let hot_map: std::collections::HashMap<&str, u64> = reply
|
|
||||||
.hot_refs
|
|
||||||
.iter()
|
|
||||||
.map(|hr| (hr.fingerprint_hex.as_str(), hr.hit_count))
|
|
||||||
.collect();
|
|
||||||
|
|
||||||
// blob_id → size_bytes
|
|
||||||
let blob_map: std::collections::HashMap<&str, u64> = reply
|
|
||||||
.blobs_sample
|
|
||||||
.iter()
|
|
||||||
.map(|b| (b.blob_id_hex.as_str(), b.size_bytes))
|
|
||||||
.collect();
|
|
||||||
|
|
||||||
for dr in &reply.refs_sample {
|
|
||||||
let size_bytes = match blob_map.get(dr.blob_id_hex.as_str()) {
|
|
||||||
Some(&s) if s > 0 => s,
|
|
||||||
_ => continue, // skip unknown-size entries
|
|
||||||
};
|
|
||||||
let hit_count = *hot_map.get(dr.fingerprint_hex.as_str()).unwrap_or(&0);
|
|
||||||
let pollution_score = size_bytes as f64 / (hit_count as f64 + 1.0);
|
|
||||||
candidates.push(PollutionCandidate {
|
|
||||||
node: node.clone(),
|
|
||||||
fingerprint_hex: dr.fingerprint_hex.clone(),
|
|
||||||
blob_id_hex: dr.blob_id_hex.clone(),
|
|
||||||
size_bytes,
|
|
||||||
hit_count,
|
|
||||||
pollution_score,
|
|
||||||
});
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Highest pollution score first (largest cold blobs at the top).
|
|
||||||
candidates.sort_by(|a, b| {
|
|
||||||
b.pollution_score
|
|
||||||
.partial_cmp(&a.pollution_score)
|
|
||||||
.unwrap_or(std::cmp::Ordering::Equal)
|
|
||||||
});
|
|
||||||
candidates.truncate(50);
|
|
||||||
Json(candidates)
|
|
||||||
}
|
|
||||||
|
|
||||||
// ─────────────────────────────────────────────────────────────────
|
|
||||||
|
|
||||||
pub fn build(state: Arc<V2State>) -> Router {
|
pub fn build(state: Arc<V2State>) -> Router {
|
||||||
// Spawn the sweeper. 15s tick is a reasonable balance: quick
|
// Spawn the sweeper. 15s tick is a reasonable balance: quick
|
||||||
// enough that a mid-wizard-close cleanup feels prompt, slow
|
// enough that a mid-wizard-close cleanup feels prompt, slow
|
||||||
@@ -1765,16 +1421,9 @@ pub fn build(state: Arc<V2State>) -> Router {
|
|||||||
},
|
},
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
// Spawn the metrics ring-buffer poller (Phase B).
|
|
||||||
tokio::spawn(metrics_poller(state.clone()));
|
|
||||||
Router::new()
|
Router::new()
|
||||||
.route("/api/v2/fleet", get(handle_fleet))
|
.route("/api/v2/fleet", get(handle_fleet))
|
||||||
.route("/api/v2/node/:name/status", get(handle_node_status))
|
.route("/api/v2/node/:name/status", get(handle_node_status))
|
||||||
.route("/api/v2/node/:name/metrics-history", get(handle_metrics_history))
|
|
||||||
.route("/api/v2/anomalies", get(handle_anomalies))
|
|
||||||
.route("/api/v2/hot-refs", get(handle_hot_refs))
|
|
||||||
.route("/api/v2/pollution", get(handle_pollution))
|
|
||||||
.route("/api/v2/maintenance", get(handle_maintenance))
|
|
||||||
.route("/api/v2/storage/blobs", get(handle_blobs))
|
.route("/api/v2/storage/blobs", get(handle_blobs))
|
||||||
.route("/api/v2/storage/tags", get(handle_tags))
|
.route("/api/v2/storage/tags", get(handle_tags))
|
||||||
.route("/api/v2/storage/refs", get(handle_refs))
|
.route("/api/v2/storage/refs", get(handle_refs))
|
||||||
@@ -1798,6 +1447,14 @@ pub fn build(state: Arc<V2State>) -> Router {
|
|||||||
.route("/api/v2/sessions/:id/commit", post(handle_commit_session))
|
.route("/api/v2/sessions/:id/commit", post(handle_commit_session))
|
||||||
.route("/api/v2/repos/ensure", post(handle_repos_ensure))
|
.route("/api/v2/repos/ensure", post(handle_repos_ensure))
|
||||||
.route("/api/v2/repos/release", post(handle_repos_release))
|
.route("/api/v2/repos/release", post(handle_repos_release))
|
||||||
|
.route(
|
||||||
|
"/api/v2/node/:name/shutdown-prep/check",
|
||||||
|
post(handle_shutdown_prep_check),
|
||||||
|
)
|
||||||
|
.route(
|
||||||
|
"/api/v2/node/:name/shutdown-prep/execute",
|
||||||
|
post(handle_shutdown_prep_execute),
|
||||||
|
)
|
||||||
.route_layer(axum::middleware::from_fn_with_state(state.clone(), v2_auth))
|
.route_layer(axum::middleware::from_fn_with_state(state.clone(), v2_auth))
|
||||||
.with_state(state)
|
.with_state(state)
|
||||||
}
|
}
|
||||||
|
|||||||
Generated
+32
-345
@@ -10,7 +10,6 @@
|
|||||||
"dependencies": {
|
"dependencies": {
|
||||||
"react": "^19.2.6",
|
"react": "^19.2.6",
|
||||||
"react-dom": "^19.2.6",
|
"react-dom": "^19.2.6",
|
||||||
"recharts": "^3.10.0",
|
|
||||||
"wouter": "^3.7.1"
|
"wouter": "^3.7.1"
|
||||||
},
|
},
|
||||||
"devDependencies": {
|
"devDependencies": {
|
||||||
@@ -177,31 +176,6 @@
|
|||||||
"url": "https://github.com/sponsors/Boshen"
|
"url": "https://github.com/sponsors/Boshen"
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
"node_modules/@reduxjs/toolkit": {
|
|
||||||
"version": "2.12.0",
|
|
||||||
"resolved": "https://registry.npmjs.org/@reduxjs/toolkit/-/toolkit-2.12.0.tgz",
|
|
||||||
"integrity": "sha512-KiT+RzZbp6mQET+Mg+h2c97+9j1sNflUxQkIHI7Yuzf6Peu+OYpmkn6nbHWmLLWj+1ZODUJFwGZ7gx3L9R9EOw==",
|
|
||||||
"dependencies": {
|
|
||||||
"@standard-schema/spec": "^1.0.0",
|
|
||||||
"@standard-schema/utils": "^0.3.0",
|
|
||||||
"immer": "^11.0.0",
|
|
||||||
"redux": "^5.0.1",
|
|
||||||
"redux-thunk": "^3.1.0",
|
|
||||||
"reselect": "^5.1.0"
|
|
||||||
},
|
|
||||||
"peerDependencies": {
|
|
||||||
"react": "^16.9.0 || ^17.0.0 || ^18 || ^19",
|
|
||||||
"react-redux": "^7.2.1 || ^8.1.3 || ^9.0.0"
|
|
||||||
},
|
|
||||||
"peerDependenciesMeta": {
|
|
||||||
"react": {
|
|
||||||
"optional": true
|
|
||||||
},
|
|
||||||
"react-redux": {
|
|
||||||
"optional": true
|
|
||||||
}
|
|
||||||
}
|
|
||||||
},
|
|
||||||
"node_modules/@rolldown/binding-android-arm64": {
|
"node_modules/@rolldown/binding-android-arm64": {
|
||||||
"version": "1.1.5",
|
"version": "1.1.5",
|
||||||
"resolved": "https://registry.npmjs.org/@rolldown/binding-android-arm64/-/binding-android-arm64-1.1.5.tgz",
|
"resolved": "https://registry.npmjs.org/@rolldown/binding-android-arm64/-/binding-android-arm64-1.1.5.tgz",
|
||||||
@@ -295,6 +269,9 @@
|
|||||||
"arm64"
|
"arm64"
|
||||||
],
|
],
|
||||||
"dev": true,
|
"dev": true,
|
||||||
|
"libc": [
|
||||||
|
"glibc"
|
||||||
|
],
|
||||||
"license": "MIT",
|
"license": "MIT",
|
||||||
"optional": true,
|
"optional": true,
|
||||||
"os": [
|
"os": [
|
||||||
@@ -312,6 +289,9 @@
|
|||||||
"arm64"
|
"arm64"
|
||||||
],
|
],
|
||||||
"dev": true,
|
"dev": true,
|
||||||
|
"libc": [
|
||||||
|
"musl"
|
||||||
|
],
|
||||||
"license": "MIT",
|
"license": "MIT",
|
||||||
"optional": true,
|
"optional": true,
|
||||||
"os": [
|
"os": [
|
||||||
@@ -329,6 +309,9 @@
|
|||||||
"ppc64"
|
"ppc64"
|
||||||
],
|
],
|
||||||
"dev": true,
|
"dev": true,
|
||||||
|
"libc": [
|
||||||
|
"glibc"
|
||||||
|
],
|
||||||
"license": "MIT",
|
"license": "MIT",
|
||||||
"optional": true,
|
"optional": true,
|
||||||
"os": [
|
"os": [
|
||||||
@@ -346,6 +329,9 @@
|
|||||||
"s390x"
|
"s390x"
|
||||||
],
|
],
|
||||||
"dev": true,
|
"dev": true,
|
||||||
|
"libc": [
|
||||||
|
"glibc"
|
||||||
|
],
|
||||||
"license": "MIT",
|
"license": "MIT",
|
||||||
"optional": true,
|
"optional": true,
|
||||||
"os": [
|
"os": [
|
||||||
@@ -363,6 +349,9 @@
|
|||||||
"x64"
|
"x64"
|
||||||
],
|
],
|
||||||
"dev": true,
|
"dev": true,
|
||||||
|
"libc": [
|
||||||
|
"glibc"
|
||||||
|
],
|
||||||
"license": "MIT",
|
"license": "MIT",
|
||||||
"optional": true,
|
"optional": true,
|
||||||
"os": [
|
"os": [
|
||||||
@@ -380,6 +369,9 @@
|
|||||||
"x64"
|
"x64"
|
||||||
],
|
],
|
||||||
"dev": true,
|
"dev": true,
|
||||||
|
"libc": [
|
||||||
|
"musl"
|
||||||
|
],
|
||||||
"license": "MIT",
|
"license": "MIT",
|
||||||
"optional": true,
|
"optional": true,
|
||||||
"os": [
|
"os": [
|
||||||
@@ -466,16 +458,6 @@
|
|||||||
"dev": true,
|
"dev": true,
|
||||||
"license": "MIT"
|
"license": "MIT"
|
||||||
},
|
},
|
||||||
"node_modules/@standard-schema/spec": {
|
|
||||||
"version": "1.1.0",
|
|
||||||
"resolved": "https://registry.npmjs.org/@standard-schema/spec/-/spec-1.1.0.tgz",
|
|
||||||
"integrity": "sha512-l2aFy5jALhniG5HgqrD6jXLi/rUWrKvqN/qJx6yoJsgKhblVd+iqqU4RCXavm/jPityDo5TCvKMnpjKnOriy0w=="
|
|
||||||
},
|
|
||||||
"node_modules/@standard-schema/utils": {
|
|
||||||
"version": "0.3.0",
|
|
||||||
"resolved": "https://registry.npmjs.org/@standard-schema/utils/-/utils-0.3.0.tgz",
|
|
||||||
"integrity": "sha512-e7Mew686owMaPJVNNLs55PUvgz371nKgwsc4vxE49zsODpJEnxgxRo2y/OKrqueavXgZNMDVj3DdHFlaSAeU8g=="
|
|
||||||
},
|
|
||||||
"node_modules/@tybys/wasm-util": {
|
"node_modules/@tybys/wasm-util": {
|
||||||
"version": "0.10.3",
|
"version": "0.10.3",
|
||||||
"resolved": "https://registry.npmjs.org/@tybys/wasm-util/-/wasm-util-0.10.3.tgz",
|
"resolved": "https://registry.npmjs.org/@tybys/wasm-util/-/wasm-util-0.10.3.tgz",
|
||||||
@@ -487,65 +469,11 @@
|
|||||||
"tslib": "^2.4.0"
|
"tslib": "^2.4.0"
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
"node_modules/@types/d3-array": {
|
|
||||||
"version": "3.2.2",
|
|
||||||
"resolved": "https://registry.npmjs.org/@types/d3-array/-/d3-array-3.2.2.tgz",
|
|
||||||
"integrity": "sha512-hOLWVbm7uRza0BYXpIIW5pxfrKe0W+D5lrFiAEYR+pb6w3N2SwSMaJbXdUfSEv+dT4MfHBLtn5js0LAWaO6otw=="
|
|
||||||
},
|
|
||||||
"node_modules/@types/d3-color": {
|
|
||||||
"version": "3.1.3",
|
|
||||||
"resolved": "https://registry.npmjs.org/@types/d3-color/-/d3-color-3.1.3.tgz",
|
|
||||||
"integrity": "sha512-iO90scth9WAbmgv7ogoq57O9YpKmFBbmoEoCHDB2xMBY0+/KVrqAaCDyCE16dUspeOvIxFFRI+0sEtqDqy2b4A=="
|
|
||||||
},
|
|
||||||
"node_modules/@types/d3-ease": {
|
|
||||||
"version": "3.0.2",
|
|
||||||
"resolved": "https://registry.npmjs.org/@types/d3-ease/-/d3-ease-3.0.2.tgz",
|
|
||||||
"integrity": "sha512-NcV1JjO5oDzoK26oMzbILE6HW7uVXOHLQvHshBUW4UMdZGfiY6v5BeQwh9a9tCzv+CeefZQHJt5SRgK154RtiA=="
|
|
||||||
},
|
|
||||||
"node_modules/@types/d3-interpolate": {
|
|
||||||
"version": "3.0.4",
|
|
||||||
"resolved": "https://registry.npmjs.org/@types/d3-interpolate/-/d3-interpolate-3.0.4.tgz",
|
|
||||||
"integrity": "sha512-mgLPETlrpVV1YRJIglr4Ez47g7Yxjl1lj7YKsiMCb27VJH9W8NVM6Bb9d8kkpG/uAQS5AmbA48q2IAolKKo1MA==",
|
|
||||||
"dependencies": {
|
|
||||||
"@types/d3-color": "*"
|
|
||||||
}
|
|
||||||
},
|
|
||||||
"node_modules/@types/d3-path": {
|
|
||||||
"version": "3.1.1",
|
|
||||||
"resolved": "https://registry.npmjs.org/@types/d3-path/-/d3-path-3.1.1.tgz",
|
|
||||||
"integrity": "sha512-VMZBYyQvbGmWyWVea0EHs/BwLgxc+MKi1zLDCONksozI4YJMcTt8ZEuIR4Sb1MMTE8MMW49v0IwI5+b7RmfWlg=="
|
|
||||||
},
|
|
||||||
"node_modules/@types/d3-scale": {
|
|
||||||
"version": "4.0.9",
|
|
||||||
"resolved": "https://registry.npmjs.org/@types/d3-scale/-/d3-scale-4.0.9.tgz",
|
|
||||||
"integrity": "sha512-dLmtwB8zkAeO/juAMfnV+sItKjlsw2lKdZVVy6LRr0cBmegxSABiLEpGVmSJJ8O08i4+sGR6qQtb6WtuwJdvVw==",
|
|
||||||
"dependencies": {
|
|
||||||
"@types/d3-time": "*"
|
|
||||||
}
|
|
||||||
},
|
|
||||||
"node_modules/@types/d3-shape": {
|
|
||||||
"version": "3.1.8",
|
|
||||||
"resolved": "https://registry.npmjs.org/@types/d3-shape/-/d3-shape-3.1.8.tgz",
|
|
||||||
"integrity": "sha512-lae0iWfcDeR7qt7rA88BNiqdvPS5pFVPpo5OfjElwNaT2yyekbM0C9vK+yqBqEmHr6lDkRnYNoTBYlAgJa7a4w==",
|
|
||||||
"dependencies": {
|
|
||||||
"@types/d3-path": "*"
|
|
||||||
}
|
|
||||||
},
|
|
||||||
"node_modules/@types/d3-time": {
|
|
||||||
"version": "3.0.4",
|
|
||||||
"resolved": "https://registry.npmjs.org/@types/d3-time/-/d3-time-3.0.4.tgz",
|
|
||||||
"integrity": "sha512-yuzZug1nkAAaBlBBikKZTgzCeA+k1uy4ZFwWANOfKw5z5LRhV0gNA7gNkKm7HoK+HRN0wX3EkxGk0fpbWhmB7g=="
|
|
||||||
},
|
|
||||||
"node_modules/@types/d3-timer": {
|
|
||||||
"version": "3.0.2",
|
|
||||||
"resolved": "https://registry.npmjs.org/@types/d3-timer/-/d3-timer-3.0.2.tgz",
|
|
||||||
"integrity": "sha512-Ps3T8E8dZDam6fUyNiMkekK3XUsaUEik+idO9/YjPtfj2qruF8tFBXS7XhtE4iIXBLxhmLjP3SXpLhVf21I9Lw=="
|
|
||||||
},
|
|
||||||
"node_modules/@types/react": {
|
"node_modules/@types/react": {
|
||||||
"version": "19.2.17",
|
"version": "19.2.17",
|
||||||
"resolved": "https://registry.npmjs.org/@types/react/-/react-19.2.17.tgz",
|
"resolved": "https://registry.npmjs.org/@types/react/-/react-19.2.17.tgz",
|
||||||
"integrity": "sha512-MXfmqaVPEVgkBT/aY0aGCkRWWtByiYQXo3xdQ8r5RzuFrPiRn8Gar2tQdXSUQ2GKV3bkXckek89V8wQBY2Q/Aw==",
|
"integrity": "sha512-MXfmqaVPEVgkBT/aY0aGCkRWWtByiYQXo3xdQ8r5RzuFrPiRn8Gar2tQdXSUQ2GKV3bkXckek89V8wQBY2Q/Aw==",
|
||||||
"devOptional": true,
|
"dev": true,
|
||||||
"license": "MIT",
|
"license": "MIT",
|
||||||
"dependencies": {
|
"dependencies": {
|
||||||
"csstype": "^3.2.2"
|
"csstype": "^3.2.2"
|
||||||
@@ -561,11 +489,6 @@
|
|||||||
"@types/react": "^19.2.0"
|
"@types/react": "^19.2.0"
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
"node_modules/@types/use-sync-external-store": {
|
|
||||||
"version": "0.0.6",
|
|
||||||
"resolved": "https://registry.npmjs.org/@types/use-sync-external-store/-/use-sync-external-store-0.0.6.tgz",
|
|
||||||
"integrity": "sha512-zFDAD+tlpf2r4asuHEj0XH6pY6i0g5NeAHPn+15wk3BV6JA69eERFXC1gyGThDkVa1zCyKr5jox1+2LbV/AMLg=="
|
|
||||||
},
|
|
||||||
"node_modules/@vitejs/plugin-react": {
|
"node_modules/@vitejs/plugin-react": {
|
||||||
"version": "6.0.3",
|
"version": "6.0.3",
|
||||||
"resolved": "https://registry.npmjs.org/@vitejs/plugin-react/-/plugin-react-6.0.3.tgz",
|
"resolved": "https://registry.npmjs.org/@vitejs/plugin-react/-/plugin-react-6.0.3.tgz",
|
||||||
@@ -799,14 +722,6 @@
|
|||||||
"node": ">= 6"
|
"node": ">= 6"
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
"node_modules/clsx": {
|
|
||||||
"version": "2.1.1",
|
|
||||||
"resolved": "https://registry.npmjs.org/clsx/-/clsx-2.1.1.tgz",
|
|
||||||
"integrity": "sha512-eYm0QWBtUrBWZWG0d386OGAw16Z995PiOVo2B7bjWSbHedGl5e0ZWaq65kOGgUSNesEIDkB9ISbTg/JK9dhCZA==",
|
|
||||||
"engines": {
|
|
||||||
"node": ">=6"
|
|
||||||
}
|
|
||||||
},
|
|
||||||
"node_modules/commander": {
|
"node_modules/commander": {
|
||||||
"version": "4.1.1",
|
"version": "4.1.1",
|
||||||
"resolved": "https://registry.npmjs.org/commander/-/commander-4.1.1.tgz",
|
"resolved": "https://registry.npmjs.org/commander/-/commander-4.1.1.tgz",
|
||||||
@@ -834,124 +749,9 @@
|
|||||||
"version": "3.2.3",
|
"version": "3.2.3",
|
||||||
"resolved": "https://registry.npmjs.org/csstype/-/csstype-3.2.3.tgz",
|
"resolved": "https://registry.npmjs.org/csstype/-/csstype-3.2.3.tgz",
|
||||||
"integrity": "sha512-z1HGKcYy2xA8AGQfwrn0PAy+PB7X/GSj3UVJW9qKyn43xWa+gl5nXmU4qqLMRzWVLFC8KusUX8T/0kCiOYpAIQ==",
|
"integrity": "sha512-z1HGKcYy2xA8AGQfwrn0PAy+PB7X/GSj3UVJW9qKyn43xWa+gl5nXmU4qqLMRzWVLFC8KusUX8T/0kCiOYpAIQ==",
|
||||||
"devOptional": true,
|
"dev": true,
|
||||||
"license": "MIT"
|
"license": "MIT"
|
||||||
},
|
},
|
||||||
"node_modules/d3-array": {
|
|
||||||
"version": "3.2.4",
|
|
||||||
"resolved": "https://registry.npmjs.org/d3-array/-/d3-array-3.2.4.tgz",
|
|
||||||
"integrity": "sha512-tdQAmyA18i4J7wprpYq8ClcxZy3SC31QMeByyCFyRt7BVHdREQZ5lpzoe5mFEYZUWe+oq8HBvk9JjpibyEV4Jg==",
|
|
||||||
"dependencies": {
|
|
||||||
"internmap": "1 - 2"
|
|
||||||
},
|
|
||||||
"engines": {
|
|
||||||
"node": ">=12"
|
|
||||||
}
|
|
||||||
},
|
|
||||||
"node_modules/d3-color": {
|
|
||||||
"version": "3.1.0",
|
|
||||||
"resolved": "https://registry.npmjs.org/d3-color/-/d3-color-3.1.0.tgz",
|
|
||||||
"integrity": "sha512-zg/chbXyeBtMQ1LbD/WSoW2DpC3I0mpmPdW+ynRTj/x2DAWYrIY7qeZIHidozwV24m4iavr15lNwIwLxRmOxhA==",
|
|
||||||
"engines": {
|
|
||||||
"node": ">=12"
|
|
||||||
}
|
|
||||||
},
|
|
||||||
"node_modules/d3-ease": {
|
|
||||||
"version": "3.0.1",
|
|
||||||
"resolved": "https://registry.npmjs.org/d3-ease/-/d3-ease-3.0.1.tgz",
|
|
||||||
"integrity": "sha512-wR/XK3D3XcLIZwpbvQwQ5fK+8Ykds1ip7A2Txe0yxncXSdq1L9skcG7blcedkOX+ZcgxGAmLX1FrRGbADwzi0w==",
|
|
||||||
"engines": {
|
|
||||||
"node": ">=12"
|
|
||||||
}
|
|
||||||
},
|
|
||||||
"node_modules/d3-format": {
|
|
||||||
"version": "3.1.2",
|
|
||||||
"resolved": "https://registry.npmjs.org/d3-format/-/d3-format-3.1.2.tgz",
|
|
||||||
"integrity": "sha512-AJDdYOdnyRDV5b6ArilzCPPwc1ejkHcoyFarqlPqT7zRYjhavcT3uSrqcMvsgh2CgoPbK3RCwyHaVyxYcP2Arg==",
|
|
||||||
"engines": {
|
|
||||||
"node": ">=12"
|
|
||||||
}
|
|
||||||
},
|
|
||||||
"node_modules/d3-interpolate": {
|
|
||||||
"version": "3.0.1",
|
|
||||||
"resolved": "https://registry.npmjs.org/d3-interpolate/-/d3-interpolate-3.0.1.tgz",
|
|
||||||
"integrity": "sha512-3bYs1rOD33uo8aqJfKP3JWPAibgw8Zm2+L9vBKEHJ2Rg+viTR7o5Mmv5mZcieN+FRYaAOWX5SJATX6k1PWz72g==",
|
|
||||||
"dependencies": {
|
|
||||||
"d3-color": "1 - 3"
|
|
||||||
},
|
|
||||||
"engines": {
|
|
||||||
"node": ">=12"
|
|
||||||
}
|
|
||||||
},
|
|
||||||
"node_modules/d3-path": {
|
|
||||||
"version": "3.1.0",
|
|
||||||
"resolved": "https://registry.npmjs.org/d3-path/-/d3-path-3.1.0.tgz",
|
|
||||||
"integrity": "sha512-p3KP5HCf/bvjBSSKuXid6Zqijx7wIfNW+J/maPs+iwR35at5JCbLUT0LzF1cnjbCHWhqzQTIN2Jpe8pRebIEFQ==",
|
|
||||||
"engines": {
|
|
||||||
"node": ">=12"
|
|
||||||
}
|
|
||||||
},
|
|
||||||
"node_modules/d3-scale": {
|
|
||||||
"version": "4.0.2",
|
|
||||||
"resolved": "https://registry.npmjs.org/d3-scale/-/d3-scale-4.0.2.tgz",
|
|
||||||
"integrity": "sha512-GZW464g1SH7ag3Y7hXjf8RoUuAFIqklOAq3MRl4OaWabTFJY9PN/E1YklhXLh+OQ3fM9yS2nOkCoS+WLZ6kvxQ==",
|
|
||||||
"dependencies": {
|
|
||||||
"d3-array": "2.10.0 - 3",
|
|
||||||
"d3-format": "1 - 3",
|
|
||||||
"d3-interpolate": "1.2.0 - 3",
|
|
||||||
"d3-time": "2.1.1 - 3",
|
|
||||||
"d3-time-format": "2 - 4"
|
|
||||||
},
|
|
||||||
"engines": {
|
|
||||||
"node": ">=12"
|
|
||||||
}
|
|
||||||
},
|
|
||||||
"node_modules/d3-shape": {
|
|
||||||
"version": "3.2.0",
|
|
||||||
"resolved": "https://registry.npmjs.org/d3-shape/-/d3-shape-3.2.0.tgz",
|
|
||||||
"integrity": "sha512-SaLBuwGm3MOViRq2ABk3eLoxwZELpH6zhl3FbAoJ7Vm1gofKx6El1Ib5z23NUEhF9AsGl7y+dzLe5Cw2AArGTA==",
|
|
||||||
"dependencies": {
|
|
||||||
"d3-path": "^3.1.0"
|
|
||||||
},
|
|
||||||
"engines": {
|
|
||||||
"node": ">=12"
|
|
||||||
}
|
|
||||||
},
|
|
||||||
"node_modules/d3-time": {
|
|
||||||
"version": "3.1.0",
|
|
||||||
"resolved": "https://registry.npmjs.org/d3-time/-/d3-time-3.1.0.tgz",
|
|
||||||
"integrity": "sha512-VqKjzBLejbSMT4IgbmVgDjpkYrNWUYJnbCGo874u7MMKIWsILRX+OpX/gTk8MqjpT1A/c6HY2dCA77ZN0lkQ2Q==",
|
|
||||||
"dependencies": {
|
|
||||||
"d3-array": "2 - 3"
|
|
||||||
},
|
|
||||||
"engines": {
|
|
||||||
"node": ">=12"
|
|
||||||
}
|
|
||||||
},
|
|
||||||
"node_modules/d3-time-format": {
|
|
||||||
"version": "4.1.0",
|
|
||||||
"resolved": "https://registry.npmjs.org/d3-time-format/-/d3-time-format-4.1.0.tgz",
|
|
||||||
"integrity": "sha512-dJxPBlzC7NugB2PDLwo9Q8JiTR3M3e4/XANkreKSUxF8vvXKqm1Yfq4Q5dl8budlunRVlUUaDUgFt7eA8D6NLg==",
|
|
||||||
"dependencies": {
|
|
||||||
"d3-time": "1 - 3"
|
|
||||||
},
|
|
||||||
"engines": {
|
|
||||||
"node": ">=12"
|
|
||||||
}
|
|
||||||
},
|
|
||||||
"node_modules/d3-timer": {
|
|
||||||
"version": "3.0.1",
|
|
||||||
"resolved": "https://registry.npmjs.org/d3-timer/-/d3-timer-3.0.1.tgz",
|
|
||||||
"integrity": "sha512-ndfJ/JxxMd3nw31uyKoY2naivF+r29V+Lc0svZxe1JvvIRmi8hUsrMvdOwgS1o6uBHmiz91geQ0ylPP0aj1VUA==",
|
|
||||||
"engines": {
|
|
||||||
"node": ">=12"
|
|
||||||
}
|
|
||||||
},
|
|
||||||
"node_modules/decimal.js-light": {
|
|
||||||
"version": "2.5.1",
|
|
||||||
"resolved": "https://registry.npmjs.org/decimal.js-light/-/decimal.js-light-2.5.1.tgz",
|
|
||||||
"integrity": "sha512-qIMFpTMZmny+MMIitAB6D7iVPEorVw6YQRWkvarTkT4tBeSLLiHzcwj6q0MmYSFCiVpiqPJTJEYIrpcPzVEIvg=="
|
|
||||||
},
|
|
||||||
"node_modules/detect-libc": {
|
"node_modules/detect-libc": {
|
||||||
"version": "2.1.2",
|
"version": "2.1.2",
|
||||||
"resolved": "https://registry.npmjs.org/detect-libc/-/detect-libc-2.1.2.tgz",
|
"resolved": "https://registry.npmjs.org/detect-libc/-/detect-libc-2.1.2.tgz",
|
||||||
@@ -993,11 +793,6 @@
|
|||||||
"node": ">= 0.4"
|
"node": ">= 0.4"
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
"node_modules/es-toolkit": {
|
|
||||||
"version": "1.49.0",
|
|
||||||
"resolved": "https://registry.npmjs.org/es-toolkit/-/es-toolkit-1.49.0.tgz",
|
|
||||||
"integrity": "sha512-G5iZ6Pc/FNRY/soKZHC+TxGDD83rHUDXxzaWhGCX44vAv/tMs56WMusnm/KMNK+luUPsgA9U28cGr4RDlSzL2g=="
|
|
||||||
},
|
|
||||||
"node_modules/escalade": {
|
"node_modules/escalade": {
|
||||||
"version": "3.2.0",
|
"version": "3.2.0",
|
||||||
"resolved": "https://registry.npmjs.org/escalade/-/escalade-3.2.0.tgz",
|
"resolved": "https://registry.npmjs.org/escalade/-/escalade-3.2.0.tgz",
|
||||||
@@ -1008,11 +803,6 @@
|
|||||||
"node": ">=6"
|
"node": ">=6"
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
"node_modules/eventemitter3": {
|
|
||||||
"version": "5.0.4",
|
|
||||||
"resolved": "https://registry.npmjs.org/eventemitter3/-/eventemitter3-5.0.4.tgz",
|
|
||||||
"integrity": "sha512-mlsTRyGaPBjPedk6Bvw+aqbsXDtoAyAzm5MO7JgU+yVRyMQ5O8bD4Kcci7BS85f93veegeCPkL8R4GLClnjLFw=="
|
|
||||||
},
|
|
||||||
"node_modules/fast-glob": {
|
"node_modules/fast-glob": {
|
||||||
"version": "3.3.3",
|
"version": "3.3.3",
|
||||||
"resolved": "https://registry.npmjs.org/fast-glob/-/fast-glob-3.3.3.tgz",
|
"resolved": "https://registry.npmjs.org/fast-glob/-/fast-glob-3.3.3.tgz",
|
||||||
@@ -1131,23 +921,6 @@
|
|||||||
"node": ">= 0.4"
|
"node": ">= 0.4"
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
"node_modules/immer": {
|
|
||||||
"version": "11.1.15",
|
|
||||||
"resolved": "https://registry.npmjs.org/immer/-/immer-11.1.15.tgz",
|
|
||||||
"integrity": "sha512-VrNANlmnWQnh5COXIIOQXM9oOJw7naGKlBT74ZOOR6lpVXc3gFEu9FJLDFcpCJ2j+NWr8TIwtWD//T6ZX6TKiQ==",
|
|
||||||
"funding": {
|
|
||||||
"type": "opencollective",
|
|
||||||
"url": "https://opencollective.com/immer"
|
|
||||||
}
|
|
||||||
},
|
|
||||||
"node_modules/internmap": {
|
|
||||||
"version": "2.0.3",
|
|
||||||
"resolved": "https://registry.npmjs.org/internmap/-/internmap-2.0.3.tgz",
|
|
||||||
"integrity": "sha512-5Hh7Y1wQbvY5ooGgPbDaL5iYLAPzMTUrjMulskHLH6wnv/A+1q5rgEaiuqEjB+oxGXIVZs1FF+R/KPN3ZSQYYg==",
|
|
||||||
"engines": {
|
|
||||||
"node": ">=12"
|
|
||||||
}
|
|
||||||
},
|
|
||||||
"node_modules/is-binary-path": {
|
"node_modules/is-binary-path": {
|
||||||
"version": "2.1.0",
|
"version": "2.1.0",
|
||||||
"resolved": "https://registry.npmjs.org/is-binary-path/-/is-binary-path-2.1.0.tgz",
|
"resolved": "https://registry.npmjs.org/is-binary-path/-/is-binary-path-2.1.0.tgz",
|
||||||
@@ -1363,6 +1136,9 @@
|
|||||||
"arm64"
|
"arm64"
|
||||||
],
|
],
|
||||||
"dev": true,
|
"dev": true,
|
||||||
|
"libc": [
|
||||||
|
"glibc"
|
||||||
|
],
|
||||||
"license": "MPL-2.0",
|
"license": "MPL-2.0",
|
||||||
"optional": true,
|
"optional": true,
|
||||||
"os": [
|
"os": [
|
||||||
@@ -1384,6 +1160,9 @@
|
|||||||
"arm64"
|
"arm64"
|
||||||
],
|
],
|
||||||
"dev": true,
|
"dev": true,
|
||||||
|
"libc": [
|
||||||
|
"musl"
|
||||||
|
],
|
||||||
"license": "MPL-2.0",
|
"license": "MPL-2.0",
|
||||||
"optional": true,
|
"optional": true,
|
||||||
"os": [
|
"os": [
|
||||||
@@ -1405,6 +1184,9 @@
|
|||||||
"x64"
|
"x64"
|
||||||
],
|
],
|
||||||
"dev": true,
|
"dev": true,
|
||||||
|
"libc": [
|
||||||
|
"glibc"
|
||||||
|
],
|
||||||
"license": "MPL-2.0",
|
"license": "MPL-2.0",
|
||||||
"optional": true,
|
"optional": true,
|
||||||
"os": [
|
"os": [
|
||||||
@@ -1426,6 +1208,9 @@
|
|||||||
"x64"
|
"x64"
|
||||||
],
|
],
|
||||||
"dev": true,
|
"dev": true,
|
||||||
|
"libc": [
|
||||||
|
"musl"
|
||||||
|
],
|
||||||
"license": "MPL-2.0",
|
"license": "MPL-2.0",
|
||||||
"optional": true,
|
"optional": true,
|
||||||
"os": [
|
"os": [
|
||||||
@@ -1854,34 +1639,6 @@
|
|||||||
"react": "^19.2.7"
|
"react": "^19.2.7"
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
"node_modules/react-is": {
|
|
||||||
"version": "19.2.8",
|
|
||||||
"resolved": "https://registry.npmjs.org/react-is/-/react-is-19.2.8.tgz",
|
|
||||||
"integrity": "sha512-s5un28nYxKJw5gvUHyW5PCC28CvBqLu9r3cWgzHT4Vo/5fqqkFcdRYsGcKf50WMPpjjFZS5d76fn3YCo2njKwQ==",
|
|
||||||
"peer": true
|
|
||||||
},
|
|
||||||
"node_modules/react-redux": {
|
|
||||||
"version": "9.3.0",
|
|
||||||
"resolved": "https://registry.npmjs.org/react-redux/-/react-redux-9.3.0.tgz",
|
|
||||||
"integrity": "sha512-KQopgqFo/p/fgmAs5qz6p5RWaNAzq40WAu7fJIXnQpYxFPbJYtsJPWvGeF2rOBaY/kEuV77AVsX8TsQzKm+A/g==",
|
|
||||||
"dependencies": {
|
|
||||||
"@types/use-sync-external-store": "^0.0.6",
|
|
||||||
"use-sync-external-store": "^1.4.0"
|
|
||||||
},
|
|
||||||
"peerDependencies": {
|
|
||||||
"@types/react": "^18.2.25 || ^19",
|
|
||||||
"react": "^18.0 || ^19",
|
|
||||||
"redux": "^5.0.0"
|
|
||||||
},
|
|
||||||
"peerDependenciesMeta": {
|
|
||||||
"@types/react": {
|
|
||||||
"optional": true
|
|
||||||
},
|
|
||||||
"redux": {
|
|
||||||
"optional": true
|
|
||||||
}
|
|
||||||
}
|
|
||||||
},
|
|
||||||
"node_modules/read-cache": {
|
"node_modules/read-cache": {
|
||||||
"version": "1.0.0",
|
"version": "1.0.0",
|
||||||
"resolved": "https://registry.npmjs.org/read-cache/-/read-cache-1.0.0.tgz",
|
"resolved": "https://registry.npmjs.org/read-cache/-/read-cache-1.0.0.tgz",
|
||||||
@@ -1905,45 +1662,6 @@
|
|||||||
"node": ">=8.10.0"
|
"node": ">=8.10.0"
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
"node_modules/recharts": {
|
|
||||||
"version": "3.10.0",
|
|
||||||
"resolved": "https://registry.npmjs.org/recharts/-/recharts-3.10.0.tgz",
|
|
||||||
"integrity": "sha512-wulMvfncpIlmu2uFtRU/mE5/+NiVtASXkw2KdwJTdHs3WsASX0WxZlX+rpKgyn5BDbIhkPtCpUKkB9XNK5KE0w==",
|
|
||||||
"dependencies": {
|
|
||||||
"@reduxjs/toolkit": "^1.9.0 || 2.x.x",
|
|
||||||
"clsx": "^2.1.1",
|
|
||||||
"decimal.js-light": "^2.5.1",
|
|
||||||
"es-toolkit": "^1.39.3",
|
|
||||||
"eventemitter3": "^5.0.1",
|
|
||||||
"immer": "^11.1.8",
|
|
||||||
"react-redux": "8.x.x || 9.x.x",
|
|
||||||
"reselect": "5.2.0",
|
|
||||||
"tiny-invariant": "^1.3.3",
|
|
||||||
"use-sync-external-store": "^1.2.2",
|
|
||||||
"victory-vendor": "^37.0.2"
|
|
||||||
},
|
|
||||||
"engines": {
|
|
||||||
"node": ">=18"
|
|
||||||
},
|
|
||||||
"peerDependencies": {
|
|
||||||
"react": "^16.8.0 || ^17.0.0 || ^18.0.0 || ^19.0.0",
|
|
||||||
"react-dom": "^16.0.0 || ^17.0.0 || ^18.0.0 || ^19.0.0",
|
|
||||||
"react-is": "^16.8.0 || ^17.0.0 || ^18.0.0 || ^19.0.0"
|
|
||||||
}
|
|
||||||
},
|
|
||||||
"node_modules/redux": {
|
|
||||||
"version": "5.0.1",
|
|
||||||
"resolved": "https://registry.npmjs.org/redux/-/redux-5.0.1.tgz",
|
|
||||||
"integrity": "sha512-M9/ELqF6fy8FwmkpnF0S3YKOqMyoWJ4+CS5Efg2ct3oY9daQvd/Pc71FpGZsVsbl3Cpb+IIcjBDUnnyBdQbq4w=="
|
|
||||||
},
|
|
||||||
"node_modules/redux-thunk": {
|
|
||||||
"version": "3.1.0",
|
|
||||||
"resolved": "https://registry.npmjs.org/redux-thunk/-/redux-thunk-3.1.0.tgz",
|
|
||||||
"integrity": "sha512-NW2r5T6ksUKXCabzhL9z+h206HQw/NJkcLm1GPImRQ8IzfXwRGqjVhKJGauHirT0DAuyy6hjdnMZaRoAcy0Klw==",
|
|
||||||
"peerDependencies": {
|
|
||||||
"redux": "^5.0.0"
|
|
||||||
}
|
|
||||||
},
|
|
||||||
"node_modules/regexparam": {
|
"node_modules/regexparam": {
|
||||||
"version": "3.0.0",
|
"version": "3.0.0",
|
||||||
"resolved": "https://registry.npmjs.org/regexparam/-/regexparam-3.0.0.tgz",
|
"resolved": "https://registry.npmjs.org/regexparam/-/regexparam-3.0.0.tgz",
|
||||||
@@ -1953,11 +1671,6 @@
|
|||||||
"node": ">=8"
|
"node": ">=8"
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
"node_modules/reselect": {
|
|
||||||
"version": "5.2.0",
|
|
||||||
"resolved": "https://registry.npmjs.org/reselect/-/reselect-5.2.0.tgz",
|
|
||||||
"integrity": "sha512-AgZ3UOZm3YndfrJ4OYjgrT7bmCm/1iqkjvEfH/oYjzh6PD2qw4QuT3jjnXIrpdt4MTpMXclMT3lXbmRY+XRakw=="
|
|
||||||
},
|
|
||||||
"node_modules/resolve": {
|
"node_modules/resolve": {
|
||||||
"version": "1.22.12",
|
"version": "1.22.12",
|
||||||
"resolved": "https://registry.npmjs.org/resolve/-/resolve-1.22.12.tgz",
|
"resolved": "https://registry.npmjs.org/resolve/-/resolve-1.22.12.tgz",
|
||||||
@@ -2162,11 +1875,6 @@
|
|||||||
"node": ">=0.8"
|
"node": ">=0.8"
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
"node_modules/tiny-invariant": {
|
|
||||||
"version": "1.3.3",
|
|
||||||
"resolved": "https://registry.npmjs.org/tiny-invariant/-/tiny-invariant-1.3.3.tgz",
|
|
||||||
"integrity": "sha512-+FbBPE1o9QAYvviau/qC5SE3caw21q3xkvWKBtja5vgqOWIHHJ3ioaq1VPfn/Szqctz2bU/oYeKd9/z5BL+PVg=="
|
|
||||||
},
|
|
||||||
"node_modules/tinyglobby": {
|
"node_modules/tinyglobby": {
|
||||||
"version": "0.2.17",
|
"version": "0.2.17",
|
||||||
"resolved": "https://registry.npmjs.org/tinyglobby/-/tinyglobby-0.2.17.tgz",
|
"resolved": "https://registry.npmjs.org/tinyglobby/-/tinyglobby-0.2.17.tgz",
|
||||||
@@ -2304,27 +2012,6 @@
|
|||||||
"dev": true,
|
"dev": true,
|
||||||
"license": "MIT"
|
"license": "MIT"
|
||||||
},
|
},
|
||||||
"node_modules/victory-vendor": {
|
|
||||||
"version": "37.3.6",
|
|
||||||
"resolved": "https://registry.npmjs.org/victory-vendor/-/victory-vendor-37.3.6.tgz",
|
|
||||||
"integrity": "sha512-SbPDPdDBYp+5MJHhBCAyI7wKM3d5ivekigc2Dk2s7pgbZ9wIgIBYGVw4zGHBml/qTFbexrofXW6Gu4noGxrOwQ==",
|
|
||||||
"dependencies": {
|
|
||||||
"@types/d3-array": "^3.0.3",
|
|
||||||
"@types/d3-ease": "^3.0.0",
|
|
||||||
"@types/d3-interpolate": "^3.0.1",
|
|
||||||
"@types/d3-scale": "^4.0.2",
|
|
||||||
"@types/d3-shape": "^3.1.0",
|
|
||||||
"@types/d3-time": "^3.0.0",
|
|
||||||
"@types/d3-timer": "^3.0.0",
|
|
||||||
"d3-array": "^3.1.6",
|
|
||||||
"d3-ease": "^3.0.1",
|
|
||||||
"d3-interpolate": "^3.0.1",
|
|
||||||
"d3-scale": "^4.0.2",
|
|
||||||
"d3-shape": "^3.1.0",
|
|
||||||
"d3-time": "^3.0.0",
|
|
||||||
"d3-timer": "^3.0.1"
|
|
||||||
}
|
|
||||||
},
|
|
||||||
"node_modules/vite": {
|
"node_modules/vite": {
|
||||||
"version": "8.1.4",
|
"version": "8.1.4",
|
||||||
"resolved": "https://registry.npmjs.org/vite/-/vite-8.1.4.tgz",
|
"resolved": "https://registry.npmjs.org/vite/-/vite-8.1.4.tgz",
|
||||||
|
|||||||
@@ -11,7 +11,6 @@
|
|||||||
"dependencies": {
|
"dependencies": {
|
||||||
"react": "^19.2.6",
|
"react": "^19.2.6",
|
||||||
"react-dom": "^19.2.6",
|
"react-dom": "^19.2.6",
|
||||||
"recharts": "^3.10.0",
|
|
||||||
"wouter": "^3.7.1"
|
"wouter": "^3.7.1"
|
||||||
},
|
},
|
||||||
"devDependencies": {
|
"devDependencies": {
|
||||||
|
|||||||
@@ -1,36 +0,0 @@
|
|||||||
import { jsx as _jsx, jsxs as _jsxs } from "react/jsx-runtime";
|
|
||||||
import { useEffect, useState } from 'react';
|
|
||||||
import { api, fmtBytes, fmtAge } from '../lib/api';
|
|
||||||
function kindLabel(kind) {
|
|
||||||
if (kind === 'gc_orphan')
|
|
||||||
return 'Orphan GC';
|
|
||||||
if (kind === 'gc_eviction_scored')
|
|
||||||
return 'Scored Eviction';
|
|
||||||
if (kind === 'scrub')
|
|
||||||
return 'Scrub';
|
|
||||||
return kind;
|
|
||||||
}
|
|
||||||
function kindColor(kind) {
|
|
||||||
if (kind === 'gc_eviction_scored')
|
|
||||||
return 'text-amber-300';
|
|
||||||
if (kind === 'scrub')
|
|
||||||
return 'text-blue-400';
|
|
||||||
return 'text-slate-300';
|
|
||||||
}
|
|
||||||
export function MaintenancePanel() {
|
|
||||||
const [rows, setRows] = useState([]);
|
|
||||||
const [err, setErr] = useState(null);
|
|
||||||
const [loading, setLoading] = useState(true);
|
|
||||||
useEffect(() => {
|
|
||||||
api
|
|
||||||
.maintenance()
|
|
||||||
.then((r) => { setRows(r); setErr(null); })
|
|
||||||
.catch((e) => setErr(String(e)))
|
|
||||||
.finally(() => setLoading(false));
|
|
||||||
}, []);
|
|
||||||
return (_jsxs("div", { className: "rounded border border-slate-800 bg-slate-900/40 p-4", children: [_jsx("h2", { className: "text-sm font-semibold text-slate-300 uppercase tracking-wider mb-3", children: "GC / Maintenance Log" }), loading && _jsx("div", { className: "text-slate-500 text-sm", children: "Loading\u2026" }), err && _jsx("div", { className: "text-amber-400 text-sm", children: err }), !loading && !err && rows.length === 0 && (_jsx("div", { className: "text-slate-500 text-sm italic", children: "No maintenance events recorded yet." })), rows.length > 0 && (_jsx("div", { className: "overflow-x-auto", children: _jsxs("table", { className: "w-full text-sm border-collapse", children: [_jsx("thead", { children: _jsxs("tr", { className: "text-left text-xs text-slate-500 uppercase tracking-wider border-b border-slate-800", children: [_jsx("th", { className: "pr-3 pb-2", children: "Age" }), _jsx("th", { className: "pr-3 pb-2", children: "Node" }), _jsx("th", { className: "pr-3 pb-2", children: "Type" }), _jsx("th", { className: "pr-3 pb-2 text-right", children: "Scanned" }), _jsx("th", { className: "pr-3 pb-2 text-right", children: "Removed" }), _jsx("th", { className: "pr-3 pb-2 text-right", children: "Reclaimed" }), _jsx("th", { className: "pb-2", children: "Status" })] }) }), _jsx("tbody", { children: rows.map((row, i) => (_jsxs("tr", { className: "border-b border-slate-800/50 hover:bg-slate-800/20", children: [_jsx("td", { className: "pr-3 py-1.5 text-slate-400 whitespace-nowrap", children: fmtAge(row.unix_ts) }), _jsx("td", { className: "pr-3 py-1.5 font-mono text-xs text-slate-300", children: row.node }), _jsx("td", { className: `pr-3 py-1.5 font-medium ${kindColor(row.kind)}`, children: kindLabel(row.kind) }), _jsx("td", { className: "pr-3 py-1.5 text-right text-slate-300", children: row.chunks_scanned.toLocaleString() }), _jsx("td", { className: "pr-3 py-1.5 text-right text-slate-300", children: row.chunks_removed > 0
|
|
||||||
? _jsx("span", { className: "text-amber-300", children: row.chunks_removed.toLocaleString() })
|
|
||||||
: _jsx("span", { className: "text-slate-600", children: "0" }) }), _jsx("td", { className: "pr-3 py-1.5 text-right text-slate-300", children: row.bytes_reclaimed > 0 ? fmtBytes(row.bytes_reclaimed) : '—' }), _jsx("td", { className: "py-1.5", children: row.error
|
|
||||||
? _jsx("span", { className: "text-red-400 text-xs", children: row.error })
|
|
||||||
: _jsx("span", { className: "text-emerald-500 text-xs", children: "ok" }) })] }, i))) })] }) }))] }));
|
|
||||||
}
|
|
||||||
@@ -1,93 +0,0 @@
|
|||||||
import { useEffect, useState } from 'react';
|
|
||||||
import { api, MaintenanceRow, fmtBytes, fmtAge } from '../lib/api';
|
|
||||||
|
|
||||||
function kindLabel(kind: string): string {
|
|
||||||
if (kind === 'gc_orphan') return 'Orphan GC';
|
|
||||||
if (kind === 'gc_eviction_scored') return 'Scored Eviction';
|
|
||||||
if (kind === 'scrub') return 'Scrub';
|
|
||||||
return kind;
|
|
||||||
}
|
|
||||||
|
|
||||||
function kindColor(kind: string): string {
|
|
||||||
if (kind === 'gc_eviction_scored') return 'text-amber-300';
|
|
||||||
if (kind === 'scrub') return 'text-blue-400';
|
|
||||||
return 'text-slate-300';
|
|
||||||
}
|
|
||||||
|
|
||||||
export function MaintenancePanel() {
|
|
||||||
const [rows, setRows] = useState<MaintenanceRow[]>([]);
|
|
||||||
const [err, setErr] = useState<string | null>(null);
|
|
||||||
const [loading, setLoading] = useState(true);
|
|
||||||
|
|
||||||
useEffect(() => {
|
|
||||||
api
|
|
||||||
.maintenance()
|
|
||||||
.then((r) => { setRows(r); setErr(null); })
|
|
||||||
.catch((e) => setErr(String(e)))
|
|
||||||
.finally(() => setLoading(false));
|
|
||||||
}, []);
|
|
||||||
|
|
||||||
return (
|
|
||||||
<div className="rounded border border-slate-800 bg-slate-900/40 p-4">
|
|
||||||
<h2 className="text-sm font-semibold text-slate-300 uppercase tracking-wider mb-3">
|
|
||||||
GC / Maintenance Log
|
|
||||||
</h2>
|
|
||||||
|
|
||||||
{loading && <div className="text-slate-500 text-sm">Loading…</div>}
|
|
||||||
{err && <div className="text-amber-400 text-sm">{err}</div>}
|
|
||||||
|
|
||||||
{!loading && !err && rows.length === 0 && (
|
|
||||||
<div className="text-slate-500 text-sm italic">No maintenance events recorded yet.</div>
|
|
||||||
)}
|
|
||||||
|
|
||||||
{rows.length > 0 && (
|
|
||||||
<div className="overflow-x-auto">
|
|
||||||
<table className="w-full text-sm border-collapse">
|
|
||||||
<thead>
|
|
||||||
<tr className="text-left text-xs text-slate-500 uppercase tracking-wider border-b border-slate-800">
|
|
||||||
<th className="pr-3 pb-2">Age</th>
|
|
||||||
<th className="pr-3 pb-2">Node</th>
|
|
||||||
<th className="pr-3 pb-2">Type</th>
|
|
||||||
<th className="pr-3 pb-2 text-right">Scanned</th>
|
|
||||||
<th className="pr-3 pb-2 text-right">Removed</th>
|
|
||||||
<th className="pr-3 pb-2 text-right">Reclaimed</th>
|
|
||||||
<th className="pb-2">Status</th>
|
|
||||||
</tr>
|
|
||||||
</thead>
|
|
||||||
<tbody>
|
|
||||||
{rows.map((row, i) => (
|
|
||||||
<tr key={i} className="border-b border-slate-800/50 hover:bg-slate-800/20">
|
|
||||||
<td className="pr-3 py-1.5 text-slate-400 whitespace-nowrap">
|
|
||||||
{fmtAge(row.unix_ts)}
|
|
||||||
</td>
|
|
||||||
<td className="pr-3 py-1.5 font-mono text-xs text-slate-300">
|
|
||||||
{row.node}
|
|
||||||
</td>
|
|
||||||
<td className={`pr-3 py-1.5 font-medium ${kindColor(row.kind)}`}>
|
|
||||||
{kindLabel(row.kind)}
|
|
||||||
</td>
|
|
||||||
<td className="pr-3 py-1.5 text-right text-slate-300">
|
|
||||||
{row.chunks_scanned.toLocaleString()}
|
|
||||||
</td>
|
|
||||||
<td className="pr-3 py-1.5 text-right text-slate-300">
|
|
||||||
{row.chunks_removed > 0
|
|
||||||
? <span className="text-amber-300">{row.chunks_removed.toLocaleString()}</span>
|
|
||||||
: <span className="text-slate-600">0</span>}
|
|
||||||
</td>
|
|
||||||
<td className="pr-3 py-1.5 text-right text-slate-300">
|
|
||||||
{row.bytes_reclaimed > 0 ? fmtBytes(row.bytes_reclaimed) : '—'}
|
|
||||||
</td>
|
|
||||||
<td className="py-1.5">
|
|
||||||
{row.error
|
|
||||||
? <span className="text-red-400 text-xs">{row.error}</span>
|
|
||||||
: <span className="text-emerald-500 text-xs">ok</span>}
|
|
||||||
</td>
|
|
||||||
</tr>
|
|
||||||
))}
|
|
||||||
</tbody>
|
|
||||||
</table>
|
|
||||||
</div>
|
|
||||||
)}
|
|
||||||
</div>
|
|
||||||
);
|
|
||||||
}
|
|
||||||
@@ -1,12 +1,10 @@
|
|||||||
import { jsx as _jsx, jsxs as _jsxs, Fragment as _Fragment } from "react/jsx-runtime";
|
import { jsx as _jsx, jsxs as _jsxs, Fragment as _Fragment } from "react/jsx-runtime";
|
||||||
import { Link } from 'wouter';
|
import { Link } from 'wouter';
|
||||||
import { fmtBytes, fmtUptime } from '../lib/api';
|
|
||||||
import { StorageBar } from './StorageBar';
|
import { StorageBar } from './StorageBar';
|
||||||
import { NodeHistorySparklines } from './NodeHistorySparklines';
|
|
||||||
/// Human-oriented node card for the FleetHealth landing.
|
/// Human-oriented node card for the FleetHealth landing.
|
||||||
/// Shows: overall health traffic-light, storage bars, mount state,
|
/// Shows: overall health traffic-light, storage bars, mount state,
|
||||||
/// cache hit rate, next scheduled job. No hex, no primitives.
|
/// cache hit rate, next scheduled job. No hex, no primitives.
|
||||||
export function NodeCard({ node, anomalyLevel }) {
|
export function NodeCard({ node }) {
|
||||||
const health = healthOf(node);
|
const health = healthOf(node);
|
||||||
const border = {
|
const border = {
|
||||||
ok: 'border-emerald-700 hover:border-emerald-500',
|
ok: 'border-emerald-700 hover:border-emerald-500',
|
||||||
@@ -24,40 +22,13 @@ export function NodeCard({ node, anomalyLevel }) {
|
|||||||
'block rounded-lg bg-slate-900 border transition-colors',
|
'block rounded-lg bg-slate-900 border transition-colors',
|
||||||
'p-5 space-y-4',
|
'p-5 space-y-4',
|
||||||
border,
|
border,
|
||||||
].join(' '), children: [_jsxs("div", { className: "flex items-center justify-between", children: [_jsxs("div", { className: "flex items-center gap-2", children: [_jsx("span", { className: `inline-block w-2.5 h-2.5 rounded-full ${dot}` }), _jsx("span", { className: "text-lg font-semibold text-slate-100", children: node.node_name }), anomalyLevel === 'alert' && (_jsx("span", { className: "text-xs font-mono px-1.5 py-0.5 rounded bg-red-900/60 text-red-300 border border-red-800", children: "anomaly" })), anomalyLevel === 'warn' && (_jsx("span", { className: "text-xs font-mono px-1.5 py-0.5 rounded bg-amber-900/60 text-amber-300 border border-amber-800", children: "drift" }))] }), _jsx("span", { className: "text-xs text-slate-500 font-mono", children: node.zone || '—' })] }), !node.online && (_jsx("div", { className: "text-sm text-red-400 break-words", children: node.error ?? 'offline' })), node.online && (_jsxs(_Fragment, { children: [node.filesystem && (_jsx(StorageBar, { label: node.filesystem.available_bytes != null
|
].join(' '), children: [_jsxs("div", { className: "flex items-center justify-between", children: [_jsxs("div", { className: "flex items-center gap-2", children: [_jsx("span", { className: `inline-block w-2.5 h-2.5 rounded-full ${dot}` }), _jsx("span", { className: "text-lg font-semibold text-slate-100", children: node.node_name })] }), _jsx("span", { className: "text-xs text-slate-500 font-mono", children: node.zone || '—' })] }), !node.online && (_jsx("div", { className: "text-sm text-red-400 break-words", children: node.error ?? 'offline' })), node.online && (_jsxs(_Fragment, { children: [node.filesystem && (_jsx(StorageBar, { label: "disk", used: node.filesystem.used_bytes, total: node.filesystem.total_bytes })), node.hot && node.hot.max_bytes > 0 && (_jsx(StorageBar, { label: "hot tier", used: node.hot.used_bytes, total: node.hot.max_bytes, pinned: node.hot.pinned_bytes ?? undefined })), _jsxs("div", { className: "grid grid-cols-2 gap-y-1 text-sm", children: [_jsx("span", { className: "text-slate-500", children: "mount" }), _jsx("span", { className: "text-right font-mono text-xs", children: node.mount?.active ? (_jsx("span", { className: "text-emerald-300", children: "\u2713 mounted" })) : (_jsx("span", { className: "text-slate-500", children: "not mounted" })) }), _jsx("span", { className: "text-slate-500", children: "cache hit rate" }), _jsx("span", { className: "text-right font-mono text-xs", children: node.cache && node.cache.hits + node.cache.misses > 0
|
||||||
? `disk — ${fmtBytes(node.filesystem.available_bytes)} free`
|
|
||||||
: 'disk', used: node.filesystem.used_bytes, total: node.filesystem.total_bytes })), node.hot && node.hot.max_bytes > 0 && (_jsx(StorageBar, { label: "hot tier", used: node.hot.used_bytes, total: node.hot.max_bytes, pinned: node.hot.pinned_bytes ?? undefined })), _jsxs("div", { className: "grid grid-cols-2 gap-y-1 text-sm", children: [_jsx("span", { className: "text-slate-500", children: "uptime" }), _jsx("span", { className: "text-right font-mono text-xs", children: node.daemon_started_unix
|
|
||||||
? _jsx("span", { className: "text-slate-300", children: fmtUptime(node.daemon_started_unix) })
|
|
||||||
: _jsx("span", { className: "text-slate-500", children: "\u2014" }) }), _jsx("span", { className: "text-slate-500", children: "mount" }), _jsx("span", { className: "text-right font-mono text-xs", children: node.mount?.active ? (_jsx("span", { className: "text-emerald-300", children: "\u2713 mounted" })) : (_jsx("span", { className: "text-slate-500", children: "not mounted" })) }), _jsx("span", { className: "text-slate-500", children: "cache hit rate" }), _jsx("span", { className: "text-right font-mono text-xs", children: node.cache && node.cache.hits + node.cache.misses > 0
|
|
||||||
? `${Math.round(node.cache.hit_rate * 100)}%`
|
? `${Math.round(node.cache.hit_rate * 100)}%`
|
||||||
: _jsx("span", { className: "text-slate-500", children: "idle" }) }), dedupRate(node) !== null && (_jsxs(_Fragment, { children: [_jsx("span", { className: "text-slate-500", children: "dedup efficiency" }), _jsxs("span", { className: "text-right font-mono text-xs text-slate-300", children: [dedupRate(node), "% chunks skipped"] })] })), node.cpu_load_1m != null && (_jsxs(_Fragment, { children: [_jsx("span", { className: "text-slate-500", children: "cpu load" }), _jsx("span", { className: [
|
: _jsx("span", { className: "text-slate-500", children: "idle" }) }), _jsx("span", { className: "text-slate-500", children: "next scheduled job" }), _jsx("span", { className: "text-right font-mono text-xs", children: nextTimer(node) })] })] })), _jsx("div", { className: "pt-1 border-t border-slate-800 text-xs text-slate-500", children: "view detail \u00B7 maintenance & shutdown prep \u2192" })] }) }));
|
||||||
'text-right font-mono text-xs',
|
|
||||||
node.cpu_load_1m > 4 ? 'text-amber-300' : 'text-slate-300',
|
|
||||||
].join(' '), children: node.cpu_load_1m.toFixed(2) })] })), _jsx("span", { className: "text-slate-500", children: "next scheduled job" }), _jsx("span", { className: "text-right font-mono text-xs", children: nextTimer(node) })] }), failedTimers(node).length > 0 && (_jsx("div", { className: "space-y-0.5", children: failedTimers(node).map((t) => (_jsxs("div", { className: "text-xs text-amber-400 font-mono truncate", children: ["\u26A0 ", t.unit.replace(/^clawstor-/, '').replace(/\.timer$/, ''), ": ", t.last_result] }, t.unit))) })), _jsx(NodeHistorySparklines, { nodeName: node.node_name, hotMaxBytes: node.hot?.max_bytes ?? 0 })] }))] }) }));
|
|
||||||
}
|
|
||||||
/** HasChunk dedup efficiency as an integer percent, or null if no data. */
|
|
||||||
function dedupRate(n) {
|
|
||||||
const c = n.cache;
|
|
||||||
if (!c)
|
|
||||||
return null;
|
|
||||||
const total = (c.has_chunk_hits ?? 0) + (c.has_chunk_misses ?? 0);
|
|
||||||
if (total === 0)
|
|
||||||
return null;
|
|
||||||
return Math.round(((c.has_chunk_hits ?? 0) / total) * 100);
|
|
||||||
}
|
|
||||||
/** Timers whose last_result is a non-success string. */
|
|
||||||
function failedTimers(n) {
|
|
||||||
return n.timers.filter((t) => t.last_result && t.last_result !== 'success');
|
|
||||||
}
|
}
|
||||||
function healthOf(n) {
|
function healthOf(n) {
|
||||||
if (!n.online)
|
if (!n.online)
|
||||||
return 'err';
|
return 'err';
|
||||||
// Server-side health_score already accounts for hot/fs/cpu/timers.
|
|
||||||
if (n.health_score >= 3)
|
|
||||||
return 'err';
|
|
||||||
if (n.health_score >= 1)
|
|
||||||
return 'warn';
|
|
||||||
// Client-side fallback for older daemons that don't send health_score.
|
|
||||||
const fsPct = n.filesystem
|
const fsPct = n.filesystem
|
||||||
? n.filesystem.used_bytes / Math.max(n.filesystem.total_bytes, 1)
|
? n.filesystem.used_bytes / Math.max(n.filesystem.total_bytes, 1)
|
||||||
: 0;
|
: 0;
|
||||||
|
|||||||
@@ -1,17 +1,15 @@
|
|||||||
import { Link } from 'wouter';
|
import { Link } from 'wouter';
|
||||||
import { NodeStatusV2, fmtBytes, fmtUptime } from '../lib/api';
|
import { NodeStatusV2 } from '../lib/api';
|
||||||
import { StorageBar } from './StorageBar';
|
import { StorageBar } from './StorageBar';
|
||||||
import { NodeHistorySparklines } from './NodeHistorySparklines';
|
|
||||||
|
|
||||||
interface Props {
|
interface Props {
|
||||||
node: NodeStatusV2;
|
node: NodeStatusV2;
|
||||||
anomalyLevel?: 'ok' | 'warn' | 'alert';
|
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Human-oriented node card for the FleetHealth landing.
|
/// Human-oriented node card for the FleetHealth landing.
|
||||||
/// Shows: overall health traffic-light, storage bars, mount state,
|
/// Shows: overall health traffic-light, storage bars, mount state,
|
||||||
/// cache hit rate, next scheduled job. No hex, no primitives.
|
/// cache hit rate, next scheduled job. No hex, no primitives.
|
||||||
export function NodeCard({ node, anomalyLevel }: Props) {
|
export function NodeCard({ node }: Props) {
|
||||||
const health = healthOf(node);
|
const health = healthOf(node);
|
||||||
const border = {
|
const border = {
|
||||||
ok: 'border-emerald-700 hover:border-emerald-500',
|
ok: 'border-emerald-700 hover:border-emerald-500',
|
||||||
@@ -42,16 +40,6 @@ export function NodeCard({ node, anomalyLevel }: Props) {
|
|||||||
<span className="text-lg font-semibold text-slate-100">
|
<span className="text-lg font-semibold text-slate-100">
|
||||||
{node.node_name}
|
{node.node_name}
|
||||||
</span>
|
</span>
|
||||||
{anomalyLevel === 'alert' && (
|
|
||||||
<span className="text-xs font-mono px-1.5 py-0.5 rounded bg-red-900/60 text-red-300 border border-red-800">
|
|
||||||
anomaly
|
|
||||||
</span>
|
|
||||||
)}
|
|
||||||
{anomalyLevel === 'warn' && (
|
|
||||||
<span className="text-xs font-mono px-1.5 py-0.5 rounded bg-amber-900/60 text-amber-300 border border-amber-800">
|
|
||||||
drift
|
|
||||||
</span>
|
|
||||||
)}
|
|
||||||
</div>
|
</div>
|
||||||
<span className="text-xs text-slate-500 font-mono">
|
<span className="text-xs text-slate-500 font-mono">
|
||||||
{node.zone || '—'}
|
{node.zone || '—'}
|
||||||
@@ -70,11 +58,7 @@ export function NodeCard({ node, anomalyLevel }: Props) {
|
|||||||
{/* Storage bars */}
|
{/* Storage bars */}
|
||||||
{node.filesystem && (
|
{node.filesystem && (
|
||||||
<StorageBar
|
<StorageBar
|
||||||
label={
|
label="disk"
|
||||||
node.filesystem.available_bytes != null
|
|
||||||
? `disk — ${fmtBytes(node.filesystem.available_bytes)} free`
|
|
||||||
: 'disk'
|
|
||||||
}
|
|
||||||
used={node.filesystem.used_bytes}
|
used={node.filesystem.used_bytes}
|
||||||
total={node.filesystem.total_bytes}
|
total={node.filesystem.total_bytes}
|
||||||
/>
|
/>
|
||||||
@@ -90,12 +74,6 @@ export function NodeCard({ node, anomalyLevel }: Props) {
|
|||||||
|
|
||||||
{/* One-liner facts */}
|
{/* One-liner facts */}
|
||||||
<div className="grid grid-cols-2 gap-y-1 text-sm">
|
<div className="grid grid-cols-2 gap-y-1 text-sm">
|
||||||
<span className="text-slate-500">uptime</span>
|
|
||||||
<span className="text-right font-mono text-xs">
|
|
||||||
{node.daemon_started_unix
|
|
||||||
? <span className="text-slate-300">{fmtUptime(node.daemon_started_unix)}</span>
|
|
||||||
: <span className="text-slate-500">—</span>}
|
|
||||||
</span>
|
|
||||||
<span className="text-slate-500">mount</span>
|
<span className="text-slate-500">mount</span>
|
||||||
<span className="text-right font-mono text-xs">
|
<span className="text-right font-mono text-xs">
|
||||||
{node.mount?.active ? (
|
{node.mount?.active ? (
|
||||||
@@ -110,76 +88,24 @@ export function NodeCard({ node, anomalyLevel }: Props) {
|
|||||||
? `${Math.round(node.cache.hit_rate * 100)}%`
|
? `${Math.round(node.cache.hit_rate * 100)}%`
|
||||||
: <span className="text-slate-500">idle</span>}
|
: <span className="text-slate-500">idle</span>}
|
||||||
</span>
|
</span>
|
||||||
{dedupRate(node) !== null && (
|
|
||||||
<>
|
|
||||||
<span className="text-slate-500">dedup efficiency</span>
|
|
||||||
<span className="text-right font-mono text-xs text-slate-300">
|
|
||||||
{dedupRate(node)}% chunks skipped
|
|
||||||
</span>
|
|
||||||
</>
|
|
||||||
)}
|
|
||||||
{node.cpu_load_1m != null && (
|
|
||||||
<>
|
|
||||||
<span className="text-slate-500">cpu load</span>
|
|
||||||
<span className={[
|
|
||||||
'text-right font-mono text-xs',
|
|
||||||
node.cpu_load_1m > 4 ? 'text-amber-300' : 'text-slate-300',
|
|
||||||
].join(' ')}>
|
|
||||||
{node.cpu_load_1m.toFixed(2)}
|
|
||||||
</span>
|
|
||||||
</>
|
|
||||||
)}
|
|
||||||
<span className="text-slate-500">next scheduled job</span>
|
<span className="text-slate-500">next scheduled job</span>
|
||||||
<span className="text-right font-mono text-xs">
|
<span className="text-right font-mono text-xs">
|
||||||
{nextTimer(node)}
|
{nextTimer(node)}
|
||||||
</span>
|
</span>
|
||||||
</div>
|
</div>
|
||||||
|
|
||||||
{/* Failed timer callout (T1.3) */}
|
|
||||||
{failedTimers(node).length > 0 && (
|
|
||||||
<div className="space-y-0.5">
|
|
||||||
{failedTimers(node).map((t) => (
|
|
||||||
<div key={t.unit} className="text-xs text-amber-400 font-mono truncate">
|
|
||||||
⚠ {t.unit.replace(/^clawstor-/, '').replace(/\.timer$/, '')}: {t.last_result}
|
|
||||||
</div>
|
|
||||||
))}
|
|
||||||
</div>
|
|
||||||
)}
|
|
||||||
|
|
||||||
{/* Sparklines (Phase B) — fetches its own 1h history */}
|
|
||||||
<NodeHistorySparklines
|
|
||||||
nodeName={node.node_name}
|
|
||||||
hotMaxBytes={node.hot?.max_bytes ?? 0}
|
|
||||||
/>
|
|
||||||
</>
|
</>
|
||||||
)}
|
)}
|
||||||
|
|
||||||
|
<div className="pt-1 border-t border-slate-800 text-xs text-slate-500">
|
||||||
|
view detail · maintenance & shutdown prep →
|
||||||
|
</div>
|
||||||
</a>
|
</a>
|
||||||
</Link>
|
</Link>
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
/** HasChunk dedup efficiency as an integer percent, or null if no data. */
|
|
||||||
function dedupRate(n: NodeStatusV2): number | null {
|
|
||||||
const c = n.cache;
|
|
||||||
if (!c) return null;
|
|
||||||
const total = (c.has_chunk_hits ?? 0) + (c.has_chunk_misses ?? 0);
|
|
||||||
if (total === 0) return null;
|
|
||||||
return Math.round(((c.has_chunk_hits ?? 0) / total) * 100);
|
|
||||||
}
|
|
||||||
|
|
||||||
/** Timers whose last_result is a non-success string. */
|
|
||||||
function failedTimers(n: NodeStatusV2) {
|
|
||||||
return n.timers.filter(
|
|
||||||
(t) => t.last_result && t.last_result !== 'success'
|
|
||||||
);
|
|
||||||
}
|
|
||||||
|
|
||||||
function healthOf(n: NodeStatusV2): 'ok' | 'warn' | 'err' | 'idle' {
|
function healthOf(n: NodeStatusV2): 'ok' | 'warn' | 'err' | 'idle' {
|
||||||
if (!n.online) return 'err';
|
if (!n.online) return 'err';
|
||||||
// Server-side health_score already accounts for hot/fs/cpu/timers.
|
|
||||||
if (n.health_score >= 3) return 'err';
|
|
||||||
if (n.health_score >= 1) return 'warn';
|
|
||||||
// Client-side fallback for older daemons that don't send health_score.
|
|
||||||
const fsPct = n.filesystem
|
const fsPct = n.filesystem
|
||||||
? n.filesystem.used_bytes / Math.max(n.filesystem.total_bytes, 1)
|
? n.filesystem.used_bytes / Math.max(n.filesystem.total_bytes, 1)
|
||||||
: 0;
|
: 0;
|
||||||
|
|||||||
@@ -1,37 +0,0 @@
|
|||||||
import { jsx as _jsx, jsxs as _jsxs } from "react/jsx-runtime";
|
|
||||||
import { useEffect, useState } from 'react';
|
|
||||||
import { AreaChart, Area, LineChart, Line, ResponsiveContainer, Tooltip, ReferenceLine, } from 'recharts';
|
|
||||||
import { api, fmtBytes } from '../lib/api';
|
|
||||||
// Compact tooltip for sparklines — shows a single value line.
|
|
||||||
function SparkTooltip({ active, payload, label: _label, formatter, }) {
|
|
||||||
if (!active || !payload?.length)
|
|
||||||
return null;
|
|
||||||
return (_jsx("div", { className: "rounded bg-slate-800 border border-slate-700 px-2 py-1 text-xs text-slate-200 shadow-lg", children: formatter(payload[0].value) }));
|
|
||||||
}
|
|
||||||
export function NodeHistorySparklines({ nodeName, hotMaxBytes }) {
|
|
||||||
const [samples, setSamples] = useState([]);
|
|
||||||
useEffect(() => {
|
|
||||||
let cancelled = false;
|
|
||||||
api
|
|
||||||
.metricsHistory(nodeName, 60)
|
|
||||||
.then((data) => {
|
|
||||||
if (!cancelled)
|
|
||||||
setSamples(data);
|
|
||||||
})
|
|
||||||
.catch(() => { });
|
|
||||||
return () => {
|
|
||||||
cancelled = true;
|
|
||||||
};
|
|
||||||
}, [nodeName]);
|
|
||||||
if (samples.length < 2)
|
|
||||||
return null;
|
|
||||||
// Recharts data arrays
|
|
||||||
const hotData = samples.map((s) => ({ t: s.unix_ts, v: s.hot_used_bytes }));
|
|
||||||
const hitData = samples.map((s) => ({
|
|
||||||
t: s.unix_ts,
|
|
||||||
v: Math.round(s.cache_hit_rate * 100),
|
|
||||||
}));
|
|
||||||
// Hot-tier ceiling reference line (90% of max = gc threshold)
|
|
||||||
const gcLine = hotMaxBytes > 0 ? hotMaxBytes * 0.9 : null;
|
|
||||||
return (_jsxs("div", { className: "space-y-3", children: [hotMaxBytes > 0 && (_jsxs("div", { children: [_jsxs("div", { className: "flex justify-between text-xs text-slate-500 mb-0.5", children: [_jsx("span", { children: "hot tier \u2014 1h history" }), _jsx("span", { children: fmtBytes(hotData[hotData.length - 1].v) })] }), _jsx(ResponsiveContainer, { width: "100%", height: 40, children: _jsxs(AreaChart, { data: hotData, margin: { top: 2, right: 0, bottom: 2, left: 0 }, children: [_jsx("defs", { children: _jsxs("linearGradient", { id: `hg-${nodeName}`, x1: "0", y1: "0", x2: "0", y2: "1", children: [_jsx("stop", { offset: "5%", stopColor: "#3b82f6", stopOpacity: 0.4 }), _jsx("stop", { offset: "95%", stopColor: "#3b82f6", stopOpacity: 0 })] }) }), gcLine && (_jsx(ReferenceLine, { y: gcLine, stroke: "#ef4444", strokeDasharray: "3 3", strokeWidth: 1 })), _jsx(Area, { type: "monotone", dataKey: "v", stroke: "#3b82f6", strokeWidth: 1.5, fill: `url(#hg-${nodeName})`, dot: false, isAnimationActive: false }), _jsx(Tooltip, { content: _jsx(SparkTooltip, { formatter: (v) => fmtBytes(v) }) })] }) })] })), _jsxs("div", { children: [_jsxs("div", { className: "flex justify-between text-xs text-slate-500 mb-0.5", children: [_jsx("span", { children: "cache hit rate \u2014 1h history" }), _jsxs("span", { children: [hitData[hitData.length - 1].v, "%"] })] }), _jsx(ResponsiveContainer, { width: "100%", height: 40, children: _jsxs(LineChart, { data: hitData, margin: { top: 2, right: 0, bottom: 2, left: 0 }, children: [_jsx(Line, { type: "monotone", dataKey: "v", stroke: "#10b981", strokeWidth: 1.5, dot: false, isAnimationActive: false }), _jsx(Tooltip, { content: _jsx(SparkTooltip, { formatter: (v) => `${v}%` }) })] }) })] })] }));
|
|
||||||
}
|
|
||||||
@@ -1,136 +0,0 @@
|
|||||||
import { useEffect, useState } from 'react';
|
|
||||||
import {
|
|
||||||
AreaChart,
|
|
||||||
Area,
|
|
||||||
LineChart,
|
|
||||||
Line,
|
|
||||||
ResponsiveContainer,
|
|
||||||
Tooltip,
|
|
||||||
ReferenceLine,
|
|
||||||
} from 'recharts';
|
|
||||||
import { api, MetricSample, fmtBytes } from '../lib/api';
|
|
||||||
|
|
||||||
interface Props {
|
|
||||||
nodeName: string;
|
|
||||||
hotMaxBytes: number;
|
|
||||||
}
|
|
||||||
|
|
||||||
// Compact tooltip for sparklines — shows a single value line.
|
|
||||||
function SparkTooltip({
|
|
||||||
active,
|
|
||||||
payload,
|
|
||||||
label: _label,
|
|
||||||
formatter,
|
|
||||||
}: {
|
|
||||||
active?: boolean;
|
|
||||||
payload?: { value: number }[];
|
|
||||||
label?: unknown;
|
|
||||||
formatter: (v: number) => string;
|
|
||||||
}) {
|
|
||||||
if (!active || !payload?.length) return null;
|
|
||||||
return (
|
|
||||||
<div className="rounded bg-slate-800 border border-slate-700 px-2 py-1 text-xs text-slate-200 shadow-lg">
|
|
||||||
{formatter(payload[0].value)}
|
|
||||||
</div>
|
|
||||||
);
|
|
||||||
}
|
|
||||||
|
|
||||||
export function NodeHistorySparklines({ nodeName, hotMaxBytes }: Props) {
|
|
||||||
const [samples, setSamples] = useState<MetricSample[]>([]);
|
|
||||||
|
|
||||||
useEffect(() => {
|
|
||||||
let cancelled = false;
|
|
||||||
api
|
|
||||||
.metricsHistory(nodeName, 60)
|
|
||||||
.then((data) => {
|
|
||||||
if (!cancelled) setSamples(data);
|
|
||||||
})
|
|
||||||
.catch(() => {});
|
|
||||||
return () => {
|
|
||||||
cancelled = true;
|
|
||||||
};
|
|
||||||
}, [nodeName]);
|
|
||||||
|
|
||||||
if (samples.length < 2) return null;
|
|
||||||
|
|
||||||
// Recharts data arrays
|
|
||||||
const hotData = samples.map((s) => ({ t: s.unix_ts, v: s.hot_used_bytes }));
|
|
||||||
const hitData = samples.map((s) => ({
|
|
||||||
t: s.unix_ts,
|
|
||||||
v: Math.round(s.cache_hit_rate * 100),
|
|
||||||
}));
|
|
||||||
|
|
||||||
// Hot-tier ceiling reference line (90% of max = gc threshold)
|
|
||||||
const gcLine = hotMaxBytes > 0 ? hotMaxBytes * 0.9 : null;
|
|
||||||
|
|
||||||
return (
|
|
||||||
<div className="space-y-3">
|
|
||||||
{/* Hot-tier usage sparkline */}
|
|
||||||
{hotMaxBytes > 0 && (
|
|
||||||
<div>
|
|
||||||
<div className="flex justify-between text-xs text-slate-500 mb-0.5">
|
|
||||||
<span>hot tier — 1h history</span>
|
|
||||||
<span>{fmtBytes(hotData[hotData.length - 1].v)}</span>
|
|
||||||
</div>
|
|
||||||
<ResponsiveContainer width="100%" height={40}>
|
|
||||||
<AreaChart data={hotData} margin={{ top: 2, right: 0, bottom: 2, left: 0 }}>
|
|
||||||
<defs>
|
|
||||||
<linearGradient id={`hg-${nodeName}`} x1="0" y1="0" x2="0" y2="1">
|
|
||||||
<stop offset="5%" stopColor="#3b82f6" stopOpacity={0.4} />
|
|
||||||
<stop offset="95%" stopColor="#3b82f6" stopOpacity={0} />
|
|
||||||
</linearGradient>
|
|
||||||
</defs>
|
|
||||||
{gcLine && (
|
|
||||||
<ReferenceLine
|
|
||||||
y={gcLine}
|
|
||||||
stroke="#ef4444"
|
|
||||||
strokeDasharray="3 3"
|
|
||||||
strokeWidth={1}
|
|
||||||
/>
|
|
||||||
)}
|
|
||||||
<Area
|
|
||||||
type="monotone"
|
|
||||||
dataKey="v"
|
|
||||||
stroke="#3b82f6"
|
|
||||||
strokeWidth={1.5}
|
|
||||||
fill={`url(#hg-${nodeName})`}
|
|
||||||
dot={false}
|
|
||||||
isAnimationActive={false}
|
|
||||||
/>
|
|
||||||
<Tooltip
|
|
||||||
content={
|
|
||||||
<SparkTooltip formatter={(v) => fmtBytes(v)} />
|
|
||||||
}
|
|
||||||
/>
|
|
||||||
</AreaChart>
|
|
||||||
</ResponsiveContainer>
|
|
||||||
</div>
|
|
||||||
)}
|
|
||||||
|
|
||||||
{/* Cache hit-rate sparkline */}
|
|
||||||
<div>
|
|
||||||
<div className="flex justify-between text-xs text-slate-500 mb-0.5">
|
|
||||||
<span>cache hit rate — 1h history</span>
|
|
||||||
<span>{hitData[hitData.length - 1].v}%</span>
|
|
||||||
</div>
|
|
||||||
<ResponsiveContainer width="100%" height={40}>
|
|
||||||
<LineChart data={hitData} margin={{ top: 2, right: 0, bottom: 2, left: 0 }}>
|
|
||||||
<Line
|
|
||||||
type="monotone"
|
|
||||||
dataKey="v"
|
|
||||||
stroke="#10b981"
|
|
||||||
strokeWidth={1.5}
|
|
||||||
dot={false}
|
|
||||||
isAnimationActive={false}
|
|
||||||
/>
|
|
||||||
<Tooltip
|
|
||||||
content={
|
|
||||||
<SparkTooltip formatter={(v) => `${v}%`} />
|
|
||||||
}
|
|
||||||
/>
|
|
||||||
</LineChart>
|
|
||||||
</ResponsiveContainer>
|
|
||||||
</div>
|
|
||||||
</div>
|
|
||||||
);
|
|
||||||
}
|
|
||||||
@@ -1,40 +0,0 @@
|
|||||||
import { jsx as _jsx, jsxs as _jsxs } from "react/jsx-runtime";
|
|
||||||
import { useEffect, useState } from 'react';
|
|
||||||
import { api, fmtBytes } from '../lib/api';
|
|
||||||
function ScoreBar({ score, max }) {
|
|
||||||
const pct = max > 0 ? Math.min(100, (score / max) * 100) : 0;
|
|
||||||
const color = pct > 66
|
|
||||||
? 'bg-red-500'
|
|
||||||
: pct > 33
|
|
||||||
? 'bg-amber-500'
|
|
||||||
: 'bg-emerald-500';
|
|
||||||
return (_jsx("div", { className: "w-20 h-1.5 bg-zinc-700 rounded-full overflow-hidden", children: _jsx("div", { className: `h-full rounded-full ${color}`, style: { width: `${pct}%` } }) }));
|
|
||||||
}
|
|
||||||
export function PollutionPanel() {
|
|
||||||
const [candidates, setCandidates] = useState([]);
|
|
||||||
const [loading, setLoading] = useState(true);
|
|
||||||
const [error, setError] = useState(null);
|
|
||||||
useEffect(() => {
|
|
||||||
api
|
|
||||||
.pollution()
|
|
||||||
.then((rows) => {
|
|
||||||
setCandidates(rows);
|
|
||||||
setLoading(false);
|
|
||||||
})
|
|
||||||
.catch((e) => {
|
|
||||||
setError(String(e));
|
|
||||||
setLoading(false);
|
|
||||||
});
|
|
||||||
}, []);
|
|
||||||
if (loading) {
|
|
||||||
return (_jsx("div", { className: "rounded-xl border border-zinc-700/50 bg-zinc-800/40 p-4 text-zinc-400 text-sm", children: "Loading pollution candidates\u2026" }));
|
|
||||||
}
|
|
||||||
if (error) {
|
|
||||||
return (_jsx("div", { className: "rounded-xl border border-red-700/40 bg-zinc-800/40 p-4 text-red-400 text-sm", children: error }));
|
|
||||||
}
|
|
||||||
if (candidates.length === 0) {
|
|
||||||
return (_jsxs("div", { className: "rounded-xl border border-zinc-700/50 bg-zinc-800/40 p-4", children: [_jsx("h2", { className: "text-sm font-semibold text-zinc-300 mb-1", children: "Cache Pollution" }), _jsx("p", { className: "text-zinc-500 text-sm", children: "No blobs tracked yet \u2014 pollution scores accumulate as builds run." })] }));
|
|
||||||
}
|
|
||||||
const maxScore = candidates[0]?.pollution_score ?? 1;
|
|
||||||
return (_jsxs("div", { className: "rounded-xl border border-zinc-700/50 bg-zinc-800/40 p-4", children: [_jsxs("div", { className: "flex items-baseline justify-between mb-3", children: [_jsxs("h2", { className: "text-sm font-semibold text-zinc-300", children: ["Cache Pollution", _jsx("span", { className: "ml-2 text-xs text-zinc-500 font-normal", children: "T2.8" })] }), _jsxs("span", { className: "text-xs text-zinc-500", children: [candidates.length, " entries"] })] }), _jsx("p", { className: "text-xs text-zinc-500 mb-3", children: "Large blobs with few recorded accesses since last restart. High-score entries consume hot-tier space disproportionate to their build value." }), _jsx("div", { className: "overflow-x-auto", children: _jsxs("table", { className: "w-full text-xs text-left", children: [_jsx("thead", { children: _jsxs("tr", { className: "text-zinc-500 border-b border-zinc-700/50", children: [_jsx("th", { className: "pb-2 pr-4 font-medium", children: "Node" }), _jsx("th", { className: "pb-2 pr-4 font-medium", children: "Fingerprint" }), _jsx("th", { className: "pb-2 pr-4 font-medium text-right", children: "Size" }), _jsx("th", { className: "pb-2 pr-4 font-medium text-right", children: "Hits" }), _jsx("th", { className: "pb-2 font-medium", children: "Score" })] }) }), _jsx("tbody", { children: candidates.map((c) => (_jsxs("tr", { className: "border-b border-zinc-700/30 last:border-0 hover:bg-zinc-700/20", children: [_jsx("td", { className: "py-2 pr-4", children: _jsx("span", { className: "text-zinc-400 font-mono", children: c.node }) }), _jsx("td", { className: "py-2 pr-4", children: _jsxs("code", { className: "font-mono text-xs bg-zinc-700 px-1 py-0.5 rounded", children: [c.fingerprint_hex.slice(0, 12), "\u2026"] }) }), _jsx("td", { className: "py-2 pr-4 text-right text-zinc-300 font-mono", children: fmtBytes(c.size_bytes) }), _jsx("td", { className: "py-2 pr-4 text-right", children: _jsx("span", { className: c.hit_count === 0 ? 'text-red-400' : 'text-zinc-400', children: c.hit_count === 0 ? '0 ✗' : c.hit_count.toLocaleString() }) }), _jsx("td", { className: "py-2", children: _jsx(ScoreBar, { score: c.pollution_score, max: maxScore }) })] }, `${c.node}-${c.fingerprint_hex}`))) })] }) })] }));
|
|
||||||
}
|
|
||||||
@@ -1,124 +0,0 @@
|
|||||||
import { useEffect, useState } from 'react';
|
|
||||||
import { api, PollutionCandidate, fmtBytes } from '../lib/api';
|
|
||||||
|
|
||||||
function ScoreBar({ score, max }: { score: number; max: number }) {
|
|
||||||
const pct = max > 0 ? Math.min(100, (score / max) * 100) : 0;
|
|
||||||
const color =
|
|
||||||
pct > 66
|
|
||||||
? 'bg-red-500'
|
|
||||||
: pct > 33
|
|
||||||
? 'bg-amber-500'
|
|
||||||
: 'bg-emerald-500';
|
|
||||||
return (
|
|
||||||
<div className="w-20 h-1.5 bg-zinc-700 rounded-full overflow-hidden">
|
|
||||||
<div className={`h-full rounded-full ${color}`} style={{ width: `${pct}%` }} />
|
|
||||||
</div>
|
|
||||||
);
|
|
||||||
}
|
|
||||||
|
|
||||||
export function PollutionPanel() {
|
|
||||||
const [candidates, setCandidates] = useState<PollutionCandidate[]>([]);
|
|
||||||
const [loading, setLoading] = useState(true);
|
|
||||||
const [error, setError] = useState<string | null>(null);
|
|
||||||
|
|
||||||
useEffect(() => {
|
|
||||||
api
|
|
||||||
.pollution()
|
|
||||||
.then((rows) => {
|
|
||||||
setCandidates(rows);
|
|
||||||
setLoading(false);
|
|
||||||
})
|
|
||||||
.catch((e) => {
|
|
||||||
setError(String(e));
|
|
||||||
setLoading(false);
|
|
||||||
});
|
|
||||||
}, []);
|
|
||||||
|
|
||||||
if (loading) {
|
|
||||||
return (
|
|
||||||
<div className="rounded-xl border border-zinc-700/50 bg-zinc-800/40 p-4 text-zinc-400 text-sm">
|
|
||||||
Loading pollution candidates…
|
|
||||||
</div>
|
|
||||||
);
|
|
||||||
}
|
|
||||||
|
|
||||||
if (error) {
|
|
||||||
return (
|
|
||||||
<div className="rounded-xl border border-red-700/40 bg-zinc-800/40 p-4 text-red-400 text-sm">
|
|
||||||
{error}
|
|
||||||
</div>
|
|
||||||
);
|
|
||||||
}
|
|
||||||
|
|
||||||
if (candidates.length === 0) {
|
|
||||||
return (
|
|
||||||
<div className="rounded-xl border border-zinc-700/50 bg-zinc-800/40 p-4">
|
|
||||||
<h2 className="text-sm font-semibold text-zinc-300 mb-1">Cache Pollution</h2>
|
|
||||||
<p className="text-zinc-500 text-sm">
|
|
||||||
No blobs tracked yet — pollution scores accumulate as builds run.
|
|
||||||
</p>
|
|
||||||
</div>
|
|
||||||
);
|
|
||||||
}
|
|
||||||
|
|
||||||
const maxScore = candidates[0]?.pollution_score ?? 1;
|
|
||||||
|
|
||||||
return (
|
|
||||||
<div className="rounded-xl border border-zinc-700/50 bg-zinc-800/40 p-4">
|
|
||||||
<div className="flex items-baseline justify-between mb-3">
|
|
||||||
<h2 className="text-sm font-semibold text-zinc-300">
|
|
||||||
Cache Pollution
|
|
||||||
<span className="ml-2 text-xs text-zinc-500 font-normal">T2.8</span>
|
|
||||||
</h2>
|
|
||||||
<span className="text-xs text-zinc-500">{candidates.length} entries</span>
|
|
||||||
</div>
|
|
||||||
|
|
||||||
<p className="text-xs text-zinc-500 mb-3">
|
|
||||||
Large blobs with few recorded accesses since last restart. High-score entries
|
|
||||||
consume hot-tier space disproportionate to their build value.
|
|
||||||
</p>
|
|
||||||
|
|
||||||
<div className="overflow-x-auto">
|
|
||||||
<table className="w-full text-xs text-left">
|
|
||||||
<thead>
|
|
||||||
<tr className="text-zinc-500 border-b border-zinc-700/50">
|
|
||||||
<th className="pb-2 pr-4 font-medium">Node</th>
|
|
||||||
<th className="pb-2 pr-4 font-medium">Fingerprint</th>
|
|
||||||
<th className="pb-2 pr-4 font-medium text-right">Size</th>
|
|
||||||
<th className="pb-2 pr-4 font-medium text-right">Hits</th>
|
|
||||||
<th className="pb-2 font-medium">Score</th>
|
|
||||||
</tr>
|
|
||||||
</thead>
|
|
||||||
<tbody>
|
|
||||||
{candidates.map((c) => (
|
|
||||||
<tr
|
|
||||||
key={`${c.node}-${c.fingerprint_hex}`}
|
|
||||||
className="border-b border-zinc-700/30 last:border-0 hover:bg-zinc-700/20"
|
|
||||||
>
|
|
||||||
<td className="py-2 pr-4">
|
|
||||||
<span className="text-zinc-400 font-mono">{c.node}</span>
|
|
||||||
</td>
|
|
||||||
<td className="py-2 pr-4">
|
|
||||||
<code className="font-mono text-xs bg-zinc-700 px-1 py-0.5 rounded">
|
|
||||||
{c.fingerprint_hex.slice(0, 12)}…
|
|
||||||
</code>
|
|
||||||
</td>
|
|
||||||
<td className="py-2 pr-4 text-right text-zinc-300 font-mono">
|
|
||||||
{fmtBytes(c.size_bytes)}
|
|
||||||
</td>
|
|
||||||
<td className="py-2 pr-4 text-right">
|
|
||||||
<span className={c.hit_count === 0 ? 'text-red-400' : 'text-zinc-400'}>
|
|
||||||
{c.hit_count === 0 ? '0 ✗' : c.hit_count.toLocaleString()}
|
|
||||||
</span>
|
|
||||||
</td>
|
|
||||||
<td className="py-2">
|
|
||||||
<ScoreBar score={c.pollution_score} max={maxScore} />
|
|
||||||
</td>
|
|
||||||
</tr>
|
|
||||||
))}
|
|
||||||
</tbody>
|
|
||||||
</table>
|
|
||||||
</div>
|
|
||||||
</div>
|
|
||||||
);
|
|
||||||
}
|
|
||||||
@@ -0,0 +1,44 @@
|
|||||||
|
import { jsx as _jsx, jsxs as _jsxs, Fragment as _Fragment } from "react/jsx-runtime";
|
||||||
|
import { useState } from 'react';
|
||||||
|
import { api } from '../lib/api';
|
||||||
|
/**
|
||||||
|
* Node-maintenance panel: runs `safe-shutdown-prep.sh --dry-run` on
|
||||||
|
* demand (safe, read-mostly, never stops anything) and — only once
|
||||||
|
* that comes back ready — unlocks a type-to-confirm button that
|
||||||
|
* starts the real run.
|
||||||
|
*
|
||||||
|
* The real run is fire-and-forget by necessity: its own steps stop
|
||||||
|
* this node's daemon, which is what's serving this very page, so
|
||||||
|
* there is no way to stream a live result past that point. Once
|
||||||
|
* started, the UI says so plainly and points at the on-disk log for
|
||||||
|
* the full report.
|
||||||
|
*/
|
||||||
|
export function ShutdownPrepPanel({ name }) {
|
||||||
|
const [check, setCheck] = useState({ phase: 'idle' });
|
||||||
|
const [exec, setExec] = useState({ phase: 'idle' });
|
||||||
|
const [confirmText, setConfirmText] = useState('');
|
||||||
|
const runCheck = () => {
|
||||||
|
setCheck({ phase: 'checking' });
|
||||||
|
setExec({ phase: 'idle' });
|
||||||
|
setConfirmText('');
|
||||||
|
api
|
||||||
|
.shutdownPrepCheck(name)
|
||||||
|
.then((r) => setCheck({ phase: 'done', ready: r.ready, output: r.output }))
|
||||||
|
.catch((e) => setCheck({ phase: 'error', message: String(e) }));
|
||||||
|
};
|
||||||
|
const runExecute = () => {
|
||||||
|
if (confirmText !== name)
|
||||||
|
return;
|
||||||
|
setExec({ phase: 'starting' });
|
||||||
|
api
|
||||||
|
.shutdownPrepExecute(name, confirmText)
|
||||||
|
.then((r) => setExec({ phase: 'started', message: r.message }))
|
||||||
|
.catch((e) => setExec({ phase: 'error', message: String(e) }));
|
||||||
|
};
|
||||||
|
const ready = check.phase === 'done' && check.ready;
|
||||||
|
return (_jsxs("div", { className: "rounded border border-slate-800 bg-slate-900/40 p-4 text-sm space-y-3", children: [_jsxs("div", { className: "flex items-center justify-between", children: [_jsx("div", { className: "text-slate-500 uppercase text-xs tracking-wider", children: "shutdown prep" }), _jsx("button", { onClick: runCheck, disabled: check.phase === 'checking', className: "px-3 py-1.5 rounded bg-slate-800 hover:bg-slate-700 text-slate-200 text-xs disabled:opacity-50", children: check.phase === 'checking' ? 'checking…' : 'check readiness for shutdown' })] }), _jsxs("p", { className: "text-xs text-slate-500", children: ["Runs a dry-run of the pre-shutdown checklist on ", _jsx("span", { className: "font-mono", children: name }), ' ', "\u2014 active builds, pending peer sync, a final snapshot + replicate to cold. Nothing is stopped or unmounted by this check."] }), check.phase === 'error' && (_jsx("div", { className: "rounded border border-red-800 bg-red-950/40 p-3 text-red-300 text-xs", children: check.message })), check.phase === 'done' && (_jsxs(_Fragment, { children: [_jsx("div", { className: `rounded border p-3 text-xs ${check.ready
|
||||||
|
? 'border-emerald-800 bg-emerald-950/40 text-emerald-300'
|
||||||
|
: 'border-amber-800 bg-amber-950/40 text-amber-300'}`, children: check.ready
|
||||||
|
? '✓ ready — safe to start the real shutdown-prep run'
|
||||||
|
: '! not ready — see output below (active build or un-synced changes are the usual cause)' }), _jsx("pre", { className: "max-h-72 overflow-auto rounded bg-slate-950 border border-slate-800 p-3 text-[11px] leading-relaxed text-slate-300 whitespace-pre-wrap", children: check.output })] })), ready && exec.phase !== 'started' && (_jsxs("div", { className: "rounded border border-red-900 bg-red-950/30 p-3 space-y-2", children: [_jsxs("div", { className: "text-red-300 text-xs", children: ["This starts the real run: stops maintenance timers, the dashboard, the storage daemon (gossip announces departure to peers), and unmounts FUSE on", ' ', _jsx("span", { className: "font-mono", children: name }), ". The node's dashboard connection will drop partway through \u2014 that's expected, not an error. It does ", _jsx("strong", { children: "not" }), " power the machine off; do that yourself once it's gone dark."] }), _jsxs("div", { className: "flex items-center gap-2", children: [_jsx("input", { value: confirmText, onChange: (e) => setConfirmText(e.target.value), placeholder: `type "${name}" to confirm`, className: "flex-1 rounded bg-slate-900 border border-slate-700 px-2 py-1.5 text-xs font-mono text-slate-200 placeholder:text-slate-600" }), _jsx("button", { onClick: runExecute, disabled: confirmText !== name || exec.phase === 'starting', className: "px-3 py-1.5 rounded bg-red-900 hover:bg-red-800 text-red-100 text-xs disabled:opacity-40 disabled:cursor-not-allowed whitespace-nowrap", children: exec.phase === 'starting' ? 'starting…' : `stop services on ${name}` })] })] })), exec.phase === 'error' && (_jsx("div", { className: "rounded border border-red-800 bg-red-950/40 p-3 text-red-300 text-xs", children: exec.message })), exec.phase === 'started' && (_jsx("div", { className: "rounded border border-sky-800 bg-sky-950/30 p-3 text-sky-300 text-xs", children: exec.message }))] }));
|
||||||
|
}
|
||||||
@@ -0,0 +1,144 @@
|
|||||||
|
import { useState } from 'react';
|
||||||
|
import { api } from '../lib/api';
|
||||||
|
|
||||||
|
interface Props {
|
||||||
|
name: string;
|
||||||
|
}
|
||||||
|
|
||||||
|
type CheckState =
|
||||||
|
| { phase: 'idle' }
|
||||||
|
| { phase: 'checking' }
|
||||||
|
| { phase: 'done'; ready: boolean; output: string }
|
||||||
|
| { phase: 'error'; message: string };
|
||||||
|
|
||||||
|
type ExecState =
|
||||||
|
| { phase: 'idle' }
|
||||||
|
| { phase: 'starting' }
|
||||||
|
| { phase: 'started'; message: string }
|
||||||
|
| { phase: 'error'; message: string };
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Node-maintenance panel: runs `safe-shutdown-prep.sh --dry-run` on
|
||||||
|
* demand (safe, read-mostly, never stops anything) and — only once
|
||||||
|
* that comes back ready — unlocks a type-to-confirm button that
|
||||||
|
* starts the real run.
|
||||||
|
*
|
||||||
|
* The real run is fire-and-forget by necessity: its own steps stop
|
||||||
|
* this node's daemon, which is what's serving this very page, so
|
||||||
|
* there is no way to stream a live result past that point. Once
|
||||||
|
* started, the UI says so plainly and points at the on-disk log for
|
||||||
|
* the full report.
|
||||||
|
*/
|
||||||
|
export function ShutdownPrepPanel({ name }: Props) {
|
||||||
|
const [check, setCheck] = useState<CheckState>({ phase: 'idle' });
|
||||||
|
const [exec, setExec] = useState<ExecState>({ phase: 'idle' });
|
||||||
|
const [confirmText, setConfirmText] = useState('');
|
||||||
|
|
||||||
|
const runCheck = () => {
|
||||||
|
setCheck({ phase: 'checking' });
|
||||||
|
setExec({ phase: 'idle' });
|
||||||
|
setConfirmText('');
|
||||||
|
api
|
||||||
|
.shutdownPrepCheck(name)
|
||||||
|
.then((r) => setCheck({ phase: 'done', ready: r.ready, output: r.output }))
|
||||||
|
.catch((e) => setCheck({ phase: 'error', message: String(e) }));
|
||||||
|
};
|
||||||
|
|
||||||
|
const runExecute = () => {
|
||||||
|
if (confirmText !== name) return;
|
||||||
|
setExec({ phase: 'starting' });
|
||||||
|
api
|
||||||
|
.shutdownPrepExecute(name, confirmText)
|
||||||
|
.then((r) => setExec({ phase: 'started', message: r.message }))
|
||||||
|
.catch((e) => setExec({ phase: 'error', message: String(e) }));
|
||||||
|
};
|
||||||
|
|
||||||
|
const ready = check.phase === 'done' && check.ready;
|
||||||
|
|
||||||
|
return (
|
||||||
|
<div className="rounded border border-slate-800 bg-slate-900/40 p-4 text-sm space-y-3">
|
||||||
|
<div className="flex items-center justify-between">
|
||||||
|
<div className="text-slate-500 uppercase text-xs tracking-wider">
|
||||||
|
shutdown prep
|
||||||
|
</div>
|
||||||
|
<button
|
||||||
|
onClick={runCheck}
|
||||||
|
disabled={check.phase === 'checking'}
|
||||||
|
className="px-3 py-1.5 rounded bg-slate-800 hover:bg-slate-700 text-slate-200 text-xs disabled:opacity-50"
|
||||||
|
>
|
||||||
|
{check.phase === 'checking' ? 'checking…' : 'check readiness for shutdown'}
|
||||||
|
</button>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<p className="text-xs text-slate-500">
|
||||||
|
Runs a dry-run of the pre-shutdown checklist on <span className="font-mono">{name}</span>{' '}
|
||||||
|
— active builds, pending peer sync, a final snapshot + replicate to cold. Nothing is
|
||||||
|
stopped or unmounted by this check.
|
||||||
|
</p>
|
||||||
|
|
||||||
|
{check.phase === 'error' && (
|
||||||
|
<div className="rounded border border-red-800 bg-red-950/40 p-3 text-red-300 text-xs">
|
||||||
|
{check.message}
|
||||||
|
</div>
|
||||||
|
)}
|
||||||
|
|
||||||
|
{check.phase === 'done' && (
|
||||||
|
<>
|
||||||
|
<div
|
||||||
|
className={`rounded border p-3 text-xs ${
|
||||||
|
check.ready
|
||||||
|
? 'border-emerald-800 bg-emerald-950/40 text-emerald-300'
|
||||||
|
: 'border-amber-800 bg-amber-950/40 text-amber-300'
|
||||||
|
}`}
|
||||||
|
>
|
||||||
|
{check.ready
|
||||||
|
? '✓ ready — safe to start the real shutdown-prep run'
|
||||||
|
: '! not ready — see output below (active build or un-synced changes are the usual cause)'}
|
||||||
|
</div>
|
||||||
|
<pre className="max-h-72 overflow-auto rounded bg-slate-950 border border-slate-800 p-3 text-[11px] leading-relaxed text-slate-300 whitespace-pre-wrap">
|
||||||
|
{check.output}
|
||||||
|
</pre>
|
||||||
|
</>
|
||||||
|
)}
|
||||||
|
|
||||||
|
{ready && exec.phase !== 'started' && (
|
||||||
|
<div className="rounded border border-red-900 bg-red-950/30 p-3 space-y-2">
|
||||||
|
<div className="text-red-300 text-xs">
|
||||||
|
This starts the real run: stops maintenance timers, the dashboard, the storage
|
||||||
|
daemon (gossip announces departure to peers), and unmounts FUSE on{' '}
|
||||||
|
<span className="font-mono">{name}</span>. The node's dashboard connection will drop
|
||||||
|
partway through — that's expected, not an error. It does <strong>not</strong> power
|
||||||
|
the machine off; do that yourself once it's gone dark.
|
||||||
|
</div>
|
||||||
|
<div className="flex items-center gap-2">
|
||||||
|
<input
|
||||||
|
value={confirmText}
|
||||||
|
onChange={(e) => setConfirmText(e.target.value)}
|
||||||
|
placeholder={`type "${name}" to confirm`}
|
||||||
|
className="flex-1 rounded bg-slate-900 border border-slate-700 px-2 py-1.5 text-xs font-mono text-slate-200 placeholder:text-slate-600"
|
||||||
|
/>
|
||||||
|
<button
|
||||||
|
onClick={runExecute}
|
||||||
|
disabled={confirmText !== name || exec.phase === 'starting'}
|
||||||
|
className="px-3 py-1.5 rounded bg-red-900 hover:bg-red-800 text-red-100 text-xs disabled:opacity-40 disabled:cursor-not-allowed whitespace-nowrap"
|
||||||
|
>
|
||||||
|
{exec.phase === 'starting' ? 'starting…' : `stop services on ${name}`}
|
||||||
|
</button>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
)}
|
||||||
|
|
||||||
|
{exec.phase === 'error' && (
|
||||||
|
<div className="rounded border border-red-800 bg-red-950/40 p-3 text-red-300 text-xs">
|
||||||
|
{exec.message}
|
||||||
|
</div>
|
||||||
|
)}
|
||||||
|
|
||||||
|
{exec.phase === 'started' && (
|
||||||
|
<div className="rounded border border-sky-800 bg-sky-950/30 p-3 text-sky-300 text-xs">
|
||||||
|
{exec.message}
|
||||||
|
</div>
|
||||||
|
)}
|
||||||
|
</div>
|
||||||
|
);
|
||||||
|
}
|
||||||
@@ -1,39 +0,0 @@
|
|||||||
import { jsxs as _jsxs, jsx as _jsx } from "react/jsx-runtime";
|
|
||||||
import { useEffect, useState } from 'react';
|
|
||||||
import { api, fmtAge } from '../lib/api';
|
|
||||||
function FpBadge({ hex }) {
|
|
||||||
return (_jsxs("code", { className: "font-mono text-xs bg-zinc-700 px-1 py-0.5 rounded", children: [hex.slice(0, 12), "\u2026"] }));
|
|
||||||
}
|
|
||||||
function NodePill({ name, variant }) {
|
|
||||||
const colors = variant === 'hot'
|
|
||||||
? 'bg-orange-900/50 text-orange-300 border border-orange-700/40'
|
|
||||||
: 'bg-zinc-700/60 text-zinc-400 border border-zinc-600/40';
|
|
||||||
return (_jsx("span", { className: `inline-block px-1.5 py-0.5 rounded text-xs mr-1 ${colors}`, children: name }));
|
|
||||||
}
|
|
||||||
export function WarmingCandidatesPanel() {
|
|
||||||
const [candidates, setCandidates] = useState([]);
|
|
||||||
const [loading, setLoading] = useState(true);
|
|
||||||
const [error, setError] = useState(null);
|
|
||||||
useEffect(() => {
|
|
||||||
api
|
|
||||||
.hotRefs()
|
|
||||||
.then((rows) => {
|
|
||||||
setCandidates(rows);
|
|
||||||
setLoading(false);
|
|
||||||
})
|
|
||||||
.catch((e) => {
|
|
||||||
setError(String(e));
|
|
||||||
setLoading(false);
|
|
||||||
});
|
|
||||||
}, []);
|
|
||||||
if (loading) {
|
|
||||||
return (_jsx("div", { className: "rounded-xl border border-zinc-700/50 bg-zinc-800/40 p-4 text-zinc-400 text-sm", children: "Loading warming candidates\u2026" }));
|
|
||||||
}
|
|
||||||
if (error) {
|
|
||||||
return (_jsx("div", { className: "rounded-xl border border-red-700/40 bg-zinc-800/40 p-4 text-red-400 text-sm", children: error }));
|
|
||||||
}
|
|
||||||
if (candidates.length === 0) {
|
|
||||||
return (_jsxs("div", { className: "rounded-xl border border-zinc-700/50 bg-zinc-800/40 p-4", children: [_jsx("h2", { className: "text-sm font-semibold text-zinc-300 mb-1", children: "Warming Candidates" }), _jsx("p", { className: "text-zinc-500 text-sm", children: "No asymmetric cache entries yet \u2014 all hot fingerprints are present on every node, or no access data has been collected." })] }));
|
|
||||||
}
|
|
||||||
return (_jsxs("div", { className: "rounded-xl border border-zinc-700/50 bg-zinc-800/40 p-4", children: [_jsxs("div", { className: "flex items-baseline justify-between mb-3", children: [_jsxs("h2", { className: "text-sm font-semibold text-zinc-300", children: ["Warming Candidates", _jsx("span", { className: "ml-2 text-xs text-zinc-500 font-normal", children: "T2.7" })] }), _jsxs("span", { className: "text-xs text-zinc-500", children: [candidates.length, " fingerprints"] })] }), _jsx("p", { className: "text-xs text-zinc-500 mb-3", children: "Fingerprints frequently accessed on some nodes but absent on others. Pre-positioning these blobs eliminates cross-node forwarding on the next build." }), _jsx("div", { className: "overflow-x-auto", children: _jsxs("table", { className: "w-full text-xs text-left", children: [_jsx("thead", { children: _jsxs("tr", { className: "text-zinc-500 border-b border-zinc-700/50", children: [_jsx("th", { className: "pb-2 pr-4 font-medium", children: "Fingerprint" }), _jsx("th", { className: "pb-2 pr-4 font-medium", children: "Hot on" }), _jsx("th", { className: "pb-2 pr-4 font-medium", children: "Missing on" }), _jsx("th", { className: "pb-2 pr-4 font-medium text-right", children: "Hits" }), _jsx("th", { className: "pb-2 font-medium text-right", children: "Last seen" })] }) }), _jsx("tbody", { children: candidates.map((c) => (_jsxs("tr", { className: "border-b border-zinc-700/30 last:border-0 hover:bg-zinc-700/20", children: [_jsx("td", { className: "py-2 pr-4", children: _jsx(FpBadge, { hex: c.fingerprint_hex }) }), _jsx("td", { className: "py-2 pr-4", children: c.hot_on.map((n) => (_jsx(NodePill, { name: n, variant: "hot" }, n))) }), _jsx("td", { className: "py-2 pr-4", children: c.missing_on.map((n) => (_jsx(NodePill, { name: n, variant: "missing" }, n))) }), _jsx("td", { className: "py-2 pr-4 text-right text-zinc-300 font-mono", children: c.max_hit_count.toLocaleString() }), _jsx("td", { className: "py-2 text-right text-zinc-400", children: c.last_hit_unix > 0 ? fmtAge(c.last_hit_unix) : '—' })] }, c.fingerprint_hex))) })] }) })] }));
|
|
||||||
}
|
|
||||||
@@ -1,128 +0,0 @@
|
|||||||
import { useEffect, useState } from 'react';
|
|
||||||
import { api, WarmingCandidate, fmtAge } from '../lib/api';
|
|
||||||
|
|
||||||
function FpBadge({ hex }: { hex: string }) {
|
|
||||||
return (
|
|
||||||
<code className="font-mono text-xs bg-zinc-700 px-1 py-0.5 rounded">
|
|
||||||
{hex.slice(0, 12)}…
|
|
||||||
</code>
|
|
||||||
);
|
|
||||||
}
|
|
||||||
|
|
||||||
function NodePill({ name, variant }: { name: string; variant: 'hot' | 'missing' }) {
|
|
||||||
const colors =
|
|
||||||
variant === 'hot'
|
|
||||||
? 'bg-orange-900/50 text-orange-300 border border-orange-700/40'
|
|
||||||
: 'bg-zinc-700/60 text-zinc-400 border border-zinc-600/40';
|
|
||||||
return (
|
|
||||||
<span className={`inline-block px-1.5 py-0.5 rounded text-xs mr-1 ${colors}`}>
|
|
||||||
{name}
|
|
||||||
</span>
|
|
||||||
);
|
|
||||||
}
|
|
||||||
|
|
||||||
export function WarmingCandidatesPanel() {
|
|
||||||
const [candidates, setCandidates] = useState<WarmingCandidate[]>([]);
|
|
||||||
const [loading, setLoading] = useState(true);
|
|
||||||
const [error, setError] = useState<string | null>(null);
|
|
||||||
|
|
||||||
useEffect(() => {
|
|
||||||
api
|
|
||||||
.hotRefs()
|
|
||||||
.then((rows) => {
|
|
||||||
setCandidates(rows);
|
|
||||||
setLoading(false);
|
|
||||||
})
|
|
||||||
.catch((e) => {
|
|
||||||
setError(String(e));
|
|
||||||
setLoading(false);
|
|
||||||
});
|
|
||||||
}, []);
|
|
||||||
|
|
||||||
if (loading) {
|
|
||||||
return (
|
|
||||||
<div className="rounded-xl border border-zinc-700/50 bg-zinc-800/40 p-4 text-zinc-400 text-sm">
|
|
||||||
Loading warming candidates…
|
|
||||||
</div>
|
|
||||||
);
|
|
||||||
}
|
|
||||||
|
|
||||||
if (error) {
|
|
||||||
return (
|
|
||||||
<div className="rounded-xl border border-red-700/40 bg-zinc-800/40 p-4 text-red-400 text-sm">
|
|
||||||
{error}
|
|
||||||
</div>
|
|
||||||
);
|
|
||||||
}
|
|
||||||
|
|
||||||
if (candidates.length === 0) {
|
|
||||||
return (
|
|
||||||
<div className="rounded-xl border border-zinc-700/50 bg-zinc-800/40 p-4">
|
|
||||||
<h2 className="text-sm font-semibold text-zinc-300 mb-1">Warming Candidates</h2>
|
|
||||||
<p className="text-zinc-500 text-sm">
|
|
||||||
No asymmetric cache entries yet — all hot fingerprints are present on every node, or
|
|
||||||
no access data has been collected.
|
|
||||||
</p>
|
|
||||||
</div>
|
|
||||||
);
|
|
||||||
}
|
|
||||||
|
|
||||||
return (
|
|
||||||
<div className="rounded-xl border border-zinc-700/50 bg-zinc-800/40 p-4">
|
|
||||||
<div className="flex items-baseline justify-between mb-3">
|
|
||||||
<h2 className="text-sm font-semibold text-zinc-300">
|
|
||||||
Warming Candidates
|
|
||||||
<span className="ml-2 text-xs text-zinc-500 font-normal">T2.7</span>
|
|
||||||
</h2>
|
|
||||||
<span className="text-xs text-zinc-500">{candidates.length} fingerprints</span>
|
|
||||||
</div>
|
|
||||||
|
|
||||||
<p className="text-xs text-zinc-500 mb-3">
|
|
||||||
Fingerprints frequently accessed on some nodes but absent on others. Pre-positioning these
|
|
||||||
blobs eliminates cross-node forwarding on the next build.
|
|
||||||
</p>
|
|
||||||
|
|
||||||
<div className="overflow-x-auto">
|
|
||||||
<table className="w-full text-xs text-left">
|
|
||||||
<thead>
|
|
||||||
<tr className="text-zinc-500 border-b border-zinc-700/50">
|
|
||||||
<th className="pb-2 pr-4 font-medium">Fingerprint</th>
|
|
||||||
<th className="pb-2 pr-4 font-medium">Hot on</th>
|
|
||||||
<th className="pb-2 pr-4 font-medium">Missing on</th>
|
|
||||||
<th className="pb-2 pr-4 font-medium text-right">Hits</th>
|
|
||||||
<th className="pb-2 font-medium text-right">Last seen</th>
|
|
||||||
</tr>
|
|
||||||
</thead>
|
|
||||||
<tbody>
|
|
||||||
{candidates.map((c) => (
|
|
||||||
<tr
|
|
||||||
key={c.fingerprint_hex}
|
|
||||||
className="border-b border-zinc-700/30 last:border-0 hover:bg-zinc-700/20"
|
|
||||||
>
|
|
||||||
<td className="py-2 pr-4">
|
|
||||||
<FpBadge hex={c.fingerprint_hex} />
|
|
||||||
</td>
|
|
||||||
<td className="py-2 pr-4">
|
|
||||||
{c.hot_on.map((n) => (
|
|
||||||
<NodePill key={n} name={n} variant="hot" />
|
|
||||||
))}
|
|
||||||
</td>
|
|
||||||
<td className="py-2 pr-4">
|
|
||||||
{c.missing_on.map((n) => (
|
|
||||||
<NodePill key={n} name={n} variant="missing" />
|
|
||||||
))}
|
|
||||||
</td>
|
|
||||||
<td className="py-2 pr-4 text-right text-zinc-300 font-mono">
|
|
||||||
{c.max_hit_count.toLocaleString()}
|
|
||||||
</td>
|
|
||||||
<td className="py-2 text-right text-zinc-400">
|
|
||||||
{c.last_hit_unix > 0 ? fmtAge(c.last_hit_unix) : '—'}
|
|
||||||
</td>
|
|
||||||
</tr>
|
|
||||||
))}
|
|
||||||
</tbody>
|
|
||||||
</table>
|
|
||||||
</div>
|
|
||||||
</div>
|
|
||||||
);
|
|
||||||
}
|
|
||||||
+17
-18
@@ -23,20 +23,32 @@ async function get(path) {
|
|||||||
}
|
}
|
||||||
return resp.json();
|
return resp.json();
|
||||||
}
|
}
|
||||||
|
async function post(path, body) {
|
||||||
|
const full = `${API_BASE}${path}`;
|
||||||
|
const resp = await fetch(full, {
|
||||||
|
method: 'POST',
|
||||||
|
headers: { Accept: 'application/json', 'Content-Type': 'application/json' },
|
||||||
|
body: body === undefined ? undefined : JSON.stringify(body),
|
||||||
|
});
|
||||||
|
if (!resp.ok) {
|
||||||
|
const text = await resp.text().catch(() => '');
|
||||||
|
throw new Error(`${full} → ${resp.status} ${resp.statusText}${text ? `: ${text}` : ''}`);
|
||||||
|
}
|
||||||
|
return resp.json();
|
||||||
|
}
|
||||||
export const api = {
|
export const api = {
|
||||||
fleet: () => get('/v2/fleet'),
|
fleet: () => get('/v2/fleet'),
|
||||||
projects: () => get('/v2/projects'),
|
projects: () => get('/v2/projects'),
|
||||||
nodeStatus: (name) => get(`/v2/node/${name}/status`),
|
nodeStatus: (name) => get(`/v2/node/${name}/status`),
|
||||||
metricsHistory: (name, limit = 60) => get(`/v2/node/${name}/metrics-history?limit=${limit}`),
|
|
||||||
anomalies: () => get('/v2/anomalies'),
|
|
||||||
hotRefs: () => get('/v2/hot-refs'),
|
|
||||||
pollution: () => get('/v2/pollution'),
|
|
||||||
maintenance: () => get('/v2/maintenance'),
|
|
||||||
blobs: (limit = 200, offset = 0) => get(`/v2/storage/blobs?limit=${limit}&offset=${offset}`),
|
blobs: (limit = 200, offset = 0) => get(`/v2/storage/blobs?limit=${limit}&offset=${offset}`),
|
||||||
tags: (prefix = '') => get(`/v2/storage/tags?prefix=${encodeURIComponent(prefix)}`),
|
tags: (prefix = '') => get(`/v2/storage/tags?prefix=${encodeURIComponent(prefix)}`),
|
||||||
refs: (limit = 200, offset = 0) => get(`/v2/storage/refs?limit=${limit}&offset=${offset}`),
|
refs: (limit = 200, offset = 0) => get(`/v2/storage/refs?limit=${limit}&offset=${offset}`),
|
||||||
snapshots: () => get('/v2/storage/snapshots'),
|
snapshots: () => get('/v2/storage/snapshots'),
|
||||||
refTracking: (repo = '') => get(`/v2/storage/ref-tracking?repo=${encodeURIComponent(repo)}`),
|
refTracking: (repo = '') => get(`/v2/storage/ref-tracking?repo=${encodeURIComponent(repo)}`),
|
||||||
|
shutdownPrepCheck: (name) => post(`/v2/node/${name}/shutdown-prep/check`),
|
||||||
|
shutdownPrepExecute: (name, confirmNodeName) => post(`/v2/node/${name}/shutdown-prep/execute`, {
|
||||||
|
confirm_node_name: confirmNodeName,
|
||||||
|
}),
|
||||||
};
|
};
|
||||||
/** Format bytes as MB / GB / TB as needed. */
|
/** Format bytes as MB / GB / TB as needed. */
|
||||||
export function fmtBytes(n) {
|
export function fmtBytes(n) {
|
||||||
@@ -62,16 +74,3 @@ export function fmtAge(unix) {
|
|||||||
return `${Math.floor(diff / 3600)}h ago`;
|
return `${Math.floor(diff / 3600)}h ago`;
|
||||||
return `${Math.floor(diff / 86400)}d ago`;
|
return `${Math.floor(diff / 86400)}d ago`;
|
||||||
}
|
}
|
||||||
/** "up 3d 14h" from a daemon start unix timestamp. */
|
|
||||||
export function fmtUptime(startedUnix) {
|
|
||||||
const now = Math.floor(Date.now() / 1000);
|
|
||||||
const secs = Math.max(0, now - startedUnix);
|
|
||||||
const days = Math.floor(secs / 86400);
|
|
||||||
const hours = Math.floor((secs % 86400) / 3600);
|
|
||||||
const mins = Math.floor((secs % 3600) / 60);
|
|
||||||
if (days > 0)
|
|
||||||
return `up ${days}d ${hours}h`;
|
|
||||||
if (hours > 0)
|
|
||||||
return `up ${hours}h ${mins}m`;
|
|
||||||
return `up ${mins}m`;
|
|
||||||
}
|
|
||||||
|
|||||||
+32
-83
@@ -25,14 +25,6 @@ export interface CacheSummary {
|
|||||||
bytes_served: number;
|
bytes_served: number;
|
||||||
bytes_ingested: number;
|
bytes_ingested: number;
|
||||||
hit_rate: number;
|
hit_rate: number;
|
||||||
// Per-type breakdown (available when backend >= Phase A).
|
|
||||||
get_ref_hits: number;
|
|
||||||
get_ref_misses: number;
|
|
||||||
get_tag_hits: number;
|
|
||||||
get_tag_misses: number;
|
|
||||||
/** HasChunk probes — non-zero means partial-sync dedup is active. */
|
|
||||||
has_chunk_hits: number;
|
|
||||||
has_chunk_misses: number;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
export interface TimerStatus {
|
export interface TimerStatus {
|
||||||
@@ -57,12 +49,6 @@ export interface NodeStatusV2 {
|
|||||||
mount: MountStatus | null;
|
mount: MountStatus | null;
|
||||||
cache: CacheSummary | null;
|
cache: CacheSummary | null;
|
||||||
timers: TimerStatus[];
|
timers: TimerStatus[];
|
||||||
/** Unix timestamp (seconds) when the daemon last started. */
|
|
||||||
daemon_started_unix: number | null;
|
|
||||||
/** 1-minute load average from /proc/loadavg. Null on non-Linux. */
|
|
||||||
cpu_load_1m: number | null;
|
|
||||||
/** Lifeguard health score: 0=healthy, 1-2=degraded, 3+=stressed. */
|
|
||||||
health_score: number;
|
|
||||||
online: boolean;
|
online: boolean;
|
||||||
error: string | null;
|
error: string | null;
|
||||||
}
|
}
|
||||||
@@ -133,6 +119,20 @@ async function get<T>(path: string): Promise<T> {
|
|||||||
return resp.json();
|
return resp.json();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
async function post<T>(path: string, body?: unknown): Promise<T> {
|
||||||
|
const full = `${API_BASE}${path}`;
|
||||||
|
const resp = await fetch(full, {
|
||||||
|
method: 'POST',
|
||||||
|
headers: { Accept: 'application/json', 'Content-Type': 'application/json' },
|
||||||
|
body: body === undefined ? undefined : JSON.stringify(body),
|
||||||
|
});
|
||||||
|
if (!resp.ok) {
|
||||||
|
const text = await resp.text().catch(() => '');
|
||||||
|
throw new Error(`${full} → ${resp.status} ${resp.statusText}${text ? `: ${text}` : ''}`);
|
||||||
|
}
|
||||||
|
return resp.json();
|
||||||
|
}
|
||||||
|
|
||||||
// All paths are relative to API_BASE. E.g. '/v2/fleet' becomes
|
// All paths are relative to API_BASE. E.g. '/v2/fleet' becomes
|
||||||
// '/api/v2/fleet' locally or '/clawstor/api/v2/fleet' via Tailscale.
|
// '/api/v2/fleet' locally or '/clawstor/api/v2/fleet' via Tailscale.
|
||||||
export interface ProjectRow {
|
export interface ProjectRow {
|
||||||
@@ -146,67 +146,10 @@ export interface ProjectRow {
|
|||||||
tier: 'active' | 'recent' | 'idle' | string;
|
tier: 'active' | 'recent' | 'idle' | string;
|
||||||
}
|
}
|
||||||
|
|
||||||
/** One 1-minute snapshot from the server-side metrics ring buffer. */
|
|
||||||
export interface MetricSample {
|
|
||||||
unix_ts: number;
|
|
||||||
hot_used_bytes: number;
|
|
||||||
hot_max_bytes: number;
|
|
||||||
cache_hit_rate: number;
|
|
||||||
cache_hits: number;
|
|
||||||
cache_misses: number;
|
|
||||||
has_chunk_hits: number;
|
|
||||||
has_chunk_misses: number;
|
|
||||||
fs_used_bytes: number;
|
|
||||||
fs_total_bytes: number;
|
|
||||||
fs_available_bytes: number;
|
|
||||||
/** Sum of z² across 4 drift-adapted metrics. Absent until 10 samples collected. */
|
|
||||||
anomaly_score?: number;
|
|
||||||
}
|
|
||||||
|
|
||||||
export interface AnomalyStatus {
|
|
||||||
node: string;
|
|
||||||
score: number;
|
|
||||||
level: 'ok' | 'warn' | 'alert';
|
|
||||||
samples_used: number;
|
|
||||||
}
|
|
||||||
|
|
||||||
export interface WarmingCandidate {
|
|
||||||
fingerprint_hex: string;
|
|
||||||
hot_on: string[];
|
|
||||||
missing_on: string[];
|
|
||||||
max_hit_count: number;
|
|
||||||
last_hit_unix: number;
|
|
||||||
}
|
|
||||||
|
|
||||||
export interface PollutionCandidate {
|
|
||||||
node: string;
|
|
||||||
fingerprint_hex: string;
|
|
||||||
blob_id_hex: string;
|
|
||||||
size_bytes: number;
|
|
||||||
hit_count: number;
|
|
||||||
pollution_score: number;
|
|
||||||
}
|
|
||||||
|
|
||||||
export interface MaintenanceRow {
|
|
||||||
node: string;
|
|
||||||
kind: string;
|
|
||||||
unix_ts: number;
|
|
||||||
chunks_scanned: number;
|
|
||||||
chunks_removed: number;
|
|
||||||
bytes_reclaimed: number;
|
|
||||||
error: string | null;
|
|
||||||
}
|
|
||||||
|
|
||||||
export const api = {
|
export const api = {
|
||||||
fleet: () => get<FleetSnapshot>('/v2/fleet'),
|
fleet: () => get<FleetSnapshot>('/v2/fleet'),
|
||||||
projects: () => get<ProjectRow[]>('/v2/projects'),
|
projects: () => get<ProjectRow[]>('/v2/projects'),
|
||||||
nodeStatus: (name: string) => get<NodeStatusV2>(`/v2/node/${name}/status`),
|
nodeStatus: (name: string) => get<NodeStatusV2>(`/v2/node/${name}/status`),
|
||||||
metricsHistory: (name: string, limit = 60) =>
|
|
||||||
get<MetricSample[]>(`/v2/node/${name}/metrics-history?limit=${limit}`),
|
|
||||||
anomalies: () => get<AnomalyStatus[]>('/v2/anomalies'),
|
|
||||||
hotRefs: () => get<WarmingCandidate[]>('/v2/hot-refs'),
|
|
||||||
pollution: () => get<PollutionCandidate[]>('/v2/pollution'),
|
|
||||||
maintenance: () => get<MaintenanceRow[]>('/v2/maintenance'),
|
|
||||||
blobs: (limit = 200, offset = 0) =>
|
blobs: (limit = 200, offset = 0) =>
|
||||||
get<BlobSummary[]>(`/v2/storage/blobs?limit=${limit}&offset=${offset}`),
|
get<BlobSummary[]>(`/v2/storage/blobs?limit=${limit}&offset=${offset}`),
|
||||||
tags: (prefix = '') =>
|
tags: (prefix = '') =>
|
||||||
@@ -216,8 +159,26 @@ export const api = {
|
|||||||
snapshots: () => get<SnapshotSummary[]>('/v2/storage/snapshots'),
|
snapshots: () => get<SnapshotSummary[]>('/v2/storage/snapshots'),
|
||||||
refTracking: (repo = '') =>
|
refTracking: (repo = '') =>
|
||||||
get<RefTrackingItem[]>(`/v2/storage/ref-tracking?repo=${encodeURIComponent(repo)}`),
|
get<RefTrackingItem[]>(`/v2/storage/ref-tracking?repo=${encodeURIComponent(repo)}`),
|
||||||
|
shutdownPrepCheck: (name: string) =>
|
||||||
|
post<ShutdownPrepCheckResponse>(`/v2/node/${name}/shutdown-prep/check`),
|
||||||
|
shutdownPrepExecute: (name: string, confirmNodeName: string) =>
|
||||||
|
post<ShutdownPrepExecuteResponse>(`/v2/node/${name}/shutdown-prep/execute`, {
|
||||||
|
confirm_node_name: confirmNodeName,
|
||||||
|
}),
|
||||||
};
|
};
|
||||||
|
|
||||||
|
export interface ShutdownPrepCheckResponse {
|
||||||
|
node: string;
|
||||||
|
ready: boolean;
|
||||||
|
output: string;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface ShutdownPrepExecuteResponse {
|
||||||
|
node: string;
|
||||||
|
started: boolean;
|
||||||
|
message: string;
|
||||||
|
}
|
||||||
|
|
||||||
/** Format bytes as MB / GB / TB as needed. */
|
/** Format bytes as MB / GB / TB as needed. */
|
||||||
export function fmtBytes(n: number): string {
|
export function fmtBytes(n: number): string {
|
||||||
if (n < 1024) return `${n} B`;
|
if (n < 1024) return `${n} B`;
|
||||||
@@ -236,15 +197,3 @@ export function fmtAge(unix: number): string {
|
|||||||
if (diff < 86400) return `${Math.floor(diff / 3600)}h ago`;
|
if (diff < 86400) return `${Math.floor(diff / 3600)}h ago`;
|
||||||
return `${Math.floor(diff / 86400)}d ago`;
|
return `${Math.floor(diff / 86400)}d ago`;
|
||||||
}
|
}
|
||||||
|
|
||||||
/** "up 3d 14h" from a daemon start unix timestamp. */
|
|
||||||
export function fmtUptime(startedUnix: number): string {
|
|
||||||
const now = Math.floor(Date.now() / 1000);
|
|
||||||
const secs = Math.max(0, now - startedUnix);
|
|
||||||
const days = Math.floor(secs / 86400);
|
|
||||||
const hours = Math.floor((secs % 86400) / 3600);
|
|
||||||
const mins = Math.floor((secs % 3600) / 60);
|
|
||||||
if (days > 0) return `up ${days}d ${hours}h`;
|
|
||||||
if (hours > 0) return `up ${hours}h ${mins}m`;
|
|
||||||
return `up ${mins}m`;
|
|
||||||
}
|
|
||||||
|
|||||||
@@ -3,13 +3,10 @@ import { useEffect, useState } from 'react';
|
|||||||
import { api, fmtBytes, fmtAge } from '../lib/api';
|
import { api, fmtBytes, fmtAge } from '../lib/api';
|
||||||
import { NodeCard } from '../components/NodeCard';
|
import { NodeCard } from '../components/NodeCard';
|
||||||
import { ProjectsPanel } from '../components/ProjectsPanel';
|
import { ProjectsPanel } from '../components/ProjectsPanel';
|
||||||
import { WarmingCandidatesPanel } from '../components/WarmingCandidatesPanel';
|
|
||||||
import { PollutionPanel } from '../components/PollutionPanel';
|
|
||||||
// FleetHealth landing — human-oriented single-pane-of-glass.
|
// FleetHealth landing — human-oriented single-pane-of-glass.
|
||||||
// Polls the aggregator's /api/v2/fleet every 10 s.
|
// Polls the aggregator's /api/v2/fleet every 10 s.
|
||||||
export function CommandCenter() {
|
export function CommandCenter() {
|
||||||
const [fleet, setFleet] = useState(null);
|
const [fleet, setFleet] = useState(null);
|
||||||
const [anomalies, setAnomalies] = useState([]);
|
|
||||||
const [err, setErr] = useState(null);
|
const [err, setErr] = useState(null);
|
||||||
const [tick, setTick] = useState(0);
|
const [tick, setTick] = useState(0);
|
||||||
useEffect(() => {
|
useEffect(() => {
|
||||||
@@ -20,7 +17,6 @@ export function CommandCenter() {
|
|||||||
setErr(null);
|
setErr(null);
|
||||||
})
|
})
|
||||||
.catch((e) => setErr(String(e)));
|
.catch((e) => setErr(String(e)));
|
||||||
api.anomalies().then(setAnomalies).catch(() => { });
|
|
||||||
}, [tick]);
|
}, [tick]);
|
||||||
useEffect(() => {
|
useEffect(() => {
|
||||||
const id = setInterval(() => setTick((t) => t + 1), 10_000);
|
const id = setInterval(() => setTick((t) => t + 1), 10_000);
|
||||||
@@ -36,10 +32,6 @@ export function CommandCenter() {
|
|||||||
mounted: a.mounted + (n.mount?.active ? 1 : 0),
|
mounted: a.mounted + (n.mount?.active ? 1 : 0),
|
||||||
}), { diskUsed: 0, diskTotal: 0, hotUsed: 0, hotMax: 0, online: 0, mounted: 0 })
|
}), { diskUsed: 0, diskTotal: 0, hotUsed: 0, hotMax: 0, online: 0, mounted: 0 })
|
||||||
: null;
|
: null;
|
||||||
// Build a node→level lookup for NodeCard props.
|
return (_jsxs("div", { className: "space-y-6", children: [_jsxs("div", { children: [_jsx("h1", { className: "text-2xl font-semibold text-slate-100", children: "Fleet health" }), _jsx("div", { className: "text-sm text-slate-500 mt-1", children: fleet && (_jsxs(_Fragment, { children: [_jsx("span", { className: "text-emerald-300", children: totals?.online }), "/", fleet.nodes.length, " nodes online", ' · ', totals?.mounted, "/", fleet.nodes.length, " mounted", ' · ', "updated ", _jsx("span", { className: "font-mono", children: fmtAge(fleet.fetched_at_unix) }), ' · ', "hosted by ", _jsx("span", { className: "font-mono text-slate-300", children: fleet.aggregator_name })] })) })] }), err && (_jsx("div", { className: "rounded border border-red-800 bg-red-950/40 p-3 text-red-300 text-sm", children: err })), totals && totals.diskTotal > 0 && (_jsxs("div", { className: "rounded-lg border border-slate-800 bg-slate-900/60 p-4 flex items-center justify-between", children: [_jsxs("div", { children: [_jsx("div", { className: "text-xs uppercase tracking-wider text-slate-500", children: "Fleet-wide storage" }), _jsxs("div", { className: "text-2xl font-semibold font-mono mt-1", children: [fmtBytes(totals.diskUsed), ' ', _jsxs("span", { className: "text-slate-500 text-lg", children: ["/ ", fmtBytes(totals.diskTotal)] })] })] }), _jsxs("div", { className: "text-right text-slate-400 text-sm", children: [_jsxs("div", { children: [Math.round((totals.diskUsed / totals.diskTotal) * 100), "% used"] }), _jsxs("div", { className: "text-xs text-slate-500 mt-1", children: ["hot tier: ", fmtBytes(totals.hotUsed), " / ", fmtBytes(totals.hotMax)] })] })] })), _jsxs("section", { children: [_jsx("h2", { className: "text-lg font-semibold text-slate-100 mb-3", children: "Nodes" }), _jsxs("div", { className: "grid grid-cols-1 sm:grid-cols-2 lg:grid-cols-3 gap-4", children: [fleet?.nodes.map((n) => (_jsx(NodeCard, { node: n }, n.node_name))), !fleet &&
|
||||||
const anomalyMap = Object.fromEntries(anomalies.map((a) => [a.node, a.level]));
|
[1, 2, 3].map((i) => (_jsx("div", { className: "rounded-lg border border-slate-800 bg-slate-900/40 p-5 h-64 animate-pulse" }, i)))] })] }), _jsx(ProjectsPanel, {})] }));
|
||||||
const alertNodes = anomalies.filter((a) => a.level === 'alert');
|
|
||||||
const warnNodes = anomalies.filter((a) => a.level === 'warn');
|
|
||||||
return (_jsxs("div", { className: "space-y-6", children: [_jsxs("div", { children: [_jsx("h1", { className: "text-2xl font-semibold text-slate-100", children: "Fleet health" }), _jsx("div", { className: "text-sm text-slate-500 mt-1", children: fleet && (_jsxs(_Fragment, { children: [_jsx("span", { className: "text-emerald-300", children: totals?.online }), "/", fleet.nodes.length, " nodes online", ' · ', totals?.mounted, "/", fleet.nodes.length, " mounted", ' · ', "updated ", _jsx("span", { className: "font-mono", children: fmtAge(fleet.fetched_at_unix) }), ' · ', "hosted by ", _jsx("span", { className: "font-mono text-slate-300", children: fleet.aggregator_name })] })) })] }), err && (_jsx("div", { className: "rounded border border-red-800 bg-red-950/40 p-3 text-red-300 text-sm", children: err })), alertNodes.length > 0 && (_jsxs("div", { className: "rounded border border-red-800 bg-red-950/40 px-4 py-3 text-sm flex items-start gap-3", children: [_jsx("span", { className: "text-red-400 font-bold mt-0.5", children: "\u25CF" }), _jsxs("div", { children: [_jsx("span", { className: "text-red-300 font-semibold", children: "Metric anomaly detected \u2014 " }), _jsxs("span", { className: "text-red-200", children: [alertNodes.map((a) => a.node).join(', '), " deviating >2\u03C3 from baseline", alertNodes.length === 1 && ` (score ${alertNodes[0].score.toFixed(1)})`] })] })] })), alertNodes.length === 0 && warnNodes.length > 0 && (_jsxs("div", { className: "rounded border border-amber-800 bg-amber-950/40 px-4 py-3 text-sm flex items-start gap-3", children: [_jsx("span", { className: "text-amber-400 font-bold mt-0.5", children: "\u25CF" }), _jsxs("div", { children: [_jsx("span", { className: "text-amber-300 font-semibold", children: "Metric drift \u2014 " }), _jsxs("span", { className: "text-amber-200", children: [warnNodes.map((a) => a.node).join(', '), " showing unusual patterns"] })] })] })), totals && totals.diskTotal > 0 && (_jsxs("div", { className: "rounded-lg border border-slate-800 bg-slate-900/60 p-4 flex items-center justify-between", children: [_jsxs("div", { children: [_jsx("div", { className: "text-xs uppercase tracking-wider text-slate-500", children: "Fleet-wide storage" }), _jsxs("div", { className: "text-2xl font-semibold font-mono mt-1", children: [fmtBytes(totals.diskUsed), ' ', _jsxs("span", { className: "text-slate-500 text-lg", children: ["/ ", fmtBytes(totals.diskTotal)] })] })] }), _jsxs("div", { className: "text-right text-slate-400 text-sm", children: [_jsxs("div", { children: [Math.round((totals.diskUsed / totals.diskTotal) * 100), "% used"] }), _jsxs("div", { className: "text-xs text-slate-500 mt-1", children: ["hot tier: ", fmtBytes(totals.hotUsed), " / ", fmtBytes(totals.hotMax)] })] })] })), _jsxs("section", { children: [_jsx("h2", { className: "text-lg font-semibold text-slate-100 mb-3", children: "Nodes" }), _jsxs("div", { className: "grid grid-cols-1 sm:grid-cols-2 lg:grid-cols-3 gap-4", children: [fleet?.nodes.map((n) => (_jsx(NodeCard, { node: n, anomalyLevel: anomalyMap[n.node_name] }, n.node_name))), !fleet &&
|
|
||||||
[1, 2, 3].map((i) => (_jsx("div", { className: "rounded-lg border border-slate-800 bg-slate-900/40 p-5 h-64 animate-pulse" }, i)))] })] }), _jsx(ProjectsPanel, {}), _jsx(WarmingCandidatesPanel, {}), _jsx(PollutionPanel, {})] }));
|
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1,15 +1,12 @@
|
|||||||
import { useEffect, useState } from 'react';
|
import { useEffect, useState } from 'react';
|
||||||
import { api, FleetSnapshot, AnomalyStatus, fmtBytes, fmtAge } from '../lib/api';
|
import { api, FleetSnapshot, fmtBytes, fmtAge } from '../lib/api';
|
||||||
import { NodeCard } from '../components/NodeCard';
|
import { NodeCard } from '../components/NodeCard';
|
||||||
import { ProjectsPanel } from '../components/ProjectsPanel';
|
import { ProjectsPanel } from '../components/ProjectsPanel';
|
||||||
import { WarmingCandidatesPanel } from '../components/WarmingCandidatesPanel';
|
|
||||||
import { PollutionPanel } from '../components/PollutionPanel';
|
|
||||||
|
|
||||||
// FleetHealth landing — human-oriented single-pane-of-glass.
|
// FleetHealth landing — human-oriented single-pane-of-glass.
|
||||||
// Polls the aggregator's /api/v2/fleet every 10 s.
|
// Polls the aggregator's /api/v2/fleet every 10 s.
|
||||||
export function CommandCenter() {
|
export function CommandCenter() {
|
||||||
const [fleet, setFleet] = useState<FleetSnapshot | null>(null);
|
const [fleet, setFleet] = useState<FleetSnapshot | null>(null);
|
||||||
const [anomalies, setAnomalies] = useState<AnomalyStatus[]>([]);
|
|
||||||
const [err, setErr] = useState<string | null>(null);
|
const [err, setErr] = useState<string | null>(null);
|
||||||
const [tick, setTick] = useState(0);
|
const [tick, setTick] = useState(0);
|
||||||
|
|
||||||
@@ -21,7 +18,6 @@ export function CommandCenter() {
|
|||||||
setErr(null);
|
setErr(null);
|
||||||
})
|
})
|
||||||
.catch((e) => setErr(String(e)));
|
.catch((e) => setErr(String(e)));
|
||||||
api.anomalies().then(setAnomalies).catch(() => {});
|
|
||||||
}, [tick]);
|
}, [tick]);
|
||||||
|
|
||||||
useEffect(() => {
|
useEffect(() => {
|
||||||
@@ -43,13 +39,6 @@ export function CommandCenter() {
|
|||||||
)
|
)
|
||||||
: null;
|
: null;
|
||||||
|
|
||||||
// Build a node→level lookup for NodeCard props.
|
|
||||||
const anomalyMap = Object.fromEntries(
|
|
||||||
anomalies.map((a) => [a.node, a.level] as const)
|
|
||||||
);
|
|
||||||
const alertNodes = anomalies.filter((a) => a.level === 'alert');
|
|
||||||
const warnNodes = anomalies.filter((a) => a.level === 'warn');
|
|
||||||
|
|
||||||
return (
|
return (
|
||||||
<div className="space-y-6">
|
<div className="space-y-6">
|
||||||
<div>
|
<div>
|
||||||
@@ -76,31 +65,6 @@ export function CommandCenter() {
|
|||||||
</div>
|
</div>
|
||||||
)}
|
)}
|
||||||
|
|
||||||
{/* Anomaly banners (only shown once the ring buffer has 10+ samples) */}
|
|
||||||
{alertNodes.length > 0 && (
|
|
||||||
<div className="rounded border border-red-800 bg-red-950/40 px-4 py-3 text-sm flex items-start gap-3">
|
|
||||||
<span className="text-red-400 font-bold mt-0.5">●</span>
|
|
||||||
<div>
|
|
||||||
<span className="text-red-300 font-semibold">Metric anomaly detected — </span>
|
|
||||||
<span className="text-red-200">
|
|
||||||
{alertNodes.map((a) => a.node).join(', ')} deviating >2σ from baseline
|
|
||||||
{alertNodes.length === 1 && ` (score ${alertNodes[0].score.toFixed(1)})`}
|
|
||||||
</span>
|
|
||||||
</div>
|
|
||||||
</div>
|
|
||||||
)}
|
|
||||||
{alertNodes.length === 0 && warnNodes.length > 0 && (
|
|
||||||
<div className="rounded border border-amber-800 bg-amber-950/40 px-4 py-3 text-sm flex items-start gap-3">
|
|
||||||
<span className="text-amber-400 font-bold mt-0.5">●</span>
|
|
||||||
<div>
|
|
||||||
<span className="text-amber-300 font-semibold">Metric drift — </span>
|
|
||||||
<span className="text-amber-200">
|
|
||||||
{warnNodes.map((a) => a.node).join(', ')} showing unusual patterns
|
|
||||||
</span>
|
|
||||||
</div>
|
|
||||||
</div>
|
|
||||||
)}
|
|
||||||
|
|
||||||
{totals && totals.diskTotal > 0 && (
|
{totals && totals.diskTotal > 0 && (
|
||||||
<div className="rounded-lg border border-slate-800 bg-slate-900/60 p-4 flex items-center justify-between">
|
<div className="rounded-lg border border-slate-800 bg-slate-900/60 p-4 flex items-center justify-between">
|
||||||
<div>
|
<div>
|
||||||
@@ -125,11 +89,7 @@ export function CommandCenter() {
|
|||||||
<h2 className="text-lg font-semibold text-slate-100 mb-3">Nodes</h2>
|
<h2 className="text-lg font-semibold text-slate-100 mb-3">Nodes</h2>
|
||||||
<div className="grid grid-cols-1 sm:grid-cols-2 lg:grid-cols-3 gap-4">
|
<div className="grid grid-cols-1 sm:grid-cols-2 lg:grid-cols-3 gap-4">
|
||||||
{fleet?.nodes.map((n) => (
|
{fleet?.nodes.map((n) => (
|
||||||
<NodeCard
|
<NodeCard key={n.node_name} node={n} />
|
||||||
key={n.node_name}
|
|
||||||
node={n}
|
|
||||||
anomalyLevel={anomalyMap[n.node_name]}
|
|
||||||
/>
|
|
||||||
))}
|
))}
|
||||||
{!fleet &&
|
{!fleet &&
|
||||||
[1, 2, 3].map((i) => (
|
[1, 2, 3].map((i) => (
|
||||||
@@ -142,10 +102,6 @@ export function CommandCenter() {
|
|||||||
</section>
|
</section>
|
||||||
|
|
||||||
<ProjectsPanel />
|
<ProjectsPanel />
|
||||||
|
|
||||||
<WarmingCandidatesPanel />
|
|
||||||
|
|
||||||
<PollutionPanel />
|
|
||||||
</div>
|
</div>
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -3,7 +3,7 @@ import { useEffect, useState } from 'react';
|
|||||||
import { Link } from 'wouter';
|
import { Link } from 'wouter';
|
||||||
import { api, fmtBytes } from '../lib/api';
|
import { api, fmtBytes } from '../lib/api';
|
||||||
import { StatTile } from '../components/StatTile';
|
import { StatTile } from '../components/StatTile';
|
||||||
import { MaintenancePanel } from '../components/MaintenancePanel';
|
import { ShutdownPrepPanel } from '../components/ShutdownPrepPanel';
|
||||||
export function NodeDetail({ name }) {
|
export function NodeDetail({ name }) {
|
||||||
const [status, setStatus] = useState(null);
|
const [status, setStatus] = useState(null);
|
||||||
const [err, setErr] = useState(null);
|
const [err, setErr] = useState(null);
|
||||||
@@ -16,5 +16,5 @@ export function NodeDetail({ name }) {
|
|||||||
})
|
})
|
||||||
.catch((e) => setErr(String(e)));
|
.catch((e) => setErr(String(e)));
|
||||||
}, [name]);
|
}, [name]);
|
||||||
return (_jsxs("div", { className: "space-y-6", children: [_jsxs("div", { children: [_jsx(Link, { href: "/", children: _jsx("a", { className: "text-sm text-slate-500 hover:text-slate-300", children: "\u2190 fleet" }) }), _jsx("h1", { className: "text-2xl font-semibold text-slate-100 mt-2", children: name })] }), err && (_jsxs("div", { className: "rounded border border-amber-800 bg-amber-950/40 p-3 text-amber-300 text-sm", children: [err, _jsxs("div", { className: "mt-2 text-xs text-slate-400", children: ["Cross-node lookup lands in a follow-on PR. Until then this page shows detail only when you're already viewing the dashboard hosted by ", name, ". Try opening", ' ', _jsxs("span", { className: "font-mono", children: ["http://", name, ":7700/v2/#/nodes/", name] }), ' ', "directly."] })] })), status && (_jsxs(_Fragment, { children: [_jsxs("div", { className: "grid grid-cols-2 sm:grid-cols-3 lg:grid-cols-6 gap-3", children: [_jsx(StatTile, { label: "blobs", value: status.blob_count.toLocaleString(), color: "ok" }), _jsx(StatTile, { label: "tags", value: status.tag_count, color: "ok" }), _jsx(StatTile, { label: "refs", value: status.ref_count, color: "ok" }), _jsx(StatTile, { label: "snapshots", value: status.snapshot_count, color: "ok" }), _jsx(StatTile, { label: "ref-tracking", value: status.ref_tracking_count, color: "ok" }), _jsx(StatTile, { label: "store size", value: fmtBytes(status.blob_store_bytes), color: "ok" })] }), _jsxs("div", { className: "rounded border border-slate-800 bg-slate-900/40 p-4 text-sm", children: [_jsx("div", { className: "text-slate-500 uppercase text-xs tracking-wider", children: "blob store root" }), _jsx("div", { className: "font-mono mt-1", children: status.blob_store_root ?? '—' })] })] })), _jsx(MaintenancePanel, {})] }));
|
return (_jsxs("div", { className: "space-y-6", children: [_jsxs("div", { children: [_jsx(Link, { href: "/", children: _jsx("a", { className: "text-sm text-slate-500 hover:text-slate-300", children: "\u2190 fleet" }) }), _jsx("h1", { className: "text-2xl font-semibold text-slate-100 mt-2", children: name })] }), err && (_jsxs("div", { className: "rounded border border-amber-800 bg-amber-950/40 p-3 text-amber-300 text-sm", children: [err, _jsxs("div", { className: "mt-2 text-xs text-slate-400", children: ["Cross-node lookup lands in a follow-on PR. Until then this page shows detail only when you're already viewing the dashboard hosted by ", name, ". Try opening", ' ', _jsxs("span", { className: "font-mono", children: ["http://", name, ":7700/v2/#/nodes/", name] }), ' ', "directly."] })] })), status && (_jsxs(_Fragment, { children: [_jsxs("div", { className: "grid grid-cols-2 sm:grid-cols-3 lg:grid-cols-6 gap-3", children: [_jsx(StatTile, { label: "blobs", value: status.blob_count.toLocaleString(), color: "ok" }), _jsx(StatTile, { label: "tags", value: status.tag_count, color: "ok" }), _jsx(StatTile, { label: "refs", value: status.ref_count, color: "ok" }), _jsx(StatTile, { label: "snapshots", value: status.snapshot_count, color: "ok" }), _jsx(StatTile, { label: "ref-tracking", value: status.ref_tracking_count, color: "ok" }), _jsx(StatTile, { label: "store size", value: fmtBytes(status.blob_store_bytes), color: "ok" })] }), _jsxs("div", { className: "rounded border border-slate-800 bg-slate-900/40 p-4 text-sm", children: [_jsx("div", { className: "text-slate-500 uppercase text-xs tracking-wider", children: "blob store root" }), _jsx("div", { className: "font-mono mt-1", children: status.blob_store_root ?? '—' })] }), _jsx(ShutdownPrepPanel, { name: name })] }))] }));
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -2,7 +2,7 @@ import { useEffect, useState } from 'react';
|
|||||||
import { Link } from 'wouter';
|
import { Link } from 'wouter';
|
||||||
import { api, NodeStatusV2, fmtBytes } from '../lib/api';
|
import { api, NodeStatusV2, fmtBytes } from '../lib/api';
|
||||||
import { StatTile } from '../components/StatTile';
|
import { StatTile } from '../components/StatTile';
|
||||||
import { MaintenancePanel } from '../components/MaintenancePanel';
|
import { ShutdownPrepPanel } from '../components/ShutdownPrepPanel';
|
||||||
|
|
||||||
interface Props {
|
interface Props {
|
||||||
name: string;
|
name: string;
|
||||||
@@ -68,10 +68,10 @@ export function NodeDetail({ name }: Props) {
|
|||||||
</div>
|
</div>
|
||||||
<div className="font-mono mt-1">{status.blob_store_root ?? '—'}</div>
|
<div className="font-mono mt-1">{status.blob_store_root ?? '—'}</div>
|
||||||
</div>
|
</div>
|
||||||
|
|
||||||
|
<ShutdownPrepPanel name={name} />
|
||||||
</>
|
</>
|
||||||
)}
|
)}
|
||||||
|
|
||||||
<MaintenancePanel />
|
|
||||||
</div>
|
</div>
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1 +1 @@
|
|||||||
{"root":["./src/App.tsx","./src/main.tsx","./src/components/MaintenancePanel.tsx","./src/components/NodeCard.tsx","./src/components/NodeHistorySparklines.tsx","./src/components/PollutionPanel.tsx","./src/components/ProjectsPanel.tsx","./src/components/StatTile.tsx","./src/components/StorageBar.tsx","./src/components/WarmingCandidatesPanel.tsx","./src/lib/api.ts","./src/pages/CommandCenter.tsx","./src/pages/NodeDetail.tsx","./src/pages/RefTrackingPage.tsx","./src/pages/StorageBrowser.tsx"],"version":"6.0.3"}
|
{"root":["./src/App.tsx","./src/main.tsx","./src/components/NodeCard.tsx","./src/components/ProjectsPanel.tsx","./src/components/ShutdownPrepPanel.tsx","./src/components/StatTile.tsx","./src/components/StorageBar.tsx","./src/lib/api.ts","./src/pages/CommandCenter.tsx","./src/pages/NodeDetail.tsx","./src/pages/RefTrackingPage.tsx","./src/pages/StorageBrowser.tsx"],"version":"6.0.3"}
|
||||||
@@ -6,12 +6,15 @@ import react from '@vitejs/plugin-react';
|
|||||||
// During local dev the daemon proxies /api/v2/* on :7700 so
|
// During local dev the daemon proxies /api/v2/* on :7700 so
|
||||||
// `vite dev` on :5173 can hit it via server.proxy.
|
// `vite dev` on :5173 can hit it via server.proxy.
|
||||||
export default defineConfig({
|
export default defineConfig({
|
||||||
// Relative base so the SPA works under any mount point (/v2/,
|
// Absolute base matching the backend's actual mount point
|
||||||
// /clawstor/, or bare /). The no-trailing-slash edge case
|
// (`serve.rs` nests the v2 static dir at `/v2` via
|
||||||
// (browser treats the path as a file and strips the last segment)
|
// `nest_service("/v2", …)`). A prior `/clawstor/` base assumed a
|
||||||
// is handled server-side: serve.rs redirects /v2 → /v2/ and
|
// Tailscale Serve path mapping that was never actually configured
|
||||||
// /clawstor → /clawstor/ before serving the SPA.
|
// on any node (checked `tailscale serve status` on tank +
|
||||||
base: './',
|
// architect: neither proxies a `/clawstor` path) — that base
|
||||||
|
// silently broke direct `:7700/v2/` access, the only access
|
||||||
|
// pattern that's actually live.
|
||||||
|
base: '/v2/',
|
||||||
plugins: [react()],
|
plugins: [react()],
|
||||||
server: {
|
server: {
|
||||||
port: 5173,
|
port: 5173,
|
||||||
|
|||||||
Executable
+238
@@ -0,0 +1,238 @@
|
|||||||
|
#!/usr/bin/env bash
|
||||||
|
# safe-shutdown-prep.sh — bring a clawstor node to a clean, safe stop
|
||||||
|
# before hardware maintenance (parts replacement, drive swap, etc.).
|
||||||
|
#
|
||||||
|
# Run this ON the node you're about to power off. It does NOT power
|
||||||
|
# the machine off itself — the last line of output tells you the
|
||||||
|
# command to run for that, once everything above it is clean.
|
||||||
|
#
|
||||||
|
# What it does, in order:
|
||||||
|
# 1. Refuse to proceed if a cargo/rustc build is active against a
|
||||||
|
# tracked project's warm_path (unless --force).
|
||||||
|
# 2. Refuse to proceed if the sync queue has pending jobs peers
|
||||||
|
# haven't received yet (unless --force). Gives it one chance to
|
||||||
|
# drain via `claw-store sync <project>` before failing.
|
||||||
|
# 3. Take a final ZFS snapshot of the warm tier + replicate it to
|
||||||
|
# the configured cold peer, and wait for both to finish.
|
||||||
|
# 4. Stop the four maintenance timers (scrub/gc/ref-sweep/
|
||||||
|
# snapshot-rotate) so nothing fires mid-shutdown or immediately
|
||||||
|
# after next boot before you've verified the node.
|
||||||
|
# 5. Stop claw-store-serve.service (dashboard) — no data risk, just
|
||||||
|
# tidy.
|
||||||
|
# 6. Stop claw-store.service gracefully. The unit's
|
||||||
|
# TimeoutStopSec=60 gives the daemon's SIGTERM handler room to
|
||||||
|
# let gossip announce this node's departure to peers before the
|
||||||
|
# process exits — skipping this step means peers only notice via
|
||||||
|
# the failure detector's dead_node_grace_period (10s) instead of
|
||||||
|
# an immediate clean departure.
|
||||||
|
# 7. Stop claw-fuse.service and verify the mount is actually gone
|
||||||
|
# (retries a lazy unmount if the clean one doesn't take).
|
||||||
|
# 8. Sync filesystem buffers and print zpool health for the warm
|
||||||
|
# tier's pool — warns (does not block) if the pool is degraded,
|
||||||
|
# since that's independently worth knowing before you touch
|
||||||
|
# hardware.
|
||||||
|
#
|
||||||
|
# Flags:
|
||||||
|
# --force Skip the active-build and pending-sync guards.
|
||||||
|
# Everything else (steps 3-8) still runs.
|
||||||
|
# --export-zpool Additionally `zpool export` the warm-tier pool
|
||||||
|
# at the end — only do this if you're physically
|
||||||
|
# removing the storage drives, not for e.g. a RAM
|
||||||
|
# or PSU swap. Requires a matching `zpool import`
|
||||||
|
# after the node is back up before claw-store.service
|
||||||
|
# will find its data again.
|
||||||
|
# --skip-replicate Skip step 3 (snapshot + replicate). Use only if
|
||||||
|
# you already know cold tier is current, or this
|
||||||
|
# node has no [replication] configured.
|
||||||
|
# --dry-run Run every check (steps 1-2) and the snapshot/
|
||||||
|
# replicate (step 3) for real, but only print what
|
||||||
|
# steps 4-8 (stop timers/services, unmount, zpool
|
||||||
|
# export) would do instead of doing them. Use this
|
||||||
|
# first to verify the script sees your node's
|
||||||
|
# actual state correctly before trusting it live.
|
||||||
|
set -uo pipefail
|
||||||
|
|
||||||
|
CONFIG=${CLAWSTOR_CONFIG:-/etc/claw-store/config.toml}
|
||||||
|
BIN=${CLAWSTOR_BIN:-/usr/local/bin/claw-store}
|
||||||
|
SYNC_QUEUE=/var/lib/claw-store/sync-queue.toml
|
||||||
|
FORCE=0
|
||||||
|
EXPORT_ZPOOL=0
|
||||||
|
SKIP_REPLICATE=0
|
||||||
|
DRY_RUN=0
|
||||||
|
|
||||||
|
for arg in "$@"; do
|
||||||
|
case "$arg" in
|
||||||
|
--force) FORCE=1 ;;
|
||||||
|
--export-zpool) EXPORT_ZPOOL=1 ;;
|
||||||
|
--skip-replicate) SKIP_REPLICATE=1 ;;
|
||||||
|
--dry-run) DRY_RUN=1 ;;
|
||||||
|
*) echo "unknown flag: $arg" >&2; exit 2 ;;
|
||||||
|
esac
|
||||||
|
done
|
||||||
|
|
||||||
|
run() {
|
||||||
|
# Gate an actual state-changing command behind --dry-run.
|
||||||
|
if [ "$DRY_RUN" -eq 1 ]; then
|
||||||
|
echo " [dry-run] would run: $*"
|
||||||
|
return 0
|
||||||
|
fi
|
||||||
|
"$@"
|
||||||
|
}
|
||||||
|
|
||||||
|
hr() { printf '%.0s─' {1..66}; echo; }
|
||||||
|
step() { hr; echo "▶ $1"; hr; }
|
||||||
|
ok() { echo " ✓ $1"; }
|
||||||
|
warn() { echo " ! $1"; }
|
||||||
|
fail() { echo " ✗ $1" >&2; }
|
||||||
|
|
||||||
|
NODE=$(hostname)
|
||||||
|
echo "safe-shutdown-prep — $NODE — $(date -Iseconds)"
|
||||||
|
|
||||||
|
# ── 1. Active builds ────────────────────────────────────────────────
|
||||||
|
step "checking for active cargo/rustc builds"
|
||||||
|
ACTIVE=$(pgrep -af 'cargo|rustc' | grep -v "safe-shutdown-prep\|grep" || true)
|
||||||
|
if [ -n "$ACTIVE" ]; then
|
||||||
|
echo "$ACTIVE" | sed 's/^/ /'
|
||||||
|
if [ "$FORCE" -eq 1 ]; then
|
||||||
|
warn "active build(s) found — continuing anyway (--force)"
|
||||||
|
else
|
||||||
|
fail "active build(s) found on this node. A build in progress against"
|
||||||
|
fail "the warm tier can be interrupted mid-write by an unmount/shutdown."
|
||||||
|
fail "Wait for it to finish, or re-run with --force to proceed anyway."
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
else
|
||||||
|
ok "no active cargo/rustc processes"
|
||||||
|
fi
|
||||||
|
|
||||||
|
# ── 2. Sync queue ────────────────────────────────────────────────────
|
||||||
|
step "checking sync queue for pending peer pushes"
|
||||||
|
if [ -f "$SYNC_QUEUE" ]; then
|
||||||
|
DEPTH=$(grep -c '^project = ' "$SYNC_QUEUE" 2>/dev/null); DEPTH=${DEPTH:-0}
|
||||||
|
else
|
||||||
|
DEPTH=0
|
||||||
|
fi
|
||||||
|
if [ "$DEPTH" -gt 0 ]; then
|
||||||
|
warn "$DEPTH pending sync job(s) in $SYNC_QUEUE — attempting to drain"
|
||||||
|
PROJECTS=$(grep '^project = ' "$SYNC_QUEUE" | sed 's/project = "\(.*\)"/\1/')
|
||||||
|
for p in $PROJECTS; do
|
||||||
|
echo " syncing $p ..."
|
||||||
|
"$BIN" --config "$CONFIG" sync "$p" || warn "sync failed for $p"
|
||||||
|
done
|
||||||
|
DEPTH_AFTER=$(grep -c '^project = ' "$SYNC_QUEUE" 2>/dev/null); DEPTH_AFTER=${DEPTH_AFTER:-0}
|
||||||
|
if [ "$DEPTH_AFTER" -gt 0 ]; then
|
||||||
|
if [ "$FORCE" -eq 1 ]; then
|
||||||
|
warn "$DEPTH_AFTER job(s) still pending — continuing anyway (--force)"
|
||||||
|
else
|
||||||
|
fail "$DEPTH_AFTER sync job(s) still pending after drain attempt."
|
||||||
|
fail "Peers may be unreachable, or the push is failing for another"
|
||||||
|
fail "reason. Re-run with --force to shut down anyway (those changes"
|
||||||
|
fail "will catch up once this node is back and the daemon retries)."
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
else
|
||||||
|
ok "sync queue drained"
|
||||||
|
fi
|
||||||
|
else
|
||||||
|
ok "sync queue empty"
|
||||||
|
fi
|
||||||
|
|
||||||
|
# ── 3. Final snapshot + replicate to cold ───────────────────────────
|
||||||
|
if [ "$SKIP_REPLICATE" -eq 1 ]; then
|
||||||
|
step "skipping snapshot + replicate (--skip-replicate)"
|
||||||
|
else
|
||||||
|
step "taking final snapshot + replicating to cold tier"
|
||||||
|
if "$BIN" --config "$CONFIG" snapshot; then
|
||||||
|
ok "snapshot created"
|
||||||
|
else
|
||||||
|
warn "snapshot command failed — check output above"
|
||||||
|
fi
|
||||||
|
if "$BIN" --config "$CONFIG" replicate; then
|
||||||
|
ok "replication to cold tier complete"
|
||||||
|
else
|
||||||
|
warn "replicate command failed or not configured — check output above"
|
||||||
|
warn "([replication] section may be absent on this node; that's fine)"
|
||||||
|
fi
|
||||||
|
fi
|
||||||
|
|
||||||
|
# ── 4. Stop maintenance timers ──────────────────────────────────────
|
||||||
|
step "stopping maintenance timers"
|
||||||
|
for t in clawstor-scrub clawstor-gc clawstor-ref-sweep clawstor-snapshot-rotate; do
|
||||||
|
run systemctl --user stop "$t.timer" 2>/dev/null && ok "$t.timer stopped" || warn "$t.timer not running or not found"
|
||||||
|
done
|
||||||
|
|
||||||
|
# ── 5. Stop dashboard ────────────────────────────────────────────────
|
||||||
|
step "stopping claw-store-serve.service"
|
||||||
|
if systemctl is-active --quiet claw-store-serve.service 2>/dev/null; then
|
||||||
|
run sudo systemctl stop claw-store-serve.service && ok "stopped" || fail "failed to stop"
|
||||||
|
else
|
||||||
|
ok "not running"
|
||||||
|
fi
|
||||||
|
|
||||||
|
# ── 6. Stop daemon (gossip departure) ───────────────────────────────
|
||||||
|
step "stopping claw-store.service (gossip will announce departure)"
|
||||||
|
if systemctl is-active --quiet claw-store.service 2>/dev/null; then
|
||||||
|
run sudo systemctl stop claw-store.service && ok "stopped cleanly" || fail "failed to stop — check 'systemctl status claw-store.service'"
|
||||||
|
else
|
||||||
|
ok "not running"
|
||||||
|
fi
|
||||||
|
|
||||||
|
# ── 7. Unmount FUSE ──────────────────────────────────────────────────
|
||||||
|
step "unmounting FUSE"
|
||||||
|
if systemctl is-active --quiet claw-fuse.service 2>/dev/null; then
|
||||||
|
run sudo systemctl stop claw-fuse.service
|
||||||
|
sleep 1
|
||||||
|
fi
|
||||||
|
if [ "$DRY_RUN" -eq 1 ]; then
|
||||||
|
if mount | grep -q "type fuse.clawstor"; then
|
||||||
|
warn "still mounted (expected — nothing was actually stopped in --dry-run)"
|
||||||
|
else
|
||||||
|
ok "already unmounted"
|
||||||
|
fi
|
||||||
|
else
|
||||||
|
if mount | grep -q "type fuse.clawstor"; then
|
||||||
|
warn "still mounted after service stop — trying lazy unmount"
|
||||||
|
sudo umount -l ~/clawstor-mount 2>/dev/null
|
||||||
|
sleep 1
|
||||||
|
fi
|
||||||
|
if mount | grep -q "type fuse.clawstor"; then
|
||||||
|
fail "FUSE mount would not come down: $(mount | grep 'fuse.clawstor')"
|
||||||
|
fail "Do not power off until this is resolved — an unclean FUSE"
|
||||||
|
fail "unmount can leave a stale mountpoint that needs manual cleanup"
|
||||||
|
fail "on next boot."
|
||||||
|
exit 1
|
||||||
|
else
|
||||||
|
ok "unmounted"
|
||||||
|
fi
|
||||||
|
fi
|
||||||
|
|
||||||
|
# ── 8. Flush + pool health ──────────────────────────────────────────
|
||||||
|
step "flushing filesystem buffers"
|
||||||
|
sync
|
||||||
|
ok "sync complete"
|
||||||
|
|
||||||
|
step "zpool health check"
|
||||||
|
if ! command -v zpool >/dev/null 2>&1; then
|
||||||
|
ok "no zpool binary on this node — warm tier is not ZFS-backed here, nothing to check"
|
||||||
|
else
|
||||||
|
POOL=$(df --output=source /slab 2>/dev/null | tail -1 | tr -d '[:space:]')
|
||||||
|
if [ -n "$POOL" ] && [ "$POOL" != "none" ]; then
|
||||||
|
echo " pool: $POOL"
|
||||||
|
zpool status -x "$POOL" 2>&1 | sed 's/^/ /'
|
||||||
|
if [ "$EXPORT_ZPOOL" -eq 1 ]; then
|
||||||
|
step "exporting $POOL (--export-zpool)"
|
||||||
|
run sudo zpool export "$POOL" && ok "exported — remember: zpool import $POOL after reboot" || fail "export failed"
|
||||||
|
fi
|
||||||
|
else
|
||||||
|
warn "zpool present but /slab isn't a recognizable ZFS mount"
|
||||||
|
fi
|
||||||
|
fi
|
||||||
|
|
||||||
|
hr
|
||||||
|
if [ "$DRY_RUN" -eq 1 ]; then
|
||||||
|
echo "DRY RUN COMPLETE — nothing was actually stopped or unmounted."
|
||||||
|
echo "Re-run without --dry-run when ready to actually prep for shutdown."
|
||||||
|
else
|
||||||
|
echo "SAFE TO POWER OFF — run: sudo shutdown -h now"
|
||||||
|
fi
|
||||||
|
hr
|
||||||
Reference in New Issue
Block a user