Compare commits
8
Commits
2d0c225f98
..
main
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
bc2996e536 | ||
|
|
ef02b22b72 | ||
|
|
cc26052cfe | ||
|
|
11a259b762 | ||
|
|
ec24f37d90 | ||
|
|
e5fcb8b3f4 | ||
|
|
9debd84e95 | ||
|
|
f38efc7096 |
@@ -655,6 +655,17 @@ fn decode_nibble(b: u8) -> Option<u8> {
|
||||
|
||||
/// statvfs on the given path. Uses libc directly — cheap enough
|
||||
/// that we don't need to cache. Silent on error (returns None).
|
||||
///
|
||||
/// Bug fix 2026-08-02: `used` was computed as `total - f_bavail`.
|
||||
/// `f_bavail` is space available to an *unprivileged* user, which
|
||||
/// excludes ext4's reserved-blocks-for-root margin (~5% of the
|
||||
/// filesystem by default) — so that formula silently folded the
|
||||
/// entire reserved margin into "used", overstating usage by exactly
|
||||
/// that amount on every node (worse on bigger disks: ~21GB on a
|
||||
/// 466GB root, proportionally more on multi-TB ones). `df`'s Used
|
||||
/// column is `total - f_bfree` (raw free blocks, root-reserved or
|
||||
/// not) — matching that here is what makes the dashboard agree with
|
||||
/// `df` instead of silently running high.
|
||||
fn filesystem_usage(path: &std::path::Path) -> Option<FilesystemUsage> {
|
||||
let cpath = std::ffi::CString::new(path.as_os_str().to_str()?).ok()?;
|
||||
// SAFETY: statvfs writes to a zero-initialised struct; we
|
||||
@@ -667,7 +678,8 @@ fn filesystem_usage(path: &std::path::Path) -> Option<FilesystemUsage> {
|
||||
let bsize = stat.f_frsize as u64;
|
||||
let total = stat.f_blocks as u64 * bsize;
|
||||
let avail = stat.f_bavail as u64 * bsize;
|
||||
let used = total.saturating_sub(avail);
|
||||
let free = stat.f_bfree as u64 * bsize;
|
||||
let used = total.saturating_sub(free);
|
||||
Some(FilesystemUsage {
|
||||
mount_point: path.display().to_string(),
|
||||
total_bytes: total,
|
||||
@@ -676,6 +688,34 @@ fn filesystem_usage(path: &std::path::Path) -> Option<FilesystemUsage> {
|
||||
})
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod filesystem_usage_tests {
|
||||
use super::filesystem_usage;
|
||||
|
||||
#[test]
|
||||
fn used_plus_available_never_exceeds_total() {
|
||||
// `available` is a subset of the space `used` now excludes
|
||||
// (used = total - free, and available = f_bavail <= f_bfree),
|
||||
// so this invariant holds regardless of any root-reserved
|
||||
// margin -- the regression this guards against is `used`
|
||||
// being computed as `total - available`, which collapses
|
||||
// that margin into `used` and can make `used + available`
|
||||
// overshoot `total` in the other direction (it wouldn't here,
|
||||
// but the two numbers would silently disagree with `df`).
|
||||
let fs = filesystem_usage(std::path::Path::new(".")).expect("statvfs on cwd");
|
||||
assert!(fs.total_bytes > 0);
|
||||
assert!(fs.used_bytes <= fs.total_bytes);
|
||||
assert!(fs.available_bytes <= fs.total_bytes.saturating_sub(fs.used_bytes) + 4096,
|
||||
"available ({}) should fit within total-used ({}) modulo one block; got used={}, total={}",
|
||||
fs.available_bytes, fs.total_bytes.saturating_sub(fs.used_bytes), fs.used_bytes, fs.total_bytes);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn missing_path_returns_none() {
|
||||
assert!(filesystem_usage(std::path::Path::new("/this/path/does/not/exist/at/all")).is_none());
|
||||
}
|
||||
}
|
||||
|
||||
/// Query systemd for a user-scope timer's next-fire + last result.
|
||||
/// Shells out to systemctl. Silent on any failure — dashboards
|
||||
/// should degrade to "unknown" rather than 500.
|
||||
|
||||
@@ -87,16 +87,24 @@ pub async fn check() -> Result<ShutdownPrepCheckReply> {
|
||||
if !script.exists() {
|
||||
bail!("shutdown-prep script not found at {}", script.display());
|
||||
}
|
||||
// `2>&1` inside the shell merges stderr into stdout *before*
|
||||
// either stream is piped back to us, preserving true
|
||||
// chronological order. Capturing stdout/stderr separately (as
|
||||
// `Command::output()` does by default) and concatenating them
|
||||
// after the fact loses interleaving entirely — every stderr line
|
||||
// lands at the very end regardless of when it was actually
|
||||
// printed, which makes a mid-script warning (e.g. "replicate not
|
||||
// configured on this node") look like a failure that happened
|
||||
// after "DRY RUN COMPLETE".
|
||||
let run = Command::new("bash")
|
||||
.arg(&script)
|
||||
.arg("--dry-run")
|
||||
.arg("-c")
|
||||
.arg(format!("{} --dry-run 2>&1", script.display()))
|
||||
.output();
|
||||
let output = timeout(CHECK_TIMEOUT, run)
|
||||
.await
|
||||
.context("shutdown-prep --dry-run timed out")?
|
||||
.context("spawning shutdown-prep --dry-run")?;
|
||||
let mut combined = String::from_utf8_lossy(&output.stdout).into_owned();
|
||||
combined.push_str(&String::from_utf8_lossy(&output.stderr));
|
||||
let combined = String::from_utf8_lossy(&output.stdout).into_owned();
|
||||
Ok(ShutdownPrepCheckReply {
|
||||
ready: output.status.success(),
|
||||
output: combined,
|
||||
|
||||
@@ -27,12 +27,26 @@ pub struct HotConfig {
|
||||
#[derive(Debug, Clone, Deserialize, Serialize)]
|
||||
pub struct WarmConfig {
|
||||
pub projects_path: PathBuf,
|
||||
/// Dataset name for `zfs`/`zpool` operations against the warm
|
||||
/// tier. The literal value `"none"` means this node's warm tier
|
||||
/// is a plain directory, not ZFS-backed (e.g. a build node with
|
||||
/// no ZFS pool) — snapshot/replicate become no-ops instead of
|
||||
/// erroring on a missing `zfs`/`zpool` binary. See
|
||||
/// [`WarmConfig::zfs_enabled`].
|
||||
pub zfs_dataset: String,
|
||||
pub snapshot_retain_hours: u64,
|
||||
pub snapshot_retain_days: u64,
|
||||
pub snapshot_retain_weeks: u64,
|
||||
}
|
||||
|
||||
impl WarmConfig {
|
||||
/// `false` when `zfs_dataset = "none"` — this node's warm tier
|
||||
/// has no ZFS pool underneath it.
|
||||
pub fn zfs_enabled(&self) -> bool {
|
||||
self.zfs_dataset != "none"
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Deserialize, Serialize)]
|
||||
pub struct ColdConfig {
|
||||
pub archive_path: PathBuf,
|
||||
|
||||
+16
-10
@@ -188,19 +188,25 @@ pub async fn run(cfg: Config, mut manifest: Manifest) -> Result<()> {
|
||||
}
|
||||
}
|
||||
_ = snap_tick.tick() => {
|
||||
let ts = chrono::Utc::now().format("%Y-%m-%d-%H%M").to_string();
|
||||
tracing::info!("taking snapshot {}", ts);
|
||||
if let Err(e) = snapshot::run_snapshot_cycle(
|
||||
&zfs, &cfg.warm.zfs_dataset, &ts,
|
||||
cfg.warm.snapshot_retain_hours as usize,
|
||||
cfg.warm.snapshot_retain_days as usize,
|
||||
cfg.warm.snapshot_retain_weeks as usize,
|
||||
) {
|
||||
tracing::error!("snapshot failed: {:#}", e);
|
||||
if !cfg.warm.zfs_enabled() {
|
||||
tracing::debug!("skipping snapshot tick — zfs_dataset = \"none\" on this node");
|
||||
} else {
|
||||
let ts = chrono::Utc::now().format("%Y-%m-%d-%H%M").to_string();
|
||||
tracing::info!("taking snapshot {}", ts);
|
||||
if let Err(e) = snapshot::run_snapshot_cycle(
|
||||
&zfs, &cfg.warm.zfs_dataset, &ts,
|
||||
cfg.warm.snapshot_retain_hours as usize,
|
||||
cfg.warm.snapshot_retain_days as usize,
|
||||
cfg.warm.snapshot_retain_weeks as usize,
|
||||
) {
|
||||
tracing::error!("snapshot failed: {:#}", e);
|
||||
}
|
||||
}
|
||||
}
|
||||
_ = repl_tick.tick() => {
|
||||
if let Some(rep) = &cfg.replication {
|
||||
if !cfg.warm.zfs_enabled() {
|
||||
tracing::debug!("skipping replication tick — zfs_dataset = \"none\" on this node");
|
||||
} else if let Some(rep) = &cfg.replication {
|
||||
if let (Some(host), Some(user), Some(dest)) = (
|
||||
&rep.send_to_host, &rep.send_to_user, &rep.cold_dataset_on_peer
|
||||
) {
|
||||
|
||||
@@ -1502,6 +1502,10 @@ fn cmd_gc(cfg: &Config, manifest: &mut Manifest) -> Result<()> {
|
||||
}
|
||||
|
||||
fn cmd_snapshot(cfg: &Config, zfs: &SystemZfs) -> Result<()> {
|
||||
if !cfg.warm.zfs_enabled() {
|
||||
println!("Warm tier is not ZFS-backed on this node (zfs_dataset = \"none\") — nothing to snapshot.");
|
||||
return Ok(());
|
||||
}
|
||||
let ts = chrono::Utc::now().format("%Y-%m-%d-%H%M").to_string();
|
||||
snapshot::run_snapshot_cycle(
|
||||
zfs, &cfg.warm.zfs_dataset, &ts,
|
||||
@@ -1514,6 +1518,10 @@ fn cmd_snapshot(cfg: &Config, zfs: &SystemZfs) -> Result<()> {
|
||||
}
|
||||
|
||||
fn cmd_list_snapshots(cfg: &Config, zfs: &SystemZfs, _project: &str) -> Result<()> {
|
||||
if !cfg.warm.zfs_enabled() {
|
||||
println!("Warm tier is not ZFS-backed on this node (zfs_dataset = \"none\").");
|
||||
return Ok(());
|
||||
}
|
||||
let snaps = zfs.list_snapshots(&cfg.warm.zfs_dataset)?;
|
||||
if snaps.is_empty() { println!("No snapshots found."); return Ok(()); }
|
||||
for s in &snaps { println!(" {}", s); }
|
||||
@@ -1531,6 +1539,10 @@ fn cmd_restore(cfg: &Config, zfs: &SystemZfs, project: &str, snap: &str) -> Resu
|
||||
}
|
||||
|
||||
fn cmd_replicate(cfg: &Config, zfs: &SystemZfs) -> Result<()> {
|
||||
if !cfg.warm.zfs_enabled() {
|
||||
println!("Warm tier is not ZFS-backed on this node (zfs_dataset = \"none\") — nothing to replicate.");
|
||||
return Ok(());
|
||||
}
|
||||
let rep = cfg.replication.as_ref()
|
||||
.context("no replication config — this node does not replicate")?;
|
||||
let host = rep.send_to_host.as_ref().context("send_to_host not set")?;
|
||||
|
||||
+146
-4
@@ -66,6 +66,23 @@ pub fn replicate_to_cold(
|
||||
remote_user: &str,
|
||||
remote_host: &str,
|
||||
remote_dataset: &str,
|
||||
) -> Result<()> {
|
||||
replicate_to_cold_with_state_path(
|
||||
zfs, dataset, remote_user, remote_host, remote_dataset,
|
||||
std::path::Path::new(LAST_REPLICATED_PATH),
|
||||
)
|
||||
}
|
||||
|
||||
/// Same as [`replicate_to_cold`] with the state-file path injectable
|
||||
/// -- lets tests exercise the baseline-recovery logic against a temp
|
||||
/// file instead of the real `/var/lib/claw-store/...` path.
|
||||
pub fn replicate_to_cold_with_state_path(
|
||||
zfs: &dyn ZfsOps,
|
||||
dataset: &str,
|
||||
remote_user: &str,
|
||||
remote_host: &str,
|
||||
remote_dataset: &str,
|
||||
state_path: &std::path::Path,
|
||||
) -> Result<()> {
|
||||
let snaps = zfs.list_snapshots(dataset)?;
|
||||
let latest = match snaps.last().cloned() {
|
||||
@@ -77,12 +94,64 @@ pub fn replicate_to_cold(
|
||||
};
|
||||
|
||||
// Use the last successfully replicated snapshot as the incremental base.
|
||||
// Only valid if it still exists in the current snapshot list.
|
||||
let prev = std::fs::read_to_string(LAST_REPLICATED_PATH)
|
||||
// Only valid if it still exists in the current snapshot list -- local
|
||||
// retention (snapshot_retain_hours) can prune it out from under us
|
||||
// between replication runs (e.g. a daemon restart resets the
|
||||
// replication tick's 24h timer without resetting the hourly-snapshot
|
||||
// pruning tick, so a slow/interrupted replication cadence can let the
|
||||
// recorded snapshot age out locally before it's ever used again).
|
||||
let recorded = std::fs::read_to_string(state_path)
|
||||
.ok()
|
||||
.map(|s| s.trim().to_string())
|
||||
.filter(|s| !s.is_empty() && snaps.contains(s));
|
||||
|
||||
// Recorded snapshot is gone -- don't give up and fall back to a full
|
||||
// send (which fails outright against a non-empty destination, as
|
||||
// opposed to just being wasteful). Ask the remote what it actually
|
||||
// has and find the newest snapshot both sides still share, by tag
|
||||
// (the `@kind-timestamp` suffix -- dataset paths differ between
|
||||
// source and destination, e.g. slab/projects vs
|
||||
// data/archive/tank-projects, but tags are written identically).
|
||||
let prev = match recorded {
|
||||
Some(p) => Some(p),
|
||||
None => {
|
||||
match zfs.list_remote_snapshots(remote_user, remote_host, remote_dataset) {
|
||||
Ok(remote_snaps) => {
|
||||
let remote_tags: std::collections::HashSet<&str> = remote_snaps
|
||||
.iter()
|
||||
.filter_map(|s| s.split_once('@').map(|(_, tag)| tag))
|
||||
.collect();
|
||||
let fallback = snaps
|
||||
.iter()
|
||||
.rev()
|
||||
.skip(1) // exclude `latest` itself
|
||||
.find(|s| {
|
||||
s.split_once('@')
|
||||
.map(|(_, tag)| remote_tags.contains(tag))
|
||||
.unwrap_or(false)
|
||||
})
|
||||
.cloned();
|
||||
if let Some(f) = &fallback {
|
||||
tracing::warn!(
|
||||
"recorded replication baseline was pruned locally; \
|
||||
recovered a common snapshot from the remote instead: {}",
|
||||
f
|
||||
);
|
||||
}
|
||||
fallback
|
||||
}
|
||||
Err(e) => {
|
||||
tracing::warn!(
|
||||
error = %e,
|
||||
"could not query remote snapshots to recover a replication baseline; \
|
||||
falling back to full send"
|
||||
);
|
||||
None
|
||||
}
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
if prev.as_deref() == Some(latest.as_str()) {
|
||||
tracing::info!("replication already up to date ({})", latest);
|
||||
return Ok(());
|
||||
@@ -99,10 +168,10 @@ pub fn replicate_to_cold(
|
||||
zfs.send_to_remote(&latest, prev.as_deref(), remote_user, remote_host, remote_dataset)?;
|
||||
|
||||
// Record this snapshot as the new baseline for the next incremental send.
|
||||
if let Some(parent) = std::path::Path::new(LAST_REPLICATED_PATH).parent() {
|
||||
if let Some(parent) = state_path.parent() {
|
||||
let _ = std::fs::create_dir_all(parent);
|
||||
}
|
||||
std::fs::write(LAST_REPLICATED_PATH, &latest)?;
|
||||
std::fs::write(state_path, &latest)?;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
@@ -177,4 +246,77 @@ mod tests {
|
||||
assert!(!is_sunday_midnight("2026-06-29-0000")); // Monday
|
||||
assert!(!is_sunday_midnight("2026-06-28-0100")); // Sunday but not midnight
|
||||
}
|
||||
|
||||
/// Unique temp path per test so parallel test runs don't clobber
|
||||
/// each other's replication-baseline state file.
|
||||
fn temp_state_path(name: &str) -> std::path::PathBuf {
|
||||
std::env::temp_dir().join(format!("clawstor-test-last-replicated-{name}-{}", std::process::id()))
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn replicate_uses_recorded_baseline_when_still_present() {
|
||||
let zfs = MockZfs::default();
|
||||
zfs.snapshot("slab/projects", "hourly-1").unwrap();
|
||||
zfs.snapshot("slab/projects", "hourly-2").unwrap();
|
||||
let state = temp_state_path("recorded-present");
|
||||
std::fs::write(&state, "slab/projects@hourly-1").unwrap();
|
||||
|
||||
replicate_to_cold_with_state_path(
|
||||
&zfs, "slab/projects", "user", "host", "remote/ds", &state,
|
||||
).unwrap();
|
||||
|
||||
let sends = zfs.sends();
|
||||
assert_eq!(sends.len(), 1);
|
||||
assert_eq!(sends[0].0, "slab/projects@hourly-2");
|
||||
assert_eq!(sends[0].1.as_deref(), Some("slab/projects@hourly-1"));
|
||||
let _ = std::fs::remove_file(&state);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn replicate_recovers_baseline_from_remote_when_recorded_one_was_pruned() {
|
||||
let zfs = MockZfs::default();
|
||||
// Local retention already pruned hourly-1 -- only hourly-2 and
|
||||
// hourly-3 remain locally. The remote, however, still has
|
||||
// hourly-2 (it just hasn't received hourly-3 yet).
|
||||
zfs.snapshot("slab/projects", "hourly-2").unwrap();
|
||||
zfs.snapshot("slab/projects", "hourly-3").unwrap();
|
||||
zfs.set_remote_snapshots(vec![
|
||||
"remote/ds@hourly-1".to_string(),
|
||||
"remote/ds@hourly-2".to_string(),
|
||||
]);
|
||||
let state = temp_state_path("recovers-from-remote");
|
||||
// Recorded baseline (hourly-1) no longer exists locally.
|
||||
std::fs::write(&state, "slab/projects@hourly-1").unwrap();
|
||||
|
||||
replicate_to_cold_with_state_path(
|
||||
&zfs, "slab/projects", "user", "host", "remote/ds", &state,
|
||||
).unwrap();
|
||||
|
||||
let sends = zfs.sends();
|
||||
assert_eq!(sends.len(), 1);
|
||||
assert_eq!(sends[0].0, "slab/projects@hourly-3");
|
||||
// Recovered hourly-2 as the base by matching tags against the
|
||||
// remote's actual snapshot list, NOT a full send.
|
||||
assert_eq!(sends[0].1.as_deref(), Some("slab/projects@hourly-2"));
|
||||
let _ = std::fs::remove_file(&state);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn replicate_falls_back_to_full_send_when_truly_no_common_snapshot() {
|
||||
let zfs = MockZfs::default();
|
||||
zfs.snapshot("slab/projects", "hourly-9").unwrap();
|
||||
zfs.set_remote_snapshots(vec!["remote/ds@hourly-1".to_string()]);
|
||||
let state = temp_state_path("no-common-snapshot");
|
||||
let _ = std::fs::remove_file(&state); // no recorded baseline at all
|
||||
|
||||
replicate_to_cold_with_state_path(
|
||||
&zfs, "slab/projects", "user", "host", "remote/ds", &state,
|
||||
).unwrap();
|
||||
|
||||
let sends = zfs.sends();
|
||||
assert_eq!(sends.len(), 1);
|
||||
assert_eq!(sends[0].0, "slab/projects@hourly-9");
|
||||
assert_eq!(sends[0].1, None, "no common tag exists -- must fall back to full send");
|
||||
let _ = std::fs::remove_file(&state);
|
||||
}
|
||||
}
|
||||
|
||||
+50
-1
@@ -13,6 +13,12 @@ pub trait ZfsOps: Send + Sync {
|
||||
remote_user: &str, remote_host: &str,
|
||||
remote_dataset: &str) -> Result<()>;
|
||||
fn clone_snapshot(&self, snapshot: &str, dest_dataset: &str) -> Result<()>;
|
||||
/// List snapshot names (full `dataset@tag` form, remote-side
|
||||
/// naming) currently on a remote dataset over SSH. Used to find a
|
||||
/// real incremental base when the locally-recorded one has been
|
||||
/// pruned -- see `snapshot::replicate_to_cold`.
|
||||
fn list_remote_snapshots(&self, remote_user: &str, remote_host: &str,
|
||||
remote_dataset: &str) -> Result<Vec<String>>;
|
||||
}
|
||||
|
||||
pub struct SystemZfs;
|
||||
@@ -89,11 +95,48 @@ impl ZfsOps for SystemZfs {
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn list_remote_snapshots(&self, remote_user: &str, remote_host: &str,
|
||||
remote_dataset: &str) -> Result<Vec<String>> {
|
||||
let out = std::process::Command::new("ssh")
|
||||
.args([
|
||||
&format!("{remote_user}@{remote_host}"),
|
||||
"zfs", "list", "-H", "-t", "snapshot", "-o", "name", "-r", remote_dataset,
|
||||
])
|
||||
.output()
|
||||
.context("running ssh zfs list on remote")?;
|
||||
if !out.status.success() {
|
||||
bail!("remote zfs list failed: {}", String::from_utf8_lossy(&out.stderr));
|
||||
}
|
||||
Ok(String::from_utf8_lossy(&out.stdout)
|
||||
.lines()
|
||||
.filter(|l| !l.is_empty())
|
||||
.map(String::from)
|
||||
.collect())
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Default, Clone)]
|
||||
pub struct MockZfs {
|
||||
snapshots: Arc<Mutex<Vec<String>>>,
|
||||
/// Configurable via `set_remote_snapshots` -- what
|
||||
/// `list_remote_snapshots` returns, for exercising the
|
||||
/// baseline-recovery path in `snapshot::replicate_to_cold`.
|
||||
remote_snapshots: Arc<Mutex<Vec<String>>>,
|
||||
/// Every `send_to_remote` call, recorded as `(snapshot,
|
||||
/// incremental_from)`, so tests can assert which base was
|
||||
/// actually used.
|
||||
sends: Arc<Mutex<Vec<(String, Option<String>)>>>,
|
||||
}
|
||||
|
||||
impl MockZfs {
|
||||
pub fn set_remote_snapshots(&self, snaps: Vec<String>) {
|
||||
*self.remote_snapshots.lock().unwrap() = snaps;
|
||||
}
|
||||
|
||||
pub fn sends(&self) -> Vec<(String, Option<String>)> {
|
||||
self.sends.lock().unwrap().clone()
|
||||
}
|
||||
}
|
||||
|
||||
impl ZfsOps for MockZfs {
|
||||
@@ -115,8 +158,9 @@ impl ZfsOps for MockZfs {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn send_to_remote(&self, _snap: &str, _incr: Option<&str>,
|
||||
fn send_to_remote(&self, snap: &str, incr: Option<&str>,
|
||||
_user: &str, _host: &str, _dest: &str) -> Result<()> {
|
||||
self.sends.lock().unwrap().push((snap.to_string(), incr.map(String::from)));
|
||||
Ok(())
|
||||
}
|
||||
|
||||
@@ -125,6 +169,11 @@ impl ZfsOps for MockZfs {
|
||||
.push(format!("{} -> {}", snapshot, dest_dataset));
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn list_remote_snapshots(&self, _remote_user: &str, _remote_host: &str,
|
||||
_remote_dataset: &str) -> Result<Vec<String>> {
|
||||
Ok(self.remote_snapshots.lock().unwrap().clone())
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
|
||||
@@ -24,7 +24,7 @@ export function NodeCard({ node }) {
|
||||
border,
|
||||
].join(' '), children: [_jsxs("div", { className: "flex items-center justify-between", children: [_jsxs("div", { className: "flex items-center gap-2", children: [_jsx("span", { className: `inline-block w-2.5 h-2.5 rounded-full ${dot}` }), _jsx("span", { className: "text-lg font-semibold text-slate-100", children: node.node_name })] }), _jsx("span", { className: "text-xs text-slate-500 font-mono", children: node.zone || '—' })] }), !node.online && (_jsx("div", { className: "text-sm text-red-400 break-words", children: node.error ?? 'offline' })), node.online && (_jsxs(_Fragment, { children: [node.filesystem && (_jsx(StorageBar, { label: "disk", used: node.filesystem.used_bytes, total: node.filesystem.total_bytes })), node.hot && node.hot.max_bytes > 0 && (_jsx(StorageBar, { label: "hot tier", used: node.hot.used_bytes, total: node.hot.max_bytes, pinned: node.hot.pinned_bytes ?? undefined })), _jsxs("div", { className: "grid grid-cols-2 gap-y-1 text-sm", children: [_jsx("span", { className: "text-slate-500", children: "mount" }), _jsx("span", { className: "text-right font-mono text-xs", children: node.mount?.active ? (_jsx("span", { className: "text-emerald-300", children: "\u2713 mounted" })) : (_jsx("span", { className: "text-slate-500", children: "not mounted" })) }), _jsx("span", { className: "text-slate-500", children: "cache hit rate" }), _jsx("span", { className: "text-right font-mono text-xs", children: node.cache && node.cache.hits + node.cache.misses > 0
|
||||
? `${Math.round(node.cache.hit_rate * 100)}%`
|
||||
: _jsx("span", { className: "text-slate-500", children: "idle" }) }), _jsx("span", { className: "text-slate-500", children: "next scheduled job" }), _jsx("span", { className: "text-right font-mono text-xs", children: nextTimer(node) })] })] }))] }) }));
|
||||
: _jsx("span", { className: "text-slate-500", children: "idle" }) }), _jsx("span", { className: "text-slate-500", children: "next scheduled job" }), _jsx("span", { className: "text-right font-mono text-xs", children: nextTimer(node) })] })] })), _jsx("div", { className: "pt-1 border-t border-slate-800 text-xs text-slate-500", children: "view detail \u00B7 maintenance & shutdown prep \u2192" })] }) }));
|
||||
}
|
||||
function healthOf(n) {
|
||||
if (!n.online)
|
||||
|
||||
@@ -95,6 +95,10 @@ export function NodeCard({ node }: Props) {
|
||||
</div>
|
||||
</>
|
||||
)}
|
||||
|
||||
<div className="pt-1 border-t border-slate-800 text-xs text-slate-500">
|
||||
view detail · maintenance & shutdown prep →
|
||||
</div>
|
||||
</a>
|
||||
</Link>
|
||||
);
|
||||
|
||||
@@ -211,13 +211,20 @@ step "flushing filesystem buffers"
|
||||
sync
|
||||
ok "sync complete"
|
||||
|
||||
POOL=$(df --output=source /slab 2>/dev/null | tail -1 | tr -d '[:space:]')
|
||||
if [ -n "$POOL" ] && [ "$POOL" != "none" ]; then
|
||||
step "zpool health check ($POOL)"
|
||||
zpool status -x "$POOL" 2>&1 | sed 's/^/ /'
|
||||
if [ "$EXPORT_ZPOOL" -eq 1 ]; then
|
||||
step "exporting $POOL (--export-zpool)"
|
||||
run sudo zpool export "$POOL" && ok "exported — remember: zpool import $POOL after reboot" || fail "export failed"
|
||||
step "zpool health check"
|
||||
if ! command -v zpool >/dev/null 2>&1; then
|
||||
ok "no zpool binary on this node — warm tier is not ZFS-backed here, nothing to check"
|
||||
else
|
||||
POOL=$(df --output=source /slab 2>/dev/null | tail -1 | tr -d '[:space:]')
|
||||
if [ -n "$POOL" ] && [ "$POOL" != "none" ]; then
|
||||
echo " pool: $POOL"
|
||||
zpool status -x "$POOL" 2>&1 | sed 's/^/ /'
|
||||
if [ "$EXPORT_ZPOOL" -eq 1 ]; then
|
||||
step "exporting $POOL (--export-zpool)"
|
||||
run sudo zpool export "$POOL" && ok "exported — remember: zpool import $POOL after reboot" || fail "export failed"
|
||||
fi
|
||||
else
|
||||
warn "zpool present but /slab isn't a recognizable ZFS mount"
|
||||
fi
|
||||
fi
|
||||
|
||||
|
||||
Reference in New Issue
Block a user