Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
f38efc7096 | ||
|
|
2d0c225f98 | ||
|
|
4ea1cbed2e | ||
|
|
6a4bc09cbb | ||
|
|
fe815db981 | ||
|
|
4132937021 | ||
|
|
334cd068d2 | ||
|
|
f30ea04ab6 | ||
|
|
bb24c77676 | ||
|
|
75d822d0f2 | ||
|
|
dd2b90872a | ||
|
|
5c9bc7eb9c | ||
|
|
5c1d962bf2 | ||
|
|
a3fe1d147c |
@@ -26,6 +26,7 @@ pub mod refs;
|
|||||||
pub mod repo_ensure;
|
pub mod repo_ensure;
|
||||||
pub mod rpc;
|
pub mod rpc;
|
||||||
pub mod services;
|
pub mod services;
|
||||||
|
pub mod shutdown_prep;
|
||||||
pub mod snapshot;
|
pub mod snapshot;
|
||||||
pub mod tags;
|
pub mod tags;
|
||||||
pub mod tailscale;
|
pub mod tailscale;
|
||||||
|
|||||||
@@ -217,6 +217,22 @@ pub enum Method {
|
|||||||
/// `payload`: JSON [`crate::cluster::repo_ensure::RepoReleaseRequest`].
|
/// `payload`: JSON [`crate::cluster::repo_ensure::RepoReleaseRequest`].
|
||||||
/// Reply: JSON [`crate::cluster::repo_ensure::RepoReleaseReply`].
|
/// Reply: JSON [`crate::cluster::repo_ensure::RepoReleaseReply`].
|
||||||
RepoRelease = 0x1f,
|
RepoRelease = 0x1f,
|
||||||
|
/// Runs `safe-shutdown-prep.sh --dry-run` to completion on this
|
||||||
|
/// node and returns the full report. Never stops anything —
|
||||||
|
/// dry-run only, safe to call repeatedly.
|
||||||
|
///
|
||||||
|
/// `payload`: JSON [`crate::cluster::shutdown_prep::ShutdownPrepCheckRequest`].
|
||||||
|
/// Reply: JSON [`crate::cluster::shutdown_prep::ShutdownPrepCheckReply`].
|
||||||
|
ShutdownPrepCheck = 0x20,
|
||||||
|
/// Starts the real `safe-shutdown-prep.sh` run in a detached
|
||||||
|
/// systemd scope and returns immediately — the script's own step
|
||||||
|
/// stops `claw-store.service`, so this RPC connection cannot
|
||||||
|
/// outlive full completion. See
|
||||||
|
/// [`crate::cluster::shutdown_prep`] module docs.
|
||||||
|
///
|
||||||
|
/// `payload`: JSON [`crate::cluster::shutdown_prep::ShutdownPrepExecuteRequest`].
|
||||||
|
/// Reply: JSON [`crate::cluster::shutdown_prep::ShutdownPrepExecuteReply`].
|
||||||
|
ShutdownPrepExecute = 0x21,
|
||||||
}
|
}
|
||||||
|
|
||||||
impl Method {
|
impl Method {
|
||||||
@@ -255,6 +271,8 @@ impl Method {
|
|||||||
0x1d => Some(Method::DashboardStorage),
|
0x1d => Some(Method::DashboardStorage),
|
||||||
0x1e => Some(Method::RepoEnsure),
|
0x1e => Some(Method::RepoEnsure),
|
||||||
0x1f => Some(Method::RepoRelease),
|
0x1f => Some(Method::RepoRelease),
|
||||||
|
0x20 => Some(Method::ShutdownPrepCheck),
|
||||||
|
0x21 => Some(Method::ShutdownPrepExecute),
|
||||||
_ => None,
|
_ => None,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -1176,6 +1194,26 @@ impl RpcRouter {
|
|||||||
.context("encoding RepoReleaseReply as JSON")?;
|
.context("encoding RepoReleaseReply as JSON")?;
|
||||||
Ok(HandlerOutcome::Reply(json))
|
Ok(HandlerOutcome::Reply(json))
|
||||||
}
|
}
|
||||||
|
Method::ShutdownPrepCheck => {
|
||||||
|
let reply = crate::cluster::shutdown_prep::check().await?;
|
||||||
|
let json = serde_json::to_vec(&reply)
|
||||||
|
.context("encoding ShutdownPrepCheckReply as JSON")?;
|
||||||
|
Ok(HandlerOutcome::Reply(json))
|
||||||
|
}
|
||||||
|
Method::ShutdownPrepExecute => {
|
||||||
|
let req: crate::cluster::shutdown_prep::ShutdownPrepExecuteRequest =
|
||||||
|
match serde_json::from_slice(payload) {
|
||||||
|
Ok(r) => r,
|
||||||
|
Err(_) => {
|
||||||
|
return Ok(HandlerOutcome::Error(ErrorCode::InvalidRequest))
|
||||||
|
}
|
||||||
|
};
|
||||||
|
let reply =
|
||||||
|
crate::cluster::shutdown_prep::execute(&self.local_name, &req).await?;
|
||||||
|
let json = serde_json::to_vec(&reply)
|
||||||
|
.context("encoding ShutdownPrepExecuteReply as JSON")?;
|
||||||
|
Ok(HandlerOutcome::Reply(json))
|
||||||
|
}
|
||||||
Method::BlobStat => {
|
Method::BlobStat => {
|
||||||
let store = match &self.blob_store {
|
let store = match &self.blob_store {
|
||||||
Some(s) => s,
|
Some(s) => s,
|
||||||
|
|||||||
@@ -229,6 +229,45 @@ pub async fn call_repo_release(
|
|||||||
serde_json::from_slice(&reply).context("decoding RepoReleaseReply JSON")
|
serde_json::from_slice(&reply).context("decoding RepoReleaseReply JSON")
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Convenience wrapper for [`Method::ShutdownPrepCheck`]. Runs
|
||||||
|
/// `safe-shutdown-prep.sh --dry-run` on the connected peer and waits
|
||||||
|
/// for the full report. Never stops anything on the peer.
|
||||||
|
pub async fn call_shutdown_prep_check(
|
||||||
|
conn: &Connection,
|
||||||
|
) -> Result<crate::cluster::shutdown_prep::ShutdownPrepCheckReply> {
|
||||||
|
let req = crate::cluster::shutdown_prep::ShutdownPrepCheckRequest {};
|
||||||
|
let payload = serde_json::to_vec(&req).context("encoding ShutdownPrepCheckRequest")?;
|
||||||
|
let reply = rpc_call(conn, Method::ShutdownPrepCheck, &payload).await?;
|
||||||
|
if reply.len() == 1 {
|
||||||
|
if let Some(code) = decode_error(reply[0]) {
|
||||||
|
bail!("peer replied with error: {}", code.describe());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
serde_json::from_slice(&reply).context("decoding ShutdownPrepCheckReply JSON")
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Convenience wrapper for [`Method::ShutdownPrepExecute`]. Starts the
|
||||||
|
/// real shutdown-prep run on the connected peer (detached — this call
|
||||||
|
/// returns as soon as the peer confirms it started, not when it
|
||||||
|
/// finishes, since the peer's own daemon stops itself partway
|
||||||
|
/// through).
|
||||||
|
pub async fn call_shutdown_prep_execute(
|
||||||
|
conn: &Connection,
|
||||||
|
confirm_node_name: &str,
|
||||||
|
) -> Result<crate::cluster::shutdown_prep::ShutdownPrepExecuteReply> {
|
||||||
|
let req = crate::cluster::shutdown_prep::ShutdownPrepExecuteRequest {
|
||||||
|
confirm_node_name: confirm_node_name.to_string(),
|
||||||
|
};
|
||||||
|
let payload = serde_json::to_vec(&req).context("encoding ShutdownPrepExecuteRequest")?;
|
||||||
|
let reply = rpc_call(conn, Method::ShutdownPrepExecute, &payload).await?;
|
||||||
|
if reply.len() == 1 {
|
||||||
|
if let Some(code) = decode_error(reply[0]) {
|
||||||
|
bail!("peer replied with error: {}", code.describe());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
serde_json::from_slice(&reply).context("decoding ShutdownPrepExecuteReply JSON")
|
||||||
|
}
|
||||||
|
|
||||||
/// Recognise a single-byte reply as one of our error codes. Returns
|
/// Recognise a single-byte reply as one of our error codes. Returns
|
||||||
/// `None` for any other single-byte value (which is a valid reply,
|
/// `None` for any other single-byte value (which is a valid reply,
|
||||||
/// just an unusually short one).
|
/// just an unusually short one).
|
||||||
|
|||||||
@@ -0,0 +1,166 @@
|
|||||||
|
//! Peer-side wiring for `deploy/scripts/safe-shutdown-prep.sh`,
|
||||||
|
//! surfaced through the RPC layer so the dashboard-v2 aggregator can
|
||||||
|
//! offer a "prepare this node for shutdown" action.
|
||||||
|
//!
|
||||||
|
//! Split into two RPCs deliberately:
|
||||||
|
//!
|
||||||
|
//! - [`Method::ShutdownPrepCheck`] runs the script's `--dry-run` mode
|
||||||
|
//! and waits for it to finish. Dry-run never stops this node's own
|
||||||
|
//! daemon, so the RPC connection survives to deliver the full
|
||||||
|
//! report — this is the part a browser can meaningfully show.
|
||||||
|
//! - [`Method::ShutdownPrepExecute`] runs the real script, which (by
|
||||||
|
//! design) stops `claw-store.service` — i.e. the very process
|
||||||
|
//! handling this RPC. There is no way to stream a live result past
|
||||||
|
//! that point, so this RPC detaches the script into its own
|
||||||
|
//! transient systemd scope (outside this daemon's service cgroup,
|
||||||
|
//! so `systemctl stop claw-store.service` doesn't take the script
|
||||||
|
//! down with it) and returns immediately. The full report lands in
|
||||||
|
//! `SHUTDOWN_PREP_LOG` for whoever is physically at the machine (or
|
||||||
|
//! over SSH) to read once the node has gone dark.
|
||||||
|
//!
|
||||||
|
//! [`Method::ShutdownPrepCheck`]: crate::cluster::rpc::Method::ShutdownPrepCheck
|
||||||
|
//! [`Method::ShutdownPrepExecute`]: crate::cluster::rpc::Method::ShutdownPrepExecute
|
||||||
|
|
||||||
|
use anyhow::{bail, Context, Result};
|
||||||
|
use serde::{Deserialize, Serialize};
|
||||||
|
use std::path::PathBuf;
|
||||||
|
use std::time::Duration;
|
||||||
|
use tokio::process::Command;
|
||||||
|
use tokio::time::timeout;
|
||||||
|
|
||||||
|
/// Dry-run does a real snapshot + replicate, which can legitimately
|
||||||
|
/// take a while on a large delta. Generous but bounded so a stuck
|
||||||
|
/// peer connection doesn't hang the RPC forever.
|
||||||
|
const CHECK_TIMEOUT: Duration = Duration::from_secs(300);
|
||||||
|
|
||||||
|
/// Where the real run's output lands once this node's daemon (and
|
||||||
|
/// therefore this RPC connection) is gone.
|
||||||
|
pub const SHUTDOWN_PREP_LOG: &str = "/var/lib/claw-store/shutdown-prep.log";
|
||||||
|
|
||||||
|
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
|
||||||
|
pub struct ShutdownPrepCheckRequest {}
|
||||||
|
|
||||||
|
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
|
||||||
|
pub struct ShutdownPrepCheckReply {
|
||||||
|
/// True iff the script exited 0 (every guard passed, "SAFE TO
|
||||||
|
/// POWER OFF" printed for the checked steps).
|
||||||
|
pub ready: bool,
|
||||||
|
/// Full combined stdout+stderr from `--dry-run`.
|
||||||
|
pub output: String,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
|
||||||
|
pub struct ShutdownPrepExecuteRequest {
|
||||||
|
/// Defense in depth beyond RPC targeting: the caller must name
|
||||||
|
/// the exact node it thinks it's shutting down. Checked against
|
||||||
|
/// this node's own configured name before anything runs.
|
||||||
|
pub confirm_node_name: String,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
|
||||||
|
pub struct ShutdownPrepExecuteReply {
|
||||||
|
pub started: bool,
|
||||||
|
pub message: String,
|
||||||
|
pub log_path: String,
|
||||||
|
}
|
||||||
|
|
||||||
|
fn script_path() -> PathBuf {
|
||||||
|
if let Ok(p) = std::env::var("CLAWSTOR_SHUTDOWN_SCRIPT") {
|
||||||
|
if !p.is_empty() {
|
||||||
|
return PathBuf::from(p);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if let Ok(home) = std::env::var("HOME") {
|
||||||
|
if !home.is_empty() {
|
||||||
|
return PathBuf::from(home)
|
||||||
|
.join("clawstor-deploy/scripts/safe-shutdown-prep.sh");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
PathBuf::from("/usr/local/share/claw-store/safe-shutdown-prep.sh")
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Run `safe-shutdown-prep.sh --dry-run` to completion and report the
|
||||||
|
/// full output. Never stops anything on this node — safe to call any
|
||||||
|
/// time, including repeatedly.
|
||||||
|
pub async fn check() -> Result<ShutdownPrepCheckReply> {
|
||||||
|
let script = script_path();
|
||||||
|
if !script.exists() {
|
||||||
|
bail!("shutdown-prep script not found at {}", script.display());
|
||||||
|
}
|
||||||
|
let run = Command::new("bash")
|
||||||
|
.arg(&script)
|
||||||
|
.arg("--dry-run")
|
||||||
|
.output();
|
||||||
|
let output = timeout(CHECK_TIMEOUT, run)
|
||||||
|
.await
|
||||||
|
.context("shutdown-prep --dry-run timed out")?
|
||||||
|
.context("spawning shutdown-prep --dry-run")?;
|
||||||
|
let mut combined = String::from_utf8_lossy(&output.stdout).into_owned();
|
||||||
|
combined.push_str(&String::from_utf8_lossy(&output.stderr));
|
||||||
|
Ok(ShutdownPrepCheckReply {
|
||||||
|
ready: output.status.success(),
|
||||||
|
output: combined,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Kick off the real (non-dry-run) script in a transient systemd
|
||||||
|
/// scope detached from this daemon's own service cgroup, then return
|
||||||
|
/// immediately without waiting for it. The script's own step 6 stops
|
||||||
|
/// `claw-store.service` — waiting for it to exit here would mean
|
||||||
|
/// waiting for our own process to be killed.
|
||||||
|
pub async fn execute(local_node_name: &str, req: &ShutdownPrepExecuteRequest) -> Result<ShutdownPrepExecuteReply> {
|
||||||
|
if req.confirm_node_name != local_node_name {
|
||||||
|
bail!(
|
||||||
|
"confirm_node_name '{}' does not match this node ('{}') — refusing",
|
||||||
|
req.confirm_node_name,
|
||||||
|
local_node_name
|
||||||
|
);
|
||||||
|
}
|
||||||
|
let script = script_path();
|
||||||
|
if !script.exists() {
|
||||||
|
bail!("shutdown-prep script not found at {}", script.display());
|
||||||
|
}
|
||||||
|
let unit = format!(
|
||||||
|
"clawstor-shutdown-prep-{}",
|
||||||
|
std::time::SystemTime::now()
|
||||||
|
.duration_since(std::time::UNIX_EPOCH)
|
||||||
|
.map(|d| d.as_secs())
|
||||||
|
.unwrap_or(0)
|
||||||
|
);
|
||||||
|
// `--user --scope` places this under the user session's cgroup
|
||||||
|
// tree (/user.slice/...), a sibling of — not a descendant of —
|
||||||
|
// /system.slice/claw-store.service. `systemctl stop
|
||||||
|
// claw-store.service` only tears down its own cgroup, so this
|
||||||
|
// keeps running (and completes step 6, which stops that very
|
||||||
|
// service) unaffected.
|
||||||
|
// Deliberately no --force: the real run re-checks active builds
|
||||||
|
// and the sync queue itself, even though `check()` may have run
|
||||||
|
// moments ago — state can change between the two clicks, and
|
||||||
|
// re-validating is cheap.
|
||||||
|
let cmd = format!(
|
||||||
|
"{} >> {} 2>&1",
|
||||||
|
script.display(),
|
||||||
|
SHUTDOWN_PREP_LOG
|
||||||
|
);
|
||||||
|
let spawn = Command::new("systemd-run")
|
||||||
|
.arg("--user")
|
||||||
|
.arg("--scope")
|
||||||
|
.arg("--collect")
|
||||||
|
.arg(format!("--unit={unit}"))
|
||||||
|
.arg("bash")
|
||||||
|
.arg("-c")
|
||||||
|
.arg(&cmd)
|
||||||
|
.spawn();
|
||||||
|
match spawn {
|
||||||
|
Ok(_child) => Ok(ShutdownPrepExecuteReply {
|
||||||
|
started: true,
|
||||||
|
message: format!(
|
||||||
|
"shutdown-prep started on {local_node_name} as transient unit {unit}. \
|
||||||
|
This node's daemon (and dashboard) will go offline as part of the \
|
||||||
|
process — that is expected. Full output: {SHUTDOWN_PREP_LOG}."
|
||||||
|
),
|
||||||
|
log_path: SHUTDOWN_PREP_LOG.to_string(),
|
||||||
|
}),
|
||||||
|
Err(e) => Err(e).context("spawning systemd-run for shutdown-prep"),
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -27,12 +27,26 @@ pub struct HotConfig {
|
|||||||
#[derive(Debug, Clone, Deserialize, Serialize)]
|
#[derive(Debug, Clone, Deserialize, Serialize)]
|
||||||
pub struct WarmConfig {
|
pub struct WarmConfig {
|
||||||
pub projects_path: PathBuf,
|
pub projects_path: PathBuf,
|
||||||
|
/// Dataset name for `zfs`/`zpool` operations against the warm
|
||||||
|
/// tier. The literal value `"none"` means this node's warm tier
|
||||||
|
/// is a plain directory, not ZFS-backed (e.g. a build node with
|
||||||
|
/// no ZFS pool) — snapshot/replicate become no-ops instead of
|
||||||
|
/// erroring on a missing `zfs`/`zpool` binary. See
|
||||||
|
/// [`WarmConfig::zfs_enabled`].
|
||||||
pub zfs_dataset: String,
|
pub zfs_dataset: String,
|
||||||
pub snapshot_retain_hours: u64,
|
pub snapshot_retain_hours: u64,
|
||||||
pub snapshot_retain_days: u64,
|
pub snapshot_retain_days: u64,
|
||||||
pub snapshot_retain_weeks: u64,
|
pub snapshot_retain_weeks: u64,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
impl WarmConfig {
|
||||||
|
/// `false` when `zfs_dataset = "none"` — this node's warm tier
|
||||||
|
/// has no ZFS pool underneath it.
|
||||||
|
pub fn zfs_enabled(&self) -> bool {
|
||||||
|
self.zfs_dataset != "none"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
#[derive(Debug, Clone, Deserialize, Serialize)]
|
#[derive(Debug, Clone, Deserialize, Serialize)]
|
||||||
pub struct ColdConfig {
|
pub struct ColdConfig {
|
||||||
pub archive_path: PathBuf,
|
pub archive_path: PathBuf,
|
||||||
|
|||||||
+17
-11
@@ -7,7 +7,7 @@ use crate::manifest::Manifest;
|
|||||||
use crate::snapshot;
|
use crate::snapshot;
|
||||||
use crate::sync::{SyncQueue, drain_sync_queue};
|
use crate::sync::{SyncQueue, drain_sync_queue};
|
||||||
use crate::zfs::SystemZfs;
|
use crate::zfs::SystemZfs;
|
||||||
use anyhow::Result;
|
use anyhow::{Context, Result};
|
||||||
use chrono::Utc;
|
use chrono::Utc;
|
||||||
use sysinfo::{ProcessRefreshKind, RefreshKind, System};
|
use sysinfo::{ProcessRefreshKind, RefreshKind, System};
|
||||||
use tokio::time::{interval, Duration};
|
use tokio::time::{interval, Duration};
|
||||||
@@ -36,7 +36,14 @@ pub async fn run(cfg: Config, mut manifest: Manifest) -> Result<()> {
|
|||||||
let hot_dir = cfg.hot.path.clone();
|
let hot_dir = cfg.hot.path.clone();
|
||||||
let hot_max_bytes = cfg.hot.max_gb.saturating_mul(1024 * 1024 * 1024);
|
let hot_max_bytes = cfg.hot.max_gb.saturating_mul(1024 * 1024 * 1024);
|
||||||
let blob_root = cluster_cfg.blob_store_root.clone();
|
let blob_root = cluster_cfg.blob_store_root.clone();
|
||||||
match ClusterServices::start(
|
// Fail fast rather than degrade silently: a bind failure here is
|
||||||
|
// almost always a boot-time race against DHCP/network-online
|
||||||
|
// (the bind address isn't assigned to the interface yet). The
|
||||||
|
// systemd unit has `Restart=on-failure`; exiting lets it retry
|
||||||
|
// a few seconds later once the network is actually up, instead
|
||||||
|
// of leaving the daemon running indefinitely with no gossip,
|
||||||
|
// RPC, or Prometheus endpoint and no visible failure state.
|
||||||
|
let svc = ClusterServices::start(
|
||||||
cluster_cfg,
|
cluster_cfg,
|
||||||
cfg.node.name.clone(),
|
cfg.node.name.clone(),
|
||||||
hot_dir,
|
hot_dir,
|
||||||
@@ -44,8 +51,7 @@ pub async fn run(cfg: Config, mut manifest: Manifest) -> Result<()> {
|
|||||||
blob_root,
|
blob_root,
|
||||||
)
|
)
|
||||||
.await
|
.await
|
||||||
{
|
.context("starting cluster services")?;
|
||||||
Ok(svc) => {
|
|
||||||
tracing::info!(
|
tracing::info!(
|
||||||
rpc_enabled = svc.rpc_enabled(),
|
rpc_enabled = svc.rpc_enabled(),
|
||||||
blob_store_enabled = svc.blob_store_enabled(),
|
blob_store_enabled = svc.blob_store_enabled(),
|
||||||
@@ -54,12 +60,6 @@ pub async fn run(cfg: Config, mut manifest: Manifest) -> Result<()> {
|
|||||||
);
|
);
|
||||||
Some(svc)
|
Some(svc)
|
||||||
}
|
}
|
||||||
Err(e) => {
|
|
||||||
tracing::error!(error = %e, "cluster services failed to start; continuing without cluster");
|
|
||||||
None
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
None => {
|
None => {
|
||||||
tracing::info!("no [cluster] section in config; running standalone");
|
tracing::info!("no [cluster] section in config; running standalone");
|
||||||
None
|
None
|
||||||
@@ -188,6 +188,9 @@ pub async fn run(cfg: Config, mut manifest: Manifest) -> Result<()> {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
_ = snap_tick.tick() => {
|
_ = snap_tick.tick() => {
|
||||||
|
if !cfg.warm.zfs_enabled() {
|
||||||
|
tracing::debug!("skipping snapshot tick — zfs_dataset = \"none\" on this node");
|
||||||
|
} else {
|
||||||
let ts = chrono::Utc::now().format("%Y-%m-%d-%H%M").to_string();
|
let ts = chrono::Utc::now().format("%Y-%m-%d-%H%M").to_string();
|
||||||
tracing::info!("taking snapshot {}", ts);
|
tracing::info!("taking snapshot {}", ts);
|
||||||
if let Err(e) = snapshot::run_snapshot_cycle(
|
if let Err(e) = snapshot::run_snapshot_cycle(
|
||||||
@@ -199,8 +202,11 @@ pub async fn run(cfg: Config, mut manifest: Manifest) -> Result<()> {
|
|||||||
tracing::error!("snapshot failed: {:#}", e);
|
tracing::error!("snapshot failed: {:#}", e);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
}
|
||||||
_ = repl_tick.tick() => {
|
_ = repl_tick.tick() => {
|
||||||
if let Some(rep) = &cfg.replication {
|
if !cfg.warm.zfs_enabled() {
|
||||||
|
tracing::debug!("skipping replication tick — zfs_dataset = \"none\" on this node");
|
||||||
|
} else if let Some(rep) = &cfg.replication {
|
||||||
if let (Some(host), Some(user), Some(dest)) = (
|
if let (Some(host), Some(user), Some(dest)) = (
|
||||||
&rep.send_to_host, &rep.send_to_user, &rep.cold_dataset_on_peer
|
&rep.send_to_host, &rep.send_to_user, &rep.cold_dataset_on_peer
|
||||||
) {
|
) {
|
||||||
|
|||||||
@@ -1502,6 +1502,10 @@ fn cmd_gc(cfg: &Config, manifest: &mut Manifest) -> Result<()> {
|
|||||||
}
|
}
|
||||||
|
|
||||||
fn cmd_snapshot(cfg: &Config, zfs: &SystemZfs) -> Result<()> {
|
fn cmd_snapshot(cfg: &Config, zfs: &SystemZfs) -> Result<()> {
|
||||||
|
if !cfg.warm.zfs_enabled() {
|
||||||
|
println!("Warm tier is not ZFS-backed on this node (zfs_dataset = \"none\") — nothing to snapshot.");
|
||||||
|
return Ok(());
|
||||||
|
}
|
||||||
let ts = chrono::Utc::now().format("%Y-%m-%d-%H%M").to_string();
|
let ts = chrono::Utc::now().format("%Y-%m-%d-%H%M").to_string();
|
||||||
snapshot::run_snapshot_cycle(
|
snapshot::run_snapshot_cycle(
|
||||||
zfs, &cfg.warm.zfs_dataset, &ts,
|
zfs, &cfg.warm.zfs_dataset, &ts,
|
||||||
@@ -1514,6 +1518,10 @@ fn cmd_snapshot(cfg: &Config, zfs: &SystemZfs) -> Result<()> {
|
|||||||
}
|
}
|
||||||
|
|
||||||
fn cmd_list_snapshots(cfg: &Config, zfs: &SystemZfs, _project: &str) -> Result<()> {
|
fn cmd_list_snapshots(cfg: &Config, zfs: &SystemZfs, _project: &str) -> Result<()> {
|
||||||
|
if !cfg.warm.zfs_enabled() {
|
||||||
|
println!("Warm tier is not ZFS-backed on this node (zfs_dataset = \"none\").");
|
||||||
|
return Ok(());
|
||||||
|
}
|
||||||
let snaps = zfs.list_snapshots(&cfg.warm.zfs_dataset)?;
|
let snaps = zfs.list_snapshots(&cfg.warm.zfs_dataset)?;
|
||||||
if snaps.is_empty() { println!("No snapshots found."); return Ok(()); }
|
if snaps.is_empty() { println!("No snapshots found."); return Ok(()); }
|
||||||
for s in &snaps { println!(" {}", s); }
|
for s in &snaps { println!(" {}", s); }
|
||||||
@@ -1531,6 +1539,10 @@ fn cmd_restore(cfg: &Config, zfs: &SystemZfs, project: &str, snap: &str) -> Resu
|
|||||||
}
|
}
|
||||||
|
|
||||||
fn cmd_replicate(cfg: &Config, zfs: &SystemZfs) -> Result<()> {
|
fn cmd_replicate(cfg: &Config, zfs: &SystemZfs) -> Result<()> {
|
||||||
|
if !cfg.warm.zfs_enabled() {
|
||||||
|
println!("Warm tier is not ZFS-backed on this node (zfs_dataset = \"none\") — nothing to replicate.");
|
||||||
|
return Ok(());
|
||||||
|
}
|
||||||
let rep = cfg.replication.as_ref()
|
let rep = cfg.replication.as_ref()
|
||||||
.context("no replication config — this node does not replicate")?;
|
.context("no replication config — this node does not replicate")?;
|
||||||
let host = rep.send_to_host.as_ref().context("send_to_host not set")?;
|
let host = rep.send_to_host.as_ref().context("send_to_host not set")?;
|
||||||
|
|||||||
+130
-1
@@ -91,9 +91,30 @@ impl V2State {
|
|||||||
.join("aggregator-sessions.json");
|
.join("aggregator-sessions.json");
|
||||||
let sessions = SessionStore::load(sessions_path)
|
let sessions = SessionStore::load(sessions_path)
|
||||||
.map_err(|e| anyhow::anyhow!("loading session store: {e}"))?;
|
.map_err(|e| anyhow::anyhow!("loading session store: {e}"))?;
|
||||||
|
// Bug fix 2026-07-31: the fleet view previously never included
|
||||||
|
// the node actually serving the dashboard — `cluster.peers` is
|
||||||
|
// by definition every *other* node, so hitting a given node's
|
||||||
|
// `/api/v2/fleet` directly silently dropped that node from its
|
||||||
|
// own view (looked like "node X is missing" from the UI, even
|
||||||
|
// though X was perfectly healthy — it just never queried
|
||||||
|
// itself). Fix: synthesize a self `PeerEntry` from our own
|
||||||
|
// gossip bind address and include it in the fan-out list, same
|
||||||
|
// as any other peer. `peer_rpc_addr` derives the RPC port from
|
||||||
|
// `lan_addr`/`tailscale_addr` via the fleet's +1 convention, so
|
||||||
|
// this resolves to the same `bind_rpc_lan`/`bind_rpc_tailscale`
|
||||||
|
// the daemon actually listens on.
|
||||||
|
let self_peer = PeerEntry {
|
||||||
|
name: cfg.node.name.clone(),
|
||||||
|
zone: cluster.zone.clone(),
|
||||||
|
lan_addr: cluster.bind_lan,
|
||||||
|
tailscale_addr: cluster.bind_tailscale,
|
||||||
|
};
|
||||||
|
let mut peers = cluster.peers.clone();
|
||||||
|
peers.push(self_peer);
|
||||||
|
|
||||||
Ok(Self {
|
Ok(Self {
|
||||||
aggregator_name: cfg.node.name.clone(),
|
aggregator_name: cfg.node.name.clone(),
|
||||||
peers: cluster.peers.clone(),
|
peers,
|
||||||
client: std::sync::Arc::new(client),
|
client: std::sync::Arc::new(client),
|
||||||
default_rpc_port_offset: 1,
|
default_rpc_port_offset: 1,
|
||||||
api_token: cfg.api_token.clone(),
|
api_token: cfg.api_token.clone(),
|
||||||
@@ -549,6 +570,19 @@ impl AuthedCaller {
|
|||||||
/// in the request body. Explicit user-supplied workspace is only
|
/// in the request body. Explicit user-supplied workspace is only
|
||||||
/// honored for admin/open — a namespaced caller supplying a
|
/// honored for admin/open — a namespaced caller supplying a
|
||||||
/// mismatched workspace is a forbidden write.
|
/// mismatched workspace is a forbidden write.
|
||||||
|
/// Gate for fleet-infrastructure actions (shutdown-prep) that
|
||||||
|
/// have nothing to do with a tag/repo namespace — a namespaced
|
||||||
|
/// per-app token has no business stopping a node's services.
|
||||||
|
pub fn require_admin(&self) -> Result<(), (StatusCode, String)> {
|
||||||
|
match self {
|
||||||
|
AuthedCaller::Admin | AuthedCaller::Open => Ok(()),
|
||||||
|
AuthedCaller::Namespaced { .. } => Err((
|
||||||
|
StatusCode::FORBIDDEN,
|
||||||
|
"this action requires an admin token".to_string(),
|
||||||
|
)),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
pub fn resolve_workspace(&self, requested: Option<&str>) -> Result<String, (StatusCode, String)> {
|
pub fn resolve_workspace(&self, requested: Option<&str>) -> Result<String, (StatusCode, String)> {
|
||||||
match self {
|
match self {
|
||||||
AuthedCaller::Admin | AuthedCaller::Open => match requested {
|
AuthedCaller::Admin | AuthedCaller::Open => match requested {
|
||||||
@@ -845,6 +879,93 @@ async fn handle_repos_release(
|
|||||||
}))
|
}))
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// ── shutdown-prep (targets one specific node, not a fan-out) ──────
|
||||||
|
|
||||||
|
#[derive(Serialize)]
|
||||||
|
pub struct ShutdownPrepCheckResponse {
|
||||||
|
pub node: String,
|
||||||
|
pub ready: bool,
|
||||||
|
pub output: String,
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn handle_shutdown_prep_check(
|
||||||
|
State(s): State<Arc<V2State>>,
|
||||||
|
Path(name): Path<String>,
|
||||||
|
axum::extract::Extension(caller): axum::extract::Extension<AuthedCaller>,
|
||||||
|
) -> Result<Json<ShutdownPrepCheckResponse>, (StatusCode, String)> {
|
||||||
|
caller.require_admin()?;
|
||||||
|
let peer = s
|
||||||
|
.peers
|
||||||
|
.iter()
|
||||||
|
.find(|p| p.name == name)
|
||||||
|
.cloned()
|
||||||
|
.ok_or((StatusCode::NOT_FOUND, format!("no peer named {name}")))?;
|
||||||
|
let conn = s
|
||||||
|
.dial(&peer)
|
||||||
|
.await
|
||||||
|
.map_err(|e| (StatusCode::BAD_GATEWAY, format!("connecting to {name}: {e:#}")))?;
|
||||||
|
let reply = crate::cluster::rpc::call_shutdown_prep_check(&conn).await;
|
||||||
|
conn.close(quinn::VarInt::from_u32(0), b"done");
|
||||||
|
let reply = reply.map_err(|e| (StatusCode::BAD_GATEWAY, format!("{e:#}")))?;
|
||||||
|
Ok(Json(ShutdownPrepCheckResponse {
|
||||||
|
node: name,
|
||||||
|
ready: reply.ready,
|
||||||
|
output: reply.output,
|
||||||
|
}))
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Deserialize)]
|
||||||
|
pub struct ShutdownPrepExecuteBody {
|
||||||
|
/// Must equal the target node's own name — a second, server-side
|
||||||
|
/// confirmation beyond "the operator clicked the right button in
|
||||||
|
/// the UI". Checked again on the peer itself in
|
||||||
|
/// `shutdown_prep::execute`.
|
||||||
|
pub confirm_node_name: String,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Serialize)]
|
||||||
|
pub struct ShutdownPrepExecuteResponse {
|
||||||
|
pub node: String,
|
||||||
|
pub started: bool,
|
||||||
|
pub message: String,
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn handle_shutdown_prep_execute(
|
||||||
|
State(s): State<Arc<V2State>>,
|
||||||
|
Path(name): Path<String>,
|
||||||
|
axum::extract::Extension(caller): axum::extract::Extension<AuthedCaller>,
|
||||||
|
Json(body): Json<ShutdownPrepExecuteBody>,
|
||||||
|
) -> Result<Json<ShutdownPrepExecuteResponse>, (StatusCode, String)> {
|
||||||
|
caller.require_admin()?;
|
||||||
|
if body.confirm_node_name != name {
|
||||||
|
return Err((
|
||||||
|
StatusCode::BAD_REQUEST,
|
||||||
|
format!(
|
||||||
|
"confirm_node_name '{}' does not match target node '{name}'",
|
||||||
|
body.confirm_node_name
|
||||||
|
),
|
||||||
|
));
|
||||||
|
}
|
||||||
|
let peer = s
|
||||||
|
.peers
|
||||||
|
.iter()
|
||||||
|
.find(|p| p.name == name)
|
||||||
|
.cloned()
|
||||||
|
.ok_or((StatusCode::NOT_FOUND, format!("no peer named {name}")))?;
|
||||||
|
let conn = s
|
||||||
|
.dial(&peer)
|
||||||
|
.await
|
||||||
|
.map_err(|e| (StatusCode::BAD_GATEWAY, format!("connecting to {name}: {e:#}")))?;
|
||||||
|
let reply = crate::cluster::rpc::call_shutdown_prep_execute(&conn, &name).await;
|
||||||
|
conn.close(quinn::VarInt::from_u32(0), b"done");
|
||||||
|
let reply = reply.map_err(|e| (StatusCode::BAD_GATEWAY, format!("{e:#}")))?;
|
||||||
|
Ok(Json(ShutdownPrepExecuteResponse {
|
||||||
|
node: name,
|
||||||
|
started: reply.started,
|
||||||
|
message: reply.message,
|
||||||
|
}))
|
||||||
|
}
|
||||||
|
|
||||||
async fn fanout_repo_ensure(
|
async fn fanout_repo_ensure(
|
||||||
s: &V2State,
|
s: &V2State,
|
||||||
req: &crate::cluster::repo_ensure::RepoEnsureRequest,
|
req: &crate::cluster::repo_ensure::RepoEnsureRequest,
|
||||||
@@ -1326,6 +1447,14 @@ pub fn build(state: Arc<V2State>) -> Router {
|
|||||||
.route("/api/v2/sessions/:id/commit", post(handle_commit_session))
|
.route("/api/v2/sessions/:id/commit", post(handle_commit_session))
|
||||||
.route("/api/v2/repos/ensure", post(handle_repos_ensure))
|
.route("/api/v2/repos/ensure", post(handle_repos_ensure))
|
||||||
.route("/api/v2/repos/release", post(handle_repos_release))
|
.route("/api/v2/repos/release", post(handle_repos_release))
|
||||||
|
.route(
|
||||||
|
"/api/v2/node/:name/shutdown-prep/check",
|
||||||
|
post(handle_shutdown_prep_check),
|
||||||
|
)
|
||||||
|
.route(
|
||||||
|
"/api/v2/node/:name/shutdown-prep/execute",
|
||||||
|
post(handle_shutdown_prep_execute),
|
||||||
|
)
|
||||||
.route_layer(axum::middleware::from_fn_with_state(state.clone(), v2_auth))
|
.route_layer(axum::middleware::from_fn_with_state(state.clone(), v2_auth))
|
||||||
.with_state(state)
|
.with_state(state)
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -24,7 +24,7 @@ export function NodeCard({ node }) {
|
|||||||
border,
|
border,
|
||||||
].join(' '), children: [_jsxs("div", { className: "flex items-center justify-between", children: [_jsxs("div", { className: "flex items-center gap-2", children: [_jsx("span", { className: `inline-block w-2.5 h-2.5 rounded-full ${dot}` }), _jsx("span", { className: "text-lg font-semibold text-slate-100", children: node.node_name })] }), _jsx("span", { className: "text-xs text-slate-500 font-mono", children: node.zone || '—' })] }), !node.online && (_jsx("div", { className: "text-sm text-red-400 break-words", children: node.error ?? 'offline' })), node.online && (_jsxs(_Fragment, { children: [node.filesystem && (_jsx(StorageBar, { label: "disk", used: node.filesystem.used_bytes, total: node.filesystem.total_bytes })), node.hot && node.hot.max_bytes > 0 && (_jsx(StorageBar, { label: "hot tier", used: node.hot.used_bytes, total: node.hot.max_bytes, pinned: node.hot.pinned_bytes ?? undefined })), _jsxs("div", { className: "grid grid-cols-2 gap-y-1 text-sm", children: [_jsx("span", { className: "text-slate-500", children: "mount" }), _jsx("span", { className: "text-right font-mono text-xs", children: node.mount?.active ? (_jsx("span", { className: "text-emerald-300", children: "\u2713 mounted" })) : (_jsx("span", { className: "text-slate-500", children: "not mounted" })) }), _jsx("span", { className: "text-slate-500", children: "cache hit rate" }), _jsx("span", { className: "text-right font-mono text-xs", children: node.cache && node.cache.hits + node.cache.misses > 0
|
].join(' '), children: [_jsxs("div", { className: "flex items-center justify-between", children: [_jsxs("div", { className: "flex items-center gap-2", children: [_jsx("span", { className: `inline-block w-2.5 h-2.5 rounded-full ${dot}` }), _jsx("span", { className: "text-lg font-semibold text-slate-100", children: node.node_name })] }), _jsx("span", { className: "text-xs text-slate-500 font-mono", children: node.zone || '—' })] }), !node.online && (_jsx("div", { className: "text-sm text-red-400 break-words", children: node.error ?? 'offline' })), node.online && (_jsxs(_Fragment, { children: [node.filesystem && (_jsx(StorageBar, { label: "disk", used: node.filesystem.used_bytes, total: node.filesystem.total_bytes })), node.hot && node.hot.max_bytes > 0 && (_jsx(StorageBar, { label: "hot tier", used: node.hot.used_bytes, total: node.hot.max_bytes, pinned: node.hot.pinned_bytes ?? undefined })), _jsxs("div", { className: "grid grid-cols-2 gap-y-1 text-sm", children: [_jsx("span", { className: "text-slate-500", children: "mount" }), _jsx("span", { className: "text-right font-mono text-xs", children: node.mount?.active ? (_jsx("span", { className: "text-emerald-300", children: "\u2713 mounted" })) : (_jsx("span", { className: "text-slate-500", children: "not mounted" })) }), _jsx("span", { className: "text-slate-500", children: "cache hit rate" }), _jsx("span", { className: "text-right font-mono text-xs", children: node.cache && node.cache.hits + node.cache.misses > 0
|
||||||
? `${Math.round(node.cache.hit_rate * 100)}%`
|
? `${Math.round(node.cache.hit_rate * 100)}%`
|
||||||
: _jsx("span", { className: "text-slate-500", children: "idle" }) }), _jsx("span", { className: "text-slate-500", children: "next scheduled job" }), _jsx("span", { className: "text-right font-mono text-xs", children: nextTimer(node) })] })] }))] }) }));
|
: _jsx("span", { className: "text-slate-500", children: "idle" }) }), _jsx("span", { className: "text-slate-500", children: "next scheduled job" }), _jsx("span", { className: "text-right font-mono text-xs", children: nextTimer(node) })] })] })), _jsx("div", { className: "pt-1 border-t border-slate-800 text-xs text-slate-500", children: "view detail \u00B7 maintenance & shutdown prep \u2192" })] }) }));
|
||||||
}
|
}
|
||||||
function healthOf(n) {
|
function healthOf(n) {
|
||||||
if (!n.online)
|
if (!n.online)
|
||||||
|
|||||||
@@ -95,6 +95,10 @@ export function NodeCard({ node }: Props) {
|
|||||||
</div>
|
</div>
|
||||||
</>
|
</>
|
||||||
)}
|
)}
|
||||||
|
|
||||||
|
<div className="pt-1 border-t border-slate-800 text-xs text-slate-500">
|
||||||
|
view detail · maintenance & shutdown prep →
|
||||||
|
</div>
|
||||||
</a>
|
</a>
|
||||||
</Link>
|
</Link>
|
||||||
);
|
);
|
||||||
|
|||||||
@@ -0,0 +1,44 @@
|
|||||||
|
import { jsx as _jsx, jsxs as _jsxs, Fragment as _Fragment } from "react/jsx-runtime";
|
||||||
|
import { useState } from 'react';
|
||||||
|
import { api } from '../lib/api';
|
||||||
|
/**
|
||||||
|
* Node-maintenance panel: runs `safe-shutdown-prep.sh --dry-run` on
|
||||||
|
* demand (safe, read-mostly, never stops anything) and — only once
|
||||||
|
* that comes back ready — unlocks a type-to-confirm button that
|
||||||
|
* starts the real run.
|
||||||
|
*
|
||||||
|
* The real run is fire-and-forget by necessity: its own steps stop
|
||||||
|
* this node's daemon, which is what's serving this very page, so
|
||||||
|
* there is no way to stream a live result past that point. Once
|
||||||
|
* started, the UI says so plainly and points at the on-disk log for
|
||||||
|
* the full report.
|
||||||
|
*/
|
||||||
|
export function ShutdownPrepPanel({ name }) {
|
||||||
|
const [check, setCheck] = useState({ phase: 'idle' });
|
||||||
|
const [exec, setExec] = useState({ phase: 'idle' });
|
||||||
|
const [confirmText, setConfirmText] = useState('');
|
||||||
|
const runCheck = () => {
|
||||||
|
setCheck({ phase: 'checking' });
|
||||||
|
setExec({ phase: 'idle' });
|
||||||
|
setConfirmText('');
|
||||||
|
api
|
||||||
|
.shutdownPrepCheck(name)
|
||||||
|
.then((r) => setCheck({ phase: 'done', ready: r.ready, output: r.output }))
|
||||||
|
.catch((e) => setCheck({ phase: 'error', message: String(e) }));
|
||||||
|
};
|
||||||
|
const runExecute = () => {
|
||||||
|
if (confirmText !== name)
|
||||||
|
return;
|
||||||
|
setExec({ phase: 'starting' });
|
||||||
|
api
|
||||||
|
.shutdownPrepExecute(name, confirmText)
|
||||||
|
.then((r) => setExec({ phase: 'started', message: r.message }))
|
||||||
|
.catch((e) => setExec({ phase: 'error', message: String(e) }));
|
||||||
|
};
|
||||||
|
const ready = check.phase === 'done' && check.ready;
|
||||||
|
return (_jsxs("div", { className: "rounded border border-slate-800 bg-slate-900/40 p-4 text-sm space-y-3", children: [_jsxs("div", { className: "flex items-center justify-between", children: [_jsx("div", { className: "text-slate-500 uppercase text-xs tracking-wider", children: "shutdown prep" }), _jsx("button", { onClick: runCheck, disabled: check.phase === 'checking', className: "px-3 py-1.5 rounded bg-slate-800 hover:bg-slate-700 text-slate-200 text-xs disabled:opacity-50", children: check.phase === 'checking' ? 'checking…' : 'check readiness for shutdown' })] }), _jsxs("p", { className: "text-xs text-slate-500", children: ["Runs a dry-run of the pre-shutdown checklist on ", _jsx("span", { className: "font-mono", children: name }), ' ', "\u2014 active builds, pending peer sync, a final snapshot + replicate to cold. Nothing is stopped or unmounted by this check."] }), check.phase === 'error' && (_jsx("div", { className: "rounded border border-red-800 bg-red-950/40 p-3 text-red-300 text-xs", children: check.message })), check.phase === 'done' && (_jsxs(_Fragment, { children: [_jsx("div", { className: `rounded border p-3 text-xs ${check.ready
|
||||||
|
? 'border-emerald-800 bg-emerald-950/40 text-emerald-300'
|
||||||
|
: 'border-amber-800 bg-amber-950/40 text-amber-300'}`, children: check.ready
|
||||||
|
? '✓ ready — safe to start the real shutdown-prep run'
|
||||||
|
: '! not ready — see output below (active build or un-synced changes are the usual cause)' }), _jsx("pre", { className: "max-h-72 overflow-auto rounded bg-slate-950 border border-slate-800 p-3 text-[11px] leading-relaxed text-slate-300 whitespace-pre-wrap", children: check.output })] })), ready && exec.phase !== 'started' && (_jsxs("div", { className: "rounded border border-red-900 bg-red-950/30 p-3 space-y-2", children: [_jsxs("div", { className: "text-red-300 text-xs", children: ["This starts the real run: stops maintenance timers, the dashboard, the storage daemon (gossip announces departure to peers), and unmounts FUSE on", ' ', _jsx("span", { className: "font-mono", children: name }), ". The node's dashboard connection will drop partway through \u2014 that's expected, not an error. It does ", _jsx("strong", { children: "not" }), " power the machine off; do that yourself once it's gone dark."] }), _jsxs("div", { className: "flex items-center gap-2", children: [_jsx("input", { value: confirmText, onChange: (e) => setConfirmText(e.target.value), placeholder: `type "${name}" to confirm`, className: "flex-1 rounded bg-slate-900 border border-slate-700 px-2 py-1.5 text-xs font-mono text-slate-200 placeholder:text-slate-600" }), _jsx("button", { onClick: runExecute, disabled: confirmText !== name || exec.phase === 'starting', className: "px-3 py-1.5 rounded bg-red-900 hover:bg-red-800 text-red-100 text-xs disabled:opacity-40 disabled:cursor-not-allowed whitespace-nowrap", children: exec.phase === 'starting' ? 'starting…' : `stop services on ${name}` })] })] })), exec.phase === 'error' && (_jsx("div", { className: "rounded border border-red-800 bg-red-950/40 p-3 text-red-300 text-xs", children: exec.message })), exec.phase === 'started' && (_jsx("div", { className: "rounded border border-sky-800 bg-sky-950/30 p-3 text-sky-300 text-xs", children: exec.message }))] }));
|
||||||
|
}
|
||||||
@@ -0,0 +1,144 @@
|
|||||||
|
import { useState } from 'react';
|
||||||
|
import { api } from '../lib/api';
|
||||||
|
|
||||||
|
interface Props {
|
||||||
|
name: string;
|
||||||
|
}
|
||||||
|
|
||||||
|
type CheckState =
|
||||||
|
| { phase: 'idle' }
|
||||||
|
| { phase: 'checking' }
|
||||||
|
| { phase: 'done'; ready: boolean; output: string }
|
||||||
|
| { phase: 'error'; message: string };
|
||||||
|
|
||||||
|
type ExecState =
|
||||||
|
| { phase: 'idle' }
|
||||||
|
| { phase: 'starting' }
|
||||||
|
| { phase: 'started'; message: string }
|
||||||
|
| { phase: 'error'; message: string };
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Node-maintenance panel: runs `safe-shutdown-prep.sh --dry-run` on
|
||||||
|
* demand (safe, read-mostly, never stops anything) and — only once
|
||||||
|
* that comes back ready — unlocks a type-to-confirm button that
|
||||||
|
* starts the real run.
|
||||||
|
*
|
||||||
|
* The real run is fire-and-forget by necessity: its own steps stop
|
||||||
|
* this node's daemon, which is what's serving this very page, so
|
||||||
|
* there is no way to stream a live result past that point. Once
|
||||||
|
* started, the UI says so plainly and points at the on-disk log for
|
||||||
|
* the full report.
|
||||||
|
*/
|
||||||
|
export function ShutdownPrepPanel({ name }: Props) {
|
||||||
|
const [check, setCheck] = useState<CheckState>({ phase: 'idle' });
|
||||||
|
const [exec, setExec] = useState<ExecState>({ phase: 'idle' });
|
||||||
|
const [confirmText, setConfirmText] = useState('');
|
||||||
|
|
||||||
|
const runCheck = () => {
|
||||||
|
setCheck({ phase: 'checking' });
|
||||||
|
setExec({ phase: 'idle' });
|
||||||
|
setConfirmText('');
|
||||||
|
api
|
||||||
|
.shutdownPrepCheck(name)
|
||||||
|
.then((r) => setCheck({ phase: 'done', ready: r.ready, output: r.output }))
|
||||||
|
.catch((e) => setCheck({ phase: 'error', message: String(e) }));
|
||||||
|
};
|
||||||
|
|
||||||
|
const runExecute = () => {
|
||||||
|
if (confirmText !== name) return;
|
||||||
|
setExec({ phase: 'starting' });
|
||||||
|
api
|
||||||
|
.shutdownPrepExecute(name, confirmText)
|
||||||
|
.then((r) => setExec({ phase: 'started', message: r.message }))
|
||||||
|
.catch((e) => setExec({ phase: 'error', message: String(e) }));
|
||||||
|
};
|
||||||
|
|
||||||
|
const ready = check.phase === 'done' && check.ready;
|
||||||
|
|
||||||
|
return (
|
||||||
|
<div className="rounded border border-slate-800 bg-slate-900/40 p-4 text-sm space-y-3">
|
||||||
|
<div className="flex items-center justify-between">
|
||||||
|
<div className="text-slate-500 uppercase text-xs tracking-wider">
|
||||||
|
shutdown prep
|
||||||
|
</div>
|
||||||
|
<button
|
||||||
|
onClick={runCheck}
|
||||||
|
disabled={check.phase === 'checking'}
|
||||||
|
className="px-3 py-1.5 rounded bg-slate-800 hover:bg-slate-700 text-slate-200 text-xs disabled:opacity-50"
|
||||||
|
>
|
||||||
|
{check.phase === 'checking' ? 'checking…' : 'check readiness for shutdown'}
|
||||||
|
</button>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<p className="text-xs text-slate-500">
|
||||||
|
Runs a dry-run of the pre-shutdown checklist on <span className="font-mono">{name}</span>{' '}
|
||||||
|
— active builds, pending peer sync, a final snapshot + replicate to cold. Nothing is
|
||||||
|
stopped or unmounted by this check.
|
||||||
|
</p>
|
||||||
|
|
||||||
|
{check.phase === 'error' && (
|
||||||
|
<div className="rounded border border-red-800 bg-red-950/40 p-3 text-red-300 text-xs">
|
||||||
|
{check.message}
|
||||||
|
</div>
|
||||||
|
)}
|
||||||
|
|
||||||
|
{check.phase === 'done' && (
|
||||||
|
<>
|
||||||
|
<div
|
||||||
|
className={`rounded border p-3 text-xs ${
|
||||||
|
check.ready
|
||||||
|
? 'border-emerald-800 bg-emerald-950/40 text-emerald-300'
|
||||||
|
: 'border-amber-800 bg-amber-950/40 text-amber-300'
|
||||||
|
}`}
|
||||||
|
>
|
||||||
|
{check.ready
|
||||||
|
? '✓ ready — safe to start the real shutdown-prep run'
|
||||||
|
: '! not ready — see output below (active build or un-synced changes are the usual cause)'}
|
||||||
|
</div>
|
||||||
|
<pre className="max-h-72 overflow-auto rounded bg-slate-950 border border-slate-800 p-3 text-[11px] leading-relaxed text-slate-300 whitespace-pre-wrap">
|
||||||
|
{check.output}
|
||||||
|
</pre>
|
||||||
|
</>
|
||||||
|
)}
|
||||||
|
|
||||||
|
{ready && exec.phase !== 'started' && (
|
||||||
|
<div className="rounded border border-red-900 bg-red-950/30 p-3 space-y-2">
|
||||||
|
<div className="text-red-300 text-xs">
|
||||||
|
This starts the real run: stops maintenance timers, the dashboard, the storage
|
||||||
|
daemon (gossip announces departure to peers), and unmounts FUSE on{' '}
|
||||||
|
<span className="font-mono">{name}</span>. The node's dashboard connection will drop
|
||||||
|
partway through — that's expected, not an error. It does <strong>not</strong> power
|
||||||
|
the machine off; do that yourself once it's gone dark.
|
||||||
|
</div>
|
||||||
|
<div className="flex items-center gap-2">
|
||||||
|
<input
|
||||||
|
value={confirmText}
|
||||||
|
onChange={(e) => setConfirmText(e.target.value)}
|
||||||
|
placeholder={`type "${name}" to confirm`}
|
||||||
|
className="flex-1 rounded bg-slate-900 border border-slate-700 px-2 py-1.5 text-xs font-mono text-slate-200 placeholder:text-slate-600"
|
||||||
|
/>
|
||||||
|
<button
|
||||||
|
onClick={runExecute}
|
||||||
|
disabled={confirmText !== name || exec.phase === 'starting'}
|
||||||
|
className="px-3 py-1.5 rounded bg-red-900 hover:bg-red-800 text-red-100 text-xs disabled:opacity-40 disabled:cursor-not-allowed whitespace-nowrap"
|
||||||
|
>
|
||||||
|
{exec.phase === 'starting' ? 'starting…' : `stop services on ${name}`}
|
||||||
|
</button>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
)}
|
||||||
|
|
||||||
|
{exec.phase === 'error' && (
|
||||||
|
<div className="rounded border border-red-800 bg-red-950/40 p-3 text-red-300 text-xs">
|
||||||
|
{exec.message}
|
||||||
|
</div>
|
||||||
|
)}
|
||||||
|
|
||||||
|
{exec.phase === 'started' && (
|
||||||
|
<div className="rounded border border-sky-800 bg-sky-950/30 p-3 text-sky-300 text-xs">
|
||||||
|
{exec.message}
|
||||||
|
</div>
|
||||||
|
)}
|
||||||
|
</div>
|
||||||
|
);
|
||||||
|
}
|
||||||
@@ -23,6 +23,19 @@ async function get(path) {
|
|||||||
}
|
}
|
||||||
return resp.json();
|
return resp.json();
|
||||||
}
|
}
|
||||||
|
async function post(path, body) {
|
||||||
|
const full = `${API_BASE}${path}`;
|
||||||
|
const resp = await fetch(full, {
|
||||||
|
method: 'POST',
|
||||||
|
headers: { Accept: 'application/json', 'Content-Type': 'application/json' },
|
||||||
|
body: body === undefined ? undefined : JSON.stringify(body),
|
||||||
|
});
|
||||||
|
if (!resp.ok) {
|
||||||
|
const text = await resp.text().catch(() => '');
|
||||||
|
throw new Error(`${full} → ${resp.status} ${resp.statusText}${text ? `: ${text}` : ''}`);
|
||||||
|
}
|
||||||
|
return resp.json();
|
||||||
|
}
|
||||||
export const api = {
|
export const api = {
|
||||||
fleet: () => get('/v2/fleet'),
|
fleet: () => get('/v2/fleet'),
|
||||||
projects: () => get('/v2/projects'),
|
projects: () => get('/v2/projects'),
|
||||||
@@ -32,6 +45,10 @@ export const api = {
|
|||||||
refs: (limit = 200, offset = 0) => get(`/v2/storage/refs?limit=${limit}&offset=${offset}`),
|
refs: (limit = 200, offset = 0) => get(`/v2/storage/refs?limit=${limit}&offset=${offset}`),
|
||||||
snapshots: () => get('/v2/storage/snapshots'),
|
snapshots: () => get('/v2/storage/snapshots'),
|
||||||
refTracking: (repo = '') => get(`/v2/storage/ref-tracking?repo=${encodeURIComponent(repo)}`),
|
refTracking: (repo = '') => get(`/v2/storage/ref-tracking?repo=${encodeURIComponent(repo)}`),
|
||||||
|
shutdownPrepCheck: (name) => post(`/v2/node/${name}/shutdown-prep/check`),
|
||||||
|
shutdownPrepExecute: (name, confirmNodeName) => post(`/v2/node/${name}/shutdown-prep/execute`, {
|
||||||
|
confirm_node_name: confirmNodeName,
|
||||||
|
}),
|
||||||
};
|
};
|
||||||
/** Format bytes as MB / GB / TB as needed. */
|
/** Format bytes as MB / GB / TB as needed. */
|
||||||
export function fmtBytes(n) {
|
export function fmtBytes(n) {
|
||||||
|
|||||||
@@ -119,6 +119,20 @@ async function get<T>(path: string): Promise<T> {
|
|||||||
return resp.json();
|
return resp.json();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
async function post<T>(path: string, body?: unknown): Promise<T> {
|
||||||
|
const full = `${API_BASE}${path}`;
|
||||||
|
const resp = await fetch(full, {
|
||||||
|
method: 'POST',
|
||||||
|
headers: { Accept: 'application/json', 'Content-Type': 'application/json' },
|
||||||
|
body: body === undefined ? undefined : JSON.stringify(body),
|
||||||
|
});
|
||||||
|
if (!resp.ok) {
|
||||||
|
const text = await resp.text().catch(() => '');
|
||||||
|
throw new Error(`${full} → ${resp.status} ${resp.statusText}${text ? `: ${text}` : ''}`);
|
||||||
|
}
|
||||||
|
return resp.json();
|
||||||
|
}
|
||||||
|
|
||||||
// All paths are relative to API_BASE. E.g. '/v2/fleet' becomes
|
// All paths are relative to API_BASE. E.g. '/v2/fleet' becomes
|
||||||
// '/api/v2/fleet' locally or '/clawstor/api/v2/fleet' via Tailscale.
|
// '/api/v2/fleet' locally or '/clawstor/api/v2/fleet' via Tailscale.
|
||||||
export interface ProjectRow {
|
export interface ProjectRow {
|
||||||
@@ -145,8 +159,26 @@ export const api = {
|
|||||||
snapshots: () => get<SnapshotSummary[]>('/v2/storage/snapshots'),
|
snapshots: () => get<SnapshotSummary[]>('/v2/storage/snapshots'),
|
||||||
refTracking: (repo = '') =>
|
refTracking: (repo = '') =>
|
||||||
get<RefTrackingItem[]>(`/v2/storage/ref-tracking?repo=${encodeURIComponent(repo)}`),
|
get<RefTrackingItem[]>(`/v2/storage/ref-tracking?repo=${encodeURIComponent(repo)}`),
|
||||||
|
shutdownPrepCheck: (name: string) =>
|
||||||
|
post<ShutdownPrepCheckResponse>(`/v2/node/${name}/shutdown-prep/check`),
|
||||||
|
shutdownPrepExecute: (name: string, confirmNodeName: string) =>
|
||||||
|
post<ShutdownPrepExecuteResponse>(`/v2/node/${name}/shutdown-prep/execute`, {
|
||||||
|
confirm_node_name: confirmNodeName,
|
||||||
|
}),
|
||||||
};
|
};
|
||||||
|
|
||||||
|
export interface ShutdownPrepCheckResponse {
|
||||||
|
node: string;
|
||||||
|
ready: boolean;
|
||||||
|
output: string;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface ShutdownPrepExecuteResponse {
|
||||||
|
node: string;
|
||||||
|
started: boolean;
|
||||||
|
message: string;
|
||||||
|
}
|
||||||
|
|
||||||
/** Format bytes as MB / GB / TB as needed. */
|
/** Format bytes as MB / GB / TB as needed. */
|
||||||
export function fmtBytes(n: number): string {
|
export function fmtBytes(n: number): string {
|
||||||
if (n < 1024) return `${n} B`;
|
if (n < 1024) return `${n} B`;
|
||||||
|
|||||||
@@ -3,6 +3,7 @@ import { useEffect, useState } from 'react';
|
|||||||
import { Link } from 'wouter';
|
import { Link } from 'wouter';
|
||||||
import { api, fmtBytes } from '../lib/api';
|
import { api, fmtBytes } from '../lib/api';
|
||||||
import { StatTile } from '../components/StatTile';
|
import { StatTile } from '../components/StatTile';
|
||||||
|
import { ShutdownPrepPanel } from '../components/ShutdownPrepPanel';
|
||||||
export function NodeDetail({ name }) {
|
export function NodeDetail({ name }) {
|
||||||
const [status, setStatus] = useState(null);
|
const [status, setStatus] = useState(null);
|
||||||
const [err, setErr] = useState(null);
|
const [err, setErr] = useState(null);
|
||||||
@@ -15,5 +16,5 @@ export function NodeDetail({ name }) {
|
|||||||
})
|
})
|
||||||
.catch((e) => setErr(String(e)));
|
.catch((e) => setErr(String(e)));
|
||||||
}, [name]);
|
}, [name]);
|
||||||
return (_jsxs("div", { className: "space-y-6", children: [_jsxs("div", { children: [_jsx(Link, { href: "/", children: _jsx("a", { className: "text-sm text-slate-500 hover:text-slate-300", children: "\u2190 fleet" }) }), _jsx("h1", { className: "text-2xl font-semibold text-slate-100 mt-2", children: name })] }), err && (_jsxs("div", { className: "rounded border border-amber-800 bg-amber-950/40 p-3 text-amber-300 text-sm", children: [err, _jsxs("div", { className: "mt-2 text-xs text-slate-400", children: ["Cross-node lookup lands in a follow-on PR. Until then this page shows detail only when you're already viewing the dashboard hosted by ", name, ". Try opening", ' ', _jsxs("span", { className: "font-mono", children: ["http://", name, ":7700/v2/#/nodes/", name] }), ' ', "directly."] })] })), status && (_jsxs(_Fragment, { children: [_jsxs("div", { className: "grid grid-cols-2 sm:grid-cols-3 lg:grid-cols-6 gap-3", children: [_jsx(StatTile, { label: "blobs", value: status.blob_count.toLocaleString(), color: "ok" }), _jsx(StatTile, { label: "tags", value: status.tag_count, color: "ok" }), _jsx(StatTile, { label: "refs", value: status.ref_count, color: "ok" }), _jsx(StatTile, { label: "snapshots", value: status.snapshot_count, color: "ok" }), _jsx(StatTile, { label: "ref-tracking", value: status.ref_tracking_count, color: "ok" }), _jsx(StatTile, { label: "store size", value: fmtBytes(status.blob_store_bytes), color: "ok" })] }), _jsxs("div", { className: "rounded border border-slate-800 bg-slate-900/40 p-4 text-sm", children: [_jsx("div", { className: "text-slate-500 uppercase text-xs tracking-wider", children: "blob store root" }), _jsx("div", { className: "font-mono mt-1", children: status.blob_store_root ?? '—' })] })] }))] }));
|
return (_jsxs("div", { className: "space-y-6", children: [_jsxs("div", { children: [_jsx(Link, { href: "/", children: _jsx("a", { className: "text-sm text-slate-500 hover:text-slate-300", children: "\u2190 fleet" }) }), _jsx("h1", { className: "text-2xl font-semibold text-slate-100 mt-2", children: name })] }), err && (_jsxs("div", { className: "rounded border border-amber-800 bg-amber-950/40 p-3 text-amber-300 text-sm", children: [err, _jsxs("div", { className: "mt-2 text-xs text-slate-400", children: ["Cross-node lookup lands in a follow-on PR. Until then this page shows detail only when you're already viewing the dashboard hosted by ", name, ". Try opening", ' ', _jsxs("span", { className: "font-mono", children: ["http://", name, ":7700/v2/#/nodes/", name] }), ' ', "directly."] })] })), status && (_jsxs(_Fragment, { children: [_jsxs("div", { className: "grid grid-cols-2 sm:grid-cols-3 lg:grid-cols-6 gap-3", children: [_jsx(StatTile, { label: "blobs", value: status.blob_count.toLocaleString(), color: "ok" }), _jsx(StatTile, { label: "tags", value: status.tag_count, color: "ok" }), _jsx(StatTile, { label: "refs", value: status.ref_count, color: "ok" }), _jsx(StatTile, { label: "snapshots", value: status.snapshot_count, color: "ok" }), _jsx(StatTile, { label: "ref-tracking", value: status.ref_tracking_count, color: "ok" }), _jsx(StatTile, { label: "store size", value: fmtBytes(status.blob_store_bytes), color: "ok" })] }), _jsxs("div", { className: "rounded border border-slate-800 bg-slate-900/40 p-4 text-sm", children: [_jsx("div", { className: "text-slate-500 uppercase text-xs tracking-wider", children: "blob store root" }), _jsx("div", { className: "font-mono mt-1", children: status.blob_store_root ?? '—' })] }), _jsx(ShutdownPrepPanel, { name: name })] }))] }));
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -2,6 +2,7 @@ import { useEffect, useState } from 'react';
|
|||||||
import { Link } from 'wouter';
|
import { Link } from 'wouter';
|
||||||
import { api, NodeStatusV2, fmtBytes } from '../lib/api';
|
import { api, NodeStatusV2, fmtBytes } from '../lib/api';
|
||||||
import { StatTile } from '../components/StatTile';
|
import { StatTile } from '../components/StatTile';
|
||||||
|
import { ShutdownPrepPanel } from '../components/ShutdownPrepPanel';
|
||||||
|
|
||||||
interface Props {
|
interface Props {
|
||||||
name: string;
|
name: string;
|
||||||
@@ -67,6 +68,8 @@ export function NodeDetail({ name }: Props) {
|
|||||||
</div>
|
</div>
|
||||||
<div className="font-mono mt-1">{status.blob_store_root ?? '—'}</div>
|
<div className="font-mono mt-1">{status.blob_store_root ?? '—'}</div>
|
||||||
</div>
|
</div>
|
||||||
|
|
||||||
|
<ShutdownPrepPanel name={name} />
|
||||||
</>
|
</>
|
||||||
)}
|
)}
|
||||||
</div>
|
</div>
|
||||||
|
|||||||
@@ -1 +1 @@
|
|||||||
{"root":["./src/app.tsx","./src/main.tsx","./src/components/nodecard.tsx","./src/components/projectspanel.tsx","./src/components/stattile.tsx","./src/components/storagebar.tsx","./src/lib/api.ts","./src/pages/commandcenter.tsx","./src/pages/nodedetail.tsx","./src/pages/reftrackingpage.tsx","./src/pages/storagebrowser.tsx"],"version":"6.0.3"}
|
{"root":["./src/App.tsx","./src/main.tsx","./src/components/NodeCard.tsx","./src/components/ProjectsPanel.tsx","./src/components/ShutdownPrepPanel.tsx","./src/components/StatTile.tsx","./src/components/StorageBar.tsx","./src/lib/api.ts","./src/pages/CommandCenter.tsx","./src/pages/NodeDetail.tsx","./src/pages/RefTrackingPage.tsx","./src/pages/StorageBrowser.tsx"],"version":"6.0.3"}
|
||||||
@@ -6,13 +6,15 @@ import react from '@vitejs/plugin-react';
|
|||||||
// During local dev the daemon proxies /api/v2/* on :7700 so
|
// During local dev the daemon proxies /api/v2/* on :7700 so
|
||||||
// `vite dev` on :5173 can hit it via server.proxy.
|
// `vite dev` on :5173 can hit it via server.proxy.
|
||||||
export default defineConfig({
|
export default defineConfig({
|
||||||
// Absolute base tied to the deploy path. Prior try was `./`
|
// Absolute base matching the backend's actual mount point
|
||||||
// (fully relative) which broke when the user hit `/clawstor`
|
// (`serve.rs` nests the v2 static dir at `/v2` via
|
||||||
// without trailing slash — browser resolves `./assets/…`
|
// `nest_service("/v2", …)`). A prior `/clawstor/` base assumed a
|
||||||
// against `/clawstor` treated as a file, gives `/assets/…`,
|
// Tailscale Serve path mapping that was never actually configured
|
||||||
// 404 from Tailscale. Absolute `/clawstor/` sidesteps the
|
// on any node (checked `tailscale serve status` on tank +
|
||||||
// slash / no-slash ambiguity.
|
// architect: neither proxies a `/clawstor` path) — that base
|
||||||
base: '/clawstor/',
|
// silently broke direct `:7700/v2/` access, the only access
|
||||||
|
// pattern that's actually live.
|
||||||
|
base: '/v2/',
|
||||||
plugins: [react()],
|
plugins: [react()],
|
||||||
server: {
|
server: {
|
||||||
port: 5173,
|
port: 5173,
|
||||||
|
|||||||
Executable
+238
@@ -0,0 +1,238 @@
|
|||||||
|
#!/usr/bin/env bash
|
||||||
|
# safe-shutdown-prep.sh — bring a clawstor node to a clean, safe stop
|
||||||
|
# before hardware maintenance (parts replacement, drive swap, etc.).
|
||||||
|
#
|
||||||
|
# Run this ON the node you're about to power off. It does NOT power
|
||||||
|
# the machine off itself — the last line of output tells you the
|
||||||
|
# command to run for that, once everything above it is clean.
|
||||||
|
#
|
||||||
|
# What it does, in order:
|
||||||
|
# 1. Refuse to proceed if a cargo/rustc build is active against a
|
||||||
|
# tracked project's warm_path (unless --force).
|
||||||
|
# 2. Refuse to proceed if the sync queue has pending jobs peers
|
||||||
|
# haven't received yet (unless --force). Gives it one chance to
|
||||||
|
# drain via `claw-store sync <project>` before failing.
|
||||||
|
# 3. Take a final ZFS snapshot of the warm tier + replicate it to
|
||||||
|
# the configured cold peer, and wait for both to finish.
|
||||||
|
# 4. Stop the four maintenance timers (scrub/gc/ref-sweep/
|
||||||
|
# snapshot-rotate) so nothing fires mid-shutdown or immediately
|
||||||
|
# after next boot before you've verified the node.
|
||||||
|
# 5. Stop claw-store-serve.service (dashboard) — no data risk, just
|
||||||
|
# tidy.
|
||||||
|
# 6. Stop claw-store.service gracefully. The unit's
|
||||||
|
# TimeoutStopSec=60 gives the daemon's SIGTERM handler room to
|
||||||
|
# let gossip announce this node's departure to peers before the
|
||||||
|
# process exits — skipping this step means peers only notice via
|
||||||
|
# the failure detector's dead_node_grace_period (10s) instead of
|
||||||
|
# an immediate clean departure.
|
||||||
|
# 7. Stop claw-fuse.service and verify the mount is actually gone
|
||||||
|
# (retries a lazy unmount if the clean one doesn't take).
|
||||||
|
# 8. Sync filesystem buffers and print zpool health for the warm
|
||||||
|
# tier's pool — warns (does not block) if the pool is degraded,
|
||||||
|
# since that's independently worth knowing before you touch
|
||||||
|
# hardware.
|
||||||
|
#
|
||||||
|
# Flags:
|
||||||
|
# --force Skip the active-build and pending-sync guards.
|
||||||
|
# Everything else (steps 3-8) still runs.
|
||||||
|
# --export-zpool Additionally `zpool export` the warm-tier pool
|
||||||
|
# at the end — only do this if you're physically
|
||||||
|
# removing the storage drives, not for e.g. a RAM
|
||||||
|
# or PSU swap. Requires a matching `zpool import`
|
||||||
|
# after the node is back up before claw-store.service
|
||||||
|
# will find its data again.
|
||||||
|
# --skip-replicate Skip step 3 (snapshot + replicate). Use only if
|
||||||
|
# you already know cold tier is current, or this
|
||||||
|
# node has no [replication] configured.
|
||||||
|
# --dry-run Run every check (steps 1-2) and the snapshot/
|
||||||
|
# replicate (step 3) for real, but only print what
|
||||||
|
# steps 4-8 (stop timers/services, unmount, zpool
|
||||||
|
# export) would do instead of doing them. Use this
|
||||||
|
# first to verify the script sees your node's
|
||||||
|
# actual state correctly before trusting it live.
|
||||||
|
set -uo pipefail
|
||||||
|
|
||||||
|
CONFIG=${CLAWSTOR_CONFIG:-/etc/claw-store/config.toml}
|
||||||
|
BIN=${CLAWSTOR_BIN:-/usr/local/bin/claw-store}
|
||||||
|
SYNC_QUEUE=/var/lib/claw-store/sync-queue.toml
|
||||||
|
FORCE=0
|
||||||
|
EXPORT_ZPOOL=0
|
||||||
|
SKIP_REPLICATE=0
|
||||||
|
DRY_RUN=0
|
||||||
|
|
||||||
|
for arg in "$@"; do
|
||||||
|
case "$arg" in
|
||||||
|
--force) FORCE=1 ;;
|
||||||
|
--export-zpool) EXPORT_ZPOOL=1 ;;
|
||||||
|
--skip-replicate) SKIP_REPLICATE=1 ;;
|
||||||
|
--dry-run) DRY_RUN=1 ;;
|
||||||
|
*) echo "unknown flag: $arg" >&2; exit 2 ;;
|
||||||
|
esac
|
||||||
|
done
|
||||||
|
|
||||||
|
run() {
|
||||||
|
# Gate an actual state-changing command behind --dry-run.
|
||||||
|
if [ "$DRY_RUN" -eq 1 ]; then
|
||||||
|
echo " [dry-run] would run: $*"
|
||||||
|
return 0
|
||||||
|
fi
|
||||||
|
"$@"
|
||||||
|
}
|
||||||
|
|
||||||
|
hr() { printf '%.0s─' {1..66}; echo; }
|
||||||
|
step() { hr; echo "▶ $1"; hr; }
|
||||||
|
ok() { echo " ✓ $1"; }
|
||||||
|
warn() { echo " ! $1"; }
|
||||||
|
fail() { echo " ✗ $1" >&2; }
|
||||||
|
|
||||||
|
NODE=$(hostname)
|
||||||
|
echo "safe-shutdown-prep — $NODE — $(date -Iseconds)"
|
||||||
|
|
||||||
|
# ── 1. Active builds ────────────────────────────────────────────────
|
||||||
|
step "checking for active cargo/rustc builds"
|
||||||
|
ACTIVE=$(pgrep -af 'cargo|rustc' | grep -v "safe-shutdown-prep\|grep" || true)
|
||||||
|
if [ -n "$ACTIVE" ]; then
|
||||||
|
echo "$ACTIVE" | sed 's/^/ /'
|
||||||
|
if [ "$FORCE" -eq 1 ]; then
|
||||||
|
warn "active build(s) found — continuing anyway (--force)"
|
||||||
|
else
|
||||||
|
fail "active build(s) found on this node. A build in progress against"
|
||||||
|
fail "the warm tier can be interrupted mid-write by an unmount/shutdown."
|
||||||
|
fail "Wait for it to finish, or re-run with --force to proceed anyway."
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
else
|
||||||
|
ok "no active cargo/rustc processes"
|
||||||
|
fi
|
||||||
|
|
||||||
|
# ── 2. Sync queue ────────────────────────────────────────────────────
|
||||||
|
step "checking sync queue for pending peer pushes"
|
||||||
|
if [ -f "$SYNC_QUEUE" ]; then
|
||||||
|
DEPTH=$(grep -c '^project = ' "$SYNC_QUEUE" 2>/dev/null); DEPTH=${DEPTH:-0}
|
||||||
|
else
|
||||||
|
DEPTH=0
|
||||||
|
fi
|
||||||
|
if [ "$DEPTH" -gt 0 ]; then
|
||||||
|
warn "$DEPTH pending sync job(s) in $SYNC_QUEUE — attempting to drain"
|
||||||
|
PROJECTS=$(grep '^project = ' "$SYNC_QUEUE" | sed 's/project = "\(.*\)"/\1/')
|
||||||
|
for p in $PROJECTS; do
|
||||||
|
echo " syncing $p ..."
|
||||||
|
"$BIN" --config "$CONFIG" sync "$p" || warn "sync failed for $p"
|
||||||
|
done
|
||||||
|
DEPTH_AFTER=$(grep -c '^project = ' "$SYNC_QUEUE" 2>/dev/null); DEPTH_AFTER=${DEPTH_AFTER:-0}
|
||||||
|
if [ "$DEPTH_AFTER" -gt 0 ]; then
|
||||||
|
if [ "$FORCE" -eq 1 ]; then
|
||||||
|
warn "$DEPTH_AFTER job(s) still pending — continuing anyway (--force)"
|
||||||
|
else
|
||||||
|
fail "$DEPTH_AFTER sync job(s) still pending after drain attempt."
|
||||||
|
fail "Peers may be unreachable, or the push is failing for another"
|
||||||
|
fail "reason. Re-run with --force to shut down anyway (those changes"
|
||||||
|
fail "will catch up once this node is back and the daemon retries)."
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
else
|
||||||
|
ok "sync queue drained"
|
||||||
|
fi
|
||||||
|
else
|
||||||
|
ok "sync queue empty"
|
||||||
|
fi
|
||||||
|
|
||||||
|
# ── 3. Final snapshot + replicate to cold ───────────────────────────
|
||||||
|
if [ "$SKIP_REPLICATE" -eq 1 ]; then
|
||||||
|
step "skipping snapshot + replicate (--skip-replicate)"
|
||||||
|
else
|
||||||
|
step "taking final snapshot + replicating to cold tier"
|
||||||
|
if "$BIN" --config "$CONFIG" snapshot; then
|
||||||
|
ok "snapshot created"
|
||||||
|
else
|
||||||
|
warn "snapshot command failed — check output above"
|
||||||
|
fi
|
||||||
|
if "$BIN" --config "$CONFIG" replicate; then
|
||||||
|
ok "replication to cold tier complete"
|
||||||
|
else
|
||||||
|
warn "replicate command failed or not configured — check output above"
|
||||||
|
warn "([replication] section may be absent on this node; that's fine)"
|
||||||
|
fi
|
||||||
|
fi
|
||||||
|
|
||||||
|
# ── 4. Stop maintenance timers ──────────────────────────────────────
|
||||||
|
step "stopping maintenance timers"
|
||||||
|
for t in clawstor-scrub clawstor-gc clawstor-ref-sweep clawstor-snapshot-rotate; do
|
||||||
|
run systemctl --user stop "$t.timer" 2>/dev/null && ok "$t.timer stopped" || warn "$t.timer not running or not found"
|
||||||
|
done
|
||||||
|
|
||||||
|
# ── 5. Stop dashboard ────────────────────────────────────────────────
|
||||||
|
step "stopping claw-store-serve.service"
|
||||||
|
if systemctl is-active --quiet claw-store-serve.service 2>/dev/null; then
|
||||||
|
run sudo systemctl stop claw-store-serve.service && ok "stopped" || fail "failed to stop"
|
||||||
|
else
|
||||||
|
ok "not running"
|
||||||
|
fi
|
||||||
|
|
||||||
|
# ── 6. Stop daemon (gossip departure) ───────────────────────────────
|
||||||
|
step "stopping claw-store.service (gossip will announce departure)"
|
||||||
|
if systemctl is-active --quiet claw-store.service 2>/dev/null; then
|
||||||
|
run sudo systemctl stop claw-store.service && ok "stopped cleanly" || fail "failed to stop — check 'systemctl status claw-store.service'"
|
||||||
|
else
|
||||||
|
ok "not running"
|
||||||
|
fi
|
||||||
|
|
||||||
|
# ── 7. Unmount FUSE ──────────────────────────────────────────────────
|
||||||
|
step "unmounting FUSE"
|
||||||
|
if systemctl is-active --quiet claw-fuse.service 2>/dev/null; then
|
||||||
|
run sudo systemctl stop claw-fuse.service
|
||||||
|
sleep 1
|
||||||
|
fi
|
||||||
|
if [ "$DRY_RUN" -eq 1 ]; then
|
||||||
|
if mount | grep -q "type fuse.clawstor"; then
|
||||||
|
warn "still mounted (expected — nothing was actually stopped in --dry-run)"
|
||||||
|
else
|
||||||
|
ok "already unmounted"
|
||||||
|
fi
|
||||||
|
else
|
||||||
|
if mount | grep -q "type fuse.clawstor"; then
|
||||||
|
warn "still mounted after service stop — trying lazy unmount"
|
||||||
|
sudo umount -l ~/clawstor-mount 2>/dev/null
|
||||||
|
sleep 1
|
||||||
|
fi
|
||||||
|
if mount | grep -q "type fuse.clawstor"; then
|
||||||
|
fail "FUSE mount would not come down: $(mount | grep 'fuse.clawstor')"
|
||||||
|
fail "Do not power off until this is resolved — an unclean FUSE"
|
||||||
|
fail "unmount can leave a stale mountpoint that needs manual cleanup"
|
||||||
|
fail "on next boot."
|
||||||
|
exit 1
|
||||||
|
else
|
||||||
|
ok "unmounted"
|
||||||
|
fi
|
||||||
|
fi
|
||||||
|
|
||||||
|
# ── 8. Flush + pool health ──────────────────────────────────────────
|
||||||
|
step "flushing filesystem buffers"
|
||||||
|
sync
|
||||||
|
ok "sync complete"
|
||||||
|
|
||||||
|
step "zpool health check"
|
||||||
|
if ! command -v zpool >/dev/null 2>&1; then
|
||||||
|
ok "no zpool binary on this node — warm tier is not ZFS-backed here, nothing to check"
|
||||||
|
else
|
||||||
|
POOL=$(df --output=source /slab 2>/dev/null | tail -1 | tr -d '[:space:]')
|
||||||
|
if [ -n "$POOL" ] && [ "$POOL" != "none" ]; then
|
||||||
|
echo " pool: $POOL"
|
||||||
|
zpool status -x "$POOL" 2>&1 | sed 's/^/ /'
|
||||||
|
if [ "$EXPORT_ZPOOL" -eq 1 ]; then
|
||||||
|
step "exporting $POOL (--export-zpool)"
|
||||||
|
run sudo zpool export "$POOL" && ok "exported — remember: zpool import $POOL after reboot" || fail "export failed"
|
||||||
|
fi
|
||||||
|
else
|
||||||
|
warn "zpool present but /slab isn't a recognizable ZFS mount"
|
||||||
|
fi
|
||||||
|
fi
|
||||||
|
|
||||||
|
hr
|
||||||
|
if [ "$DRY_RUN" -eq 1 ]; then
|
||||||
|
echo "DRY RUN COMPLETE — nothing was actually stopped or unmounted."
|
||||||
|
echo "Re-run without --dry-run when ready to actually prep for shutdown."
|
||||||
|
else
|
||||||
|
echo "SAFE TO POWER OFF — run: sudo shutdown -h now"
|
||||||
|
fi
|
||||||
|
hr
|
||||||
Reference in New Issue
Block a user