Fix stdout/stderr ordering in ShutdownPrepCheck

Command::output() captures stdout and stderr as two separate
buffers. check() was concatenating stdout-then-stderr, which throws
away chronological order entirely -- every stderr line (e.g.
"Error: send_to_host not set" from `claw-store replicate` on a node
with no downstream replication target, which is normal and expected
on architect) landed at the very end of the report regardless of
when it actually printed, making a mid-script, already-handled
condition look like a failure that happened after "DRY RUN
COMPLETE".

Fix: invoke via `bash -c "script --dry-run 2>&1"` so stderr merges
into stdout inside the shell, before either stream reaches us --
true chronological order preserved, single buffer to read.

Verified on architect: the send_to_host message now appears exactly
where it happens, inside the "taking final snapshot + replicating"
step, with DRY RUN COMPLETE correctly last.

Co-Authored-By: Claude Sonnet 5 <[email protected]>

Makefile

@@ -32,7 +32,7 @@ install-systemd:
 install-dashboard:
 	cd dashboard && npm ci && npm run build
 	install -dm755 $(INSTALL_STATIC)
-	cp -r dashboard/dist/. $(INSTALL_STATIC)/
+	cp -r claw-store/static/. $(INSTALL_STATIC)/
 	@echo "Dashboard installed to $(INSTALL_STATIC)"
 
 ## Install node-specific config (NODE=architect|tank)

claw-store/src/cargo_init.rs

@@ -21,8 +21,14 @@ pub fn write_cargo_config(warm_path: &Path, hot_target_path: &Path) -> Result<()
         String::new()
     };
 
+    // Drop any leading copies of our own marker comment before stripping the
+    // [build] section — it sits above the [build] header, outside the range
+    // strip_section tracks, so without this it would survive every
+    // regenerate cycle and duplicate one more time.
+    let existing = strip_leading_marker(&existing);
+
     // Remove old [build] block (from its header to the next section or EOF).
-    let stripped = strip_section(&existing, "build");
+    let stripped = strip_section(existing, "build");
 
     let new_content = format!(
         "# claw-store managed — do not edit manually\n\
@@ -37,6 +43,21 @@ pub fn write_cargo_config(warm_path: &Path, hot_target_path: &Path) -> Result<()
         .with_context(|| format!("writing {}", config_path.display()))
 }
 
+const MANAGED_MARKER: &str = "# claw-store managed — do not edit manually";
+
+/// Strip leading copies of `MANAGED_MARKER`, one per line, from the start of `src`.
+fn strip_leading_marker(src: &str) -> &str {
+    let mut rest = src;
+    while let Some(line_end) = rest.find('\n') {
+        if rest[..line_end].trim() == MANAGED_MARKER {
+            rest = &rest[line_end + 1..];
+        } else {
+            break;
+        }
+    }
+    rest
+}
+
 /// Remove a TOML section `[name]` and all its key=value lines from `src`,
 /// stopping at the next `[section]` header or EOF.
 fn strip_section(src: &str, name: &str) -> String {
@@ -102,6 +123,46 @@ mod tests {
         assert!(verify_cargo_config(&warm, &hot).unwrap());
     }
 
+    #[test]
+    fn test_write_cargo_config_idempotent_no_duplicate_marker() {
+        let dir = TempDir::new().unwrap();
+        let warm = dir.path().join("proj");
+        std::fs::create_dir_all(&warm).unwrap();
+        let hot: std::path::PathBuf = "/hot/targets/proj".into();
+
+        write_cargo_config(&warm, &hot).unwrap();
+        write_cargo_config(&warm, &hot).unwrap();
+        write_cargo_config(&warm, &hot).unwrap();
+
+        let content = std::fs::read_to_string(warm.join(".cargo/config.toml")).unwrap();
+        assert_eq!(content.matches("claw-store managed").count(), 1);
+    }
+
+    #[test]
+    fn test_write_cargo_config_heals_existing_duplicate_marker() {
+        let dir = TempDir::new().unwrap();
+        let warm = dir.path().join("proj");
+        let cargo_dir = warm.join(".cargo");
+        std::fs::create_dir_all(&cargo_dir).unwrap();
+        std::fs::write(
+            cargo_dir.join("config.toml"),
+            "# claw-store managed — do not edit manually\n\
+             [build]\n\
+             target-dir = \"/hot/targets/proj\"\n\
+             # claw-store managed — do not edit manually\n\
+             [env]\n\
+             FOO = \"bar\"\n",
+        )
+        .unwrap();
+
+        let hot: std::path::PathBuf = "/hot/targets/proj".into();
+        write_cargo_config(&warm, &hot).unwrap();
+
+        let content = std::fs::read_to_string(cargo_dir.join("config.toml")).unwrap();
+        assert_eq!(content.matches("claw-store managed").count(), 1);
+        assert!(content.contains("FOO = \"bar\""));
+    }
+
     #[test]
     fn test_verify_cargo_config_detects_missing() {
         let dir = TempDir::new().unwrap();

claw-store/src/cluster.rs

@@ -26,6 +26,7 @@ pub mod refs;
 pub mod repo_ensure;
 pub mod rpc;
 pub mod services;
+pub mod shutdown_prep;
 pub mod snapshot;
 pub mod tags;
 pub mod tailscale;

claw-store/src/cluster/rpc.rs

@@ -217,6 +217,22 @@ pub enum Method {
     /// `payload`: JSON [`crate::cluster::repo_ensure::RepoReleaseRequest`].
     /// Reply: JSON [`crate::cluster::repo_ensure::RepoReleaseReply`].
     RepoRelease = 0x1f,
+    /// Runs `safe-shutdown-prep.sh --dry-run` to completion on this
+    /// node and returns the full report. Never stops anything —
+    /// dry-run only, safe to call repeatedly.
+    ///
+    /// `payload`: JSON [`crate::cluster::shutdown_prep::ShutdownPrepCheckRequest`].
+    /// Reply: JSON [`crate::cluster::shutdown_prep::ShutdownPrepCheckReply`].
+    ShutdownPrepCheck = 0x20,
+    /// Starts the real `safe-shutdown-prep.sh` run in a detached
+    /// systemd scope and returns immediately — the script's own step
+    /// stops `claw-store.service`, so this RPC connection cannot
+    /// outlive full completion. See
+    /// [`crate::cluster::shutdown_prep`] module docs.
+    ///
+    /// `payload`: JSON [`crate::cluster::shutdown_prep::ShutdownPrepExecuteRequest`].
+    /// Reply: JSON [`crate::cluster::shutdown_prep::ShutdownPrepExecuteReply`].
+    ShutdownPrepExecute = 0x21,
 }
 
 impl Method {
@@ -255,6 +271,8 @@ impl Method {
             0x1d => Some(Method::DashboardStorage),
             0x1e => Some(Method::RepoEnsure),
             0x1f => Some(Method::RepoRelease),
+            0x20 => Some(Method::ShutdownPrepCheck),
+            0x21 => Some(Method::ShutdownPrepExecute),
             _ => None,
         }
     }
@@ -1176,6 +1194,26 @@ impl RpcRouter {
                     .context("encoding RepoReleaseReply as JSON")?;
                 Ok(HandlerOutcome::Reply(json))
             }
+            Method::ShutdownPrepCheck => {
+                let reply = crate::cluster::shutdown_prep::check().await?;
+                let json = serde_json::to_vec(&reply)
+                    .context("encoding ShutdownPrepCheckReply as JSON")?;
+                Ok(HandlerOutcome::Reply(json))
+            }
+            Method::ShutdownPrepExecute => {
+                let req: crate::cluster::shutdown_prep::ShutdownPrepExecuteRequest =
+                    match serde_json::from_slice(payload) {
+                        Ok(r) => r,
+                        Err(_) => {
+                            return Ok(HandlerOutcome::Error(ErrorCode::InvalidRequest))
+                        }
+                    };
+                let reply =
+                    crate::cluster::shutdown_prep::execute(&self.local_name, &req).await?;
+                let json = serde_json::to_vec(&reply)
+                    .context("encoding ShutdownPrepExecuteReply as JSON")?;
+                Ok(HandlerOutcome::Reply(json))
+            }
             Method::BlobStat => {
                 let store = match &self.blob_store {
                     Some(s) => s,

claw-store/src/cluster/rpc/client.rs

@@ -229,6 +229,45 @@ pub async fn call_repo_release(
     serde_json::from_slice(&reply).context("decoding RepoReleaseReply JSON")
 }
 
+/// Convenience wrapper for [`Method::ShutdownPrepCheck`]. Runs
+/// `safe-shutdown-prep.sh --dry-run` on the connected peer and waits
+/// for the full report. Never stops anything on the peer.
+pub async fn call_shutdown_prep_check(
+    conn: &Connection,
+) -> Result<crate::cluster::shutdown_prep::ShutdownPrepCheckReply> {
+    let req = crate::cluster::shutdown_prep::ShutdownPrepCheckRequest {};
+    let payload = serde_json::to_vec(&req).context("encoding ShutdownPrepCheckRequest")?;
+    let reply = rpc_call(conn, Method::ShutdownPrepCheck, &payload).await?;
+    if reply.len() == 1 {
+        if let Some(code) = decode_error(reply[0]) {
+            bail!("peer replied with error: {}", code.describe());
+        }
+    }
+    serde_json::from_slice(&reply).context("decoding ShutdownPrepCheckReply JSON")
+}
+
+/// Convenience wrapper for [`Method::ShutdownPrepExecute`]. Starts the
+/// real shutdown-prep run on the connected peer (detached — this call
+/// returns as soon as the peer confirms it started, not when it
+/// finishes, since the peer's own daemon stops itself partway
+/// through).
+pub async fn call_shutdown_prep_execute(
+    conn: &Connection,
+    confirm_node_name: &str,
+) -> Result<crate::cluster::shutdown_prep::ShutdownPrepExecuteReply> {
+    let req = crate::cluster::shutdown_prep::ShutdownPrepExecuteRequest {
+        confirm_node_name: confirm_node_name.to_string(),
+    };
+    let payload = serde_json::to_vec(&req).context("encoding ShutdownPrepExecuteRequest")?;
+    let reply = rpc_call(conn, Method::ShutdownPrepExecute, &payload).await?;
+    if reply.len() == 1 {
+        if let Some(code) = decode_error(reply[0]) {
+            bail!("peer replied with error: {}", code.describe());
+        }
+    }
+    serde_json::from_slice(&reply).context("decoding ShutdownPrepExecuteReply JSON")
+}
+
 /// Recognise a single-byte reply as one of our error codes. Returns
 /// `None` for any other single-byte value (which is a valid reply,
 /// just an unusually short one).

claw-store/src/cluster/shutdown_prep.rs

@@ -0,0 +1,174 @@
+//! Peer-side wiring for `deploy/scripts/safe-shutdown-prep.sh`,
+//! surfaced through the RPC layer so the dashboard-v2 aggregator can
+//! offer a "prepare this node for shutdown" action.
+//!
+//! Split into two RPCs deliberately:
+//!
+//! - [`Method::ShutdownPrepCheck`] runs the script's `--dry-run` mode
+//!   and waits for it to finish. Dry-run never stops this node's own
+//!   daemon, so the RPC connection survives to deliver the full
+//!   report — this is the part a browser can meaningfully show.
+//! - [`Method::ShutdownPrepExecute`] runs the real script, which (by
+//!   design) stops `claw-store.service` — i.e. the very process
+//!   handling this RPC. There is no way to stream a live result past
+//!   that point, so this RPC detaches the script into its own
+//!   transient systemd scope (outside this daemon's service cgroup,
+//!   so `systemctl stop claw-store.service` doesn't take the script
+//!   down with it) and returns immediately. The full report lands in
+//!   `SHUTDOWN_PREP_LOG` for whoever is physically at the machine (or
+//!   over SSH) to read once the node has gone dark.
+//!
+//! [`Method::ShutdownPrepCheck`]: crate::cluster::rpc::Method::ShutdownPrepCheck
+//! [`Method::ShutdownPrepExecute`]: crate::cluster::rpc::Method::ShutdownPrepExecute
+
+use anyhow::{bail, Context, Result};
+use serde::{Deserialize, Serialize};
+use std::path::PathBuf;
+use std::time::Duration;
+use tokio::process::Command;
+use tokio::time::timeout;
+
+/// Dry-run does a real snapshot + replicate, which can legitimately
+/// take a while on a large delta. Generous but bounded so a stuck
+/// peer connection doesn't hang the RPC forever.
+const CHECK_TIMEOUT: Duration = Duration::from_secs(300);
+
+/// Where the real run's output lands once this node's daemon (and
+/// therefore this RPC connection) is gone.
+pub const SHUTDOWN_PREP_LOG: &str = "/var/lib/claw-store/shutdown-prep.log";
+
+#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
+pub struct ShutdownPrepCheckRequest {}
+
+#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
+pub struct ShutdownPrepCheckReply {
+    /// True iff the script exited 0 (every guard passed, "SAFE TO
+    /// POWER OFF" printed for the checked steps).
+    pub ready: bool,
+    /// Full combined stdout+stderr from `--dry-run`.
+    pub output: String,
+}
+
+#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
+pub struct ShutdownPrepExecuteRequest {
+    /// Defense in depth beyond RPC targeting: the caller must name
+    /// the exact node it thinks it's shutting down. Checked against
+    /// this node's own configured name before anything runs.
+    pub confirm_node_name: String,
+}
+
+#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
+pub struct ShutdownPrepExecuteReply {
+    pub started: bool,
+    pub message: String,
+    pub log_path: String,
+}
+
+fn script_path() -> PathBuf {
+    if let Ok(p) = std::env::var("CLAWSTOR_SHUTDOWN_SCRIPT") {
+        if !p.is_empty() {
+            return PathBuf::from(p);
+        }
+    }
+    if let Ok(home) = std::env::var("HOME") {
+        if !home.is_empty() {
+            return PathBuf::from(home)
+                .join("clawstor-deploy/scripts/safe-shutdown-prep.sh");
+        }
+    }
+    PathBuf::from("/usr/local/share/claw-store/safe-shutdown-prep.sh")
+}
+
+/// Run `safe-shutdown-prep.sh --dry-run` to completion and report the
+/// full output. Never stops anything on this node — safe to call any
+/// time, including repeatedly.
+pub async fn check() -> Result<ShutdownPrepCheckReply> {
+    let script = script_path();
+    if !script.exists() {
+        bail!("shutdown-prep script not found at {}", script.display());
+    }
+    // `2>&1` inside the shell merges stderr into stdout *before*
+    // either stream is piped back to us, preserving true
+    // chronological order. Capturing stdout/stderr separately (as
+    // `Command::output()` does by default) and concatenating them
+    // after the fact loses interleaving entirely — every stderr line
+    // lands at the very end regardless of when it was actually
+    // printed, which makes a mid-script warning (e.g. "replicate not
+    // configured on this node") look like a failure that happened
+    // after "DRY RUN COMPLETE".
+    let run = Command::new("bash")
+        .arg("-c")
+        .arg(format!("{} --dry-run 2>&1", script.display()))
+        .output();
+    let output = timeout(CHECK_TIMEOUT, run)
+        .await
+        .context("shutdown-prep --dry-run timed out")?
+        .context("spawning shutdown-prep --dry-run")?;
+    let combined = String::from_utf8_lossy(&output.stdout).into_owned();
+    Ok(ShutdownPrepCheckReply {
+        ready: output.status.success(),
+        output: combined,
+    })
+}
+
+/// Kick off the real (non-dry-run) script in a transient systemd
+/// scope detached from this daemon's own service cgroup, then return
+/// immediately without waiting for it. The script's own step 6 stops
+/// `claw-store.service` — waiting for it to exit here would mean
+/// waiting for our own process to be killed.
+pub async fn execute(local_node_name: &str, req: &ShutdownPrepExecuteRequest) -> Result<ShutdownPrepExecuteReply> {
+    if req.confirm_node_name != local_node_name {
+        bail!(
+            "confirm_node_name '{}' does not match this node ('{}') — refusing",
+            req.confirm_node_name,
+            local_node_name
+        );
+    }
+    let script = script_path();
+    if !script.exists() {
+        bail!("shutdown-prep script not found at {}", script.display());
+    }
+    let unit = format!(
+        "clawstor-shutdown-prep-{}",
+        std::time::SystemTime::now()
+            .duration_since(std::time::UNIX_EPOCH)
+            .map(|d| d.as_secs())
+            .unwrap_or(0)
+    );
+    // `--user --scope` places this under the user session's cgroup
+    // tree (/user.slice/...), a sibling of — not a descendant of —
+    // /system.slice/claw-store.service. `systemctl stop
+    // claw-store.service` only tears down its own cgroup, so this
+    // keeps running (and completes step 6, which stops that very
+    // service) unaffected.
+    // Deliberately no --force: the real run re-checks active builds
+    // and the sync queue itself, even though `check()` may have run
+    // moments ago — state can change between the two clicks, and
+    // re-validating is cheap.
+    let cmd = format!(
+        "{} >> {} 2>&1",
+        script.display(),
+        SHUTDOWN_PREP_LOG
+    );
+    let spawn = Command::new("systemd-run")
+        .arg("--user")
+        .arg("--scope")
+        .arg("--collect")
+        .arg(format!("--unit={unit}"))
+        .arg("bash")
+        .arg("-c")
+        .arg(&cmd)
+        .spawn();
+    match spawn {
+        Ok(_child) => Ok(ShutdownPrepExecuteReply {
+            started: true,
+            message: format!(
+                "shutdown-prep started on {local_node_name} as transient unit {unit}. \
+                 This node's daemon (and dashboard) will go offline as part of the \
+                 process — that is expected. Full output: {SHUTDOWN_PREP_LOG}."
+            ),
+            log_path: SHUTDOWN_PREP_LOG.to_string(),
+        }),
+        Err(e) => Err(e).context("spawning systemd-run for shutdown-prep"),
+    }
+}

claw-store/src/config.rs

@@ -27,12 +27,26 @@ pub struct HotConfig {
 #[derive(Debug, Clone, Deserialize, Serialize)]
 pub struct WarmConfig {
     pub projects_path: PathBuf,
+    /// Dataset name for `zfs`/`zpool` operations against the warm
+    /// tier. The literal value `"none"` means this node's warm tier
+    /// is a plain directory, not ZFS-backed (e.g. a build node with
+    /// no ZFS pool) — snapshot/replicate become no-ops instead of
+    /// erroring on a missing `zfs`/`zpool` binary. See
+    /// [`WarmConfig::zfs_enabled`].
     pub zfs_dataset: String,
     pub snapshot_retain_hours: u64,
     pub snapshot_retain_days: u64,
     pub snapshot_retain_weeks: u64,
 }
 
+impl WarmConfig {
+    /// `false` when `zfs_dataset = "none"` — this node's warm tier
+    /// has no ZFS pool underneath it.
+    pub fn zfs_enabled(&self) -> bool {
+        self.zfs_dataset != "none"
+    }
+}
+
 #[derive(Debug, Clone, Deserialize, Serialize)]
 pub struct ColdConfig {
     pub archive_path: PathBuf,

claw-store/src/daemon.rs

@@ -188,19 +188,25 @@ pub async fn run(cfg: Config, mut manifest: Manifest) -> Result<()> {
                 }
             }
             _ = snap_tick.tick() => {
-                let ts = chrono::Utc::now().format("%Y-%m-%d-%H%M").to_string();
+                if !cfg.warm.zfs_enabled() {
-                tracing::info!("taking snapshot {}", ts);
+                    tracing::debug!("skipping snapshot tick — zfs_dataset = \"none\" on this node");
-                if let Err(e) = snapshot::run_snapshot_cycle(
+                } else {
-                    &zfs, &cfg.warm.zfs_dataset, &ts,
+                    let ts = chrono::Utc::now().format("%Y-%m-%d-%H%M").to_string();
-                    cfg.warm.snapshot_retain_hours as usize,
+                    tracing::info!("taking snapshot {}", ts);
-                    cfg.warm.snapshot_retain_days as usize,
+                    if let Err(e) = snapshot::run_snapshot_cycle(
-                    cfg.warm.snapshot_retain_weeks as usize,
+                        &zfs, &cfg.warm.zfs_dataset, &ts,
-                ) {
+                        cfg.warm.snapshot_retain_hours as usize,
-                    tracing::error!("snapshot failed: {:#}", e);
+                        cfg.warm.snapshot_retain_days as usize,
+                        cfg.warm.snapshot_retain_weeks as usize,
+                    ) {
+                        tracing::error!("snapshot failed: {:#}", e);
+                    }
                 }
             }
             _ = repl_tick.tick() => {
-                if let Some(rep) = &cfg.replication {
+                if !cfg.warm.zfs_enabled() {
+                    tracing::debug!("skipping replication tick — zfs_dataset = \"none\" on this node");
+                } else if let Some(rep) = &cfg.replication {
                     if let (Some(host), Some(user), Some(dest)) = (
                         &rep.send_to_host, &rep.send_to_user, &rep.cold_dataset_on_peer
                     ) {

claw-store/src/main.rs

@@ -1502,6 +1502,10 @@ fn cmd_gc(cfg: &Config, manifest: &mut Manifest) -> Result<()> {
 }
 
 fn cmd_snapshot(cfg: &Config, zfs: &SystemZfs) -> Result<()> {
+    if !cfg.warm.zfs_enabled() {
+        println!("Warm tier is not ZFS-backed on this node (zfs_dataset = \"none\") — nothing to snapshot.");
+        return Ok(());
+    }
     let ts = chrono::Utc::now().format("%Y-%m-%d-%H%M").to_string();
     snapshot::run_snapshot_cycle(
         zfs, &cfg.warm.zfs_dataset, &ts,
@@ -1514,6 +1518,10 @@ fn cmd_snapshot(cfg: &Config, zfs: &SystemZfs) -> Result<()> {
 }
 
 fn cmd_list_snapshots(cfg: &Config, zfs: &SystemZfs, _project: &str) -> Result<()> {
+    if !cfg.warm.zfs_enabled() {
+        println!("Warm tier is not ZFS-backed on this node (zfs_dataset = \"none\").");
+        return Ok(());
+    }
     let snaps = zfs.list_snapshots(&cfg.warm.zfs_dataset)?;
     if snaps.is_empty() { println!("No snapshots found."); return Ok(()); }
     for s in &snaps { println!("  {}", s); }
@@ -1531,6 +1539,10 @@ fn cmd_restore(cfg: &Config, zfs: &SystemZfs, project: &str, snap: &str) -> Resu
 }
 
 fn cmd_replicate(cfg: &Config, zfs: &SystemZfs) -> Result<()> {
+    if !cfg.warm.zfs_enabled() {
+        println!("Warm tier is not ZFS-backed on this node (zfs_dataset = \"none\") — nothing to replicate.");
+        return Ok(());
+    }
     let rep = cfg.replication.as_ref()
         .context("no replication config — this node does not replicate")?;
     let host = rep.send_to_host.as_ref().context("send_to_host not set")?;

claw-store/src/serve_v2.rs

@@ -570,6 +570,19 @@ impl AuthedCaller {
     /// in the request body. Explicit user-supplied workspace is only
     /// honored for admin/open — a namespaced caller supplying a
     /// mismatched workspace is a forbidden write.
+    /// Gate for fleet-infrastructure actions (shutdown-prep) that
+    /// have nothing to do with a tag/repo namespace — a namespaced
+    /// per-app token has no business stopping a node's services.
+    pub fn require_admin(&self) -> Result<(), (StatusCode, String)> {
+        match self {
+            AuthedCaller::Admin | AuthedCaller::Open => Ok(()),
+            AuthedCaller::Namespaced { .. } => Err((
+                StatusCode::FORBIDDEN,
+                "this action requires an admin token".to_string(),
+            )),
+        }
+    }
+
     pub fn resolve_workspace(&self, requested: Option<&str>) -> Result<String, (StatusCode, String)> {
         match self {
             AuthedCaller::Admin | AuthedCaller::Open => match requested {
@@ -866,6 +879,93 @@ async fn handle_repos_release(
     }))
 }
 
+// ── shutdown-prep (targets one specific node, not a fan-out) ──────
+
+#[derive(Serialize)]
+pub struct ShutdownPrepCheckResponse {
+    pub node: String,
+    pub ready: bool,
+    pub output: String,
+}
+
+async fn handle_shutdown_prep_check(
+    State(s): State<Arc<V2State>>,
+    Path(name): Path<String>,
+    axum::extract::Extension(caller): axum::extract::Extension<AuthedCaller>,
+) -> Result<Json<ShutdownPrepCheckResponse>, (StatusCode, String)> {
+    caller.require_admin()?;
+    let peer = s
+        .peers
+        .iter()
+        .find(|p| p.name == name)
+        .cloned()
+        .ok_or((StatusCode::NOT_FOUND, format!("no peer named {name}")))?;
+    let conn = s
+        .dial(&peer)
+        .await
+        .map_err(|e| (StatusCode::BAD_GATEWAY, format!("connecting to {name}: {e:#}")))?;
+    let reply = crate::cluster::rpc::call_shutdown_prep_check(&conn).await;
+    conn.close(quinn::VarInt::from_u32(0), b"done");
+    let reply = reply.map_err(|e| (StatusCode::BAD_GATEWAY, format!("{e:#}")))?;
+    Ok(Json(ShutdownPrepCheckResponse {
+        node: name,
+        ready: reply.ready,
+        output: reply.output,
+    }))
+}
+
+#[derive(Deserialize)]
+pub struct ShutdownPrepExecuteBody {
+    /// Must equal the target node's own name — a second, server-side
+    /// confirmation beyond "the operator clicked the right button in
+    /// the UI". Checked again on the peer itself in
+    /// `shutdown_prep::execute`.
+    pub confirm_node_name: String,
+}
+
+#[derive(Serialize)]
+pub struct ShutdownPrepExecuteResponse {
+    pub node: String,
+    pub started: bool,
+    pub message: String,
+}
+
+async fn handle_shutdown_prep_execute(
+    State(s): State<Arc<V2State>>,
+    Path(name): Path<String>,
+    axum::extract::Extension(caller): axum::extract::Extension<AuthedCaller>,
+    Json(body): Json<ShutdownPrepExecuteBody>,
+) -> Result<Json<ShutdownPrepExecuteResponse>, (StatusCode, String)> {
+    caller.require_admin()?;
+    if body.confirm_node_name != name {
+        return Err((
+            StatusCode::BAD_REQUEST,
+            format!(
+                "confirm_node_name '{}' does not match target node '{name}'",
+                body.confirm_node_name
+            ),
+        ));
+    }
+    let peer = s
+        .peers
+        .iter()
+        .find(|p| p.name == name)
+        .cloned()
+        .ok_or((StatusCode::NOT_FOUND, format!("no peer named {name}")))?;
+    let conn = s
+        .dial(&peer)
+        .await
+        .map_err(|e| (StatusCode::BAD_GATEWAY, format!("connecting to {name}: {e:#}")))?;
+    let reply = crate::cluster::rpc::call_shutdown_prep_execute(&conn, &name).await;
+    conn.close(quinn::VarInt::from_u32(0), b"done");
+    let reply = reply.map_err(|e| (StatusCode::BAD_GATEWAY, format!("{e:#}")))?;
+    Ok(Json(ShutdownPrepExecuteResponse {
+        node: name,
+        started: reply.started,
+        message: reply.message,
+    }))
+}
+
 async fn fanout_repo_ensure(
     s: &V2State,
     req: &crate::cluster::repo_ensure::RepoEnsureRequest,
@@ -1347,6 +1447,14 @@ pub fn build(state: Arc<V2State>) -> Router {
         .route("/api/v2/sessions/:id/commit", post(handle_commit_session))
         .route("/api/v2/repos/ensure", post(handle_repos_ensure))
         .route("/api/v2/repos/release", post(handle_repos_release))
+        .route(
+            "/api/v2/node/:name/shutdown-prep/check",
+            post(handle_shutdown_prep_check),
+        )
+        .route(
+            "/api/v2/node/:name/shutdown-prep/execute",
+            post(handle_shutdown_prep_execute),
+        )
         .route_layer(axum::middleware::from_fn_with_state(state.clone(), v2_auth))
         .with_state(state)
 }

config/architect.toml

@@ -19,6 +19,33 @@ archive_path = "/data/archive"
 zfs_dataset = "data/archive"
 retain_weeks = 12
 
+[cluster]
+zone = "fabric-10g"
+bind_lan = "10.0.0.13:7701"
+prom_bind = "0.0.0.0:7703"
+bind_rpc_lan = "10.0.0.13:7702"
+bind_rpc_tailscale = "100.104.171.32:7702"
+blob_store_root = "/home/osobh/clawstor-deploy/data"
+
+[[cluster.peers]]
+name = "tank"
+zone = "fabric-10g"
+lan_addr = "10.0.0.14:7701"
+rpc_lan_addr = "10.10.0.10:7702"
+tailscale_addr = "100.108.129.81:7702"
+
+[[cluster.peers]]
+name = "morpheus"
+zone = "lan-1g"
+lan_addr = "10.0.0.5:7701"
+rpc_lan_addr = "10.0.0.5:7702"
+tailscale_addr = "100.123.224.84:7702"
+
+[cluster.tls]
+ca_cert = "/home/osobh/clawstor-deploy/tls/ca.crt"
+node_cert = "/home/osobh/clawstor-deploy/tls/node.crt"
+node_key = "/home/osobh/clawstor-deploy/tls/node.key"
+
 [replication]
 receive_from_peer = true
 peer_user = "osobh"

config/morpheus.toml

@@ -0,0 +1,43 @@
+[node]
+name = "morpheus"
+role = "secondary"
+
+[hot]
+path = "/hot/targets"
+max_gb = 80
+stale_hours = 48
+
+[warm]
+projects_path = "/slab/projects"
+zfs_dataset = "none"
+snapshot_retain_hours = 24
+snapshot_retain_days = 7
+snapshot_retain_weeks = 4
+
+[cluster]
+zone = "lan-1g"
+# Morpheus has no direct 10G to Architect/Tank — use main LAN for all traffic
+bind_lan = "10.0.0.5:7701"
+prom_bind = "0.0.0.0:7703"
+bind_rpc_lan = "10.0.0.5:7702"
+bind_rpc_tailscale = "100.123.224.84:7702"
+blob_store_root = "/home/osobh/clawstor-deploy/data"
+
+[[cluster.peers]]
+name = "architect"
+zone = "fabric-10g"
+lan_addr = "10.0.0.13:7701"
+rpc_lan_addr = "10.0.0.13:7702"
+tailscale_addr = "100.104.171.32:7702"
+
+[[cluster.peers]]
+name = "tank"
+zone = "fabric-10g"
+lan_addr = "10.0.0.14:7701"
+rpc_lan_addr = "10.0.0.14:7702"
+tailscale_addr = "100.108.129.81:7702"
+
+[cluster.tls]
+ca_cert = "/home/osobh/clawstor-deploy/tls/ca.crt"
+node_cert = "/home/osobh/clawstor-deploy/tls/node.crt"
+node_key = "/home/osobh/clawstor-deploy/tls/node.key"

config/tank.toml

@@ -14,13 +14,35 @@ snapshot_retain_hours = 24
 snapshot_retain_days = 7
 snapshot_retain_weeks = 4
 
+[cluster]
+zone = "fabric-10g"
+bind_lan = "10.0.0.14:7701"
+prom_bind = "0.0.0.0:7703"
+bind_rpc_lan = "10.0.0.14:7702"
+bind_rpc_tailscale = "100.108.129.81:7702"
+blob_store_root = "/home/osobh/clawstor-deploy/data"
+
+[[cluster.peers]]
+name = "architect"
+zone = "fabric-10g"
+lan_addr = "10.0.0.13:7701"
+rpc_lan_addr = "10.10.0.9:7702"
+tailscale_addr = "100.104.171.32:7702"
+
+[[cluster.peers]]
+name = "morpheus"
+zone = "lan-1g"
+lan_addr = "10.0.0.5:7701"
+rpc_lan_addr = "10.0.0.5:7702"
+tailscale_addr = "100.123.224.84:7702"
+
+[cluster.tls]
+ca_cert = "/home/osobh/clawstor-deploy/tls/ca.crt"
+node_cert = "/home/osobh/clawstor-deploy/tls/node.crt"
+node_key = "/home/osobh/clawstor-deploy/tls/node.key"
+
 [replication]
-# 10.10.0.9 is architect-fab-tank — the dedicated 10G fabric link between
-# the two nodes. Intentionally used for replication to maximise bandwidth;
-# architect's primary LAN address is 10.0.0.13.
 send_to_host = "10.10.0.9"
 send_to_user = "osobh"
 cold_dataset_on_peer = "data/archive/tank-projects"
-# nightly_at is reserved for future use; replication schedule is currently
-# controlled by the claw-store-replicate.timer systemd unit.
 nightly_at = "03:30"

dashboard-v2/src/components/NodeCard.js

@@ -24,7 +24,7 @@ export function NodeCard({ node }) {
                 border,
             ].join(' '), children: [_jsxs("div", { className: "flex items-center justify-between", children: [_jsxs("div", { className: "flex items-center gap-2", children: [_jsx("span", { className: `inline-block w-2.5 h-2.5 rounded-full ${dot}` }), _jsx("span", { className: "text-lg font-semibold text-slate-100", children: node.node_name })] }), _jsx("span", { className: "text-xs text-slate-500 font-mono", children: node.zone || '—' })] }), !node.online && (_jsx("div", { className: "text-sm text-red-400 break-words", children: node.error ?? 'offline' })), node.online && (_jsxs(_Fragment, { children: [node.filesystem && (_jsx(StorageBar, { label: "disk", used: node.filesystem.used_bytes, total: node.filesystem.total_bytes })), node.hot && node.hot.max_bytes > 0 && (_jsx(StorageBar, { label: "hot tier", used: node.hot.used_bytes, total: node.hot.max_bytes, pinned: node.hot.pinned_bytes ?? undefined })), _jsxs("div", { className: "grid grid-cols-2 gap-y-1 text-sm", children: [_jsx("span", { className: "text-slate-500", children: "mount" }), _jsx("span", { className: "text-right font-mono text-xs", children: node.mount?.active ? (_jsx("span", { className: "text-emerald-300", children: "\u2713 mounted" })) : (_jsx("span", { className: "text-slate-500", children: "not mounted" })) }), _jsx("span", { className: "text-slate-500", children: "cache hit rate" }), _jsx("span", { className: "text-right font-mono text-xs", children: node.cache && node.cache.hits + node.cache.misses > 0
                                         ? `${Math.round(node.cache.hit_rate * 100)}%`
-                                        : _jsx("span", { className: "text-slate-500", children: "idle" }) }), _jsx("span", { className: "text-slate-500", children: "next scheduled job" }), _jsx("span", { className: "text-right font-mono text-xs", children: nextTimer(node) })] })] }))] }) }));
+                                        : _jsx("span", { className: "text-slate-500", children: "idle" }) }), _jsx("span", { className: "text-slate-500", children: "next scheduled job" }), _jsx("span", { className: "text-right font-mono text-xs", children: nextTimer(node) })] })] })), _jsx("div", { className: "pt-1 border-t border-slate-800 text-xs text-slate-500", children: "view detail \u00B7 maintenance & shutdown prep \u2192" })] }) }));
 }
 function healthOf(n) {
     if (!n.online)

dashboard-v2/src/components/NodeCard.tsx

@@ -95,6 +95,10 @@ export function NodeCard({ node }: Props) {
             </div>
           </>
         )}
+
+        <div className="pt-1 border-t border-slate-800 text-xs text-slate-500">
+          view detail · maintenance & shutdown prep →
+        </div>
       </a>
     </Link>
   );

dashboard-v2/src/components/ShutdownPrepPanel.js

@@ -0,0 +1,44 @@
+import { jsx as _jsx, jsxs as _jsxs, Fragment as _Fragment } from "react/jsx-runtime";
+import { useState } from 'react';
+import { api } from '../lib/api';
+/**
+ * Node-maintenance panel: runs `safe-shutdown-prep.sh --dry-run` on
+ * demand (safe, read-mostly, never stops anything) and — only once
+ * that comes back ready — unlocks a type-to-confirm button that
+ * starts the real run.
+ *
+ * The real run is fire-and-forget by necessity: its own steps stop
+ * this node's daemon, which is what's serving this very page, so
+ * there is no way to stream a live result past that point. Once
+ * started, the UI says so plainly and points at the on-disk log for
+ * the full report.
+ */
+export function ShutdownPrepPanel({ name }) {
+    const [check, setCheck] = useState({ phase: 'idle' });
+    const [exec, setExec] = useState({ phase: 'idle' });
+    const [confirmText, setConfirmText] = useState('');
+    const runCheck = () => {
+        setCheck({ phase: 'checking' });
+        setExec({ phase: 'idle' });
+        setConfirmText('');
+        api
+            .shutdownPrepCheck(name)
+            .then((r) => setCheck({ phase: 'done', ready: r.ready, output: r.output }))
+            .catch((e) => setCheck({ phase: 'error', message: String(e) }));
+    };
+    const runExecute = () => {
+        if (confirmText !== name)
+            return;
+        setExec({ phase: 'starting' });
+        api
+            .shutdownPrepExecute(name, confirmText)
+            .then((r) => setExec({ phase: 'started', message: r.message }))
+            .catch((e) => setExec({ phase: 'error', message: String(e) }));
+    };
+    const ready = check.phase === 'done' && check.ready;
+    return (_jsxs("div", { className: "rounded border border-slate-800 bg-slate-900/40 p-4 text-sm space-y-3", children: [_jsxs("div", { className: "flex items-center justify-between", children: [_jsx("div", { className: "text-slate-500 uppercase text-xs tracking-wider", children: "shutdown prep" }), _jsx("button", { onClick: runCheck, disabled: check.phase === 'checking', className: "px-3 py-1.5 rounded bg-slate-800 hover:bg-slate-700 text-slate-200 text-xs disabled:opacity-50", children: check.phase === 'checking' ? 'checking…' : 'check readiness for shutdown' })] }), _jsxs("p", { className: "text-xs text-slate-500", children: ["Runs a dry-run of the pre-shutdown checklist on ", _jsx("span", { className: "font-mono", children: name }), ' ', "\u2014 active builds, pending peer sync, a final snapshot + replicate to cold. Nothing is stopped or unmounted by this check."] }), check.phase === 'error' && (_jsx("div", { className: "rounded border border-red-800 bg-red-950/40 p-3 text-red-300 text-xs", children: check.message })), check.phase === 'done' && (_jsxs(_Fragment, { children: [_jsx("div", { className: `rounded border p-3 text-xs ${check.ready
+                            ? 'border-emerald-800 bg-emerald-950/40 text-emerald-300'
+                            : 'border-amber-800 bg-amber-950/40 text-amber-300'}`, children: check.ready
+                            ? '✓ ready — safe to start the real shutdown-prep run'
+                            : '! not ready — see output below (active build or un-synced changes are the usual cause)' }), _jsx("pre", { className: "max-h-72 overflow-auto rounded bg-slate-950 border border-slate-800 p-3 text-[11px] leading-relaxed text-slate-300 whitespace-pre-wrap", children: check.output })] })), ready && exec.phase !== 'started' && (_jsxs("div", { className: "rounded border border-red-900 bg-red-950/30 p-3 space-y-2", children: [_jsxs("div", { className: "text-red-300 text-xs", children: ["This starts the real run: stops maintenance timers, the dashboard, the storage daemon (gossip announces departure to peers), and unmounts FUSE on", ' ', _jsx("span", { className: "font-mono", children: name }), ". The node's dashboard connection will drop partway through \u2014 that's expected, not an error. It does ", _jsx("strong", { children: "not" }), " power the machine off; do that yourself once it's gone dark."] }), _jsxs("div", { className: "flex items-center gap-2", children: [_jsx("input", { value: confirmText, onChange: (e) => setConfirmText(e.target.value), placeholder: `type "${name}" to confirm`, className: "flex-1 rounded bg-slate-900 border border-slate-700 px-2 py-1.5 text-xs font-mono text-slate-200 placeholder:text-slate-600" }), _jsx("button", { onClick: runExecute, disabled: confirmText !== name || exec.phase === 'starting', className: "px-3 py-1.5 rounded bg-red-900 hover:bg-red-800 text-red-100 text-xs disabled:opacity-40 disabled:cursor-not-allowed whitespace-nowrap", children: exec.phase === 'starting' ? 'starting…' : `stop services on ${name}` })] })] })), exec.phase === 'error' && (_jsx("div", { className: "rounded border border-red-800 bg-red-950/40 p-3 text-red-300 text-xs", children: exec.message })), exec.phase === 'started' && (_jsx("div", { className: "rounded border border-sky-800 bg-sky-950/30 p-3 text-sky-300 text-xs", children: exec.message }))] }));
+}

dashboard-v2/src/components/ShutdownPrepPanel.tsx

@@ -0,0 +1,144 @@
+import { useState } from 'react';
+import { api } from '../lib/api';
+
+interface Props {
+  name: string;
+}
+
+type CheckState =
+  | { phase: 'idle' }
+  | { phase: 'checking' }
+  | { phase: 'done'; ready: boolean; output: string }
+  | { phase: 'error'; message: string };
+
+type ExecState =
+  | { phase: 'idle' }
+  | { phase: 'starting' }
+  | { phase: 'started'; message: string }
+  | { phase: 'error'; message: string };
+
+/**
+ * Node-maintenance panel: runs `safe-shutdown-prep.sh --dry-run` on
+ * demand (safe, read-mostly, never stops anything) and — only once
+ * that comes back ready — unlocks a type-to-confirm button that
+ * starts the real run.
+ *
+ * The real run is fire-and-forget by necessity: its own steps stop
+ * this node's daemon, which is what's serving this very page, so
+ * there is no way to stream a live result past that point. Once
+ * started, the UI says so plainly and points at the on-disk log for
+ * the full report.
+ */
+export function ShutdownPrepPanel({ name }: Props) {
+  const [check, setCheck] = useState<CheckState>({ phase: 'idle' });
+  const [exec, setExec] = useState<ExecState>({ phase: 'idle' });
+  const [confirmText, setConfirmText] = useState('');
+
+  const runCheck = () => {
+    setCheck({ phase: 'checking' });
+    setExec({ phase: 'idle' });
+    setConfirmText('');
+    api
+      .shutdownPrepCheck(name)
+      .then((r) => setCheck({ phase: 'done', ready: r.ready, output: r.output }))
+      .catch((e) => setCheck({ phase: 'error', message: String(e) }));
+  };
+
+  const runExecute = () => {
+    if (confirmText !== name) return;
+    setExec({ phase: 'starting' });
+    api
+      .shutdownPrepExecute(name, confirmText)
+      .then((r) => setExec({ phase: 'started', message: r.message }))
+      .catch((e) => setExec({ phase: 'error', message: String(e) }));
+  };
+
+  const ready = check.phase === 'done' && check.ready;
+
+  return (
+    <div className="rounded border border-slate-800 bg-slate-900/40 p-4 text-sm space-y-3">
+      <div className="flex items-center justify-between">
+        <div className="text-slate-500 uppercase text-xs tracking-wider">
+          shutdown prep
+        </div>
+        <button
+          onClick={runCheck}
+          disabled={check.phase === 'checking'}
+          className="px-3 py-1.5 rounded bg-slate-800 hover:bg-slate-700 text-slate-200 text-xs disabled:opacity-50"
+        >
+          {check.phase === 'checking' ? 'checking…' : 'check readiness for shutdown'}
+        </button>
+      </div>
+
+      <p className="text-xs text-slate-500">
+        Runs a dry-run of the pre-shutdown checklist on <span className="font-mono">{name}</span>{' '}
+        — active builds, pending peer sync, a final snapshot + replicate to cold. Nothing is
+        stopped or unmounted by this check.
+      </p>
+
+      {check.phase === 'error' && (
+        <div className="rounded border border-red-800 bg-red-950/40 p-3 text-red-300 text-xs">
+          {check.message}
+        </div>
+      )}
+
+      {check.phase === 'done' && (
+        <>
+          <div
+            className={`rounded border p-3 text-xs ${
+              check.ready
+                ? 'border-emerald-800 bg-emerald-950/40 text-emerald-300'
+                : 'border-amber-800 bg-amber-950/40 text-amber-300'
+            }`}
+          >
+            {check.ready
+              ? '✓ ready — safe to start the real shutdown-prep run'
+              : '! not ready — see output below (active build or un-synced changes are the usual cause)'}
+          </div>
+          <pre className="max-h-72 overflow-auto rounded bg-slate-950 border border-slate-800 p-3 text-[11px] leading-relaxed text-slate-300 whitespace-pre-wrap">
+            {check.output}
+          </pre>
+        </>
+      )}
+
+      {ready && exec.phase !== 'started' && (
+        <div className="rounded border border-red-900 bg-red-950/30 p-3 space-y-2">
+          <div className="text-red-300 text-xs">
+            This starts the real run: stops maintenance timers, the dashboard, the storage
+            daemon (gossip announces departure to peers), and unmounts FUSE on{' '}
+            <span className="font-mono">{name}</span>. The node's dashboard connection will drop
+            partway through — that's expected, not an error. It does <strong>not</strong> power
+            the machine off; do that yourself once it's gone dark.
+          </div>
+          <div className="flex items-center gap-2">
+            <input
+              value={confirmText}
+              onChange={(e) => setConfirmText(e.target.value)}
+              placeholder={`type "${name}" to confirm`}
+              className="flex-1 rounded bg-slate-900 border border-slate-700 px-2 py-1.5 text-xs font-mono text-slate-200 placeholder:text-slate-600"
+            />
+            <button
+              onClick={runExecute}
+              disabled={confirmText !== name || exec.phase === 'starting'}
+              className="px-3 py-1.5 rounded bg-red-900 hover:bg-red-800 text-red-100 text-xs disabled:opacity-40 disabled:cursor-not-allowed whitespace-nowrap"
+            >
+              {exec.phase === 'starting' ? 'starting…' : `stop services on ${name}`}
+            </button>
+          </div>
+        </div>
+      )}
+
+      {exec.phase === 'error' && (
+        <div className="rounded border border-red-800 bg-red-950/40 p-3 text-red-300 text-xs">
+          {exec.message}
+        </div>
+      )}
+
+      {exec.phase === 'started' && (
+        <div className="rounded border border-sky-800 bg-sky-950/30 p-3 text-sky-300 text-xs">
+          {exec.message}
+        </div>
+      )}
+    </div>
+  );
+}

dashboard-v2/src/lib/api.js

@@ -23,6 +23,19 @@ async function get(path) {
     }
     return resp.json();
 }
+async function post(path, body) {
+    const full = `${API_BASE}${path}`;
+    const resp = await fetch(full, {
+        method: 'POST',
+        headers: { Accept: 'application/json', 'Content-Type': 'application/json' },
+        body: body === undefined ? undefined : JSON.stringify(body),
+    });
+    if (!resp.ok) {
+        const text = await resp.text().catch(() => '');
+        throw new Error(`${full} → ${resp.status} ${resp.statusText}${text ? `: ${text}` : ''}`);
+    }
+    return resp.json();
+}
 export const api = {
     fleet: () => get('/v2/fleet'),
     projects: () => get('/v2/projects'),
@@ -32,6 +45,10 @@ export const api = {
     refs: (limit = 200, offset = 0) => get(`/v2/storage/refs?limit=${limit}&offset=${offset}`),
     snapshots: () => get('/v2/storage/snapshots'),
     refTracking: (repo = '') => get(`/v2/storage/ref-tracking?repo=${encodeURIComponent(repo)}`),
+    shutdownPrepCheck: (name) => post(`/v2/node/${name}/shutdown-prep/check`),
+    shutdownPrepExecute: (name, confirmNodeName) => post(`/v2/node/${name}/shutdown-prep/execute`, {
+        confirm_node_name: confirmNodeName,
+    }),
 };
 /** Format bytes as MB / GB / TB as needed. */
 export function fmtBytes(n) {

dashboard-v2/src/lib/api.ts

@@ -119,6 +119,20 @@ async function get<T>(path: string): Promise<T> {
   return resp.json();
 }
 
+async function post<T>(path: string, body?: unknown): Promise<T> {
+  const full = `${API_BASE}${path}`;
+  const resp = await fetch(full, {
+    method: 'POST',
+    headers: { Accept: 'application/json', 'Content-Type': 'application/json' },
+    body: body === undefined ? undefined : JSON.stringify(body),
+  });
+  if (!resp.ok) {
+    const text = await resp.text().catch(() => '');
+    throw new Error(`${full} → ${resp.status} ${resp.statusText}${text ? `: ${text}` : ''}`);
+  }
+  return resp.json();
+}
+
 // All paths are relative to API_BASE. E.g. '/v2/fleet' becomes
 // '/api/v2/fleet' locally or '/clawstor/api/v2/fleet' via Tailscale.
 export interface ProjectRow {
@@ -145,8 +159,26 @@ export const api = {
   snapshots: () => get<SnapshotSummary[]>('/v2/storage/snapshots'),
   refTracking: (repo = '') =>
     get<RefTrackingItem[]>(`/v2/storage/ref-tracking?repo=${encodeURIComponent(repo)}`),
+  shutdownPrepCheck: (name: string) =>
+    post<ShutdownPrepCheckResponse>(`/v2/node/${name}/shutdown-prep/check`),
+  shutdownPrepExecute: (name: string, confirmNodeName: string) =>
+    post<ShutdownPrepExecuteResponse>(`/v2/node/${name}/shutdown-prep/execute`, {
+      confirm_node_name: confirmNodeName,
+    }),
 };
 
+export interface ShutdownPrepCheckResponse {
+  node: string;
+  ready: boolean;
+  output: string;
+}
+
+export interface ShutdownPrepExecuteResponse {
+  node: string;
+  started: boolean;
+  message: string;
+}
+
 /** Format bytes as MB / GB / TB as needed. */
 export function fmtBytes(n: number): string {
   if (n < 1024) return `${n} B`;

dashboard-v2/src/pages/NodeDetail.js

@@ -3,6 +3,7 @@ import { useEffect, useState } from 'react';
 import { Link } from 'wouter';
 import { api, fmtBytes } from '../lib/api';
 import { StatTile } from '../components/StatTile';
+import { ShutdownPrepPanel } from '../components/ShutdownPrepPanel';
 export function NodeDetail({ name }) {
     const [status, setStatus] = useState(null);
     const [err, setErr] = useState(null);
@@ -15,5 +16,5 @@ export function NodeDetail({ name }) {
         })
             .catch((e) => setErr(String(e)));
     }, [name]);
-    return (_jsxs("div", { className: "space-y-6", children: [_jsxs("div", { children: [_jsx(Link, { href: "/", children: _jsx("a", { className: "text-sm text-slate-500 hover:text-slate-300", children: "\u2190 fleet" }) }), _jsx("h1", { className: "text-2xl font-semibold text-slate-100 mt-2", children: name })] }), err && (_jsxs("div", { className: "rounded border border-amber-800 bg-amber-950/40 p-3 text-amber-300 text-sm", children: [err, _jsxs("div", { className: "mt-2 text-xs text-slate-400", children: ["Cross-node lookup lands in a follow-on PR. Until then this page shows detail only when you're already viewing the dashboard hosted by ", name, ". Try opening", ' ', _jsxs("span", { className: "font-mono", children: ["http://", name, ":7700/v2/#/nodes/", name] }), ' ', "directly."] })] })), status && (_jsxs(_Fragment, { children: [_jsxs("div", { className: "grid grid-cols-2 sm:grid-cols-3 lg:grid-cols-6 gap-3", children: [_jsx(StatTile, { label: "blobs", value: status.blob_count.toLocaleString(), color: "ok" }), _jsx(StatTile, { label: "tags", value: status.tag_count, color: "ok" }), _jsx(StatTile, { label: "refs", value: status.ref_count, color: "ok" }), _jsx(StatTile, { label: "snapshots", value: status.snapshot_count, color: "ok" }), _jsx(StatTile, { label: "ref-tracking", value: status.ref_tracking_count, color: "ok" }), _jsx(StatTile, { label: "store size", value: fmtBytes(status.blob_store_bytes), color: "ok" })] }), _jsxs("div", { className: "rounded border border-slate-800 bg-slate-900/40 p-4 text-sm", children: [_jsx("div", { className: "text-slate-500 uppercase text-xs tracking-wider", children: "blob store root" }), _jsx("div", { className: "font-mono mt-1", children: status.blob_store_root ?? '—' })] })] }))] }));
+    return (_jsxs("div", { className: "space-y-6", children: [_jsxs("div", { children: [_jsx(Link, { href: "/", children: _jsx("a", { className: "text-sm text-slate-500 hover:text-slate-300", children: "\u2190 fleet" }) }), _jsx("h1", { className: "text-2xl font-semibold text-slate-100 mt-2", children: name })] }), err && (_jsxs("div", { className: "rounded border border-amber-800 bg-amber-950/40 p-3 text-amber-300 text-sm", children: [err, _jsxs("div", { className: "mt-2 text-xs text-slate-400", children: ["Cross-node lookup lands in a follow-on PR. Until then this page shows detail only when you're already viewing the dashboard hosted by ", name, ". Try opening", ' ', _jsxs("span", { className: "font-mono", children: ["http://", name, ":7700/v2/#/nodes/", name] }), ' ', "directly."] })] })), status && (_jsxs(_Fragment, { children: [_jsxs("div", { className: "grid grid-cols-2 sm:grid-cols-3 lg:grid-cols-6 gap-3", children: [_jsx(StatTile, { label: "blobs", value: status.blob_count.toLocaleString(), color: "ok" }), _jsx(StatTile, { label: "tags", value: status.tag_count, color: "ok" }), _jsx(StatTile, { label: "refs", value: status.ref_count, color: "ok" }), _jsx(StatTile, { label: "snapshots", value: status.snapshot_count, color: "ok" }), _jsx(StatTile, { label: "ref-tracking", value: status.ref_tracking_count, color: "ok" }), _jsx(StatTile, { label: "store size", value: fmtBytes(status.blob_store_bytes), color: "ok" })] }), _jsxs("div", { className: "rounded border border-slate-800 bg-slate-900/40 p-4 text-sm", children: [_jsx("div", { className: "text-slate-500 uppercase text-xs tracking-wider", children: "blob store root" }), _jsx("div", { className: "font-mono mt-1", children: status.blob_store_root ?? '—' })] }), _jsx(ShutdownPrepPanel, { name: name })] }))] }));
 }

dashboard-v2/src/pages/NodeDetail.tsx

@@ -2,6 +2,7 @@ import { useEffect, useState } from 'react';
 import { Link } from 'wouter';
 import { api, NodeStatusV2, fmtBytes } from '../lib/api';
 import { StatTile } from '../components/StatTile';
+import { ShutdownPrepPanel } from '../components/ShutdownPrepPanel';
 
 interface Props {
   name: string;
@@ -67,6 +68,8 @@ export function NodeDetail({ name }: Props) {
             </div>
             <div className="font-mono mt-1">{status.blob_store_root ?? '—'}</div>
           </div>
+
+          <ShutdownPrepPanel name={name} />
         </>
       )}
     </div>

dashboard-v2/tsconfig.tsbuildinfo

@@ -1 +1 @@
-{"root":["./src/app.tsx","./src/main.tsx","./src/components/nodecard.tsx","./src/components/projectspanel.tsx","./src/components/stattile.tsx","./src/components/storagebar.tsx","./src/lib/api.ts","./src/pages/commandcenter.tsx","./src/pages/nodedetail.tsx","./src/pages/reftrackingpage.tsx","./src/pages/storagebrowser.tsx"],"version":"6.0.3"}
+{"root":["./src/App.tsx","./src/main.tsx","./src/components/NodeCard.tsx","./src/components/ProjectsPanel.tsx","./src/components/ShutdownPrepPanel.tsx","./src/components/StatTile.tsx","./src/components/StorageBar.tsx","./src/lib/api.ts","./src/pages/CommandCenter.tsx","./src/pages/NodeDetail.tsx","./src/pages/RefTrackingPage.tsx","./src/pages/StorageBrowser.tsx"],"version":"6.0.3"}

dashboard-v2/vite.config.ts

@@ -6,13 +6,15 @@ import react from '@vitejs/plugin-react';
 // During local dev the daemon proxies /api/v2/* on :7700 so
 // `vite dev` on :5173 can hit it via server.proxy.
 export default defineConfig({
-  // Absolute base tied to the deploy path. Prior try was `./`
+  // Absolute base matching the backend's actual mount point
-  // (fully relative) which broke when the user hit `/clawstor`
+  // (`serve.rs` nests the v2 static dir at `/v2` via
-  // without trailing slash — browser resolves `./assets/…`
+  // `nest_service("/v2", …)`). A prior `/clawstor/` base assumed a
-  // against `/clawstor` treated as a file, gives `/assets/…`,
+  // Tailscale Serve path mapping that was never actually configured
-  // 404 from Tailscale. Absolute `/clawstor/` sidesteps the
+  // on any node (checked `tailscale serve status` on tank +
-  // slash / no-slash ambiguity.
+  // architect: neither proxies a `/clawstor` path) — that base
-  base: '/clawstor/',
+  // silently broke direct `:7700/v2/` access, the only access
+  // pattern that's actually live.
+  base: '/v2/',
   plugins: [react()],
   server: {
     port: 5173,

deploy/scripts/safe-shutdown-prep.sh

@@ -0,0 +1,238 @@
+#!/usr/bin/env bash
+# safe-shutdown-prep.sh — bring a clawstor node to a clean, safe stop
+# before hardware maintenance (parts replacement, drive swap, etc.).
+#
+# Run this ON the node you're about to power off. It does NOT power
+# the machine off itself — the last line of output tells you the
+# command to run for that, once everything above it is clean.
+#
+# What it does, in order:
+#   1. Refuse to proceed if a cargo/rustc build is active against a
+#      tracked project's warm_path (unless --force).
+#   2. Refuse to proceed if the sync queue has pending jobs peers
+#      haven't received yet (unless --force). Gives it one chance to
+#      drain via `claw-store sync <project>` before failing.
+#   3. Take a final ZFS snapshot of the warm tier + replicate it to
+#      the configured cold peer, and wait for both to finish.
+#   4. Stop the four maintenance timers (scrub/gc/ref-sweep/
+#      snapshot-rotate) so nothing fires mid-shutdown or immediately
+#      after next boot before you've verified the node.
+#   5. Stop claw-store-serve.service (dashboard) — no data risk, just
+#      tidy.
+#   6. Stop claw-store.service gracefully. The unit's
+#      TimeoutStopSec=60 gives the daemon's SIGTERM handler room to
+#      let gossip announce this node's departure to peers before the
+#      process exits — skipping this step means peers only notice via
+#      the failure detector's dead_node_grace_period (10s) instead of
+#      an immediate clean departure.
+#   7. Stop claw-fuse.service and verify the mount is actually gone
+#      (retries a lazy unmount if the clean one doesn't take).
+#   8. Sync filesystem buffers and print zpool health for the warm
+#      tier's pool — warns (does not block) if the pool is degraded,
+#      since that's independently worth knowing before you touch
+#      hardware.
+#
+# Flags:
+#   --force          Skip the active-build and pending-sync guards.
+#                     Everything else (steps 3-8) still runs.
+#   --export-zpool    Additionally `zpool export` the warm-tier pool
+#                     at the end — only do this if you're physically
+#                     removing the storage drives, not for e.g. a RAM
+#                     or PSU swap. Requires a matching `zpool import`
+#                     after the node is back up before claw-store.service
+#                     will find its data again.
+#   --skip-replicate  Skip step 3 (snapshot + replicate). Use only if
+#                     you already know cold tier is current, or this
+#                     node has no [replication] configured.
+#   --dry-run         Run every check (steps 1-2) and the snapshot/
+#                     replicate (step 3) for real, but only print what
+#                     steps 4-8 (stop timers/services, unmount, zpool
+#                     export) would do instead of doing them. Use this
+#                     first to verify the script sees your node's
+#                     actual state correctly before trusting it live.
+set -uo pipefail
+
+CONFIG=${CLAWSTOR_CONFIG:-/etc/claw-store/config.toml}
+BIN=${CLAWSTOR_BIN:-/usr/local/bin/claw-store}
+SYNC_QUEUE=/var/lib/claw-store/sync-queue.toml
+FORCE=0
+EXPORT_ZPOOL=0
+SKIP_REPLICATE=0
+DRY_RUN=0
+
+for arg in "$@"; do
+  case "$arg" in
+    --force) FORCE=1 ;;
+    --export-zpool) EXPORT_ZPOOL=1 ;;
+    --skip-replicate) SKIP_REPLICATE=1 ;;
+    --dry-run) DRY_RUN=1 ;;
+    *) echo "unknown flag: $arg" >&2; exit 2 ;;
+  esac
+done
+
+run() {
+  # Gate an actual state-changing command behind --dry-run.
+  if [ "$DRY_RUN" -eq 1 ]; then
+    echo "  [dry-run] would run: $*"
+    return 0
+  fi
+  "$@"
+}
+
+hr() { printf '%.0s─' {1..66}; echo; }
+step() { hr; echo "▶ $1"; hr; }
+ok()   { echo "  ✓ $1"; }
+warn() { echo "  ! $1"; }
+fail() { echo "  ✗ $1" >&2; }
+
+NODE=$(hostname)
+echo "safe-shutdown-prep — $NODE — $(date -Iseconds)"
+
+# ── 1. Active builds ────────────────────────────────────────────────
+step "checking for active cargo/rustc builds"
+ACTIVE=$(pgrep -af 'cargo|rustc' | grep -v "safe-shutdown-prep\|grep" || true)
+if [ -n "$ACTIVE" ]; then
+  echo "$ACTIVE" | sed 's/^/  /'
+  if [ "$FORCE" -eq 1 ]; then
+    warn "active build(s) found — continuing anyway (--force)"
+  else
+    fail "active build(s) found on this node. A build in progress against"
+    fail "the warm tier can be interrupted mid-write by an unmount/shutdown."
+    fail "Wait for it to finish, or re-run with --force to proceed anyway."
+    exit 1
+  fi
+else
+  ok "no active cargo/rustc processes"
+fi
+
+# ── 2. Sync queue ────────────────────────────────────────────────────
+step "checking sync queue for pending peer pushes"
+if [ -f "$SYNC_QUEUE" ]; then
+  DEPTH=$(grep -c '^project = ' "$SYNC_QUEUE" 2>/dev/null); DEPTH=${DEPTH:-0}
+else
+  DEPTH=0
+fi
+if [ "$DEPTH" -gt 0 ]; then
+  warn "$DEPTH pending sync job(s) in $SYNC_QUEUE — attempting to drain"
+  PROJECTS=$(grep '^project = ' "$SYNC_QUEUE" | sed 's/project = "\(.*\)"/\1/')
+  for p in $PROJECTS; do
+    echo "  syncing $p ..."
+    "$BIN" --config "$CONFIG" sync "$p" || warn "sync failed for $p"
+  done
+  DEPTH_AFTER=$(grep -c '^project = ' "$SYNC_QUEUE" 2>/dev/null); DEPTH_AFTER=${DEPTH_AFTER:-0}
+  if [ "$DEPTH_AFTER" -gt 0 ]; then
+    if [ "$FORCE" -eq 1 ]; then
+      warn "$DEPTH_AFTER job(s) still pending — continuing anyway (--force)"
+    else
+      fail "$DEPTH_AFTER sync job(s) still pending after drain attempt."
+      fail "Peers may be unreachable, or the push is failing for another"
+      fail "reason. Re-run with --force to shut down anyway (those changes"
+      fail "will catch up once this node is back and the daemon retries)."
+      exit 1
+    fi
+  else
+    ok "sync queue drained"
+  fi
+else
+  ok "sync queue empty"
+fi
+
+# ── 3. Final snapshot + replicate to cold ───────────────────────────
+if [ "$SKIP_REPLICATE" -eq 1 ]; then
+  step "skipping snapshot + replicate (--skip-replicate)"
+else
+  step "taking final snapshot + replicating to cold tier"
+  if "$BIN" --config "$CONFIG" snapshot; then
+    ok "snapshot created"
+  else
+    warn "snapshot command failed — check output above"
+  fi
+  if "$BIN" --config "$CONFIG" replicate; then
+    ok "replication to cold tier complete"
+  else
+    warn "replicate command failed or not configured — check output above"
+    warn "([replication] section may be absent on this node; that's fine)"
+  fi
+fi
+
+# ── 4. Stop maintenance timers ──────────────────────────────────────
+step "stopping maintenance timers"
+for t in clawstor-scrub clawstor-gc clawstor-ref-sweep clawstor-snapshot-rotate; do
+  run systemctl --user stop "$t.timer" 2>/dev/null && ok "$t.timer stopped" || warn "$t.timer not running or not found"
+done
+
+# ── 5. Stop dashboard ────────────────────────────────────────────────
+step "stopping claw-store-serve.service"
+if systemctl is-active --quiet claw-store-serve.service 2>/dev/null; then
+  run sudo systemctl stop claw-store-serve.service && ok "stopped" || fail "failed to stop"
+else
+  ok "not running"
+fi
+
+# ── 6. Stop daemon (gossip departure) ───────────────────────────────
+step "stopping claw-store.service (gossip will announce departure)"
+if systemctl is-active --quiet claw-store.service 2>/dev/null; then
+  run sudo systemctl stop claw-store.service && ok "stopped cleanly" || fail "failed to stop — check 'systemctl status claw-store.service'"
+else
+  ok "not running"
+fi
+
+# ── 7. Unmount FUSE ──────────────────────────────────────────────────
+step "unmounting FUSE"
+if systemctl is-active --quiet claw-fuse.service 2>/dev/null; then
+  run sudo systemctl stop claw-fuse.service
+  sleep 1
+fi
+if [ "$DRY_RUN" -eq 1 ]; then
+  if mount | grep -q "type fuse.clawstor"; then
+    warn "still mounted (expected — nothing was actually stopped in --dry-run)"
+  else
+    ok "already unmounted"
+  fi
+else
+  if mount | grep -q "type fuse.clawstor"; then
+    warn "still mounted after service stop — trying lazy unmount"
+    sudo umount -l ~/clawstor-mount 2>/dev/null
+    sleep 1
+  fi
+  if mount | grep -q "type fuse.clawstor"; then
+    fail "FUSE mount would not come down: $(mount | grep 'fuse.clawstor')"
+    fail "Do not power off until this is resolved — an unclean FUSE"
+    fail "unmount can leave a stale mountpoint that needs manual cleanup"
+    fail "on next boot."
+    exit 1
+  else
+    ok "unmounted"
+  fi
+fi
+
+# ── 8. Flush + pool health ──────────────────────────────────────────
+step "flushing filesystem buffers"
+sync
+ok "sync complete"
+
+step "zpool health check"
+if ! command -v zpool >/dev/null 2>&1; then
+  ok "no zpool binary on this node — warm tier is not ZFS-backed here, nothing to check"
+else
+  POOL=$(df --output=source /slab 2>/dev/null | tail -1 | tr -d '[:space:]')
+  if [ -n "$POOL" ] && [ "$POOL" != "none" ]; then
+    echo "  pool: $POOL"
+    zpool status -x "$POOL" 2>&1 | sed 's/^/  /'
+    if [ "$EXPORT_ZPOOL" -eq 1 ]; then
+      step "exporting $POOL (--export-zpool)"
+      run sudo zpool export "$POOL" && ok "exported — remember: zpool import $POOL after reboot" || fail "export failed"
+    fi
+  else
+    warn "zpool present but /slab isn't a recognizable ZFS mount"
+  fi
+fi
+
+hr
+if [ "$DRY_RUN" -eq 1 ]; then
+  echo "DRY RUN COMPLETE — nothing was actually stopped or unmounted."
+  echo "Re-run without --dry-run when ready to actually prep for shutdown."
+else
+  echo "SAFE TO POWER OFF — run: sudo shutdown -h now"
+fi
+hr

systemd/claw-fuse.service

@@ -0,0 +1,17 @@
+[Unit]
+Description=clawstor FUSE mount
+After=claw-store.service
+Requires=claw-store.service
+
+[Service]
+Type=simple
+User=osobh
+Environment=PATH=/home/osobh/.cargo/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin
+ExecStartPre=/bin/mkdir -p /home/osobh/clawstor-mount
+ExecStart=/usr/local/bin/claw-fuse --data-dir /home/osobh/clawstor-deploy/data --mount /home/osobh/clawstor-mount
+ExecStop=/bin/fusermount -u /home/osobh/clawstor-mount
+Restart=on-failure
+RestartSec=30
+
+[Install]
+WantedBy=multi-user.target

systemd/claw-store-serve.service

@@ -11,7 +11,7 @@ Wants=network-online.target
 [Service]
 Type=simple
 User=osobh
-ExecStart=/usr/local/bin/claw-store serve --port 7700 --static-dir /usr/share/claw-store/static
+ExecStart=/usr/local/bin/claw-store serve --port 7700 --static-dir /usr/share/claw-store/static --v2-static-dir /usr/share/claw-store/v2
 Restart=on-failure
 RestartSec=15
 Environment=RUST_LOG=info

systemd/claw-store.service

@@ -1,15 +1,19 @@
 [Unit]
-Description=claw-store fleet storage daemon
+Description=clawstor cluster daemon (gossip + QUIC blob store + ZFS snapshots)
-After=zfs-mount.service network.target
+After=network-online.target
-Wants=zfs-mount.service
+Wants=network-online.target
 
 [Service]
 Type=simple
 User=osobh
+Environment=PATH=/home/osobh/.cargo/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin
+Environment=RUST_LOG=info
 ExecStart=/usr/local/bin/claw-store daemon
 Restart=on-failure
-RestartSec=30
+RestartSec=15
-Environment=RUST_LOG=info
+TimeoutStopSec=60
+ProtectSystem=strict
+ReadWritePaths=/var/lib/claw-store /hot/targets /slab/projects /home/osobh/clawstor-deploy
 
 [Install]
 WantedBy=multi-user.target