deploy: add safe-shutdown-prep.sh for hardware maintenance

Node-local script an operator runs before powering a node off for
parts replacement. Guards against shutting down mid-build or with
un-pushed sync jobs, takes a final snapshot + replicates to cold,
stops the maintenance timers and dashboard, gracefully stops the
daemon (giving gossip its TimeoutStopSec=60 window to announce
departure to peers instead of relying on the 10s failure-detector
timeout), and cleanly unmounts FUSE before declaring the node safe
to power off. --dry-run runs every check for real but only prints
what the stop/unmount steps would do.

Co-Authored-By: Claude Sonnet 5 <[email protected]>

Makefile

@@ -32,7 +32,7 @@ install-systemd:
 install-dashboard:
 	cd dashboard && npm ci && npm run build
 	install -dm755 $(INSTALL_STATIC)
-	cp -r dashboard/dist/. $(INSTALL_STATIC)/
+	cp -r claw-store/static/. $(INSTALL_STATIC)/
 	@echo "Dashboard installed to $(INSTALL_STATIC)"
 
 ## Install node-specific config (NODE=architect|tank)

claw-store/src/cargo_init.rs

@@ -21,8 +21,14 @@ pub fn write_cargo_config(warm_path: &Path, hot_target_path: &Path) -> Result<()
         String::new()
     };
 
+    // Drop any leading copies of our own marker comment before stripping the
+    // [build] section — it sits above the [build] header, outside the range
+    // strip_section tracks, so without this it would survive every
+    // regenerate cycle and duplicate one more time.
+    let existing = strip_leading_marker(&existing);
+
     // Remove old [build] block (from its header to the next section or EOF).
-    let stripped = strip_section(&existing, "build");
+    let stripped = strip_section(existing, "build");
 
     let new_content = format!(
         "# claw-store managed — do not edit manually\n\
@@ -37,6 +43,21 @@ pub fn write_cargo_config(warm_path: &Path, hot_target_path: &Path) -> Result<()
         .with_context(|| format!("writing {}", config_path.display()))
 }
 
+const MANAGED_MARKER: &str = "# claw-store managed — do not edit manually";
+
+/// Strip leading copies of `MANAGED_MARKER`, one per line, from the start of `src`.
+fn strip_leading_marker(src: &str) -> &str {
+    let mut rest = src;
+    while let Some(line_end) = rest.find('\n') {
+        if rest[..line_end].trim() == MANAGED_MARKER {
+            rest = &rest[line_end + 1..];
+        } else {
+            break;
+        }
+    }
+    rest
+}
+
 /// Remove a TOML section `[name]` and all its key=value lines from `src`,
 /// stopping at the next `[section]` header or EOF.
 fn strip_section(src: &str, name: &str) -> String {
@@ -102,6 +123,46 @@ mod tests {
         assert!(verify_cargo_config(&warm, &hot).unwrap());
     }
 
+    #[test]
+    fn test_write_cargo_config_idempotent_no_duplicate_marker() {
+        let dir = TempDir::new().unwrap();
+        let warm = dir.path().join("proj");
+        std::fs::create_dir_all(&warm).unwrap();
+        let hot: std::path::PathBuf = "/hot/targets/proj".into();
+
+        write_cargo_config(&warm, &hot).unwrap();
+        write_cargo_config(&warm, &hot).unwrap();
+        write_cargo_config(&warm, &hot).unwrap();
+
+        let content = std::fs::read_to_string(warm.join(".cargo/config.toml")).unwrap();
+        assert_eq!(content.matches("claw-store managed").count(), 1);
+    }
+
+    #[test]
+    fn test_write_cargo_config_heals_existing_duplicate_marker() {
+        let dir = TempDir::new().unwrap();
+        let warm = dir.path().join("proj");
+        let cargo_dir = warm.join(".cargo");
+        std::fs::create_dir_all(&cargo_dir).unwrap();
+        std::fs::write(
+            cargo_dir.join("config.toml"),
+            "# claw-store managed — do not edit manually\n\
+             [build]\n\
+             target-dir = \"/hot/targets/proj\"\n\
+             # claw-store managed — do not edit manually\n\
+             [env]\n\
+             FOO = \"bar\"\n",
+        )
+        .unwrap();
+
+        let hot: std::path::PathBuf = "/hot/targets/proj".into();
+        write_cargo_config(&warm, &hot).unwrap();
+
+        let content = std::fs::read_to_string(cargo_dir.join("config.toml")).unwrap();
+        assert_eq!(content.matches("claw-store managed").count(), 1);
+        assert!(content.contains("FOO = \"bar\""));
+    }
+
     #[test]
     fn test_verify_cargo_config_detects_missing() {
         let dir = TempDir::new().unwrap();

config/architect.toml

@@ -19,6 +19,33 @@ archive_path = "/data/archive"
 zfs_dataset = "data/archive"
 retain_weeks = 12
 
+[cluster]
+zone = "fabric-10g"
+bind_lan = "10.0.0.13:7701"
+prom_bind = "0.0.0.0:7703"
+bind_rpc_lan = "10.0.0.13:7702"
+bind_rpc_tailscale = "100.104.171.32:7702"
+blob_store_root = "/home/osobh/clawstor-deploy/data"
+
+[[cluster.peers]]
+name = "tank"
+zone = "fabric-10g"
+lan_addr = "10.0.0.14:7701"
+rpc_lan_addr = "10.10.0.10:7702"
+tailscale_addr = "100.108.129.81:7702"
+
+[[cluster.peers]]
+name = "morpheus"
+zone = "lan-1g"
+lan_addr = "10.0.0.5:7701"
+rpc_lan_addr = "10.0.0.5:7702"
+tailscale_addr = "100.123.224.84:7702"
+
+[cluster.tls]
+ca_cert = "/home/osobh/clawstor-deploy/tls/ca.crt"
+node_cert = "/home/osobh/clawstor-deploy/tls/node.crt"
+node_key = "/home/osobh/clawstor-deploy/tls/node.key"
+
 [replication]
 receive_from_peer = true
 peer_user = "osobh"

config/morpheus.toml

@@ -0,0 +1,43 @@
+[node]
+name = "morpheus"
+role = "secondary"
+
+[hot]
+path = "/hot/targets"
+max_gb = 80
+stale_hours = 48
+
+[warm]
+projects_path = "/slab/projects"
+zfs_dataset = "none"
+snapshot_retain_hours = 24
+snapshot_retain_days = 7
+snapshot_retain_weeks = 4
+
+[cluster]
+zone = "lan-1g"
+# Morpheus has no direct 10G to Architect/Tank — use main LAN for all traffic
+bind_lan = "10.0.0.5:7701"
+prom_bind = "0.0.0.0:7703"
+bind_rpc_lan = "10.0.0.5:7702"
+bind_rpc_tailscale = "100.123.224.84:7702"
+blob_store_root = "/home/osobh/clawstor-deploy/data"
+
+[[cluster.peers]]
+name = "architect"
+zone = "fabric-10g"
+lan_addr = "10.0.0.13:7701"
+rpc_lan_addr = "10.0.0.13:7702"
+tailscale_addr = "100.104.171.32:7702"
+
+[[cluster.peers]]
+name = "tank"
+zone = "fabric-10g"
+lan_addr = "10.0.0.14:7701"
+rpc_lan_addr = "10.0.0.14:7702"
+tailscale_addr = "100.108.129.81:7702"
+
+[cluster.tls]
+ca_cert = "/home/osobh/clawstor-deploy/tls/ca.crt"
+node_cert = "/home/osobh/clawstor-deploy/tls/node.crt"
+node_key = "/home/osobh/clawstor-deploy/tls/node.key"

config/tank.toml

@@ -14,13 +14,35 @@ snapshot_retain_hours = 24
 snapshot_retain_days = 7
 snapshot_retain_weeks = 4
 
+[cluster]
+zone = "fabric-10g"
+bind_lan = "10.0.0.14:7701"
+prom_bind = "0.0.0.0:7703"
+bind_rpc_lan = "10.0.0.14:7702"
+bind_rpc_tailscale = "100.108.129.81:7702"
+blob_store_root = "/home/osobh/clawstor-deploy/data"
+
+[[cluster.peers]]
+name = "architect"
+zone = "fabric-10g"
+lan_addr = "10.0.0.13:7701"
+rpc_lan_addr = "10.10.0.9:7702"
+tailscale_addr = "100.104.171.32:7702"
+
+[[cluster.peers]]
+name = "morpheus"
+zone = "lan-1g"
+lan_addr = "10.0.0.5:7701"
+rpc_lan_addr = "10.0.0.5:7702"
+tailscale_addr = "100.123.224.84:7702"
+
+[cluster.tls]
+ca_cert = "/home/osobh/clawstor-deploy/tls/ca.crt"
+node_cert = "/home/osobh/clawstor-deploy/tls/node.crt"
+node_key = "/home/osobh/clawstor-deploy/tls/node.key"
+
 [replication]
-# 10.10.0.9 is architect-fab-tank — the dedicated 10G fabric link between
-# the two nodes. Intentionally used for replication to maximise bandwidth;
-# architect's primary LAN address is 10.0.0.13.
 send_to_host = "10.10.0.9"
 send_to_user = "osobh"
 cold_dataset_on_peer = "data/archive/tank-projects"
-# nightly_at is reserved for future use; replication schedule is currently
-# controlled by the claw-store-replicate.timer systemd unit.
 nightly_at = "03:30"

dashboard-v2/tsconfig.tsbuildinfo

@@ -1 +1 @@
-{"root":["./src/app.tsx","./src/main.tsx","./src/components/nodecard.tsx","./src/components/projectspanel.tsx","./src/components/stattile.tsx","./src/components/storagebar.tsx","./src/lib/api.ts","./src/pages/commandcenter.tsx","./src/pages/nodedetail.tsx","./src/pages/reftrackingpage.tsx","./src/pages/storagebrowser.tsx"],"version":"6.0.3"}
+{"root":["./src/App.tsx","./src/main.tsx","./src/components/NodeCard.tsx","./src/components/ProjectsPanel.tsx","./src/components/StatTile.tsx","./src/components/StorageBar.tsx","./src/lib/api.ts","./src/pages/CommandCenter.tsx","./src/pages/NodeDetail.tsx","./src/pages/RefTrackingPage.tsx","./src/pages/StorageBrowser.tsx"],"version":"6.0.3"}

dashboard-v2/vite.config.ts

@@ -6,13 +6,15 @@ import react from '@vitejs/plugin-react';
 // During local dev the daemon proxies /api/v2/* on :7700 so
 // `vite dev` on :5173 can hit it via server.proxy.
 export default defineConfig({
-  // Absolute base tied to the deploy path. Prior try was `./`
+  // Absolute base matching the backend's actual mount point
-  // (fully relative) which broke when the user hit `/clawstor`
+  // (`serve.rs` nests the v2 static dir at `/v2` via
-  // without trailing slash — browser resolves `./assets/…`
+  // `nest_service("/v2", …)`). A prior `/clawstor/` base assumed a
-  // against `/clawstor` treated as a file, gives `/assets/…`,
+  // Tailscale Serve path mapping that was never actually configured
-  // 404 from Tailscale. Absolute `/clawstor/` sidesteps the
+  // on any node (checked `tailscale serve status` on tank +
-  // slash / no-slash ambiguity.
+  // architect: neither proxies a `/clawstor` path) — that base
-  base: '/clawstor/',
+  // silently broke direct `:7700/v2/` access, the only access
+  // pattern that's actually live.
+  base: '/v2/',
   plugins: [react()],
   server: {
     port: 5173,

deploy/scripts/safe-shutdown-prep.sh

@@ -0,0 +1,231 @@
+#!/usr/bin/env bash
+# safe-shutdown-prep.sh — bring a clawstor node to a clean, safe stop
+# before hardware maintenance (parts replacement, drive swap, etc.).
+#
+# Run this ON the node you're about to power off. It does NOT power
+# the machine off itself — the last line of output tells you the
+# command to run for that, once everything above it is clean.
+#
+# What it does, in order:
+#   1. Refuse to proceed if a cargo/rustc build is active against a
+#      tracked project's warm_path (unless --force).
+#   2. Refuse to proceed if the sync queue has pending jobs peers
+#      haven't received yet (unless --force). Gives it one chance to
+#      drain via `claw-store sync <project>` before failing.
+#   3. Take a final ZFS snapshot of the warm tier + replicate it to
+#      the configured cold peer, and wait for both to finish.
+#   4. Stop the four maintenance timers (scrub/gc/ref-sweep/
+#      snapshot-rotate) so nothing fires mid-shutdown or immediately
+#      after next boot before you've verified the node.
+#   5. Stop claw-store-serve.service (dashboard) — no data risk, just
+#      tidy.
+#   6. Stop claw-store.service gracefully. The unit's
+#      TimeoutStopSec=60 gives the daemon's SIGTERM handler room to
+#      let gossip announce this node's departure to peers before the
+#      process exits — skipping this step means peers only notice via
+#      the failure detector's dead_node_grace_period (10s) instead of
+#      an immediate clean departure.
+#   7. Stop claw-fuse.service and verify the mount is actually gone
+#      (retries a lazy unmount if the clean one doesn't take).
+#   8. Sync filesystem buffers and print zpool health for the warm
+#      tier's pool — warns (does not block) if the pool is degraded,
+#      since that's independently worth knowing before you touch
+#      hardware.
+#
+# Flags:
+#   --force          Skip the active-build and pending-sync guards.
+#                     Everything else (steps 3-8) still runs.
+#   --export-zpool    Additionally `zpool export` the warm-tier pool
+#                     at the end — only do this if you're physically
+#                     removing the storage drives, not for e.g. a RAM
+#                     or PSU swap. Requires a matching `zpool import`
+#                     after the node is back up before claw-store.service
+#                     will find its data again.
+#   --skip-replicate  Skip step 3 (snapshot + replicate). Use only if
+#                     you already know cold tier is current, or this
+#                     node has no [replication] configured.
+#   --dry-run         Run every check (steps 1-2) and the snapshot/
+#                     replicate (step 3) for real, but only print what
+#                     steps 4-8 (stop timers/services, unmount, zpool
+#                     export) would do instead of doing them. Use this
+#                     first to verify the script sees your node's
+#                     actual state correctly before trusting it live.
+set -uo pipefail
+
+CONFIG=${CLAWSTOR_CONFIG:-/etc/claw-store/config.toml}
+BIN=${CLAWSTOR_BIN:-/usr/local/bin/claw-store}
+SYNC_QUEUE=/var/lib/claw-store/sync-queue.toml
+FORCE=0
+EXPORT_ZPOOL=0
+SKIP_REPLICATE=0
+DRY_RUN=0
+
+for arg in "$@"; do
+  case "$arg" in
+    --force) FORCE=1 ;;
+    --export-zpool) EXPORT_ZPOOL=1 ;;
+    --skip-replicate) SKIP_REPLICATE=1 ;;
+    --dry-run) DRY_RUN=1 ;;
+    *) echo "unknown flag: $arg" >&2; exit 2 ;;
+  esac
+done
+
+run() {
+  # Gate an actual state-changing command behind --dry-run.
+  if [ "$DRY_RUN" -eq 1 ]; then
+    echo "  [dry-run] would run: $*"
+    return 0
+  fi
+  "$@"
+}
+
+hr() { printf '%.0s─' {1..66}; echo; }
+step() { hr; echo "▶ $1"; hr; }
+ok()   { echo "  ✓ $1"; }
+warn() { echo "  ! $1"; }
+fail() { echo "  ✗ $1" >&2; }
+
+NODE=$(hostname)
+echo "safe-shutdown-prep — $NODE — $(date -Iseconds)"
+
+# ── 1. Active builds ────────────────────────────────────────────────
+step "checking for active cargo/rustc builds"
+ACTIVE=$(pgrep -af 'cargo|rustc' | grep -v "safe-shutdown-prep\|grep" || true)
+if [ -n "$ACTIVE" ]; then
+  echo "$ACTIVE" | sed 's/^/  /'
+  if [ "$FORCE" -eq 1 ]; then
+    warn "active build(s) found — continuing anyway (--force)"
+  else
+    fail "active build(s) found on this node. A build in progress against"
+    fail "the warm tier can be interrupted mid-write by an unmount/shutdown."
+    fail "Wait for it to finish, or re-run with --force to proceed anyway."
+    exit 1
+  fi
+else
+  ok "no active cargo/rustc processes"
+fi
+
+# ── 2. Sync queue ────────────────────────────────────────────────────
+step "checking sync queue for pending peer pushes"
+if [ -f "$SYNC_QUEUE" ]; then
+  DEPTH=$(grep -c '^project = ' "$SYNC_QUEUE" 2>/dev/null); DEPTH=${DEPTH:-0}
+else
+  DEPTH=0
+fi
+if [ "$DEPTH" -gt 0 ]; then
+  warn "$DEPTH pending sync job(s) in $SYNC_QUEUE — attempting to drain"
+  PROJECTS=$(grep '^project = ' "$SYNC_QUEUE" | sed 's/project = "\(.*\)"/\1/')
+  for p in $PROJECTS; do
+    echo "  syncing $p ..."
+    "$BIN" --config "$CONFIG" sync "$p" || warn "sync failed for $p"
+  done
+  DEPTH_AFTER=$(grep -c '^project = ' "$SYNC_QUEUE" 2>/dev/null); DEPTH_AFTER=${DEPTH_AFTER:-0}
+  if [ "$DEPTH_AFTER" -gt 0 ]; then
+    if [ "$FORCE" -eq 1 ]; then
+      warn "$DEPTH_AFTER job(s) still pending — continuing anyway (--force)"
+    else
+      fail "$DEPTH_AFTER sync job(s) still pending after drain attempt."
+      fail "Peers may be unreachable, or the push is failing for another"
+      fail "reason. Re-run with --force to shut down anyway (those changes"
+      fail "will catch up once this node is back and the daemon retries)."
+      exit 1
+    fi
+  else
+    ok "sync queue drained"
+  fi
+else
+  ok "sync queue empty"
+fi
+
+# ── 3. Final snapshot + replicate to cold ───────────────────────────
+if [ "$SKIP_REPLICATE" -eq 1 ]; then
+  step "skipping snapshot + replicate (--skip-replicate)"
+else
+  step "taking final snapshot + replicating to cold tier"
+  if "$BIN" --config "$CONFIG" snapshot; then
+    ok "snapshot created"
+  else
+    warn "snapshot command failed — check output above"
+  fi
+  if "$BIN" --config "$CONFIG" replicate; then
+    ok "replication to cold tier complete"
+  else
+    warn "replicate command failed or not configured — check output above"
+    warn "([replication] section may be absent on this node; that's fine)"
+  fi
+fi
+
+# ── 4. Stop maintenance timers ──────────────────────────────────────
+step "stopping maintenance timers"
+for t in clawstor-scrub clawstor-gc clawstor-ref-sweep clawstor-snapshot-rotate; do
+  run systemctl --user stop "$t.timer" 2>/dev/null && ok "$t.timer stopped" || warn "$t.timer not running or not found"
+done
+
+# ── 5. Stop dashboard ────────────────────────────────────────────────
+step "stopping claw-store-serve.service"
+if systemctl is-active --quiet claw-store-serve.service 2>/dev/null; then
+  run sudo systemctl stop claw-store-serve.service && ok "stopped" || fail "failed to stop"
+else
+  ok "not running"
+fi
+
+# ── 6. Stop daemon (gossip departure) ───────────────────────────────
+step "stopping claw-store.service (gossip will announce departure)"
+if systemctl is-active --quiet claw-store.service 2>/dev/null; then
+  run sudo systemctl stop claw-store.service && ok "stopped cleanly" || fail "failed to stop — check 'systemctl status claw-store.service'"
+else
+  ok "not running"
+fi
+
+# ── 7. Unmount FUSE ──────────────────────────────────────────────────
+step "unmounting FUSE"
+if systemctl is-active --quiet claw-fuse.service 2>/dev/null; then
+  run sudo systemctl stop claw-fuse.service
+  sleep 1
+fi
+if [ "$DRY_RUN" -eq 1 ]; then
+  if mount | grep -q "type fuse.clawstor"; then
+    warn "still mounted (expected — nothing was actually stopped in --dry-run)"
+  else
+    ok "already unmounted"
+  fi
+else
+  if mount | grep -q "type fuse.clawstor"; then
+    warn "still mounted after service stop — trying lazy unmount"
+    sudo umount -l ~/clawstor-mount 2>/dev/null
+    sleep 1
+  fi
+  if mount | grep -q "type fuse.clawstor"; then
+    fail "FUSE mount would not come down: $(mount | grep 'fuse.clawstor')"
+    fail "Do not power off until this is resolved — an unclean FUSE"
+    fail "unmount can leave a stale mountpoint that needs manual cleanup"
+    fail "on next boot."
+    exit 1
+  else
+    ok "unmounted"
+  fi
+fi
+
+# ── 8. Flush + pool health ──────────────────────────────────────────
+step "flushing filesystem buffers"
+sync
+ok "sync complete"
+
+POOL=$(mount | awk '/on \/slab / {print $1}')
+if [ -n "$POOL" ]; then
+  step "zpool health check ($POOL)"
+  zpool status -x "$POOL" 2>&1 | sed 's/^/  /'
+  if [ "$EXPORT_ZPOOL" -eq 1 ]; then
+    step "exporting $POOL (--export-zpool)"
+    run sudo zpool export "$POOL" && ok "exported — remember: zpool import $POOL after reboot" || fail "export failed"
+  fi
+fi
+
+hr
+if [ "$DRY_RUN" -eq 1 ]; then
+  echo "DRY RUN COMPLETE — nothing was actually stopped or unmounted."
+  echo "Re-run without --dry-run when ready to actually prep for shutdown."
+else
+  echo "SAFE TO POWER OFF — run: sudo shutdown -h now"
+fi
+hr

systemd/claw-fuse.service

@@ -0,0 +1,17 @@
+[Unit]
+Description=clawstor FUSE mount
+After=claw-store.service
+Requires=claw-store.service
+
+[Service]
+Type=simple
+User=osobh
+Environment=PATH=/home/osobh/.cargo/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin
+ExecStartPre=/bin/mkdir -p /home/osobh/clawstor-mount
+ExecStart=/usr/local/bin/claw-fuse --data-dir /home/osobh/clawstor-deploy/data --mount /home/osobh/clawstor-mount
+ExecStop=/bin/fusermount -u /home/osobh/clawstor-mount
+Restart=on-failure
+RestartSec=30
+
+[Install]
+WantedBy=multi-user.target

systemd/claw-store-serve.service

@@ -11,7 +11,7 @@ Wants=network-online.target
 [Service]
 Type=simple
 User=osobh
-ExecStart=/usr/local/bin/claw-store serve --port 7700 --static-dir /usr/share/claw-store/static
+ExecStart=/usr/local/bin/claw-store serve --port 7700 --static-dir /usr/share/claw-store/static --v2-static-dir /usr/share/claw-store/v2
 Restart=on-failure
 RestartSec=15
 Environment=RUST_LOG=info

systemd/claw-store.service

@@ -1,15 +1,19 @@
 [Unit]
-Description=claw-store fleet storage daemon
+Description=clawstor cluster daemon (gossip + QUIC blob store + ZFS snapshots)
-After=zfs-mount.service network.target
+After=network-online.target
-Wants=zfs-mount.service
+Wants=network-online.target
 
 [Service]
 Type=simple
 User=osobh
+Environment=PATH=/home/osobh/.cargo/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin
+Environment=RUST_LOG=info
 ExecStart=/usr/local/bin/claw-store daemon
 Restart=on-failure
-RestartSec=30
+RestartSec=15
-Environment=RUST_LOG=info
+TimeoutStopSec=60
+ProtectSystem=strict
+ReadWritePaths=/var/lib/claw-store /hot/targets /slab/projects /home/osobh/clawstor-deploy
 
 [Install]
 WantedBy=multi-user.target