Recover replication baseline from remote when locally pruned

Found during a routine fleet health sweep: tank's daily replication
to architect had silently broken. replicate_to_cold() records the
last-replicated snapshot name in a local state file and reuses it as
the incremental send base next time -- but never handles the case
where local snapshot retention (snapshot_retain_hours = 24) prunes
that exact snapshot before the next replication run. When that
happens it silently falls back to a FULL send, which then hard-fails
against a non-empty destination ("must destroy them to overwrite
it").

Root cause of the timing gap: the daemon's replication tick is a
24-hour interval, same order of magnitude as local retention. Every
claw-store.service restart resets that tick's countdown without
resetting the hourly-snapshot pruning tick, so a day of frequent
restarts (routine during active deployment work) is enough for the
two to drift out of sync -- the recorded baseline ages out locally
before replication ever gets to reuse it.

Fix: when the recorded baseline is gone, query the remote's actual
snapshot list over SSH (new ZfsOps::list_remote_snapshots) and find
the newest snapshot both sides still share by tag, rather than
giving up and attempting a full send. Only truly falls back to full
when no common snapshot exists anywhere. Verified live against tank
-> architect: correctly recovered daily-2026-08-01-0000 as the base
and completed an incremental send.

Added test coverage for all three paths (recorded baseline present,
recovered from remote, no common snapshot found) -- replicate_to_cold
had none before this, since LAST_REPLICATED_PATH was a hardcoded
absolute path with no way to inject a test double. Split into a
public wrapper plus replicate_to_cold_with_state_path() so tests can
use a temp file instead of touching real /var/lib/claw-store state.

Co-Authored-By: Claude Sonnet 5 <[email protected]>

claw-store/src/cluster/shutdown_prep.rs

@@ -87,16 +87,24 @@ pub async fn check() -> Result<ShutdownPrepCheckReply> {
     if !script.exists() {
         bail!("shutdown-prep script not found at {}", script.display());
     }
+    // `2>&1` inside the shell merges stderr into stdout *before*
+    // either stream is piped back to us, preserving true
+    // chronological order. Capturing stdout/stderr separately (as
+    // `Command::output()` does by default) and concatenating them
+    // after the fact loses interleaving entirely — every stderr line
+    // lands at the very end regardless of when it was actually
+    // printed, which makes a mid-script warning (e.g. "replicate not
+    // configured on this node") look like a failure that happened
+    // after "DRY RUN COMPLETE".
     let run = Command::new("bash")
-        .arg(&script)
-        .arg("--dry-run")
+        .arg("-c")
+        .arg(format!("{} --dry-run 2>&1", script.display()))
         .output();
     let output = timeout(CHECK_TIMEOUT, run)
         .await
         .context("shutdown-prep --dry-run timed out")?
         .context("spawning shutdown-prep --dry-run")?;
-    let mut combined = String::from_utf8_lossy(&output.stdout).into_owned();
-    combined.push_str(&String::from_utf8_lossy(&output.stderr));
+    let combined = String::from_utf8_lossy(&output.stdout).into_owned();
     Ok(ShutdownPrepCheckReply {
         ready: output.status.success(),
         output: combined,

claw-store/src/config.rs

@@ -27,12 +27,26 @@ pub struct HotConfig {
 #[derive(Debug, Clone, Deserialize, Serialize)]
 pub struct WarmConfig {
     pub projects_path: PathBuf,
+    /// Dataset name for `zfs`/`zpool` operations against the warm
+    /// tier. The literal value `"none"` means this node's warm tier
+    /// is a plain directory, not ZFS-backed (e.g. a build node with
+    /// no ZFS pool) — snapshot/replicate become no-ops instead of
+    /// erroring on a missing `zfs`/`zpool` binary. See
+    /// [`WarmConfig::zfs_enabled`].
     pub zfs_dataset: String,
     pub snapshot_retain_hours: u64,
     pub snapshot_retain_days: u64,
     pub snapshot_retain_weeks: u64,
 }
 
+impl WarmConfig {
+    /// `false` when `zfs_dataset = "none"` — this node's warm tier
+    /// has no ZFS pool underneath it.
+    pub fn zfs_enabled(&self) -> bool {
+        self.zfs_dataset != "none"
+    }
+}
+
 #[derive(Debug, Clone, Deserialize, Serialize)]
 pub struct ColdConfig {
     pub archive_path: PathBuf,

claw-store/src/daemon.rs

@@ -188,19 +188,25 @@ pub async fn run(cfg: Config, mut manifest: Manifest) -> Result<()> {
                 }
             }
             _ = snap_tick.tick() => {
-                let ts = chrono::Utc::now().format("%Y-%m-%d-%H%M").to_string();
-                tracing::info!("taking snapshot {}", ts);
-                if let Err(e) = snapshot::run_snapshot_cycle(
-                    &zfs, &cfg.warm.zfs_dataset, &ts,
-                    cfg.warm.snapshot_retain_hours as usize,
-                    cfg.warm.snapshot_retain_days as usize,
-                    cfg.warm.snapshot_retain_weeks as usize,
-                ) {
-                    tracing::error!("snapshot failed: {:#}", e);
+                if !cfg.warm.zfs_enabled() {
+                    tracing::debug!("skipping snapshot tick — zfs_dataset = \"none\" on this node");
+                } else {
+                    let ts = chrono::Utc::now().format("%Y-%m-%d-%H%M").to_string();
+                    tracing::info!("taking snapshot {}", ts);
+                    if let Err(e) = snapshot::run_snapshot_cycle(
+                        &zfs, &cfg.warm.zfs_dataset, &ts,
+                        cfg.warm.snapshot_retain_hours as usize,
+                        cfg.warm.snapshot_retain_days as usize,
+                        cfg.warm.snapshot_retain_weeks as usize,
+                    ) {
+                        tracing::error!("snapshot failed: {:#}", e);
+                    }
                 }
             }
             _ = repl_tick.tick() => {
-                if let Some(rep) = &cfg.replication {
+                if !cfg.warm.zfs_enabled() {
+                    tracing::debug!("skipping replication tick — zfs_dataset = \"none\" on this node");
+                } else if let Some(rep) = &cfg.replication {
                     if let (Some(host), Some(user), Some(dest)) = (
                         &rep.send_to_host, &rep.send_to_user, &rep.cold_dataset_on_peer
                     ) {

claw-store/src/main.rs

@@ -1502,6 +1502,10 @@ fn cmd_gc(cfg: &Config, manifest: &mut Manifest) -> Result<()> {
 }
 
 fn cmd_snapshot(cfg: &Config, zfs: &SystemZfs) -> Result<()> {
+    if !cfg.warm.zfs_enabled() {
+        println!("Warm tier is not ZFS-backed on this node (zfs_dataset = \"none\") — nothing to snapshot.");
+        return Ok(());
+    }
     let ts = chrono::Utc::now().format("%Y-%m-%d-%H%M").to_string();
     snapshot::run_snapshot_cycle(
         zfs, &cfg.warm.zfs_dataset, &ts,
@@ -1514,6 +1518,10 @@ fn cmd_snapshot(cfg: &Config, zfs: &SystemZfs) -> Result<()> {
 }
 
 fn cmd_list_snapshots(cfg: &Config, zfs: &SystemZfs, _project: &str) -> Result<()> {
+    if !cfg.warm.zfs_enabled() {
+        println!("Warm tier is not ZFS-backed on this node (zfs_dataset = \"none\").");
+        return Ok(());
+    }
     let snaps = zfs.list_snapshots(&cfg.warm.zfs_dataset)?;
     if snaps.is_empty() { println!("No snapshots found."); return Ok(()); }
     for s in &snaps { println!("  {}", s); }
@@ -1531,6 +1539,10 @@ fn cmd_restore(cfg: &Config, zfs: &SystemZfs, project: &str, snap: &str) -> Resu
 }
 
 fn cmd_replicate(cfg: &Config, zfs: &SystemZfs) -> Result<()> {
+    if !cfg.warm.zfs_enabled() {
+        println!("Warm tier is not ZFS-backed on this node (zfs_dataset = \"none\") — nothing to replicate.");
+        return Ok(());
+    }
     let rep = cfg.replication.as_ref()
         .context("no replication config — this node does not replicate")?;
     let host = rep.send_to_host.as_ref().context("send_to_host not set")?;

claw-store/src/snapshot.rs

@@ -66,6 +66,23 @@ pub fn replicate_to_cold(
     remote_user: &str,
     remote_host: &str,
     remote_dataset: &str,
+) -> Result<()> {
+    replicate_to_cold_with_state_path(
+        zfs, dataset, remote_user, remote_host, remote_dataset,
+        std::path::Path::new(LAST_REPLICATED_PATH),
+    )
+}
+
+/// Same as [`replicate_to_cold`] with the state-file path injectable
+/// -- lets tests exercise the baseline-recovery logic against a temp
+/// file instead of the real `/var/lib/claw-store/...` path.
+pub fn replicate_to_cold_with_state_path(
+    zfs: &dyn ZfsOps,
+    dataset: &str,
+    remote_user: &str,
+    remote_host: &str,
+    remote_dataset: &str,
+    state_path: &std::path::Path,
 ) -> Result<()> {
     let snaps = zfs.list_snapshots(dataset)?;
     let latest = match snaps.last().cloned() {
@@ -77,12 +94,64 @@ pub fn replicate_to_cold(
     };
 
     // Use the last successfully replicated snapshot as the incremental base.
-    // Only valid if it still exists in the current snapshot list.
-    let prev = std::fs::read_to_string(LAST_REPLICATED_PATH)
+    // Only valid if it still exists in the current snapshot list -- local
+    // retention (snapshot_retain_hours) can prune it out from under us
+    // between replication runs (e.g. a daemon restart resets the
+    // replication tick's 24h timer without resetting the hourly-snapshot
+    // pruning tick, so a slow/interrupted replication cadence can let the
+    // recorded snapshot age out locally before it's ever used again).
+    let recorded = std::fs::read_to_string(state_path)
         .ok()
         .map(|s| s.trim().to_string())
         .filter(|s| !s.is_empty() && snaps.contains(s));
 
+    // Recorded snapshot is gone -- don't give up and fall back to a full
+    // send (which fails outright against a non-empty destination, as
+    // opposed to just being wasteful). Ask the remote what it actually
+    // has and find the newest snapshot both sides still share, by tag
+    // (the `@kind-timestamp` suffix -- dataset paths differ between
+    // source and destination, e.g. slab/projects vs
+    // data/archive/tank-projects, but tags are written identically).
+    let prev = match recorded {
+        Some(p) => Some(p),
+        None => {
+            match zfs.list_remote_snapshots(remote_user, remote_host, remote_dataset) {
+                Ok(remote_snaps) => {
+                    let remote_tags: std::collections::HashSet<&str> = remote_snaps
+                        .iter()
+                        .filter_map(|s| s.split_once('@').map(|(_, tag)| tag))
+                        .collect();
+                    let fallback = snaps
+                        .iter()
+                        .rev()
+                        .skip(1) // exclude `latest` itself
+                        .find(|s| {
+                            s.split_once('@')
+                                .map(|(_, tag)| remote_tags.contains(tag))
+                                .unwrap_or(false)
+                        })
+                        .cloned();
+                    if let Some(f) = &fallback {
+                        tracing::warn!(
+                            "recorded replication baseline was pruned locally; \
+                             recovered a common snapshot from the remote instead: {}",
+                            f
+                        );
+                    }
+                    fallback
+                }
+                Err(e) => {
+                    tracing::warn!(
+                        error = %e,
+                        "could not query remote snapshots to recover a replication baseline; \
+                         falling back to full send"
+                    );
+                    None
+                }
+            }
+        }
+    };
+
     if prev.as_deref() == Some(latest.as_str()) {
         tracing::info!("replication already up to date ({})", latest);
         return Ok(());
@@ -99,10 +168,10 @@ pub fn replicate_to_cold(
     zfs.send_to_remote(&latest, prev.as_deref(), remote_user, remote_host, remote_dataset)?;
 
     // Record this snapshot as the new baseline for the next incremental send.
-    if let Some(parent) = std::path::Path::new(LAST_REPLICATED_PATH).parent() {
+    if let Some(parent) = state_path.parent() {
         let _ = std::fs::create_dir_all(parent);
     }
-    std::fs::write(LAST_REPLICATED_PATH, &latest)?;
+    std::fs::write(state_path, &latest)?;
 
     Ok(())
 }
@@ -177,4 +246,77 @@ mod tests {
         assert!(!is_sunday_midnight("2026-06-29-0000")); // Monday
         assert!(!is_sunday_midnight("2026-06-28-0100")); // Sunday but not midnight
     }
+
+    /// Unique temp path per test so parallel test runs don't clobber
+    /// each other's replication-baseline state file.
+    fn temp_state_path(name: &str) -> std::path::PathBuf {
+        std::env::temp_dir().join(format!("clawstor-test-last-replicated-{name}-{}", std::process::id()))
+    }
+
+    #[test]
+    fn replicate_uses_recorded_baseline_when_still_present() {
+        let zfs = MockZfs::default();
+        zfs.snapshot("slab/projects", "hourly-1").unwrap();
+        zfs.snapshot("slab/projects", "hourly-2").unwrap();
+        let state = temp_state_path("recorded-present");
+        std::fs::write(&state, "slab/projects@hourly-1").unwrap();
+
+        replicate_to_cold_with_state_path(
+            &zfs, "slab/projects", "user", "host", "remote/ds", &state,
+        ).unwrap();
+
+        let sends = zfs.sends();
+        assert_eq!(sends.len(), 1);
+        assert_eq!(sends[0].0, "slab/projects@hourly-2");
+        assert_eq!(sends[0].1.as_deref(), Some("slab/projects@hourly-1"));
+        let _ = std::fs::remove_file(&state);
+    }
+
+    #[test]
+    fn replicate_recovers_baseline_from_remote_when_recorded_one_was_pruned() {
+        let zfs = MockZfs::default();
+        // Local retention already pruned hourly-1 -- only hourly-2 and
+        // hourly-3 remain locally. The remote, however, still has
+        // hourly-2 (it just hasn't received hourly-3 yet).
+        zfs.snapshot("slab/projects", "hourly-2").unwrap();
+        zfs.snapshot("slab/projects", "hourly-3").unwrap();
+        zfs.set_remote_snapshots(vec![
+            "remote/ds@hourly-1".to_string(),
+            "remote/ds@hourly-2".to_string(),
+        ]);
+        let state = temp_state_path("recovers-from-remote");
+        // Recorded baseline (hourly-1) no longer exists locally.
+        std::fs::write(&state, "slab/projects@hourly-1").unwrap();
+
+        replicate_to_cold_with_state_path(
+            &zfs, "slab/projects", "user", "host", "remote/ds", &state,
+        ).unwrap();
+
+        let sends = zfs.sends();
+        assert_eq!(sends.len(), 1);
+        assert_eq!(sends[0].0, "slab/projects@hourly-3");
+        // Recovered hourly-2 as the base by matching tags against the
+        // remote's actual snapshot list, NOT a full send.
+        assert_eq!(sends[0].1.as_deref(), Some("slab/projects@hourly-2"));
+        let _ = std::fs::remove_file(&state);
+    }
+
+    #[test]
+    fn replicate_falls_back_to_full_send_when_truly_no_common_snapshot() {
+        let zfs = MockZfs::default();
+        zfs.snapshot("slab/projects", "hourly-9").unwrap();
+        zfs.set_remote_snapshots(vec!["remote/ds@hourly-1".to_string()]);
+        let state = temp_state_path("no-common-snapshot");
+        let _ = std::fs::remove_file(&state); // no recorded baseline at all
+
+        replicate_to_cold_with_state_path(
+            &zfs, "slab/projects", "user", "host", "remote/ds", &state,
+        ).unwrap();
+
+        let sends = zfs.sends();
+        assert_eq!(sends.len(), 1);
+        assert_eq!(sends[0].0, "slab/projects@hourly-9");
+        assert_eq!(sends[0].1, None, "no common tag exists -- must fall back to full send");
+        let _ = std::fs::remove_file(&state);
+    }
 }

claw-store/src/zfs.rs

@@ -13,6 +13,12 @@ pub trait ZfsOps: Send + Sync {
                       remote_user: &str, remote_host: &str,
                       remote_dataset: &str) -> Result<()>;
     fn clone_snapshot(&self, snapshot: &str, dest_dataset: &str) -> Result<()>;
+    /// List snapshot names (full `dataset@tag` form, remote-side
+    /// naming) currently on a remote dataset over SSH. Used to find a
+    /// real incremental base when the locally-recorded one has been
+    /// pruned -- see `snapshot::replicate_to_cold`.
+    fn list_remote_snapshots(&self, remote_user: &str, remote_host: &str,
+                              remote_dataset: &str) -> Result<Vec<String>>;
 }
 
 pub struct SystemZfs;
@@ -89,11 +95,48 @@ impl ZfsOps for SystemZfs {
         }
         Ok(())
     }
+
+    fn list_remote_snapshots(&self, remote_user: &str, remote_host: &str,
+                              remote_dataset: &str) -> Result<Vec<String>> {
+        let out = std::process::Command::new("ssh")
+            .args([
+                &format!("{remote_user}@{remote_host}"),
+                "zfs", "list", "-H", "-t", "snapshot", "-o", "name", "-r", remote_dataset,
+            ])
+            .output()
+            .context("running ssh zfs list on remote")?;
+        if !out.status.success() {
+            bail!("remote zfs list failed: {}", String::from_utf8_lossy(&out.stderr));
+        }
+        Ok(String::from_utf8_lossy(&out.stdout)
+            .lines()
+            .filter(|l| !l.is_empty())
+            .map(String::from)
+            .collect())
+    }
 }
 
 #[derive(Default, Clone)]
 pub struct MockZfs {
     snapshots: Arc<Mutex<Vec<String>>>,
+    /// Configurable via `set_remote_snapshots` -- what
+    /// `list_remote_snapshots` returns, for exercising the
+    /// baseline-recovery path in `snapshot::replicate_to_cold`.
+    remote_snapshots: Arc<Mutex<Vec<String>>>,
+    /// Every `send_to_remote` call, recorded as `(snapshot,
+    /// incremental_from)`, so tests can assert which base was
+    /// actually used.
+    sends: Arc<Mutex<Vec<(String, Option<String>)>>>,
+}
+
+impl MockZfs {
+    pub fn set_remote_snapshots(&self, snaps: Vec<String>) {
+        *self.remote_snapshots.lock().unwrap() = snaps;
+    }
+
+    pub fn sends(&self) -> Vec<(String, Option<String>)> {
+        self.sends.lock().unwrap().clone()
+    }
 }
 
 impl ZfsOps for MockZfs {
@@ -115,8 +158,9 @@ impl ZfsOps for MockZfs {
         Ok(())
     }
 
-    fn send_to_remote(&self, _snap: &str, _incr: Option<&str>,
+    fn send_to_remote(&self, snap: &str, incr: Option<&str>,
                       _user: &str, _host: &str, _dest: &str) -> Result<()> {
+        self.sends.lock().unwrap().push((snap.to_string(), incr.map(String::from)));
         Ok(())
     }
 
@@ -125,6 +169,11 @@ impl ZfsOps for MockZfs {
             .push(format!("{} -> {}", snapshot, dest_dataset));
         Ok(())
     }
+
+    fn list_remote_snapshots(&self, _remote_user: &str, _remote_host: &str,
+                              _remote_dataset: &str) -> Result<Vec<String>> {
+        Ok(self.remote_snapshots.lock().unwrap().clone())
+    }
 }
 
 #[cfg(test)]

dashboard-v2/src/components/NodeCard.js

@@ -24,7 +24,7 @@ export function NodeCard({ node }) {
                 border,
             ].join(' '), children: [_jsxs("div", { className: "flex items-center justify-between", children: [_jsxs("div", { className: "flex items-center gap-2", children: [_jsx("span", { className: `inline-block w-2.5 h-2.5 rounded-full ${dot}` }), _jsx("span", { className: "text-lg font-semibold text-slate-100", children: node.node_name })] }), _jsx("span", { className: "text-xs text-slate-500 font-mono", children: node.zone || '—' })] }), !node.online && (_jsx("div", { className: "text-sm text-red-400 break-words", children: node.error ?? 'offline' })), node.online && (_jsxs(_Fragment, { children: [node.filesystem && (_jsx(StorageBar, { label: "disk", used: node.filesystem.used_bytes, total: node.filesystem.total_bytes })), node.hot && node.hot.max_bytes > 0 && (_jsx(StorageBar, { label: "hot tier", used: node.hot.used_bytes, total: node.hot.max_bytes, pinned: node.hot.pinned_bytes ?? undefined })), _jsxs("div", { className: "grid grid-cols-2 gap-y-1 text-sm", children: [_jsx("span", { className: "text-slate-500", children: "mount" }), _jsx("span", { className: "text-right font-mono text-xs", children: node.mount?.active ? (_jsx("span", { className: "text-emerald-300", children: "\u2713 mounted" })) : (_jsx("span", { className: "text-slate-500", children: "not mounted" })) }), _jsx("span", { className: "text-slate-500", children: "cache hit rate" }), _jsx("span", { className: "text-right font-mono text-xs", children: node.cache && node.cache.hits + node.cache.misses > 0
                                         ? `${Math.round(node.cache.hit_rate * 100)}%`
-                                        : _jsx("span", { className: "text-slate-500", children: "idle" }) }), _jsx("span", { className: "text-slate-500", children: "next scheduled job" }), _jsx("span", { className: "text-right font-mono text-xs", children: nextTimer(node) })] })] }))] }) }));
+                                        : _jsx("span", { className: "text-slate-500", children: "idle" }) }), _jsx("span", { className: "text-slate-500", children: "next scheduled job" }), _jsx("span", { className: "text-right font-mono text-xs", children: nextTimer(node) })] })] })), _jsx("div", { className: "pt-1 border-t border-slate-800 text-xs text-slate-500", children: "view detail \u00B7 maintenance & shutdown prep \u2192" })] }) }));
 }
 function healthOf(n) {
     if (!n.online)

dashboard-v2/src/components/NodeCard.tsx

@@ -95,6 +95,10 @@ export function NodeCard({ node }: Props) {
             </div>
           </>
         )}
+
+        <div className="pt-1 border-t border-slate-800 text-xs text-slate-500">
+          view detail · maintenance & shutdown prep →
+        </div>
       </a>
     </Link>
   );

deploy/scripts/safe-shutdown-prep.sh

@@ -211,13 +211,20 @@ step "flushing filesystem buffers"
 sync
 ok "sync complete"
 
-POOL=$(df --output=source /slab 2>/dev/null | tail -1 | tr -d '[:space:]')
-if [ -n "$POOL" ] && [ "$POOL" != "none" ]; then
-  step "zpool health check ($POOL)"
-  zpool status -x "$POOL" 2>&1 | sed 's/^/  /'
-  if [ "$EXPORT_ZPOOL" -eq 1 ]; then
-    step "exporting $POOL (--export-zpool)"
-    run sudo zpool export "$POOL" && ok "exported — remember: zpool import $POOL after reboot" || fail "export failed"
+step "zpool health check"
+if ! command -v zpool >/dev/null 2>&1; then
+  ok "no zpool binary on this node — warm tier is not ZFS-backed here, nothing to check"
+else
+  POOL=$(df --output=source /slab 2>/dev/null | tail -1 | tr -d '[:space:]')
+  if [ -n "$POOL" ] && [ "$POOL" != "none" ]; then
+    echo "  pool: $POOL"
+    zpool status -x "$POOL" 2>&1 | sed 's/^/  /'
+    if [ "$EXPORT_ZPOOL" -eq 1 ]; then
+      step "exporting $POOL (--export-zpool)"
+      run sudo zpool export "$POOL" && ok "exported — remember: zpool import $POOL after reboot" || fail "export failed"
+    fi
+  else
+    warn "zpool present but /slab isn't a recognizable ZFS mount"
   fi
 fi