Recover replication baseline from remote when locally pruned

Found during a routine fleet health sweep: tank's daily replication
to architect had silently broken. replicate_to_cold() records the
last-replicated snapshot name in a local state file and reuses it as
the incremental send base next time -- but never handles the case
where local snapshot retention (snapshot_retain_hours = 24) prunes
that exact snapshot before the next replication run. When that
happens it silently falls back to a FULL send, which then hard-fails
against a non-empty destination ("must destroy them to overwrite
it").

Root cause of the timing gap: the daemon's replication tick is a
24-hour interval, same order of magnitude as local retention. Every
claw-store.service restart resets that tick's countdown without
resetting the hourly-snapshot pruning tick, so a day of frequent
restarts (routine during active deployment work) is enough for the
two to drift out of sync -- the recorded baseline ages out locally
before replication ever gets to reuse it.

Fix: when the recorded baseline is gone, query the remote's actual
snapshot list over SSH (new ZfsOps::list_remote_snapshots) and find
the newest snapshot both sides still share by tag, rather than
giving up and attempting a full send. Only truly falls back to full
when no common snapshot exists anywhere. Verified live against tank
-> architect: correctly recovered daily-2026-08-01-0000 as the base
and completed an incremental send.

Added test coverage for all three paths (recorded baseline present,
recovered from remote, no common snapshot found) -- replicate_to_cold
had none before this, since LAST_REPLICATED_PATH was a hardcoded
absolute path with no way to inject a test double. Split into a
public wrapper plus replicate_to_cold_with_state_path() so tests can
use a temp file instead of touching real /var/lib/claw-store state.

Co-Authored-By: Claude Sonnet 5 <[email protected]>

claw-store/src/snapshot.rs

@@ -66,6 +66,23 @@ pub fn replicate_to_cold(
     remote_user: &str,
     remote_host: &str,
     remote_dataset: &str,
+) -> Result<()> {
+    replicate_to_cold_with_state_path(
+        zfs, dataset, remote_user, remote_host, remote_dataset,
+        std::path::Path::new(LAST_REPLICATED_PATH),
+    )
+}
+
+/// Same as [`replicate_to_cold`] with the state-file path injectable
+/// -- lets tests exercise the baseline-recovery logic against a temp
+/// file instead of the real `/var/lib/claw-store/...` path.
+pub fn replicate_to_cold_with_state_path(
+    zfs: &dyn ZfsOps,
+    dataset: &str,
+    remote_user: &str,
+    remote_host: &str,
+    remote_dataset: &str,
+    state_path: &std::path::Path,
 ) -> Result<()> {
     let snaps = zfs.list_snapshots(dataset)?;
     let latest = match snaps.last().cloned() {
@@ -77,12 +94,64 @@ pub fn replicate_to_cold(
     };
 
     // Use the last successfully replicated snapshot as the incremental base.
-    // Only valid if it still exists in the current snapshot list.
-    let prev = std::fs::read_to_string(LAST_REPLICATED_PATH)
+    // Only valid if it still exists in the current snapshot list -- local
+    // retention (snapshot_retain_hours) can prune it out from under us
+    // between replication runs (e.g. a daemon restart resets the
+    // replication tick's 24h timer without resetting the hourly-snapshot
+    // pruning tick, so a slow/interrupted replication cadence can let the
+    // recorded snapshot age out locally before it's ever used again).
+    let recorded = std::fs::read_to_string(state_path)
         .ok()
         .map(|s| s.trim().to_string())
         .filter(|s| !s.is_empty() && snaps.contains(s));
 
+    // Recorded snapshot is gone -- don't give up and fall back to a full
+    // send (which fails outright against a non-empty destination, as
+    // opposed to just being wasteful). Ask the remote what it actually
+    // has and find the newest snapshot both sides still share, by tag
+    // (the `@kind-timestamp` suffix -- dataset paths differ between
+    // source and destination, e.g. slab/projects vs
+    // data/archive/tank-projects, but tags are written identically).
+    let prev = match recorded {
+        Some(p) => Some(p),
+        None => {
+            match zfs.list_remote_snapshots(remote_user, remote_host, remote_dataset) {
+                Ok(remote_snaps) => {
+                    let remote_tags: std::collections::HashSet<&str> = remote_snaps
+                        .iter()
+                        .filter_map(|s| s.split_once('@').map(|(_, tag)| tag))
+                        .collect();
+                    let fallback = snaps
+                        .iter()
+                        .rev()
+                        .skip(1) // exclude `latest` itself
+                        .find(|s| {
+                            s.split_once('@')
+                                .map(|(_, tag)| remote_tags.contains(tag))
+                                .unwrap_or(false)
+                        })
+                        .cloned();
+                    if let Some(f) = &fallback {
+                        tracing::warn!(
+                            "recorded replication baseline was pruned locally; \
+                             recovered a common snapshot from the remote instead: {}",
+                            f
+                        );
+                    }
+                    fallback
+                }
+                Err(e) => {
+                    tracing::warn!(
+                        error = %e,
+                        "could not query remote snapshots to recover a replication baseline; \
+                         falling back to full send"
+                    );
+                    None
+                }
+            }
+        }
+    };
+
     if prev.as_deref() == Some(latest.as_str()) {
         tracing::info!("replication already up to date ({})", latest);
         return Ok(());
@@ -99,10 +168,10 @@ pub fn replicate_to_cold(
     zfs.send_to_remote(&latest, prev.as_deref(), remote_user, remote_host, remote_dataset)?;
 
     // Record this snapshot as the new baseline for the next incremental send.
-    if let Some(parent) = std::path::Path::new(LAST_REPLICATED_PATH).parent() {
+    if let Some(parent) = state_path.parent() {
         let _ = std::fs::create_dir_all(parent);
     }
-    std::fs::write(LAST_REPLICATED_PATH, &latest)?;
+    std::fs::write(state_path, &latest)?;
 
     Ok(())
 }
@@ -177,4 +246,77 @@ mod tests {
         assert!(!is_sunday_midnight("2026-06-29-0000")); // Monday
         assert!(!is_sunday_midnight("2026-06-28-0100")); // Sunday but not midnight
     }
+
+    /// Unique temp path per test so parallel test runs don't clobber
+    /// each other's replication-baseline state file.
+    fn temp_state_path(name: &str) -> std::path::PathBuf {
+        std::env::temp_dir().join(format!("clawstor-test-last-replicated-{name}-{}", std::process::id()))
+    }
+
+    #[test]
+    fn replicate_uses_recorded_baseline_when_still_present() {
+        let zfs = MockZfs::default();
+        zfs.snapshot("slab/projects", "hourly-1").unwrap();
+        zfs.snapshot("slab/projects", "hourly-2").unwrap();
+        let state = temp_state_path("recorded-present");
+        std::fs::write(&state, "slab/projects@hourly-1").unwrap();
+
+        replicate_to_cold_with_state_path(
+            &zfs, "slab/projects", "user", "host", "remote/ds", &state,
+        ).unwrap();
+
+        let sends = zfs.sends();
+        assert_eq!(sends.len(), 1);
+        assert_eq!(sends[0].0, "slab/projects@hourly-2");
+        assert_eq!(sends[0].1.as_deref(), Some("slab/projects@hourly-1"));
+        let _ = std::fs::remove_file(&state);
+    }
+
+    #[test]
+    fn replicate_recovers_baseline_from_remote_when_recorded_one_was_pruned() {
+        let zfs = MockZfs::default();
+        // Local retention already pruned hourly-1 -- only hourly-2 and
+        // hourly-3 remain locally. The remote, however, still has
+        // hourly-2 (it just hasn't received hourly-3 yet).
+        zfs.snapshot("slab/projects", "hourly-2").unwrap();
+        zfs.snapshot("slab/projects", "hourly-3").unwrap();
+        zfs.set_remote_snapshots(vec![
+            "remote/ds@hourly-1".to_string(),
+            "remote/ds@hourly-2".to_string(),
+        ]);
+        let state = temp_state_path("recovers-from-remote");
+        // Recorded baseline (hourly-1) no longer exists locally.
+        std::fs::write(&state, "slab/projects@hourly-1").unwrap();
+
+        replicate_to_cold_with_state_path(
+            &zfs, "slab/projects", "user", "host", "remote/ds", &state,
+        ).unwrap();
+
+        let sends = zfs.sends();
+        assert_eq!(sends.len(), 1);
+        assert_eq!(sends[0].0, "slab/projects@hourly-3");
+        // Recovered hourly-2 as the base by matching tags against the
+        // remote's actual snapshot list, NOT a full send.
+        assert_eq!(sends[0].1.as_deref(), Some("slab/projects@hourly-2"));
+        let _ = std::fs::remove_file(&state);
+    }
+
+    #[test]
+    fn replicate_falls_back_to_full_send_when_truly_no_common_snapshot() {
+        let zfs = MockZfs::default();
+        zfs.snapshot("slab/projects", "hourly-9").unwrap();
+        zfs.set_remote_snapshots(vec!["remote/ds@hourly-1".to_string()]);
+        let state = temp_state_path("no-common-snapshot");
+        let _ = std::fs::remove_file(&state); // no recorded baseline at all
+
+        replicate_to_cold_with_state_path(
+            &zfs, "slab/projects", "user", "host", "remote/ds", &state,
+        ).unwrap();
+
+        let sends = zfs.sends();
+        assert_eq!(sends.len(), 1);
+        assert_eq!(sends[0].0, "slab/projects@hourly-9");
+        assert_eq!(sends[0].1, None, "no common tag exists -- must fall back to full send");
+        let _ = std::fs::remove_file(&state);
+    }
 }