claw-store/src/snapshot.rs

@@ -66,6 +66,23 @@ pub fn replicate_to_cold(
     remote_user: &str,
     remote_host: &str,
     remote_dataset: &str,
+) -> Result<()> {
+    replicate_to_cold_with_state_path(
+        zfs, dataset, remote_user, remote_host, remote_dataset,
+        std::path::Path::new(LAST_REPLICATED_PATH),
+    )
+}
+
+/// Same as [`replicate_to_cold`] with the state-file path injectable
+/// -- lets tests exercise the baseline-recovery logic against a temp
+/// file instead of the real `/var/lib/claw-store/...` path.
+pub fn replicate_to_cold_with_state_path(
+    zfs: &dyn ZfsOps,
+    dataset: &str,
+    remote_user: &str,
+    remote_host: &str,
+    remote_dataset: &str,
+    state_path: &std::path::Path,
 ) -> Result<()> {
     let snaps = zfs.list_snapshots(dataset)?;
     let latest = match snaps.last().cloned() {
@@ -77,12 +94,64 @@ pub fn replicate_to_cold(
     };
 
     // Use the last successfully replicated snapshot as the incremental base.
-    // Only valid if it still exists in the current snapshot list.
+    // Only valid if it still exists in the current snapshot list -- local
-    let prev = std::fs::read_to_string(LAST_REPLICATED_PATH)
+    // retention (snapshot_retain_hours) can prune it out from under us
+    // between replication runs (e.g. a daemon restart resets the
+    // replication tick's 24h timer without resetting the hourly-snapshot
+    // pruning tick, so a slow/interrupted replication cadence can let the
+    // recorded snapshot age out locally before it's ever used again).
+    let recorded = std::fs::read_to_string(state_path)
         .ok()
         .map(|s| s.trim().to_string())
         .filter(|s| !s.is_empty() && snaps.contains(s));
 
+    // Recorded snapshot is gone -- don't give up and fall back to a full
+    // send (which fails outright against a non-empty destination, as
+    // opposed to just being wasteful). Ask the remote what it actually
+    // has and find the newest snapshot both sides still share, by tag
+    // (the `@kind-timestamp` suffix -- dataset paths differ between
+    // source and destination, e.g. slab/projects vs
+    // data/archive/tank-projects, but tags are written identically).
+    let prev = match recorded {
+        Some(p) => Some(p),
+        None => {
+            match zfs.list_remote_snapshots(remote_user, remote_host, remote_dataset) {
+                Ok(remote_snaps) => {
+                    let remote_tags: std::collections::HashSet<&str> = remote_snaps
+                        .iter()
+                        .filter_map(|s| s.split_once('@').map(|(_, tag)| tag))
+                        .collect();
+                    let fallback = snaps
+                        .iter()
+                        .rev()
+                        .skip(1) // exclude `latest` itself
+                        .find(|s| {
+                            s.split_once('@')
+                                .map(|(_, tag)| remote_tags.contains(tag))
+                                .unwrap_or(false)
+                        })
+                        .cloned();
+                    if let Some(f) = &fallback {
+                        tracing::warn!(
+                            "recorded replication baseline was pruned locally; \
+                             recovered a common snapshot from the remote instead: {}",
+                            f
+                        );
+                    }
+                    fallback
+                }
+                Err(e) => {
+                    tracing::warn!(
+                        error = %e,
+                        "could not query remote snapshots to recover a replication baseline; \
+                         falling back to full send"
+                    );
+                    None
+                }
+            }
+        }
+    };
+
     if prev.as_deref() == Some(latest.as_str()) {
         tracing::info!("replication already up to date ({})", latest);
         return Ok(());
@@ -99,10 +168,10 @@ pub fn replicate_to_cold(
     zfs.send_to_remote(&latest, prev.as_deref(), remote_user, remote_host, remote_dataset)?;
 
     // Record this snapshot as the new baseline for the next incremental send.
-    if let Some(parent) = std::path::Path::new(LAST_REPLICATED_PATH).parent() {
+    if let Some(parent) = state_path.parent() {
         let _ = std::fs::create_dir_all(parent);
     }
-    std::fs::write(LAST_REPLICATED_PATH, &latest)?;
+    std::fs::write(state_path, &latest)?;
 
     Ok(())
 }
@@ -177,4 +246,77 @@ mod tests {
         assert!(!is_sunday_midnight("2026-06-29-0000")); // Monday
         assert!(!is_sunday_midnight("2026-06-28-0100")); // Sunday but not midnight
     }
+
+    /// Unique temp path per test so parallel test runs don't clobber
+    /// each other's replication-baseline state file.
+    fn temp_state_path(name: &str) -> std::path::PathBuf {
+        std::env::temp_dir().join(format!("clawstor-test-last-replicated-{name}-{}", std::process::id()))
+    }
+
+    #[test]
+    fn replicate_uses_recorded_baseline_when_still_present() {
+        let zfs = MockZfs::default();
+        zfs.snapshot("slab/projects", "hourly-1").unwrap();
+        zfs.snapshot("slab/projects", "hourly-2").unwrap();
+        let state = temp_state_path("recorded-present");
+        std::fs::write(&state, "slab/projects@hourly-1").unwrap();
+
+        replicate_to_cold_with_state_path(
+            &zfs, "slab/projects", "user", "host", "remote/ds", &state,
+        ).unwrap();
+
+        let sends = zfs.sends();
+        assert_eq!(sends.len(), 1);
+        assert_eq!(sends[0].0, "slab/projects@hourly-2");
+        assert_eq!(sends[0].1.as_deref(), Some("slab/projects@hourly-1"));
+        let _ = std::fs::remove_file(&state);
+    }
+
+    #[test]
+    fn replicate_recovers_baseline_from_remote_when_recorded_one_was_pruned() {
+        let zfs = MockZfs::default();
+        // Local retention already pruned hourly-1 -- only hourly-2 and
+        // hourly-3 remain locally. The remote, however, still has
+        // hourly-2 (it just hasn't received hourly-3 yet).
+        zfs.snapshot("slab/projects", "hourly-2").unwrap();
+        zfs.snapshot("slab/projects", "hourly-3").unwrap();
+        zfs.set_remote_snapshots(vec![
+            "remote/ds@hourly-1".to_string(),
+            "remote/ds@hourly-2".to_string(),
+        ]);
+        let state = temp_state_path("recovers-from-remote");
+        // Recorded baseline (hourly-1) no longer exists locally.
+        std::fs::write(&state, "slab/projects@hourly-1").unwrap();
+
+        replicate_to_cold_with_state_path(
+            &zfs, "slab/projects", "user", "host", "remote/ds", &state,
+        ).unwrap();
+
+        let sends = zfs.sends();
+        assert_eq!(sends.len(), 1);
+        assert_eq!(sends[0].0, "slab/projects@hourly-3");
+        // Recovered hourly-2 as the base by matching tags against the
+        // remote's actual snapshot list, NOT a full send.
+        assert_eq!(sends[0].1.as_deref(), Some("slab/projects@hourly-2"));
+        let _ = std::fs::remove_file(&state);
+    }
+
+    #[test]
+    fn replicate_falls_back_to_full_send_when_truly_no_common_snapshot() {
+        let zfs = MockZfs::default();
+        zfs.snapshot("slab/projects", "hourly-9").unwrap();
+        zfs.set_remote_snapshots(vec!["remote/ds@hourly-1".to_string()]);
+        let state = temp_state_path("no-common-snapshot");
+        let _ = std::fs::remove_file(&state); // no recorded baseline at all
+
+        replicate_to_cold_with_state_path(
+            &zfs, "slab/projects", "user", "host", "remote/ds", &state,
+        ).unwrap();
+
+        let sends = zfs.sends();
+        assert_eq!(sends.len(), 1);
+        assert_eq!(sends[0].0, "slab/projects@hourly-9");
+        assert_eq!(sends[0].1, None, "no common tag exists -- must fall back to full send");
+        let _ = std::fs::remove_file(&state);
+    }
 }

claw-store/src/zfs.rs

@@ -13,6 +13,12 @@ pub trait ZfsOps: Send + Sync {
                       remote_user: &str, remote_host: &str,
                       remote_dataset: &str) -> Result<()>;
     fn clone_snapshot(&self, snapshot: &str, dest_dataset: &str) -> Result<()>;
+    /// List snapshot names (full `dataset@tag` form, remote-side
+    /// naming) currently on a remote dataset over SSH. Used to find a
+    /// real incremental base when the locally-recorded one has been
+    /// pruned -- see `snapshot::replicate_to_cold`.
+    fn list_remote_snapshots(&self, remote_user: &str, remote_host: &str,
+                              remote_dataset: &str) -> Result<Vec<String>>;
 }
 
 pub struct SystemZfs;
@@ -89,11 +95,48 @@ impl ZfsOps for SystemZfs {
         }
         Ok(())
     }
+
+    fn list_remote_snapshots(&self, remote_user: &str, remote_host: &str,
+                              remote_dataset: &str) -> Result<Vec<String>> {
+        let out = std::process::Command::new("ssh")
+            .args([
+                &format!("{remote_user}@{remote_host}"),
+                "zfs", "list", "-H", "-t", "snapshot", "-o", "name", "-r", remote_dataset,
+            ])
+            .output()
+            .context("running ssh zfs list on remote")?;
+        if !out.status.success() {
+            bail!("remote zfs list failed: {}", String::from_utf8_lossy(&out.stderr));
+        }
+        Ok(String::from_utf8_lossy(&out.stdout)
+            .lines()
+            .filter(|l| !l.is_empty())
+            .map(String::from)
+            .collect())
+    }
 }
 
 #[derive(Default, Clone)]
 pub struct MockZfs {
     snapshots: Arc<Mutex<Vec<String>>>,
+    /// Configurable via `set_remote_snapshots` -- what
+    /// `list_remote_snapshots` returns, for exercising the
+    /// baseline-recovery path in `snapshot::replicate_to_cold`.
+    remote_snapshots: Arc<Mutex<Vec<String>>>,
+    /// Every `send_to_remote` call, recorded as `(snapshot,
+    /// incremental_from)`, so tests can assert which base was
+    /// actually used.
+    sends: Arc<Mutex<Vec<(String, Option<String>)>>>,
+}
+
+impl MockZfs {
+    pub fn set_remote_snapshots(&self, snaps: Vec<String>) {
+        *self.remote_snapshots.lock().unwrap() = snaps;
+    }
+
+    pub fn sends(&self) -> Vec<(String, Option<String>)> {
+        self.sends.lock().unwrap().clone()
+    }
 }
 
 impl ZfsOps for MockZfs {
@@ -115,8 +158,9 @@ impl ZfsOps for MockZfs {
         Ok(())
     }
 
-    fn send_to_remote(&self, _snap: &str, _incr: Option<&str>,
+    fn send_to_remote(&self, snap: &str, incr: Option<&str>,
                       _user: &str, _host: &str, _dest: &str) -> Result<()> {
+        self.sends.lock().unwrap().push((snap.to_string(), incr.map(String::from)));
         Ok(())
     }
 
@@ -125,6 +169,11 @@ impl ZfsOps for MockZfs {
             .push(format!("{} -> {}", snapshot, dest_dataset));
         Ok(())
     }
+
+    fn list_remote_snapshots(&self, _remote_user: &str, _remote_host: &str,
+                              _remote_dataset: &str) -> Result<Vec<String>> {
+        Ok(self.remote_snapshots.lock().unwrap().clone())
+    }
 }
 
 #[cfg(test)]