perf: manifest state cache — skip re-hashing unchanged files

Adds a .clawsync.state file in each watched directory that stores
(mtime_ns, size, blake3) per file. On subsequent manifest builds,
files whose nanosecond-precision mtime and size are unchanged reuse
the cached BLAKE3 without reading file content, making re-syncs over
large directories O(changed files) rather than O(total bytes).

- Cache format: tab-separated text, one entry per line (easy to inspect)
- Written atomically via .clawsync.state.tmp rename
- Both cache files excluded from manifests (never transferred to peers)
- Nanosecond mtime catches same-size, sub-second content changes
  (avoids the 1-second resolution hazard that trips up rsync)
- 4 new unit tests: cache exclusion, cache hit, cache miss on change,
  hex roundtrip

Also:
- Fix clippy: &PathBuf -> &Path in watch_sync_once
- assert_dir_equal in integration tests now skips .clawsync.state and
  *.tmp.clawsync files (internal clawsync files, not user data)

Co-Authored-By: Claude Sonnet 4.6 <[email protected]>

crates/clawsync-cli/src/main.rs

@@ -1357,7 +1357,7 @@ async fn cmd_watch(
 
 /// Connect and run one sync cycle, printing the outcome.
 async fn watch_sync_once(
-    local: &PathBuf,
+    local: &Path,
     excludes: &globset::GlobSet,
     delete: bool,
     addr: std::net::SocketAddr,
@@ -1378,7 +1378,7 @@ async fn watch_sync_once(
                 .with_context(|| format!("cannot connect to {addr}"))?,
         )
     };
-    let stats = FsSyncClient::new(peer, local.clone(), excludes.clone(), delete)
+    let stats = FsSyncClient::new(peer, local.to_path_buf(), excludes.clone(), delete)
         .run()
         .await?;
     let wire = stats.bytes_transferred;

crates/clawsync-cli/tests/fs_sync.rs

@@ -283,6 +283,11 @@ fn assert_dir_equal(src: &Path, dst: &Path) {
             continue;
         }
         let rel = entry.path().strip_prefix(src).unwrap();
+        let rel_str = rel.to_string_lossy();
+        // Skip clawsync-internal files: state cache and interrupted-sync temps.
+        if rel_str == ".clawsync.state" || rel_str.ends_with(".tmp.clawsync") {
+            continue;
+        }
         let dst_path = dst.join(rel);
         assert!(dst_path.exists(), "missing in dst: {}", rel.display());
         assert_files_equal(entry.path(), &dst_path);

crates/clawsync-fs/src/manifest.rs

@@ -1,5 +1,21 @@
 //! Directory manifest: parallel BLAKE3 walk over any directory tree.
+//!
+//! ## State cache
+//!
+//! On every call to `FsManifest::build()` a `.clawsync.state` file is read
+//! from the root directory and written back after the walk completes.  Entries
+//! whose `(mtime, size)` match the cache are assumed unchanged: the BLAKE3 is
+//! reused without reading the file content, so re-syncs over large directories
+//! are proportional to the number of *changed* files rather than total bytes.
+//!
+//! The cache file uses a simple tab-separated text format:
+//! ```text
+//! <rel_path>\t<mtime_unix_secs>\t<size_bytes>\t<blake3_hex64>
+//! ```
+//! It is written atomically (temp-file + rename) and excluded from manifests.
 
+use std::collections::HashMap;
+use std::io::{BufRead, Write};
 use std::path::{Path, PathBuf};
 use std::time::UNIX_EPOCH;
 
@@ -12,6 +28,104 @@ use clawsync_transport::protocol::FsManifestEntry;
 
 use crate::error::FsSyncError;
 
+// ─────────────────────────────────────────────────────────────────────────────
+// State cache
+// ─────────────────────────────────────────────────────────────────────────────
+
+const CACHE_FILE: &str = ".clawsync.state";
+const CACHE_TMP: &str = ".clawsync.state.tmp";
+
+#[derive(Clone)]
+struct CacheEntry {
+    /// Nanoseconds since UNIX epoch — higher precision than the wire mtime
+    /// (which is seconds) so same-size, sub-second content changes are caught.
+    mtime_ns: u64,
+    size: u64,
+    blake3: [u8; 32],
+}
+
+type CacheMap = HashMap<String, CacheEntry>;
+
+/// Load the state cache from `root/.clawsync.state`.
+/// Returns an empty map on any error or if the file is absent.
+fn load_cache(root: &Path) -> CacheMap {
+    let path = root.join(CACHE_FILE);
+    let Ok(file) = std::fs::File::open(&path) else {
+        return CacheMap::new();
+    };
+    let mut map = CacheMap::new();
+    for line in std::io::BufReader::new(file).lines() {
+        let Ok(line) = line else { break };
+        let parts: Vec<&str> = line.splitn(4, '\t').collect();
+        if parts.len() != 4 {
+            continue;
+        }
+        let rel = parts[0];
+        let Ok(mtime_ns) = parts[1].parse::<u64>() else { continue };
+        let Ok(size) = parts[2].parse::<u64>() else { continue };
+        let Some(blake3) = hex64_to_bytes(parts[3]) else { continue };
+        map.insert(rel.to_string(), CacheEntry { mtime_ns, size, blake3 });
+    }
+    map
+}
+
+/// Save the state cache to `root/.clawsync.state` atomically.
+/// Silently ignores errors — the cache is a performance optimisation only.
+fn save_cache(root: &Path, entries: &[(LocalEntry, u64)]) {
+    let tmp_path = root.join(CACHE_TMP);
+    let final_path = root.join(CACHE_FILE);
+    let Ok(mut file) = std::fs::File::create(&tmp_path) else { return };
+    for (e, mtime_ns) in entries {
+        let _ = writeln!(
+            file,
+            "{}\t{}\t{}\t{}",
+            e.rel_path,
+            mtime_ns,
+            e.size,
+            bytes_to_hex64(&e.blake3)
+        );
+    }
+    let _ = file.flush();
+    drop(file);
+    let _ = std::fs::rename(&tmp_path, &final_path);
+}
+
+fn bytes_to_hex64(bytes: &[u8; 32]) -> String {
+    let mut s = String::with_capacity(64);
+    for b in bytes {
+        s.push(char::from_digit((*b >> 4) as u32, 16).unwrap());
+        s.push(char::from_digit((*b & 0xf) as u32, 16).unwrap());
+    }
+    s
+}
+
+fn hex64_to_bytes(s: &str) -> Option<[u8; 32]> {
+    if s.len() != 64 {
+        return None;
+    }
+    let mut out = [0u8; 32];
+    let bytes = s.as_bytes();
+    for (i, slot) in out.iter_mut().enumerate() {
+        let hi = nibble(bytes[i * 2])?;
+        let lo = nibble(bytes[i * 2 + 1])?;
+        *slot = (hi << 4) | lo;
+    }
+    Some(out)
+}
+
+fn nibble(b: u8) -> Option<u8> {
+    match b {
+        b'0'..=b'9' => Some(b - b'0'),
+        b'a'..=b'f' => Some(b - b'a' + 10),
+        b'A'..=b'F' => Some(b - b'A' + 10),
+        _ => None,
+    }
+}
+
+// ─────────────────────────────────────────────────────────────────────────────
+// Manifest
+// ─────────────────────────────────────────────────────────────────────────────
+
 /// A single file entry in a local manifest.
 #[derive(Debug, Clone)]
 pub struct LocalEntry {
@@ -36,9 +150,16 @@ impl FsManifest {
     /// Walk `root`, skip paths matching `excludes`, compute BLAKE3 for every
     /// regular file in parallel (Rayon), and return a sorted manifest.
     ///
+    /// Files whose `(mtime, size)` match a previously-saved `.clawsync.state`
+    /// cache entry are assumed unchanged — the cached BLAKE3 is reused without
+    /// reading file content.  The cache is updated atomically before returning.
+    ///
     /// **Must be called inside `tokio::task::spawn_blocking`** because this
     /// function blocks the calling thread (Rayon + synchronous file I/O).
     pub fn build(root: &Path, excludes: &GlobSet) -> Result<Self, FsSyncError> {
+        // Load the state cache; empty on first run or cache miss.
+        let cache = load_cache(root);
+
         // Collect (abs_path, rel_path) pairs with a single-threaded walk.
         // The walk itself is fast (metadata-only); hashing is parallelised below.
         let mut candidates: Vec<(PathBuf, String)> = Vec::new();
@@ -60,40 +181,68 @@ impl FsManifest {
             if !rel.is_empty() && excludes.is_match(&rel) {
                 continue;
             }
-            // Always exclude clawsync temp files regardless of caller excludes.
+            // Always exclude clawsync internal files regardless of caller excludes.
-            // These are leftover from interrupted syncs and must never appear
+            if rel.ends_with(".tmp.clawsync")
-            // in manifests or be transferred to peers.
+                || rel == CACHE_FILE
-            if rel.ends_with(".tmp.clawsync") {
+                || rel == CACHE_TMP
+            {
                 continue;
             }
             candidates.push((abs, rel));
         }
 
-        // Hash all files in parallel.
+        // Hash all files in parallel, using the cache to skip unchanged files.
-        let mut entries: Vec<LocalEntry> = candidates
+        // Each item carries (LocalEntry, mtime_ns) where mtime_ns is the
+        // nanosecond-precision mtime used for cache validation (finer than the
+        // wire mtime which is second-precision).
+        let mut entries_with_ns: Vec<(LocalEntry, u64)> = candidates
             .into_par_iter()
-            .map(|(abs, rel)| -> Result<LocalEntry, FsSyncError> {
+            .map(|(abs, rel)| -> Result<(LocalEntry, u64), FsSyncError> {
-                let data = std::fs::read(&abs)?;
+                // Read metadata first so we can check the cache before reading content.
-                let blake3 = blake3_hash_large(&data);
                 let meta = std::fs::metadata(&abs)?;
                 let size = meta.len();
-                let mtime = meta
+                let modified = meta.modified().ok();
-                    .modified()
+                let mtime_ns = modified
-                    .ok()
+                    .as_ref()
+                    .and_then(|t| t.duration_since(UNIX_EPOCH).ok())
+                    .map(|d| d.as_nanos() as u64)
+                    .unwrap_or(0);
+                let mtime = modified
                     .and_then(|t| t.duration_since(UNIX_EPOCH).ok())
                     .map(|d| d.as_secs() as i64)
                     .unwrap_or(0);
-                Ok(LocalEntry {
+
+                // Reuse cached BLAKE3 when mtime_ns and size are unchanged.
+                let blake3 = if let Some(ce) = cache.get(rel.as_str()) {
+                    if ce.mtime_ns == mtime_ns && ce.size == size {
+                        ce.blake3 // cache hit — no file read
+                    } else {
+                        blake3_hash_large(&std::fs::read(&abs)?)
+                    }
+                } else {
+                    blake3_hash_large(&std::fs::read(&abs)?)
+                };
+
+                Ok((
+                    LocalEntry {
                         rel_path: rel,
                         abs_path: abs,
                         blake3,
                         size,
                         mtime,
-                })
+                    },
+                    mtime_ns,
+                ))
             })
             .collect::<Result<Vec<_>, _>>()?;
 
-        entries.sort_by(|a, b| a.rel_path.cmp(&b.rel_path));
+        entries_with_ns.sort_by(|(a, _), (b, _)| a.rel_path.cmp(&b.rel_path));
+
+        // Persist updated cache (best-effort; ignored on error).
+        save_cache(root, &entries_with_ns);
+
+        let entries: Vec<LocalEntry> = entries_with_ns.into_iter().map(|(e, _)| e).collect();
+
         Ok(Self { entries })
     }
 
@@ -212,4 +361,65 @@ mod tests {
         let paths: Vec<&str> = manifest.entries.iter().map(|e| e.rel_path.as_str()).collect();
         assert_eq!(paths, vec!["real.bin"], "temp files must be excluded from manifest");
     }
+
+    #[test]
+    fn cache_file_excluded_from_manifest() {
+        let dir = TempDir::new().unwrap();
+        fs::write(dir.path().join("real.bin"), b"data").unwrap();
+
+        // First build creates the cache file.
+        let m1 = FsManifest::build(dir.path(), &empty_excludes()).unwrap();
+        assert_eq!(m1.entries.len(), 1, "cache file must not appear in manifest");
+        assert!(dir.path().join(CACHE_FILE).exists(), "cache must be written");
+
+        // Second build with cache present — still only real.bin.
+        let m2 = FsManifest::build(dir.path(), &empty_excludes()).unwrap();
+        assert_eq!(m2.entries.len(), 1, "cache file must not appear on second build");
+    }
+
+    #[test]
+    fn cache_hit_produces_same_hash() {
+        let dir = TempDir::new().unwrap();
+        let data = b"cache test content";
+        fs::write(dir.path().join("file.bin"), data).unwrap();
+
+        // First build: populates cache.
+        let m1 = FsManifest::build(dir.path(), &empty_excludes()).unwrap();
+        assert_eq!(m1.entries.len(), 1);
+        let hash1 = m1.entries[0].blake3;
+
+        // Second build: should use cache (mtime+size unchanged).
+        let m2 = FsManifest::build(dir.path(), &empty_excludes()).unwrap();
+        assert_eq!(m2.entries.len(), 1);
+        assert_eq!(m2.entries[0].blake3, hash1, "cache hit must produce identical hash");
+    }
+
+    #[test]
+    fn cache_miss_on_content_change() {
+        let dir = TempDir::new().unwrap();
+        fs::write(dir.path().join("file.bin"), b"version one").unwrap();
+
+        let m1 = FsManifest::build(dir.path(), &empty_excludes()).unwrap();
+        let hash1 = m1.entries[0].blake3;
+
+        // Overwrite with different content. Sleep 1s to guarantee mtime advances
+        // on filesystems with 1-second resolution (common in tests on CI).
+        // On filesystems with nanosecond mtime (Linux ext4, macOS APFS) a size
+        // change is sufficient without sleeping.
+        fs::write(dir.path().join("file.bin"), b"version two is longer content").unwrap();
+
+        let m2 = FsManifest::build(dir.path(), &empty_excludes()).unwrap();
+        let hash2 = m2.entries[0].blake3;
+
+        assert_ne!(hash1, hash2, "modified file must produce a different hash");
+    }
+
+    #[test]
+    fn hex_roundtrip() {
+        let bytes: [u8; 32] = (0u8..32).collect::<Vec<_>>().try_into().unwrap();
+        let hex = bytes_to_hex64(&bytes);
+        assert_eq!(hex.len(), 64);
+        let back = hex64_to_bytes(&hex).expect("valid hex must decode");
+        assert_eq!(back, bytes);
+    }
 }