Initial commit: ClawSync v0.1.0

8-crate pure-Rust workspace for revision-aware HDF5 sync. ## Crates - clawhdf5-onion: ClawOnion VFD — page-level versioned HDF5 storage, binary format, writer/reader, branch DAG, GC, snapshots, provenance - clawsync-core: BLAKE3, xxHash3, FastCDC (+ SIMD NEON), zstd/lz4 - clawsync-onion: IBLT sketch, Merkle tree differ, packet differ/merger, ClawSyncManifest, SyncSelector - clawsync-hdf5: dataset-level manifest, differ, patcher, wire payload reconstruction (apply_received_payloads) - clawsync-transport: TCP, QUIC (quinn 0.11/TLS 1.3), SyncPeer abstraction, length-prefixed rkyv wire protocol (21 SyncMessage variants) - clawsync-agent: OnionMemory, SyncScheduler, TcpSyncBackend, PeerCapabilities negotiation - clawsync-fs: CDC-based delta sync for any file type; FsSyncClient/Server, W=16 pipelining, atomic writes - clawsync-cli: push/pull/serve/hdf5-sync/serve-hdf5/sync/serve-fs + all local management commands; --quic on all network commands ## Key features - IBLT pre-flight: O(revision count) vs rsync's O(file size) - W=16 sliding-window push: 13–15x speedup over stop-and-wait at WAN RTT - Dataset-granular HDF5 sync: only modified datasets transferred - CDC delta for any file type: insertion-stable chunk boundaries - Full revision DAG: branch, merge, rollback, export, snapshot, GC - QUIC transport: TLS 1.3, per-message streams via quinn 0.11 ## Tests ~573 passing (default features); ~589 with --features simd-cdc ## Performance (Apple Silicon) - Reconstruct rev=100: 68 µs (target ≤ 1 ms) - BLAKE3 Rayon 1 MB: 10.3 GiB/s (target ≥ 5 GB/s) - GC 500 revisions: 20.6 µs (target ≤ 2 s) - W=16 vs W=1 at 5 ms RTT: 14.8x speedup - No-op pre-flight at 16 MB: 4 ms vs rsync 35 ms (7.8x) Co-Authored-By: Claude Sonnet 4.6 <[email protected]>
2026-04-04 18:41:22 -05:00
commit 260e15f5b6
102 changed files with 30098 additions and 0 deletions
@@ -0,0 +1,555 @@
+//! Content-defined Merkle tree over OnionFile revision BLAKE3 hashes.
+//!
+//! Enables O(log N) identification of missing revisions between two nodes,
+//! compared to O(N) for a flat revision-set comparison.
+//!
+//! ## Layout
+//!
+//! The tree is a balanced binary tree stored in 1-indexed BFS (heap) order:
+//!
+//! ```text
+//! capacity = next_power_of_two(leaf_count)
+//! root           → index 1
+//! left  child(i) → index 2i
+//! right child(i) → index 2i+1
+//! leaf  k        → index capacity + k   (k = 0-based ordinal)
+//! ```
+//!
+//! Padding leaves (when `leaf_count < capacity`) have all-zero hashes.
+//! Internal node hash = `BLAKE3(left_child_hash || right_child_hash)`.
+//!
+//! ## Serialisation wire format
+//!
+//! ```text
+//! magic:      [u8; 4]  = b"MERK"
+//! version:    u8       = 1
+//! leaf_count: u64 LE
+//! entries:    leaf_count × { revision: u64 LE, blake3: [u8; 32] }
+//! ```
+//!
+//! Size: `5 + 8 + N × 40` bytes.  For N = 1 000 this is ~40 KB, vs the
+//! O(N × 60 B) flat `ClawSyncManifest`.
+//!
+//! ## References
+//!
+//! arXiv 2104.02158 — content-defined Merkle trees for versioned storage.
+
+#![forbid(unsafe_code)]
+
+use blake3::Hasher as Blake3Hasher;
+
+// ─────────────────────────────────────────────────────────────────────────────
+// Wire constants
+// ─────────────────────────────────────────────────────────────────────────────
+
+const MAGIC: &[u8; 4] = b"MERK";
+const VERSION: u8 = 1;
+
+// ─────────────────────────────────────────────────────────────────────────────
+// Public types
+// ─────────────────────────────────────────────────────────────────────────────
+
+/// A single node of the revision Merkle tree.
+#[derive(Clone, Debug, PartialEq, Eq)]
+pub struct MerkleNode {
+    /// BLAKE3 hash of this node's subtree.
+    pub hash: [u8; 32],
+    /// Lowest revision number covered by this subtree.
+    pub rev_lo: u64,
+    /// Highest revision number covered by this subtree.
+    pub rev_hi: u64,
+    /// `true` when this node is a leaf (single revision).
+    pub is_leaf: bool,
+}
+
+/// Outcome of one step in a Merkle tree diff walk.
+#[derive(Clone, Debug, PartialEq, Eq)]
+pub enum WalkStep {
+    /// Subtree [rev_lo, rev_hi] is identical on both sides — skip.
+    Skip { rev_lo: u64, rev_hi: u64 },
+    /// Subtree differs; descend into children.
+    Descend { node: MerkleNode },
+    /// Leaf differs; this revision needs to be transferred.
+    Leaf { revision: u64, hash: [u8; 32] },
+}
+
+/// Balanced binary Merkle tree over OnionFile revision BLAKE3 hashes.
+///
+/// Built from a sorted list of `(revision_number, blake3_hash)` pairs.
+#[derive(Clone, Debug, PartialEq, Eq)]
+pub struct RevisionMerkleTree {
+    /// 1-indexed BFS hashes.  Index 0 is unused.
+    /// Size: 2 * capacity + 1.
+    hashes: Vec<[u8; 32]>,
+    /// Revision number for each leaf (0-based ordinal).
+    leaf_revisions: Vec<u64>,
+    /// Number of actual revisions (leaves with real hashes).
+    pub leaf_count: usize,
+    /// Smallest power of 2 ≥ leaf_count.
+    pub capacity: usize,
+}
+
+impl RevisionMerkleTree {
+    // ── Construction ──────────────────────────────────────────────────────────
+
+    /// Build from a sorted slice of `(revision, blake3_hash)` pairs.
+    ///
+    /// The slice **must** be sorted by `revision` in ascending order.
+    pub fn build(entries: &[(u64, [u8; 32])]) -> Self {
+        let leaf_count = entries.len();
+        let capacity = if leaf_count == 0 { 1 } else { leaf_count.next_power_of_two() };
+
+        // 1-indexed BFS array; index 0 unused.
+        let mut hashes = vec![[0u8; 32]; 2 * capacity + 1];
+        let mut leaf_revisions = Vec::with_capacity(leaf_count);
+
+        // Place leaf hashes
+        for (i, (rev, hash)) in entries.iter().enumerate() {
+            hashes[capacity + i] = *hash;
+            leaf_revisions.push(*rev);
+        }
+        // Indices [capacity + leaf_count, 2*capacity) are padding zeros
+
+        // Build internal nodes bottom-up
+        build_internal(&mut hashes, capacity);
+
+        RevisionMerkleTree { hashes, leaf_revisions, leaf_count, capacity }
+    }
+
+    // ── Queries ───────────────────────────────────────────────────────────────
+
+    /// Root hash of the tree.  Two trees with the same root hash have identical
+    /// revision sets (assuming no BLAKE3 collisions).
+    pub fn root_hash(&self) -> [u8; 32] {
+        if self.leaf_count == 0 {
+            [0u8; 32]
+        } else {
+            self.hashes[1]
+        }
+    }
+
+    /// Return the revision numbers that `self` has but `other` lacks (or
+    /// has a different hash for).
+    ///
+    /// The returned list is sorted in ascending revision order.
+    ///
+    /// Algorithm: iterative BFS.  At each node, if hashes agree the subtree is
+    /// skipped (O(1)).  Only differing subtrees are descended into.  Total work
+    /// is O((D + 1) × log N) for D differing revisions.
+    pub fn diff_missing_revisions(&self, other: &RevisionMerkleTree) -> Vec<u64> {
+        if self.leaf_count == 0 {
+            return Vec::new();
+        }
+
+        // Expand other to self's capacity so BFS indices align.
+        let other_hashes = other.padded_hashes(self.capacity);
+
+        let mut result = Vec::new();
+
+        // Quick exit if roots match
+        if self.hashes[1] == other_hashes[1] {
+            return result;
+        }
+
+        // Iterative BFS walk
+        let mut queue: std::collections::VecDeque<usize> = std::collections::VecDeque::new();
+        queue.push_back(1);
+
+        while let Some(node_idx) = queue.pop_front() {
+            if self.hashes[node_idx] == other_hashes[node_idx] {
+                // Subtree matches — skip entirely
+                continue;
+            }
+
+            if node_idx >= self.capacity {
+                // Leaf node
+                let leaf_ord = node_idx - self.capacity;
+                if leaf_ord < self.leaf_count {
+                    result.push(self.leaf_revisions[leaf_ord]);
+                }
+            } else {
+                // Internal node — descend
+                queue.push_back(2 * node_idx);
+                queue.push_back(2 * node_idx + 1);
+            }
+        }
+
+        result.sort_unstable();
+        result
+    }
+
+    /// Return a `MerkleNode` for the root.
+    pub fn root_node(&self) -> Option<MerkleNode> {
+        if self.leaf_count == 0 {
+            return None;
+        }
+        Some(self.node_at(1))
+    }
+
+    // ── Serialisation ─────────────────────────────────────────────────────────
+
+    /// Serialise into the compact wire format:
+    /// `MERK | version(1) | leaf_count(u64 LE) | entries[]`
+    pub fn serialise(&self) -> Vec<u8> {
+        let mut out = Vec::with_capacity(5 + 8 + self.leaf_count * 40);
+        out.extend_from_slice(MAGIC);
+        out.push(VERSION);
+        out.extend_from_slice(&(self.leaf_count as u64).to_le_bytes());
+        for (i, &rev) in self.leaf_revisions.iter().enumerate() {
+            out.extend_from_slice(&rev.to_le_bytes());
+            out.extend_from_slice(&self.hashes[self.capacity + i]);
+        }
+        out
+    }
+
+    /// Deserialise from wire format.
+    pub fn deserialise(data: &[u8]) -> Result<Self, MerkleError> {
+        if data.len() < 5 + 8 {
+            return Err(MerkleError::TruncatedHeader);
+        }
+        if &data[0..4] != MAGIC {
+            return Err(MerkleError::BadMagic);
+        }
+        if data[4] != VERSION {
+            return Err(MerkleError::UnknownVersion(data[4]));
+        }
+        let leaf_count = u64::from_le_bytes(data[5..13].try_into().unwrap()) as usize;
+        let expected_len = 5 + 8 + leaf_count * 40;
+        if data.len() < expected_len {
+            return Err(MerkleError::TruncatedPayload {
+                expected: expected_len,
+                got: data.len(),
+            });
+        }
+
+        let mut entries = Vec::with_capacity(leaf_count);
+        let mut off = 13usize;
+        for _ in 0..leaf_count {
+            let rev = u64::from_le_bytes(data[off..off + 8].try_into().unwrap());
+            let mut hash = [0u8; 32];
+            hash.copy_from_slice(&data[off + 8..off + 40]);
+            entries.push((rev, hash));
+            off += 40;
+        }
+
+        Ok(Self::build(&entries))
+    }
+
+    // ── Internal helpers ──────────────────────────────────────────────────────
+
+    /// Return a copy of this tree's BFS hash array padded (or already matching)
+    /// to `target_capacity`.
+    ///
+    /// If `target_capacity > self.capacity`, the leaves are placed at the same
+    /// ordinal positions in the larger tree and internal nodes are recomputed.
+    /// If `target_capacity == self.capacity`, a cheap clone is returned.
+    pub(crate) fn padded_hashes(&self, target_capacity: usize) -> Vec<[u8; 32]> {
+        if target_capacity == self.capacity {
+            return self.hashes.clone();
+        }
+        let mut h = vec![[0u8; 32]; 2 * target_capacity + 1];
+        // Only copy leaves that fit within target_capacity (handles both
+        // expanding a smaller tree and projecting a larger tree down).
+        let copyable = self.leaf_count.min(target_capacity);
+        for i in 0..copyable {
+            h[target_capacity + i] = self.hashes[self.capacity + i];
+        }
+        build_internal(&mut h, target_capacity);
+        h
+    }
+
+    /// Retrieve the `MerkleNode` metadata for BFS index `i`.
+    fn node_at(&self, i: usize) -> MerkleNode {
+        let (lo, hi, is_leaf) = self.node_range(i);
+        MerkleNode {
+            hash: self.hashes[i],
+            rev_lo: lo,
+            rev_hi: hi,
+            is_leaf,
+        }
+    }
+
+    /// Compute the [rev_lo, rev_hi] range and leaf flag for BFS index `i`.
+    fn node_range(&self, i: usize) -> (u64, u64, bool) {
+        if i >= self.capacity {
+            // Leaf
+            let ord = i - self.capacity;
+            let rev = if ord < self.leaf_count {
+                self.leaf_revisions[ord]
+            } else {
+                u64::MAX
+            };
+            (rev, rev, true)
+        } else {
+            // Internal: find leftmost/rightmost leaf under this node
+            let depth = (i.ilog2()) as usize;
+            let span = self.capacity >> depth;
+            let lo_ord = (i - (1 << depth)) * span;
+            let hi_ord = lo_ord + span - 1;
+            let lo_rev = if lo_ord < self.leaf_count {
+                self.leaf_revisions[lo_ord]
+            } else {
+                u64::MAX
+            };
+            let hi_rev = if hi_ord < self.leaf_count {
+                self.leaf_revisions[hi_ord]
+            } else if lo_ord < self.leaf_count {
+                *self.leaf_revisions.last().unwrap()
+            } else {
+                u64::MAX
+            };
+            (lo_rev, hi_rev, false)
+        }
+    }
+}
+
+// ─────────────────────────────────────────────────────────────────────────────
+// Internal helpers
+// ─────────────────────────────────────────────────────────────────────────────
+
+/// Build internal nodes of a 1-indexed BFS tree bottom-up.
+/// `hashes` must have length `2 * capacity + 1`.
+fn build_internal(hashes: &mut [[u8; 32]], capacity: usize) {
+    for i in (1..capacity).rev() {
+        let left  = hashes[2 * i];
+        let right = hashes[2 * i + 1];
+        hashes[i] = node_hash(&left, &right);
+    }
+}
+
+/// Hash two child hashes together: `BLAKE3(left || right)`.
+#[inline]
+fn node_hash(left: &[u8; 32], right: &[u8; 32]) -> [u8; 32] {
+    let mut h = Blake3Hasher::new();
+    h.update(left);
+    h.update(right);
+    *h.finalize().as_bytes()
+}
+
+// ─────────────────────────────────────────────────────────────────────────────
+// Error type
+// ─────────────────────────────────────────────────────────────────────────────
+
+/// Errors from Merkle tree serialisation/deserialisation.
+#[derive(Debug, thiserror::Error, PartialEq, Eq)]
+pub enum MerkleError {
+    #[error("truncated header (< 13 bytes)")]
+    TruncatedHeader,
+    #[error("bad magic bytes")]
+    BadMagic,
+    #[error("unknown format version {0}")]
+    UnknownVersion(u8),
+    #[error("truncated payload: expected {expected} bytes, got {got}")]
+    TruncatedPayload { expected: usize, got: usize },
+}
+
+// ─────────────────────────────────────────────────────────────────────────────
+// Tests
+// ─────────────────────────────────────────────────────────────────────────────
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use proptest::prelude::*;
+
+    fn rev_hash(r: u64) -> [u8; 32] {
+        *blake3::hash(&r.to_le_bytes()).as_bytes()
+    }
+
+    fn build_n(n: usize) -> RevisionMerkleTree {
+        let entries: Vec<(u64, [u8; 32])> = (0..n as u64).map(|r| (r, rev_hash(r))).collect();
+        RevisionMerkleTree::build(&entries)
+    }
+
+    // ── basic structure ───────────────────────────────────────────────────────
+
+    #[test]
+    fn tree_empty() {
+        let t = RevisionMerkleTree::build(&[]);
+        assert_eq!(t.leaf_count, 0);
+        assert_eq!(t.capacity, 1);
+        assert_eq!(t.root_hash(), [0u8; 32]);
+        assert!(t.root_node().is_none());
+    }
+
+    #[test]
+    fn tree_single_leaf() {
+        let h = rev_hash(42);
+        let t = RevisionMerkleTree::build(&[(42, h)]);
+        assert_eq!(t.leaf_count, 1);
+        assert_eq!(t.capacity, 1);
+        assert_eq!(t.root_hash(), h);
+    }
+
+    #[test]
+    fn tree_two_leaves() {
+        let t = build_n(2);
+        assert_eq!(t.capacity, 2);
+        let expected_root = node_hash(&rev_hash(0), &rev_hash(1));
+        assert_eq!(t.root_hash(), expected_root);
+    }
+
+    #[test]
+    fn tree_depth_is_log_n() {
+        // For 1000 leaves, depth should be ≤ ceil(log2(1000)) = 10
+        let t = build_n(1000);
+        assert_eq!(t.capacity, 1024); // next power of 2
+        // Depth = log2(capacity) = 10
+        let depth = t.capacity.ilog2() as usize;
+        assert!(depth <= 10, "depth {depth} > 10");
+        // Also verify no root is all zeros (would mean build failed)
+        assert_ne!(t.root_hash(), [0u8; 32]);
+    }
+
+    #[test]
+    fn tree_identical_trees_no_diff() {
+        let t = build_n(50);
+        let diff = t.diff_missing_revisions(&t.clone());
+        assert!(diff.is_empty(), "identical trees should have empty diff");
+    }
+
+    #[test]
+    fn tree_one_new_revision() {
+        let base = build_n(5);
+        let extended = build_n(6); // has rev 5 that base lacks
+        let diff = extended.diff_missing_revisions(&base);
+        assert_eq!(diff, vec![5]);
+    }
+
+    #[test]
+    fn tree_multiple_new_revisions() {
+        let base = build_n(3);
+        let extended = build_n(7);
+        let diff = extended.diff_missing_revisions(&base);
+        assert_eq!(diff, vec![3, 4, 5, 6]);
+    }
+
+    #[test]
+    fn tree_remote_ahead_is_empty_diff() {
+        // If remote has MORE revisions than local, from local's perspective
+        // there's nothing "missing on remote" beyond local's range
+        let local = build_n(5);
+        let remote = build_n(10);
+        // local.diff_missing_revisions(remote) → what local has that remote doesn't
+        // local has 0..4, remote has 0..9: local has nothing remote lacks
+        let diff = local.diff_missing_revisions(&remote);
+        assert!(diff.is_empty());
+    }
+
+    #[test]
+    fn tree_changed_hash_mid_range() {
+        // One revision in the middle has a different hash (e.g., corruption)
+        let mut entries: Vec<(u64, [u8; 32])> = (0..10u64).map(|r| (r, rev_hash(r))).collect();
+        let t1 = RevisionMerkleTree::build(&entries);
+        entries[5].1 = [0xFFu8; 32]; // alter rev 5
+        let t2 = RevisionMerkleTree::build(&entries);
+        let diff = t1.diff_missing_revisions(&t2);
+        assert_eq!(diff, vec![5]);
+    }
+
+    // ── serialisation ─────────────────────────────────────────────────────────
+
+    #[test]
+    fn tree_roundtrip_serialisation() {
+        let t = build_n(100);
+        let bytes = t.serialise();
+        let t2 = RevisionMerkleTree::deserialise(&bytes).unwrap();
+        assert_eq!(t2.leaf_count, 100);
+        assert_eq!(t2.root_hash(), t.root_hash());
+    }
+
+    #[test]
+    fn tree_serialise_size_linear() {
+        let t = build_n(1000);
+        let bytes = t.serialise();
+        // 5 (header) + 8 (count) + 1000 * 40 (entries)
+        assert_eq!(bytes.len(), 5 + 8 + 1000 * 40);
+    }
+
+    #[test]
+    fn deserialise_bad_magic_errors() {
+        let mut bytes = build_n(5).serialise();
+        bytes[0] = 0xFF;
+        assert_eq!(
+            RevisionMerkleTree::deserialise(&bytes),
+            Err(MerkleError::BadMagic)
+        );
+    }
+
+    #[test]
+    fn deserialise_truncated_errors() {
+        // Provide the 13-byte header (MERK + version + leaf_count=5) but no
+        // entry bytes — this gets past the header check and hits TruncatedPayload.
+        let full = build_n(5).serialise();
+        let header_only = &full[..13]; // 5 + 8 bytes
+        assert_eq!(
+            RevisionMerkleTree::deserialise(header_only),
+            Err(MerkleError::TruncatedPayload { expected: 5 + 8 + 5 * 40, got: 13 })
+        );
+    }
+
+    #[test]
+    fn deserialise_unknown_version_errors() {
+        let mut bytes = build_n(3).serialise();
+        bytes[4] = 99;
+        assert_eq!(
+            RevisionMerkleTree::deserialise(&bytes),
+            Err(MerkleError::UnknownVersion(99))
+        );
+    }
+
+    // ── walk correctness ──────────────────────────────────────────────────────
+
+    #[test]
+    fn diff_empty_local_vs_empty_remote() {
+        let t = RevisionMerkleTree::build(&[]);
+        assert!(t.diff_missing_revisions(&t.clone()).is_empty());
+    }
+
+    #[test]
+    fn diff_empty_local_vs_nonempty_remote() {
+        let local = RevisionMerkleTree::build(&[]);
+        let remote = build_n(5);
+        // local has nothing, so nothing is "missing on remote"
+        let diff = local.diff_missing_revisions(&remote);
+        assert!(diff.is_empty());
+    }
+
+    // ── proptest ──────────────────────────────────────────────────────────────
+
+    proptest! {
+        /// For any two random revision sets A ⊇ B (A = local, B = remote),
+        /// `diff_missing_revisions` correctly identifies A \ B.
+        #[test]
+        fn prop_diff_identifies_all_differences(
+            total in 1usize..500,
+            keep_frac in 0.0f64..1.0,
+        ) {
+            let all_entries: Vec<(u64, [u8; 32])> = (0..total as u64)
+                .map(|r| (r, rev_hash(r)))
+                .collect();
+            let keep_count = ((total as f64 * keep_frac) as usize).max(0);
+            let remote_entries = &all_entries[..keep_count];
+
+            let local  = RevisionMerkleTree::build(&all_entries);
+            let remote = RevisionMerkleTree::build(remote_entries);
+
+            let diff = local.diff_missing_revisions(&remote);
+
+            // Expected: revisions keep_count..total
+            let expected: Vec<u64> = (keep_count as u64..total as u64).collect();
+            prop_assert_eq!(diff, expected);
+        }
+
+        /// Roundtrip: serialise then deserialise preserves root hash and leaf count.
+        #[test]
+        fn prop_serialise_roundtrip(n in 0usize..300) {
+            let entries: Vec<(u64, [u8; 32])> = (0..n as u64).map(|r| (r, rev_hash(r))).collect();
+            let t = RevisionMerkleTree::build(&entries);
+            let bytes = t.serialise();
+            let t2 = RevisionMerkleTree::deserialise(&bytes).unwrap();
+            prop_assert_eq!(t2.root_hash(), t.root_hash());
+            prop_assert_eq!(t2.leaf_count, t.leaf_count);
+        }
+    }
+}