Initial commit: ClawSync v0.1.0

8-crate pure-Rust workspace for revision-aware HDF5 sync. ## Crates - clawhdf5-onion: ClawOnion VFD — page-level versioned HDF5 storage, binary format, writer/reader, branch DAG, GC, snapshots, provenance - clawsync-core: BLAKE3, xxHash3, FastCDC (+ SIMD NEON), zstd/lz4 - clawsync-onion: IBLT sketch, Merkle tree differ, packet differ/merger, ClawSyncManifest, SyncSelector - clawsync-hdf5: dataset-level manifest, differ, patcher, wire payload reconstruction (apply_received_payloads) - clawsync-transport: TCP, QUIC (quinn 0.11/TLS 1.3), SyncPeer abstraction, length-prefixed rkyv wire protocol (21 SyncMessage variants) - clawsync-agent: OnionMemory, SyncScheduler, TcpSyncBackend, PeerCapabilities negotiation - clawsync-fs: CDC-based delta sync for any file type; FsSyncClient/Server, W=16 pipelining, atomic writes - clawsync-cli: push/pull/serve/hdf5-sync/serve-hdf5/sync/serve-fs + all local management commands; --quic on all network commands ## Key features - IBLT pre-flight: O(revision count) vs rsync's O(file size) - W=16 sliding-window push: 13–15x speedup over stop-and-wait at WAN RTT - Dataset-granular HDF5 sync: only modified datasets transferred - CDC delta for any file type: insertion-stable chunk boundaries - Full revision DAG: branch, merge, rollback, export, snapshot, GC - QUIC transport: TLS 1.3, per-message streams via quinn 0.11 ## Tests ~573 passing (default features); ~589 with --features simd-cdc ## Performance (Apple Silicon) - Reconstruct rev=100: 68 µs (target ≤ 1 ms) - BLAKE3 Rayon 1 MB: 10.3 GiB/s (target ≥ 5 GB/s) - GC 500 revisions: 20.6 µs (target ≤ 2 s) - W=16 vs W=1 at 5 ms RTT: 14.8x speedup - No-op pre-flight at 16 MB: 4 ms vs rsync 35 ms (7.8x) Co-Authored-By: Claude Sonnet 4.6 <[email protected]>
2026-04-04 18:41:22 -05:00
commit 260e15f5b6
102 changed files with 30098 additions and 0 deletions
@@ -0,0 +1,382 @@
+//! Integration tests for branch-selective ClawSync.
+//!
+//! These tests exercise the full pipeline:
+//!   differ::diff_revisions → selector::filter_packets → merger::merge_packets
+//!
+//! Scenarios covered:
+//! - Sync only a named branch to an empty remote
+//! - UpTo selector limits revisions regardless of branch
+//! - BranchAt selector combines branch + revision ceiling
+//! - Multi-branch source: only target branch lands on remote
+//! - Verify page data integrity after branch-selective merge
+//! - SyncSelector::All passes everything through (baseline)
+
+use clawhdf5_onion::branch::MergeStrategy;
+use clawhdf5_onion::format::NO_PARENT;
+use clawhdf5_onion::writer::OnionFile;
+use clawsync_onion::differ::diff_revisions;
+use clawsync_onion::merger::merge_packets;
+use clawsync_onion::selector::{filter_packets, remote_head_for_selector, SyncSelector};
+use tempfile::NamedTempFile;
+
+// ─────────────────────────────────────────────────────────────────────────────
+// Helpers
+// ─────────────────────────────────────────────────────────────────────────────
+
+fn tmp_onion(n_main: u8) -> OnionFile {
+    let f = NamedTempFile::new().unwrap();
+    let h5 = f.path().with_extension("h5");
+    std::fs::write(&h5, b"\x89HDF\r\n\x1a\n").unwrap();
+    let mut onion = OnionFile::create(&h5, 4096).unwrap();
+    for i in 0..n_main {
+        let mut s = onion.begin_session(None).unwrap();
+        s.record_page(0, &vec![i; 4096]);
+        onion.commit_session(s, Some(&format!("main-{i}"))).unwrap();
+    }
+    onion
+}
+
+fn empty_onion() -> OnionFile {
+    let f = NamedTempFile::new().unwrap();
+    let h5 = f.path().with_extension("h5");
+    std::fs::write(&h5, b"\x89HDF\r\n\x1a\n").unwrap();
+    OnionFile::create(&h5, 4096).unwrap()
+}
+
+/// Build an onion with `n_main` revisions on main, then fork "feat" and add
+/// `n_feat` more revisions on feat.
+fn branched_onion(n_main: u8, n_feat: u8) -> OnionFile {
+    let mut onion = tmp_onion(n_main);
+    onion.create_branch("feat", "main").unwrap();
+    let feat_id = onion.branch_by_name("feat").unwrap().id;
+    for i in 0..n_feat {
+        let mut s = onion.begin_session(Some(feat_id)).unwrap();
+        s.record_page(0, &vec![0x80 + i; 4096]);
+        onion.commit_session(s, Some(&format!("feat-{i}"))).unwrap();
+    }
+    onion
+}
+
+// ─────────────────────────────────────────────────────────────────────────────
+// SyncSelector::All — baseline
+// ─────────────────────────────────────────────────────────────────────────────
+
+#[test]
+fn all_selector_transfers_every_revision() {
+    let src = tmp_onion(5);
+    let mut dst = empty_onion();
+
+    let packets = diff_revisions(&src, NO_PARENT).unwrap();
+    let filtered = filter_packets(packets, &SyncSelector::All, &src).unwrap();
+    let stats = merge_packets(&mut dst, filtered, false).unwrap();
+
+    assert_eq!(stats.revisions_merged, 5);
+    assert_eq!(dst.revision_count(), 5);
+}
+
+// ─────────────────────────────────────────────────────────────────────────────
+// SyncSelector::UpTo
+// ─────────────────────────────────────────────────────────────────────────────
+
+#[test]
+fn up_to_selector_limits_revisions() {
+    let src = tmp_onion(6);
+    let mut dst = empty_onion();
+
+    let packets = diff_revisions(&src, NO_PARENT).unwrap();
+    let filtered = filter_packets(packets, &SyncSelector::UpTo(3), &src).unwrap();
+
+    // Revisions 0, 1, 2, 3 → 4 packets
+    assert_eq!(filtered.len(), 4);
+    assert_eq!(filtered.last().unwrap().revision, 3);
+
+    let stats = merge_packets(&mut dst, filtered, false).unwrap();
+    assert_eq!(stats.revisions_merged, 4);
+    assert_eq!(dst.revision_count(), 4);
+}
+
+#[test]
+fn up_to_revision_zero_sends_only_first() {
+    let src = tmp_onion(4);
+    let packets = diff_revisions(&src, NO_PARENT).unwrap();
+    let filtered = filter_packets(packets, &SyncSelector::UpTo(0), &src).unwrap();
+    assert_eq!(filtered.len(), 1);
+    assert_eq!(filtered[0].revision, 0);
+}
+
+#[test]
+fn up_to_beyond_head_sends_all() {
+    let src = tmp_onion(3);
+    let packets = diff_revisions(&src, NO_PARENT).unwrap();
+    let filtered = filter_packets(packets, &SyncSelector::UpTo(999), &src).unwrap();
+    assert_eq!(filtered.len(), 3);
+}
+
+// ─────────────────────────────────────────────────────────────────────────────
+// SyncSelector::Branch
+// ─────────────────────────────────────────────────────────────────────────────
+
+#[test]
+fn branch_selector_main_only_sends_main_revisions() {
+    // 3 main + 2 feat revisions
+    let src = branched_onion(3, 2);
+    let mut dst = empty_onion();
+
+    let packets = diff_revisions(&src, NO_PARENT).unwrap();
+    assert_eq!(packets.len(), 5); // all revisions
+
+    let filtered = filter_packets(packets, &SyncSelector::Branch("main".into()), &src).unwrap();
+    assert_eq!(filtered.len(), 3);
+    assert!(filtered.iter().all(|p| p.branch_id == 0));
+
+    let stats = merge_packets(&mut dst, filtered, false).unwrap();
+    assert_eq!(stats.revisions_merged, 3);
+    assert_eq!(dst.revision_count(), 3);
+}
+
+#[test]
+fn branch_selector_feat_only_sends_feat_revisions() {
+    let src = branched_onion(3, 4);
+    let feat_id = src.branch_by_name("feat").unwrap().id;
+
+    let packets = diff_revisions(&src, NO_PARENT).unwrap();
+    let filtered = filter_packets(packets, &SyncSelector::Branch("feat".into()), &src).unwrap();
+
+    assert_eq!(filtered.len(), 4);
+    assert!(filtered.iter().all(|p| p.branch_id == feat_id));
+}
+
+#[test]
+fn branch_selector_nonexistent_errors() {
+    let src = tmp_onion(2);
+    let packets = diff_revisions(&src, NO_PARENT).unwrap();
+    let result = filter_packets(packets, &SyncSelector::Branch("nope".into()), &src);
+    assert!(result.is_err());
+}
+
+// ─────────────────────────────────────────────────────────────────────────────
+// SyncSelector::BranchAt
+// ─────────────────────────────────────────────────────────────────────────────
+
+#[test]
+fn branch_at_filters_branch_and_revision() {
+    // 2 main revisions (0, 1), then 4 feat revisions (2, 3, 4, 5)
+    let src = branched_onion(2, 4);
+    let feat_id = src.branch_by_name("feat").unwrap().id;
+
+    let packets = diff_revisions(&src, NO_PARENT).unwrap();
+    // BranchAt("feat", 3) → feat packets with revision ≤ 3 → revisions 2, 3
+    let filtered =
+        filter_packets(packets, &SyncSelector::BranchAt("feat".into(), 3), &src).unwrap();
+
+    assert_eq!(filtered.len(), 2);
+    assert!(filtered.iter().all(|p| p.branch_id == feat_id));
+    assert!(filtered.iter().all(|p| p.revision <= 3));
+}
+
+#[test]
+fn branch_at_main_with_ceiling() {
+    let src = tmp_onion(5);
+
+    let packets = diff_revisions(&src, NO_PARENT).unwrap();
+    let filtered =
+        filter_packets(packets, &SyncSelector::BranchAt("main".into(), 2), &src).unwrap();
+
+    assert_eq!(filtered.len(), 3); // revisions 0, 1, 2
+    assert_eq!(filtered.last().unwrap().revision, 2);
+}
+
+#[test]
+fn branch_at_nonexistent_branch_errors() {
+    let src = tmp_onion(3);
+    let packets = diff_revisions(&src, NO_PARENT).unwrap();
+    let result = filter_packets(packets, &SyncSelector::BranchAt("ghost".into(), 1), &src);
+    assert!(result.is_err());
+}
+
+// ─────────────────────────────────────────────────────────────────────────────
+// Multi-branch source → branch-filtered merge
+// ─────────────────────────────────────────────────────────────────────────────
+
+#[test]
+fn merge_feat_branch_only_to_empty_remote() {
+    let src = branched_onion(3, 3);
+    let feat_id = src.branch_by_name("feat").unwrap().id;
+    let mut dst = empty_onion();
+
+    let packets = diff_revisions(&src, NO_PARENT).unwrap();
+    let filtered = filter_packets(packets, &SyncSelector::Branch("feat".into()), &src).unwrap();
+
+    // Only feat packets → 3 revisions
+    assert_eq!(filtered.len(), 3);
+
+    let stats = merge_packets(&mut dst, filtered, false).unwrap();
+    assert_eq!(stats.revisions_merged, 3);
+
+    // The remote should have a feat branch with those revisions
+    let dst_revs = dst.list_revisions();
+    assert_eq!(dst_revs.len(), 3);
+    assert!(dst_revs.iter().all(|r| r.branch_id == feat_id));
+}
+
+#[test]
+fn merge_preserves_page_content_after_branch_filter() {
+    let src = branched_onion(2, 2);
+
+    let mut dst = empty_onion();
+    let packets = diff_revisions(&src, NO_PARENT).unwrap();
+    let filtered = filter_packets(packets, &SyncSelector::Branch("main".into()), &src).unwrap();
+    merge_packets(&mut dst, filtered, false).unwrap();
+
+    // Main rev 0: page filled with 0x00, rev 1: filled with 0x01
+    let rev0_pages = dst.revision_pages(0).unwrap();
+    assert!(rev0_pages[0].1.iter().all(|&b| b == 0x00));
+
+    let rev1_pages = dst.revision_pages(1).unwrap();
+    assert!(rev1_pages[0].1.iter().all(|&b| b == 0x01));
+}
+
+// ─────────────────────────────────────────────────────────────────────────────
+// remote_head_for_selector
+// ─────────────────────────────────────────────────────────────────────────────
+
+#[test]
+fn remote_head_empty_remote() {
+    assert_eq!(remote_head_for_selector(&SyncSelector::All, 0), NO_PARENT);
+}
+
+#[test]
+fn remote_head_non_empty_remote() {
+    // Remote has 4 revisions (0..=3) → HEAD = 3
+    assert_eq!(remote_head_for_selector(&SyncSelector::All, 4), 3);
+}
+
+#[test]
+fn remote_head_single_revision() {
+    assert_eq!(remote_head_for_selector(&SyncSelector::UpTo(5), 1), 0);
+}
+
+// ─────────────────────────────────────────────────────────────────────────────
+// Incremental sync (diff from non-zero remote_head)
+// ─────────────────────────────────────────────────────────────────────────────
+
+#[test]
+fn incremental_sync_sends_only_new_revisions() {
+    let src = tmp_onion(6);
+
+    // Remote already has revisions 0..=2; diff from HEAD=2
+    let packets = diff_revisions(&src, 2).unwrap();
+    let filtered = filter_packets(packets, &SyncSelector::All, &src).unwrap();
+
+    assert_eq!(filtered.len(), 3); // revisions 3, 4, 5
+
+    let mut dst = tmp_onion(3); // dst already has 0, 1, 2
+    let stats = merge_packets(&mut dst, filtered, false).unwrap();
+    assert_eq!(stats.revisions_merged, 3);
+    assert_eq!(dst.revision_count(), 6);
+}
+
+#[test]
+fn incremental_branch_sync_sends_only_new_branch_revisions() {
+    let src = branched_onion(3, 4); // main: 0,1,2  feat: 3,4,5,6
+    let feat_id = src.branch_by_name("feat").unwrap().id;
+
+    // Simulate remote already having feat revisions 3, 4 (remote_head=4)
+    let remote_head = remote_head_for_selector(&SyncSelector::Branch("feat".into()), 5);
+
+    let packets = diff_revisions(&src, remote_head).unwrap();
+    let filtered = filter_packets(packets, &SyncSelector::Branch("feat".into()), &src).unwrap();
+
+    // Should send feat revisions 5, 6 only
+    assert!(filtered.iter().all(|p| p.branch_id == feat_id));
+    assert!(filtered.iter().all(|p| p.revision > remote_head));
+}
+
+// ─────────────────────────────────────────────────────────────────────────────
+// Merge with branched source using ensure_branch_id auto-registration
+// ─────────────────────────────────────────────────────────────────────────────
+
+#[test]
+fn merge_auto_registers_unknown_branch() {
+    // Source: 2 main + 2 feat revisions; send ALL to empty remote
+    let src = branched_onion(2, 2);
+    let mut dst = empty_onion();
+
+    let packets = diff_revisions(&src, NO_PARENT).unwrap();
+    // All 4 packets — includes both main (branch_id=0) and feat (branch_id=1)
+    let stats = merge_packets(&mut dst, packets, false).unwrap();
+
+    assert_eq!(stats.revisions_merged, 4);
+    // The remote should have both branches registered
+    let branches = dst.list_branches();
+    assert!(branches.len() >= 2, "expected at least 2 branches, got {:?}", branches.len());
+}
+
+// ─────────────────────────────────────────────────────────────────────────────
+// Merge-then-branch on destination (fork from synced state)
+// ─────────────────────────────────────────────────────────────────────────────
+
+#[test]
+fn can_branch_on_remote_after_branch_selective_sync() {
+    let src = tmp_onion(3);
+    let mut dst = empty_onion();
+
+    let packets = diff_revisions(&src, NO_PARENT).unwrap();
+    let filtered = filter_packets(packets, &SyncSelector::Branch("main".into()), &src).unwrap();
+    merge_packets(&mut dst, filtered, false).unwrap();
+
+    // The remote should now be able to fork its own branch
+    let result = dst.create_branch("experiment", "main");
+    assert!(result.is_ok(), "should be able to create branch after sync: {result:?}");
+}
+
+// ─────────────────────────────────────────────────────────────────────────────
+// Idempotency across selectors
+// ─────────────────────────────────────────────────────────────────────────────
+
+#[test]
+fn branch_filtered_merge_is_idempotent() {
+    let src = branched_onion(3, 2);
+    let mut dst = empty_onion();
+
+    let packets1 = diff_revisions(&src, NO_PARENT).unwrap();
+    let filtered1 =
+        filter_packets(packets1, &SyncSelector::Branch("main".into()), &src).unwrap();
+    merge_packets(&mut dst, filtered1, false).unwrap();
+
+    // Apply again — should skip all
+    let packets2 = diff_revisions(&src, NO_PARENT).unwrap();
+    let filtered2 =
+        filter_packets(packets2, &SyncSelector::Branch("main".into()), &src).unwrap();
+    let stats2 = merge_packets(&mut dst, filtered2, false).unwrap();
+
+    assert_eq!(stats2.revisions_merged, 0);
+    assert_eq!(stats2.revisions_skipped, 3);
+}
+
+// ─────────────────────────────────────────────────────────────────────────────
+// merge-then-verify via merge_into (end-to-end with branching)
+// ─────────────────────────────────────────────────────────────────────────────
+
+#[test]
+fn synced_remote_can_merge_branches() {
+    let src = branched_onion(3, 2);
+    let mut dst = empty_onion();
+
+    // Sync everything
+    let packets = diff_revisions(&src, NO_PARENT).unwrap();
+    merge_packets(&mut dst, packets, false).unwrap();
+
+    // The feat branch was auto-registered as "branch-{id}" during sync
+    let feat_id = src.branch_by_name("feat").unwrap().id;
+    let dst_feat_name = format!("branch-{feat_id}");
+    let result = dst.merge_into(&dst_feat_name, "main", MergeStrategy::LatestWins);
+    assert!(result.is_ok(), "merge_into failed: {result:?}");
+    // main should now have an extra revision
+    let main_revs: Vec<_> = dst
+        .list_revisions()
+        .into_iter()
+        .filter(|r| r.branch_id == 0)
+        .collect();
+    assert!(main_revs.len() >= 4); // 3 original + 1 merge commit
+}