Initial commit: ClawSync v0.1.0

8-crate pure-Rust workspace for revision-aware HDF5 sync. ## Crates - clawhdf5-onion: ClawOnion VFD — page-level versioned HDF5 storage, binary format, writer/reader, branch DAG, GC, snapshots, provenance - clawsync-core: BLAKE3, xxHash3, FastCDC (+ SIMD NEON), zstd/lz4 - clawsync-onion: IBLT sketch, Merkle tree differ, packet differ/merger, ClawSyncManifest, SyncSelector - clawsync-hdf5: dataset-level manifest, differ, patcher, wire payload reconstruction (apply_received_payloads) - clawsync-transport: TCP, QUIC (quinn 0.11/TLS 1.3), SyncPeer abstraction, length-prefixed rkyv wire protocol (21 SyncMessage variants) - clawsync-agent: OnionMemory, SyncScheduler, TcpSyncBackend, PeerCapabilities negotiation - clawsync-fs: CDC-based delta sync for any file type; FsSyncClient/Server, W=16 pipelining, atomic writes - clawsync-cli: push/pull/serve/hdf5-sync/serve-hdf5/sync/serve-fs + all local management commands; --quic on all network commands ## Key features - IBLT pre-flight: O(revision count) vs rsync's O(file size) - W=16 sliding-window push: 13–15x speedup over stop-and-wait at WAN RTT - Dataset-granular HDF5 sync: only modified datasets transferred - CDC delta for any file type: insertion-stable chunk boundaries - Full revision DAG: branch, merge, rollback, export, snapshot, GC - QUIC transport: TLS 1.3, per-message streams via quinn 0.11 ## Tests ~573 passing (default features); ~589 with --features simd-cdc ## Performance (Apple Silicon) - Reconstruct rev=100: 68 µs (target ≤ 1 ms) - BLAKE3 Rayon 1 MB: 10.3 GiB/s (target ≥ 5 GB/s) - GC 500 revisions: 20.6 µs (target ≤ 2 s) - W=16 vs W=1 at 5 ms RTT: 14.8x speedup - No-op pre-flight at 16 MB: 4 ms vs rsync 35 ms (7.8x) Co-Authored-By: Claude Sonnet 4.6 <[email protected]>
2026-04-04 18:41:22 -05:00
commit 260e15f5b6
102 changed files with 30098 additions and 0 deletions
@@ -0,0 +1,148 @@
+//! Sync selector: filter which revisions to include in a sync operation.
+//!
+//! Used with `--branch` CLI flag to sync only a subset of the DAG.
+
+use clawhdf5_onion::format::NO_PARENT;
+use clawhdf5_onion::writer::OnionFile;
+
+use crate::error::SyncOnionError;
+use crate::packet::OnionLayerPacket;
+
+/// Selector controlling which revisions participate in a sync.
+#[derive(Debug, Clone)]
+pub enum SyncSelector {
+    /// All revisions on all branches (default).
+    All,
+    /// Only revisions on the named branch.
+    Branch(String),
+    /// Only revisions up to (and including) a specific revision number.
+    UpTo(u64),
+    /// Only the named branch, up to a specific revision.
+    BranchAt(String, u64),
+}
+
+/// Filter a set of packets according to `selector`, keeping only those that
+/// match.  This is applied on the sender side before transmission.
+pub fn filter_packets(
+    packets: Vec<OnionLayerPacket>,
+    selector: &SyncSelector,
+    onion: &OnionFile,
+) -> Result<Vec<OnionLayerPacket>, SyncOnionError> {
+    match selector {
+        SyncSelector::All => Ok(packets),
+
+        SyncSelector::Branch(name) => {
+            let branch = onion
+                .branch_by_name(name)
+                .ok_or_else(|| SyncOnionError::BranchNotFound(name.clone()))?;
+            let branch_id = branch.id;
+            Ok(packets
+                .into_iter()
+                .filter(|p| p.branch_id == branch_id)
+                .collect())
+        }
+
+        SyncSelector::UpTo(max_rev) => Ok(packets
+            .into_iter()
+            .filter(|p| p.revision <= *max_rev)
+            .collect()),
+
+        SyncSelector::BranchAt(name, max_rev) => {
+            let branch = onion
+                .branch_by_name(name)
+                .ok_or_else(|| SyncOnionError::BranchNotFound(name.clone()))?;
+            let branch_id = branch.id;
+            Ok(packets
+                .into_iter()
+                .filter(|p| p.branch_id == branch_id && p.revision <= *max_rev)
+                .collect())
+        }
+    }
+}
+
+/// Compute the remote HEAD for a given selector on the local `OnionFile`.
+///
+/// Returns [`NO_PARENT`] if no revisions match (full sync needed).
+pub fn remote_head_for_selector(
+    _selector: &SyncSelector,
+    remote_revision_count: u64,
+) -> u64 {
+    // For a simple linear sync, remote HEAD = remote_revision_count - 1
+    // (or NO_PARENT if empty).
+    if remote_revision_count == 0 {
+        NO_PARENT
+    } else {
+        remote_revision_count - 1
+    }
+}
+
+// ─────────────────────────────────────────────────────────────────────────────
+// Tests
+// ─────────────────────────────────────────────────────────────────────────────
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use crate::differ::diff_revisions;
+    use clawhdf5_onion::format::NO_PARENT;
+    use clawhdf5_onion::writer::OnionFile;
+    use tempfile::NamedTempFile;
+
+    fn make_onion(n: u8) -> OnionFile {
+        let f = NamedTempFile::new().unwrap();
+        let h5 = f.path().with_extension("h5");
+        std::fs::write(&h5, b"\x89HDF\r\n\x1a\n").unwrap();
+        let mut onion = OnionFile::create(&h5, 4096).unwrap();
+        for i in 0..n {
+            let mut s = onion.begin_session(None).unwrap();
+            s.record_page(0, &vec![i; 4096]);
+            onion.commit_session(s, None).unwrap();
+        }
+        onion
+    }
+
+    #[test]
+    fn selector_all_passes_everything() {
+        let onion = make_onion(4);
+        let packets = diff_revisions(&onion, NO_PARENT).unwrap();
+        let filtered = filter_packets(packets, &SyncSelector::All, &onion).unwrap();
+        assert_eq!(filtered.len(), 4);
+    }
+
+    #[test]
+    fn selector_up_to() {
+        let onion = make_onion(5);
+        let packets = diff_revisions(&onion, NO_PARENT).unwrap();
+        let filtered = filter_packets(packets, &SyncSelector::UpTo(2), &onion).unwrap();
+        assert_eq!(filtered.len(), 3); // revisions 0, 1, 2
+    }
+
+    #[test]
+    fn selector_branch_main() {
+        let onion = make_onion(3);
+        let packets = diff_revisions(&onion, NO_PARENT).unwrap();
+        // All revisions are on main (branch_id = 0)
+        let filtered =
+            filter_packets(packets, &SyncSelector::Branch("main".to_string()), &onion).unwrap();
+        assert_eq!(filtered.len(), 3);
+    }
+
+    #[test]
+    fn selector_branch_nonexistent_errors() {
+        let onion = make_onion(2);
+        let packets = diff_revisions(&onion, NO_PARENT).unwrap();
+        let result =
+            filter_packets(packets, &SyncSelector::Branch("no-such".to_string()), &onion);
+        assert!(matches!(result, Err(SyncOnionError::BranchNotFound(_))));
+    }
+
+    #[test]
+    fn remote_head_empty() {
+        assert_eq!(remote_head_for_selector(&SyncSelector::All, 0), NO_PARENT);
+    }
+
+    #[test]
+    fn remote_head_non_empty() {
+        assert_eq!(remote_head_for_selector(&SyncSelector::All, 5), 4);
+    }
+}