Initial commit: ClawSync v0.1.0

8-crate pure-Rust workspace for revision-aware HDF5 sync. ## Crates - clawhdf5-onion: ClawOnion VFD — page-level versioned HDF5 storage, binary format, writer/reader, branch DAG, GC, snapshots, provenance - clawsync-core: BLAKE3, xxHash3, FastCDC (+ SIMD NEON), zstd/lz4 - clawsync-onion: IBLT sketch, Merkle tree differ, packet differ/merger, ClawSyncManifest, SyncSelector - clawsync-hdf5: dataset-level manifest, differ, patcher, wire payload reconstruction (apply_received_payloads) - clawsync-transport: TCP, QUIC (quinn 0.11/TLS 1.3), SyncPeer abstraction, length-prefixed rkyv wire protocol (21 SyncMessage variants) - clawsync-agent: OnionMemory, SyncScheduler, TcpSyncBackend, PeerCapabilities negotiation - clawsync-fs: CDC-based delta sync for any file type; FsSyncClient/Server, W=16 pipelining, atomic writes - clawsync-cli: push/pull/serve/hdf5-sync/serve-hdf5/sync/serve-fs + all local management commands; --quic on all network commands ## Key features - IBLT pre-flight: O(revision count) vs rsync's O(file size) - W=16 sliding-window push: 13–15x speedup over stop-and-wait at WAN RTT - Dataset-granular HDF5 sync: only modified datasets transferred - CDC delta for any file type: insertion-stable chunk boundaries - Full revision DAG: branch, merge, rollback, export, snapshot, GC - QUIC transport: TLS 1.3, per-message streams via quinn 0.11 ## Tests ~573 passing (default features); ~589 with --features simd-cdc ## Performance (Apple Silicon) - Reconstruct rev=100: 68 µs (target ≤ 1 ms) - BLAKE3 Rayon 1 MB: 10.3 GiB/s (target ≥ 5 GB/s) - GC 500 revisions: 20.6 µs (target ≤ 2 s) - W=16 vs W=1 at 5 ms RTT: 14.8x speedup - No-op pre-flight at 16 MB: 4 ms vs rsync 35 ms (7.8x) Co-Authored-By: Claude Sonnet 4.6 <[email protected]>
2026-04-04 18:41:22 -05:00
commit 260e15f5b6
102 changed files with 30098 additions and 0 deletions
@@ -0,0 +1,108 @@
+//! Criterion benchmarks for IBLT set reconciliation.
+//!
+//! Groups:
+//! - `iblt_manifest/encode/N`     — build + serialise an IBLT sketch
+//! - `iblt_manifest/decode/D`     — decode a difference sketch with D elements
+//! - `iblt_manifest/roundtrip/N/D` — full encode-send-decode cycle
+//! - `iblt_manifest/size/N`       — report sketch bytes vs flat manifest bytes
+//!
+//! Run:
+//!   cargo bench -p clawsync-onion -- iblt_bench
+
+use clawsync_onion::iblt::{IbltSketch, DEFAULT_HASH_COUNT, MIN_CELLS};
+use criterion::{BenchmarkId, Criterion, black_box, criterion_group, criterion_main};
+
+// ─────────────────────────────────────────────────────────────────────────────
+
+const SEED: u64 = 0x0123_4567_89AB_CDEF;
+
+fn keys(n: usize) -> Vec<u64> {
+    (0..n as u64).collect()
+}
+
+fn diff_sketches(n: usize, d: usize) -> (IbltSketch, Vec<u8>) {
+    // A has keys 0..n, B has keys 0..(n-d)
+    let a_keys = keys(n);
+    let b_keys = keys(n.saturating_sub(d));
+    // Use generous m to ensure reliable decode
+    let m = ((d * 4).max(MIN_CELLS)).max(IbltSketch::recommended_cells(n));
+    let mut a = IbltSketch::new(m, DEFAULT_HASH_COUNT, SEED);
+    let mut b = IbltSketch::new(m, DEFAULT_HASH_COUNT, SEED);
+    for &k in &a_keys { a.insert(k); }
+    for &k in &b_keys { b.insert(k); }
+    let b_bytes = b.to_bytes();
+    a.subtract(&b);
+    (a, b_bytes)
+}
+
+// ─────────────────────────────────────────────────────────────────────────────
+
+fn bench_iblt(c: &mut Criterion) {
+    let ns = [10usize, 100, 1_000, 10_000];
+    let ds = [0usize, 1, 10, 50];
+
+    // ── Encode (build + serialise) ────────────────────────────────────────────
+    let mut group = c.benchmark_group("iblt_manifest/encode");
+    for &n in &ns {
+        let ks = keys(n);
+        group.bench_with_input(BenchmarkId::from_parameter(n), &ks, |b, k| {
+            b.iter(|| {
+                let s = IbltSketch::from_keys(black_box(k), SEED);
+                black_box(s.to_bytes())
+            });
+        });
+    }
+    group.finish();
+
+    // ── Decode diff sketch ────────────────────────────────────────────────────
+    let mut group = c.benchmark_group("iblt_manifest/decode");
+    for &d in &ds {
+        let n = 1_000usize;
+        let (diff_sketch, _) = diff_sketches(n, d);
+        group.bench_with_input(BenchmarkId::from_parameter(d), &diff_sketch, |b, s| {
+            b.iter(|| black_box(s.decode()));
+        });
+    }
+    group.finish();
+
+    // ── Full roundtrip (encode A, encode B, subtract, decode) ─────────────────
+    let mut group = c.benchmark_group("iblt_manifest/roundtrip");
+    for &n in &[100usize, 1_000, 10_000] {
+        for &d in &[0usize, 10] {
+            let a_keys = keys(n);
+            let b_keys = keys(n.saturating_sub(d));
+            group.bench_with_input(
+                BenchmarkId::new(format!("n{n}"), format!("d{d}")),
+                &(a_keys, b_keys),
+                |b, (ak, bk)| {
+                    b.iter(|| {
+                        let m = (d * 4).max(MIN_CELLS).max(IbltSketch::recommended_cells(n));
+                        let mut a = IbltSketch::new(m, DEFAULT_HASH_COUNT, SEED);
+                        let mut b_sk = IbltSketch::new(m, DEFAULT_HASH_COUNT, SEED);
+                        for &k in black_box(ak) { a.insert(k); }
+                        for &k in black_box(bk) { b_sk.insert(k); }
+                        a.subtract(&b_sk);
+                        black_box(a.decode())
+                    });
+                },
+            );
+        }
+    }
+    group.finish();
+
+    // ── Wire size table (not timed) ───────────────────────────────────────────
+    println!("\n=== IBLT sketch size vs flat manifest (N × 60 B) ===");
+    println!("{:>8}  {:>10}  {:>12}  {:>10}", "N", "IBLT (B)", "Flat (B)", "ratio");
+    for &n in &[10usize, 100, 1_000, 10_000, 100_000] {
+        let ks = keys(n);
+        let sz = IbltSketch::from_keys(&ks, SEED).to_bytes().len();
+        let flat = n * 60;
+        println!(
+            "{:>8}  {:>10}  {:>12}  {:>9.1}×",
+            n, sz, flat, flat as f64 / sz as f64
+        );
+    }
+}
+
+criterion_group!(benches, bench_iblt);
+criterion_main!(benches);