Initial commit: ClawSync v0.1.0

8-crate pure-Rust workspace for revision-aware HDF5 sync. ## Crates - clawhdf5-onion: ClawOnion VFD — page-level versioned HDF5 storage, binary format, writer/reader, branch DAG, GC, snapshots, provenance - clawsync-core: BLAKE3, xxHash3, FastCDC (+ SIMD NEON), zstd/lz4 - clawsync-onion: IBLT sketch, Merkle tree differ, packet differ/merger, ClawSyncManifest, SyncSelector - clawsync-hdf5: dataset-level manifest, differ, patcher, wire payload reconstruction (apply_received_payloads) - clawsync-transport: TCP, QUIC (quinn 0.11/TLS 1.3), SyncPeer abstraction, length-prefixed rkyv wire protocol (21 SyncMessage variants) - clawsync-agent: OnionMemory, SyncScheduler, TcpSyncBackend, PeerCapabilities negotiation - clawsync-fs: CDC-based delta sync for any file type; FsSyncClient/Server, W=16 pipelining, atomic writes - clawsync-cli: push/pull/serve/hdf5-sync/serve-hdf5/sync/serve-fs + all local management commands; --quic on all network commands ## Key features - IBLT pre-flight: O(revision count) vs rsync's O(file size) - W=16 sliding-window push: 13–15x speedup over stop-and-wait at WAN RTT - Dataset-granular HDF5 sync: only modified datasets transferred - CDC delta for any file type: insertion-stable chunk boundaries - Full revision DAG: branch, merge, rollback, export, snapshot, GC - QUIC transport: TLS 1.3, per-message streams via quinn 0.11 ## Tests ~573 passing (default features); ~589 with --features simd-cdc ## Performance (Apple Silicon) - Reconstruct rev=100: 68 µs (target ≤ 1 ms) - BLAKE3 Rayon 1 MB: 10.3 GiB/s (target ≥ 5 GB/s) - GC 500 revisions: 20.6 µs (target ≤ 2 s) - W=16 vs W=1 at 5 ms RTT: 14.8x speedup - No-op pre-flight at 16 MB: 4 ms vs rsync 35 ms (7.8x) Co-Authored-By: Claude Sonnet 4.6 <[email protected]>
2026-04-04 18:41:22 -05:00
commit 260e15f5b6
102 changed files with 30098 additions and 0 deletions
@@ -0,0 +1,108 @@
+//! Criterion benchmarks for epoch-based (lazy) GC vs immediate GC.
+//!
+//! Measures:
+//! - `gc_keepn_immediate/N` — existing O(N·P) immediate compact path
+//! - `gc_epoch_flip/N`      — new O(N) mark-only path
+//! - `flush_after_epoch_gc/N` — deferred compaction cost inside flush()
+//!
+//! Run:
+//!   cargo bench -p clawhdf5-onion -- epoch_gc
+
+use clawhdf5_onion::gc::GcPolicy;
+use clawhdf5_onion::writer::OnionFile;
+use criterion::{BenchmarkId, Criterion, black_box, criterion_group, criterion_main};
+use tempfile::NamedTempFile;
+
+const PAGE_SIZE: u32 = 4096;
+
+// ─────────────────────────────────────────────────────────────────────────────
+// Helpers
+// ─────────────────────────────────────────────────────────────────────────────
+
+fn make_onion(n: usize) -> (NamedTempFile, std::path::PathBuf, OnionFile) {
+    let tmp = NamedTempFile::new().unwrap();
+    let h5 = tmp.path().with_extension("h5");
+    std::fs::write(&h5, b"\x89HDF\r\n\x1a\n").unwrap();
+    let mut onion = OnionFile::create(&h5, PAGE_SIZE).unwrap();
+    for i in 0..n {
+        let mut s = onion.begin_session(None).unwrap();
+        s.record_page(0, &vec![(i % 256) as u8; PAGE_SIZE as usize]);
+        onion.commit_session(s, None).unwrap();
+    }
+    (tmp, h5, onion)
+}
+
+// ─────────────────────────────────────────────────────────────────────────────
+// Benchmarks
+// ─────────────────────────────────────────────────────────────────────────────
+
+fn bench_epoch_gc(c: &mut Criterion) {
+    let sizes = [100usize, 500, 1000];
+
+    // ── Immediate GC (baseline) ───────────────────────────────────────────────
+    let mut group = c.benchmark_group("epoch_gc/immediate");
+    for &n in &sizes {
+        group.bench_with_input(BenchmarkId::new("keep_last_half", n), &n, |b, &n| {
+            b.iter_with_setup(
+                || make_onion(n),
+                |(_tmp, _h5, mut onion)| {
+                    let _ = onion.gc(black_box(GcPolicy::KeepLastN((n / 2) as u64))).unwrap();
+                },
+            );
+        });
+    }
+    group.finish();
+
+    // ── Epoch flip (mark only, no compaction) ─────────────────────────────────
+    let mut group = c.benchmark_group("epoch_gc/epoch_flip_mark");
+    for &n in &sizes {
+        group.bench_with_input(BenchmarkId::new("keep_last_half", n), &n, |b, &n| {
+            b.iter_with_setup(
+                || make_onion(n),
+                |(_tmp, _h5, mut onion)| {
+                    let _ = onion.gc(black_box(GcPolicy::EpochFlip(Box::new(
+                        GcPolicy::KeepLastN((n / 2) as u64),
+                    )))).unwrap();
+                },
+            );
+        });
+    }
+    group.finish();
+
+    // ── Flush after epoch flip (deferred compaction cost) ────────────────────
+    let mut group = c.benchmark_group("epoch_gc/flush_after_epoch_flip");
+    for &n in &sizes {
+        group.bench_with_input(BenchmarkId::new("keep_last_half", n), &n, |b, &n| {
+            b.iter_with_setup(
+                || {
+                    let (tmp, h5, mut onion) = make_onion(n);
+                    onion.gc(GcPolicy::EpochFlip(Box::new(
+                        GcPolicy::KeepLastN((n / 2) as u64),
+                    ))).unwrap();
+                    (tmp, h5, onion)
+                },
+                |(_tmp, _h5, mut onion)| {
+                    onion.flush().unwrap();
+                },
+            );
+        });
+    }
+    group.finish();
+
+    // ── Comparison: flush without any GC (baseline for flush cost) ───────────
+    let mut group = c.benchmark_group("epoch_gc/flush_no_gc");
+    for &n in &sizes {
+        group.bench_with_input(BenchmarkId::new("n_revisions", n), &n, |b, &n| {
+            b.iter_with_setup(
+                || make_onion(n),
+                |(_tmp, _h5, mut onion)| {
+                    onion.flush().unwrap();
+                },
+            );
+        });
+    }
+    group.finish();
+}
+
+criterion_group!(benches, bench_epoch_gc);
+criterion_main!(benches);