Initial commit: ClawSync v0.1.0
8-crate pure-Rust workspace for revision-aware HDF5 sync. ## Crates - clawhdf5-onion: ClawOnion VFD — page-level versioned HDF5 storage, binary format, writer/reader, branch DAG, GC, snapshots, provenance - clawsync-core: BLAKE3, xxHash3, FastCDC (+ SIMD NEON), zstd/lz4 - clawsync-onion: IBLT sketch, Merkle tree differ, packet differ/merger, ClawSyncManifest, SyncSelector - clawsync-hdf5: dataset-level manifest, differ, patcher, wire payload reconstruction (apply_received_payloads) - clawsync-transport: TCP, QUIC (quinn 0.11/TLS 1.3), SyncPeer abstraction, length-prefixed rkyv wire protocol (21 SyncMessage variants) - clawsync-agent: OnionMemory, SyncScheduler, TcpSyncBackend, PeerCapabilities negotiation - clawsync-fs: CDC-based delta sync for any file type; FsSyncClient/Server, W=16 pipelining, atomic writes - clawsync-cli: push/pull/serve/hdf5-sync/serve-hdf5/sync/serve-fs + all local management commands; --quic on all network commands ## Key features - IBLT pre-flight: O(revision count) vs rsync's O(file size) - W=16 sliding-window push: 13–15x speedup over stop-and-wait at WAN RTT - Dataset-granular HDF5 sync: only modified datasets transferred - CDC delta for any file type: insertion-stable chunk boundaries - Full revision DAG: branch, merge, rollback, export, snapshot, GC - QUIC transport: TLS 1.3, per-message streams via quinn 0.11 ## Tests ~573 passing (default features); ~589 with --features simd-cdc ## Performance (Apple Silicon) - Reconstruct rev=100: 68 µs (target ≤ 1 ms) - BLAKE3 Rayon 1 MB: 10.3 GiB/s (target ≥ 5 GB/s) - GC 500 revisions: 20.6 µs (target ≤ 2 s) - W=16 vs W=1 at 5 ms RTT: 14.8x speedup - No-op pre-flight at 16 MB: 4 ms vs rsync 35 ms (7.8x) Co-Authored-By: Claude Sonnet 4.6 <[email protected]>
This commit is contained in:
osobh
@@ -0,0 +1,119 @@
|
||||
//! Criterion benchmarks for the revision Merkle tree.
|
||||
//!
|
||||
//! Groups:
|
||||
//! - `merkle_index/build_tree/N` — build tree from N revisions
|
||||
//! - `merkle_index/diff_walk/N/D` — diff walk with D differing revisions
|
||||
//! - `merkle_index/serialise/N` — serialise N-leaf tree
|
||||
//! - `merkle_index/deserialise/N` — deserialise N-leaf tree
|
||||
//! - `merkle_index/serialised_size/N` — report bytes vs flat manifest
|
||||
//!
|
||||
//! Run:
|
||||
//! cargo bench -p clawhdf5-onion -- merkle_bench
|
||||
|
||||
use clawhdf5_onion::merkle::RevisionMerkleTree;
|
||||
use criterion::{BenchmarkId, Criterion, black_box, criterion_group, criterion_main};
|
||||
|
||||
fn rev_hash(r: u64) -> [u8; 32] {
|
||||
*blake3::hash(&r.to_le_bytes()).as_bytes()
|
||||
}
|
||||
|
||||
fn make_entries(n: usize) -> Vec<(u64, [u8; 32])> {
|
||||
(0..n as u64).map(|r| (r, rev_hash(r))).collect()
|
||||
}
|
||||
|
||||
// ─────────────────────────────────────────────────────────────────────────────
|
||||
|
||||
fn bench_merkle(c: &mut Criterion) {
|
||||
let ns = [100usize, 1_000, 10_000];
|
||||
|
||||
// ── Build ─────────────────────────────────────────────────────────────────
|
||||
let mut group = c.benchmark_group("merkle_index/build_tree");
|
||||
for &n in &ns {
|
||||
let entries = make_entries(n);
|
||||
group.bench_with_input(BenchmarkId::from_parameter(n), &entries, |b, e| {
|
||||
b.iter(|| RevisionMerkleTree::build(black_box(e)));
|
||||
});
|
||||
}
|
||||
group.finish();
|
||||
|
||||
// ── Diff walk: D=0 (all match) ────────────────────────────────────────────
|
||||
let mut group = c.benchmark_group("merkle_index/diff_walk_d0");
|
||||
for &n in &ns {
|
||||
let tree = RevisionMerkleTree::build(&make_entries(n));
|
||||
group.bench_with_input(BenchmarkId::from_parameter(n), &tree, |b, t| {
|
||||
b.iter(|| t.diff_missing_revisions(black_box(t)));
|
||||
});
|
||||
}
|
||||
group.finish();
|
||||
|
||||
// ── Diff walk: D=1 ───────────────────────────────────────────────────────
|
||||
let mut group = c.benchmark_group("merkle_index/diff_walk_d1");
|
||||
for &n in &ns {
|
||||
let full = RevisionMerkleTree::build(&make_entries(n));
|
||||
let minus1 = RevisionMerkleTree::build(&make_entries(n - 1));
|
||||
group.bench_with_input(BenchmarkId::from_parameter(n), &(full, minus1), |b, (f, m)| {
|
||||
b.iter(|| f.diff_missing_revisions(black_box(m)));
|
||||
});
|
||||
}
|
||||
group.finish();
|
||||
|
||||
// ── Diff walk: D=10 ──────────────────────────────────────────────────────
|
||||
let mut group = c.benchmark_group("merkle_index/diff_walk_d10");
|
||||
for &n in &ns {
|
||||
if n < 20 { continue; }
|
||||
let full = RevisionMerkleTree::build(&make_entries(n));
|
||||
let base = RevisionMerkleTree::build(&make_entries(n - 10));
|
||||
group.bench_with_input(BenchmarkId::from_parameter(n), &(full, base), |b, (f, base)| {
|
||||
b.iter(|| f.diff_missing_revisions(black_box(base)));
|
||||
});
|
||||
}
|
||||
group.finish();
|
||||
|
||||
// ── Diff walk: D=100 ─────────────────────────────────────────────────────
|
||||
let mut group = c.benchmark_group("merkle_index/diff_walk_d100");
|
||||
for &n in &[1_000usize, 10_000] {
|
||||
let full = RevisionMerkleTree::build(&make_entries(n));
|
||||
let base = RevisionMerkleTree::build(&make_entries(n - 100));
|
||||
group.bench_with_input(BenchmarkId::from_parameter(n), &(full, base), |b, (f, base)| {
|
||||
b.iter(|| f.diff_missing_revisions(black_box(base)));
|
||||
});
|
||||
}
|
||||
group.finish();
|
||||
|
||||
// ── Serialise ─────────────────────────────────────────────────────────────
|
||||
let mut group = c.benchmark_group("merkle_index/serialise");
|
||||
for &n in &ns {
|
||||
let tree = RevisionMerkleTree::build(&make_entries(n));
|
||||
group.bench_with_input(BenchmarkId::from_parameter(n), &tree, |b, t| {
|
||||
b.iter(|| black_box(t.serialise()));
|
||||
});
|
||||
}
|
||||
group.finish();
|
||||
|
||||
// ── Deserialise ───────────────────────────────────────────────────────────
|
||||
let mut group = c.benchmark_group("merkle_index/deserialise");
|
||||
for &n in &ns {
|
||||
let bytes = RevisionMerkleTree::build(&make_entries(n)).serialise();
|
||||
group.bench_with_input(BenchmarkId::from_parameter(n), &bytes, |b, b_| {
|
||||
b.iter(|| RevisionMerkleTree::deserialise(black_box(b_)).unwrap());
|
||||
});
|
||||
}
|
||||
group.finish();
|
||||
|
||||
// ── Serialised size report (not timed) ───────────────────────────────────
|
||||
println!("\n=== Serialised size vs flat manifest (N × 60 B) ===");
|
||||
println!("{:>8} {:>12} {:>12} {:>10}", "N", "Merkle (B)", "Flat (B)", "ratio");
|
||||
for &n in &[10usize, 100, 1_000, 10_000] {
|
||||
let tree = RevisionMerkleTree::build(&make_entries(n));
|
||||
let merkle_sz = tree.serialise().len();
|
||||
let flat_sz = n * 60;
|
||||
println!(
|
||||
"{:>8} {:>12} {:>12} {:>10.2}x",
|
||||
n, merkle_sz, flat_sz,
|
||||
flat_sz as f64 / merkle_sz as f64
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
criterion_group!(benches, bench_merkle);
|
||||
criterion_main!(benches);
|
||||
Reference in New Issue
Block a user