Initial commit: ClawSync v0.1.0
8-crate pure-Rust workspace for revision-aware HDF5 sync. ## Crates - clawhdf5-onion: ClawOnion VFD — page-level versioned HDF5 storage, binary format, writer/reader, branch DAG, GC, snapshots, provenance - clawsync-core: BLAKE3, xxHash3, FastCDC (+ SIMD NEON), zstd/lz4 - clawsync-onion: IBLT sketch, Merkle tree differ, packet differ/merger, ClawSyncManifest, SyncSelector - clawsync-hdf5: dataset-level manifest, differ, patcher, wire payload reconstruction (apply_received_payloads) - clawsync-transport: TCP, QUIC (quinn 0.11/TLS 1.3), SyncPeer abstraction, length-prefixed rkyv wire protocol (21 SyncMessage variants) - clawsync-agent: OnionMemory, SyncScheduler, TcpSyncBackend, PeerCapabilities negotiation - clawsync-fs: CDC-based delta sync for any file type; FsSyncClient/Server, W=16 pipelining, atomic writes - clawsync-cli: push/pull/serve/hdf5-sync/serve-hdf5/sync/serve-fs + all local management commands; --quic on all network commands ## Key features - IBLT pre-flight: O(revision count) vs rsync's O(file size) - W=16 sliding-window push: 13–15x speedup over stop-and-wait at WAN RTT - Dataset-granular HDF5 sync: only modified datasets transferred - CDC delta for any file type: insertion-stable chunk boundaries - Full revision DAG: branch, merge, rollback, export, snapshot, GC - QUIC transport: TLS 1.3, per-message streams via quinn 0.11 ## Tests ~573 passing (default features); ~589 with --features simd-cdc ## Performance (Apple Silicon) - Reconstruct rev=100: 68 µs (target ≤ 1 ms) - BLAKE3 Rayon 1 MB: 10.3 GiB/s (target ≥ 5 GB/s) - GC 500 revisions: 20.6 µs (target ≤ 2 s) - W=16 vs W=1 at 5 ms RTT: 14.8x speedup - No-op pre-flight at 16 MB: 4 ms vs rsync 35 ms (7.8x) Co-Authored-By: Claude Sonnet 4.6 <[email protected]>
This commit is contained in:
osobh
@@ -0,0 +1,115 @@
|
||||
//! Criterion benchmarks: FastCDC vs SIMD Gear hash CDC.
|
||||
//!
|
||||
//! Run:
|
||||
//! cargo bench -p clawsync-core --features simd-cdc -- cdc_bench
|
||||
//!
|
||||
//! Groups:
|
||||
//! - `cdc/fastcdc/{data_type}/{size}` — baseline (FastCDC from `fastcdc` crate)
|
||||
//! - `cdc/simd/{data_type}/{size}` — SIMD Gear hash path
|
||||
//! - `cdc/scalar/{data_type}/{size}` — scalar Gear hash (same algorithm, no SIMD skip)
|
||||
|
||||
use clawsync_core::cdc::{chunk_data, chunk_data_simd};
|
||||
#[cfg(feature = "simd-cdc")]
|
||||
use clawsync_core::simd_cdc::chunk_scalar;
|
||||
use criterion::{BenchmarkId, Criterion, Throughput, black_box, criterion_group, criterion_main};
|
||||
|
||||
// ─────────────────────────────────────────────────────────────────────────────
|
||||
// Data generators
|
||||
// ─────────────────────────────────────────────────────────────────────────────
|
||||
|
||||
fn random_data(n: usize) -> Vec<u8> {
|
||||
(0..n)
|
||||
.map(|i| {
|
||||
let v = (i as u64)
|
||||
.wrapping_mul(6364136223846793005)
|
||||
.wrapping_add(1442695040888963407);
|
||||
(v ^ (v >> 33)) as u8
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
/// Simulates IEEE 754 float array bytes: exponent bytes cluster in 0x3F-0x45,
|
||||
/// mantissa bytes spread across 0x00-0xFF.
|
||||
fn float_data(n: usize) -> Vec<u8> {
|
||||
let mut v = Vec::with_capacity(n);
|
||||
let mut i = 0usize;
|
||||
while i + 4 <= n {
|
||||
// Synthetic f32: sign(0) | exp(127±small) | mantissa
|
||||
let exp_byte = 0x3Fu8.wrapping_add((i % 16) as u8);
|
||||
let mant0 = (i.wrapping_mul(2654435769) >> 24) as u8;
|
||||
let mant1 = (i.wrapping_mul(0x811c9dc5) >> 16) as u8;
|
||||
let mant2 = (i.wrapping_mul(0x01000193)) as u8;
|
||||
v.push(mant2); // byte 0: low mantissa (often < 64 → hot)
|
||||
v.push(mant1);
|
||||
v.push(mant0);
|
||||
v.push(exp_byte); // byte 3: exponent (often 0x3F-0x45 ≥ 64 → cold)
|
||||
i += 4;
|
||||
}
|
||||
v.truncate(n);
|
||||
v
|
||||
}
|
||||
|
||||
/// All bytes are cold (≥ 64) — worst case for SIMD skip, exercises batch-shift path.
|
||||
fn cold_data(n: usize) -> Vec<u8> {
|
||||
(0..n).map(|i| 64u8 + ((i * 7) % 192) as u8).collect()
|
||||
}
|
||||
|
||||
// ─────────────────────────────────────────────────────────────────────────────
|
||||
// Benchmark
|
||||
// ─────────────────────────────────────────────────────────────────────────────
|
||||
|
||||
fn bench_cdc(c: &mut Criterion) {
|
||||
let datasets: &[(&str, fn(usize) -> Vec<u8>)] = &[
|
||||
("random", random_data),
|
||||
("float", float_data),
|
||||
("cold_only", cold_data),
|
||||
];
|
||||
let sizes = [1 << 20, 4 << 20, 16 << 20]; // 1 MB, 4 MB, 16 MB
|
||||
|
||||
for &(label, make) in datasets {
|
||||
// ── FastCDC (baseline) ───────────────────────────────────────────────
|
||||
let mut group = c.benchmark_group(format!("cdc/fastcdc/{label}"));
|
||||
for &size in &sizes {
|
||||
let data = make(size);
|
||||
group.throughput(Throughput::Bytes(size as u64));
|
||||
group.bench_with_input(
|
||||
BenchmarkId::new("chunk_data", format!("{}MB", size >> 20)),
|
||||
&data,
|
||||
|b, d| b.iter(|| chunk_data(black_box(d))),
|
||||
);
|
||||
}
|
||||
group.finish();
|
||||
|
||||
// ── SIMD Gear hash ───────────────────────────────────────────────────
|
||||
let mut group = c.benchmark_group(format!("cdc/simd/{label}"));
|
||||
for &size in &sizes {
|
||||
let data = make(size);
|
||||
group.throughput(Throughput::Bytes(size as u64));
|
||||
group.bench_with_input(
|
||||
BenchmarkId::new("chunk_data_simd", format!("{}MB", size >> 20)),
|
||||
&data,
|
||||
|b, d| b.iter(|| chunk_data_simd(black_box(d))),
|
||||
);
|
||||
}
|
||||
group.finish();
|
||||
|
||||
// ── Scalar Gear hash (no SIMD skip) ──────────────────────────────────
|
||||
#[cfg(feature = "simd-cdc")]
|
||||
{
|
||||
let mut group = c.benchmark_group(format!("cdc/scalar/{label}"));
|
||||
for &size in &sizes {
|
||||
let data = make(size);
|
||||
group.throughput(Throughput::Bytes(size as u64));
|
||||
group.bench_with_input(
|
||||
BenchmarkId::new("chunk_scalar", format!("{}MB", size >> 20)),
|
||||
&data,
|
||||
|b, d| b.iter(|| chunk_scalar(black_box(d), 8192, 65536, 262144)),
|
||||
);
|
||||
}
|
||||
group.finish();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
criterion_group!(benches, bench_cdc);
|
||||
criterion_main!(benches);
|
||||
Reference in New Issue
Block a user