Initial commit: ClawSync v0.1.0
8-crate pure-Rust workspace for revision-aware HDF5 sync. ## Crates - clawhdf5-onion: ClawOnion VFD — page-level versioned HDF5 storage, binary format, writer/reader, branch DAG, GC, snapshots, provenance - clawsync-core: BLAKE3, xxHash3, FastCDC (+ SIMD NEON), zstd/lz4 - clawsync-onion: IBLT sketch, Merkle tree differ, packet differ/merger, ClawSyncManifest, SyncSelector - clawsync-hdf5: dataset-level manifest, differ, patcher, wire payload reconstruction (apply_received_payloads) - clawsync-transport: TCP, QUIC (quinn 0.11/TLS 1.3), SyncPeer abstraction, length-prefixed rkyv wire protocol (21 SyncMessage variants) - clawsync-agent: OnionMemory, SyncScheduler, TcpSyncBackend, PeerCapabilities negotiation - clawsync-fs: CDC-based delta sync for any file type; FsSyncClient/Server, W=16 pipelining, atomic writes - clawsync-cli: push/pull/serve/hdf5-sync/serve-hdf5/sync/serve-fs + all local management commands; --quic on all network commands ## Key features - IBLT pre-flight: O(revision count) vs rsync's O(file size) - W=16 sliding-window push: 13–15x speedup over stop-and-wait at WAN RTT - Dataset-granular HDF5 sync: only modified datasets transferred - CDC delta for any file type: insertion-stable chunk boundaries - Full revision DAG: branch, merge, rollback, export, snapshot, GC - QUIC transport: TLS 1.3, per-message streams via quinn 0.11 ## Tests ~573 passing (default features); ~589 with --features simd-cdc ## Performance (Apple Silicon) - Reconstruct rev=100: 68 µs (target ≤ 1 ms) - BLAKE3 Rayon 1 MB: 10.3 GiB/s (target ≥ 5 GB/s) - GC 500 revisions: 20.6 µs (target ≤ 2 s) - W=16 vs W=1 at 5 ms RTT: 14.8x speedup - No-op pre-flight at 16 MB: 4 ms vs rsync 35 ms (7.8x) Co-Authored-By: Claude Sonnet 4.6 <[email protected]>
This commit is contained in:
osobh
@@ -0,0 +1,145 @@
|
||||
//! Criterion benchmarks for the TDT byte-interleaving transform + ZstdTdt codec.
|
||||
//!
|
||||
//! Measures compression ratio and throughput for common HDF5 data types,
|
||||
//! comparing plain Zstd against TDT-transformed Zstd.
|
||||
//!
|
||||
//! Run:
|
||||
//! cargo bench -p clawhdf5-onion -- tdt_compress
|
||||
|
||||
use clawhdf5_onion::compress::{compress_page, decompress_page};
|
||||
use clawhdf5_onion::format::Codec;
|
||||
use clawhdf5_onion::tdt;
|
||||
use criterion::{BenchmarkId, Criterion, Throughput, black_box, criterion_group, criterion_main};
|
||||
|
||||
// ─────────────────────────────────────────────────────────────────────────────
|
||||
// Test data generators
|
||||
// ─────────────────────────────────────────────────────────────────────────────
|
||||
|
||||
/// Smooth f32: values from a sine wave — good compressibility, representative
|
||||
/// of neural network activation or weight tensors.
|
||||
fn make_f32_smooth(n_bytes: usize) -> Vec<u8> {
|
||||
let n_floats = n_bytes / 4;
|
||||
let mut out = Vec::with_capacity(n_bytes);
|
||||
for i in 0..n_floats {
|
||||
let v = ((i as f64 / 256.0).sin() as f32).to_le_bytes();
|
||||
out.extend_from_slice(&v);
|
||||
}
|
||||
// Pad to exact size if n_bytes % 4 != 0.
|
||||
out.resize(n_bytes, 0);
|
||||
out
|
||||
}
|
||||
|
||||
/// Random f32: worst case for compression.
|
||||
fn make_f32_random(n_bytes: usize) -> Vec<u8> {
|
||||
// Deterministic LCG so benchmarks are reproducible.
|
||||
let mut state = 0x_dead_beef_u64;
|
||||
let mut out = Vec::with_capacity(n_bytes);
|
||||
while out.len() < n_bytes {
|
||||
state = state.wrapping_mul(6364136223846793005).wrapping_add(1442695040888963407);
|
||||
out.extend_from_slice(&(state as u32).to_le_bytes());
|
||||
}
|
||||
out.truncate(n_bytes);
|
||||
out
|
||||
}
|
||||
|
||||
/// Random i32 data.
|
||||
fn make_int32_random(n_bytes: usize) -> Vec<u8> {
|
||||
make_f32_random(n_bytes) // same byte distribution
|
||||
}
|
||||
|
||||
/// Sequential i8 data (highly compressible, TDT should be neutral).
|
||||
fn make_int8_seq(n_bytes: usize) -> Vec<u8> {
|
||||
(0u8..=255).cycle().take(n_bytes).collect()
|
||||
}
|
||||
|
||||
// ─────────────────────────────────────────────────────────────────────────────
|
||||
// Helpers
|
||||
// ─────────────────────────────────────────────────────────────────────────────
|
||||
|
||||
/// Run compress + decompress roundtrip and return compressed size.
|
||||
fn bench_compress(data: &[u8], codec: Codec) -> usize {
|
||||
let compressed = compress_page(black_box(data), codec).unwrap();
|
||||
let size = compressed.len();
|
||||
let _ = decompress_page(black_box(&compressed), codec, data.len() as u32).unwrap();
|
||||
size
|
||||
}
|
||||
|
||||
// ─────────────────────────────────────────────────────────────────────────────
|
||||
// Benchmark groups
|
||||
// ─────────────────────────────────────────────────────────────────────────────
|
||||
|
||||
fn bench_tdt_compress(c: &mut Criterion) {
|
||||
let sizes = [4096usize, 65536];
|
||||
|
||||
let datasets: &[(&str, fn(usize) -> Vec<u8>, usize)] = &[
|
||||
("f32_smooth", make_f32_smooth, 4),
|
||||
("f32_random", make_f32_random, 4),
|
||||
("int32_random", make_int32_random, 4),
|
||||
("int8_seq", make_int8_seq, 1),
|
||||
];
|
||||
|
||||
let codecs = [
|
||||
("zstd", Codec::Zstd),
|
||||
("zstd_tdt", Codec::ZstdTdt),
|
||||
];
|
||||
|
||||
// ── Throughput benchmark ──────────────────────────────────────────────────
|
||||
let mut group = c.benchmark_group("tdt_compress/throughput");
|
||||
for &size in &sizes {
|
||||
for &(dtype, make, _width) in datasets {
|
||||
let data = make(size);
|
||||
group.throughput(Throughput::Bytes(size as u64));
|
||||
for &(codec_name, codec) in &codecs {
|
||||
let id = BenchmarkId::new(
|
||||
format!("{codec_name}/{dtype}"),
|
||||
format!("{size}B"),
|
||||
);
|
||||
group.bench_with_input(id, &data, |b, d| {
|
||||
b.iter(|| bench_compress(d, codec));
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
group.finish();
|
||||
|
||||
// ── TDT transform only (encode + decode, without compression) ────────────
|
||||
let mut group = c.benchmark_group("tdt_compress/transform_only");
|
||||
for &size in &sizes {
|
||||
for &(dtype, make, width) in datasets {
|
||||
let data = make(size);
|
||||
group.throughput(Throughput::Bytes(size as u64));
|
||||
|
||||
let id = BenchmarkId::new(format!("encode/{dtype}"), format!("{size}B"));
|
||||
group.bench_with_input(id, &data, |b, d| {
|
||||
b.iter(|| tdt::encode(black_box(d), width));
|
||||
});
|
||||
|
||||
let encoded = tdt::encode(&data, width);
|
||||
let id = BenchmarkId::new(format!("decode/{dtype}"), format!("{size}B"));
|
||||
group.bench_with_input(id, &encoded, |b, enc| {
|
||||
b.iter(|| tdt::decode(black_box(enc), width, size));
|
||||
});
|
||||
}
|
||||
}
|
||||
group.finish();
|
||||
|
||||
// ── Compression ratio summary (printed, not timed) ───────────────────────
|
||||
// Run once outside Criterion to print ratio comparison.
|
||||
println!("\n─── TDT compression ratio summary ───");
|
||||
println!("{:<20} {:>8} {:>10} {:>10} {:>8}",
|
||||
"dataset/size", "orig", "zstd", "zstd_tdt", "savings");
|
||||
for &size in &sizes {
|
||||
for &(dtype, make, _w) in datasets {
|
||||
let data = make(size);
|
||||
let zstd_size = compress_page(&data, Codec::Zstd).unwrap().len();
|
||||
let tdt_size = compress_page(&data, Codec::ZstdTdt).unwrap().len();
|
||||
let savings_pct = 100.0 * (1.0 - tdt_size as f64 / zstd_size as f64);
|
||||
println!("{:<20} {:>8} {:>10} {:>10} {:>7.1}%",
|
||||
format!("{dtype}/{size}B"), size, zstd_size, tdt_size, savings_pct);
|
||||
}
|
||||
}
|
||||
println!();
|
||||
}
|
||||
|
||||
criterion_group!(benches, bench_tdt_compress);
|
||||
criterion_main!(benches);
|
||||
Reference in New Issue
Block a user