Initial commit: ClawSync v0.1.0
8-crate pure-Rust workspace for revision-aware HDF5 sync. ## Crates - clawhdf5-onion: ClawOnion VFD — page-level versioned HDF5 storage, binary format, writer/reader, branch DAG, GC, snapshots, provenance - clawsync-core: BLAKE3, xxHash3, FastCDC (+ SIMD NEON), zstd/lz4 - clawsync-onion: IBLT sketch, Merkle tree differ, packet differ/merger, ClawSyncManifest, SyncSelector - clawsync-hdf5: dataset-level manifest, differ, patcher, wire payload reconstruction (apply_received_payloads) - clawsync-transport: TCP, QUIC (quinn 0.11/TLS 1.3), SyncPeer abstraction, length-prefixed rkyv wire protocol (21 SyncMessage variants) - clawsync-agent: OnionMemory, SyncScheduler, TcpSyncBackend, PeerCapabilities negotiation - clawsync-fs: CDC-based delta sync for any file type; FsSyncClient/Server, W=16 pipelining, atomic writes - clawsync-cli: push/pull/serve/hdf5-sync/serve-hdf5/sync/serve-fs + all local management commands; --quic on all network commands ## Key features - IBLT pre-flight: O(revision count) vs rsync's O(file size) - W=16 sliding-window push: 13–15x speedup over stop-and-wait at WAN RTT - Dataset-granular HDF5 sync: only modified datasets transferred - CDC delta for any file type: insertion-stable chunk boundaries - Full revision DAG: branch, merge, rollback, export, snapshot, GC - QUIC transport: TLS 1.3, per-message streams via quinn 0.11 ## Tests ~573 passing (default features); ~589 with --features simd-cdc ## Performance (Apple Silicon) - Reconstruct rev=100: 68 µs (target ≤ 1 ms) - BLAKE3 Rayon 1 MB: 10.3 GiB/s (target ≥ 5 GB/s) - GC 500 revisions: 20.6 µs (target ≤ 2 s) - W=16 vs W=1 at 5 ms RTT: 14.8x speedup - No-op pre-flight at 16 MB: 4 ms vs rsync 35 ms (7.8x) Co-Authored-By: Claude Sonnet 4.6 <[email protected]>
This commit is contained in:
osobh
@@ -0,0 +1,108 @@
|
||||
//! Criterion benchmarks for epoch-based (lazy) GC vs immediate GC.
|
||||
//!
|
||||
//! Measures:
|
||||
//! - `gc_keepn_immediate/N` — existing O(N·P) immediate compact path
|
||||
//! - `gc_epoch_flip/N` — new O(N) mark-only path
|
||||
//! - `flush_after_epoch_gc/N` — deferred compaction cost inside flush()
|
||||
//!
|
||||
//! Run:
|
||||
//! cargo bench -p clawhdf5-onion -- epoch_gc
|
||||
|
||||
use clawhdf5_onion::gc::GcPolicy;
|
||||
use clawhdf5_onion::writer::OnionFile;
|
||||
use criterion::{BenchmarkId, Criterion, black_box, criterion_group, criterion_main};
|
||||
use tempfile::NamedTempFile;
|
||||
|
||||
const PAGE_SIZE: u32 = 4096;
|
||||
|
||||
// ─────────────────────────────────────────────────────────────────────────────
|
||||
// Helpers
|
||||
// ─────────────────────────────────────────────────────────────────────────────
|
||||
|
||||
fn make_onion(n: usize) -> (NamedTempFile, std::path::PathBuf, OnionFile) {
|
||||
let tmp = NamedTempFile::new().unwrap();
|
||||
let h5 = tmp.path().with_extension("h5");
|
||||
std::fs::write(&h5, b"\x89HDF\r\n\x1a\n").unwrap();
|
||||
let mut onion = OnionFile::create(&h5, PAGE_SIZE).unwrap();
|
||||
for i in 0..n {
|
||||
let mut s = onion.begin_session(None).unwrap();
|
||||
s.record_page(0, &vec![(i % 256) as u8; PAGE_SIZE as usize]);
|
||||
onion.commit_session(s, None).unwrap();
|
||||
}
|
||||
(tmp, h5, onion)
|
||||
}
|
||||
|
||||
// ─────────────────────────────────────────────────────────────────────────────
|
||||
// Benchmarks
|
||||
// ─────────────────────────────────────────────────────────────────────────────
|
||||
|
||||
fn bench_epoch_gc(c: &mut Criterion) {
|
||||
let sizes = [100usize, 500, 1000];
|
||||
|
||||
// ── Immediate GC (baseline) ───────────────────────────────────────────────
|
||||
let mut group = c.benchmark_group("epoch_gc/immediate");
|
||||
for &n in &sizes {
|
||||
group.bench_with_input(BenchmarkId::new("keep_last_half", n), &n, |b, &n| {
|
||||
b.iter_with_setup(
|
||||
|| make_onion(n),
|
||||
|(_tmp, _h5, mut onion)| {
|
||||
let _ = onion.gc(black_box(GcPolicy::KeepLastN((n / 2) as u64))).unwrap();
|
||||
},
|
||||
);
|
||||
});
|
||||
}
|
||||
group.finish();
|
||||
|
||||
// ── Epoch flip (mark only, no compaction) ─────────────────────────────────
|
||||
let mut group = c.benchmark_group("epoch_gc/epoch_flip_mark");
|
||||
for &n in &sizes {
|
||||
group.bench_with_input(BenchmarkId::new("keep_last_half", n), &n, |b, &n| {
|
||||
b.iter_with_setup(
|
||||
|| make_onion(n),
|
||||
|(_tmp, _h5, mut onion)| {
|
||||
let _ = onion.gc(black_box(GcPolicy::EpochFlip(Box::new(
|
||||
GcPolicy::KeepLastN((n / 2) as u64),
|
||||
)))).unwrap();
|
||||
},
|
||||
);
|
||||
});
|
||||
}
|
||||
group.finish();
|
||||
|
||||
// ── Flush after epoch flip (deferred compaction cost) ────────────────────
|
||||
let mut group = c.benchmark_group("epoch_gc/flush_after_epoch_flip");
|
||||
for &n in &sizes {
|
||||
group.bench_with_input(BenchmarkId::new("keep_last_half", n), &n, |b, &n| {
|
||||
b.iter_with_setup(
|
||||
|| {
|
||||
let (tmp, h5, mut onion) = make_onion(n);
|
||||
onion.gc(GcPolicy::EpochFlip(Box::new(
|
||||
GcPolicy::KeepLastN((n / 2) as u64),
|
||||
))).unwrap();
|
||||
(tmp, h5, onion)
|
||||
},
|
||||
|(_tmp, _h5, mut onion)| {
|
||||
onion.flush().unwrap();
|
||||
},
|
||||
);
|
||||
});
|
||||
}
|
||||
group.finish();
|
||||
|
||||
// ── Comparison: flush without any GC (baseline for flush cost) ───────────
|
||||
let mut group = c.benchmark_group("epoch_gc/flush_no_gc");
|
||||
for &n in &sizes {
|
||||
group.bench_with_input(BenchmarkId::new("n_revisions", n), &n, |b, &n| {
|
||||
b.iter_with_setup(
|
||||
|| make_onion(n),
|
||||
|(_tmp, _h5, mut onion)| {
|
||||
onion.flush().unwrap();
|
||||
},
|
||||
);
|
||||
});
|
||||
}
|
||||
group.finish();
|
||||
}
|
||||
|
||||
criterion_group!(benches, bench_epoch_gc);
|
||||
criterion_main!(benches);
|
||||
@@ -0,0 +1,119 @@
|
||||
//! Criterion benchmarks for the revision Merkle tree.
|
||||
//!
|
||||
//! Groups:
|
||||
//! - `merkle_index/build_tree/N` — build tree from N revisions
|
||||
//! - `merkle_index/diff_walk/N/D` — diff walk with D differing revisions
|
||||
//! - `merkle_index/serialise/N` — serialise N-leaf tree
|
||||
//! - `merkle_index/deserialise/N` — deserialise N-leaf tree
|
||||
//! - `merkle_index/serialised_size/N` — report bytes vs flat manifest
|
||||
//!
|
||||
//! Run:
|
||||
//! cargo bench -p clawhdf5-onion -- merkle_bench
|
||||
|
||||
use clawhdf5_onion::merkle::RevisionMerkleTree;
|
||||
use criterion::{BenchmarkId, Criterion, black_box, criterion_group, criterion_main};
|
||||
|
||||
fn rev_hash(r: u64) -> [u8; 32] {
|
||||
*blake3::hash(&r.to_le_bytes()).as_bytes()
|
||||
}
|
||||
|
||||
fn make_entries(n: usize) -> Vec<(u64, [u8; 32])> {
|
||||
(0..n as u64).map(|r| (r, rev_hash(r))).collect()
|
||||
}
|
||||
|
||||
// ─────────────────────────────────────────────────────────────────────────────
|
||||
|
||||
fn bench_merkle(c: &mut Criterion) {
|
||||
let ns = [100usize, 1_000, 10_000];
|
||||
|
||||
// ── Build ─────────────────────────────────────────────────────────────────
|
||||
let mut group = c.benchmark_group("merkle_index/build_tree");
|
||||
for &n in &ns {
|
||||
let entries = make_entries(n);
|
||||
group.bench_with_input(BenchmarkId::from_parameter(n), &entries, |b, e| {
|
||||
b.iter(|| RevisionMerkleTree::build(black_box(e)));
|
||||
});
|
||||
}
|
||||
group.finish();
|
||||
|
||||
// ── Diff walk: D=0 (all match) ────────────────────────────────────────────
|
||||
let mut group = c.benchmark_group("merkle_index/diff_walk_d0");
|
||||
for &n in &ns {
|
||||
let tree = RevisionMerkleTree::build(&make_entries(n));
|
||||
group.bench_with_input(BenchmarkId::from_parameter(n), &tree, |b, t| {
|
||||
b.iter(|| t.diff_missing_revisions(black_box(t)));
|
||||
});
|
||||
}
|
||||
group.finish();
|
||||
|
||||
// ── Diff walk: D=1 ───────────────────────────────────────────────────────
|
||||
let mut group = c.benchmark_group("merkle_index/diff_walk_d1");
|
||||
for &n in &ns {
|
||||
let full = RevisionMerkleTree::build(&make_entries(n));
|
||||
let minus1 = RevisionMerkleTree::build(&make_entries(n - 1));
|
||||
group.bench_with_input(BenchmarkId::from_parameter(n), &(full, minus1), |b, (f, m)| {
|
||||
b.iter(|| f.diff_missing_revisions(black_box(m)));
|
||||
});
|
||||
}
|
||||
group.finish();
|
||||
|
||||
// ── Diff walk: D=10 ──────────────────────────────────────────────────────
|
||||
let mut group = c.benchmark_group("merkle_index/diff_walk_d10");
|
||||
for &n in &ns {
|
||||
if n < 20 { continue; }
|
||||
let full = RevisionMerkleTree::build(&make_entries(n));
|
||||
let base = RevisionMerkleTree::build(&make_entries(n - 10));
|
||||
group.bench_with_input(BenchmarkId::from_parameter(n), &(full, base), |b, (f, base)| {
|
||||
b.iter(|| f.diff_missing_revisions(black_box(base)));
|
||||
});
|
||||
}
|
||||
group.finish();
|
||||
|
||||
// ── Diff walk: D=100 ─────────────────────────────────────────────────────
|
||||
let mut group = c.benchmark_group("merkle_index/diff_walk_d100");
|
||||
for &n in &[1_000usize, 10_000] {
|
||||
let full = RevisionMerkleTree::build(&make_entries(n));
|
||||
let base = RevisionMerkleTree::build(&make_entries(n - 100));
|
||||
group.bench_with_input(BenchmarkId::from_parameter(n), &(full, base), |b, (f, base)| {
|
||||
b.iter(|| f.diff_missing_revisions(black_box(base)));
|
||||
});
|
||||
}
|
||||
group.finish();
|
||||
|
||||
// ── Serialise ─────────────────────────────────────────────────────────────
|
||||
let mut group = c.benchmark_group("merkle_index/serialise");
|
||||
for &n in &ns {
|
||||
let tree = RevisionMerkleTree::build(&make_entries(n));
|
||||
group.bench_with_input(BenchmarkId::from_parameter(n), &tree, |b, t| {
|
||||
b.iter(|| black_box(t.serialise()));
|
||||
});
|
||||
}
|
||||
group.finish();
|
||||
|
||||
// ── Deserialise ───────────────────────────────────────────────────────────
|
||||
let mut group = c.benchmark_group("merkle_index/deserialise");
|
||||
for &n in &ns {
|
||||
let bytes = RevisionMerkleTree::build(&make_entries(n)).serialise();
|
||||
group.bench_with_input(BenchmarkId::from_parameter(n), &bytes, |b, b_| {
|
||||
b.iter(|| RevisionMerkleTree::deserialise(black_box(b_)).unwrap());
|
||||
});
|
||||
}
|
||||
group.finish();
|
||||
|
||||
// ── Serialised size report (not timed) ───────────────────────────────────
|
||||
println!("\n=== Serialised size vs flat manifest (N × 60 B) ===");
|
||||
println!("{:>8} {:>12} {:>12} {:>10}", "N", "Merkle (B)", "Flat (B)", "ratio");
|
||||
for &n in &[10usize, 100, 1_000, 10_000] {
|
||||
let tree = RevisionMerkleTree::build(&make_entries(n));
|
||||
let merkle_sz = tree.serialise().len();
|
||||
let flat_sz = n * 60;
|
||||
println!(
|
||||
"{:>8} {:>12} {:>12} {:>10.2}x",
|
||||
n, merkle_sz, flat_sz,
|
||||
flat_sz as f64 / merkle_sz as f64
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
criterion_group!(benches, bench_merkle);
|
||||
criterion_main!(benches);
|
||||
@@ -0,0 +1,276 @@
|
||||
//! Criterion benchmark suite for `clawhdf5-onion`.
|
||||
//!
|
||||
//! Targets from the Phase 5 PRD:
|
||||
//!
|
||||
//! | Benchmark | Target |
|
||||
//! |----------------------------------------|---------------|
|
||||
//! | Onion write overhead (vs unversioned) | ≤ 10% |
|
||||
//! | open(rev=100) — no snapshot | ≤ 1 ms |
|
||||
//! | open(rev=1000) — with snapshot | ≤ 5 ms |
|
||||
//! | Manifest generation 100 revisions | ≤ 50 ms |
|
||||
//! | BLAKE3 verify throughput | ≥ 5 GB/s |
|
||||
//! | Branch fork | ≤ 100 µs |
|
||||
//! | GC keep_last_1000 on 10K revisions | ≤ 2 s |
|
||||
|
||||
use clawhdf5_onion::gc::GcPolicy;
|
||||
use clawhdf5_onion::writer::OnionFile;
|
||||
use criterion::{BenchmarkId, Criterion, Throughput, black_box, criterion_group, criterion_main};
|
||||
use tempfile::NamedTempFile;
|
||||
|
||||
// ─────────────────────────────────────────────────────────────────────────────
|
||||
// Helpers
|
||||
// ─────────────────────────────────────────────────────────────────────────────
|
||||
|
||||
const PAGE_SIZE: u32 = 4096;
|
||||
|
||||
fn tmp_onion_with_n(n: usize) -> (NamedTempFile, std::path::PathBuf, OnionFile, Vec<u8>) {
|
||||
let f = NamedTempFile::new().unwrap();
|
||||
let h5 = f.path().with_extension("h5");
|
||||
let base = b"\x89HDF\r\n\x1a\n".to_vec();
|
||||
std::fs::write(&h5, &base).unwrap();
|
||||
let mut onion = OnionFile::create(&h5, PAGE_SIZE).unwrap();
|
||||
for i in 0..n {
|
||||
let mut s = onion.begin_session(None).unwrap();
|
||||
s.record_page(0, &vec![(i % 256) as u8; PAGE_SIZE as usize]);
|
||||
onion.commit_session(s, None).unwrap();
|
||||
}
|
||||
(f, h5, onion, base)
|
||||
}
|
||||
|
||||
// ─────────────────────────────────────────────────────────────────────────────
|
||||
// Benchmarks
|
||||
// ─────────────────────────────────────────────────────────────────────────────
|
||||
|
||||
/// Benchmark: single revision write (commit_session overhead).
|
||||
fn bench_write_one_revision(c: &mut Criterion) {
|
||||
let mut group = c.benchmark_group("write");
|
||||
group.throughput(Throughput::Elements(1));
|
||||
|
||||
group.bench_function("commit_session/1_page", |b| {
|
||||
let f = NamedTempFile::new().unwrap();
|
||||
let h5 = f.path().with_extension("h5");
|
||||
std::fs::write(&h5, b"\x89HDF\r\n\x1a\n").unwrap();
|
||||
let mut onion = OnionFile::create(&h5, PAGE_SIZE).unwrap();
|
||||
b.iter(|| {
|
||||
let mut s = onion.begin_session(None).unwrap();
|
||||
s.record_page(0, black_box(&vec![0xABu8; PAGE_SIZE as usize]));
|
||||
black_box(onion.commit_session(s, None).unwrap());
|
||||
});
|
||||
});
|
||||
|
||||
group.bench_function("commit_session/4_pages", |b| {
|
||||
let f = NamedTempFile::new().unwrap();
|
||||
let h5 = f.path().with_extension("h5");
|
||||
std::fs::write(&h5, b"\x89HDF\r\n\x1a\n").unwrap();
|
||||
let mut onion = OnionFile::create(&h5, PAGE_SIZE).unwrap();
|
||||
b.iter(|| {
|
||||
let mut s = onion.begin_session(None).unwrap();
|
||||
for i in 0u64..4 {
|
||||
s.record_page(i * PAGE_SIZE as u64, black_box(&vec![0xCDu8; PAGE_SIZE as usize]));
|
||||
}
|
||||
black_box(onion.commit_session(s, None).unwrap());
|
||||
});
|
||||
});
|
||||
|
||||
group.finish();
|
||||
}
|
||||
|
||||
/// Benchmark: reconstruct_revision at various depths.
|
||||
fn bench_reconstruct_revision(c: &mut Criterion) {
|
||||
let mut group = c.benchmark_group("reconstruct");
|
||||
|
||||
for &depth in &[10usize, 50, 100] {
|
||||
let (_f, _h5, onion, base) = tmp_onion_with_n(depth);
|
||||
let target_rev = (depth - 1) as u64;
|
||||
|
||||
group.bench_with_input(
|
||||
BenchmarkId::new("no_snapshot", depth),
|
||||
&depth,
|
||||
|b, _| {
|
||||
b.iter(|| {
|
||||
black_box(onion.reconstruct_revision(target_rev, black_box(&base)).unwrap());
|
||||
});
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
// With snapshot: create a real snapshot at midpoint via create_snapshot()
|
||||
for &depth in &[100usize, 200] {
|
||||
let (_f, _h5, mut onion, base) = tmp_onion_with_n(depth);
|
||||
// Insert a snapshot halfway through
|
||||
onion.create_snapshot(&base, Some("mid-snap")).unwrap();
|
||||
// Add a few more revisions after the snapshot
|
||||
for i in 0u8..5 {
|
||||
let mut s = onion.begin_session(None).unwrap();
|
||||
s.record_page(0, &vec![i; PAGE_SIZE as usize]);
|
||||
onion.commit_session(s, None).unwrap();
|
||||
}
|
||||
let target_rev = onion.revision_count() - 1;
|
||||
|
||||
group.bench_with_input(
|
||||
BenchmarkId::new("with_snapshot", depth),
|
||||
&depth,
|
||||
|b, _| {
|
||||
b.iter(|| {
|
||||
black_box(onion.reconstruct_revision(target_rev, black_box(&base)).unwrap());
|
||||
});
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
group.finish();
|
||||
}
|
||||
|
||||
/// Benchmark: list_revisions (manifest generation).
|
||||
fn bench_manifest_generation(c: &mut Criterion) {
|
||||
let mut group = c.benchmark_group("manifest");
|
||||
|
||||
for &n in &[10usize, 100, 500] {
|
||||
let (_f, _h5, onion, _base) = tmp_onion_with_n(n);
|
||||
|
||||
group.bench_with_input(BenchmarkId::new("list_revisions", n), &n, |b, _| {
|
||||
b.iter(|| {
|
||||
black_box(onion.list_revisions());
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
group.finish();
|
||||
}
|
||||
|
||||
/// Benchmark: BLAKE3 hashing throughput — single-threaded vs Rayon.
|
||||
fn bench_blake3_throughput(c: &mut Criterion) {
|
||||
let mut group = c.benchmark_group("blake3");
|
||||
|
||||
// Single-threaded path (best for small pages — dispatch overhead dominates above).
|
||||
for &size_kb in &[4usize, 64, 1024, 4096] {
|
||||
let data = vec![0xAAu8; size_kb * 1024];
|
||||
group.throughput(Throughput::Bytes(data.len() as u64));
|
||||
|
||||
group.bench_with_input(
|
||||
BenchmarkId::new("single", format!("{size_kb}KB")),
|
||||
&data,
|
||||
|b, d| {
|
||||
b.iter(|| {
|
||||
black_box(clawsync_core::checksum::blake3_hash(black_box(d)));
|
||||
});
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
// Rayon tree-parallel path (best for large inputs >= 128 KB).
|
||||
for &size_kb in &[128usize, 512, 1024, 4096] {
|
||||
let data = vec![0xAAu8; size_kb * 1024];
|
||||
group.throughput(Throughput::Bytes(data.len() as u64));
|
||||
|
||||
group.bench_with_input(
|
||||
BenchmarkId::new("rayon", format!("{size_kb}KB")),
|
||||
&data,
|
||||
|b, d| {
|
||||
b.iter(|| {
|
||||
black_box(clawsync_core::checksum::blake3_hash_large(black_box(d)));
|
||||
});
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
// Batch parallel: hash N independent 4 KB pages simultaneously.
|
||||
for &n_pages in &[16usize, 64, 256] {
|
||||
let pages: Vec<Vec<u8>> = (0..n_pages).map(|i| vec![(i % 256) as u8; 4096]).collect();
|
||||
let page_refs: Vec<&[u8]> = pages.iter().map(|p| p.as_slice()).collect();
|
||||
group.throughput(Throughput::Bytes((n_pages * 4096) as u64));
|
||||
|
||||
group.bench_with_input(
|
||||
BenchmarkId::new("batch_pages", format!("{n_pages}x4KB")),
|
||||
&page_refs,
|
||||
|b, refs| {
|
||||
b.iter(|| {
|
||||
black_box(clawsync_core::checksum::blake3_hash_batch(black_box(refs)));
|
||||
});
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
group.finish();
|
||||
}
|
||||
|
||||
/// Benchmark: branch fork.
|
||||
fn bench_branch_fork(c: &mut Criterion) {
|
||||
c.bench_function("branch/fork", |b| {
|
||||
let f = NamedTempFile::new().unwrap();
|
||||
let h5 = f.path().with_extension("h5");
|
||||
std::fs::write(&h5, b"\x89HDF\r\n\x1a\n").unwrap();
|
||||
let mut onion = OnionFile::create(&h5, PAGE_SIZE).unwrap();
|
||||
|
||||
let mut s = onion.begin_session(None).unwrap();
|
||||
s.record_page(0, &vec![0u8; PAGE_SIZE as usize]);
|
||||
onion.commit_session(s, None).unwrap();
|
||||
|
||||
let mut fork_count = 0usize;
|
||||
b.iter(|| {
|
||||
let name = format!("bench-{fork_count}");
|
||||
fork_count += 1;
|
||||
black_box(onion.create_branch(&name, "main").unwrap());
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
/// Benchmark: GC keep_last_N.
|
||||
fn bench_gc(c: &mut Criterion) {
|
||||
let mut group = c.benchmark_group("gc");
|
||||
group.warm_up_time(std::time::Duration::from_millis(200));
|
||||
group.measurement_time(std::time::Duration::from_secs(3));
|
||||
|
||||
for &(total, keep) in &[(200usize, 50usize), (500, 100)] {
|
||||
group.bench_with_input(
|
||||
BenchmarkId::new(format!("keep_last_{keep}"), total),
|
||||
&(total, keep),
|
||||
|b, &(total, keep)| {
|
||||
b.iter_with_setup(
|
||||
|| tmp_onion_with_n(total).2,
|
||||
|mut onion| {
|
||||
black_box(onion.gc(GcPolicy::KeepLastN(keep as u64)).unwrap());
|
||||
},
|
||||
);
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
group.finish();
|
||||
}
|
||||
|
||||
/// Benchmark: snapshot creation overhead.
|
||||
fn bench_snapshot(c: &mut Criterion) {
|
||||
let mut group = c.benchmark_group("snapshot");
|
||||
|
||||
for &n_revs in &[5usize, 20, 50] {
|
||||
group.bench_with_input(
|
||||
BenchmarkId::new("create_snapshot", n_revs),
|
||||
&n_revs,
|
||||
|b, &n_revs| {
|
||||
let (_f, _h5, mut onion, base) = tmp_onion_with_n(n_revs);
|
||||
b.iter(|| {
|
||||
black_box(onion.create_snapshot(&base, Some("bench-snap")).unwrap());
|
||||
});
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
group.finish();
|
||||
}
|
||||
|
||||
// ─────────────────────────────────────────────────────────────────────────────
|
||||
// Registration
|
||||
// ─────────────────────────────────────────────────────────────────────────────
|
||||
|
||||
criterion_group!(
|
||||
benches,
|
||||
bench_write_one_revision,
|
||||
bench_reconstruct_revision,
|
||||
bench_manifest_generation,
|
||||
bench_blake3_throughput,
|
||||
bench_branch_fork,
|
||||
bench_gc,
|
||||
bench_snapshot,
|
||||
);
|
||||
criterion_main!(benches);
|
||||
@@ -0,0 +1,145 @@
|
||||
//! Criterion benchmarks for the TDT byte-interleaving transform + ZstdTdt codec.
|
||||
//!
|
||||
//! Measures compression ratio and throughput for common HDF5 data types,
|
||||
//! comparing plain Zstd against TDT-transformed Zstd.
|
||||
//!
|
||||
//! Run:
|
||||
//! cargo bench -p clawhdf5-onion -- tdt_compress
|
||||
|
||||
use clawhdf5_onion::compress::{compress_page, decompress_page};
|
||||
use clawhdf5_onion::format::Codec;
|
||||
use clawhdf5_onion::tdt;
|
||||
use criterion::{BenchmarkId, Criterion, Throughput, black_box, criterion_group, criterion_main};
|
||||
|
||||
// ─────────────────────────────────────────────────────────────────────────────
|
||||
// Test data generators
|
||||
// ─────────────────────────────────────────────────────────────────────────────
|
||||
|
||||
/// Smooth f32: values from a sine wave — good compressibility, representative
|
||||
/// of neural network activation or weight tensors.
|
||||
fn make_f32_smooth(n_bytes: usize) -> Vec<u8> {
|
||||
let n_floats = n_bytes / 4;
|
||||
let mut out = Vec::with_capacity(n_bytes);
|
||||
for i in 0..n_floats {
|
||||
let v = ((i as f64 / 256.0).sin() as f32).to_le_bytes();
|
||||
out.extend_from_slice(&v);
|
||||
}
|
||||
// Pad to exact size if n_bytes % 4 != 0.
|
||||
out.resize(n_bytes, 0);
|
||||
out
|
||||
}
|
||||
|
||||
/// Random f32: worst case for compression.
|
||||
fn make_f32_random(n_bytes: usize) -> Vec<u8> {
|
||||
// Deterministic LCG so benchmarks are reproducible.
|
||||
let mut state = 0x_dead_beef_u64;
|
||||
let mut out = Vec::with_capacity(n_bytes);
|
||||
while out.len() < n_bytes {
|
||||
state = state.wrapping_mul(6364136223846793005).wrapping_add(1442695040888963407);
|
||||
out.extend_from_slice(&(state as u32).to_le_bytes());
|
||||
}
|
||||
out.truncate(n_bytes);
|
||||
out
|
||||
}
|
||||
|
||||
/// Random i32 data.
|
||||
fn make_int32_random(n_bytes: usize) -> Vec<u8> {
|
||||
make_f32_random(n_bytes) // same byte distribution
|
||||
}
|
||||
|
||||
/// Sequential i8 data (highly compressible, TDT should be neutral).
|
||||
fn make_int8_seq(n_bytes: usize) -> Vec<u8> {
|
||||
(0u8..=255).cycle().take(n_bytes).collect()
|
||||
}
|
||||
|
||||
// ─────────────────────────────────────────────────────────────────────────────
|
||||
// Helpers
|
||||
// ─────────────────────────────────────────────────────────────────────────────
|
||||
|
||||
/// Run compress + decompress roundtrip and return compressed size.
|
||||
fn bench_compress(data: &[u8], codec: Codec) -> usize {
|
||||
let compressed = compress_page(black_box(data), codec).unwrap();
|
||||
let size = compressed.len();
|
||||
let _ = decompress_page(black_box(&compressed), codec, data.len() as u32).unwrap();
|
||||
size
|
||||
}
|
||||
|
||||
// ─────────────────────────────────────────────────────────────────────────────
|
||||
// Benchmark groups
|
||||
// ─────────────────────────────────────────────────────────────────────────────
|
||||
|
||||
fn bench_tdt_compress(c: &mut Criterion) {
|
||||
let sizes = [4096usize, 65536];
|
||||
|
||||
let datasets: &[(&str, fn(usize) -> Vec<u8>, usize)] = &[
|
||||
("f32_smooth", make_f32_smooth, 4),
|
||||
("f32_random", make_f32_random, 4),
|
||||
("int32_random", make_int32_random, 4),
|
||||
("int8_seq", make_int8_seq, 1),
|
||||
];
|
||||
|
||||
let codecs = [
|
||||
("zstd", Codec::Zstd),
|
||||
("zstd_tdt", Codec::ZstdTdt),
|
||||
];
|
||||
|
||||
// ── Throughput benchmark ──────────────────────────────────────────────────
|
||||
let mut group = c.benchmark_group("tdt_compress/throughput");
|
||||
for &size in &sizes {
|
||||
for &(dtype, make, _width) in datasets {
|
||||
let data = make(size);
|
||||
group.throughput(Throughput::Bytes(size as u64));
|
||||
for &(codec_name, codec) in &codecs {
|
||||
let id = BenchmarkId::new(
|
||||
format!("{codec_name}/{dtype}"),
|
||||
format!("{size}B"),
|
||||
);
|
||||
group.bench_with_input(id, &data, |b, d| {
|
||||
b.iter(|| bench_compress(d, codec));
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
group.finish();
|
||||
|
||||
// ── TDT transform only (encode + decode, without compression) ────────────
|
||||
let mut group = c.benchmark_group("tdt_compress/transform_only");
|
||||
for &size in &sizes {
|
||||
for &(dtype, make, width) in datasets {
|
||||
let data = make(size);
|
||||
group.throughput(Throughput::Bytes(size as u64));
|
||||
|
||||
let id = BenchmarkId::new(format!("encode/{dtype}"), format!("{size}B"));
|
||||
group.bench_with_input(id, &data, |b, d| {
|
||||
b.iter(|| tdt::encode(black_box(d), width));
|
||||
});
|
||||
|
||||
let encoded = tdt::encode(&data, width);
|
||||
let id = BenchmarkId::new(format!("decode/{dtype}"), format!("{size}B"));
|
||||
group.bench_with_input(id, &encoded, |b, enc| {
|
||||
b.iter(|| tdt::decode(black_box(enc), width, size));
|
||||
});
|
||||
}
|
||||
}
|
||||
group.finish();
|
||||
|
||||
// ── Compression ratio summary (printed, not timed) ───────────────────────
|
||||
// Run once outside Criterion to print ratio comparison.
|
||||
println!("\n─── TDT compression ratio summary ───");
|
||||
println!("{:<20} {:>8} {:>10} {:>10} {:>8}",
|
||||
"dataset/size", "orig", "zstd", "zstd_tdt", "savings");
|
||||
for &size in &sizes {
|
||||
for &(dtype, make, _w) in datasets {
|
||||
let data = make(size);
|
||||
let zstd_size = compress_page(&data, Codec::Zstd).unwrap().len();
|
||||
let tdt_size = compress_page(&data, Codec::ZstdTdt).unwrap().len();
|
||||
let savings_pct = 100.0 * (1.0 - tdt_size as f64 / zstd_size as f64);
|
||||
println!("{:<20} {:>8} {:>10} {:>10} {:>7.1}%",
|
||||
format!("{dtype}/{size}B"), size, zstd_size, tdt_size, savings_pct);
|
||||
}
|
||||
}
|
||||
println!();
|
||||
}
|
||||
|
||||
criterion_group!(benches, bench_tdt_compress);
|
||||
criterion_main!(benches);
|
||||
Reference in New Issue
Block a user