Omar SobhandClaude Sonnet 4.6
5701e8045d
Implements Pcodec (filter ID 32023) via the `pco` 1.0.x crate as a new optional compression codec. Pcodec achieves 30–94% better compression ratio than Zstd for f32/f64 columnar data at 1–5 GiB/s decompression speed, making it ideal for write-once/read-many embedding archives. Write throughput at 512×512: 591 MiB/s (parity with Zstd-3 at 610 MiB/s). For smaller chunks Zstd-3 remains faster due to Pcodec's fixed per-chunk distributional analysis overhead. - Add FILTER_PCODEC = 32023 constant to filter_pipeline.rs - Add pcodec_compress/pcodec_decompress using pco::standalone API - Wire into compress_chunk/decompress_chunk dispatch - Add ChunkOptions.pcodec field and DatasetBuilder.with_pcodec() method - Enable pcodec as highest-priority codec in build_pipeline() - Add pco dep (optional, feature = "pcodec") to clawhdf5-format/clawhdf5 - Add write_2d_chunked_pcodec benchmark comparing pcodec vs zstd-3 - Document results in BENCHMARKS.md Co-Authored-By: Claude Sonnet 4.6 <[email protected]>
69 lines
1.7 KiB
TOML
69 lines
1.7 KiB
TOML
[package]
|
|
name = "clawhdf5-bench"
|
|
version = "2.1.0"
|
|
edition = "2024"
|
|
description = "Benchmark harnesses for clawhdf5-agent (Track 8)"
|
|
license = "MIT"
|
|
|
|
[[bin]]
|
|
name = "longmemeval_bench"
|
|
path = "src/bin/longmemeval_bench.rs"
|
|
|
|
[[bin]]
|
|
name = "memory_arena"
|
|
path = "src/bin/memory_arena.rs"
|
|
|
|
[[bin]]
|
|
name = "footprint_bench"
|
|
path = "src/bin/footprint_bench.rs"
|
|
|
|
[[bin]]
|
|
name = "consolidation_efficiency"
|
|
path = "src/bin/consolidation_efficiency.rs"
|
|
|
|
[[bin]]
|
|
name = "ephemeral_perf"
|
|
path = "src/bin/ephemeral_perf.rs"
|
|
|
|
[[bin]]
|
|
name = "mpi_io_bench"
|
|
path = "src/bin/mpi_io_bench.rs"
|
|
required-features = ["mpi-io"]
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# h5bench-equivalent Criterion benchmarks
|
|
# ---------------------------------------------------------------------------
|
|
|
|
[[bench]]
|
|
name = "h5bench_write"
|
|
harness = false
|
|
|
|
[[bench]]
|
|
name = "h5bench_read"
|
|
harness = false
|
|
|
|
[[bench]]
|
|
name = "h5bench_meta"
|
|
harness = false
|
|
|
|
[dependencies]
|
|
clawhdf5-agent = { path = "../clawhdf5-agent" }
|
|
clawhdf5-io = { path = "../clawhdf5-io" }
|
|
mpi = { version = "0.8", optional = true }
|
|
serde = { version = "1", features = ["derive"] }
|
|
serde_json = "1"
|
|
tempfile = "3"
|
|
# Optional: libhdf5 C wrapper for side-by-side comparison (requires system libhdf5).
|
|
# Enable with: cargo bench -p clawhdf5-bench --features libhdf5-compare
|
|
# Uses hdf5-metno (fork of hdf5 crate) which supports HDF5 1.14.x.
|
|
hdf5 = { version = "0.12", optional = true, package = "hdf5-metno" }
|
|
|
|
[dev-dependencies]
|
|
clawhdf5 = { path = "../clawhdf5", features = ["zstd", "pcodec"] }
|
|
criterion = { version = "0.5", features = ["html_reports"] }
|
|
|
|
[features]
|
|
# When enabled, benchmarks add matching libhdf5 variants for side-by-side comparison.
|
|
libhdf5-compare = ["hdf5"]
|
|
mpi-io = ["clawhdf5-io/mpi-io", "mpi"]
|