Omar SobhandClaude Sonnet 4.6
5701e8045d
Implements Pcodec (filter ID 32023) via the `pco` 1.0.x crate as a new optional compression codec. Pcodec achieves 30–94% better compression ratio than Zstd for f32/f64 columnar data at 1–5 GiB/s decompression speed, making it ideal for write-once/read-many embedding archives. Write throughput at 512×512: 591 MiB/s (parity with Zstd-3 at 610 MiB/s). For smaller chunks Zstd-3 remains faster due to Pcodec's fixed per-chunk distributional analysis overhead. - Add FILTER_PCODEC = 32023 constant to filter_pipeline.rs - Add pcodec_compress/pcodec_decompress using pco::standalone API - Wire into compress_chunk/decompress_chunk dispatch - Add ChunkOptions.pcodec field and DatasetBuilder.with_pcodec() method - Enable pcodec as highest-priority codec in build_pipeline() - Add pco dep (optional, feature = "pcodec") to clawhdf5-format/clawhdf5 - Add write_2d_chunked_pcodec benchmark comparing pcodec vs zstd-3 - Document results in BENCHMARKS.md Co-Authored-By: Claude Sonnet 4.6 <[email protected]>
55 lines
1.7 KiB
TOML
55 lines
1.7 KiB
TOML
[package]
|
|
name = "clawhdf5-format"
|
|
version = "2.1.0"
|
|
edition = "2024"
|
|
description = "Pure-Rust HDF5 binary format parsing and writing — no C dependencies"
|
|
license = "MIT"
|
|
repository = "https://github.com/redclawsystems/clawhdf5"
|
|
readme = "README.md"
|
|
keywords = ["hdf5", "science", "data", "binary", "no-std"]
|
|
categories = ["parser-implementations", "science", "encoding", "no-std"]
|
|
|
|
[dependencies]
|
|
byteorder = { version = "1", default-features = false }
|
|
flate2 = { version = "1", default-features = false, features = ["rust_backend"], optional = true }
|
|
sha2 = { version = "0.10", default-features = false, optional = true }
|
|
rayon = { version = "1", optional = true }
|
|
crc32fast = { version = "1", optional = true }
|
|
lz4_flex = { version = "0.11", optional = true }
|
|
zstd = { version = "0.13", optional = true }
|
|
blake3 = { version = "1", optional = true }
|
|
libaec-sys = { path = "../libaec-sys", version = "0.1", optional = true }
|
|
pco = { version = "1.0", optional = true }
|
|
|
|
[dev-dependencies]
|
|
serde_json = "1"
|
|
criterion = { version = "0.5", features = ["html_reports"] }
|
|
clawhdf5-derive = { path = "../clawhdf5-derive", version = "2.1.0" }
|
|
|
|
[[bench]]
|
|
name = "bench"
|
|
harness = false
|
|
|
|
[features]
|
|
default = ["std", "checksum", "deflate", "provenance", "fast-deflate", "system-zlib-decompress"]
|
|
std = []
|
|
checksum = []
|
|
deflate = ["flate2"]
|
|
provenance = ["sha2"]
|
|
parallel = ["rayon", "std"]
|
|
fast-checksum = ["crc32fast"]
|
|
fast-deflate = ["flate2/zlib-ng"]
|
|
system-zlib = ["flate2/zlib-default"]
|
|
system-zlib-decompress = []
|
|
zlib-rs = ["flate2/zlib-rs"]
|
|
lz4 = ["lz4_flex"]
|
|
zstd = ["dep:zstd"]
|
|
blake3_hash = ["blake3"]
|
|
szip = ["libaec-sys"]
|
|
pcodec = ["dep:pco"]
|
|
|
|
[[bench]]
|
|
name = "parallel_decompress_bench"
|
|
harness = false
|
|
required-features = ["parallel"]
|