osobhandClaude Opus 5.5
7334e21c93
hdf5plugin's Blosc2 was a clear "not implemented" error. It stores each HDF5 chunk as a Blosc2 contiguous frame; for chunks of 2+ dimensions the frame holds a B2ND array whose data are cut into (padded) blocks stored one after another. filters_blosc2 (feature `blosc2`, in `plugin-filters`; the facade forwards both) decodes, following c-blosc2's decoder and hdf5-blosc2's blosc2_filter.c: - the frame: the msgpack header's fixed fields, the metalayer index, the offsets chunk (with the special zero/NaN/uninitialised offsets) and chunk lookup; - Blosc2 chunks: 16- and 32-byte headers, special chunks (zeros, NaN, uninitialised, one repeated value), split and unsplit streams, zero and run-length streams, and the filter pipeline run backwards (shuffle, shuffle with a byte-group size, bit shuffle including the version-2 and later handling of a partial group of 8, delta against the first block, truncated precision); - the codecs, shared with Blosc 1: BloscLZ, LZ4/LZ4HC, Zlib, Zstandard; - B2ND arrays: blocks gathered into C order, padding dropped, several chunks per array, and the array shape checked against the chunk shape in cd_values as the HDF5 filter does. Dictionaries, lazy chunks, variable-length blocks, user-defined codecs and registered filters (e.g. bytedelta) are errors. Uninitialised chunks read as zeros. No encoder. Tests: h5py + hdf5plugin write every codec x filter (none, shuffle, bitshuffle, delta) and levels 0-9 over the plugin-filter cases, then i1..u8/f4/f8 in 1-D to 5-D chunks with partial edge chunks, datasets of zeros, one value and NaN, and Fletcher32 before Blosc2 (plain frames for n-D chunks); clawhdf5 reads each exactly as its unfiltered twin, and truncated precision exactly as h5py reads it. Fixture frames from python-blosc2 (tests/fixtures/blosc2/generate.py) cover what hdf5plugin never writes: special chunks, delta over many blocks and odd type sizes, odd bit-shuffle blocks, forced splitting, multi-chunk B2ND arrays with a zero chunk, and the refused features. The decoder is fuzzed (random and mutated frames and chunks: no panic, output within the limit). Conformance: h5ex_d_blosc2.h5 now reads (576 of 697 ok, baseline 575). Co-Authored-By: Claude Opus 5.5 (1M context) <[email protected]>
88 lines
3.4 KiB
TOML
88 lines
3.4 KiB
TOML
[package]
|
|
name = "clawhdf5-format"
|
|
version = "2.7.0"
|
|
edition = "2024"
|
|
rust-version.workspace = true
|
|
description = "Pure-Rust HDF5 binary format parsing and writing — no C dependencies"
|
|
license = "MIT"
|
|
repository = "https://git.redclaw.dev/quantumclaw/clawhdf5"
|
|
readme = "README.md"
|
|
keywords = ["hdf5", "science", "data", "binary", "no-std"]
|
|
categories = ["parser-implementations", "science", "encoding", "no-std"]
|
|
|
|
[dependencies]
|
|
byteorder = { version = "1", default-features = false }
|
|
portable-atomic = { version = "1" }
|
|
flate2 = { version = "1", default-features = false, features = ["rust_backend"], optional = true }
|
|
sha2 = { version = "0.10", default-features = false, optional = true }
|
|
rayon = { version = "1", optional = true }
|
|
crc32fast = { version = "1", optional = true }
|
|
lz4_flex = { version = "0.11", optional = true }
|
|
zstd = { version = "0.13", optional = true }
|
|
blake3 = { version = "1", optional = true }
|
|
libaec-sys = { path = "../libaec-sys", version = "0.1", optional = true }
|
|
pco = { version = "1.0", optional = true }
|
|
# Pure-Rust Zstandard, for the plugin filters that embed zstd (bitshuffle,
|
|
# blosc). The `zstd` feature (filter 32015) links libzstd instead.
|
|
ruzstd = { version = "0.9", optional = true }
|
|
# bzip2 with its default backend, libbz2-rs-sys: a pure-Rust port of
|
|
# libbzip2 (no C is compiled, despite the -sys name).
|
|
bzip2 = { version = "0.6", optional = true }
|
|
snap = { version = "1", optional = true }
|
|
|
|
[target.'cfg(target_os = "linux")'.dependencies]
|
|
# madvise(MADV_HUGEPAGE) for large read buffers (see src/bulk_alloc.rs).
|
|
libc = { version = "0.2", default-features = false }
|
|
|
|
[dev-dependencies]
|
|
half = { workspace = true }
|
|
serde_json = "1"
|
|
criterion = { workspace = true }
|
|
clawhdf5-derive = { path = "../clawhdf5-derive", version = "2.7.0" }
|
|
|
|
[[bench]]
|
|
name = "bench"
|
|
harness = false
|
|
|
|
[features]
|
|
# Deflate backend: `zlib-rs` (pure Rust) by default. `fast-deflate` selects
|
|
# zlib-ng instead (C, built with cmake); flate2 prefers a C zlib whenever one
|
|
# is enabled, so turning it on anywhere in the build overrides the default.
|
|
default = ["std", "checksum", "deflate", "provenance", "zlib-rs", "system-zlib-decompress", "lzf"]
|
|
std = []
|
|
checksum = []
|
|
deflate = ["flate2"]
|
|
provenance = ["sha2"]
|
|
parallel = ["rayon", "std"]
|
|
fast-checksum = ["crc32fast"]
|
|
fast-deflate = ["flate2/zlib-ng"]
|
|
system-zlib = ["flate2/zlib-default"]
|
|
system-zlib-decompress = []
|
|
# `runtime_detection` gives zlib-rs `std`, which it needs to detect and use
|
|
# SIMD at runtime. flate2 enables it by default, but we build flate2 with
|
|
# default-features = false, and without it zlib-rs inflates 3.5x slower.
|
|
zlib-rs = ["flate2/zlib-rs", "flate2/runtime_detection"]
|
|
lz4 = ["lz4_flex"]
|
|
zstd = ["dep:zstd"]
|
|
blake3_hash = ["blake3"]
|
|
szip = ["libaec-sys"]
|
|
pcodec = ["dep:pco"]
|
|
# Plugin filters, pure Rust. LZF (32000) is h5py's built-in compression; it
|
|
# has no dependencies, so it is on by default.
|
|
lzf = []
|
|
# Bitshuffle (32008), with its LZ4 and Zstandard modes.
|
|
bitshuffle = ["lz4_flex", "ruzstd"]
|
|
# bzip2 (307).
|
|
bzip2 = ["dep:bzip2", "std"]
|
|
# Blosc 1 (32001) with its BloscLZ, LZ4, Snappy, Zlib and Zstandard codecs.
|
|
blosc = ["lz4_flex", "ruzstd", "snap", "deflate", "std"]
|
|
# Blosc2 (32026), read-only: frames, B2ND arrays, and the Blosc codecs above.
|
|
blosc2 = ["blosc"]
|
|
# Every plugin filter above.
|
|
plugin-filters = ["lzf", "bitshuffle", "bzip2", "blosc", "blosc2"]
|
|
|
|
[[bench]]
|
|
name = "parallel_decompress_bench"
|
|
harness = false
|
|
required-features = ["parallel"]
|