feat(format): read Blosc2 (filter 32026) in pure Rust
hdf5plugin's Blosc2 was a clear "not implemented" error. It stores each HDF5 chunk as a Blosc2 contiguous frame; for chunks of 2+ dimensions the frame holds a B2ND array whose data are cut into (padded) blocks stored one after another. filters_blosc2 (feature `blosc2`, in `plugin-filters`; the facade forwards both) decodes, following c-blosc2's decoder and hdf5-blosc2's blosc2_filter.c: - the frame: the msgpack header's fixed fields, the metalayer index, the offsets chunk (with the special zero/NaN/uninitialised offsets) and chunk lookup; - Blosc2 chunks: 16- and 32-byte headers, special chunks (zeros, NaN, uninitialised, one repeated value), split and unsplit streams, zero and run-length streams, and the filter pipeline run backwards (shuffle, shuffle with a byte-group size, bit shuffle including the version-2 and later handling of a partial group of 8, delta against the first block, truncated precision); - the codecs, shared with Blosc 1: BloscLZ, LZ4/LZ4HC, Zlib, Zstandard; - B2ND arrays: blocks gathered into C order, padding dropped, several chunks per array, and the array shape checked against the chunk shape in cd_values as the HDF5 filter does. Dictionaries, lazy chunks, variable-length blocks, user-defined codecs and registered filters (e.g. bytedelta) are errors. Uninitialised chunks read as zeros. No encoder. Tests: h5py + hdf5plugin write every codec x filter (none, shuffle, bitshuffle, delta) and levels 0-9 over the plugin-filter cases, then i1..u8/f4/f8 in 1-D to 5-D chunks with partial edge chunks, datasets of zeros, one value and NaN, and Fletcher32 before Blosc2 (plain frames for n-D chunks); clawhdf5 reads each exactly as its unfiltered twin, and truncated precision exactly as h5py reads it. Fixture frames from python-blosc2 (tests/fixtures/blosc2/generate.py) cover what hdf5plugin never writes: special chunks, delta over many blocks and odd type sizes, odd bit-shuffle blocks, forced splitting, multi-chunk B2ND arrays with a zero chunk, and the refused features. The decoder is fuzzed (random and mutated frames and chunks: no panic, output within the limit). Conformance: h5ex_d_blosc2.h5 now reads (576 of 697 ok, baseline 575). Co-Authored-By: Claude Opus 5.5 (1M context) <[email protected]>
This commit is contained in:
@@ -5,7 +5,8 @@
|
||||
//! * **Built-in filters** — a static table of the filters compiled into this
|
||||
//! build: the HDF5 standard filters (deflate, shuffle, Fletcher32, szip,
|
||||
//! N-Bit, scale-offset) and the plugin filters whose cargo features are
|
||||
//! enabled (LZ4, Zstandard, pcodec, LZF, bitshuffle, bzip2, blosc).
|
||||
//! enabled (LZ4, Zstandard, pcodec, LZF, bitshuffle, bzip2, blosc,
|
||||
//! blosc2).
|
||||
//! [`builtin_filters`] lists them.
|
||||
//! * **Registered filters** (`std` only) — codecs the application supplies
|
||||
//! for any other ID with [`register_filter`] (a [`FilterCodec`], or just a
|
||||
@@ -166,7 +167,7 @@ pub fn known_filter(id: u16) -> Option<(&'static str, Option<&'static str>)> {
|
||||
32019 => ("JPEG", None),
|
||||
32022 => ("BitGroom", None),
|
||||
32023 => ("Granular BitRound", None),
|
||||
32026 => ("Blosc2", None),
|
||||
32026 => ("Blosc2", Some("blosc2")),
|
||||
_ => return None,
|
||||
})
|
||||
}
|
||||
@@ -452,12 +453,12 @@ pub(crate) mod tests {
|
||||
#[test]
|
||||
fn unsupported_filter_error_names_the_filter() {
|
||||
let msg = FormatError::UnsupportedFilter(32026).to_string();
|
||||
assert!(msg.contains("Blosc2") && msg.contains("`blosc2`"), "{msg}");
|
||||
let msg = FormatError::UnsupportedFilter(32013).to_string();
|
||||
assert!(
|
||||
msg.contains("Blosc2") && msg.contains("not implemented"),
|
||||
msg.contains("ZFP") && msg.contains("not implemented"),
|
||||
"{msg}"
|
||||
);
|
||||
let msg = FormatError::UnsupportedFilter(32013).to_string();
|
||||
assert!(msg.contains("ZFP"), "{msg}");
|
||||
let msg = FormatError::UnsupportedFilter(32000).to_string();
|
||||
assert!(msg.contains("LZF") && msg.contains("`lzf`"), "{msg}");
|
||||
assert_eq!(
|
||||
|
||||
@@ -278,6 +278,13 @@ pub(crate) static BUILTIN_FILTERS: &[BuiltinFilter] = &[
|
||||
},
|
||||
encode: None,
|
||||
},
|
||||
#[cfg(feature = "blosc2")]
|
||||
BuiltinFilter {
|
||||
id: crate::filter_pipeline::FILTER_BLOSC2,
|
||||
name: "blosc2",
|
||||
decode: crate::filters_blosc2::blosc2_decode,
|
||||
encode: None,
|
||||
},
|
||||
];
|
||||
|
||||
/// Decode the HDF5 scale-offset filter (id 6).
|
||||
|
||||
@@ -49,7 +49,7 @@ fn le32(b: &[u8], at: usize) -> Result<usize, FormatError> {
|
||||
|
||||
/// The codec inside a Blosc frame (flags bits 5-7).
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
enum Codec {
|
||||
pub(crate) enum Codec {
|
||||
BloscLz,
|
||||
Lz4,
|
||||
Snappy,
|
||||
@@ -58,7 +58,7 @@ enum Codec {
|
||||
}
|
||||
|
||||
impl Codec {
|
||||
fn from_flags(flags: u8) -> Result<Codec, FormatError> {
|
||||
pub(crate) fn from_flags(flags: u8) -> Result<Codec, FormatError> {
|
||||
match flags >> 5 {
|
||||
0 => Ok(Codec::BloscLz),
|
||||
1 => Ok(Codec::Lz4),
|
||||
@@ -71,7 +71,7 @@ impl Codec {
|
||||
}
|
||||
|
||||
/// Decode one codec stream into exactly `dst`.
|
||||
fn decode_stream(
|
||||
pub(crate) fn decode_stream(
|
||||
codec: Codec,
|
||||
src: &[u8],
|
||||
dst: &mut [u8],
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -86,6 +86,8 @@ pub mod filters;
|
||||
mod filters_bitshuffle;
|
||||
#[cfg(feature = "blosc")]
|
||||
pub mod filters_blosc;
|
||||
#[cfg(feature = "blosc2")]
|
||||
pub mod filters_blosc2;
|
||||
#[cfg(feature = "bzip2")]
|
||||
mod filters_bzip2;
|
||||
#[cfg(feature = "lzf")]
|
||||
|
||||
Reference in New Issue
Block a user