feat(format): read Blosc2 (filter 32026) in pure Rust

hdf5plugin's Blosc2 was a clear "not implemented" error. It stores each
HDF5 chunk as a Blosc2 contiguous frame; for chunks of 2+ dimensions the
frame holds a B2ND array whose data are cut into (padded) blocks stored
one after another.

filters_blosc2 (feature `blosc2`, in `plugin-filters`; the facade
forwards both) decodes, following c-blosc2's decoder and hdf5-blosc2's
blosc2_filter.c:
- the frame: the msgpack header's fixed fields, the metalayer index, the
  offsets chunk (with the special zero/NaN/uninitialised offsets) and
  chunk lookup;
- Blosc2 chunks: 16- and 32-byte headers, special chunks (zeros, NaN,
  uninitialised, one repeated value), split and unsplit streams, zero and
  run-length streams, and the filter pipeline run backwards (shuffle,
  shuffle with a byte-group size, bit shuffle including the version-2 and
  later handling of a partial group of 8, delta against the first block,
  truncated precision);
- the codecs, shared with Blosc 1: BloscLZ, LZ4/LZ4HC, Zlib, Zstandard;
- B2ND arrays: blocks gathered into C order, padding dropped, several
  chunks per array, and the array shape checked against the chunk shape
  in cd_values as the HDF5 filter does.
Dictionaries, lazy chunks, variable-length blocks, user-defined codecs
and registered filters (e.g. bytedelta) are errors. Uninitialised chunks
read as zeros. No encoder.

Tests: h5py + hdf5plugin write every codec x filter (none, shuffle,
bitshuffle, delta) and levels 0-9 over the plugin-filter cases, then
i1..u8/f4/f8 in 1-D to 5-D chunks with partial edge chunks, datasets of
zeros, one value and NaN, and Fletcher32 before Blosc2 (plain frames for
n-D chunks); clawhdf5 reads each exactly as its unfiltered twin, and
truncated precision exactly as h5py reads it. Fixture frames from
python-blosc2 (tests/fixtures/blosc2/generate.py) cover what hdf5plugin
never writes: special chunks, delta over many blocks and odd type sizes,
odd bit-shuffle blocks, forced splitting, multi-chunk B2ND arrays with a
zero chunk, and the refused features. The decoder is fuzzed (random and
mutated frames and chunks: no panic, output within the limit).

Conformance: h5ex_d_blosc2.h5 now reads (576 of 697 ok, baseline 575).

Co-Authored-By: Claude Opus 5.5 (1M context) <[email protected]>

crates/clawhdf5-format/Cargo.toml

@@ -76,8 +76,10 @@ bitshuffle = ["lz4_flex", "ruzstd"]
 bzip2 = ["dep:bzip2", "std"]
 # Blosc 1 (32001) with its BloscLZ, LZ4, Snappy, Zlib and Zstandard codecs.
 blosc = ["lz4_flex", "ruzstd", "snap", "deflate", "std"]
+# Blosc2 (32026), read-only: frames, B2ND arrays, and the Blosc codecs above.
+blosc2 = ["blosc"]
 # Every plugin filter above.
-plugin-filters = ["lzf", "bitshuffle", "bzip2", "blosc"]
+plugin-filters = ["lzf", "bitshuffle", "bzip2", "blosc", "blosc2"]
 
 [[bench]]
 name = "parallel_decompress_bench"

crates/clawhdf5-format/src/filter_registry.rs

@@ -5,7 +5,8 @@
 //! * **Built-in filters** — a static table of the filters compiled into this
 //!   build: the HDF5 standard filters (deflate, shuffle, Fletcher32, szip,
 //!   N-Bit, scale-offset) and the plugin filters whose cargo features are
-//!   enabled (LZ4, Zstandard, pcodec, LZF, bitshuffle, bzip2, blosc).
+//!   enabled (LZ4, Zstandard, pcodec, LZF, bitshuffle, bzip2, blosc,
+//!   blosc2).
 //!   [`builtin_filters`] lists them.
 //! * **Registered filters** (`std` only) — codecs the application supplies
 //!   for any other ID with [`register_filter`] (a [`FilterCodec`], or just a
@@ -166,7 +167,7 @@ pub fn known_filter(id: u16) -> Option<(&'static str, Option<&'static str>)> {
         32019 => ("JPEG", None),
         32022 => ("BitGroom", None),
         32023 => ("Granular BitRound", None),
-        32026 => ("Blosc2", None),
+        32026 => ("Blosc2", Some("blosc2")),
         _ => return None,
     })
 }
@@ -452,12 +453,12 @@ pub(crate) mod tests {
     #[test]
     fn unsupported_filter_error_names_the_filter() {
         let msg = FormatError::UnsupportedFilter(32026).to_string();
+        assert!(msg.contains("Blosc2") && msg.contains("`blosc2`"), "{msg}");
+        let msg = FormatError::UnsupportedFilter(32013).to_string();
         assert!(
-            msg.contains("Blosc2") && msg.contains("not implemented"),
+            msg.contains("ZFP") && msg.contains("not implemented"),
             "{msg}"
         );
-        let msg = FormatError::UnsupportedFilter(32013).to_string();
-        assert!(msg.contains("ZFP"), "{msg}");
         let msg = FormatError::UnsupportedFilter(32000).to_string();
         assert!(msg.contains("LZF") && msg.contains("`lzf`"), "{msg}");
         assert_eq!(

crates/clawhdf5-format/src/filters.rs

@@ -278,6 +278,13 @@ pub(crate) static BUILTIN_FILTERS: &[BuiltinFilter] = &[
         },
         encode: None,
     },
+    #[cfg(feature = "blosc2")]
+    BuiltinFilter {
+        id: crate::filter_pipeline::FILTER_BLOSC2,
+        name: "blosc2",
+        decode: crate::filters_blosc2::blosc2_decode,
+        encode: None,
+    },
 ];
 
 /// Decode the HDF5 scale-offset filter (id 6).

crates/clawhdf5-format/src/filters_blosc.rs

@@ -49,7 +49,7 @@ fn le32(b: &[u8], at: usize) -> Result<usize, FormatError> {
 
 /// The codec inside a Blosc frame (flags bits 5-7).
 #[derive(Debug, Clone, Copy, PartialEq, Eq)]
-enum Codec {
+pub(crate) enum Codec {
     BloscLz,
     Lz4,
     Snappy,
@@ -58,7 +58,7 @@ enum Codec {
 }
 
 impl Codec {
-    fn from_flags(flags: u8) -> Result<Codec, FormatError> {
+    pub(crate) fn from_flags(flags: u8) -> Result<Codec, FormatError> {
         match flags >> 5 {
             0 => Ok(Codec::BloscLz),
             1 => Ok(Codec::Lz4),
@@ -71,7 +71,7 @@ impl Codec {
 }
 
 /// Decode one codec stream into exactly `dst`.
-fn decode_stream(
+pub(crate) fn decode_stream(
     codec: Codec,
     src: &[u8],
     dst: &mut [u8],

crates/clawhdf5-format/src/lib.rs

@@ -86,6 +86,8 @@ pub mod filters;
 mod filters_bitshuffle;
 #[cfg(feature = "blosc")]
 pub mod filters_blosc;
+#[cfg(feature = "blosc2")]
+pub mod filters_blosc2;
 #[cfg(feature = "bzip2")]
 mod filters_bzip2;
 #[cfg(feature = "lzf")]

crates/clawhdf5-format/tests/fixtures/blosc2/bytedelta.err

@@ -0,0 +1 @@
+filter 35

crates/clawhdf5-format/tests/fixtures/blosc2/generate.py

@@ -0,0 +1,129 @@
+"""Generate the Blosc2 frames the `filters_blosc2` unit tests decode.
+
+Each case is `<name>.b2f` (a Blosc2 contiguous frame, what the HDF5 Blosc2
+filter stores per chunk) and `<name>.out` (what decoding it must give: the
+first chunk of a plain frame, or the whole array in C order for a B2ND
+frame), or `<name>.err` (a frame clawhdf5 must refuse; the file holds a word
+the error message must contain).
+
+These cover what files written by h5py + hdf5plugin never contain, but a
+Blosc2 frame may: special chunks (repeated value, NaN, uninitialised), the
+delta filter over many blocks and odd type sizes, bit shuffle of blocks that
+are not a multiple of 8 elements, shuffle with a byte-group size, forced
+stream splitting, multi-chunk B2ND arrays with padded edge chunks and a
+chunk of zeros, and features clawhdf5 refuses (dictionaries, registered
+filters).
+
+Written with python-blosc2 4.13.1 (c-blosc2 3.3.4) in a scratch venv
+(`pip install blosc2`). Re-run only to regenerate:
+
+    python generate.py <this directory>
+"""
+import os
+import sys
+
+import blosc2
+import numpy as np
+
+out = sys.argv[1]
+
+
+def save(name, frame, expected):
+    with open(os.path.join(out, name + ".b2f"), "wb") as f:
+        f.write(frame)
+    with open(os.path.join(out, name + ".out"), "wb") as f:
+        f.write(expected)
+
+
+def save_err(name, frame, word):
+    with open(os.path.join(out, name + ".b2f"), "wb") as f:
+        f.write(frame)
+    with open(os.path.join(out, name + ".err"), "w") as f:
+        f.write(word)
+
+
+def plain(data, **cparams):
+    """A one-chunk super-chunk frame of `data`, as hdf5-blosc2 writes."""
+    data = np.ascontiguousarray(data)
+    cp = blosc2.CParams(typesize=data.dtype.itemsize, **cparams)
+    sc = blosc2.SChunk(chunksize=data.nbytes, cparams=cp)
+    sc.append_data(data)
+    return sc.to_cframe(), data.tobytes()
+
+
+def special(nitems, dtype, kind, value=None):
+    dt = np.dtype(dtype)
+    sc = blosc2.SChunk(chunksize=nitems * dt.itemsize,
+                       cparams=blosc2.CParams(typesize=dt.itemsize))
+    sc.fill_special(nitems, kind, value)
+    return sc.to_cframe()
+
+
+# Special chunks. A repeated value stays in the frame as a 33+ byte chunk;
+# NaN and uninitialised chunks become special offsets.
+save("value_i4", special(300, "<i4", blosc2.SpecialValue.VALUE, 123456),
+     np.full(300, 123456, "<i4").tobytes())
+save("value_f8", special(250, "<f8", blosc2.SpecialValue.VALUE, -2.5),
+     np.full(250, -2.5, "<f8").tobytes())
+save("nan_f4", special(500, "<f4", blosc2.SpecialValue.NAN),
+     np.full(500, np.nan, "<f4").tobytes())
+save("nan_f8", special(300, "<f8", blosc2.SpecialValue.NAN),
+     np.full(300, np.nan, "<f8").tobytes())
+save("zero_u2", special(2000, "<u2", blosc2.SpecialValue.ZERO), bytes(4000))
+# Uninitialised values: libhdf5 would hand back whatever memory it had;
+# clawhdf5 returns zeros.
+save("uninit_i8", special(64, "<i8", blosc2.SpecialValue.UNINIT), bytes(512))
+
+rng = np.random.default_rng(11)
+ramp = lambda n, dt: ((np.arange(n) * 7) % 1000 + rng.integers(0, 3, n)).astype(dt)
+# Slowly varying: what the delta filter is for (noise would be stored raw).
+smooth = lambda n, dt: (np.arange(n) // 3 + 1000).astype(dt)
+
+# Delta over many blocks, for type sizes 1, 2, 4, 8, 3 (bytes) and 16 (u64
+# pairs).
+for dt, n, codec in [("<u1", 2000, blosc2.Codec.LZ4), ("<i2", 1000, blosc2.Codec.BLOSCLZ),
+                     ("<i4", 700, blosc2.Codec.LZ4), ("<u8", 400, blosc2.Codec.BLOSCLZ)]:
+    save(f"delta_{np.dtype(dt).name}_{codec.name.lower()}",
+         *plain(smooth(n, dt), codec=codec, blocksize=256,
+                filters=[blosc2.Filter.DELTA], filters_meta=[0]))
+rec3 = np.frombuffer(smooth(3 * 300, "<u1").tobytes(), dtype="V3")
+save("delta_v3", *plain(rec3, blocksize=300, filters=[blosc2.Filter.DELTA], filters_meta=[0]))
+rec16 = np.frombuffer(smooth(2 * 200, "<u8").tobytes(), dtype="V16")
+save("delta_shuffle_v16", *plain(rec16, blocksize=512,
+                                 filters=[blosc2.Filter.DELTA, blosc2.Filter.SHUFFLE],
+                                 filters_meta=[0, 0]))
+
+# Bit shuffle of blocks whose element count is not a multiple of 8 (44-byte
+# blocks of 4-byte elements: 8 transposed, 3 copied).
+save("bitshuffle_odd_blocks", *plain(ramp(500, "<i4"), codec=blosc2.Codec.ZSTD, blocksize=44,
+                                     filters=[blosc2.Filter.BITSHUFFLE], filters_meta=[0]))
+# Shuffle in groups of 2 bytes of an 8-byte type (filters_meta).
+save("shuffle_meta2", *plain(ramp(500, "<i8"), codec=blosc2.Codec.ZLIB,
+                             filters=[blosc2.Filter.SHUFFLE], filters_meta=[2]))
+# Streams split per byte, and never split.
+save("always_split", *plain(ramp(1000, "<f4"), codec=blosc2.Codec.LZ4HC,
+                            splitmode=blosc2.SplitMode.ALWAYS_SPLIT))
+save("never_split", *plain(ramp(1500, "<u2"), codec=blosc2.Codec.ZSTD,
+                           splitmode=blosc2.SplitMode.NEVER_SPLIT))
+
+# B2ND arrays of several chunks whose edge chunks and blocks are padded, and
+# one whose middle chunk is all zeros (a special offset).
+for name, shape, chunks, blocks, dt in [
+    ("b2nd_2d", (37, 29), (10, 16), (4, 6), "<i4"),
+    ("b2nd_3d", (9, 10, 7), (4, 5, 3), (3, 2, 2), "<f4"),
+    ("b2nd_4d", (5, 7, 5, 6), (3, 2, 5, 4), (2, 2, 3, 3), "<u2"),
+]:
+    a = ramp(int(np.prod(shape)), dt).reshape(shape)
+    arr = blosc2.asarray(a, chunks=chunks, blocks=blocks)
+    save(name, arr.to_cframe(), a.tobytes())
+a = ramp(60 * 20, "<i2").reshape(60, 20)
+a[20:40, :] = 0
+arr = blosc2.asarray(a, chunks=(20, 20), blocks=(8, 16))
+save("b2nd_zero_chunk", arr.to_cframe(), a.tobytes())
+
+# Refused: a dictionary, and a registered filter (bytedelta).
+frame, _ = plain(ramp(4000, "<i4"), codec=blosc2.Codec.ZSTD, use_dict=True, blocksize=2048)
+save_err("zstd_dict", frame, "dictionar")
+frame, _ = plain(ramp(500, "<i4"), filters=[blosc2.Filter.SHUFFLE, blosc2.Filter.BYTEDELTA],
+                 filters_meta=[0, 4])
+save_err("bytedelta", frame, "filter 35")

crates/clawhdf5-format/tests/fixtures/blosc2/zstd_dict.err

@@ -0,0 +1 @@
+dictionar