From 7334e21c9380fe5c6360a739e3c7abd7c3f4717f Mon Sep 17 00:00:00 2001 From: osobh Date: Sat, 26 Sep 2026 10:22:22 -0500 Subject: [PATCH 1/9] feat(format): read Blosc2 (filter 32026) in pure Rust hdf5plugin's Blosc2 was a clear "not implemented" error. It stores each HDF5 chunk as a Blosc2 contiguous frame; for chunks of 2+ dimensions the frame holds a B2ND array whose data are cut into (padded) blocks stored one after another. filters_blosc2 (feature `blosc2`, in `plugin-filters`; the facade forwards both) decodes, following c-blosc2's decoder and hdf5-blosc2's blosc2_filter.c: - the frame: the msgpack header's fixed fields, the metalayer index, the offsets chunk (with the special zero/NaN/uninitialised offsets) and chunk lookup; - Blosc2 chunks: 16- and 32-byte headers, special chunks (zeros, NaN, uninitialised, one repeated value), split and unsplit streams, zero and run-length streams, and the filter pipeline run backwards (shuffle, shuffle with a byte-group size, bit shuffle including the version-2 and later handling of a partial group of 8, delta against the first block, truncated precision); - the codecs, shared with Blosc 1: BloscLZ, LZ4/LZ4HC, Zlib, Zstandard; - B2ND arrays: blocks gathered into C order, padding dropped, several chunks per array, and the array shape checked against the chunk shape in cd_values as the HDF5 filter does. Dictionaries, lazy chunks, variable-length blocks, user-defined codecs and registered filters (e.g. bytedelta) are errors. Uninitialised chunks read as zeros. No encoder. Tests: h5py + hdf5plugin write every codec x filter (none, shuffle, bitshuffle, delta) and levels 0-9 over the plugin-filter cases, then i1..u8/f4/f8 in 1-D to 5-D chunks with partial edge chunks, datasets of zeros, one value and NaN, and Fletcher32 before Blosc2 (plain frames for n-D chunks); clawhdf5 reads each exactly as its unfiltered twin, and truncated precision exactly as h5py reads it. Fixture frames from python-blosc2 (tests/fixtures/blosc2/generate.py) cover what hdf5plugin never writes: special chunks, delta over many blocks and odd type sizes, odd bit-shuffle blocks, forced splitting, multi-chunk B2ND arrays with a zero chunk, and the refused features. The decoder is fuzzed (random and mutated frames and chunks: no panic, output within the limit). Conformance: h5ex_d_blosc2.h5 now reads (576 of 697 ok, baseline 575). Co-Authored-By: Claude Opus 5.5 (1M context) --- crates/clawhdf5-format/Cargo.toml | 4 +- crates/clawhdf5-format/src/filter_registry.rs | 11 +- crates/clawhdf5-format/src/filters.rs | 7 + crates/clawhdf5-format/src/filters_blosc.rs | 6 +- crates/clawhdf5-format/src/filters_blosc2.rs | 1083 +++++++++++++++++ crates/clawhdf5-format/src/lib.rs | 2 + .../tests/fixtures/blosc2/always_split.b2f | Bin 0 -> 1650 bytes .../tests/fixtures/blosc2/always_split.out | Bin 0 -> 4000 bytes .../tests/fixtures/blosc2/b2nd_2d.b2f | Bin 0 -> 3950 bytes .../tests/fixtures/blosc2/b2nd_2d.out | Bin 0 -> 4292 bytes .../tests/fixtures/blosc2/b2nd_3d.b2f | Bin 0 -> 8108 bytes .../tests/fixtures/blosc2/b2nd_3d.out | Bin 0 -> 2520 bytes .../tests/fixtures/blosc2/b2nd_4d.b2f | Bin 0 -> 6804 bytes .../tests/fixtures/blosc2/b2nd_4d.out | Bin 0 -> 2100 bytes .../tests/fixtures/blosc2/b2nd_zero_chunk.b2f | Bin 0 -> 1849 bytes .../tests/fixtures/blosc2/b2nd_zero_chunk.out | Bin 0 -> 2400 bytes .../fixtures/blosc2/bitshuffle_odd_blocks.b2f | Bin 0 -> 2204 bytes .../fixtures/blosc2/bitshuffle_odd_blocks.out | Bin 0 -> 2000 bytes .../tests/fixtures/blosc2/bytedelta.b2f | Bin 0 -> 458 bytes .../tests/fixtures/blosc2/bytedelta.err | 1 + .../fixtures/blosc2/delta_int16_blosclz.b2f | Bin 0 -> 1601 bytes .../fixtures/blosc2/delta_int16_blosclz.out | Bin 0 -> 2000 bytes .../tests/fixtures/blosc2/delta_int32_lz4.b2f | Bin 0 -> 1584 bytes .../tests/fixtures/blosc2/delta_int32_lz4.out | Bin 0 -> 2800 bytes .../fixtures/blosc2/delta_shuffle_v16.b2f | Bin 0 -> 1019 bytes .../fixtures/blosc2/delta_shuffle_v16.out | Bin 0 -> 3200 bytes .../fixtures/blosc2/delta_uint64_blosclz.b2f | Bin 0 -> 1323 bytes .../fixtures/blosc2/delta_uint64_blosclz.out | Bin 0 -> 3200 bytes .../tests/fixtures/blosc2/delta_uint8_lz4.b2f | Bin 0 -> 1494 bytes .../tests/fixtures/blosc2/delta_uint8_lz4.out | Bin 0 -> 2000 bytes .../tests/fixtures/blosc2/delta_v3.b2f | Bin 0 -> 377 bytes .../tests/fixtures/blosc2/delta_v3.out | Bin 0 -> 900 bytes .../tests/fixtures/blosc2/generate.py | 129 ++ .../tests/fixtures/blosc2/nan_f4.b2f | Bin 0 -> 172 bytes .../tests/fixtures/blosc2/nan_f4.out | Bin 0 -> 2000 bytes .../tests/fixtures/blosc2/nan_f8.b2f | Bin 0 -> 172 bytes .../tests/fixtures/blosc2/nan_f8.out | Bin 0 -> 2400 bytes .../tests/fixtures/blosc2/never_split.b2f | Bin 0 -> 1671 bytes .../tests/fixtures/blosc2/never_split.out | Bin 0 -> 3000 bytes .../tests/fixtures/blosc2/shuffle_meta2.b2f | Bin 0 -> 1003 bytes .../tests/fixtures/blosc2/shuffle_meta2.out | Bin 0 -> 4000 bytes .../tests/fixtures/blosc2/uninit_i8.b2f | Bin 0 -> 172 bytes .../tests/fixtures/blosc2/uninit_i8.out | Bin 0 -> 512 bytes .../tests/fixtures/blosc2/value_f8.b2f | Bin 0 -> 212 bytes .../tests/fixtures/blosc2/value_f8.out | Bin 0 -> 2000 bytes .../tests/fixtures/blosc2/value_i4.b2f | Bin 0 -> 208 bytes .../tests/fixtures/blosc2/value_i4.out | Bin 0 -> 1200 bytes .../tests/fixtures/blosc2/zero_u2.b2f | Bin 0 -> 172 bytes .../tests/fixtures/blosc2/zero_u2.out | Bin 0 -> 4000 bytes .../tests/fixtures/blosc2/zstd_dict.b2f | Bin 0 -> 4978 bytes .../tests/fixtures/blosc2/zstd_dict.err | 1 + crates/clawhdf5/Cargo.toml | 3 +- .../clawhdf5/tests/plugin_filters_interop.rs | 135 +- scripts/ci-test.sh | 4 +- 54 files changed, 1366 insertions(+), 20 deletions(-) create mode 100644 crates/clawhdf5-format/src/filters_blosc2.rs create mode 100644 crates/clawhdf5-format/tests/fixtures/blosc2/always_split.b2f create mode 100644 crates/clawhdf5-format/tests/fixtures/blosc2/always_split.out create mode 100644 crates/clawhdf5-format/tests/fixtures/blosc2/b2nd_2d.b2f create mode 100644 crates/clawhdf5-format/tests/fixtures/blosc2/b2nd_2d.out create mode 100644 crates/clawhdf5-format/tests/fixtures/blosc2/b2nd_3d.b2f create mode 100644 crates/clawhdf5-format/tests/fixtures/blosc2/b2nd_3d.out create mode 100644 crates/clawhdf5-format/tests/fixtures/blosc2/b2nd_4d.b2f create mode 100644 crates/clawhdf5-format/tests/fixtures/blosc2/b2nd_4d.out create mode 100644 crates/clawhdf5-format/tests/fixtures/blosc2/b2nd_zero_chunk.b2f create mode 100644 crates/clawhdf5-format/tests/fixtures/blosc2/b2nd_zero_chunk.out create mode 100644 crates/clawhdf5-format/tests/fixtures/blosc2/bitshuffle_odd_blocks.b2f create mode 100644 crates/clawhdf5-format/tests/fixtures/blosc2/bitshuffle_odd_blocks.out create mode 100644 crates/clawhdf5-format/tests/fixtures/blosc2/bytedelta.b2f create mode 100644 crates/clawhdf5-format/tests/fixtures/blosc2/bytedelta.err create mode 100644 crates/clawhdf5-format/tests/fixtures/blosc2/delta_int16_blosclz.b2f create mode 100644 crates/clawhdf5-format/tests/fixtures/blosc2/delta_int16_blosclz.out create mode 100644 crates/clawhdf5-format/tests/fixtures/blosc2/delta_int32_lz4.b2f create mode 100644 crates/clawhdf5-format/tests/fixtures/blosc2/delta_int32_lz4.out create mode 100644 crates/clawhdf5-format/tests/fixtures/blosc2/delta_shuffle_v16.b2f create mode 100644 crates/clawhdf5-format/tests/fixtures/blosc2/delta_shuffle_v16.out create mode 100644 crates/clawhdf5-format/tests/fixtures/blosc2/delta_uint64_blosclz.b2f create mode 100644 crates/clawhdf5-format/tests/fixtures/blosc2/delta_uint64_blosclz.out create mode 100644 crates/clawhdf5-format/tests/fixtures/blosc2/delta_uint8_lz4.b2f create mode 100644 crates/clawhdf5-format/tests/fixtures/blosc2/delta_uint8_lz4.out create mode 100644 crates/clawhdf5-format/tests/fixtures/blosc2/delta_v3.b2f create mode 100644 crates/clawhdf5-format/tests/fixtures/blosc2/delta_v3.out create mode 100644 crates/clawhdf5-format/tests/fixtures/blosc2/generate.py create mode 100644 crates/clawhdf5-format/tests/fixtures/blosc2/nan_f4.b2f create mode 100644 crates/clawhdf5-format/tests/fixtures/blosc2/nan_f4.out create mode 100644 crates/clawhdf5-format/tests/fixtures/blosc2/nan_f8.b2f create mode 100644 crates/clawhdf5-format/tests/fixtures/blosc2/nan_f8.out create mode 100644 crates/clawhdf5-format/tests/fixtures/blosc2/never_split.b2f create mode 100644 crates/clawhdf5-format/tests/fixtures/blosc2/never_split.out create mode 100644 crates/clawhdf5-format/tests/fixtures/blosc2/shuffle_meta2.b2f create mode 100644 crates/clawhdf5-format/tests/fixtures/blosc2/shuffle_meta2.out create mode 100644 crates/clawhdf5-format/tests/fixtures/blosc2/uninit_i8.b2f create mode 100644 crates/clawhdf5-format/tests/fixtures/blosc2/uninit_i8.out create mode 100644 crates/clawhdf5-format/tests/fixtures/blosc2/value_f8.b2f create mode 100644 crates/clawhdf5-format/tests/fixtures/blosc2/value_f8.out create mode 100644 crates/clawhdf5-format/tests/fixtures/blosc2/value_i4.b2f create mode 100644 crates/clawhdf5-format/tests/fixtures/blosc2/value_i4.out create mode 100644 crates/clawhdf5-format/tests/fixtures/blosc2/zero_u2.b2f create mode 100644 crates/clawhdf5-format/tests/fixtures/blosc2/zero_u2.out create mode 100644 crates/clawhdf5-format/tests/fixtures/blosc2/zstd_dict.b2f create mode 100644 crates/clawhdf5-format/tests/fixtures/blosc2/zstd_dict.err diff --git a/crates/clawhdf5-format/Cargo.toml b/crates/clawhdf5-format/Cargo.toml index 1d1998c..c1ec6df 100644 --- a/crates/clawhdf5-format/Cargo.toml +++ b/crates/clawhdf5-format/Cargo.toml @@ -76,8 +76,10 @@ bitshuffle = ["lz4_flex", "ruzstd"] bzip2 = ["dep:bzip2", "std"] # Blosc 1 (32001) with its BloscLZ, LZ4, Snappy, Zlib and Zstandard codecs. blosc = ["lz4_flex", "ruzstd", "snap", "deflate", "std"] +# Blosc2 (32026), read-only: frames, B2ND arrays, and the Blosc codecs above. +blosc2 = ["blosc"] # Every plugin filter above. -plugin-filters = ["lzf", "bitshuffle", "bzip2", "blosc"] +plugin-filters = ["lzf", "bitshuffle", "bzip2", "blosc", "blosc2"] [[bench]] name = "parallel_decompress_bench" diff --git a/crates/clawhdf5-format/src/filter_registry.rs b/crates/clawhdf5-format/src/filter_registry.rs index 6d39613..442e27c 100644 --- a/crates/clawhdf5-format/src/filter_registry.rs +++ b/crates/clawhdf5-format/src/filter_registry.rs @@ -5,7 +5,8 @@ //! * **Built-in filters** — a static table of the filters compiled into this //! build: the HDF5 standard filters (deflate, shuffle, Fletcher32, szip, //! N-Bit, scale-offset) and the plugin filters whose cargo features are -//! enabled (LZ4, Zstandard, pcodec, LZF, bitshuffle, bzip2, blosc). +//! enabled (LZ4, Zstandard, pcodec, LZF, bitshuffle, bzip2, blosc, +//! blosc2). //! [`builtin_filters`] lists them. //! * **Registered filters** (`std` only) — codecs the application supplies //! for any other ID with [`register_filter`] (a [`FilterCodec`], or just a @@ -166,7 +167,7 @@ pub fn known_filter(id: u16) -> Option<(&'static str, Option<&'static str>)> { 32019 => ("JPEG", None), 32022 => ("BitGroom", None), 32023 => ("Granular BitRound", None), - 32026 => ("Blosc2", None), + 32026 => ("Blosc2", Some("blosc2")), _ => return None, }) } @@ -452,12 +453,12 @@ pub(crate) mod tests { #[test] fn unsupported_filter_error_names_the_filter() { let msg = FormatError::UnsupportedFilter(32026).to_string(); + assert!(msg.contains("Blosc2") && msg.contains("`blosc2`"), "{msg}"); + let msg = FormatError::UnsupportedFilter(32013).to_string(); assert!( - msg.contains("Blosc2") && msg.contains("not implemented"), + msg.contains("ZFP") && msg.contains("not implemented"), "{msg}" ); - let msg = FormatError::UnsupportedFilter(32013).to_string(); - assert!(msg.contains("ZFP"), "{msg}"); let msg = FormatError::UnsupportedFilter(32000).to_string(); assert!(msg.contains("LZF") && msg.contains("`lzf`"), "{msg}"); assert_eq!( diff --git a/crates/clawhdf5-format/src/filters.rs b/crates/clawhdf5-format/src/filters.rs index 36f2e96..d577eac 100644 --- a/crates/clawhdf5-format/src/filters.rs +++ b/crates/clawhdf5-format/src/filters.rs @@ -278,6 +278,13 @@ pub(crate) static BUILTIN_FILTERS: &[BuiltinFilter] = &[ }, encode: None, }, + #[cfg(feature = "blosc2")] + BuiltinFilter { + id: crate::filter_pipeline::FILTER_BLOSC2, + name: "blosc2", + decode: crate::filters_blosc2::blosc2_decode, + encode: None, + }, ]; /// Decode the HDF5 scale-offset filter (id 6). diff --git a/crates/clawhdf5-format/src/filters_blosc.rs b/crates/clawhdf5-format/src/filters_blosc.rs index 3edc964..fadfc1e 100644 --- a/crates/clawhdf5-format/src/filters_blosc.rs +++ b/crates/clawhdf5-format/src/filters_blosc.rs @@ -49,7 +49,7 @@ fn le32(b: &[u8], at: usize) -> Result { /// The codec inside a Blosc frame (flags bits 5-7). #[derive(Debug, Clone, Copy, PartialEq, Eq)] -enum Codec { +pub(crate) enum Codec { BloscLz, Lz4, Snappy, @@ -58,7 +58,7 @@ enum Codec { } impl Codec { - fn from_flags(flags: u8) -> Result { + pub(crate) fn from_flags(flags: u8) -> Result { match flags >> 5 { 0 => Ok(Codec::BloscLz), 1 => Ok(Codec::Lz4), @@ -71,7 +71,7 @@ impl Codec { } /// Decode one codec stream into exactly `dst`. -fn decode_stream( +pub(crate) fn decode_stream( codec: Codec, src: &[u8], dst: &mut [u8], diff --git a/crates/clawhdf5-format/src/filters_blosc2.rs b/crates/clawhdf5-format/src/filters_blosc2.rs new file mode 100644 index 0000000..5f332f8 --- /dev/null +++ b/crates/clawhdf5-format/src/filters_blosc2.rs @@ -0,0 +1,1083 @@ +//! Blosc2 (HDF5 filter 32026, `hdf5-blosc2`, hdf5plugin's `Blosc2`) in pure +//! Rust, read-only: the contiguous frame ("cframe") each chunk is stored +//! as, the Blosc2 chunks inside it, and the B2ND n-dimensional layout. +//! +//! References: c-blosc2's `README_CFRAME_FORMAT.rst`, +//! `README_CHUNK_FORMAT.rst` and `README_B2ND_METALAYER.rst`, and the +//! decoder in `blosc/blosc2.c` (`read_chunk_header`, `blosc_d`, +//! `pipeline_backward`), `blosc/frame.c` and `blosc/b2nd.c`, whose checks +//! this module mirrors. +//! +//! **The filter** (`blosc2_filter.c`). `cd_values`: `[0]` filter revision, +//! `[1]` block size, `[2]` type size, `[3]` chunk size in bytes, `[4]` +//! level, `[5]` filter (0 none, 1 shuffle, 2 bit shuffle, 3 delta, 4 +//! truncate precision), `[6]` codec, and for chunks of 2 to 8 (16 in newer +//! builds) dimensions `[7]` the rank and `[8..]` the chunk dimensions. Each +//! HDF5 chunk is one frame. A frame with a `b2nd` (or older `caterva`) +//! metalayer holds an n-D array the shape of the HDF5 chunk, whose data are +//! cut into blocks of `blockshape` (padded at the chunk's edges) stored one +//! after another, each in C order — so the decoded Blosc2 chunk is +//! reassembled into C order here. Otherwise the frame's first chunk is the +//! HDF5 chunk's data as it is. +//! +//! **Frame.** A msgpack header with fields at fixed offsets (header length, +//! frame length, sizes, type size, chunk size, codec flags) and a map of +//! metalayers; the chunks back to back; an "offsets" chunk (a Blosc2 chunk +//! of little-endian `i64` offsets, one per chunk, relative to the end of the +//! header — negative values encode a chunk of zeros, NaNs or uninitialised +//! values); and a msgpack trailer, which the decoder does not need. +//! +//! **Chunk.** A 16-byte Blosc header (as Blosc 1), extended to 32 bytes when +//! the shuffle and bit-shuffle flags are both set: six filter slots with +//! their metadata, the codec, and flags for dictionaries, lazy chunks and +//! "special" chunks (all zeros, NaN, uninitialised, or one repeated value +//! stored after the header). A regular chunk has a table of block starts; +//! each block is one stream, or `typesize` streams when the "do not split" +//! flag is clear and the block is not the short last one. A stream is an +//! `i32` length and the codec's output: 0 means zeros, negative with a token +//! byte means a run of one byte, and a length equal to the stream's size +//! means stored raw. Decoded blocks run through the filter pipeline +//! backwards (shuffle, bit shuffle, delta; truncate-precision needs no +//! decoding). The codecs are Blosc 1's: BloscLZ, LZ4 (and LZ4HC), Zlib and +//! Zstandard, shared with [`crate::filters_blosc`]. +//! +//! Chunks of uninitialised values read as zeros (libhdf5 hands back +//! whatever its buffer held). Not supported (a clear error, never data): +//! variable-length blocks, dictionaries, lazy chunks, user-defined codecs +//! and registered filters (e.g. bytedelta), sparse frames. + +use crate::error::FormatError; +use crate::filter_registry::FilterContext; +use crate::filters_bitshuffle::bitunshuffle_block; +use crate::filters_blosc::{Codec, blosclz_decompress, decode_stream}; + +const MIN_HEADER: usize = 16; +const EXT_HEADER: usize = 32; + +const FLAG_SHUFFLE: u8 = 0x01; +const FLAG_MEMCPYED: u8 = 0x02; +const FLAG_BITSHUFFLE: u8 = 0x04; +const FLAG_DELTA: u8 = 0x08; +const FLAG_DONT_SPLIT: u8 = 0x10; + +const B2_USEDICT: u8 = 0x01; +const B2_LAZY: u8 = 0x08; +const VL_BLOCKS: u8 = 0x01; + +const SPECIAL_ZERO: u8 = 1; +const SPECIAL_NAN: u8 = 2; +const SPECIAL_VALUE: u8 = 3; +const SPECIAL_UNINIT: u8 = 4; + +const FILTER_NONE: u8 = 0; +const FILTER_SHUFFLE: u8 = 1; +const FILTER_BITSHUFFLE: u8 = 2; +const FILTER_DELTA: u8 = 3; +const FILTER_TRUNC_PREC: u8 = 4; + +/// Chunk format versions: 3 is the Blosc 2 alpha series. +const VERSION_ALPHA: u8 = 3; +/// The newest chunk format c-blosc2 knows (6 added variable-length blocks). +const VERSION_MAX: u8 = 6; +/// The newest frame format c-blosc2 knows. +const FRAME_VERSION_MAX: u8 = 3; +const MAX_BLOCKSIZE: usize = 536_866_816; +/// `B2ND_MAX_DIM` in current c-blosc2 (8 in older builds). +const B2ND_MAX_DIM: usize = 16; + +// Frame header field offsets (c-blosc2 `frame.h`). +const FRAME_HEADER_LEN: usize = 11; +const FRAME_LEN: usize = 16; +const FRAME_FLAGS: usize = 25; +const FRAME_TYPE: usize = 26; +const FRAME_NBYTES: usize = 30; +const FRAME_CBYTES: usize = 39; +const FRAME_TYPESIZE: usize = 48; +const FRAME_CHUNKSIZE: usize = 58; +const FRAME_HEADER_MINLEN: usize = 87; +const FRAME_IDX_SIZE: usize = 89; +const FRAME_VL_BLOCKS: u8 = 0x80; + +fn err(msg: &str) -> FormatError { + FormatError::DecompressionError(format!("blosc2: {msg}")) +} + +fn le_i32(b: &[u8], at: usize) -> Result { + b.get(at..at + 4) + .map(|s| i32::from_le_bytes(s.try_into().unwrap())) + .ok_or_else(|| err("truncated chunk")) +} + +fn be(b: &[u8], at: usize) -> Result<[u8; N], FormatError> { + b.get(at..at + N) + .map(|s| s.try_into().unwrap()) + .ok_or_else(|| err("truncated frame header")) +} + +/// A parsed Blosc2 chunk header. +struct ChunkHeader { + version: u8, + flags: u8, + typesize: usize, + nbytes: usize, + blocksize: usize, + cbytes: usize, + overhead: usize, + filters: [u8; 6], + filters_meta: [u8; 6], + udcodec: u8, + blosc2_flags: u8, + special: u8, +} + +fn read_header(src: &[u8]) -> Result { + if src.len() < MIN_HEADER { + return Err(err("truncated chunk header")); + } + let version = src[0]; + let flags = src[2]; + let typesize = src[3] as usize; + let nbytes = le_i32(src, 4)?; + let blocksize = le_i32(src, 8)?; + let cbytes = le_i32(src, 12)?; + if cbytes < MIN_HEADER as i32 { + return Err(err("chunk size smaller than its header")); + } + if blocksize <= 0 || blocksize as usize > MAX_BLOCKSIZE { + return Err(err("bad block size")); + } + if typesize == 0 { + return Err(err("type size is zero")); + } + if nbytes < 0 { + return Err(err("negative decoded size")); + } + let (nbytes, mut blocksize, cbytes) = (nbytes as usize, blocksize as usize, cbytes as usize); + let mut h = ChunkHeader { + version, + flags, + typesize, + nbytes, + blocksize, + cbytes, + overhead: MIN_HEADER, + filters: [0; 6], + filters_meta: [0; 6], + udcodec: 0, + blosc2_flags: 0, + special: 0, + }; + let mut flags2 = 0; + if flags & (FLAG_SHUFFLE | FLAG_BITSHUFFLE) == FLAG_SHUFFLE | FLAG_BITSHUFFLE { + if cbytes < EXT_HEADER || src.len() < EXT_HEADER { + return Err(err("truncated extended chunk header")); + } + h.overhead = EXT_HEADER; + h.filters.copy_from_slice(&src[16..22]); + h.udcodec = src[22]; + h.filters_meta.copy_from_slice(&src[24..30]); + flags2 = src[30]; + h.blosc2_flags = src[31]; + h.special = (h.blosc2_flags >> 4) & 7; + if h.special == SPECIAL_VALUE { + let ts = cbytes - EXT_HEADER; + if ts == 0 || ts > nbytes || !nbytes.is_multiple_of(ts) { + return Err(err("bad repeated-value chunk")); + } + } else if h.special != 0 && h.special != SPECIAL_ZERO && !nbytes.is_multiple_of(typesize) { + return Err(err("decoded size is not a whole number of elements")); + } + if version == VERSION_ALPHA { + h.filters[5] = 0; + h.filters_meta[5] = 0; + } + } else { + // A Blosc 1 style header: the filters come from the flags. + if flags & FLAG_SHUFFLE != 0 { + h.filters[5] = FILTER_SHUFFLE; + } + if flags & FLAG_BITSHUFFLE != 0 { + h.filters[5] = FILTER_BITSHUFFLE; + } + if flags & FLAG_DELTA != 0 { + h.filters[4] = FILTER_DELTA; + } + } + if version > VERSION_MAX && flags2 & !VL_BLOCKS != 0 { + return Err(err(&format!("chunk format version {version} is too new"))); + } + if flags2 & VL_BLOCKS != 0 { + return Err(err("variable-length blocks are not supported")); + } + if nbytes > 0 && blocksize > nbytes { + blocksize = nbytes; + } + h.blocksize = blocksize; + if cbytes > src.len() { + return Err(err("chunk is longer than its buffer")); + } + Ok(h) +} + +/// Decompress one Blosc2 chunk (header included), refusing to produce more +/// than `limit` bytes. +pub fn blosc2_decompress_chunk(src: &[u8], limit: usize) -> Result, FormatError> { + let h = read_header(src)?; + if h.nbytes > limit { + return Err(err("decoded size exceeds the limit")); + } + let src = &src[..h.cbytes]; + if h.special > SPECIAL_UNINIT { + return Err(err(&format!("unknown special chunk type {}", h.special))); + } + let memcpyed = h.flags & FLAG_MEMCPYED != 0; + if memcpyed && h.cbytes != h.nbytes + h.overhead { + return Err(err("stored chunk has the wrong size")); + } + if h.nbytes == 0 && h.cbytes == h.overhead && h.special == 0 { + return Ok(Vec::new()); + } + let nbytes = h.nbytes; + let mut out = vec![0u8; nbytes]; + if h.special != 0 { + // Filled per block, as c-blosc2 does: each block must hold whole + // values. + let fill: Option<&[u8]> = match h.special { + SPECIAL_ZERO | SPECIAL_UNINIT => None, + SPECIAL_NAN => Some(match h.typesize { + 4 => &[0x00, 0x00, 0xc0, 0x7f], + 8 => &[0, 0, 0, 0, 0, 0, 0xf8, 0x7f], + _ => return Err(err("NaN chunk of a type that is not f32 or f64")), + }), + _ => Some(&src[EXT_HEADER..]), + }; + if let Some(value) = fill { + let ts = value.len(); + let blocksize = h.blocksize.max(1); + if !blocksize.is_multiple_of(ts) || !(nbytes % blocksize).is_multiple_of(ts) { + return Err(err("special chunk blocks are not whole values")); + } + for v in out.chunks_exact_mut(ts) { + v.copy_from_slice(value); + } + } + return Ok(out); + } + if memcpyed { + out.copy_from_slice(&src[h.overhead..]); + return Ok(out); + } + if h.blosc2_flags & B2_USEDICT != 0 { + return Err(err("dictionary-compressed chunks are not supported")); + } + if h.blosc2_flags & B2_LAZY != 0 { + return Err(err("lazy chunks are not supported")); + } + let codec = match h.flags >> 5 { + 6 => { + return Err(err(&format!( + "user-defined codec {} is not supported", + h.udcodec + ))); + } + 2 | 5 | 7 => return Err(err(&format!("unknown codec {}", h.flags >> 5))), + _ => Codec::from_flags(h.flags)?, + }; + for &f in &h.filters { + if f > FILTER_TRUNC_PREC { + return Err(err(&format!("filter {f} is not supported"))); + } + } + let blocksize = h.blocksize; + let nblocks = nbytes.div_ceil(blocksize); + let leftover = nbytes % blocksize; + let bstarts_end = nblocks + .checked_mul(4) + .and_then(|n| n.checked_add(h.overhead)) + .ok_or_else(|| err("block table overflows"))?; + if bstarts_end > src.len() { + return Err(err("block table is truncated")); + } + let dont_split = h.flags & FLAG_DONT_SPLIT != 0; + let mut tmp = vec![0u8; blocksize]; + let mut tmp2 = vec![0u8; blocksize]; + let mut zstd = None; + for j in 0..nblocks { + let is_leftover = j == nblocks - 1 && leftover > 0; + let bsize = if is_leftover { leftover } else { blocksize }; + let start = le_i32(src, h.overhead + 4 * j)?; + if start <= 0 || start as usize >= src.len() { + return Err(err("block start out of range")); + } + let mut pos = start as usize; + let nstreams = if !dont_split && !is_leftover { + h.typesize + } else { + 1 + }; + let neblock = bsize / nstreams; + if neblock == 0 || neblock * nstreams != bsize { + return Err(err("block is not a whole number of streams")); + } + let cur = &mut tmp[..bsize]; + for s in 0..nstreams { + let csize = le_i32(src, pos).map_err(|_| err("stream runs past the chunk"))?; + pos += 4; + let dst = &mut cur[s * neblock..(s + 1) * neblock]; + if csize == 0 { + dst.fill(0); + } else if csize < 0 { + let token = *src + .get(pos) + .ok_or_else(|| err("stream runs past the chunk"))?; + pos += 1; + if token & 1 == 0 || csize < -255 { + return Err(err("unsupported stream token")); + } + dst.fill((-csize) as u8); + } else { + let csize = csize as usize; + let stream = src + .get(pos..pos + csize) + .ok_or_else(|| err("stream runs past the chunk"))?; + pos += csize; + if csize == neblock { + dst.copy_from_slice(stream); + } else if codec == Codec::BloscLz { + if blosclz_decompress(stream, dst) != neblock { + return Err(err("blosclz stream is malformed")); + } + } else { + decode_stream(codec, stream, dst, &mut zstd)?; + } + } + } + let (done, rest) = out.split_at_mut(j * blocksize); + let dest = &mut rest[..bsize]; + run_filters_backward(&h, cur, &mut tmp2[..bsize], done, dest); + } + Ok(out) +} + +/// Undo the chunk's filters on one decoded block `cur`, leaving the result +/// in `dest`. `done` is the chunk decoded so far (the delta filter's +/// reference is the first block). +fn run_filters_backward( + h: &ChunkHeader, + cur: &mut [u8], + scratch: &mut [u8], + done: &[u8], + dest: &mut [u8], +) { + let bsize = cur.len(); + let ts = h.typesize; + for i in (0..6).rev() { + match h.filters[i] { + FILTER_SHUFFLE => { + let m = h.filters_meta[i] as usize; + let bytes = if m == 0 { ts } else { m }; + unshuffle(bytes, cur, scratch); + cur.copy_from_slice(scratch); + } + FILTER_BITSHUFFLE => { + let mut n = bsize / ts; + if h.version == 2 { + if !n.is_multiple_of(8) { + continue; + } + } else { + n -= n % 8; + } + let body = n * ts; + bitunshuffle_block(&cur[..body], &mut scratch[..body], n, ts); + cur[..body].copy_from_slice(&scratch[..body]); + } + FILTER_DELTA => delta_decode(done, cur, ts), + // Truncating precision is lossy and has nothing to undo. + FILTER_NONE | FILTER_TRUNC_PREC => {} + _ => unreachable!("filters are checked before decoding"), + } + } + dest.copy_from_slice(cur); +} + +/// Byte unshuffle (`unshuffle_generic`): trailing bytes that do not make a +/// whole element are copied as they are. +fn unshuffle(bytes: usize, src: &[u8], dest: &mut [u8]) { + let n = src.len() / bytes; + for i in 0..n { + for b in 0..bytes { + dest[i * bytes + b] = src[b * n + i]; + } + } + dest[n * bytes..].copy_from_slice(&src[n * bytes..]); +} + +/// `delta_decoder`: the first block is XOR-accumulated over its own +/// elements; every other block is XORed with the (decoded) first block. +/// Elements are 1, 2, 4 or 8 bytes wide (other sizes: 8 if a multiple of +/// 8, else 1); a trailing partial element is left alone. +fn delta_decode(done: &[u8], cur: &mut [u8], typesize: usize) { + let w = match typesize { + 1 | 2 | 4 | 8 => typesize, + t if t.is_multiple_of(8) => 8, + _ => 1, + }; + let n = cur.len() / w; + if done.is_empty() { + for i in 1..n { + for b in 0..w { + cur[i * w + b] ^= cur[(i - 1) * w + b]; + } + } + } else { + // The first block is at least as long as any other. + for (c, r) in cur[..n * w].iter_mut().zip(done) { + *c ^= *r; + } + } +} + +/// What the frame header says. +struct Frame<'a> { + buf: &'a [u8], + header_len: usize, + nbytes: usize, + cbytes: usize, + typesize: usize, + chunksize: usize, + nchunks: usize, + /// The decoded offsets chunk. + offsets: Vec, +} + +fn parse_frame(buf: &[u8]) -> Result, FormatError> { + if buf.len() < FRAME_HEADER_MINLEN { + return Err(err("truncated frame header")); + } + if buf[0] & 0xf0 != 0x90 || buf[1] != 0xa8 || &buf[2..10] != b"b2frame\0" { + return Err(err("not a Blosc2 frame (bad magic)")); + } + let frame_version = buf[FRAME_FLAGS] & 0x0f; + if frame_version > FRAME_VERSION_MAX { + return Err(err(&format!( + "frame format version {frame_version} is too new" + ))); + } + if buf[FRAME_FLAGS] & FRAME_VL_BLOCKS != 0 { + return Err(err("variable-length blocks are not supported")); + } + if buf[FRAME_TYPE] & 0x0f != 0 { + return Err(err("not a contiguous frame")); + } + let header_len = i32::from_be_bytes(be(buf, FRAME_HEADER_LEN)?); + let frame_len = u64::from_be_bytes(be(buf, FRAME_LEN)?); + let nbytes = i64::from_be_bytes(be(buf, FRAME_NBYTES)?); + let cbytes = i64::from_be_bytes(be(buf, FRAME_CBYTES)?); + let typesize = i32::from_be_bytes(be(buf, FRAME_TYPESIZE)?); + let chunksize = i32::from_be_bytes(be(buf, FRAME_CHUNKSIZE)?); + if header_len < FRAME_HEADER_MINLEN as i32 || header_len as u64 > frame_len { + return Err(err("bad frame header length")); + } + if frame_len > buf.len() as u64 { + return Err(err("frame is longer than the chunk")); + } + if typesize <= 0 { + return Err(err("bad type size")); + } + if nbytes < 0 || cbytes < 0 || chunksize < 0 { + return Err(err("negative size in frame header")); + } + let header_len = header_len as usize; + let buf = &buf[..frame_len as usize]; + let cbytes = usize::try_from(cbytes).map_err(|_| err("bad compressed size"))?; + let data_end = header_len + .checked_add(cbytes) + .filter(|&e| e <= buf.len()) + .ok_or_else(|| err("chunks run past the frame"))?; + let nbytes = usize::try_from(nbytes).map_err(|_| err("bad decoded size"))?; + let chunksize = chunksize as usize; + // The offsets chunk follows the data chunks. + let off_src = &buf[data_end..]; + let expected = if nbytes == 0 { + 0 + } else if chunksize > 0 { + nbytes.div_ceil(chunksize) + } else { + // Variable-size chunks: the offsets chunk tells how many. + usize::MAX + }; + let off_limit = if expected == usize::MAX { + 1 << 24 + } else { + expected + .checked_mul(8) + .ok_or_else(|| err("too many chunks"))? + }; + let offsets = if nbytes == 0 { + Vec::new() + } else { + blosc2_decompress_chunk(off_src, off_limit)? + }; + if !offsets.len().is_multiple_of(8) || (expected != usize::MAX && offsets.len() != off_limit) { + return Err(err("offsets chunk does not match the number of chunks")); + } + Ok(Frame { + buf, + header_len, + nbytes, + cbytes, + typesize: typesize as usize, + chunksize, + nchunks: offsets.len() / 8, + offsets, + }) +} + +impl Frame<'_> { + /// Decode chunk `n`, refusing more than `limit` bytes. + fn chunk(&self, n: usize, limit: usize) -> Result, FormatError> { + if n >= self.nchunks { + return Err(err("frame has no such chunk")); + } + let raw: [u8; 8] = self.offsets[8 * n..8 * n + 8].try_into().unwrap(); + let offset = i64::from_le_bytes(raw); + if offset < 0 { + // A special chunk, recorded in the offset's top byte. + if self.chunksize == 0 { + return Err(err("special chunk in a frame without a chunk size")); + } + let size = if n == self.nchunks - 1 && !self.nbytes.is_multiple_of(self.chunksize) { + self.nbytes % self.chunksize + } else { + self.chunksize + }; + if size > limit { + return Err(err("decoded size exceeds the limit")); + } + let kind = raw[7] & 7; + return match kind { + SPECIAL_ZERO | SPECIAL_UNINIT => Ok(vec![0u8; size]), + SPECIAL_NAN => { + let nan: &[u8] = match self.typesize { + 4 => &[0x00, 0x00, 0xc0, 0x7f], + 8 => &[0, 0, 0, 0, 0, 0, 0xf8, 0x7f], + _ => return Err(err("NaN chunk of a type that is not f32 or f64")), + }; + if !size.is_multiple_of(nan.len()) { + return Err(err("NaN chunk is not whole values")); + } + Ok(nan.iter().copied().cycle().take(size).collect()) + } + _ => Err(err(&format!("unknown special chunk offset {kind}"))), + }; + } + // Every chunk lies in the data section, between the header and the + // offsets chunk. + let end = self.header_len + self.cbytes; + let start = usize::try_from(offset) + .ok() + .and_then(|o| self.header_len.checked_add(o)) + .filter(|&s| s < end && end - s >= MIN_HEADER) + .ok_or_else(|| err("chunk offset out of range"))?; + blosc2_decompress_chunk(&self.buf[start..end], limit) + } + + /// The content of metalayer `name`, if the frame has it. + fn metalayer(&self, name: &[u8]) -> Result, FormatError> { + let h = &self.buf[..self.header_len]; + // [FRAME_IDX_SIZE] is the u16 index size, then a map16. + let mut p = FRAME_IDX_SIZE + 2; + if h.get(p) != Some(&0xde) { + return Err(err("bad metalayer index")); + } + let count = u16::from_be_bytes(be(h, p + 1)?) as usize; + p += 3; + if count > 16 { + return Err(err("too many metalayers")); + } + for _ in 0..count { + let tag = *h.get(p).ok_or_else(|| err("truncated metalayer index"))?; + if tag & 0xe0 != 0xa0 { + return Err(err("bad metalayer name")); + } + let len = (tag & 0x1f) as usize; + let key = h + .get(p + 1..p + 1 + len) + .ok_or_else(|| err("truncated metalayer index"))?; + p += 1 + len; + if h.get(p) != Some(&0xd2) { + return Err(err("bad metalayer offset")); + } + let off = i32::from_be_bytes(be(h, p + 1)?); + p += 5; + if key != name { + continue; + } + if off < 0 || off as usize >= h.len() { + return Err(err("metalayer offset out of range")); + } + let off = off as usize; + if h.get(off) != Some(&0xc6) { + return Err(err("bad metalayer content")); + } + let clen = u32::from_be_bytes(be(h, off + 1)?) as usize; + return h + .get(off + 5..) + .and_then(|c| c.get(..clen)) + .map(Some) + .ok_or_else(|| err("metalayer runs past the header")); + } + Ok(None) + } +} + +/// The B2ND (or Caterva) metalayer: shape, chunk shape, block shape. +#[derive(Debug, PartialEq, Eq)] +struct NdMeta { + shape: Vec, + chunkshape: Vec, + blockshape: Vec, +} + +/// A msgpack array header: fixarray or array16. +fn array_len(m: &[u8], p: &mut usize) -> Result { + let t = *m.get(*p).ok_or_else(|| err("truncated b2nd metalayer"))?; + if t & 0xf0 == 0x90 { + *p += 1; + Ok((t & 0x0f) as usize) + } else if t == 0xdc { + let n = u16::from_be_bytes(be(m, *p + 1)?) as usize; + *p += 3; + Ok(n) + } else { + Err(err("bad b2nd metalayer")) + } +} + +fn parse_nd(m: &[u8]) -> Result { + let bad = || err("bad b2nd metalayer"); + let mut p = 0; + if array_len(m, &mut p)? < 5 { + return Err(bad()); + } + // Version, then the rank: positive fixints. + let version = *m.get(p).ok_or_else(bad)?; + let ndim = *m.get(p + 1).ok_or_else(bad)? as usize; + if version > 0x7f || ndim == 0 || ndim > B2ND_MAX_DIM { + return Err(err(&format!("unsupported b2nd rank {ndim}"))); + } + p += 2; + let mut read = |wide: bool| -> Result, FormatError> { + if array_len(m, &mut p)? != ndim { + return Err(bad()); + } + (0..ndim) + .map(|_| { + let v = if wide { + if m.get(p) != Some(&0xd3) { + return Err(bad()); + } + let v = i64::from_be_bytes(be(m, p + 1)?); + p += 9; + v + } else { + if m.get(p) != Some(&0xd2) { + return Err(bad()); + } + let v = i32::from_be_bytes(be(m, p + 1)?) as i64; + p += 5; + v + }; + usize::try_from(v).map_err(|_| bad()) + }) + .collect() + }; + let shape = read(true)?; + let chunkshape = read(false)?; + let blockshape = read(false)?; + Ok(NdMeta { + shape, + chunkshape, + blockshape, + }) +} + +/// Decode a frame as the HDF5 filter does: a B2ND array into C order, or +/// else the frame's first chunk. `cd_shape` is the chunk shape recorded in +/// the filter's `cd_values` (if any), which the array's shape must match. +fn decode_frame( + input: &[u8], + limit: usize, + cd_shape: Option<&[usize]>, +) -> Result, FormatError> { + let frame = parse_frame(input)?; + let meta = match frame.metalayer(b"b2nd")? { + Some(m) => Some(m), + None => frame.metalayer(b"caterva")?, + }; + let Some(meta) = meta else { + return frame.chunk(0, limit); + }; + let nd = parse_nd(meta)?; + if let Some(cd) = cd_shape + && cd != nd.shape.as_slice() + { + return Err(err(&format!( + "array shape {:?} is not the chunk shape {cd:?}", + nd.shape + ))); + } + reassemble(&frame, &nd, limit) +} + +/// Gather a B2ND array's blocks into one C-order buffer. +fn reassemble(frame: &Frame<'_>, nd: &NdMeta, limit: usize) -> Result, FormatError> { + let ts = frame.typesize; + let ndim = nd.shape.len(); + let too_big = || err("array exceeds the chunk size"); + let mut total = ts; + let mut ext = vec![0usize; ndim]; + let mut grid = vec![0usize; ndim]; + let mut ext_bytes = ts; + for i in 0..ndim { + let (s, c, b) = (nd.shape[i], nd.chunkshape[i], nd.blockshape[i]); + if c == 0 || b == 0 || b > c { + return Err(err("bad b2nd chunk or block shape")); + } + total = total.checked_mul(s).ok_or_else(too_big)?; + ext[i] = c.div_ceil(b) * b; + ext_bytes = ext_bytes.checked_mul(ext[i]).ok_or_else(too_big)?; + grid[i] = s.div_ceil(c); + } + if total > limit { + return Err(too_big()); + } + let mut out = vec![0u8; total]; + if total == 0 { + return Ok(out); + } + // Padding to whole blocks grows a chunk by less than 2x per dimension; + // hdf5-blosc2's block shapes keep it far below 16x (a crafted frame + // could otherwise make each chunk enormous). + if ext_bytes > limit.saturating_mul(16) { + return Err(too_big()); + } + let nchunks: usize = grid.iter().product(); + if nchunks != frame.nchunks { + return Err(err("chunk count does not match the b2nd shape")); + } + let blocks_in_chunk: Vec = (0..ndim).map(|i| ext[i] / nd.blockshape[i]).collect(); + let nblocks: usize = blocks_in_chunk.iter().product(); + let block_items: usize = nd.blockshape.iter().product(); + let block_bytes = block_items * ts; + // Strides (in elements) of the output array and of a block. + let mut out_stride = vec![1usize; ndim]; + let mut blk_stride = vec![1usize; ndim]; + for i in (0..ndim.saturating_sub(1)).rev() { + out_stride[i] = out_stride[i + 1] * nd.shape[i + 1]; + blk_stride[i] = blk_stride[i + 1] * nd.blockshape[i + 1]; + } + let mut cidx = vec![0usize; ndim]; + let mut bidx = vec![0usize; ndim]; + let mut gstart = vec![0usize; ndim]; + let mut valid = vec![0usize; ndim]; + let mut pos = vec![0usize; ndim]; + for n in 0..nchunks { + unravel(n, &grid, &mut cidx); + let data = frame.chunk(n, ext_bytes)?; + if data.len() != ext_bytes { + return Err(err("chunk size does not match the b2nd shape")); + } + for b in 0..nblocks { + unravel(b, &blocks_in_chunk, &mut bidx); + let mut empty = false; + for i in 0..ndim { + let in_chunk = bidx[i] * nd.blockshape[i]; + gstart[i] = cidx[i] * nd.chunkshape[i] + in_chunk; + let lim_chunk = nd.chunkshape[i].saturating_sub(in_chunk); + let lim_shape = nd.shape[i].saturating_sub(gstart[i]); + valid[i] = nd.blockshape[i].min(lim_chunk).min(lim_shape); + empty |= valid[i] == 0; + } + if empty { + continue; + } + let block = &data[b * block_bytes..(b + 1) * block_bytes]; + // Copy row by row along the last dimension. + let row = valid[ndim - 1] * ts; + let rows: usize = valid[..ndim - 1].iter().product(); + for r in 0..rows { + unravel(r, &valid[..ndim - 1], &mut pos[..ndim - 1]); + let mut src = 0; + let mut dst = gstart[ndim - 1]; + for i in 0..ndim - 1 { + src += pos[i] * blk_stride[i]; + dst += (gstart[i] + pos[i]) * out_stride[i]; + } + out[dst * ts..dst * ts + row].copy_from_slice(&block[src * ts..src * ts + row]); + } + } + } + Ok(out) +} + +/// C-order multi-index of `n` in a grid of `dims`. +fn unravel(mut n: usize, dims: &[usize], idx: &mut [usize]) { + for i in (0..dims.len()).rev() { + idx[i] = n % dims[i]; + n /= dims[i]; + } +} + +/// Decompress one HDF5 chunk written by the Blosc2 filter (a Blosc2 frame) +/// into its data, in C order; at most `limit` bytes. +pub fn blosc2_decompress(input: &[u8], limit: usize) -> Result, FormatError> { + decode_frame(input, limit, None) +} + +/// The filter's decoder. +pub(crate) fn blosc2_decode(input: &[u8], ctx: &FilterContext<'_>) -> Result, FormatError> { + let cd = ctx.client_data(); + // cd_values[7] is the chunk rank for B2ND (> 1), and the chunk + // dimensions follow. + let cd_shape: Option> = if cd.len() >= 8 { + let rank = cd[7] as usize; + if rank < 2 || cd.len() < 8 + rank { + return Err(err("bad chunk rank in the filter parameters")); + } + Some(cd[8..8 + rank].iter().map(|&d| d as usize).collect()) + } else { + None + }; + let out = decode_frame(input, ctx.output_limit(), cd_shape.as_deref())?; + if out.is_empty() && ctx.max_output != 0 { + return Err(err("empty frame for a non-empty chunk")); + } + Ok(out) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::filter_pipeline::{FILTER_BLOSC2, FilterDescription}; + use std::path::PathBuf; + + fn fixtures() -> PathBuf { + PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("tests/fixtures/blosc2") + } + + /// Every fixture frame (see `tests/fixtures/blosc2/generate.py`), + /// sorted by name: (name, frame, expected output or error word). + #[allow(clippy::type_complexity)] + fn cases() -> Vec<(String, Vec, Result, String>)> { + let mut v = Vec::new(); + for e in std::fs::read_dir(fixtures()).unwrap() { + let p = e.unwrap().path(); + if p.extension().is_none_or(|x| x != "b2f") { + continue; + } + let name = p.file_stem().unwrap().to_string_lossy().into_owned(); + let frame = std::fs::read(&p).unwrap(); + let want = match std::fs::read(p.with_extension("out")) { + Ok(out) => Ok(out), + Err(_) => Err(std::fs::read_to_string(p.with_extension("err")).unwrap()), + }; + v.push((name, frame, want)); + } + v.sort_by(|a, b| a.0.cmp(&b.0)); + assert!(v.len() >= 20, "fixtures missing"); + v + } + + #[test] + fn fixture_frames_decode_exactly() { + for (name, frame, want) in cases() { + let got = blosc2_decompress(&frame, 1 << 20); + match want { + Ok(want) => { + let got = got.unwrap_or_else(|e| panic!("{name}: {e}")); + assert!(got == want, "{name}: decoded data differs"); + } + Err(word) => { + let e = got.expect_err(&name).to_string(); + assert!(e.contains(word.trim()), "{name}: {e}"); + } + } + } + } + + /// The output limit is enforced for every kind of frame, including the + /// padded chunks of B2ND arrays and special chunks. + #[test] + fn output_limit_is_enforced() { + for (name, frame, want) in cases() { + let Ok(want) = want else { continue }; + assert!( + blosc2_decompress(&frame, want.len() - 1).is_err(), + "{name}: decoded past the limit" + ); + assert_eq!( + blosc2_decompress(&frame, want.len()).unwrap(), + want, + "{name}" + ); + } + } + + fn desc(cd: Vec) -> FilterDescription { + FilterDescription { + filter_id: FILTER_BLOSC2, + name: None, + flags: 0, + client_data: cd, + } + } + + /// As the HDF5 filter: the B2ND array must have the chunk shape the + /// filter parameters record. + #[test] + fn b2nd_shape_must_match_the_filter_parameters() { + let frame = std::fs::read(fixtures().join("b2nd_2d.b2f")).unwrap(); + let want = std::fs::read(fixtures().join("b2nd_2d.out")).unwrap(); + let decode = |cd: Vec| { + let f = desc(cd); + blosc2_decode( + &frame, + &FilterContext { + filter: &f, + element_size: 4, + max_output: want.len(), + }, + ) + }; + assert_eq!(decode(vec![1, 0, 4, 0, 5, 1, 1, 2, 37, 29]).unwrap(), want); + assert_eq!(decode(vec![1, 0, 4, 0, 5, 1, 1]).unwrap(), want); + assert!(decode(vec![1, 0, 4, 0, 5, 1, 1, 2, 29, 37]).is_err()); + assert!(decode(vec![1, 0, 4, 0, 5, 1, 1, 3, 37, 29, 1]).is_err()); + assert!(decode(vec![1, 0, 4, 0, 5, 1, 1, 2, 37]).is_err()); + } + + /// A 32-byte extended header. + fn ext_header(flags: u8, ts: u8, nbytes: i32, blocksize: i32, cbytes: i32) -> Vec { + let mut h = vec![5u8, 1, flags | FLAG_SHUFFLE | FLAG_BITSHUFFLE, ts]; + for v in [nbytes, blocksize, cbytes] { + h.extend_from_slice(&v.to_le_bytes()); + } + h.resize(EXT_HEADER, 0); + h + } + + #[test] + fn special_chunks() { + // Zeros, uninitialised (read as zeros), NaN, one repeated value. + let mut z = ext_header(0, 4, 64, 64, 32); + z[31] = SPECIAL_ZERO << 4; + assert_eq!(blosc2_decompress_chunk(&z, 64).unwrap(), vec![0; 64]); + z[31] = SPECIAL_UNINIT << 4; + assert_eq!(blosc2_decompress_chunk(&z, 64).unwrap(), vec![0; 64]); + z[31] = SPECIAL_NAN << 4; + let nan = blosc2_decompress_chunk(&z, 64).unwrap(); + assert!( + nan.chunks(4) + .all(|c| f32::from_le_bytes(c.try_into().unwrap()).is_nan()) + ); + z[3] = 2; + assert!(blosc2_decompress_chunk(&z, 64).is_err(), "NaN of i16"); + let mut v = ext_header(0, 2, 12, 12, 35); + v[31] = SPECIAL_VALUE << 4; + v.extend_from_slice(&[1, 2, 3]); + assert_eq!( + blosc2_decompress_chunk(&v, 12).unwrap(), + [1, 2, 3].repeat(4) + ); + // The value must divide the chunk (and its blocks). + v[4] = 13; + assert!(blosc2_decompress_chunk(&v, 13).is_err()); + v[4] = 12; + v[8] = 4; + assert!(blosc2_decompress_chunk(&v, 12).is_err()); + // Special types 5 to 7 are reserved. + let mut r = ext_header(0, 4, 64, 64, 32); + r[31] = 5 << 4; + assert!(blosc2_decompress_chunk(&r, 64).is_err()); + } + + /// A regular chunk: one block of two streams (split, type size 2): a + /// run of 0x07 and a stream of zeros, then the byte shuffle undone. + #[test] + fn stream_runs_and_zero_streams() { + let mut c = ext_header(0, 2, 8, 8, 0); + c[16 + 5] = FILTER_SHUFFLE; + c.extend_from_slice(&36i32.to_le_bytes()); + c.extend_from_slice(&(-7i32).to_le_bytes()); + c.push(1); + c.extend_from_slice(&0i32.to_le_bytes()); + let n = c.len() as i32; + c[12..16].copy_from_slice(&n.to_le_bytes()); + assert_eq!( + blosc2_decompress_chunk(&c, 8).unwrap(), + [7, 0, 7, 0, 7, 0, 7, 0] + ); + // A token without the run bit is reserved. + let mut bad = c.clone(); + bad[40] = 2; + assert!(blosc2_decompress_chunk(&bad, 8).is_err()); + // Unsupported header features are errors, not data. + for (at, bit) in [(31, B2_USEDICT), (31, B2_LAZY), (30, VL_BLOCKS)] { + let mut x = c.clone(); + x[at] |= bit; + assert!( + blosc2_decompress_chunk(&x, 8).is_err(), + "byte {at} bit {bit}" + ); + } + let mut udf = c.clone(); + udf[16] = 35; + assert!(blosc2_decompress_chunk(&udf, 8).is_err()); + let mut udc = c.clone(); + udc[2] |= 6 << 5; + assert!(blosc2_decompress_chunk(&udc, 8).is_err()); + assert!(blosc2_decompress_chunk(&c, 7).is_err(), "limit"); + } + + /// Random and mutated frames: errors are fine, panics are not, and no + /// output past the limit. + #[test] + fn fuzzed_frames_never_panic() { + let limit = 20_000; + let seeds: Vec> = cases() + .into_iter() + .filter(|(_, f, w)| f.len() < 9000 && w.as_ref().is_ok_and(|w| w.len() <= limit)) + .map(|(_, f, _)| f) + .collect(); + assert!(seeds.len() >= 15); + crate::test_fuzz::fuzz_decoder(0xb2, &seeds, 20_000, limit, |s| { + blosc2_decompress(s, limit) + }); + } + + /// Blosc2 chunks on their own, mutated. + #[test] + fn fuzzed_chunks_never_panic() { + let mut seeds = Vec::new(); + for (_, f, w) in cases() { + if w.is_err() { + continue; + } + let frame = parse_frame(&f).unwrap(); + let raw: [u8; 8] = frame.offsets[..8].try_into().unwrap(); + let off = i64::from_le_bytes(raw); + if off >= 0 { + let start = frame.header_len + off as usize; + let end = frame.header_len + frame.cbytes; + let h = read_header(&f[start..end]).unwrap(); + seeds.push(f[start..start + h.cbytes].to_vec()); + } + } + assert!(seeds.len() >= 10); + crate::test_fuzz::fuzz_decoder(0xb3, &seeds, 20_000, 1 << 16, |s| { + blosc2_decompress_chunk(s, 1 << 16) + }); + } +} diff --git a/crates/clawhdf5-format/src/lib.rs b/crates/clawhdf5-format/src/lib.rs index 3e30f9d..89d324b 100644 --- a/crates/clawhdf5-format/src/lib.rs +++ b/crates/clawhdf5-format/src/lib.rs @@ -86,6 +86,8 @@ pub mod filters; mod filters_bitshuffle; #[cfg(feature = "blosc")] pub mod filters_blosc; +#[cfg(feature = "blosc2")] +pub mod filters_blosc2; #[cfg(feature = "bzip2")] mod filters_bzip2; #[cfg(feature = "lzf")] diff --git a/crates/clawhdf5-format/tests/fixtures/blosc2/always_split.b2f b/crates/clawhdf5-format/tests/fixtures/blosc2/always_split.b2f new file mode 100644 index 0000000000000000000000000000000000000000..db6322af43cc9a0dd615133bf4c3e1376a9f98fb GIT binary patch literal 1650 zcmZ8hYgkQb6n?*bI;T?(ozAHws#A(oq>FM1yF?VCLdK;+kwhiQFlI9mCheqVav6?$ zO-(Y4X+k!d+)0c}OnQtv(;Vb7G}El`V;<{yzV+;HueI0vzU%w;+EO&lD?T+gDGqf2 zVlR>-Q>5x%b@jQfyJe*JkEHvqQ)IdeAB zn*nXBM*cD3f0<4nU)^JXJs7lVfq_%P0&PnLaM&k6lE8!KC8;wE^nt(&v=t->;3Xxt zNjwCJq7qyYAV>ml3RpwO^Kz)AFqo}_C6^Ai6k4jP74&UY&82A24*784c@xA6R37{x znDa|1=1pHqZctNhAR!4hh^Dd<5wzk~z!Iy#c{8JYN(cc~x0T31Z;2OD2vE%&1xdlU za;ha~grO4zC+5K89GERUc$6|L=D>_|AS+>3sA9;3gCw;8DrPRU!&uS>>|`3jpJ6jF z3ao%i-DIHtiSU3>%q-;!$SfE$f~=a!f;lW>o;;5Xrjce4sUpVhW)2VvN$n}76Zle4 zfMYtQ7Obhydzi*3k11R^E8!}~anSHtj0xaQapqEA4oarsi-BCyMfAH0{3gaFaaySh zvzJPkg)0O#Q{=&@FEf{{8C!xObOL-W=Se40#;JK*#-cdp!*S^{wUkEd07;_rA%?;- z8DJ9ig>Box@}i+Y59SJl~u=n zJ$v~_m%?l(9%EOOF}*{KVKud>#xU6)_D zY3ugg-~Uj)zoN40xQOQ_m#@_~+`0SciQ&bozyASeq%byVr|O{A*fB>toe0=5Pj4ST zA{0Q7@{rI`VPnH1$48O)Xc8YkDd)xu=YDK2d3z`TDY|bM^O|WlD1^ zTPHWK0eo=S_$hITsTp5pX06FtSFmBz)*X9@#ets>SDg?cUAcC%;m*BBPyc%H`ke@^ zoU~h5cI?zyL`ruTH}_t>z3iBi{~!?sqr=8V5d3G8Md*^|k$_mR@3Of?lN8{hZ`~E#?&zZr1PEQ&P6?r;0;IkJp^OP{K{0yF*obixNdDBA*y(WuoR~B(g#w)xH74sHD&2=q7P2=MRaDnCH2tek zX*=t>d-(bXg+@fh#wVpOUbbQt2{YUH-L{=Yr37DjSfU@{LLI?35`5G1<~J>^XE<-A zsY-;#Mnp`v?ye+%qRfV_USv2yPZXhVyYKVfUp#jjq6y#w0s;aV?1wNo+6OUsAqVyW zIp}RB;-r0`h}Kr!n4mV3Gh-#q%C>n47h3O)CibV6&uH>1K1}`#6zqEa(+vCv_s$vv literal 0 HcmV?d00001 diff --git a/crates/clawhdf5-format/tests/fixtures/blosc2/always_split.out b/crates/clawhdf5-format/tests/fixtures/blosc2/always_split.out new file mode 100644 index 0000000000000000000000000000000000000000..a3faac59e55caf561a9670a4b6b8ef352bd0187b GIT binary patch literal 4000 zcmZwK3(!?n8He$OC?TPQ7x0Dw333Yr2{cd_@ChiC6eyUepn`~4C?Hf*t;`IKipaa&pIi^zxJHc zya_37LOZ%oOlc6>un1RS4Q|3lY{7ka80~l#yRqM-ln%t9I0F4K1S2pOC!-D1F%#$D z0$hTnSb=xqDqJ%um20sEYw;0$+H68-Hf|&kNMl!hWpJw#831) z@UZ?d_19`Uevh5zKdDcv&!~S_U%+mCYD={fI=A^hwx#?sb$_)Ry4xRs9&M@g#KEo| zir)Ie)FW_|zK?rfsrFM}Wj8<_XddL*!RBH5SF0oJN7{{2N863jkF|fTI?nDS{VDd@ zJki{yo{A|r&9!OjboC6^W~gseXSx4O``PN*=5y@NHJ_)>Ghd)yWL}`ZSbvH87VDSb z%~)!8h21j!3in;9UuBng)0Zzx8+Uf9hwxw+rXt+A8rNez?!Z?301x2_>_YO6yP^kr z;VAUSV2nh#a|))Q<<4CFg;chdLpef@*#c9lDcJLOUJag{rTJLO3;cM5mX-&F3Tf2n)9Q#!a)xl_244)i{` zQ@NA5letrSfjgNynLFudGk0n~Gj}p~%0QJnd5HZ`_YBu_r*J17kJsqAQ%W zPQ4u5Dcs2`^&Q<=&6zFO)wna753{fY*I+Gf!6w{|2k;1XB66qYPUKE+^uYi`?i`O3 z(Q@Yu{VYW8T!e7v3amuId+!)s7CJWQp26+@lX6`QmV=O+Xwrii}xmX^8R?ao;#U4 z`4u?GyKUTw+zEGT<4)mD>TB-j9_|$G)PZW`4tMe}Rqhn-)W)4EchXqbxYNp=DtC(9 z$rJ24xRbe)Pxox&PUKE{gPA*pJNYck(aW9EdAR;KjIx(IS?*N1Q@K+kcgDMRvOd1E zCg>Y?>SR0aq{f}}de<9w(o8*fDt8KZ(wi{X>^m#miQFmNDRQUfPUTMFPLVt5E$-RN zowUleaA!Stj^oUFY)4lfggfIg3rn#YH{v$jg>Yv(!kwq^JofRfyWt@8#?csn;TVna zn26I6?##ydScsN8Z_~@2B6q@_^nQE@*MmDn?vzjI*Q4dm?fN_Hzl^UTa_5_N-%{mH zxet*$KUU*Ai#zEN{0hG@NA7SZJ*9>_awk0p?v(h>dQsoWyW&ojJGqN@)>Yr+PCmfg z13m4ylNxtwPUcP;?HTSw?xYj#8+S5y z9CEx%*SS+pRVRZxkvn<1Yuu^asdA@qC!eV{?qu#%xfAXb?xgv+P|uybP+eqR44L!) z+{tn$+^H>hB6nKubmze!WX#2KtUh>rcAy zFbDH+F_vH%-tMz{x7y@RT4Qe9DIe2E?r^7Yr*NmpotMtZU-8^#^EcFcRPOA_9qzz}#m~);s*OA8cjhO}+(~~?pHaC}TDkMzNhx>ouDDYgcXH!SB!81Txp618>A91p zsHdsZT<1>aPMV?TPHx;uv;8f(6YfOrr1Kzi>U{TcCoRw~1a~rbs@$n^r^uarx$DcY z!ralFcDb{(aVO5nkvqdM72!@nxU&u$umvr59@mFEFQSWg-W}o2k?4zKF#_Svc(mM^ zsSkJNBXZ|5T#3e=@?M|aYIEaGkvn_bS!efY{mr;lzX2O@rpeq*I)?ZcfH^t-{G+R6Lt?A`4LXPwHO)D7LeOYX!uIm?|m zCs*#|#+^7PSGkirxKkT<;x|^6J86i&A$Q_8R^?6_>3+FWxRbe)PS8j0a3_9a)yADX z(afF5ozl3I8+QtK3U}h1+{&GBC!g(k?v%!z+_+P@lN)#9H&(dQ`i)h&6X)d2o%oHF KR(gI_+y4N4t(Zyx literal 0 HcmV?d00001 diff --git a/crates/clawhdf5-format/tests/fixtures/blosc2/b2nd_2d.b2f b/crates/clawhdf5-format/tests/fixtures/blosc2/b2nd_2d.b2f new file mode 100644 index 0000000000000000000000000000000000000000..a092e297936a93a2a0d0ef8263ff8a8e5adc56ab GIT binary patch literal 3950 zcma)9dtA-g7GJ+}oQiWery`}7N`;b2RIc(mxz&WoBV{;6nJ9xO-Dr9sbQ3)=k}yRj z6iLER9+L<~gmimgG8Z-8Gn#SN{{7B5bM_g3-2M5i-`?kU*4lgRwbpm7_1jkv>=?Q# zC?W(J03iPsK5&Sa=jp&~-W~3jA;=f~rVyrHOi{% zn7m-e<%~jvUPj3c5igP#8dR(8txddkmr29zD_XoOqU4JCR-VU_s?>h@8wqJ zOlGN6?kRxpW!N5I#cHCtUs53+%#Up|YtIf84y$k+sAOCNU_S=*F;4luJV*+b(7M*t z-F@7_D=a;8@u#||^@&^3caQg+I(z-RHs z@p^uiXKSY9x&p9PsB19T%I<#46JBp^a=ZjGCfRieX}7m`c76D>S8>_ds+wDJw{LeG zh%u<@Oa2$o9F1oYU+zMm-9{?Hbt{qyuF-Q4jznA7d-~jkq2anl=EEIaT7G`@<`ciD z^vs2;bqWg`oG#W=i{rsoBSwv#^vmmyp9P|_^OtIGKlpA_Y8J+x0>zQGeZ`J=Di<+R z8Tc3#yCG%AZvO>K!q==DV&mXE!QG_FSHoC&Jdq0nH%uH)1R@=M)0$iNA9uXS_ zzhwGZ)AS0;X&(D4NjdMCq{M%>2OcW+WSNCiFrXwRz$D2v>D|(!SmM-pEZO! zvJwl>G$Q?^XK#-<*fXNUW5;|S) z_vIfctE?slU=2(P9xW0Yu{t(si=*pAtV=z;MA`;sH8&d{cl=i_B4wkF_G1{{Yg3BP z%s|;U-|CqTv2}87@BH<{9|DCRrStDetMy$_W6t;3R7iR0yAWUyxHxPTfR&w-8^CJ} za#d7dI)wJ%Jbx$tR65wwv-cMsKOGg9v^8VTBro5&3l?J%%;);ojov-zkKuifr8TV3s)&NDhStt zrbG{*?(V}jwlt&OI(d>Ous}5cuAwv8p%wrN5I5qVLga}s}s<^E3QceBZ zxJ{`UdnbF%@Lw3(SI@}8#*vL}Qk%a7=6jEyzI^-XKp|H3%Mq*BC2dY~cK4X(7oa9Y zUELGl`p!ipMId5YHCH4VxAlM4JX3$D4vhtDaWwkrp1)k%aK9xvEi32X5kCaR@+h`_ zQ?z{|bG4u}vKVIX?EaM0quwPI7k|3ZxIQUm$FBX#I3_iQ&H*dH@2?zShQ{JzkK_;& z%i#`V$9HOw4uS)1mLnEj%153aAZb2C_Gc5t7Llf*#Ous5My-B(PMtM>@v`Aq!lw9e zTe0a#)se~Q6n3aNawfh{!arUkRUwh~nxH!tiL~J~ECUxUT@f8;W$T0uWA}$Y1!zN3 zYFp(NkedAyivdb0hg$R>!=@bfPT0H+>w%BoJp4Y;Gr`6PhrCHwbcuqMbrW<);69we zEEy=^OrDwjLs7|TSx5vndcbEiuxiO5x+{d&{5a8`7TBDYm7Q0#bY)Cj@-`>z*L>&w zO~6KZ!T<(pn%kM)02i**Aw;+A*qw{YaJpefG%g_Rfesbc-5dX%i8;X6pVSK@i{bWT z+<$)gwgDUv6U%8{spJFnL_wLa_0 z%E>=+q7t)RRNRIUqg*F?`I3WG+Ez-a``st(Z49ce*4=G>l9Imrz~SR_z79d-kWUp> zTSVat9_2k6TZ7?wdHk2KMKvV_k^$HekHX&K)rX!#i2Ukn_+5hr9BbYjsZX<#ua((f z^ZRJ)n_6SF?|AV>&Z^VpA~MEnbI#Al#&{9feI%cVe9f8dDU*F%Lledbn1Naokcp5C zi5&|2Dt_av_`DP*BFf5@O}TPzlpN(6cQiMNu(gj@GoKl=e>~>*`uzU-&gcD{*StTeQ&LjWlZhPUq9Da6Nja+UIQ3~l z3!bG5J?O_^MlgoSOk)lUS;-nU@(G`_mm{3wJU?)qo1{&Zl5!_mxR?7WLUA6Y64j|g zLz>ctjyz8<1~8P-yulP^GM^=Uz*;u5lP@{QG0t+4EBr#L)ZP)9$WCqwQjAiRqbfD2 zM`NC*Ezi-NJ`ChF#xRMunZrU>u$uL3<8$_Mh!dRWGS|6D>NMUFnaN3B3R9fYRHPbp zs83T`(}Aw^(4Cb+zm8@YCJK4?Ge8Xw{jFb%AMYNNfLPR^Is6cgUQIBY+ zB^`L4Ui4)!(au;VF@w1*BHH;L5kxx^na*q$k+ieH{3%I0N6e>4+PPu=o!gvUw3Cf!Cm%^WWz3bR zLA2A5=CmQ&d4XQ^XBeZowVk=1m$Hg=Y$4j&!$FR720!D^cCz?;5BVs}LzE_Ir;a)H zPE%Ubk!a^7V($zg+8Iaeo$0*GQr;)_&SrM9hXWkrG||o#ekR(9y%X(ZBNx$5QA$#d z*gG|;OJkbTmT2ci`Vj5B#_LSvEoQTjWvpTY+eq3uY(CC+e9u*)oitujv~w4KCE6)O zv{RaBrz+7-1Cn++m}BomJNWj&!3p(ax*9&UmIWlh`}U_<*%UJG+Q>4v@5y)>&mFE749~ zicp*~RHQo5PD7f}icUm3FEfClj3(N7lbOtC8LL>w7Ls-jn!n*R7rDx>q&Tbe+(iyz z?-ZmM(N0;aa%(#+JU>epx^rtgV>~DA%rZwiD~P?bo==E&_Hl%hoG13qPuwKhxr5B) zBripHkZ7kO)u>H?QWjaiX2? ziM@(9sRb!yXqrnDm3=}Hg!@d_gt$7E(Ohb6?`iFP)#gD=?6 zF-{SC=SObvU(!0SJIO}0lb^rwFlC8$YVrh)Xii%?@dCXWNVM}h<9U-=%;!B~@2qDF zpYat3IZo`IOI#&s=XPfo_vC0N4}~a7Nh(lqCC)#O7w9|oTrx*Ph!brxFv@_2f z?YvKXV{KtOyZM@E=L{G4foSJLIl*^CJJ+~LD(9AgTiYq@`5{VEo@&&l0Zn*@4m?K>`Z1W1jAJs< z&RiC=g80VT%nrU_KgT$Q*Yf8*IkUfale81xSfwdP6{4N`JjFA#BiiXf+>@i75sW2i z=Uww+R`3y<_>|r3<0vP&KL z*FL|j-_%exYst)oO*jZ(`+Iyaowj9T4_p!b(D_xYlP~>mkwP(fjNl`xJ^(&IWapu5 zyI`b)E#&dQyXbxdk&O*y*EdS>`G*nNLpgR0vK~m*hEDEFXZNM!1E~f&$Xk~EJ4;@p z_5+{E*9&Hi>JWJ_rlRbJR7z`&w`NVc~Lp5rU;Gf7a9VknS z(yoK2_U+m0v;ph4L^`^yh#bni;+IQ*iI4aZ*Z`<0{7_oMM}iQUh4PoSHjd9;80QbC z=o`RPK!1BY8k7FUp!3wxrWslFkT#OvCVSxlP@qpFoN;-`i~*`Jy` znnE7f$;YE+eiZHqm*V8i6e5phKcL)9tRTK8&gHi-Ln%uss$x^y%102%?W*&}H&&5H zVu(u0P#qdoQXV$ZyWRM!nWn9_c_c;-)3Fu!=!gpRLovP~|7zOdo=S4B?Z3sP1$_pe zzV9#jMJD$`+p(OMrp*LKmH`({XrC0EqZPmR@I*4h^lT+Q78BHk{$%SHOp`a%~@$Hci{NYVMxwR$D#!2$U|w5Xa+g-208JMy4_HdH1fYVG_!T z4C;u9sHLH|ys&F;?d0j#KKb;kZ)DQZY(Nww-Hu0aJ}$;0Qo;>fk^Xk1v?ACwyo7g1 zqIx`sx7yXQG&P9NzX0aU0}g(iIdECjrkBVsi3P+bgh`Y^{B6$iT9=Qitm49vNFavz z{FZyhy_Ic6U1ZLk< zI}h46x(rvdkt|tE8j8`y(}b)Av#A`m7EB@~?39L5LMNe>tOY`+j|ibwPwJgb{KR*^ z+r-OGylULFV*L{rR$emhrrTDp$w&|JmsT9D#w)KLI9PXO!*!(A+nIFSRz$ACMp9)0 zsr6_^da$UID35*%T)!AN@pldyNIJiF?EDwmr@QU{Djgp zqs4rYzvD)s8kgZttoJy8*>>{lik4+-f-}b_n1 zCnzf@1E#X5ptWYRdhrb-^xwYz&oA>AF29*)fGRf*13FZJrId&TcYXRzr_Q~47mV7v zqnb$=Dl$yRvP6R>;k-J4$m!O84ALdG&T4#zToiEMzdyAHkUe8qJKEL&S8-qnAwgie z#jm58&jH!brIi2R+aexD65R4TryCV;qdtP4I2RcE^a0dz$g~@%7o#5f?D+R$!x(f+ z@9Jqe`y(YBwP!_9DwdtsBM|=mQpFyl+t|0F{<0F(U)TDtrt++95kIM{9Q4c(x+GQH z2z)bi({W1pBwP|s37>>xvIdlqLt7|iYw;t|D}+-@Q;92-;$zuz6dY@d#uhJ8Ey4_E zgHcuG_kqQ-nt!5m6L--(DB(uzr4QT_W>u)KE$uk&$*sugn%B4Ag66|V-B!}_kZnlX z!LtXCTX+aRq*_VGDR#|H<>vuIY0v&7R(prYqzYN&aIu8avg3ruFHpx2b;g@p>+I*k>b9sSX+LjAda{-cTx(tSij?Fx zvl?ocInDy!hOMsXi%aoyNiKHzYdGiu%aP{t-CQ2Tt<0k(%xpt-uzK#V%=?b z3@;sh{%AFCNv3u*%PK;W+~ze9dk8bA{C(5Rx05Ep$hH|Xfj4(%4qg_ui6-)KPJ@@A z4i93hXFP+M=lJS#nXxz&b;hu>&o!s=E``~_DdrP+?n zEUopE&%Cx{{Di4jJ-qJ4ZSA#VNn=c(60-?j0B2+tvzfGw&DhEP>D3JR?HT(f#g%fj zKI93taTf44xA^8#5@(H`Sz!2x(HA5g&3#1ZwV^7*uN=m>7YqYawYVn;HD>JPSDYkF z=&@9*88bqhZz@Y!+BrbU@%L+RMXM$I(^S7$RcP|q zg0||^qo&5SmERUqaKN;MN0nwQG|cE<40z~-ilMZr;gMTLF34JFn(2VsF@sU}@mno1 zhuBPPB_xC!iK&EyZ#{80v5!cX{6n}AN6aCJn!m^mP&}EXJT682BR$O+spOlZNf#e- zTk~f^oO_(;MiR78%H*_`aGfuX_j_~5@VaYdc}aQG-{*47;*#a*Gph+CzuJA+XsQjS zf)h8NJ9`f3=DQzzoU2Do4V()}d(Yu_u6}vDoI+E_N58pVH_T<4sw5;n7O@ZF_*l|B z;@rOze@Pp*bYKB}Z2+TkjZD;kVT!Xwz0KVW)se9>$A~N!? z6O|adE;8lm-1==r%ik`$rg`$M&mkJi&gq=nHLu61`2~fsK7IQY;jc3gW$D!s{?9y{ QtKVVn68U@3AwIn6|9EUxb^rhX literal 0 HcmV?d00001 diff --git a/crates/clawhdf5-format/tests/fixtures/blosc2/b2nd_3d.out b/crates/clawhdf5-format/tests/fixtures/blosc2/b2nd_3d.out new file mode 100644 index 0000000000000000000000000000000000000000..c29f5f0ad5461edad2d9de4271b870192dbecd8c GIT binary patch literal 2520 zcmZwJY0#Eq7{>84Ygx0Ugpj40C<$4+>V1psQKS;Bk|HY07p}2n3o-U3*%FgvEMqLW z(HN2tW5_aNYqHE}LYBe!UCsT>ydO04pW{5Q>wex3_w%3QDPsM(w#4@6h|cJO9_WL_G$ zR^BLYmbb|}kDZ+DY~RIw zC%L<`&gMPMd%Npm?=AAoH;} z9w+)+C&`oL5P7O+PQ&S*8;+3}<@_x7W6WbQ-u>BfqMRfryFU+8?H6DgW;nmd*`>&R zxl+5cNUlbE9`rA*agL%uIP#4PQm~jkD(ZW zF_?t&aS1L5ck1r2xJHghN5m)yy< zJMpEtxbv+$?gV#oapxDg4lUe?4ZN$3yf^NoP0_}CSl-!Ax zk~{IW{7!z4pRmTxoha`71-(=0o#Ia7&idZrPH?C6PI0Fyy;HkW)xA^Psq{{Cl(jpt zn|XI|CvzwEk+nOSJF%agJDEF)J8_`7g*%PjNwquCfTAi zJkOoXoj3#BiP1h=yOSok*E_j|J8`~yy%QJ8>F!JKa3`1EnaiETnb~*?f1u&cFisSA zO7Gl(xoGLm=l0e36&rfr4R_j`i#vOvJ9^S>HRg zJNb6|-OjlabMY`5?ksfvG)nK(?!>F+hC6yE-o*#bxD%h?Gjr`ua3|}X%$?FZ>)rX? zGsT^M!JWjN(mQDr+1mT#PTpK{CvzupC$_cMy;Ie_QYZHoPCC-govNQZ?xX?k21@RvW1N@XIYIi(D!r3VkwZPB zcZxe{xLxNY?&Q)tqa}ALy^|-{Cra*AdM7n{r*zK$xl^@p=Y8I6#DhK22SYF&GjSK@ zVG$}U$Cvm4>rmWjgKg0PolxA_52bgCJEeDuJA+Z&8HEX$f@vtd)6$(A>^J%DO7Gku jxfAzGy%XjCI*&o`WbOoa;yE*S;$>Ofc>|?)mNxwdF;+b& literal 0 HcmV?d00001 diff --git a/crates/clawhdf5-format/tests/fixtures/blosc2/b2nd_4d.b2f b/crates/clawhdf5-format/tests/fixtures/blosc2/b2nd_4d.b2f new file mode 100644 index 0000000000000000000000000000000000000000..6fa77def7053c5fa36e16e15ab53569008c522ee GIT binary patch literal 6804 zcmbVR30zjy693l7sA-fXTL=VNcU(%pqDA(WqdtB=^|wc(bEMA7oaocWkJKx+Zgwd_t! zEs~_A=X-QW-;-K8-J?T#+Gyz^9v#y2wz<*VZWfrQ%_6fJ!V!fD4o278)&e}HrKj5_ ztE;7Z{tX=>9B9n+#n%+CtS!F!aEzO{iYPJIl4Iidyh>b7{4$wAYV^A+a-p8UV4RvJOt*@^_*0N?X+_=AEPQlG<{SAm z7U{XiJcHM=q^JFZ81b|)GR_Bv0saU|$&VnK^+L+H_$16(AiYgu@pytTH-n^wW!^Zz z1?n?#CN=L|*?2=#>_39Cp@?aPjUS+}2tjw;tN{)c(VU((DdbiUtD#zrHL z30!FWO3wtT@l2bU>J7C@ty2fpQT4f+Xr`Jkm~WXm4D`bc^poabBs09`VIh`bD+7HW zwrMXEX&S5@VI|&X&7anEf%eNbO^YU zLoEY@@`GJyg|_JIaYhj?;3t$dDj3zY1waceo#@da!%A!ErXC&AQ%i^rHO-Jn$N}LV z9l{SYMj6i->Beiu3S)<{$2efLR&7-$HCQF9v1-0rq%v%Ch(z%)P3v5(=`l^?wA>_3 z-@GjyqWQe>y>Zznr>d&EJTmXAooX+O>_zqCZEFFpX&Y+WBtFypRa;N-U*hbwCwH3G z$@jM8j%U+NTu+qb2FFNpx93w5|5hL=kh>oj|AR_lczG#4+=H(!VQ~g_?L$B*JU0hx z-b4Ngot0KHTA653Uo!fp5q<8C7&c0J%2h_|w!+kJhTa2&K@jTb+AHHM9b|4+V@<_Z zGH&LJShoeo&!SgY(EpMvH94*?pWhJ0j?Qs?#!g7h z$jWJYkGp@;@LBWHo44tZFl1!fD@#g-RB6)k{wD?{JHJKBjS47-_Me2lx=EL;eR=t% z=vprN)*UC#ULdpigWadjUm`PWe)@*3vgOKLCPkfKXkeexKt{%KW#$0%Dg$T?@EW+m zVZ7cPYSLn0G{1%8EdRbphI(leQfAK!i`um-SX*^R#jxgW+9wVfdFp&XcvPbu5($XKOLi6aIlEGFXe&MAhn;3n-xGB>!-dyJ=X*KWk$@lcip9v_lUOwU^P(u&m zaqb7r><2DuhO5a=f_94L?1U@BAA!bLtLpgJxJ;fuEe9#M$6=%6Bg7S4AOs)QVHN>A z1Uv(j1!92&t=Eq@()Bz%wy$(kUr1z!8cz>7UwXac; zoVDmlfb5T&VJ6%rIrAunVyB3$te5eNWtY_ zj2e;k__Dd%@1?duii!17zI^)J1#fTK?lX=|p8~MmeI@#c|967Xmvv(z7=W&|s_n%h};3#uFtyzbD2K|G}}h zvlBa3vroobIB*!_Cn1w(mza=4sdqb$*>3SX-6uXs&D$MjB~y4Ebm@+9lR2vge*PWd zQRwohw`~QfB~0oACb^ZEz8=R?b^7vKgj7b$w)9bW3+3>46h(Njg)1fJnA3aFgOZW9 zkRr@=hg+jP2FMG0(#`c+WOJ*V>X#A!O_V$A3_jC{P06S8DWSJ)#p;g_=6`?X_u)@Z zT(o>;?k9)GOh{R}B75(rM;}joa>m^BcQ$Qzd=C!@+}3@YDpeE$xZ?pQUvR=Wpf87} zr6}i}$hH)@9OEYPyLE(1bW3CpCB_r~E{OfjTFE7jP%;G3Erf}vhps(^`R_HjY71le zMT#`RFHEIN;!miI+p-hqzCoEgF?#{t-G)b3eu)wELs;>oB%qS(}cI-LuRbVig>~))u7Mv!N^X}IC z6GddE%t(7{?PfAwePnH5>0*a31Z5moVv7FLK^tHQ|9%Fv1%`t+Q?Rz)r#l^$yV$cK z&l$4m03Yy%rI}pdqGnzOKGcl1N^^EXxAI5e;_LOlWt58d-Nam?tVSkLRw$demFN#L zZ@+<I^Heg~dp$pY{r+v4?~Ajj zez&~%Xg|KYEX>|K{BTv6#hKW1Kp5?hn^M)9kFTxy$k?N_xErEm=y$|s^yBM%x{2N> zoniTwJvW2Qf6XWrS-)Ag-j9EJtnlg|LE&{89{K$AZ$`<;22EG4S)Z3*7*e^0V`px` z?NV$u%c0-92NQJZo5!rw>Jxw$G&2x*Q8Nz%6Eq`}(H6@bt}}lGYVvb{j0@!n3>d`2 zEx<^R)gcwa-9fMKSyqr@Dzar2a!a}jc3bzAiPN56v04J-q3(UgO`c9Bu6O^5(^%bc zkGtc*VZ8KsDurYQ8@{~oOGLi%f_*KDtDvJe&`r!Gde=bKC!RQz_;*3vQK<&*?~LTp z`1%JqMBm#EmoB49O|)w;y{5P!1G{;1Xa?pnR= x;Q<$~J;8HXP>IrIfs=>>rhg5*-DgU zjBQ3`8;;N>HKL-L`~JMAbNUxP=X>r2?m{-?Mm`il2~Rm%*%o-#?pM4)%X}2vIX0*GrO}d z2Qh_X`4XpcHWzal-{b~vk^UgVejAAcg1L0Obb`BX&3RZfqnmg=jiTB(D& zs+am{h(>FIrs!{)r`NPnYqd!`v_}VZOrPkSF6*jp=$6vOJ&`?fN5Lo>WujtKkJ`~N znn$~MI=V;i7#PE2Y)p!&F)J3u(pVGgV_WQs594s0h%<2^h>XZ?o&0zRrL6NO>oiEy z>5e`afE0|u#6)Mlb=F`VwqPgr;V}MahA1C4eK;!OY3xD z59sb(3M2zwjwgeeUUc`MX@Lym7_*H7LBaa-a0+w zFEJ=mtn+eAi#gU=VVw=JHQuw%kvM6ci;>Pc_gbf5qEiXgP#X==9BuJ5dbpi|7=afs z$vU&Jz&fk3-a5PSp>C{VfI#{Qtb%t1H0;h0>bry4_+u6i- z`M%pZ%8z-Lmw44Ww;9&SrUz6&52>^&sH*Cyp`NhLpVdvh)nCIk#yV52voK9(i*{+B z4qNB6&g%%ACnWEC9yo-u+Dbt?2n_? zIcJ^p$ZVawD1>4ti^|rigT{Cg?a>+C(Kpc+HAANu0LMSNIk` zB|2G|llfVgC0Kz~Sd;bGl&$%cb$YQMhjO%aUg1p6=j;3j*K#v=ChZ)z&Ux#6lh#fq zWmg{kK}A(Y6;)le)j-YF#_ja5&OnXOSWU9dY%S7qtR0Pz zjhvA$3P*`37Y`@xG>KNxA-cMqzA-dL#ss%BJ?6%eSQTqyW9*1MaWIZq=WJYZJDHIK z52ooSLC-& zN$WfkwcJj#XdNBnndlY$V_1x`&XkxD^Q^Nn*2SjSX`KUcJU)%{aoO!;Np$|`cFG}D z=W#SaD|EC@Pq#A|BQXJ!F#~hm&MNC{ayuWmonzKHhfDa{I=8Ho(VLtr(J94B)~Vx7 zZpOB5=Z;Q_+j)_%auydRI_naheLTeD-sB6s!fU+AUzxheIrN})id(0OYFej>TB@D8 z=vh6l!5XP?dRfyo*X^vd&PHw5`_?(ClR9IauXNp;jL4Yi+-avmRP`p;OVjCQodMqD zF)`6Pv#j&F+j%=STW4P!ayzLyS0aOTa(R;rqhz8}3-!&(P_ zEWt`Y)<$f{9vpHzAA6H8;TwE!9Wj$PIgcN!s2{7cAFDPSusPeXlOL-O2XX{o;KW2{ z5tnie-{BTN);>R0YCEZqm8$bgqI19UdXr12taTn$JwH~<$aG*^FS1cBHrYRQ7!GU+D0e0)5njM5~Hp2O8h+*#NDV;hfJQPP8Ed&uhLG05>ILsE^S#yIkfl_k-V zM{%gpY+5P1UMIAY5|Y@GZM8jCPP=S-zo9v?+urm2U+25u|Nid%-Fv^BB6nl25I28M zxD5aWw{SvXn&&IS4zhjXqzscA1#?P?#EGEP6`&39A!<7nWQqi1LLcXoEik7Sh(fbzJ~4wp3^?Q-+_a|*u?al{E|y|x(D!uipF9CQwy67TXrr$2zr&v$F+nB z07eL#qyd8s*}?`RJFzo`Fd|x%n0ESX@kMD(Lvu&>BgQO_+9G|UFIKJHyzOf@Z~x@X z+;e4>S8JQv?mm1n{Q9lBj)Ad-wVmVkJs!Tn;YVVN%ca-qZ=yGYFGj}cipr*zwwow# z0>YxA#VMz9t~XKM;0skP9SW3}U+BT;*d&ZY$;GOghFc#&7%yxnw`+9XD#!0Y0*C$* zW1=)yO47I;h~}eG&ZE=Um0cNX)<8r0Lhda@Y~3rw1Yx%qQXp$8qh&( zYu4Ae*>P4bV~JdM1VBun01gs`hBBebt(U3V$W>bi;FC{Zy!jghQ896Vh!fJj%>z}< zC5BLNK~e=9c}{!bZg>APVCh{SAlR$NT<)1E2mgFzKcOqXWf&j`aerWr0bF$uHS_Hq z5*uNrm0a^Q0UR7MDp4O{{FHYU*3}disU3ir6AenYGtM zc>TKMDPfpqNddTY5P`@`Ak|%IA!K}5AM^jCKHRVt&SPEBaA2T-s+STR)3~+Y;=Iq( zKO|BpPR_{5zff_fXMkj}=OLfQme%%;J0pdnMC7x$0u!~Ry^BiJ`C3bi%&crN5nR0k zC%ILXdwv+1L>dU7Q`}mLTKuPYm3#&B0`_?Lg%17x<{gp|KZRU1`90|XJQ<->xPb%~ zFJXpcT7a#w)&Nlr1ce`s9iuDGjZ4WmGf!PxKew>F($Lh>=6?h1`u=(r|YOkO&cqi<@_#;809_YGzJhndIsXsuW3VE*d(^Ow8A;-x3mjRFrr!JFlqX za&_I!+g+ckFIr~IwO;SIZMTO{kl=7$VVU%Yx|{9yA3hx(`HNIEFtc2P&0{a$|3GB4 zI4QmAMq_KogZ`mcqoAO~S@7AqOgoKFils&?S5;QmHce-3soaJV0TC2iKpBt~A}~RCU6lT2WcM^Iu>51K^lW00000 literal 0 HcmV?d00001 diff --git a/crates/clawhdf5-format/tests/fixtures/blosc2/b2nd_zero_chunk.out b/crates/clawhdf5-format/tests/fixtures/blosc2/b2nd_zero_chunk.out new file mode 100644 index 0000000000000000000000000000000000000000..fe7c16ec92a0ff95c712aa2320a982eea4de5db8 GIT binary patch literal 2400 zcmeI!`Ipai9L4ebx-XeQB4ppPCJJLJdyMSG6wS!KCdU?82jSSJW1TW-vaf@1?1~UI z%3(N&lc9{UP9hSHQo7%d$vOSdf8hNmJkRUg#~TPo9uz_eltmTPKux@iCU^}o=zt#R zgLg3;<1h)+kcfF$id9&Tt=NSW9KW9d~dKk6A9uXGN`)J!4fZ(&||wYhlsW z&bnG}>u*DCl#RC!Y^Kez1-9JQ+9um>d+j?rWGC#jowuLuH@j$r_O`3(>9D1YEtUg9Wi@Vs8cOL_%= z&TDvGkMiao?J?fPdwM?~;v;>WPw^R^=nH(Aul7y8-FN#wKjh#0DL>~#4&+sxk|?J- zk*JL*G)HU1qBDA-AKt?VjK^fmz--LN64lv=ZP<-(Rp+?soW*6;xsAV7=W#1w#jK1~ zvTF9C>bzpFS{r-Ix&?KH*=U=fIg=~8s&mFJ+Eu$5)Cp%^7GiOh zXB9@UHXE=>P^S}nurCL3xav&dbk63dT&z0lRc9CX@c@tU6w_7bI&bkWX7ik$&kK7g zuc$haUdJ1GOVw%TUA?yt@S#51$9udd_$R8f+}HSK)%nWztIi2O?dLrkav~oJqcqAt ztkVL~x}DB=JE${Kb*5lu*6plTo$W!L!#JTj8MuNQxC>aA<+l8)Q&x4VT1~5;Ri}M$ zI|Fn(p*qtnA*hqA+u5u-Uk7zgTDo1ZU+k9sVa^A2N(6NxRVRwg7|j@VWKY!@%#j?& zc+OxV=Wz)?3+n7vom3uY8Z(&58@$8&%*yK^15NvW1 z7cg0yyqP<=hx>V0oBR_m@+xogt~NQ`^LQa`a(S<;O|In)y{WhMH@$;4xvvlQ;Xc+U zYm-0ndA`_J`Ucz$c?8^9A!`$5fZBbnxYljpktO;!!QODF7*+3s%P(T62jNW1X$1#2R985-VO} zCE9#jW~*(3#M%>xb=-cm49k>QcO+JJ=9XAR`81zpHNM1pY|NH?Lt=GTodFVSG$*P~ z0+YCqE4Y?hB-UQlITF;lq&g483e_q2P^{YCKw`D_SnudPB-S90Q=NF9=^y)CU+SxT zy?-vTQasg&+>e%HhFs1Ca=T>)%n-+GbDaTD*|na`)&2ochKKn8 literal 0 HcmV?d00001 diff --git a/crates/clawhdf5-format/tests/fixtures/blosc2/bitshuffle_odd_blocks.b2f b/crates/clawhdf5-format/tests/fixtures/blosc2/bitshuffle_odd_blocks.b2f new file mode 100644 index 0000000000000000000000000000000000000000..6e71e429af7876af0fbed4a4adcac4b60b17c597 GIT binary patch literal 2204 zcmZA33#^WH9KiACoE%3kxs-@RF2#~XB;_(89Tg>#NUo=m2%|(pIF*@LN64)Xa~m;J zQLK~7F;WMK+zm--sK#{v{MfsldG>yu^M9T_@Af{=^WAS}!Lq)MvWNG1VL-AgNs>O_ z+Sy9XTU;UOn!fwWQD$LkaV{<@7Q6a(nT0!(WM`7Ld0+AL%U5Pxx!(Tm9ZBZlBsrKQ z8ELbM6=wQ^v*-N1`pWo<(@8mQq#Ct}>l@OHRw{a(pX--=@(3z*`O*TUq!5Ahog&E9dK1*21 zIyR6`0fp@4ASd{l^ZZGghZc{x3>B$HZR*jGrnICTkJ5#n^kpE!c!_c3FqK^9@E%L~ zfREY8mwe4`_H%?&{K_BvL;5vIQj&61q6T%S&)qbqHCc4xNuH%YgBihSCh#WHnaKhc zv4XXH!sl#d2YWcgaXgx&BxR{gv{Q#WXw3bz<{>)Ljb8L)5W|Re#_>9DF_ZZ$Vg+l7 zcJld(Z`gyhlVkiuv~!VYr??+gifE@QHK<1e?jhCABi2vwG<|rEp^T(xJJTJ{Vj5DY%R1e=DoHsiQJt7Osdk!KKR^~8=|(RGFqq*)I}?bxGmUqc zM;^;r!+P>5Alli(0giK;b6n&8PXW=+UZS0ooZ&Yvk$6|pPFX5ajoL&zjc7`=^B^7RLQi7u3?k;vC|)Ip zXeXDs}fOM5L2Wv94RElN4t{I@PI*E|C;nBRg@UL_>7SOl+r4x)jZ? zotUX8x2|9AMm5o8`4NVsG^xh)`|hMZXlQ5 z3}7&YJjiH@DWR0<%wax@si2ZoRI!<@)KJSV_Huw@{KOglq*3D}X-Z4l(2-2KkwZ`N z7|1;gWh9R=j!8VtOv+e5IZJty)x5*|Y~wS&U=RB^#0h@k5B{O?1xb=dI&JAh7qYmC z-t;4%Aw0k%jAa6)Ok*}Lv4|zS!AjP#i4XXgI=-ZygB-P&+oM@*5o#;k3HCr_}K^(au4lom2eId7_<*xR_|?Dy|`m?%YaW z1~G(TG;HT7`!hs4FOX_ynfWc&uz_f&n(ah8yNPxV6YZSlEPr#Mw~KaK(~baWJRjg+-9}?}NoS&+9Bv_xf!syh zoe_+tVLLPIb6G&#oux!OtJ%PNRPzZth<3i?Fega0^Ow1?w@M?Owqy`@C)G|b^X=qQ zKoO&OoC!=O+L=w#Sfc8`(k)wM0Ah?B^&a`IU40M-va%oJ;9IXQG`O=|LU? z7)&9aE9CecoJdJ^s2 qK>Qi{tptcgoDU$Cx;u?gT~cyR%Nv1^3}zzxTV}j&Pjk{im_VFJ6vLKF1FL zqu+VsOZREAkz794f|J~nyg_M9_KuFyU>r!qP!u>7kB&9mvp7C>B?l=d)m`AeMU7VPgnFYRrCo}^(j?Nw!N!e zUC?_}U~+X)A5c*rQb`|C$zY#Z(u?m>d;X-4s_nlu*5@ z{&#i7m4fV1fytGuEKydLX+>6P#bn#(BqwXKPHRj#*^o`zkS)r~HswvWy?@@WbJO}s zB+-un>SszKg+WSTh=wsj!zSC_KR=EM8pk9}VVb5)wtX6DWH3V+rnLEPpUVZE%OzdO zHC>r(drv&6$r;s{JgLYrRpf-Ka!OT`ZSUXHq8w0B4yhzZR5IE2{`oLED2z^uAW9LF zZSTNA3|$msau7#1#nD3v^islP+xzG3I(Ge5ez)5fWI@Vrf@%TfF%`gq*_l~+MYHmn z=477cOtyVm(vp!G$}pv6TqbB-CTU8hY06~V`%jUS3{pylXjn#Q*ks%L=d)O%ES6~n ztF&UW?Q_Us4ePYVl*0x#X#-o7$2R3nw!MGeu4C8lkx4=6Xcm0O}gv@o_u1Gtv*gJ}8TNIA646BTBA`3bYVQ1$KQzg|d5Qxo5*2 literal 0 HcmV?d00001 diff --git a/crates/clawhdf5-format/tests/fixtures/blosc2/delta_int16_blosclz.out b/crates/clawhdf5-format/tests/fixtures/blosc2/delta_int16_blosclz.out new file mode 100644 index 0000000000000000000000000000000000000000..e8918075fcd369b427c22516ac6c889feda14e8f GIT binary patch literal 2000 zcmXBUgQ8eh00q$`cc0j{ZPvDJ+qP5Nwr$&1Qrmv9t#+of{$XmWn*U7D#0rx%xxy4p ztuRf~E6mW$3bQo3!W_-5Fi-OJnys#(xnErOP66|`2Hpsm^k?bRXZs7^s=bqTtvThLuSf}ZLX^j4psulfc3H6R$M zLBU`R35IG|FkB;oks1|@)|g;9Eg!6)6@!(uauB9zJ*^*X zpbdkKv~jSBHVrn@=D`-)GT2I62is`dU^{Ie?4TWkowReXi*^lm)9%3@+B4Wodk6by z-(WxO9~__qgM)N%aEJ~K4%6Ym5jrwBN=FCB=-A*m9Uq*a6N8gbBr^-4WcWyMnuQPjIj93+~qg!Gn4zcvz1FkLt1DaXk?{si%Uc^-S=ro(rDW3&D$e zDR^101h49~;B~zbys5W>xAji&uHFmY*9XCe`Y8BVTBv0~ zE440YqqYU@)V`pDIu>+N=YlTkTF_113wo$$K`-?#=%c;`{nWo;fCd%}(%^z28d@++ Q!wW`eWWgwnE*Mk(1#5BRUjP6A literal 0 HcmV?d00001 diff --git a/crates/clawhdf5-format/tests/fixtures/blosc2/delta_int32_lz4.b2f b/crates/clawhdf5-format/tests/fixtures/blosc2/delta_int32_lz4.b2f new file mode 100644 index 0000000000000000000000000000000000000000..819f61ea83e25425fa436a5a11a9d617c67313a3 GIT binary patch literal 1584 zcma)+&r20i9L3K(=lw_?h(1#x!$mGasU$KXvDySJS{P(O(hvLLLeVHtLGj1JKR|d{ zP-svtM5$IKB-O)xiG?L)whbi{LIp+boO@>m(I_8$=AF6c`<;7c?w#o$ymq+uPEEs2 zd;m~0&I5mul7TFo4Slp{=?k6+cdN+d!BKho!UQk@XKX5xW-1+9&tvi(qBFoW5Oylu zg>(+l><$B4D+@U=?E=~KuZMtk(!jAZY$@e|kl@QGu((wq#@Nwv~x z^N1tS?jqU2(@qhGn+Fsk(RKu&*+jFMMvE|JvW~J|&~BjJNPSBLz4f-YispG57liG- zNLek|dx`dC>MO$UO&OOXsx)a=C{{h#wZ^V>o^A-gE3s;4T%@@~V_Dc4XDR0ddw^X-ri{xGRrc}O{*F{piQvYivw?ghiztf) zFL!{KJ4k&<_?Jr*+bE?pkJ2a;Hp(%|a>3pT+Q+F+h@iLJ^tz{kb4Zlk(CTR^yOGv| zN@#AgEm9OdW?SgwiHSZ*C4#3QZKDrS4hv5Il+!<>elCLPhi&v1G+)vf5jOfO%2C1I z*R!cTjf< zKcx4K@U~pb_H}DywZ04kW_#`CQRsGZJ#tLU$FnRlFKLu>{qrqdWRN9RFAJMqG} zca8Qs^@a#?+OLC`b9sBtc~l3ffyuR@I%Ih1i@4SBS`pN4Tuiw1>@cPRu2n_-y z;+2?J2(M6HNq8mYm5f(%UMYB`#d;FXbACSI9&W#N^T zS2kYRdF9}hlUFWYxq0Q`6~QYnuYA1n^NQqEfLB3Yg?JU_RfJbjUd4D7=T(ANNnWLR zmF883S6N==c$Mc>fmcOdm3UR=RfShoUe$P2=T(DOOcy)!uRgr`^6JN{Kd%A22J#xjYcQ`NyoT}`#%nmQ5xhq78pUfguQ9yF z@*2l$Jg*77Ci0rZYcj7Xyr%M+#%nsSSY9)D#qpZSYZkBByyozl%WEF5cwX~)E#S3~ z*CJkvc`f0!l-Dv|%XzKfwUXB=UaNVn;kB06I$rB}ZQ!+$*Ct+@d2QjfmDe_2+j;Ha zwUgH_Ub}hi;kB37K3@BI9pH74*CAeqc^%<(l-Ds{$9bLLb&}U9UZ;7T;dPeRIbP>^ zUEp<**Ck$;d0pXkmDe?1*LmIGb(7aEUblJO;dPhSJzn>DJ>d0_*CSq!c|GCvl-D!< EAC<{Yy#N3J literal 0 HcmV?d00001 diff --git a/crates/clawhdf5-format/tests/fixtures/blosc2/delta_shuffle_v16.b2f b/crates/clawhdf5-format/tests/fixtures/blosc2/delta_shuffle_v16.b2f new file mode 100644 index 0000000000000000000000000000000000000000..5f0cc42b8317931a2ee462d80b093cea1a592701 GIT binary patch literal 1019 zcmb7DKS%;$6n~$mUW1THS_IKmOIy_3(9#%AjSU60)zH*<1Pu*MrGla^4WXz*oq?i8 zdw(qagQl*bSd&T~=k-2MBTnn+!M*ps@B7_*-~0Xee3#qHy9Zl)JMatuTaW0_81cda z8@d-|W#W4YW2Yc9Ivs_Z_%uKo$X(vZ5DGOYc=TuYz?K1CagUsuc@u8w0PGBwRv~q` zKEbnNj4KFZG3PNZA$C=kW%5ZKZVWYT1c(2{;)!G`5sM4a8zB%3g(IOr(BX7B+#avb z?{$0rY(WdeV~JEU5swKsQ6U%zMZ%$Az~OQ_JZ`Vw=k>TX3%udrI9u1x{K!Xp`4~1? z5EdnlTSI~wATW1}L<~2}H+UUj60roz`y<1#$}WeYElM3(YKx6#t5L1h-#_ZrS|*#z z6pE#CrBou_x2JUbk8aQC_MC1n==PFsujuxgZg1%JmTvFp_MUDZ==PCr zpXfFu@P9uxly1Z5Hk@t)bQ?jpk#rkHx6yPPL$|SX8%MYCbell8iFBJpx5;#yLbs`O zn?|?kbelo9ujn?DZnNn2HQi>@Z4TY$(rq5y=F@Ef-4@dA8@er`+qZOEOt&R;TS~WO zbX!ih6?9ukw^elej&7^z_C4KxpxYX{t)<&Kx~-?%k96BWw~ch$M7Pa!+d{Xkbo+^J z+vv8PZa>p)2i2`%~SLt?*Zokv*I^Ay2?IzuB a(d{OKTKC5dM12Z1?J{6BIL`t~n^IU~s{kzaUo+-a`Zh(SY~|GB-hu4>Wp{J$MjB zb4mn79Xu+@5@loX5&J0dk&8({NwliFw$X88LqDp&ny;$9?&({1jvYF2c6j7CUI7?x z(LgOaaC;k$lwNx)&ScRu%1(v`#qM!73(P`2pD%Cno?*N;4ZQ%oCn{AlSxkTeMtgvw z=TU;zO2BUFCGxpS_BuIzr*$9obJPuCW3+dL=2OIRn0)qqVkTQ5US%uW%VaRQjbW39 zg0}HNf|G`=50%jG_G`2PMyL<`&@@u}$+J~7HdBSQEX}pTdX^~svXRcyyq@cP&?Z9) z%h@18vD|~hv3ST|Lw=6MOk{G=dqHe{oUvx;~t?a$yIVADHQ!H?tk0FjE z(e_EqG598c3NJW`my*P^XPDt6S`y~`F&(*_sDU_xRzwL8+Y+A9LAh7cQGUWHb0`p@+@XbY#iVwo8BwENK_ erH!FFx$hSh_$ey3>x!PNvVXE>2;4hg^uTX=K@=1K literal 0 HcmV?d00001 diff --git a/crates/clawhdf5-format/tests/fixtures/blosc2/delta_uint64_blosclz.out b/crates/clawhdf5-format/tests/fixtures/blosc2/delta_uint64_blosclz.out new file mode 100644 index 0000000000000000000000000000000000000000..20735223546c90661fa2654826e0ddfeddfb4f6c GIT binary patch literal 3200 zcmZ9~RZ|rJ6h&d;(p`eINFz!}3X%fS9g@-p|NAN}-OS*6TIc1Pvz~V_Jmlqq@(;iL zOSk)UdqB5`bbCa%$8>u_x2JUbk8aQC_MC1n==PFsujuxgZg1%JmTvFp_MUDZ==PCr zpXfFu@P9uxly1Z5Hk@t)bQ?jpk#rkHx6yPPL$|SX8%MYCbell8iFBJpx5;#yLbs`O zn?|?kbelo9ujn?DZnNn2HQi>@Z4TY$(rq5y=F@Ef-4@dA8@er`+qZOEOt&R;TS~WO zbX!ih6?9ukw^elej&7^z_C4KxpxYX{t)<&Kx~-?%k96BWw~ch$M7Pa!+d{Xkbo+^J z+vv8PZa>p)2i2`%~SLt?*Zokv*I^Ay2?IzuB a(d{h+SQ!sku3Nu;-P*Mv_SUW2wr}6Ib?dI( zyLau~yKn#geS7zU*f4Q)^;fT6yMF!J)vLE|-@bMC?!Ei>@7=u%V#CB=y?*`b?b~0qApv0I+#LKQn+l{$(S~ zWOSFnTu7|TVIhDkWY9wr7Q)CO4`Rc_(beNJzk-2h0>cMMVx2o8WBcqdiRlJdA{~)Q nmz9y7g98|oAglqzU;r!y(ZLi(VCl*RDP7M2C6#ZWt6%^CT@d4> literal 0 HcmV?d00001 diff --git a/crates/clawhdf5-format/tests/fixtures/blosc2/delta_uint8_lz4.out b/crates/clawhdf5-format/tests/fixtures/blosc2/delta_uint8_lz4.out new file mode 100644 index 0000000000000000000000000000000000000000..b70a6d9599651fe994a3e7abcfb7640a4284fd88 GIT binary patch literal 2000 zcmeIygH|92006*j+qP|Owr$&XTWz*&+qP}nwq5rh_xgud#>URh&cVUK$;rvZ#l_9d&BMdP%gf8h$H&jlFCZZB=g%KOK|vuQ zAz@)*5fKqlQBg55F>!Hm2?+^FNl7UwDQRhG85tQ_Sy?$bIeB?`1qB5~MMWhgC1qu0 z6%`d#RaG@LHFb4$4Gj%VO-(H=Ep2UW9UUEAU0pprJ$-$B0|Ns?Lqj7YBV%J@6B83t zQ&TfDGjnru3kwTNOG_&&D{E_O8yg#2TU$FjJ9~S32L}g7M@J_oCue787Z(>-S64ST zH+Oe;4-XGdPfsr|FK=&eA0HoIUtd2zzrTO~`uqC_1Ox;I2LAi^FDNJ|I5;>YBqTI6 zG%PGEJUl!iA|f&}GAb%6IyyQgCMGsEHZCqMK0ZDnAt5m_F)1l2IXO8cB_%aAH7zYI zJv}`mBO^02Gb<}AJ3BikCnq;IH!m+QKR>^qprEj@u&Ai0xVX5aq@=X8w5+VGyu7@k zqN1|0vZ|`8y1Kfirlz*Gwyv(OzP`Spp`o#{v8k!4xw*NerKPpCwXLnKy}iAoqocF4 zv#YDCySuxmr>D2Kx390SzrTNAU|?`?aA;^~czAeZWMp)7bZl&Fe0+RjVq$V~a%yU7 zdU|?hW@dJFc5ZHNetv#oVPSD`acOC3d3kwdWo31Bb!}~JeSLjnV`Fo3b8BmBdwY9l rXJ>bJcW-ZRe}Dhr;NbA^@aX91`1ttbs`!bW+1_+10zE8|?|7NE2aBLl-(kZv@Uvw(aZAkGA0OCZ+R zs{c1ZhoP2{A!5mrJxkWiNy({s@USon-hQfQCcimKYZal!GN*7^ty zx{M43FMz%RI*(nR|G&I_JqJUBh}_|lh7Q9}l?D6ePTlJwxKM(Tm64r;1E>vzHGmil YfV$AZ6h>eGut5Ug98gmE2D%Cc0LuthMF0Q* literal 0 HcmV?d00001 diff --git a/crates/clawhdf5-format/tests/fixtures/blosc2/delta_v3.out b/crates/clawhdf5-format/tests/fixtures/blosc2/delta_v3.out new file mode 100644 index 0000000000000000000000000000000000000000..bc826e3e48d64d9208868bccdf969ec327295470 GIT binary patch literal 900 zcmd7Q17aWs006Mswr$(CZQHi5wr$(CZQHi(?id#>URh&cVUK$;rvZ#l_9d&BMdP%gf8h$H&jlFCZWwC@3f-BqS^>EFvNz zDk>@_CMGT}E+HWyDJdx>B_%B_Eh8f%D=RA}Cnqm2ub`lysHmuC#4Z(v|xXlQ6;WMph?Y+_D3@8ICz=;-L=gwv|=H~A1?&0C# z>FMd^<>l?|?c?L)>+9?1=lAd5KYxG!fPjF&z`&rOpy1%(kdTnj(9p23u<-Ekh=_>D z$jGRusOaeEn3$N@*x0zZxcK<^goK2|#Kfeeq~zq}l$4az)YP=JwDk1!jEs!T%*?E; ztnBRUoSdB8+}ympy!`z9f`Wp=!os4WqT=G>l9H0r($ccBvhwosii(QL%F3#$s_N?M znwpy0+SS?mrsn46mX?;**4DPRw)Xb+j*gDb&d#o`uI}#co}QlG z-rl~xzW)CHfq{X+!NH-Sq2b}-k&%(n(b2K7vGMWoiHV8H$;qjysp;wInVFf{+1a_d zx%v6|g@uL1#l@wirRC-2m6es%)z!7Nwe|J&jg5`X&CRW?t?ljYot>TC-QB&tz5V_D egM)*^!^5MaqvPY_larIv)6=uFv-AJK|Nj7+jm1U) literal 0 HcmV?d00001 diff --git a/crates/clawhdf5-format/tests/fixtures/blosc2/generate.py b/crates/clawhdf5-format/tests/fixtures/blosc2/generate.py new file mode 100644 index 0000000..f79c0f7 --- /dev/null +++ b/crates/clawhdf5-format/tests/fixtures/blosc2/generate.py @@ -0,0 +1,129 @@ +"""Generate the Blosc2 frames the `filters_blosc2` unit tests decode. + +Each case is `.b2f` (a Blosc2 contiguous frame, what the HDF5 Blosc2 +filter stores per chunk) and `.out` (what decoding it must give: the +first chunk of a plain frame, or the whole array in C order for a B2ND +frame), or `.err` (a frame clawhdf5 must refuse; the file holds a word +the error message must contain). + +These cover what files written by h5py + hdf5plugin never contain, but a +Blosc2 frame may: special chunks (repeated value, NaN, uninitialised), the +delta filter over many blocks and odd type sizes, bit shuffle of blocks that +are not a multiple of 8 elements, shuffle with a byte-group size, forced +stream splitting, multi-chunk B2ND arrays with padded edge chunks and a +chunk of zeros, and features clawhdf5 refuses (dictionaries, registered +filters). + +Written with python-blosc2 4.13.1 (c-blosc2 3.3.4) in a scratch venv +(`pip install blosc2`). Re-run only to regenerate: + + python generate.py +""" +import os +import sys + +import blosc2 +import numpy as np + +out = sys.argv[1] + + +def save(name, frame, expected): + with open(os.path.join(out, name + ".b2f"), "wb") as f: + f.write(frame) + with open(os.path.join(out, name + ".out"), "wb") as f: + f.write(expected) + + +def save_err(name, frame, word): + with open(os.path.join(out, name + ".b2f"), "wb") as f: + f.write(frame) + with open(os.path.join(out, name + ".err"), "w") as f: + f.write(word) + + +def plain(data, **cparams): + """A one-chunk super-chunk frame of `data`, as hdf5-blosc2 writes.""" + data = np.ascontiguousarray(data) + cp = blosc2.CParams(typesize=data.dtype.itemsize, **cparams) + sc = blosc2.SChunk(chunksize=data.nbytes, cparams=cp) + sc.append_data(data) + return sc.to_cframe(), data.tobytes() + + +def special(nitems, dtype, kind, value=None): + dt = np.dtype(dtype) + sc = blosc2.SChunk(chunksize=nitems * dt.itemsize, + cparams=blosc2.CParams(typesize=dt.itemsize)) + sc.fill_special(nitems, kind, value) + return sc.to_cframe() + + +# Special chunks. A repeated value stays in the frame as a 33+ byte chunk; +# NaN and uninitialised chunks become special offsets. +save("value_i4", special(300, "G!2ZVfzdQDng&MGz>rP@0A(|8D*ylh literal 0 HcmV?d00001 diff --git a/crates/clawhdf5-format/tests/fixtures/blosc2/nan_f8.b2f b/crates/clawhdf5-format/tests/fixtures/blosc2/nan_f8.b2f new file mode 100644 index 0000000000000000000000000000000000000000..939321b8a1a22d1ec8d36ce0eba5c4cf325cff1e GIT binary patch literal 172 zcmbQYBFQMNC^0vc;SvJ_L*jWL0Rn552r+~*U50Qt6QB$Rup$nSN)U@P;UWXWMFz%0 zH`u@m8CfCff#l?~4D9z98167Iurji9Z~#R>SOX@{2xGwMrYVd-J!}v?=YYb>H{fzG F1^`Ikm#Z273A)J9O*?6#JvY5xEQYxcn~{O z->)Y&>hA@1_(u>LCN}ENiH;z21l2sq_%jfN{S^;$W*5RVh9Dydf<kOxRvZkJWoHb3bZ7<=_@`P&9F2cL-4Hnm-Q zjV>#gGVVB|{ zuUn<$>W8hJy+bc&7FSlWth_>EvdWr!^v$dtUA;~QpN~#V#~0i#uc~Wme>yNWg@MFH zWmPnFe&BUI=65D6Dn1QgKp<7sHn#N)jJ}#%{JO!!&Mz#bpt@7n$l{QryLUjy`IyA? z>%}GI_ZuE{J{=l+J^yhP1Bq{y-KM^0zloKd^D)1m@W{B-?7Um0l{JlR-Tf~n-zMihF`s1{IZVa;1!lsRNbX#XlCQ&ej+jB+705p>iU*;`q0a11~?R# zQ`OpQXb!A<2b?{BIVlZabi15V-_!xDzk0v?70trQzXjB^OYeZCor|YW5U`${onK76 zSM#8)vlr4_{IrHq)zCFKh_gR>{N$PQQSp#KNf~7Lq`QA?`t36C$;r26-vJ9-CwHII zp%>$lGxCZ{@76T5{M$P;J_8W9NGNUxbOE6kW0Ny;;oX$FmQDaL^B&61$|WEwt*pLB z-_+6$3I_U#OUceBlvdV0eB9MHJT<%UWdqB`BP=Pes-YzU#ef{p^ zSA+?ypa%+eIhLH6S4<>VH?(vya`1~tD#E^zMd*dtl+5b-){efx@!9#6?-*7tAu$;x zbsd9ae!*dv6RzeIfGem?Z9V-XQ*)qIEIXfwgu?cndyOn?kAOHri<;YeK@;y+zM)t- zH;c+DYiL6`4!eT&!Xx8TbMgsgjFQG~eG@Bi+m)28yjx}Dn#R`d{^7~DA9#f%6}E#K&1@XO zWf}OQ5)!2jln7MKElDe>Y40<(v~~9MNd{lFb@f88m|t1NuyI3mt?gaYs%VYv5Ie=d z$_-yvR@XK#u_BaJ)i$+t4~$OEEqz9qICg018C%#md4OYbZ-h~5qICy=PtyhWL^WCQ0iMc z=z|lp?-3LlgGONxG-|he5t=2of{FVS4YwYdoqJ%vs7jV#FHWv)2~Q3QSsjx8eprx} zf2QkkxcSa@qLFX1Pf)Itui>1T8(E;ZVtSq+8Cc@3j}Bb(qboWk2t>`Lg?fBQ*ikS$ z=ggm1{+yRB{ocda%}I*WZwf}`T1o|63PS`}lD5Zaa|jq|8Oscw^)MxA-M<``PO`zG ln3$QNeg7EpFn(UBKa0QjEEN0>7=Qf^x?x*t@bB60)qf=)>URJD literal 0 HcmV?d00001 diff --git a/crates/clawhdf5-format/tests/fixtures/blosc2/never_split.out b/crates/clawhdf5-format/tests/fixtures/blosc2/never_split.out new file mode 100644 index 0000000000000000000000000000000000000000..8a1d2f344681e2d95ae3430e9ef5b3586e7bb84c GIT binary patch literal 3000 zcmYM$hkwua7RT}PKF1cdU5OPV)YjO0tG!aS_Z~H4hN@AG*ehskp|Oh_BPdZ@?b>^* zeTy2m===N0@4ElO^YMP1*HK9>pGii^F1e+U1WQ?|BDJNFd@Ze|v-FaIGF-lu?`5jY zmIbm*Vq~LilU=f3j>#FhBsb)NJeODUK@uSa(jhYfkrzc!0u@jL_0SkC&>r3J4F+Qb z#vlUIFc*ul0&B1d+p!mia02IW4R`PuFYpc@kxWx*2F;?mwV(!T8LgzXw4pZBR@zy6 zYJVN7qjiE#(pmbWF3}kMSz~pl{;EgylwQ*7dS9REU;17XS#nEjnJtIqvqDzV%2_q5 zV@<4uwY9GHjfGm6jkR!_ZgXsrEw{C{$+p`bJ7~x3oL#Y7W|B-&OD4%Cd8D8emvT~# zI*p~dw53iT86d-CjD*V+iIgabmesOJw#shm9GBnairjKKubqw`GQb}>@i}!$QKvfU zqA6NYrw96B2*NNP6EPk0u>?P19byrOUvU^Gah^JNsq>OLs>wBtX4LE&q=obgEvuEO z(}+5)snd%(!}UArOx4-CK$q((-Kbl1x9+FTX}zd7sPkN3>ED{zQdoNPr%qlgOr7%7 zsb`I?rM0te)`vPHZJb5e44Z2UZH29~&9=kh?XaDs&NaJDofMKzGIKk5rHGW23e>41 zO{9gilP=O*25~!{&UBe83#qeKHp_O|BM0P!oRh0^M;>uIZ{?#Tg^0J)JLMNt|R zQ4R}X+liBSIYgcFa*aBV<%PTxK@y~#@bBa|V}i6Zi29ukap;sFP0pHHYTYB3i@6>b5M`zIlZiR^dYzN#_6P@PJrdI0@NvEm8nzDLa5WxdVJD} zpw4_-Y%#XMw%AVF$L*Z5i+02AS|Uj$=_SDF6qQm^i8}SAskD;z(w#a(WTcFhNp3q! zWCe9%WhZrx$Vs2h6Y9JbMRKQ;mD?%gbSmRZG^9>jbVg4M#Bh8^oynMk1>8;yHef4$ z!2xdPG<9y_4?M?T)G_trcKkJvIz_2dQLAfRZKBP!y>`_;I!H(A7!B7MPG_aAaXNeT zklW5Ry-S@J`c^+`QcGPa9xE?OU5*lc^JBOKp{{x2?8| zI=|Uz>Rh+`+>WP{RdPxJDJG@4oto0X>2#DH)EVk-@?@DM^JFP+@&@YckbQF0r*mJP z$V+ZVd6Uy16S8wVg;5;kQ4MvRPCIl#Zw%me#!zR9(~0JGyqmlm2XPE%a2dDpfZKV) zo19ovaytQr3j0 zCABn`$+BA>-sE5_%k9*%hSrQXxr_C*{?r-mbY?l77~9D0csj@Ij9s#u)JZ~}jFLri zQO7G*Wv3Iu?RYxa3FWvW45(&+VL%3v!(|`6-L_UoofSj~oc%c8a^3 zT${ye&h2#6n-OwjrL|If1jdLY+t4&TD+2j-M-5PR*~yv^0xVQyb`4 zyvd$UUlwbWj@JmCsqvFE=zorIphWU-7r zPs5%ET9E5`1*=Y-Cf3{)%j@}Y8{>55uvpQyn#J;Ze!z~Kr*qqs#Y*FJg81!};C8&8 zH}dtox3A|va67Z9vs~8j+u6!uc|AWXm)Y}&@*jD_VkJgOq-W1_u~@~ZQ^|chAuLux z&qGMWtoTK3%2Il}Fn$5j^Vk<)!(=65veV|WW&yB@O>w2Egif}s>sZ*ERNzmy_oe|WD;I|XWVnw^2$5H1ni*??v zQYR60(z54)62xM8-%fR(PJ8LfZzq&JA1f323r4!0uW>z(XU~tjp5J26|KxTANs;PP z&%fZeQ<=qTfDnE=o!qxG6r+4SpT}R&>v=4TwU^({DYu>b?D-4!9GZ;V$;6%qX<^>v zvec>Ve!({G+Zm+8*z<6mLY;-WOjq+4+{R+<=S}{d#k$F!d&PS1dY+EO%E_J=vC`D3 zNu8#=$sJtJhp^}4Y?ABwVq3{yFxKMu?RY)Ez?*!R-%d(btepII3iI2kNS!+F7wq7Q zHCV!A9D6>UH+ix9b~f=Q$ID^fU_puFblWyzn<6RO%6e8emlL~O&;ZL Y@+|&>2|bU+PVD1$PT>N3{s2$#52yv@>Hq)$ literal 0 HcmV?d00001 diff --git a/crates/clawhdf5-format/tests/fixtures/blosc2/shuffle_meta2.b2f b/crates/clawhdf5-format/tests/fixtures/blosc2/shuffle_meta2.b2f new file mode 100644 index 0000000000000000000000000000000000000000..3426ff41e2b77760cb775a10f92be50a81ce2b16 GIT binary patch literal 1003 zcmbQYBFQMNC^0vc;SvJ_L*jWL0R+simk2S0FkOc5_!mGK%=REf96*A90hqYRz;Ka) z@z4!6MzBg21}2COhRJ6c*zYkg++ko~Wh~`bz|X({!uHGz4CpFVfI7s1;uUdcucn`w zpDfY#@&5nXvr(#Bo3;yGxtFxHEc@t+8*69qno7H7E!`%vRy1VGw`D7CRi78zxU}OY z*GAQhGj|H@PR=#|_;=^z*T1JewmJ8@?dJxLX-A48I9Dgy=5(#yU|V)jPNJ{5ki{*T z=h&l|p5sqvv`80PFME_|)A;ISEy z8Ob?)xf_kY@ho;Ou;@G5X|Xn`L~`AsuRD0ZKR9Sm#kc-Z?wKR%GkV`^xy42>bhica zvn8xGkeYFaYjNyG{%`9gWZQ!atd2cQ@;UP4i5ctnM}aoI;*)P%pImG3DxvNR@AP>) zHnUwj&li1c?!t$guF@?}ToF5h(c^~sluwU*0&wElN2Cc@yI zPVdu=TGtM`Nwg^55Zs+C!FR|*Mz)-5`sPA8y`xii@NI89{kLLI(R1G#f%1Jme!bRn zcU=CGebXRnZ&5mT&h$C6>Uvk)krXaJxAK>%<-0o*=g4(g>nBt_J-7UaR!^68{leYi zi~kiae|Oz1bpN+L>-N@*oG-6SPPJ~mxOInQ`NMrXf+hBbd|my-Z1tz@`DHIWk4x9H zAH6^6a^4MY_K<&l@1DMI{eSu6ro+ae$M3(s`f>iFbM~Rff2%IP^f7cp*&R(;`JTCt z&VASsTqGr`m;Cq~E70WAx}nFV%a=Xp+c$qs+H<~tP4^JcX-tux6ti7 z(@YF^POZD!%YJCZorB?*)~jug{J7meFaGk!TlX7pF8n7yOSbPMU+d|aFMp`sT==ov zOTRp;Ub=iA?{v$l*4)V-3nC3;t{qh?O!xTLerxVTkOS3PPA_@&&d2w-w4U*!b7!7} zSp91*+vj(B+3MEIcbhUFzuuCwE6U=o?hG>&!IXcjLOtIuvNE!BaDXx$5Nkj)7c>vS Ycwl-8BQVRcL9*OAposDf6lo9#0LW;=B>(^b literal 0 HcmV?d00001 diff --git a/crates/clawhdf5-format/tests/fixtures/blosc2/shuffle_meta2.out b/crates/clawhdf5-format/tests/fixtures/blosc2/shuffle_meta2.out new file mode 100644 index 0000000000000000000000000000000000000000..da0be8caf5f846bb2d8805b1156153769c6af9e3 GIT binary patch literal 4000 zcmZ|Rdvwor9LMpYl3OlE*dca~OcY8xUEFH5%65`Dk(iS%j#A`IoE45%$(dLeCzq48 zRYO2yf*uj$$4k;3VeraX!g;T*#OC3SZ}2T*r;v%1VCC zZ@Hg8@(7Rd6wk16lZMZ!1>3R{yE2`9nZbd~WHyKMZr;O*oXp2Ki-j!Wi(JYQu3$OW zb2C3<6?gMH9$+2gYtzjAYQqlf%IMA&_SGE78yVfnwMTIrCvY|?Q|x)Gr)N!hccI=IgZhtN9<{w$>>gzUCbqXm2YqjEBFyV zVHJ0=h6i|<^*qVHc$OEqchQ~pc4wxs7yI!VMt8F89FE{vj%U1grrOgP-I;4I;3CF* zr_^4}cliOgu##VK5BIT_(H%3Tx%<_Y(Va9qoqZVHiT6&H^K8a@XN;Zf&ZEw!b0(kW ze7?k`EaNJc^IYB8?(fl^c<&^;^PB&R?wq!h-MPp;yO^DMDdW9!m3=*?! literal 0 HcmV?d00001 diff --git a/crates/clawhdf5-format/tests/fixtures/blosc2/uninit_i8.b2f b/crates/clawhdf5-format/tests/fixtures/blosc2/uninit_i8.b2f new file mode 100644 index 0000000000000000000000000000000000000000..58a2a30ca38485a13acd7aa435f63bdfd2cce945 GIT binary patch literal 172 zcmbQYBFQMNC^0vc;SvJ_L*jWL0Rn552r+~*U50R&7@!OWup$nSN)U^Q;UWVNF&?_X z23E+(3Q-RvC!b|tzsJCEhk=2Wk)4ABC<4M7FnLB815US0VFc=7gXlR26jr_gmxD0? D_(2$h literal 0 HcmV?d00001 diff --git a/crates/clawhdf5-format/tests/fixtures/blosc2/uninit_i8.out b/crates/clawhdf5-format/tests/fixtures/blosc2/uninit_i8.out new file mode 100644 index 0000000000000000000000000000000000000000..a64a5a93fb4aef4d5f63d79cb2582731b9ac5063 GIT binary patch literal 512 NcmZQz7zHCa1ONg600961 literal 0 HcmV?d00001 diff --git a/crates/clawhdf5-format/tests/fixtures/blosc2/value_f8.b2f b/crates/clawhdf5-format/tests/fixtures/blosc2/value_f8.b2f new file mode 100644 index 0000000000000000000000000000000000000000..156a4b45a5828182186c608b81325f6d55af10ca GIT binary patch literal 212 zcmbQYBFQMNC^0vc;SvJ_L*jWL0RmT+2r+~*U50SjFF+X#8X!d+AOfVE{lY~ChKmf0 zhiEY9gL=fp_C5*r)R%u literal 0 HcmV?d00001 diff --git a/crates/clawhdf5-format/tests/fixtures/blosc2/value_i4.b2f b/crates/clawhdf5-format/tests/fixtures/blosc2/value_i4.b2f new file mode 100644 index 0000000000000000000000000000000000000000..e803bc270990804c4d5d85ddf3304dd43e066e22 GIT binary patch literal 208 zcmbQYBFQMNC^0vc;SvJ_L*jWL0Rk762r+~*U50R2Hb5B+Dj-EHAOfVEWy3`VhKmf0 zhiG#K_<G!2ZVfgz9v029|$6aWAK literal 0 HcmV?d00001 diff --git a/crates/clawhdf5-format/tests/fixtures/blosc2/zero_u2.b2f b/crates/clawhdf5-format/tests/fixtures/blosc2/zero_u2.b2f new file mode 100644 index 0000000000000000000000000000000000000000..5978d316b3e51edc7adb5d00b30a9f3fe5679ecc GIT binary patch literal 172 zcmbQYBFQMNC^0vc;SvJ_L*jWL0Rn552r+~*U50S@7eE;dU`0$Il^_=Xf{P3c7a152 z-CzSNWMqY?2a=P|GO*ucV7SA;z{<$Z!2uKjVGWo(Ba8v38>cV=^{_$ooC69g-+;@( F7yz4Q8Pos( literal 0 HcmV?d00001 diff --git a/crates/clawhdf5-format/tests/fixtures/blosc2/zero_u2.out b/crates/clawhdf5-format/tests/fixtures/blosc2/zero_u2.out new file mode 100644 index 0000000000000000000000000000000000000000..1dc89f8e47e591af4c36a35046077f0ba1d1ef9d GIT binary patch literal 4000 ocmeIuF#!Mo0K%a4Pi+hzh(KY$fB^#r3>YwAz<>b*1`NCh1`nVB0RR91 literal 0 HcmV?d00001 diff --git a/crates/clawhdf5-format/tests/fixtures/blosc2/zstd_dict.b2f b/crates/clawhdf5-format/tests/fixtures/blosc2/zstd_dict.b2f new file mode 100644 index 0000000000000000000000000000000000000000..89477d8da8c9adced12097e3aca12bf7f65a733e GIT binary patch literal 4978 zcma)=c{o*T8^+f*%d8?HGG)wAQBEl&LkWpWO439ni6Y9-q)>_mO-f1w4WdpW^H@ky z5+N1MMJl1b`&nD(JLme&AK$)&uD$m1zVCBCzx#gH$@5E%R``3aUCumU7{;>(-|!mh zpCZk;@YHetXBIyAi*zYn#D{M}bh%l0EyL6@oWceH_CgLn_j-oI#8)wb&lskOVfZ;a z_`=Qbh7iM)O8>s#_mvE1fQMnCBpGHMFT+@fGfbBt!x#)p?E6Omp@V-fk!8^m% z-ZrX|2JV8NWW9?#QpCdc8~0ne{p#zNBp-bVjV>1CVD)_5oiOFq_02EdbbsdvNysRwY8yUbMLNA~7o3`$Z zjX#!r;p&at;=8qtEu9~}4DgCb$|~=POH4YOo|#ive*bavOFogIa-%i$$4|Djb)3I= zg>OJeL{!|N6KB(}XBYgw_o8J!8-gOD_8mHwd@18*eo1xR)90^0ejC7NN2-j`pD=m) zOlLRGRlXayY#+QiBOo?RUPaT;*laph6y2S?$$ne_VRtt4;~?@ z5lZTM<0qM0JGf*QmQ_D)Zs!>yH9}c^?08cPJIDDRE7k@CNA8Y0bo^{uMs`72^`oZe zuRnhqyt(^QpY=gocf`aWJ9GZ(&D_$v51zJmeE2%RFCr~Bdd%1fe^}xci+Zb1bcN&H~R>>o$f)Mjtqwl$w^6Q&?Hs(ERdkS06`6Qbtit+i;S(&1~1fn`h^j zRzG~&&Lb=(GfGW&{2vy!4)Yd!t?}O+v1{L<<2c-#d1d$No7&!d`aU2aI#f~4weR4`GilcbZ;n-2&UBjRv2tx- zXk=8}k)*SiGH(@>)zo8EpL-cW@!^VUS_YG*&9HZNM_o6D?PP?pQdMn3Q}Y@2u8WrX z_yvXU+?#Oh^o1*b-YLHK;7M!er*A*`h7294q%nB&@lcCqmi~Yjsh8nQ;4qqFbjO)m zh=ejPR~Hz0R12%tKQ|f`>)85o;f!-ZPxeX)=eS!Z@Q?lH=lD5HcRo*2OJB{$fsSuK zyF(=0IZUM>ua9Q1#?YM)Zrc6O#Rs@Aw9CaSH}Z<_J$%}R_4MhyW+s{}V{?flq-`bx>sIZv5B({X{vN?(#&+~K5@v`msT zipV=eN>&MaoNR9E*+%sKh z>-7%_-@W(Hi8B|kq35d}f#n1^QI?)2Q;42RR{3G$(Fcy4hB$8*SJXakMH~F!0aujN z^o;&M8@RYHTjL)T9t8-T!E*9Tsvb7Bzaa>SN<$3>CQ!LE%RQ(8;)EJFu{EY@TS@KP{p966bYGUG#oH~E`Ms86hq2^up59+E>?7nOq zU4a@fb{7zO=3)lKb+3;0^&R_y(SjsqmUb{zc3;u^6QOl5tNeb$vsb*LQX@eKs_RzP%^LzL9eCD zXN|~^C@rqE8Ty+*3=9lKO3%tIx=Uv(ASNxZqGdpb?=-mEJ>1SlE7hQJ-*xwK1YykT zI))&$!`y|)C&6q!NxlHq6jwcjz`2GolJW^vae>EjUm#-7zQZR|E}`YhY92MUzwP?Y z2#O7tSJNW&*}K5?A&2ecJTSssoPR@Wryvk98pnoCnaheQbX&{;RPjnyZgD023x@H7 zBP@Z~0#Qw~c643DI%U*egg7D=2o2}?jANq!IEGpd`$7nTs9=Pr&ZWZ$E0GXiy!+A% zIk16aqPeyGT=!+({=wmtyHKPX*cXhjLqJ?cQAOJj{x!?R&C7coh!vY~?9}J5D6f>${IaB(I<&dp=hT8hYs8E0WE2%e7+@n9@MlClcw+T(D#4s)>` z|4mzW#bVDC-Ak&#*N#s;{ruDdh)-y3iW$UL#3uy#>$eIksvogk3)RRdpqCJzW;wep zMSKd|9)l{twgKu|fLc%-B-hqoxP%fp62?)~IWxBqf}w-$;}w#Sg?`7Gl1)bJ+DNK4MwG$A~MGgcr}unjFN_qk?Azs*`yF~c^8zQg1`&uZmXA;^HGtrbKl}NBk|!Jt-vimq@6xEe4jj1C1an3;pld(zS&K&CrrVub0y5R65KQS+>vqRQIGEvWeq;=O{ZE_jb< zxyZ}gACnAfo_vv76GUt4`1tiFKjtfCO+8F9m=6|HMrF;JX0yUF*2iGZI9EkA9fOI} ztY$mo;L&*?#Zg2Tge1&;ZLi?oBS&k26c7bE$``R^PdrAk%QtQpg7-+e>~dt4G<1!S zq8we2sr@nTQFXAK{4zSPHwa05ARv+>d~BBUd?ek#P}1_rb7@)NeJv14w#tqx+Q{A* zO=yS$DPrMcC^H!%MgihIB0fb+XLmGs5Jk(x)2UY|;8CJ3GZTE5bO7ItB;UoX zfQ*VT@!~Z*7(q}F`xGk>&~p`dza4^3A_5j+R%oRWlY`b$Qq#dWY2!#7Wiu-FMIot- zLJ~+JGfJH@ruA&-*4r;Q3=#i$^2IAR^Ds)$tnd}1lr(&dh;3_(_WrA{8IBc)Q8nlN z8xekmA(bR6*^ig9v<-}|wUrzBxK7k&>}s2Tq!Jq^?%)!Mnfhc(+Vxu)GY2CQRZB%n z-()JQ?p1!9pe~Aj=~OL>e*Ca+01|W5OiB~$0=Go&K5zt8fGALfmRIk-^q~siwXWer zjIu5Zm-+Z%aG_>KIjI}rLqrtO!GUZ#=43>_K;j>wD)?7WK~ELlVj!bIK#c}6_|QU6 zHYHJ8jSiV2xQf-hTu`8}wH;EMP>(zTmGrV-I3E z*KV-$e_IE-0qQcKs7lWr{DP3`Ffu@X6r<3b?|TM#he+bNgDz4X21wXq0C#XX%ci8N z`lc6e$reRPUHZmTt#Hhs426Bv;XF`ElPFCMQ?^wqfKejj3*3QR-r9vR;L8U;a`LKSxjq53uMV7Ng0v b8U~NK{^ymz!ILim?vt--ysO&qKbQOmmA}MC literal 0 HcmV?d00001 diff --git a/crates/clawhdf5-format/tests/fixtures/blosc2/zstd_dict.err b/crates/clawhdf5-format/tests/fixtures/blosc2/zstd_dict.err new file mode 100644 index 0000000..658c6ba --- /dev/null +++ b/crates/clawhdf5-format/tests/fixtures/blosc2/zstd_dict.err @@ -0,0 +1 @@ +dictionar \ No newline at end of file diff --git a/crates/clawhdf5/Cargo.toml b/crates/clawhdf5/Cargo.toml index a6401fe..1eca373 100644 --- a/crates/clawhdf5/Cargo.toml +++ b/crates/clawhdf5/Cargo.toml @@ -47,8 +47,9 @@ lzf = ["clawhdf5-format/lzf"] bitshuffle = ["clawhdf5-format/bitshuffle"] bzip2 = ["clawhdf5-format/bzip2"] blosc = ["clawhdf5-format/blosc"] +blosc2 = ["clawhdf5-format/blosc2"] # Every plugin filter. -plugin-filters = ["lzf", "bitshuffle", "bzip2", "blosc"] +plugin-filters = ["lzf", "bitshuffle", "bzip2", "blosc", "blosc2"] # Dataset::verify_provenance() — recompute a dataset's SHA-256 and compare # against its stored _provenance_sha256 attribute. On by default, matching # clawhdf5-format's own default-on `provenance` feature. diff --git a/crates/clawhdf5/tests/plugin_filters_interop.rs b/crates/clawhdf5/tests/plugin_filters_interop.rs index 81cc87c..00911c6 100644 --- a/crates/clawhdf5/tests/plugin_filters_interop.rs +++ b/crates/clawhdf5/tests/plugin_filters_interop.rs @@ -77,7 +77,7 @@ except ImportError: hdf5plugin = None path = sys.argv[1] FILTERS = eval('(' + sys.argv[2] + ')') -cases = [ +cases = eval('(' + sys.argv[3] + ')') if len(sys.argv) > 3 else [ (') { let dir = tempfile::tempdir().unwrap(); let path = dir.path().join(format!("{tag}.h5")); - let n: usize = run_python(GENERATE, &[path.to_str().unwrap(), filters]) - .parse() - .unwrap(); + let mut args = vec![path.to_str().unwrap(), filters]; + args.extend(cases); + let n: usize = run_python(GENERATE, &args).parse().unwrap(); assert!(n > 0); let file = File::open(&path).unwrap(); for i in 0..n { @@ -353,6 +363,110 @@ fn blosc_written_by_clawhdf5_reads_in_hdf5plugin() { } } +#[cfg(feature = "blosc2")] +#[test] +fn blosc2_written_by_hdf5plugin_reads_exactly() { + if !have_python("h5py, hdf5plugin") { + return; + } + // Every codec hdf5plugin's Blosc2 offers, each lossless filter, and + // levels from "store" to maximum. Truncating precision is lossy, so it + // is checked separately against h5py's own reading. + check_h5py_written( + "blosc2", + r#"[(f'{c} {f} {l}', hdf5plugin.Blosc2(cname=c, clevel=l, filters=f)) + for c in ['blosclz', 'lz4', 'lz4hc', 'zlib', 'zstd'] + for f, l in [(hdf5plugin.Blosc2.NOFILTER, 5), + (hdf5plugin.Blosc2.SHUFFLE, 9), + (hdf5plugin.Blosc2.BITSHUFFLE, 1), + (hdf5plugin.Blosc2.DELTA, 5)]] + + [('blosclz level 0', hdf5plugin.Blosc2(cname='blosclz', clevel=0)), + ('zstd level 0 bitshuffle', + hdf5plugin.Blosc2(cname='zstd', clevel=0, filters=hdf5plugin.Blosc2.BITSHUFFLE))]"#, + ); +} + +/// Blosc2 over more shapes and dtypes: every integer and float width, +/// 1-D to 5-D chunks (B2ND arrays from 2-D on) with partial edge chunks and +/// block shapes that pad the chunk, datasets of zeros, of one repeated value +/// and of NaN (Blosc2's "special" chunks), and Fletcher32 before Blosc2 +/// (which makes hdf5-blosc2 fall back from B2ND to a plain frame). +#[cfg(feature = "blosc2")] +#[test] +fn blosc2_shapes_and_special_chunks_read_exactly() { + if !have_python("h5py, hdf5plugin") { + return; + } + let cases = r#"[(dt, shape, chunks, kind) + for dt in ['u2', 'f8'] + for shape, chunks in [((777,), (100,)), + ((37, 53), (10, 16)), + ((9, 10, 11), (4, 5, 3)), + ((6, 7, 5, 9), (3, 2, 5, 4))] + for kind in ['ramp', 'noise']] + + [(' Date: Sat, 26 Sep 2026 10:22:52 -0500 Subject: [PATCH 2/9] docs: Blosc2 reads (read only); ZFP is the one plugin filter left Record the `blosc2` feature in the changelog, the README's feature table and the crate table, and mark the Blosc2 half of the known "Filters" issue fixed (dated, with the conformance run that shows h5ex_d_blosc2 reading). What stays open: ZFP, writing Blosc2, and the Blosc2 features hdf5plugin never writes (dictionaries, lazy chunks, variable-length blocks, user-defined codecs and registered filters), which are errors. Co-Authored-By: Claude Opus 5.5 (1M context) --- CHANGELOG.md | 21 +++++++++++++++++++++ CLAUDE.md | 2 +- README.md | 16 +++++++++------- docs/known-issues.md | 15 ++++++++++----- 4 files changed, 41 insertions(+), 13 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 29804fc..aebeb93 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,26 @@ ## Unreleased +### Blosc2 (2026-09-26) +- **Blosc2 (filter 32026) reads, in pure Rust.** Files written with + hdf5plugin's `Blosc2` failed with `UnsupportedFilter`. New feature + `blosc2` (`clawhdf5-format` and `clawhdf5`, included in `plugin-filters`) + decodes the Blosc2 contiguous frame hdf5-blosc2 stores per chunk, the + B2ND arrays it uses for chunks of 2 or more dimensions (blocks gathered + back into C order), Blosc2 chunks with their special values (zeros, NaN, + uninitialised, one repeated value), and the shuffle, bit-shuffle, delta + and truncate-precision filters, over the BloscLZ, LZ4/LZ4HC, Zlib and + Zstandard codecs shared with Blosc 1. Read only: there is no Blosc2 + encoder. Dictionaries, lazy chunks, variable-length blocks, user-defined + codecs and registered Blosc2 filters (e.g. bytedelta) are errors; + uninitialised chunks read as zeros. Tested against h5py 3.16 + + hdf5plugin 7.1 (every codec, filter and level 0-9; 1- to 5-D chunks with + partial edge chunks; every integer width and f4/f8; datasets of zeros, + one value and NaN; Fletcher32 before Blosc2) and against frames from + python-blosc2 4.13.1 for what hdf5plugin never writes + (`crates/clawhdf5-format/tests/fixtures/blosc2/`); the decoder is + fuzzed. Conformance: 576 of 697 files ok (was 575) — h5ex_d_blosc2. + ### Concurrent reads (2026-09-26) - **Full reads of chunked datasets scale with threads again when rayon's pool has one thread.** Each full read handed its chunks to rayon to @@ -372,6 +392,7 @@ missing feature ("unsupported filter: 32026 (Blosc2, not implemented by clawhdf5)"). - **Not implemented:** Blosc2 (32026) and ZFP (32013) remain a clear error. + (Blosc2 reads since the `blosc2` feature, see above.) - **Wrong data: a chunk that decodes short read as zeros** (pre-existing, every filter). HDF5 stores every chunk at the full chunk size, so a filter pipeline that decodes to fewer bytes means a corrupt chunk; every chunk diff --git a/CLAUDE.md b/CLAUDE.md index 1cb7aba..b029c5d 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -11,7 +11,7 @@ Cargo workspace with 18 crates under `crates/` (plus `libaec-sys`, an internal F |-------|------| | `clawhdf5-format` | HDF5 binary spec parser (superblock, B-tree, heap) — also holds shared type definitions and physical constants | | `clawhdf5-io` | Read/write implementation | -| `clawhdf5-filters` | Deflate backends (zlib-rs, zlib-ng, Apple Compression); the HDF5 filter pipeline, the filter registry (`clawhdf5_format::filter_registry`) and the other codecs (LZ4, Zstd, SZIP, N-Bit, scale-offset, pcodec, and the pure-Rust plugin filters LZF, bitshuffle, bzip2, Blosc 1) live in `clawhdf5-format`. No Blosc2 or ZFP. | +| `clawhdf5-filters` | Deflate backends (zlib-rs, zlib-ng, Apple Compression); the HDF5 filter pipeline, the filter registry (`clawhdf5_format::filter_registry`) and the other codecs (LZ4, Zstd, SZIP, N-Bit, scale-offset, pcodec, and the pure-Rust plugin filters LZF, bitshuffle, bzip2, Blosc 1, and Blosc2 read-only) live in `clawhdf5-format`. No ZFP. | | `clawhdf5-derive` | Proc-macro derive for HDF5-serializable structs | | `clawhdf5` | Main facade crate | | `clawhdf5-netcdf4` | NetCDF-4 compatibility layer | diff --git a/README.md b/README.md index 4ae4d1c..25f43a7 100644 --- a/README.md +++ b/README.md @@ -668,7 +668,7 @@ clawhdf5 workspace (17 crates, ~86K lines of Rust in src/, ~104K with tests ├── Core HDF5 │ ├── clawhdf5-format — Binary parser/writer (no_std-capable), shared type definitions │ ├── clawhdf5-io — I/O abstraction (file/memory readers; optional mmap, async, HSDS, MPI) -│ ├── clawhdf5-filters — Fast deflate path (zlib-ng); the filter registry and the lz4/zstd/pcodec/szip/LZF/bitshuffle/bzip2/Blosc filters live in clawhdf5-format +│ ├── clawhdf5-filters — Fast deflate path (zlib-ng); the filter registry and the lz4/zstd/pcodec/szip/LZF/bitshuffle/bzip2/Blosc/Blosc2 filters live in clawhdf5-format │ ├── clawhdf5-derive — Proc macros │ ├── clawhdf5 — High-level API │ ├── clawhdf5-netcdf4 — NetCDF-4 support @@ -779,13 +779,15 @@ stores keep their setting. Opt out with `float16 = false` or | `bitshuffle` | no | Bitshuffle filter (id 32008) with its LZ4 and Zstandard modes: read and write. Pure Rust (lz4_flex, ruzstd) | | `bzip2` | no | bzip2 filter (id 307): read and write. Pure Rust (the `bzip2` crate's libbz2-rs-sys backend compiles no C) | | `blosc` | no | Blosc 1 filter (id 32001): reads BloscLZ, LZ4/LZ4HC, Snappy, Zlib and Zstandard frames with byte or bit shuffle; writes LZ4, Snappy, Zlib or Zstandard (not BloscLZ). Pure Rust | -| `plugin-filters` | no | All four above | +| `blosc2` | no | Blosc2 filter (id 32026), read only: hdf5plugin's frames and B2ND (n-D) chunks, BloscLZ, LZ4/LZ4HC, Zlib and Zstandard, with shuffle, bit shuffle, delta or truncated precision. Pure Rust | +| `plugin-filters` | no | All five above | -Blosc2 (32026) and ZFP (32013) are not implemented: reading them fails with -`UnsupportedFilter`, whose message names the filter. Any other filter can be -supplied at run time with `filter_registry::register_filter` (a decoder -closure, or a `FilterCodec` that also encodes). The facade (`clawhdf5`) -forwards `lzf`, `bitshuffle`, `bzip2`, `blosc` and `plugin-filters`. Write +ZFP (32013) is not implemented: reading it fails with `UnsupportedFilter`, +whose message names the filter. clawhdf5 cannot write Blosc2. Any other +filter can be supplied at run time with `filter_registry::register_filter` (a +decoder closure, or a `FilterCodec` that also encodes). The facade +(`clawhdf5`) forwards `lzf`, `bitshuffle`, `bzip2`, `blosc`, `blosc2` and +`plugin-filters`. Write with `DatasetBuilder::with_lzf()`, `with_bitshuffle(..)`, `with_bzip2(..)` and `with_blosc(..)`; h5py + hdf5plugin read the result (tested both ways in `crates/clawhdf5/tests/plugin_filters_interop.rs`). The pure-Rust Zstandard diff --git a/docs/known-issues.md b/docs/known-issues.md index 2657cdf..2c42b2f 100644 --- a/docs/known-issues.md +++ b/docs/known-issues.md @@ -210,11 +210,16 @@ fill-value item that did is fixed). bitshuffle, bzip2 and Blosc 1 (`bitshuffle`, `bzip2`, `blosc`, or `plugin-filters` for all), read and write, pure Rust; h5ex_d_lzf, h5ex_d_bshuf, h5ex_d_bzip2 and h5ex_d_blosc now read (conformance 573 of - 697 ok). **Still open:** Blosc2 (32026 — hdf5plugin stores each chunk as a - Blosc2 super-chunk frame, and n-D chunks as B2ND arrays) and ZFP (32013); - both fail with an `UnsupportedFilter` error that names the filter, and - either can be plugged in with `filter_registry::register_filter` (32023, - Granular BitRound, too, since 2026-09-26 even with the `pcodec` feature). + 697 ok). **Fixed 2026-09-26** for Blosc2 (32026, `blosc2` feature, also + in `plugin-filters`), read only: hdf5plugin's frames and B2ND arrays, every + codec and filter it offers; h5ex_d_blosc2 now reads (conformance 576 of + 697 ok, tank, `conformance/run.sh --no-fetch`). Blosc2 frames using + dictionaries, lazy chunks, variable-length blocks, user-defined codecs or + registered filters (e.g. bytedelta) are refused with an error. + **Still open:** ZFP (32013) fails with an `UnsupportedFilter` error that + names the filter, and can be plugged in with + `filter_registry::register_filter` (32023, Granular BitRound, too, since + 2026-09-26 even with the `pcodec` feature). clawhdf5 cannot write Blosc2. - **Wrong data: a chunk whose filters decode to fewer bytes than the chunk read with zeros for the missing bytes** (any filter; found reviewing the plugin filters). **Fixed 2026-09-26:** it is an error naming the chunk. A corrupt From 989335b67b5da5f9ace23783320c87e891b4a432 Mon Sep 17 00:00:00 2001 From: osobh Date: Sat, 26 Sep 2026 11:26:28 -0500 Subject: [PATCH 3/9] fix(format): bound a Blosc2 frame's offsets chunk by the HDF5 chunk size parse_frame sized the offsets chunk from the frame header's own nbytes and chunksize, so a 173-byte frame declaring 32 Mi chunks, with a 40-byte repeated-value offsets chunk, built 256 MiB (up to 2 GiB) of offsets for a 1 MiB HDF5 chunk and then returned 4 bytes. The offsets chunk is now capped at the output limit (at least 128 bytes); a frame whose nbytes/chunksize imply more chunks than that is refused before anything is allocated. tests/blosc2_alloc_bounds.rs measures peak allocation with a counting global allocator; the reviewer's frame failed it (decoded Ok(4)) before. Co-Authored-By: Claude Opus 5.5 (1M context) --- crates/clawhdf5-format/src/filters_blosc2.rs | 38 ++-- .../tests/blosc2_alloc_bounds.rs | 191 ++++++++++++++++++ 2 files changed, 213 insertions(+), 16 deletions(-) create mode 100644 crates/clawhdf5-format/tests/blosc2_alloc_bounds.rs diff --git a/crates/clawhdf5-format/src/filters_blosc2.rs b/crates/clawhdf5-format/src/filters_blosc2.rs index 5f332f8..d2b1ef0 100644 --- a/crates/clawhdf5-format/src/filters_blosc2.rs +++ b/crates/clawhdf5-format/src/filters_blosc2.rs @@ -451,7 +451,9 @@ struct Frame<'a> { offsets: Vec, } -fn parse_frame(buf: &[u8]) -> Result, FormatError> { +/// Parse a frame's header and decode its offsets chunk, holding no more +/// than an HDF5 chunk of `limit` bytes needs. +fn parse_frame(buf: &[u8], limit: usize) -> Result, FormatError> { if buf.len() < FRAME_HEADER_MINLEN { return Err(err("truncated frame header")); } @@ -497,29 +499,33 @@ fn parse_frame(buf: &[u8]) -> Result, FormatError> { .ok_or_else(|| err("chunks run past the frame"))?; let nbytes = usize::try_from(nbytes).map_err(|_| err("bad decoded size"))?; let chunksize = chunksize as usize; - // The offsets chunk follows the data chunks. + // The offsets chunk follows the data chunks: one `i64` per chunk. The + // frame header's sizes are the file's word, so they must not size it: + // it may be no larger than the HDF5 chunk (`limit`, at least 128 + // bytes), i.e. one Blosc2 chunk per 8 bytes of output. hdf5-blosc2 + // writes one chunk per frame; only python-blosc2 arrays of tiny + // chunks come near the cap. let off_src = &buf[data_end..]; - let expected = if nbytes == 0 { + let max_offsets = limit.max(128); + let off_len = if nbytes == 0 { 0 } else if chunksize > 0 { - nbytes.div_ceil(chunksize) + nbytes + .div_ceil(chunksize) + .checked_mul(8) + .filter(|&n| n <= max_offsets) + .ok_or_else(|| err("frame has more chunks than the HDF5 chunk can hold"))? } else { // Variable-size chunks: the offsets chunk tells how many. - usize::MAX - }; - let off_limit = if expected == usize::MAX { - 1 << 24 - } else { - expected - .checked_mul(8) - .ok_or_else(|| err("too many chunks"))? + max_offsets }; let offsets = if nbytes == 0 { Vec::new() } else { - blosc2_decompress_chunk(off_src, off_limit)? + blosc2_decompress_chunk(off_src, off_len)? }; - if !offsets.len().is_multiple_of(8) || (expected != usize::MAX && offsets.len() != off_limit) { + let expected = if chunksize > 0 { off_len } else { offsets.len() }; + if !offsets.len().is_multiple_of(8) || offsets.len() != expected { return Err(err("offsets chunk does not match the number of chunks")); } Ok(Frame { @@ -711,7 +717,7 @@ fn decode_frame( limit: usize, cd_shape: Option<&[usize]>, ) -> Result, FormatError> { - let frame = parse_frame(input)?; + let frame = parse_frame(input, limit)?; let meta = match frame.metalayer(b"b2nd")? { Some(m) => Some(m), None => frame.metalayer(b"caterva")?, @@ -1065,7 +1071,7 @@ mod tests { if w.is_err() { continue; } - let frame = parse_frame(&f).unwrap(); + let frame = parse_frame(&f, 1 << 20).unwrap(); let raw: [u8; 8] = frame.offsets[..8].try_into().unwrap(); let off = i64::from_le_bytes(raw); if off >= 0 { diff --git a/crates/clawhdf5-format/tests/blosc2_alloc_bounds.rs b/crates/clawhdf5-format/tests/blosc2_alloc_bounds.rs new file mode 100644 index 0000000..11e7812 --- /dev/null +++ b/crates/clawhdf5-format/tests/blosc2_alloc_bounds.rs @@ -0,0 +1,191 @@ +//! Crafted Blosc2 frames and chunks cannot make the decoder allocate out of +//! proportion to the HDF5 chunk it decodes. +//! +//! A frame's header, its offsets chunk and its chunk headers all declare +//! sizes, and the decoder used to allocate what they declared: a 173-byte +//! frame whose offsets chunk claimed 2 GiB was decoded in full before any +//! check failed. Every allocation is now bounded by the output limit (the +//! HDF5 chunk's size) and the input's length. +//! +//! Peak heap use is measured with a counting global allocator; the tests +//! share it, so each holds `SERIAL` for its whole run. +#![cfg(feature = "blosc2")] + +use std::alloc::{GlobalAlloc, Layout, System}; +use std::sync::Mutex; +use std::sync::atomic::{AtomicUsize, Ordering}; + +use clawhdf5_format::filters_blosc2::{blosc2_decompress, blosc2_decompress_chunk}; + +struct Counting; + +static CURRENT: AtomicUsize = AtomicUsize::new(0); +static PEAK: AtomicUsize = AtomicUsize::new(0); +static SERIAL: Mutex<()> = Mutex::new(()); + +unsafe impl GlobalAlloc for Counting { + unsafe fn alloc(&self, layout: Layout) -> *mut u8 { + let p = unsafe { System.alloc(layout) }; + if !p.is_null() { + let now = CURRENT.fetch_add(layout.size(), Ordering::Relaxed) + layout.size(); + PEAK.fetch_max(now, Ordering::Relaxed); + } + p + } + + unsafe fn alloc_zeroed(&self, layout: Layout) -> *mut u8 { + let p = unsafe { System.alloc_zeroed(layout) }; + if !p.is_null() { + let now = CURRENT.fetch_add(layout.size(), Ordering::Relaxed) + layout.size(); + PEAK.fetch_max(now, Ordering::Relaxed); + } + p + } + + unsafe fn dealloc(&self, ptr: *mut u8, layout: Layout) { + unsafe { System.dealloc(ptr, layout) }; + CURRENT.fetch_sub(layout.size(), Ordering::Relaxed); + } +} + +#[global_allocator] +static ALLOC: Counting = Counting; + +/// Bytes allocated at the peak of `f`, above what was live when it started. +fn peak_during(f: impl FnOnce() -> T) -> (T, usize) { + let base = CURRENT.load(Ordering::Relaxed); + PEAK.store(base, Ordering::Relaxed); + let out = f(); + (out, PEAK.load(Ordering::Relaxed).saturating_sub(base)) +} + +/// What decoding one HDF5 chunk of `limit` bytes from `input` may hold at +/// once: the output, a few blocks of scratch (each no larger than the +/// output), the offsets table, and the Zstandard decoder's state. +fn bound(limit: usize, input: &[u8]) -> usize { + 6 * limit + 2 * input.len() + (1 << 20) +} + +fn lock() -> std::sync::MutexGuard<'static, ()> { + SERIAL.lock().unwrap_or_else(|e| e.into_inner()) +} + +/// A 32-byte (extended) Blosc2 chunk header. +fn chunk_header(ts: u8, nbytes: i32, blocksize: i32, cbytes: i32, special: u8) -> Vec { + let mut c = vec![5u8, 1, 0x05, ts]; + for v in [nbytes, blocksize, cbytes] { + c.extend_from_slice(&v.to_le_bytes()); + } + c.resize(32, 0); + c[31] = special << 4; + c +} + +/// A chunk of `nbytes` bytes that repeats one value (special type 3). +fn repeated(value: &[u8], nbytes: i32, blocksize: i32) -> Vec { + let mut c = chunk_header(value.len() as u8, nbytes, blocksize, 32 + value.len() as i32, 3); + c.extend_from_slice(value); + c +} + +/// A frame offset recording a special chunk of `kind` (1 zeros, 2 NaN). +fn special_offset(kind: u8) -> [u8; 8] { + (((0x80 | kind) as i64) << 56).to_le_bytes() +} + +/// A B2ND metalayer. +fn nd_meta(shape: &[i64], chunks: &[i32], blocks: &[i32]) -> Vec { + let n = shape.len() as u8; + let mut m = vec![0x95, 0, n, 0x90 | n]; + for s in shape { + m.push(0xd3); + m.extend_from_slice(&s.to_be_bytes()); + } + for dims in [chunks, blocks] { + m.push(0x90 | n); + for d in dims { + m.push(0xd2); + m.extend_from_slice(&d.to_be_bytes()); + } + } + m +} + +/// A contiguous frame: header (with a `b2nd` metalayer if given), the data +/// chunks, then the offsets chunk. +fn frame( + meta: Option<&[u8]>, + nbytes: i64, + typesize: i32, + chunksize: i32, + data: &[u8], + offsets: &[u8], +) -> Vec { + let mut h = vec![0u8; 91]; + h[0] = 0x9e; + h[1] = 0xa8; + h[2..10].copy_from_slice(b"b2frame\0"); + h[25] = 2; + match meta { + Some(m) => { + h.extend_from_slice(&[0xde, 0, 1, 0xa4]); + h.extend_from_slice(b"b2nd"); + let at = h.len() as i32 + 5; + h.push(0xd2); + h.extend_from_slice(&at.to_be_bytes()); + h.push(0xc6); + h.extend_from_slice(&(m.len() as u32).to_be_bytes()); + h.extend_from_slice(m); + } + None => h.extend_from_slice(&[0xde, 0, 0]), + } + let header_len = h.len() as i32; + h[11..15].copy_from_slice(&header_len.to_be_bytes()); + h[30..38].copy_from_slice(&nbytes.to_be_bytes()); + h[39..47].copy_from_slice(&(data.len() as i64).to_be_bytes()); + h[48..52].copy_from_slice(&typesize.to_be_bytes()); + h[58..62].copy_from_slice(&chunksize.to_be_bytes()); + h.extend_from_slice(data); + h.extend_from_slice(offsets); + let len = h.len() as u64; + h[16..24].copy_from_slice(&len.to_be_bytes()); + h +} + +/// The frame header's own sizes must not size the offsets chunk: a frame +/// declaring 32 Mi chunks of 4 bytes, whose offsets chunk (40 bytes) says +/// "one repeated offset, 256 MiB of them", made the decoder build all +/// 256 MiB of offsets for a 1 MiB HDF5 chunk and then return 4 bytes. +#[test] +fn offsets_chunk_is_bounded_by_the_output_limit() { + let _g = lock(); + let limit = 1 << 20; + let offsets_len: i32 = 256 << 20; + let nchunks = offsets_len as i64 / 8; + let offsets = repeated(&special_offset(1), offsets_len, 64 << 20); + let f = frame(None, nchunks * 4, 4, 4, &[], &offsets); + let (r, peak) = peak_during(|| blosc2_decompress(&f, limit)); + assert!(r.is_err(), "decoded {:?} bytes", r.map(|v| v.len())); + assert!( + peak <= bound(limit, &f), + "peak {peak} bytes for a {}-byte frame", + f.len() + ); + // The same frame with a variable chunk size (0): the offsets chunk + // alone says how many chunks there are. + let f = frame(None, nchunks * 4, 4, 0, &[], &offsets); + let (r, peak) = peak_during(|| blosc2_decompress(&f, limit)); + assert!(r.is_err()); + assert!(peak <= bound(limit, &f), "chunksize 0: peak {peak} bytes"); +} + +/// A legitimate frame of this shape (one chunk, its offset special) still +/// decodes. +#[test] +fn small_frames_still_decode() { + let _g = lock(); + let offsets = repeated(&special_offset(1), 8, 8); + let f = frame(None, 64, 4, 64, &[], &offsets); + assert_eq!(blosc2_decompress(&f, 64).unwrap(), vec![0; 64]); + let _ = blosc2_decompress_chunk; +} From e05530a80519153f51d2c6845934a4bf4410e0c2 Mon Sep 17 00:00:00 2001 From: osobh Date: Sat, 26 Sep 2026 11:27:06 -0500 Subject: [PATCH 4/9] fix(format): an empty Blosc2 chunk no longer allocates its block size A chunk header with nbytes 0 and no special type kept its declared block size (up to 512 MiB): the block size was clamped to nbytes only when nbytes was positive, and the scratch blocks were allocated before the (empty) block loop, so a 20-byte chunk allocated about 1 GiB. The block size is now clamped to nbytes always, and an empty chunk returns before any scratch is allocated. A frame chunk must also decode to the size the frame header gives it (chunksize, or the remainder for the last chunk), and is decoded with that as its limit, so an empty chunk in a frame for a non-empty HDF5 chunk is an error rather than an empty result. Co-Authored-By: Claude Opus 5.5 (1M context) --- crates/clawhdf5-format/src/filters_blosc2.rs | 39 +++++++++++++------ .../tests/blosc2_alloc_bounds.rs | 33 +++++++++++++++- 2 files changed, 59 insertions(+), 13 deletions(-) diff --git a/crates/clawhdf5-format/src/filters_blosc2.rs b/crates/clawhdf5-format/src/filters_blosc2.rs index d2b1ef0..812ff24 100644 --- a/crates/clawhdf5-format/src/filters_blosc2.rs +++ b/crates/clawhdf5-format/src/filters_blosc2.rs @@ -209,7 +209,10 @@ fn read_header(src: &[u8]) -> Result { if flags2 & VL_BLOCKS != 0 { return Err(err("variable-length blocks are not supported")); } - if nbytes > 0 && blocksize > nbytes { + // c-blosc2 clamps only a non-empty chunk's block size; clamping an + // empty one too keeps its scratch blocks from being sized by the + // header (up to 512 MiB each). + if blocksize > nbytes { blocksize = nbytes; } h.blocksize = blocksize; @@ -234,9 +237,6 @@ pub fn blosc2_decompress_chunk(src: &[u8], limit: usize) -> Result, Form if memcpyed && h.cbytes != h.nbytes + h.overhead { return Err(err("stored chunk has the wrong size")); } - if h.nbytes == 0 && h.cbytes == h.overhead && h.special == 0 { - return Ok(Vec::new()); - } let nbytes = h.nbytes; let mut out = vec![0u8; nbytes]; if h.special != 0 { @@ -288,6 +288,9 @@ pub fn blosc2_decompress_chunk(src: &[u8], limit: usize) -> Result, Form return Err(err(&format!("filter {f} is not supported"))); } } + if nbytes == 0 { + return Ok(out); + } let blocksize = h.blocksize; let nblocks = nbytes.div_ceil(blocksize); let leftover = nbytes % blocksize; @@ -524,7 +527,11 @@ fn parse_frame(buf: &[u8], limit: usize) -> Result, FormatError> { } else { blosc2_decompress_chunk(off_src, off_len)? }; - let expected = if chunksize > 0 { off_len } else { offsets.len() }; + let expected = if chunksize > 0 { + off_len + } else { + offsets.len() + }; if !offsets.len().is_multiple_of(8) || offsets.len() != expected { return Err(err("offsets chunk does not match the number of chunks")); } @@ -548,15 +555,18 @@ impl Frame<'_> { } let raw: [u8; 8] = self.offsets[8 * n..8 * n + 8].try_into().unwrap(); let offset = i64::from_le_bytes(raw); + // Every chunk but the last holds `chunksize` bytes. + let size = if self.chunksize == 0 { + None + } else if n == self.nchunks - 1 && !self.nbytes.is_multiple_of(self.chunksize) { + Some(self.nbytes % self.chunksize) + } else { + Some(self.chunksize) + }; if offset < 0 { // A special chunk, recorded in the offset's top byte. - if self.chunksize == 0 { + let Some(size) = size else { return Err(err("special chunk in a frame without a chunk size")); - } - let size = if n == self.nchunks - 1 && !self.nbytes.is_multiple_of(self.chunksize) { - self.nbytes % self.chunksize - } else { - self.chunksize }; if size > limit { return Err(err("decoded size exceeds the limit")); @@ -586,7 +596,12 @@ impl Frame<'_> { .and_then(|o| self.header_len.checked_add(o)) .filter(|&s| s < end && end - s >= MIN_HEADER) .ok_or_else(|| err("chunk offset out of range"))?; - blosc2_decompress_chunk(&self.buf[start..end], limit) + let data = + blosc2_decompress_chunk(&self.buf[start..end], limit.min(size.unwrap_or(limit)))?; + if size.is_some_and(|s| s != data.len()) { + return Err(err("chunk size does not match the frame's chunk size")); + } + Ok(data) } /// The content of metalayer `name`, if the frame has it. diff --git a/crates/clawhdf5-format/tests/blosc2_alloc_bounds.rs b/crates/clawhdf5-format/tests/blosc2_alloc_bounds.rs index 11e7812..833cd53 100644 --- a/crates/clawhdf5-format/tests/blosc2_alloc_bounds.rs +++ b/crates/clawhdf5-format/tests/blosc2_alloc_bounds.rs @@ -83,7 +83,13 @@ fn chunk_header(ts: u8, nbytes: i32, blocksize: i32, cbytes: i32, special: u8) - /// A chunk of `nbytes` bytes that repeats one value (special type 3). fn repeated(value: &[u8], nbytes: i32, blocksize: i32) -> Vec { - let mut c = chunk_header(value.len() as u8, nbytes, blocksize, 32 + value.len() as i32, 3); + let mut c = chunk_header( + value.len() as u8, + nbytes, + blocksize, + 32 + value.len() as i32, + 3, + ); c.extend_from_slice(value); c } @@ -189,3 +195,28 @@ fn small_frames_still_decode() { assert_eq!(blosc2_decompress(&f, 64).unwrap(), vec![0; 64]); let _ = blosc2_decompress_chunk; } + +/// A chunk that decodes to nothing kept its declared block size (up to +/// 512 MiB) and allocated two scratch blocks of it: about 1 GiB for a +/// 20-byte chunk. +#[test] +fn empty_chunk_does_not_allocate_its_block_size() { + let _g = lock(); + let mut c = vec![5u8, 1, 0x01, 1]; + for v in [0i32, 0x1FFF_F000, 20] { + c.extend_from_slice(&v.to_le_bytes()); + } + c.resize(20, 0); + let (r, peak) = peak_during(|| blosc2_decompress_chunk(&c, 1 << 20)); + assert_eq!(r.map(|v| v.len()).unwrap_or(0), 0); + assert!( + peak <= bound(0, &c), + "peak {peak} bytes for a 20-byte chunk" + ); + // Inside a frame for a non-empty HDF5 chunk it is an error, not data. + let offsets = repeated(&0i64.to_le_bytes(), 8, 8); + let f = frame(None, 64, 4, 64, &c, &offsets); + let (r, peak) = peak_during(|| blosc2_decompress(&f, 64)); + assert!(r.is_err(), "decoded {:?}", r.map(|v| v.len())); + assert!(peak <= bound(64, &f), "in a frame: peak {peak} bytes"); +} From 22dc87b07c810665a7825a6b2a14488b89e3d54d Mon Sep 17 00:00:00 2001 From: osobh Date: Sat, 26 Sep 2026 11:30:39 -0500 Subject: [PATCH 5/9] fix(format): never hold a B2ND chunk's padding B2ND chunks were decoded whole, padding included, with up to 16x the HDF5 chunk size as their limit, so a crafted frame made each chunk allocate and fill up to 16x the output (4 GiB for a 256 MiB HDF5 chunk). The padding is the real bound (prod(ceil(c/b)*b) per chunk), but that can be 2^ndim times the array, so it is no longer held at all: - A Blosc2 chunk is now decoded block by block (decode_blocks), each block handed to a sink as it is ready, with at most three blocks of scratch. blosc2_decompress_chunk and plain frames still collect every block. - reassemble places each B2ND block straight into the output and skips blocks that are all padding (they are not decoded unless the delta filter needs the first block). A frame's NaN chunks are handed over one B2ND block at a time and its zero chunks cost nothing. - A B2ND chunk must decode to exactly its padded size, its Blosc2 blocks must be whole B2ND blocks no larger than the output, and a chunk may not be larger than the array (hdf5-blosc2's chunk is the array), so a block is never larger than the output. Peak allocation for a 10-D array padded to 13x (NaN, repeated-value and stored-block chunks) and for a 16x chunk was 4.5 MB and 17.8 MB for 315 KB and 1 MiB outputs before, and is now within the tests' bound. Blosc2 files written by hdf5plugin in 9-D and 12-D, an 8 MiB single chunk, 1x1x1 and edge-chunk shapes still read exactly. Co-Authored-By: Claude Opus 5.5 (1M context) --- crates/clawhdf5-format/src/filters_blosc2.rs | 377 +++++++++++++----- .../tests/blosc2_alloc_bounds.rs | 128 ++++++ 2 files changed, 408 insertions(+), 97 deletions(-) diff --git a/crates/clawhdf5-format/src/filters_blosc2.rs b/crates/clawhdf5-format/src/filters_blosc2.rs index 812ff24..71ad4b6 100644 --- a/crates/clawhdf5-format/src/filters_blosc2.rs +++ b/crates/clawhdf5-format/src/filters_blosc2.rs @@ -225,20 +225,59 @@ fn read_header(src: &[u8]) -> Result { /// Decompress one Blosc2 chunk (header included), refusing to produce more /// than `limit` bytes. pub fn blosc2_decompress_chunk(src: &[u8], limit: usize) -> Result, FormatError> { + let (h, src) = open_chunk(src, limit)?; + let mut out = vec![0u8; h.nbytes]; + decode_blocks(&h, src, &mut |_, _| true, &mut |at, block| { + out[at..at + block.len()].copy_from_slice(block); + })?; + Ok(out) +} + +/// Read and check a chunk's header: its decoded size must be within +/// `limit`. Returns the header and the chunk's bytes. +fn open_chunk(src: &[u8], limit: usize) -> Result<(ChunkHeader, &[u8]), FormatError> { let h = read_header(src)?; if h.nbytes > limit { return Err(err("decoded size exceeds the limit")); } - let src = &src[..h.cbytes]; if h.special > SPECIAL_UNINIT { return Err(err(&format!("unknown special chunk type {}", h.special))); } - let memcpyed = h.flags & FLAG_MEMCPYED != 0; - if memcpyed && h.cbytes != h.nbytes + h.overhead { + if h.flags & FLAG_MEMCPYED != 0 && h.cbytes != h.nbytes + h.overhead { return Err(err("stored chunk has the wrong size")); } + let src = &src[..h.cbytes]; + Ok((h, src)) +} + +/// Decode a chunk block by block into a destination the caller has +/// zeroed: `sink(offset, block)` receives each decoded block (the last may +/// be short), except blocks for which `want(offset, len)` is false and +/// the blocks of a special all-zeros chunk. Holds at most three blocks of scratch, so a +/// caller that keeps only part of each block (B2ND padding) never holds +/// the whole chunk. +fn decode_blocks( + h: &ChunkHeader, + src: &[u8], + want: &mut dyn FnMut(usize, usize) -> bool, + sink: &mut dyn FnMut(usize, &[u8]), +) -> Result<(), FormatError> { let nbytes = h.nbytes; - let mut out = vec![0u8; nbytes]; + // `read_header` clamps the block size to the decoded size. + let blocksize = h.blocksize; + let nblocks = if nbytes == 0 { + 0 + } else { + nbytes.div_ceil(blocksize) + }; + let leftover = nbytes % blocksize.max(1); + let block_len = |j: usize| { + if j == nblocks - 1 && leftover > 0 { + leftover + } else { + blocksize + } + }; if h.special != 0 { // Filled per block, as c-blosc2 does: each block must hold whole // values. @@ -253,19 +292,30 @@ pub fn blosc2_decompress_chunk(src: &[u8], limit: usize) -> Result, Form }; if let Some(value) = fill { let ts = value.len(); - let blocksize = h.blocksize.max(1); - if !blocksize.is_multiple_of(ts) || !(nbytes % blocksize).is_multiple_of(ts) { + if !blocksize.max(1).is_multiple_of(ts) || !leftover.is_multiple_of(ts) { return Err(err("special chunk blocks are not whole values")); } - for v in out.chunks_exact_mut(ts) { - v.copy_from_slice(value); + if nblocks > 0 { + let block: Vec = value.iter().copied().cycle().take(blocksize).collect(); + for j in 0..nblocks { + let len = block_len(j); + if want(j * blocksize, len) { + sink(j * blocksize, &block[..len]); + } + } } } - return Ok(out); + return Ok(()); } - if memcpyed { - out.copy_from_slice(&src[h.overhead..]); - return Ok(out); + if h.flags & FLAG_MEMCPYED != 0 { + let data = &src[h.overhead..]; + for j in 0..nblocks { + let (at, len) = (j * blocksize, block_len(j)); + if want(at, len) { + sink(at, &data[at..at + len]); + } + } + return Ok(()); } if h.blosc2_flags & B2_USEDICT != 0 { return Err(err("dictionary-compressed chunks are not supported")); @@ -289,11 +339,8 @@ pub fn blosc2_decompress_chunk(src: &[u8], limit: usize) -> Result, Form } } if nbytes == 0 { - return Ok(out); + return Ok(()); } - let blocksize = h.blocksize; - let nblocks = nbytes.div_ceil(blocksize); - let leftover = nbytes % blocksize; let bstarts_end = nblocks .checked_mul(4) .and_then(|n| n.checked_add(h.overhead)) @@ -302,12 +349,19 @@ pub fn blosc2_decompress_chunk(src: &[u8], limit: usize) -> Result, Form return Err(err("block table is truncated")); } let dont_split = h.flags & FLAG_DONT_SPLIT != 0; + // The delta filter's reference is the first block, decoded. + let delta = h.filters.contains(&FILTER_DELTA); + let mut first = Vec::new(); let mut tmp = vec![0u8; blocksize]; let mut tmp2 = vec![0u8; blocksize]; let mut zstd = None; for j in 0..nblocks { let is_leftover = j == nblocks - 1 && leftover > 0; - let bsize = if is_leftover { leftover } else { blocksize }; + let bsize = block_len(j); + let wanted = want(j * blocksize, bsize); + if !wanted && !(delta && j == 0) { + continue; + } let start = le_i32(src, h.overhead + 4 * j)?; if start <= 0 || start as usize >= src.len() { return Err(err("block start out of range")); @@ -355,23 +409,21 @@ pub fn blosc2_decompress_chunk(src: &[u8], limit: usize) -> Result, Form } } } - let (done, rest) = out.split_at_mut(j * blocksize); - let dest = &mut rest[..bsize]; - run_filters_backward(&h, cur, &mut tmp2[..bsize], done, dest); + run_filters_backward(h, cur, &mut tmp2[..bsize], &first); + if delta && j == 0 { + first = cur.to_vec(); + } + if wanted { + sink(j * blocksize, cur); + } } - Ok(out) + Ok(()) } -/// Undo the chunk's filters on one decoded block `cur`, leaving the result -/// in `dest`. `done` is the chunk decoded so far (the delta filter's -/// reference is the first block). -fn run_filters_backward( - h: &ChunkHeader, - cur: &mut [u8], - scratch: &mut [u8], - done: &[u8], - dest: &mut [u8], -) { +/// Undo the chunk's filters on one decoded block `cur`, in place. `first` +/// is the decoded first block (empty while decoding it), the delta +/// filter's reference. +fn run_filters_backward(h: &ChunkHeader, cur: &mut [u8], scratch: &mut [u8], first: &[u8]) { let bsize = cur.len(); let ts = h.typesize; for i in (0..6).rev() { @@ -395,13 +447,12 @@ fn run_filters_backward( bitunshuffle_block(&cur[..body], &mut scratch[..body], n, ts); cur[..body].copy_from_slice(&scratch[..body]); } - FILTER_DELTA => delta_decode(done, cur, ts), + FILTER_DELTA => delta_decode(first, cur, ts), // Truncating precision is lossy and has nothing to undo. FILTER_NONE | FILTER_TRUNC_PREC => {} _ => unreachable!("filters are checked before decoding"), } } - dest.copy_from_slice(cur); } /// Byte unshuffle (`unshuffle_generic`): trailing bytes that do not make a @@ -416,18 +467,19 @@ fn unshuffle(bytes: usize, src: &[u8], dest: &mut [u8]) { dest[n * bytes..].copy_from_slice(&src[n * bytes..]); } -/// `delta_decoder`: the first block is XOR-accumulated over its own -/// elements; every other block is XORed with the (decoded) first block. +/// `delta_decoder`: the first block (`first` is empty) is XOR-accumulated +/// over its own elements; every other block is XORed with the (decoded) +/// first block. /// Elements are 1, 2, 4 or 8 bytes wide (other sizes: 8 if a multiple of /// 8, else 1); a trailing partial element is left alone. -fn delta_decode(done: &[u8], cur: &mut [u8], typesize: usize) { +fn delta_decode(first: &[u8], cur: &mut [u8], typesize: usize) { let w = match typesize { 1 | 2 | 4 | 8 => typesize, t if t.is_multiple_of(8) => 8, _ => 1, }; let n = cur.len() / w; - if done.is_empty() { + if first.is_empty() { for i in 1..n { for b in 0..w { cur[i * w + b] ^= cur[(i - 1) * w + b]; @@ -435,7 +487,7 @@ fn delta_decode(done: &[u8], cur: &mut [u8], typesize: usize) { } } else { // The first block is at least as long as any other. - for (c, r) in cur[..n * w].iter_mut().zip(done) { + for (c, r) in cur[..n * w].iter_mut().zip(first) { *c ^= *r; } } @@ -548,14 +600,15 @@ fn parse_frame(buf: &[u8], limit: usize) -> Result, FormatError> { } impl Frame<'_> { - /// Decode chunk `n`, refusing more than `limit` bytes. - fn chunk(&self, n: usize, limit: usize) -> Result, FormatError> { + /// Where chunk `n` is: its size from the frame header (every chunk but + /// the last holds `chunksize` bytes; unknown if that is 0), and either + /// the special value its offset records or its bytes. + fn locate(&self, n: usize) -> Result<(Option, Located<'_>), FormatError> { if n >= self.nchunks { return Err(err("frame has no such chunk")); } let raw: [u8; 8] = self.offsets[8 * n..8 * n + 8].try_into().unwrap(); let offset = i64::from_le_bytes(raw); - // Every chunk but the last holds `chunksize` bytes. let size = if self.chunksize == 0 { None } else if n == self.nchunks - 1 && !self.nbytes.is_multiple_of(self.chunksize) { @@ -565,26 +618,20 @@ impl Frame<'_> { }; if offset < 0 { // A special chunk, recorded in the offset's top byte. - let Some(size) = size else { + if size.is_none() { return Err(err("special chunk in a frame without a chunk size")); - }; - if size > limit { - return Err(err("decoded size exceeds the limit")); } let kind = raw[7] & 7; return match kind { - SPECIAL_ZERO | SPECIAL_UNINIT => Ok(vec![0u8; size]), - SPECIAL_NAN => { - let nan: &[u8] = match self.typesize { + SPECIAL_ZERO | SPECIAL_UNINIT => Ok((size, Located::Zeros)), + SPECIAL_NAN => Ok(( + size, + Located::Fill(match self.typesize { 4 => &[0x00, 0x00, 0xc0, 0x7f], 8 => &[0, 0, 0, 0, 0, 0, 0xf8, 0x7f], _ => return Err(err("NaN chunk of a type that is not f32 or f64")), - }; - if !size.is_multiple_of(nan.len()) { - return Err(err("NaN chunk is not whole values")); - } - Ok(nan.iter().copied().cycle().take(size).collect()) - } + }), + )), _ => Err(err(&format!("unknown special chunk offset {kind}"))), }; } @@ -596,12 +643,90 @@ impl Frame<'_> { .and_then(|o| self.header_len.checked_add(o)) .filter(|&s| s < end && end - s >= MIN_HEADER) .ok_or_else(|| err("chunk offset out of range"))?; - let data = - blosc2_decompress_chunk(&self.buf[start..end], limit.min(size.unwrap_or(limit)))?; - if size.is_some_and(|s| s != data.len()) { + Ok((size, Located::Data(&self.buf[start..end]))) + } + + /// Open chunk `n` (a regular one), which must decode to the frame's + /// size for it, and at most `limit` bytes. + fn open<'s>( + &self, + size: Option, + src: &'s [u8], + limit: usize, + ) -> Result<(ChunkHeader, &'s [u8]), FormatError> { + let (h, src) = open_chunk(src, limit.min(size.unwrap_or(limit)))?; + if size.is_some_and(|s| s != h.nbytes) { return Err(err("chunk size does not match the frame's chunk size")); } - Ok(data) + Ok((h, src)) + } + + /// Decode chunk `n`, refusing more than `limit` bytes. + fn chunk(&self, n: usize, limit: usize) -> Result, FormatError> { + let (size, at) = self.locate(n)?; + let src = match at { + Located::Data(src) => src, + Located::Zeros | Located::Fill(_) => { + let size = size.unwrap_or(0); + if size > limit { + return Err(err("decoded size exceeds the limit")); + } + return match at { + Located::Fill(value) if !size.is_multiple_of(value.len()) => { + Err(err("NaN chunk is not whole values")) + } + Located::Fill(value) => Ok(value.iter().copied().cycle().take(size).collect()), + _ => Ok(vec![0u8; size]), + }; + } + }; + let (h, src) = self.open(size, src, limit)?; + let mut out = vec![0u8; h.nbytes]; + decode_blocks(&h, src, &mut |_, _| true, &mut |at, block| { + out[at..at + block.len()].copy_from_slice(block); + })?; + Ok(out) + } + + /// Decode chunk `n` (exactly `size` bytes) block by block, as + /// [`decode_blocks`], into a destination the caller has zeroed. Its + /// blocks must be whole multiples of `unit` bytes and at most + /// `max_block`; special chunks are handed over `unit` bytes at a time. + fn chunk_blocks( + &self, + n: usize, + size: usize, + unit: usize, + max_block: usize, + want: &mut dyn FnMut(usize, usize) -> bool, + sink: &mut dyn FnMut(usize, &[u8]), + ) -> Result<(), FormatError> { + let (frame_size, at) = self.locate(n)?; + if frame_size.is_some_and(|s| s != size) { + return Err(err("chunk size does not match the b2nd shape")); + } + match at { + Located::Zeros => Ok(()), + Located::Fill(value) => { + if !unit.is_multiple_of(value.len()) || !size.is_multiple_of(unit) { + return Err(err("NaN chunk is not whole values")); + } + let block: Vec = value.iter().copied().cycle().take(unit).collect(); + for k in 0..size / unit { + if want(k * unit, unit) { + sink(k * unit, &block); + } + } + Ok(()) + } + Located::Data(src) => { + let (h, src) = self.open(Some(size), src, size)?; + if h.nbytes > 0 && (!h.blocksize.is_multiple_of(unit) || h.blocksize > max_block) { + return Err(err("chunk blocks do not match the b2nd blocks")); + } + decode_blocks(&h, src, want, sink) + } + } } /// The content of metalayer `name`, if the frame has it. @@ -653,6 +778,16 @@ impl Frame<'_> { } } +/// Where a frame's chunk is. +enum Located<'a> { + /// All zeros (or uninitialised, read as zeros). + Zeros, + /// One value repeated (NaN). + Fill(&'static [u8]), + /// A Blosc2 chunk, running to the end of the data section. + Data(&'a [u8]), +} + /// The B2ND (or Caterva) metalayer: shape, chunk shape, block shape. #[derive(Debug, PartialEq, Eq)] struct NdMeta { @@ -753,6 +888,12 @@ fn decode_frame( } /// Gather a B2ND array's blocks into one C-order buffer. +/// +/// Each Blosc2 chunk holds one B2ND chunk padded to whole blocks, which can +/// be many times the array (a crafted frame's shapes are the file's word). +/// So the chunks are never decoded whole: each block is placed in the +/// output as it is decoded, blocks that are all padding are skipped, and +/// no more than a few blocks (each within the array) are held at once. fn reassemble(frame: &Frame<'_>, nd: &NdMeta, limit: usize) -> Result, FormatError> { let ts = frame.typesize; let ndim = nd.shape.len(); @@ -778,10 +919,15 @@ fn reassemble(frame: &Frame<'_>, nd: &NdMeta, limit: usize) -> Result, F if total == 0 { return Ok(out); } - // Padding to whole blocks grows a chunk by less than 2x per dimension; - // hdf5-blosc2's block shapes keep it far below 16x (a crafted frame - // could otherwise make each chunk enormous). - if ext_bytes > limit.saturating_mul(16) { + // hdf5-blosc2's chunk is the array. A chunk larger than the array + // would be padding decoded for nothing; with chunks and blocks within + // the array, a block is never larger than the output, and a chunk + // pads each dimension by less than one block. + if (0..ndim).any(|i| nd.chunkshape[i] > nd.shape[i]) { + return Err(err("b2nd chunk is larger than the array")); + } + // A Blosc2 chunk of `ext_bytes`, exactly the padded chunk. + if ext_bytes > i32::MAX as usize { return Err(too_big()); } let nchunks: usize = grid.iter().product(); @@ -789,7 +935,6 @@ fn reassemble(frame: &Frame<'_>, nd: &NdMeta, limit: usize) -> Result, F return Err(err("chunk count does not match the b2nd shape")); } let blocks_in_chunk: Vec = (0..ndim).map(|i| ext[i] / nd.blockshape[i]).collect(); - let nblocks: usize = blocks_in_chunk.iter().product(); let block_items: usize = nd.blockshape.iter().product(); let block_bytes = block_items * ts; // Strides (in elements) of the output array and of a block. @@ -799,50 +944,88 @@ fn reassemble(frame: &Frame<'_>, nd: &NdMeta, limit: usize) -> Result, F out_stride[i] = out_stride[i + 1] * nd.shape[i + 1]; blk_stride[i] = blk_stride[i + 1] * nd.blockshape[i + 1]; } + let geometry = BlockGeometry { + nd, + blocks_in_chunk: &blocks_in_chunk, + }; let mut cidx = vec![0usize; ndim]; - let mut bidx = vec![0usize; ndim]; - let mut gstart = vec![0usize; ndim]; - let mut valid = vec![0usize; ndim]; + let mut w = BlockPlace::new(ndim); + let mut p = BlockPlace::new(ndim); let mut pos = vec![0usize; ndim]; for n in 0..nchunks { unravel(n, &grid, &mut cidx); - let data = frame.chunk(n, ext_bytes)?; - if data.len() != ext_bytes { - return Err(err("chunk size does not match the b2nd shape")); - } - for b in 0..nblocks { - unravel(b, &blocks_in_chunk, &mut bidx); - let mut empty = false; - for i in 0..ndim { - let in_chunk = bidx[i] * nd.blockshape[i]; - gstart[i] = cidx[i] * nd.chunkshape[i] + in_chunk; - let lim_chunk = nd.chunkshape[i].saturating_sub(in_chunk); - let lim_shape = nd.shape[i].saturating_sub(gstart[i]); - valid[i] = nd.blockshape[i].min(lim_chunk).min(lim_shape); - empty |= valid[i] == 0; - } - if empty { - continue; - } - let block = &data[b * block_bytes..(b + 1) * block_bytes]; - // Copy row by row along the last dimension. - let row = valid[ndim - 1] * ts; - let rows: usize = valid[..ndim - 1].iter().product(); - for r in 0..rows { - unravel(r, &valid[..ndim - 1], &mut pos[..ndim - 1]); - let mut src = 0; - let mut dst = gstart[ndim - 1]; - for i in 0..ndim - 1 { - src += pos[i] * blk_stride[i]; - dst += (gstart[i] + pos[i]) * out_stride[i]; + let blocks = |at: usize, len: usize| at / block_bytes..(at + len).div_ceil(block_bytes); + let mut want = + |at: usize, len: usize| blocks(at, len).any(|b| geometry.place(b, &cidx, &mut w)); + let mut sink = |at: usize, data: &[u8]| { + for b in blocks(at, data.len()) { + if !geometry.place(b, &cidx, &mut p) { + continue; + } + let block = &data[b * block_bytes - at..][..block_bytes]; + // Copy row by row along the last dimension. + let row = p.valid[ndim - 1] * ts; + let rows: usize = p.valid[..ndim - 1].iter().product(); + for r in 0..rows { + unravel(r, &p.valid[..ndim - 1], &mut pos[..ndim - 1]); + let mut src = 0; + let mut dst = p.gstart[ndim - 1]; + for i in 0..ndim - 1 { + src += pos[i] * blk_stride[i]; + dst += (p.gstart[i] + pos[i]) * out_stride[i]; + } + out[dst * ts..dst * ts + row].copy_from_slice(&block[src * ts..src * ts + row]); } - out[dst * ts..dst * ts + row].copy_from_slice(&block[src * ts..src * ts + row]); } - } + }; + frame.chunk_blocks(n, ext_bytes, block_bytes, limit, &mut want, &mut sink)?; } Ok(out) } +/// The block layout of a B2ND array's chunks. +struct BlockGeometry<'a> { + nd: &'a NdMeta, + blocks_in_chunk: &'a [usize], +} + +/// Where one block of a chunk lands: its first element in the array and +/// how much of it (per dimension) lies within the chunk and the array. +struct BlockPlace { + bidx: Vec, + gstart: Vec, + valid: Vec, +} + +impl BlockPlace { + fn new(ndim: usize) -> BlockPlace { + BlockPlace { + bidx: vec![0; ndim], + gstart: vec![0; ndim], + valid: vec![0; ndim], + } + } +} + +impl BlockGeometry<'_> { + /// Place block `b` of the chunk at grid index `cidx`; false if it is + /// all padding. + fn place(&self, b: usize, cidx: &[usize], p: &mut BlockPlace) -> bool { + let nd = self.nd; + unravel(b, self.blocks_in_chunk, &mut p.bidx); + let mut empty = false; + for i in 0..nd.shape.len() { + let in_chunk = p.bidx[i] * nd.blockshape[i]; + p.gstart[i] = cidx[i] * nd.chunkshape[i] + in_chunk; + let lim_chunk = nd.chunkshape[i].saturating_sub(in_chunk); + let lim_shape = nd.shape[i].saturating_sub(p.gstart[i]); + p.valid[i] = nd.blockshape[i].min(lim_chunk).min(lim_shape); + empty |= p.valid[i] == 0; + } + !empty + } +} + /// C-order multi-index of `n` in a grid of `dims`. fn unravel(mut n: usize, dims: &[usize], idx: &mut [usize]) { for i in (0..dims.len()).rev() { diff --git a/crates/clawhdf5-format/tests/blosc2_alloc_bounds.rs b/crates/clawhdf5-format/tests/blosc2_alloc_bounds.rs index 833cd53..a6ba114 100644 --- a/crates/clawhdf5-format/tests/blosc2_alloc_bounds.rs +++ b/crates/clawhdf5-format/tests/blosc2_alloc_bounds.rs @@ -220,3 +220,131 @@ fn empty_chunk_does_not_allocate_its_block_size() { assert!(r.is_err(), "decoded {:?}", r.map(|v| v.len())); assert!(peak <= bound(64, &f), "in a frame: peak {peak} bytes"); } + +/// B2ND chunks were decoded whole, padding included, with up to 16x the +/// HDF5 chunk size as their limit. Blocks are now placed as they are +/// decoded, so the padding is never held. +/// +/// Ten dimensions: nine of 3 split into blocks of 2 (padded to 4) and one +/// of 4, so each chunk is 13x the array. One chunk, stored three ways: as a +/// NaN chunk in the frame's offsets, as a repeated-value chunk, and as a +/// chunk of stored (uncompressed) blocks. +#[test] +fn b2nd_padding_is_never_held() { + let _g = lock(); + let ts = 4usize; + let mut shape = vec![3i64; 9]; + shape.push(4); + let chunks: Vec = shape.iter().map(|&s| s as i32).collect(); + let mut blocks = vec![2i32; 9]; + blocks.push(4); + let meta = nd_meta(&shape, &chunks, &blocks); + let items: usize = shape.iter().product::() as usize; + let limit = items * ts; + let block_bytes = ts * blocks.iter().product::() as usize; + let ext_bytes = ts * 4usize.pow(9) * 4; + assert!(ext_bytes > 13 * limit); + let offsets = |off: [u8; 8]| repeated(&off, 8, 8); + + let value = 1.5f32.to_le_bytes(); + let stored = { + // Every block stored raw: block k holds the value k. + let mut c = chunk_header(4, ext_bytes as i32, block_bytes as i32, 0, 0); + c[2] = 0x02 | 0x10; // memcpyed, not split + c.truncate(16); + for k in 0..ext_bytes / block_bytes { + c.extend((k as f32).to_le_bytes().repeat(block_bytes / 4)); + } + let n = c.len() as i32; + c[12..16].copy_from_slice(&n.to_le_bytes()); + c + }; + let cases: Vec<(&str, Vec)> = vec![ + ( + "NaN offset", + frame( + Some(&meta), + ext_bytes as i64, + 4, + ext_bytes as i32, + &[], + &offsets(special_offset(2)), + ), + ), + ( + "repeated value", + frame( + Some(&meta), + ext_bytes as i64, + 4, + ext_bytes as i32, + &repeated(&value, ext_bytes as i32, block_bytes as i32), + &offsets(0i64.to_le_bytes()), + ), + ), + ( + "stored blocks", + frame( + Some(&meta), + ext_bytes as i64, + 4, + ext_bytes as i32, + &stored, + &offsets(0i64.to_le_bytes()), + ), + ), + ]; + for (name, f) in cases { + let (r, peak) = peak_during(|| blosc2_decompress(&f, limit)); + let out = r.unwrap_or_else(|e| panic!("{name}: {e}")); + assert_eq!(out.len(), limit, "{name}"); + match name { + "NaN offset" => assert!( + out.chunks(4) + .all(|v| f32::from_le_bytes(v.try_into().unwrap()).is_nan()) + ), + "repeated value" => assert!(out.chunks(4).all(|v| v == value)), + _ => { + // Element (i0..i9) lies in block (i0/2, .., i8/2), numbered + // in C order over a 2x..x2x1 grid of blocks. + let mut idx = [0usize; 10]; + for (e, v) in out.chunks(4).enumerate() { + let mut n = e; + for d in (0..10).rev() { + idx[d] = n % shape[d] as usize; + n /= shape[d] as usize; + } + let k = idx[..9].iter().fold(0, |k, &i| k * 2 + i / 2); + assert_eq!( + f32::from_le_bytes(v.try_into().unwrap()), + k as f32, + "{name} {e}" + ); + } + } + } + assert!( + peak <= bound(limit, &f), + "{name}: peak {peak} bytes for a {limit}-byte chunk ({}-byte frame)", + f.len() + ); + } +} + +/// A B2ND chunk larger than the array (here 16x, the old cap) is refused, +/// or at least never allocated. +#[test] +fn b2nd_chunk_larger_than_the_array_is_not_allocated() { + let _g = lock(); + let limit = 1 << 20; + let c = 16 * limit as i32; + let meta = nd_meta(&[limit as i64], &[c], &[c]); + let offsets = repeated(&special_offset(1), 8, 8); + let f = frame(Some(&meta), c as i64, 1, c, &[], &offsets); + let (r, peak) = peak_during(|| blosc2_decompress(&f, limit)); + assert!( + peak <= bound(limit, &f), + "peak {peak} bytes ({:?})", + r.map(|v| v.len()) + ); +} From d9e4dfb6e6de63a66c0238630b58c13405c767d5 Mon Sep 17 00:00:00 2001 From: osobh Date: Sat, 26 Sep 2026 11:34:28 -0500 Subject: [PATCH 6/9] fix(format): cap the Zstandard window at what the output can need ruzstd reserves a frame's declared window (up to its 100 MiB default) when a decoder is reset for a new frame, before decoding anything. The Blosc, Blosc2 and bitshuffle decoders reuse one decoder per chunk, so a Blosc2 chunk of two 16-byte streams, each declaring a 96 MiB window, allocated 128 MiB. zstd_decode_into now sets the decoder's maximum window to twice the stream's output (at least 128 KiB): c-blosc, c-blosc2 and bitshuffle compress each block in one call with its size known, so libzstd's window never exceeds the block. Found by tracking peak allocation in the Blosc2 fuzz test. Co-Authored-By: Claude Opus 5.5 (1M context) --- .../clawhdf5-format/src/filters_bitshuffle.rs | 8 +++ .../tests/blosc2_alloc_bounds.rs | 51 ++++++++++++++++++- 2 files changed, 57 insertions(+), 2 deletions(-) diff --git a/crates/clawhdf5-format/src/filters_bitshuffle.rs b/crates/clawhdf5-format/src/filters_bitshuffle.rs index d20835e..6b23291 100644 --- a/crates/clawhdf5-format/src/filters_bitshuffle.rs +++ b/crates/clawhdf5-format/src/filters_bitshuffle.rs @@ -218,12 +218,20 @@ pub(crate) fn bitshuffle_decode( } /// Decode Zstandard frames into exactly `dst`, failing if they hold more. +/// +/// ruzstd reserves a frame's declared window (by default up to 100 MiB) +/// before decoding it, so the window is capped at what the output could +/// need: twice `dst` (window sizes are rounded up), and at least 128 KiB. +/// The encoders behind these filters (c-blosc, c-blosc2, bitshuffle) +/// compress each block in one call with its size known, so libzstd's +/// window never exceeds the block. #[cfg(any(feature = "bitshuffle", feature = "blosc"))] pub(crate) fn zstd_decode_into( decoder: &mut ruzstd::decoding::FrameDecoder, frames: &[u8], dst: &mut [u8], ) -> Result { + decoder.set_max_window_size((2 * dst.len()).max(1 << 17) as u64); decoder .decode_all(frames, dst) .map_err(|e| FormatError::DecompressionError(format!("zstd: {e}"))) diff --git a/crates/clawhdf5-format/tests/blosc2_alloc_bounds.rs b/crates/clawhdf5-format/tests/blosc2_alloc_bounds.rs index a6ba114..5dc71b0 100644 --- a/crates/clawhdf5-format/tests/blosc2_alloc_bounds.rs +++ b/crates/clawhdf5-format/tests/blosc2_alloc_bounds.rs @@ -61,9 +61,11 @@ fn peak_during(f: impl FnOnce() -> T) -> (T, usize) { /// What decoding one HDF5 chunk of `limit` bytes from `input` may hold at /// once: the output, a few blocks of scratch (each no larger than the -/// output), the offsets table, and the Zstandard decoder's state. +/// output), the offsets table, and the Zstandard decoder's state, which has +/// a fixed ceiling: a window of at most 128 KiB (or twice the stream) and a +/// block's table of sequences (up to 98,303 of 12 bytes, 1.2 MB). fn bound(limit: usize, input: &[u8]) -> usize { - 6 * limit + 2 * input.len() + (1 << 20) + 6 * limit + 2 * input.len() + (2 << 20) } fn lock() -> std::sync::MutexGuard<'static, ()> { @@ -348,3 +350,48 @@ fn b2nd_chunk_larger_than_the_array_is_not_allocated() { r.map(|v| v.len()) ); } + +/// ruzstd reserves a frame's declared window (up to 100 MiB) before it +/// decodes a frame with a decoder it has used before: a Blosc2 chunk of +/// two 16-byte Zstandard streams, each declaring a 96 MiB window, +/// allocated 96 MiB. c-blosc2 compresses each block with its size known, +/// so its windows never exceed the block. +#[test] +fn zstd_window_is_bounded_by_the_output() { + let _g = lock(); + let mut z = 0xfd2f_b528u32.to_le_bytes().to_vec(); + // No single segment, no checksum; window 2^26 + 4/8 of it = 96 MiB. + z.extend_from_slice(&[0x00, (16 << 3) | 4]); + // One raw block, last, of 16 bytes. + let h = 1 | (16 << 3); + z.extend_from_slice(&[h as u8, (h >> 8) as u8, 0]); + z.extend_from_slice(&[7; 16]); + // Two blocks of 16 bytes, one stream each (not split), Zstandard + // (codec 4). + let chunk = |z: &[u8]| { + let mut c = vec![5u8, 1, 0x10 | (4 << 5), 1]; + for v in [32i32, 16, 0] { + c.extend_from_slice(&v.to_le_bytes()); + } + let first = 24 + 4 + z.len(); + c.extend_from_slice(&24i32.to_le_bytes()); + c.extend_from_slice(&(first as i32).to_le_bytes()); + for _ in 0..2 { + c.extend_from_slice(&(z.len() as i32).to_le_bytes()); + c.extend_from_slice(z); + } + let n = c.len() as i32; + c[12..16].copy_from_slice(&n.to_le_bytes()); + c + }; + let c = chunk(&z); + let (r, peak) = peak_during(|| blosc2_decompress_chunk(&c, 32)); + assert!(peak <= bound(32, &c), "peak {peak} bytes ({r:?})"); + assert!(r.is_err(), "{r:?}"); + // The same streams with a window they can use read. + z[5] = 0; + assert_eq!( + blosc2_decompress_chunk(&chunk(&z), 32).unwrap(), + vec![7; 32] + ); +} From 2ba4bc97d8ae01cfb88eac7b019fb6947e9e270a Mon Sep 17 00:00:00 2001 From: osobh Date: Sat, 26 Sep 2026 11:35:08 -0500 Subject: [PATCH 7/9] test(format): fuzz Blosc2 decoding for peak allocation The Blosc2 fuzz tests checked only for panics and the size of the output, so the offsets-chunk amplification passed all 40,000 iterations. tests/blosc2_alloc_bounds.rs now measures peak allocation (a counting global allocator) and asserts it stays within 6x the HDF5 chunk size plus twice the input plus 2 MiB (ruzstd's fixed state) for: - 20,000 mutated fixture frames, decoded with their real chunk size as the limit, with edits aimed at the frame's and chunks' size fields; - 20,000 mutated first chunks on their own; - 5,000 frames built from random header sizes, offsets chunks and B2ND shapes (padding chunk and block shapes, chunks larger than the array, NaN, zero and repeated-value chunks). Against the code before this series every test in the file fails (the fuzz tests at frame iteration 3913, a zstd window, and random frame 289, the offsets chunk); now the worst frame peaks at 0.48 of the bound. Co-Authored-By: Claude Opus 5.5 (1M context) --- crates/clawhdf5-format/src/filters_blosc2.rs | 3 +- .../tests/blosc2_alloc_bounds.rs | 244 ++++++++++++++++++ 2 files changed, 246 insertions(+), 1 deletion(-) diff --git a/crates/clawhdf5-format/src/filters_blosc2.rs b/crates/clawhdf5-format/src/filters_blosc2.rs index 71ad4b6..a70a906 100644 --- a/crates/clawhdf5-format/src/filters_blosc2.rs +++ b/crates/clawhdf5-format/src/filters_blosc2.rs @@ -1246,7 +1246,8 @@ mod tests { } /// Random and mutated frames: errors are fine, panics are not, and no - /// output past the limit. + /// output past the limit. Peak allocation is fuzzed separately, with a + /// counting allocator, in `tests/blosc2_alloc_bounds.rs`. #[test] fn fuzzed_frames_never_panic() { let limit = 20_000; diff --git a/crates/clawhdf5-format/tests/blosc2_alloc_bounds.rs b/crates/clawhdf5-format/tests/blosc2_alloc_bounds.rs index 5dc71b0..1667186 100644 --- a/crates/clawhdf5-format/tests/blosc2_alloc_bounds.rs +++ b/crates/clawhdf5-format/tests/blosc2_alloc_bounds.rs @@ -395,3 +395,247 @@ fn zstd_window_is_bounded_by_the_output() { vec![7; 32] ); } + +/// xorshift64*: deterministic, so a failure reproduces. +struct Rng(u64); + +impl Rng { + fn next(&mut self) -> u64 { + let mut x = self.0; + x ^= x >> 12; + x ^= x << 25; + x ^= x >> 27; + self.0 = x; + x.wrapping_mul(0x2545_F491_4F6C_DD1D) + } + + fn below(&mut self, n: usize) -> usize { + (self.next() % n.max(1) as u64) as usize + } + + /// A size that tends to the edges: small, a power of two, huge. + fn size(&mut self) -> i64 { + match self.below(6) { + 0 => self.below(64) as i64, + 1 => 1 << self.below(31), + 2 => i32::MAX as i64 - self.below(4096) as i64, + 3 => (1i64 << self.below(62)) + self.below(8) as i64, + 4 => MAX_BLOCK - self.below(3) as i64, + _ => self.next() as i32 as i64, + } + } +} + +const MAX_BLOCK: i64 = 536_866_816; + +/// One to four edits: bytes, or a size field written little-endian (chunk +/// headers) or big-endian (frame headers), most often at a header's size +/// fields. +fn mutate(rng: &mut Rng, seed: &[u8], data_at: usize) -> Vec { + let mut v = seed.to_vec(); + for _ in 0..1 + rng.below(4) { + let len = v.len(); + if len < 16 { + v.push(rng.next() as u8); + continue; + } + match rng.below(8) { + 0 => { + let i = rng.below(len); + v[i] ^= 1 << rng.below(8); + } + 1 => { + let i = rng.below(len); + v[i] = rng.next() as u8; + } + 2 => { + // Frame header: nbytes, cbytes (i64), typesize, chunksize. + let x = rng.size(); + match rng.below(4) { + 0 if len >= 38 => v[30..38].copy_from_slice(&x.to_be_bytes()), + 1 if len >= 47 => v[39..47].copy_from_slice(&x.to_be_bytes()), + 2 if len >= 52 => v[48..52].copy_from_slice(&(x as i32).to_be_bytes()), + _ if len >= 62 => v[58..62].copy_from_slice(&(x as i32).to_be_bytes()), + _ => {} + } + } + 3 | 4 => { + // A chunk header's nbytes, blocksize or cbytes: in the first + // data chunk, or anywhere (the offsets chunk comes last). + let at = if rng.below(2) == 0 && data_at + 16 <= len { + data_at + 4 * (1 + rng.below(3)) + } else { + rng.below(len - 3) + }; + let x = rng.size() as i32; + v[at..at + 4].copy_from_slice(&x.to_le_bytes()); + } + 5 => v.truncate(rng.below(len)), + 6 => { + let at = rng.below(len); + v[at] = [0x10, 0x20, 0x30, 0x40, 0x05, 0x07, 0x02][rng.below(7)]; + } + _ => { + let i = rng.below(len - 3); + let x = rng.size() as i32; + v[i..i + 4].copy_from_slice(&x.to_be_bytes()); + } + } + } + v +} + +/// Every fixture frame that decodes, with its decoded size. +fn seeds() -> Vec<(Vec, usize)> { + let dir = std::path::Path::new(env!("CARGO_MANIFEST_DIR")).join("tests/fixtures/blosc2"); + let mut v = Vec::new(); + for e in std::fs::read_dir(dir).unwrap() { + let p = e.unwrap().path(); + if p.extension().is_some_and(|x| x == "b2f") + && let Ok(out) = std::fs::read(p.with_extension("out")) + { + v.push((std::fs::read(&p).unwrap(), out.len())); + } + } + v.sort(); + assert!(v.len() >= 20, "fixtures missing"); + v +} + +fn header_len(frame: &[u8]) -> usize { + i32::from_be_bytes(frame[11..15].try_into().unwrap()) as usize +} + +/// Mutated fixture frames, decoded with their HDF5 chunk size as the +/// limit, and their first chunks on their own: whatever they declare, no +/// decode holds more than a small multiple of the output and the input. +#[test] +fn fuzzed_frames_and_chunks_stay_within_the_allocation_bound() { + let _g = lock(); + let seeds = seeds(); + let mut rng = Rng(0xb2a1); + let mut worst = (0.0f64, String::new()); + for i in 0..20_000 { + let (seed, limit) = &seeds[rng.below(seeds.len())]; + let f = mutate(&mut rng, seed, header_len(seed)); + let (r, peak) = peak_during(|| blosc2_decompress(&f, *limit)); + if let Ok(out) = &r { + assert!(out.len() <= *limit, "iteration {i}: output past the limit"); + } + assert!( + peak <= bound(*limit, &f), + "iteration {i}: peak {peak} bytes for a {limit}-byte chunk from {} bytes ({:?})", + f.len(), + r.map(|v| v.len()) + ); + let ratio = peak as f64 / bound(*limit, &f) as f64; + if ratio > worst.0 { + worst = ( + ratio, + format!( + "frame iteration {i}: peak {peak}, limit {limit}, input {}", + f.len() + ), + ); + } + } + for i in 0..20_000 { + let (seed, _) = &seeds[rng.below(seeds.len())]; + let at = header_len(seed); + let chunk = &seed[at..]; + let c = mutate(&mut rng, chunk, 0); + let limit = 1 << 16; + let (r, peak) = peak_during(|| blosc2_decompress_chunk(&c, limit)); + assert!( + peak <= bound(limit, &c), + "chunk iteration {i}: peak {peak} bytes from {} bytes ({:?})", + c.len(), + r.map(|v| v.len()) + ); + } + eprintln!("worst peak / bound: {:.2} ({})", worst.0, worst.1); +} + +/// Frames built from random header sizes, offsets chunks and B2ND shapes +/// (chunk and block shapes that pad, special and repeated-value chunks). +#[test] +fn random_frames_stay_within_the_allocation_bound() { + let _g = lock(); + let mut rng = Rng(0xb2a2); + for i in 0..5_000 { + let ts = [1usize, 2, 4, 8][rng.below(4)]; + let ndim = 1 + rng.below(8); + let mut shape = Vec::new(); + let mut chunks = Vec::new(); + let mut blocks = Vec::new(); + for _ in 0..ndim { + let s = 1 + rng.below(if ndim > 3 { 4 } else { 40 }); + let c = if rng.below(8) == 0 { + s * (1 + rng.below(4)) + } else { + 1 + rng.below(s) + }; + let b = 1 + rng.below(c); + shape.push(s as i64); + chunks.push(c as i32); + blocks.push(b as i32); + } + let items: usize = shape.iter().product::() as usize; + let limit = items * ts; + let meta = nd_meta(&shape, &chunks, &blocks); + let ext: usize = ts + * chunks + .iter() + .zip(&blocks) + .map(|(&c, &b)| (c as usize).div_ceil(b as usize) * b as usize) + .product::(); + let nchunks: usize = shape + .iter() + .zip(&chunks) + .map(|(&s, &c)| (s as usize).div_ceil(c as usize)) + .product(); + let block_bytes = ts * blocks.iter().product::() as usize; + let chunksize = if rng.below(4) == 0 { + rng.size() + } else { + ext as i64 + }; + let nbytes = if rng.below(4) == 0 { + rng.size() + } else { + (nchunks * ext) as i64 + }; + let off_n = if rng.below(4) == 0 { + rng.size() as i32 + } else { + 8 * nchunks as i32 + }; + let (data, off) = match rng.below(3) { + 0 => (Vec::new(), special_offset(1 + rng.below(2) as u8)), + _ => { + let bs = if rng.below(4) == 0 { + rng.size() as i32 + } else { + block_bytes as i32 + }; + let value: Vec = (0..ts).map(|_| rng.next() as u8).collect(); + let n = if rng.below(4) == 0 { + rng.size() as i32 + } else { + ext as i32 + }; + (repeated(&value, n, bs), 0i64.to_le_bytes()) + } + }; + let offsets = repeated(&off, off_n, off_n.clamp(1, 8)); + let meta = (rng.below(4) != 0).then_some(meta.as_slice()); + let f = frame(meta, nbytes, ts as i32, chunksize as i32, &data, &offsets); + let (r, peak) = peak_during(|| blosc2_decompress(&f, limit)); + assert!( + peak <= bound(limit, &f), + "iteration {i}: peak {peak} bytes for a {limit}-byte chunk ({:?}, shape {shape:?} \ + chunks {chunks:?} blocks {blocks:?})", + r.map(|v| v.len()) + ); + } +} From 7515e5dcbdd13c3399cf79f3f8371e6a92adb1e2 Mon Sep 17 00:00:00 2001 From: osobh Date: Sat, 26 Sep 2026 11:35:19 -0500 Subject: [PATCH 8/9] docs: CHANGELOG for the Blosc2 allocation bounds Co-Authored-By: Claude Opus 5.5 (1M context) --- CHANGELOG.md | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index aebeb93..51e3040 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -21,6 +21,21 @@ python-blosc2 4.13.1 for what hdf5plugin never writes (`crates/clawhdf5-format/tests/fixtures/blosc2/`); the decoder is fuzzed. Conformance: 576 of 697 files ok (was 575) — h5ex_d_blosc2. +- **A crafted Blosc2 chunk cannot allocate more than a few times its HDF5 + chunk size.** Found in review before release: the sizes a frame declares + sized the decoder's buffers. A 173-byte frame whose offsets chunk claimed + 2 GiB was decoded in full for a 1 MiB chunk; an empty chunk allocated its + declared block size twice (about 1 GiB); a B2ND chunk was decoded whole + with its padding (up to 16x the chunk); and a Zstandard stream's declared + window (up to 100 MiB) was reserved as the decoder was reused, which also + affected Blosc 1 and bitshuffle with Zstandard. Now the offsets chunk is + capped at the chunk size, block sizes are clamped to the chunk, B2ND + blocks are placed as they are decoded (padding is never held), and the + Zstandard window is capped at twice the stream's output (at least + 128 KiB). A B2ND chunk may no longer be larger than its array, which + hdf5-blosc2 never writes. `tests/blosc2_alloc_bounds.rs` measures peak + allocation for these frames and for 45,000 fuzzed ones: at most 6x the + chunk size, twice the input and 2 MiB of Zstandard state. ### Concurrent reads (2026-09-26) - **Full reads of chunked datasets scale with threads again when rayon's From 3da118d2ee953ed12b17db4e84c4a45693ad8eff Mon Sep 17 00:00:00 2001 From: osobh Date: Sat, 26 Sep 2026 11:36:00 -0500 Subject: [PATCH 9/9] style(format): iterate the chunk index in BlockGeometry::place clippy's needless_range_loop, missed before the B2ND streaming commit. Co-Authored-By: Claude Opus 5.5 (1M context) --- crates/clawhdf5-format/src/filters_blosc2.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/crates/clawhdf5-format/src/filters_blosc2.rs b/crates/clawhdf5-format/src/filters_blosc2.rs index a70a906..f4d1690 100644 --- a/crates/clawhdf5-format/src/filters_blosc2.rs +++ b/crates/clawhdf5-format/src/filters_blosc2.rs @@ -1014,9 +1014,9 @@ impl BlockGeometry<'_> { let nd = self.nd; unravel(b, self.blocks_in_chunk, &mut p.bidx); let mut empty = false; - for i in 0..nd.shape.len() { + for (i, &c) in cidx.iter().enumerate() { let in_chunk = p.bidx[i] * nd.blockshape[i]; - p.gstart[i] = cidx[i] * nd.chunkshape[i] + in_chunk; + p.gstart[i] = c * nd.chunkshape[i] + in_chunk; let lim_chunk = nd.chunkshape[i].saturating_sub(in_chunk); let lim_shape = nd.shape[i].saturating_sub(p.gstart[i]); p.valid[i] = nd.blockshape[i].min(lim_chunk).min(lim_shape);