From 7334e21c9380fe5c6360a739e3c7abd7c3f4717f Mon Sep 17 00:00:00 2001 From: osobh Date: Sat, 26 Sep 2026 10:22:22 -0500 Subject: [PATCH] feat(format): read Blosc2 (filter 32026) in pure Rust hdf5plugin's Blosc2 was a clear "not implemented" error. It stores each HDF5 chunk as a Blosc2 contiguous frame; for chunks of 2+ dimensions the frame holds a B2ND array whose data are cut into (padded) blocks stored one after another. filters_blosc2 (feature `blosc2`, in `plugin-filters`; the facade forwards both) decodes, following c-blosc2's decoder and hdf5-blosc2's blosc2_filter.c: - the frame: the msgpack header's fixed fields, the metalayer index, the offsets chunk (with the special zero/NaN/uninitialised offsets) and chunk lookup; - Blosc2 chunks: 16- and 32-byte headers, special chunks (zeros, NaN, uninitialised, one repeated value), split and unsplit streams, zero and run-length streams, and the filter pipeline run backwards (shuffle, shuffle with a byte-group size, bit shuffle including the version-2 and later handling of a partial group of 8, delta against the first block, truncated precision); - the codecs, shared with Blosc 1: BloscLZ, LZ4/LZ4HC, Zlib, Zstandard; - B2ND arrays: blocks gathered into C order, padding dropped, several chunks per array, and the array shape checked against the chunk shape in cd_values as the HDF5 filter does. Dictionaries, lazy chunks, variable-length blocks, user-defined codecs and registered filters (e.g. bytedelta) are errors. Uninitialised chunks read as zeros. No encoder. Tests: h5py + hdf5plugin write every codec x filter (none, shuffle, bitshuffle, delta) and levels 0-9 over the plugin-filter cases, then i1..u8/f4/f8 in 1-D to 5-D chunks with partial edge chunks, datasets of zeros, one value and NaN, and Fletcher32 before Blosc2 (plain frames for n-D chunks); clawhdf5 reads each exactly as its unfiltered twin, and truncated precision exactly as h5py reads it. Fixture frames from python-blosc2 (tests/fixtures/blosc2/generate.py) cover what hdf5plugin never writes: special chunks, delta over many blocks and odd type sizes, odd bit-shuffle blocks, forced splitting, multi-chunk B2ND arrays with a zero chunk, and the refused features. The decoder is fuzzed (random and mutated frames and chunks: no panic, output within the limit). Conformance: h5ex_d_blosc2.h5 now reads (576 of 697 ok, baseline 575). Co-Authored-By: Claude Opus 5.5 (1M context) --- crates/clawhdf5-format/Cargo.toml | 4 +- crates/clawhdf5-format/src/filter_registry.rs | 11 +- crates/clawhdf5-format/src/filters.rs | 7 + crates/clawhdf5-format/src/filters_blosc.rs | 6 +- crates/clawhdf5-format/src/filters_blosc2.rs | 1083 +++++++++++++++++ crates/clawhdf5-format/src/lib.rs | 2 + .../tests/fixtures/blosc2/always_split.b2f | Bin 0 -> 1650 bytes .../tests/fixtures/blosc2/always_split.out | Bin 0 -> 4000 bytes .../tests/fixtures/blosc2/b2nd_2d.b2f | Bin 0 -> 3950 bytes .../tests/fixtures/blosc2/b2nd_2d.out | Bin 0 -> 4292 bytes .../tests/fixtures/blosc2/b2nd_3d.b2f | Bin 0 -> 8108 bytes .../tests/fixtures/blosc2/b2nd_3d.out | Bin 0 -> 2520 bytes .../tests/fixtures/blosc2/b2nd_4d.b2f | Bin 0 -> 6804 bytes .../tests/fixtures/blosc2/b2nd_4d.out | Bin 0 -> 2100 bytes .../tests/fixtures/blosc2/b2nd_zero_chunk.b2f | Bin 0 -> 1849 bytes .../tests/fixtures/blosc2/b2nd_zero_chunk.out | Bin 0 -> 2400 bytes .../fixtures/blosc2/bitshuffle_odd_blocks.b2f | Bin 0 -> 2204 bytes .../fixtures/blosc2/bitshuffle_odd_blocks.out | Bin 0 -> 2000 bytes .../tests/fixtures/blosc2/bytedelta.b2f | Bin 0 -> 458 bytes .../tests/fixtures/blosc2/bytedelta.err | 1 + .../fixtures/blosc2/delta_int16_blosclz.b2f | Bin 0 -> 1601 bytes .../fixtures/blosc2/delta_int16_blosclz.out | Bin 0 -> 2000 bytes .../tests/fixtures/blosc2/delta_int32_lz4.b2f | Bin 0 -> 1584 bytes .../tests/fixtures/blosc2/delta_int32_lz4.out | Bin 0 -> 2800 bytes .../fixtures/blosc2/delta_shuffle_v16.b2f | Bin 0 -> 1019 bytes .../fixtures/blosc2/delta_shuffle_v16.out | Bin 0 -> 3200 bytes .../fixtures/blosc2/delta_uint64_blosclz.b2f | Bin 0 -> 1323 bytes .../fixtures/blosc2/delta_uint64_blosclz.out | Bin 0 -> 3200 bytes .../tests/fixtures/blosc2/delta_uint8_lz4.b2f | Bin 0 -> 1494 bytes .../tests/fixtures/blosc2/delta_uint8_lz4.out | Bin 0 -> 2000 bytes .../tests/fixtures/blosc2/delta_v3.b2f | Bin 0 -> 377 bytes .../tests/fixtures/blosc2/delta_v3.out | Bin 0 -> 900 bytes .../tests/fixtures/blosc2/generate.py | 129 ++ .../tests/fixtures/blosc2/nan_f4.b2f | Bin 0 -> 172 bytes .../tests/fixtures/blosc2/nan_f4.out | Bin 0 -> 2000 bytes .../tests/fixtures/blosc2/nan_f8.b2f | Bin 0 -> 172 bytes .../tests/fixtures/blosc2/nan_f8.out | Bin 0 -> 2400 bytes .../tests/fixtures/blosc2/never_split.b2f | Bin 0 -> 1671 bytes .../tests/fixtures/blosc2/never_split.out | Bin 0 -> 3000 bytes .../tests/fixtures/blosc2/shuffle_meta2.b2f | Bin 0 -> 1003 bytes .../tests/fixtures/blosc2/shuffle_meta2.out | Bin 0 -> 4000 bytes .../tests/fixtures/blosc2/uninit_i8.b2f | Bin 0 -> 172 bytes .../tests/fixtures/blosc2/uninit_i8.out | Bin 0 -> 512 bytes .../tests/fixtures/blosc2/value_f8.b2f | Bin 0 -> 212 bytes .../tests/fixtures/blosc2/value_f8.out | Bin 0 -> 2000 bytes .../tests/fixtures/blosc2/value_i4.b2f | Bin 0 -> 208 bytes .../tests/fixtures/blosc2/value_i4.out | Bin 0 -> 1200 bytes .../tests/fixtures/blosc2/zero_u2.b2f | Bin 0 -> 172 bytes .../tests/fixtures/blosc2/zero_u2.out | Bin 0 -> 4000 bytes .../tests/fixtures/blosc2/zstd_dict.b2f | Bin 0 -> 4978 bytes .../tests/fixtures/blosc2/zstd_dict.err | 1 + crates/clawhdf5/Cargo.toml | 3 +- .../clawhdf5/tests/plugin_filters_interop.rs | 135 +- scripts/ci-test.sh | 4 +- 54 files changed, 1366 insertions(+), 20 deletions(-) create mode 100644 crates/clawhdf5-format/src/filters_blosc2.rs create mode 100644 crates/clawhdf5-format/tests/fixtures/blosc2/always_split.b2f create mode 100644 crates/clawhdf5-format/tests/fixtures/blosc2/always_split.out create mode 100644 crates/clawhdf5-format/tests/fixtures/blosc2/b2nd_2d.b2f create mode 100644 crates/clawhdf5-format/tests/fixtures/blosc2/b2nd_2d.out create mode 100644 crates/clawhdf5-format/tests/fixtures/blosc2/b2nd_3d.b2f create mode 100644 crates/clawhdf5-format/tests/fixtures/blosc2/b2nd_3d.out create mode 100644 crates/clawhdf5-format/tests/fixtures/blosc2/b2nd_4d.b2f create mode 100644 crates/clawhdf5-format/tests/fixtures/blosc2/b2nd_4d.out create mode 100644 crates/clawhdf5-format/tests/fixtures/blosc2/b2nd_zero_chunk.b2f create mode 100644 crates/clawhdf5-format/tests/fixtures/blosc2/b2nd_zero_chunk.out create mode 100644 crates/clawhdf5-format/tests/fixtures/blosc2/bitshuffle_odd_blocks.b2f create mode 100644 crates/clawhdf5-format/tests/fixtures/blosc2/bitshuffle_odd_blocks.out create mode 100644 crates/clawhdf5-format/tests/fixtures/blosc2/bytedelta.b2f create mode 100644 crates/clawhdf5-format/tests/fixtures/blosc2/bytedelta.err create mode 100644 crates/clawhdf5-format/tests/fixtures/blosc2/delta_int16_blosclz.b2f create mode 100644 crates/clawhdf5-format/tests/fixtures/blosc2/delta_int16_blosclz.out create mode 100644 crates/clawhdf5-format/tests/fixtures/blosc2/delta_int32_lz4.b2f create mode 100644 crates/clawhdf5-format/tests/fixtures/blosc2/delta_int32_lz4.out create mode 100644 crates/clawhdf5-format/tests/fixtures/blosc2/delta_shuffle_v16.b2f create mode 100644 crates/clawhdf5-format/tests/fixtures/blosc2/delta_shuffle_v16.out create mode 100644 crates/clawhdf5-format/tests/fixtures/blosc2/delta_uint64_blosclz.b2f create mode 100644 crates/clawhdf5-format/tests/fixtures/blosc2/delta_uint64_blosclz.out create mode 100644 crates/clawhdf5-format/tests/fixtures/blosc2/delta_uint8_lz4.b2f create mode 100644 crates/clawhdf5-format/tests/fixtures/blosc2/delta_uint8_lz4.out create mode 100644 crates/clawhdf5-format/tests/fixtures/blosc2/delta_v3.b2f create mode 100644 crates/clawhdf5-format/tests/fixtures/blosc2/delta_v3.out create mode 100644 crates/clawhdf5-format/tests/fixtures/blosc2/generate.py create mode 100644 crates/clawhdf5-format/tests/fixtures/blosc2/nan_f4.b2f create mode 100644 crates/clawhdf5-format/tests/fixtures/blosc2/nan_f4.out create mode 100644 crates/clawhdf5-format/tests/fixtures/blosc2/nan_f8.b2f create mode 100644 crates/clawhdf5-format/tests/fixtures/blosc2/nan_f8.out create mode 100644 crates/clawhdf5-format/tests/fixtures/blosc2/never_split.b2f create mode 100644 crates/clawhdf5-format/tests/fixtures/blosc2/never_split.out create mode 100644 crates/clawhdf5-format/tests/fixtures/blosc2/shuffle_meta2.b2f create mode 100644 crates/clawhdf5-format/tests/fixtures/blosc2/shuffle_meta2.out create mode 100644 crates/clawhdf5-format/tests/fixtures/blosc2/uninit_i8.b2f create mode 100644 crates/clawhdf5-format/tests/fixtures/blosc2/uninit_i8.out create mode 100644 crates/clawhdf5-format/tests/fixtures/blosc2/value_f8.b2f create mode 100644 crates/clawhdf5-format/tests/fixtures/blosc2/value_f8.out create mode 100644 crates/clawhdf5-format/tests/fixtures/blosc2/value_i4.b2f create mode 100644 crates/clawhdf5-format/tests/fixtures/blosc2/value_i4.out create mode 100644 crates/clawhdf5-format/tests/fixtures/blosc2/zero_u2.b2f create mode 100644 crates/clawhdf5-format/tests/fixtures/blosc2/zero_u2.out create mode 100644 crates/clawhdf5-format/tests/fixtures/blosc2/zstd_dict.b2f create mode 100644 crates/clawhdf5-format/tests/fixtures/blosc2/zstd_dict.err diff --git a/crates/clawhdf5-format/Cargo.toml b/crates/clawhdf5-format/Cargo.toml index 1d1998c..c1ec6df 100644 --- a/crates/clawhdf5-format/Cargo.toml +++ b/crates/clawhdf5-format/Cargo.toml @@ -76,8 +76,10 @@ bitshuffle = ["lz4_flex", "ruzstd"] bzip2 = ["dep:bzip2", "std"] # Blosc 1 (32001) with its BloscLZ, LZ4, Snappy, Zlib and Zstandard codecs. blosc = ["lz4_flex", "ruzstd", "snap", "deflate", "std"] +# Blosc2 (32026), read-only: frames, B2ND arrays, and the Blosc codecs above. +blosc2 = ["blosc"] # Every plugin filter above. -plugin-filters = ["lzf", "bitshuffle", "bzip2", "blosc"] +plugin-filters = ["lzf", "bitshuffle", "bzip2", "blosc", "blosc2"] [[bench]] name = "parallel_decompress_bench" diff --git a/crates/clawhdf5-format/src/filter_registry.rs b/crates/clawhdf5-format/src/filter_registry.rs index 6d39613..442e27c 100644 --- a/crates/clawhdf5-format/src/filter_registry.rs +++ b/crates/clawhdf5-format/src/filter_registry.rs @@ -5,7 +5,8 @@ //! * **Built-in filters** — a static table of the filters compiled into this //! build: the HDF5 standard filters (deflate, shuffle, Fletcher32, szip, //! N-Bit, scale-offset) and the plugin filters whose cargo features are -//! enabled (LZ4, Zstandard, pcodec, LZF, bitshuffle, bzip2, blosc). +//! enabled (LZ4, Zstandard, pcodec, LZF, bitshuffle, bzip2, blosc, +//! blosc2). //! [`builtin_filters`] lists them. //! * **Registered filters** (`std` only) — codecs the application supplies //! for any other ID with [`register_filter`] (a [`FilterCodec`], or just a @@ -166,7 +167,7 @@ pub fn known_filter(id: u16) -> Option<(&'static str, Option<&'static str>)> { 32019 => ("JPEG", None), 32022 => ("BitGroom", None), 32023 => ("Granular BitRound", None), - 32026 => ("Blosc2", None), + 32026 => ("Blosc2", Some("blosc2")), _ => return None, }) } @@ -452,12 +453,12 @@ pub(crate) mod tests { #[test] fn unsupported_filter_error_names_the_filter() { let msg = FormatError::UnsupportedFilter(32026).to_string(); + assert!(msg.contains("Blosc2") && msg.contains("`blosc2`"), "{msg}"); + let msg = FormatError::UnsupportedFilter(32013).to_string(); assert!( - msg.contains("Blosc2") && msg.contains("not implemented"), + msg.contains("ZFP") && msg.contains("not implemented"), "{msg}" ); - let msg = FormatError::UnsupportedFilter(32013).to_string(); - assert!(msg.contains("ZFP"), "{msg}"); let msg = FormatError::UnsupportedFilter(32000).to_string(); assert!(msg.contains("LZF") && msg.contains("`lzf`"), "{msg}"); assert_eq!( diff --git a/crates/clawhdf5-format/src/filters.rs b/crates/clawhdf5-format/src/filters.rs index 36f2e96..d577eac 100644 --- a/crates/clawhdf5-format/src/filters.rs +++ b/crates/clawhdf5-format/src/filters.rs @@ -278,6 +278,13 @@ pub(crate) static BUILTIN_FILTERS: &[BuiltinFilter] = &[ }, encode: None, }, + #[cfg(feature = "blosc2")] + BuiltinFilter { + id: crate::filter_pipeline::FILTER_BLOSC2, + name: "blosc2", + decode: crate::filters_blosc2::blosc2_decode, + encode: None, + }, ]; /// Decode the HDF5 scale-offset filter (id 6). diff --git a/crates/clawhdf5-format/src/filters_blosc.rs b/crates/clawhdf5-format/src/filters_blosc.rs index 3edc964..fadfc1e 100644 --- a/crates/clawhdf5-format/src/filters_blosc.rs +++ b/crates/clawhdf5-format/src/filters_blosc.rs @@ -49,7 +49,7 @@ fn le32(b: &[u8], at: usize) -> Result { /// The codec inside a Blosc frame (flags bits 5-7). #[derive(Debug, Clone, Copy, PartialEq, Eq)] -enum Codec { +pub(crate) enum Codec { BloscLz, Lz4, Snappy, @@ -58,7 +58,7 @@ enum Codec { } impl Codec { - fn from_flags(flags: u8) -> Result { + pub(crate) fn from_flags(flags: u8) -> Result { match flags >> 5 { 0 => Ok(Codec::BloscLz), 1 => Ok(Codec::Lz4), @@ -71,7 +71,7 @@ impl Codec { } /// Decode one codec stream into exactly `dst`. -fn decode_stream( +pub(crate) fn decode_stream( codec: Codec, src: &[u8], dst: &mut [u8], diff --git a/crates/clawhdf5-format/src/filters_blosc2.rs b/crates/clawhdf5-format/src/filters_blosc2.rs new file mode 100644 index 0000000..5f332f8 --- /dev/null +++ b/crates/clawhdf5-format/src/filters_blosc2.rs @@ -0,0 +1,1083 @@ +//! Blosc2 (HDF5 filter 32026, `hdf5-blosc2`, hdf5plugin's `Blosc2`) in pure +//! Rust, read-only: the contiguous frame ("cframe") each chunk is stored +//! as, the Blosc2 chunks inside it, and the B2ND n-dimensional layout. +//! +//! References: c-blosc2's `README_CFRAME_FORMAT.rst`, +//! `README_CHUNK_FORMAT.rst` and `README_B2ND_METALAYER.rst`, and the +//! decoder in `blosc/blosc2.c` (`read_chunk_header`, `blosc_d`, +//! `pipeline_backward`), `blosc/frame.c` and `blosc/b2nd.c`, whose checks +//! this module mirrors. +//! +//! **The filter** (`blosc2_filter.c`). `cd_values`: `[0]` filter revision, +//! `[1]` block size, `[2]` type size, `[3]` chunk size in bytes, `[4]` +//! level, `[5]` filter (0 none, 1 shuffle, 2 bit shuffle, 3 delta, 4 +//! truncate precision), `[6]` codec, and for chunks of 2 to 8 (16 in newer +//! builds) dimensions `[7]` the rank and `[8..]` the chunk dimensions. Each +//! HDF5 chunk is one frame. A frame with a `b2nd` (or older `caterva`) +//! metalayer holds an n-D array the shape of the HDF5 chunk, whose data are +//! cut into blocks of `blockshape` (padded at the chunk's edges) stored one +//! after another, each in C order — so the decoded Blosc2 chunk is +//! reassembled into C order here. Otherwise the frame's first chunk is the +//! HDF5 chunk's data as it is. +//! +//! **Frame.** A msgpack header with fields at fixed offsets (header length, +//! frame length, sizes, type size, chunk size, codec flags) and a map of +//! metalayers; the chunks back to back; an "offsets" chunk (a Blosc2 chunk +//! of little-endian `i64` offsets, one per chunk, relative to the end of the +//! header — negative values encode a chunk of zeros, NaNs or uninitialised +//! values); and a msgpack trailer, which the decoder does not need. +//! +//! **Chunk.** A 16-byte Blosc header (as Blosc 1), extended to 32 bytes when +//! the shuffle and bit-shuffle flags are both set: six filter slots with +//! their metadata, the codec, and flags for dictionaries, lazy chunks and +//! "special" chunks (all zeros, NaN, uninitialised, or one repeated value +//! stored after the header). A regular chunk has a table of block starts; +//! each block is one stream, or `typesize` streams when the "do not split" +//! flag is clear and the block is not the short last one. A stream is an +//! `i32` length and the codec's output: 0 means zeros, negative with a token +//! byte means a run of one byte, and a length equal to the stream's size +//! means stored raw. Decoded blocks run through the filter pipeline +//! backwards (shuffle, bit shuffle, delta; truncate-precision needs no +//! decoding). The codecs are Blosc 1's: BloscLZ, LZ4 (and LZ4HC), Zlib and +//! Zstandard, shared with [`crate::filters_blosc`]. +//! +//! Chunks of uninitialised values read as zeros (libhdf5 hands back +//! whatever its buffer held). Not supported (a clear error, never data): +//! variable-length blocks, dictionaries, lazy chunks, user-defined codecs +//! and registered filters (e.g. bytedelta), sparse frames. + +use crate::error::FormatError; +use crate::filter_registry::FilterContext; +use crate::filters_bitshuffle::bitunshuffle_block; +use crate::filters_blosc::{Codec, blosclz_decompress, decode_stream}; + +const MIN_HEADER: usize = 16; +const EXT_HEADER: usize = 32; + +const FLAG_SHUFFLE: u8 = 0x01; +const FLAG_MEMCPYED: u8 = 0x02; +const FLAG_BITSHUFFLE: u8 = 0x04; +const FLAG_DELTA: u8 = 0x08; +const FLAG_DONT_SPLIT: u8 = 0x10; + +const B2_USEDICT: u8 = 0x01; +const B2_LAZY: u8 = 0x08; +const VL_BLOCKS: u8 = 0x01; + +const SPECIAL_ZERO: u8 = 1; +const SPECIAL_NAN: u8 = 2; +const SPECIAL_VALUE: u8 = 3; +const SPECIAL_UNINIT: u8 = 4; + +const FILTER_NONE: u8 = 0; +const FILTER_SHUFFLE: u8 = 1; +const FILTER_BITSHUFFLE: u8 = 2; +const FILTER_DELTA: u8 = 3; +const FILTER_TRUNC_PREC: u8 = 4; + +/// Chunk format versions: 3 is the Blosc 2 alpha series. +const VERSION_ALPHA: u8 = 3; +/// The newest chunk format c-blosc2 knows (6 added variable-length blocks). +const VERSION_MAX: u8 = 6; +/// The newest frame format c-blosc2 knows. +const FRAME_VERSION_MAX: u8 = 3; +const MAX_BLOCKSIZE: usize = 536_866_816; +/// `B2ND_MAX_DIM` in current c-blosc2 (8 in older builds). +const B2ND_MAX_DIM: usize = 16; + +// Frame header field offsets (c-blosc2 `frame.h`). +const FRAME_HEADER_LEN: usize = 11; +const FRAME_LEN: usize = 16; +const FRAME_FLAGS: usize = 25; +const FRAME_TYPE: usize = 26; +const FRAME_NBYTES: usize = 30; +const FRAME_CBYTES: usize = 39; +const FRAME_TYPESIZE: usize = 48; +const FRAME_CHUNKSIZE: usize = 58; +const FRAME_HEADER_MINLEN: usize = 87; +const FRAME_IDX_SIZE: usize = 89; +const FRAME_VL_BLOCKS: u8 = 0x80; + +fn err(msg: &str) -> FormatError { + FormatError::DecompressionError(format!("blosc2: {msg}")) +} + +fn le_i32(b: &[u8], at: usize) -> Result { + b.get(at..at + 4) + .map(|s| i32::from_le_bytes(s.try_into().unwrap())) + .ok_or_else(|| err("truncated chunk")) +} + +fn be(b: &[u8], at: usize) -> Result<[u8; N], FormatError> { + b.get(at..at + N) + .map(|s| s.try_into().unwrap()) + .ok_or_else(|| err("truncated frame header")) +} + +/// A parsed Blosc2 chunk header. +struct ChunkHeader { + version: u8, + flags: u8, + typesize: usize, + nbytes: usize, + blocksize: usize, + cbytes: usize, + overhead: usize, + filters: [u8; 6], + filters_meta: [u8; 6], + udcodec: u8, + blosc2_flags: u8, + special: u8, +} + +fn read_header(src: &[u8]) -> Result { + if src.len() < MIN_HEADER { + return Err(err("truncated chunk header")); + } + let version = src[0]; + let flags = src[2]; + let typesize = src[3] as usize; + let nbytes = le_i32(src, 4)?; + let blocksize = le_i32(src, 8)?; + let cbytes = le_i32(src, 12)?; + if cbytes < MIN_HEADER as i32 { + return Err(err("chunk size smaller than its header")); + } + if blocksize <= 0 || blocksize as usize > MAX_BLOCKSIZE { + return Err(err("bad block size")); + } + if typesize == 0 { + return Err(err("type size is zero")); + } + if nbytes < 0 { + return Err(err("negative decoded size")); + } + let (nbytes, mut blocksize, cbytes) = (nbytes as usize, blocksize as usize, cbytes as usize); + let mut h = ChunkHeader { + version, + flags, + typesize, + nbytes, + blocksize, + cbytes, + overhead: MIN_HEADER, + filters: [0; 6], + filters_meta: [0; 6], + udcodec: 0, + blosc2_flags: 0, + special: 0, + }; + let mut flags2 = 0; + if flags & (FLAG_SHUFFLE | FLAG_BITSHUFFLE) == FLAG_SHUFFLE | FLAG_BITSHUFFLE { + if cbytes < EXT_HEADER || src.len() < EXT_HEADER { + return Err(err("truncated extended chunk header")); + } + h.overhead = EXT_HEADER; + h.filters.copy_from_slice(&src[16..22]); + h.udcodec = src[22]; + h.filters_meta.copy_from_slice(&src[24..30]); + flags2 = src[30]; + h.blosc2_flags = src[31]; + h.special = (h.blosc2_flags >> 4) & 7; + if h.special == SPECIAL_VALUE { + let ts = cbytes - EXT_HEADER; + if ts == 0 || ts > nbytes || !nbytes.is_multiple_of(ts) { + return Err(err("bad repeated-value chunk")); + } + } else if h.special != 0 && h.special != SPECIAL_ZERO && !nbytes.is_multiple_of(typesize) { + return Err(err("decoded size is not a whole number of elements")); + } + if version == VERSION_ALPHA { + h.filters[5] = 0; + h.filters_meta[5] = 0; + } + } else { + // A Blosc 1 style header: the filters come from the flags. + if flags & FLAG_SHUFFLE != 0 { + h.filters[5] = FILTER_SHUFFLE; + } + if flags & FLAG_BITSHUFFLE != 0 { + h.filters[5] = FILTER_BITSHUFFLE; + } + if flags & FLAG_DELTA != 0 { + h.filters[4] = FILTER_DELTA; + } + } + if version > VERSION_MAX && flags2 & !VL_BLOCKS != 0 { + return Err(err(&format!("chunk format version {version} is too new"))); + } + if flags2 & VL_BLOCKS != 0 { + return Err(err("variable-length blocks are not supported")); + } + if nbytes > 0 && blocksize > nbytes { + blocksize = nbytes; + } + h.blocksize = blocksize; + if cbytes > src.len() { + return Err(err("chunk is longer than its buffer")); + } + Ok(h) +} + +/// Decompress one Blosc2 chunk (header included), refusing to produce more +/// than `limit` bytes. +pub fn blosc2_decompress_chunk(src: &[u8], limit: usize) -> Result, FormatError> { + let h = read_header(src)?; + if h.nbytes > limit { + return Err(err("decoded size exceeds the limit")); + } + let src = &src[..h.cbytes]; + if h.special > SPECIAL_UNINIT { + return Err(err(&format!("unknown special chunk type {}", h.special))); + } + let memcpyed = h.flags & FLAG_MEMCPYED != 0; + if memcpyed && h.cbytes != h.nbytes + h.overhead { + return Err(err("stored chunk has the wrong size")); + } + if h.nbytes == 0 && h.cbytes == h.overhead && h.special == 0 { + return Ok(Vec::new()); + } + let nbytes = h.nbytes; + let mut out = vec![0u8; nbytes]; + if h.special != 0 { + // Filled per block, as c-blosc2 does: each block must hold whole + // values. + let fill: Option<&[u8]> = match h.special { + SPECIAL_ZERO | SPECIAL_UNINIT => None, + SPECIAL_NAN => Some(match h.typesize { + 4 => &[0x00, 0x00, 0xc0, 0x7f], + 8 => &[0, 0, 0, 0, 0, 0, 0xf8, 0x7f], + _ => return Err(err("NaN chunk of a type that is not f32 or f64")), + }), + _ => Some(&src[EXT_HEADER..]), + }; + if let Some(value) = fill { + let ts = value.len(); + let blocksize = h.blocksize.max(1); + if !blocksize.is_multiple_of(ts) || !(nbytes % blocksize).is_multiple_of(ts) { + return Err(err("special chunk blocks are not whole values")); + } + for v in out.chunks_exact_mut(ts) { + v.copy_from_slice(value); + } + } + return Ok(out); + } + if memcpyed { + out.copy_from_slice(&src[h.overhead..]); + return Ok(out); + } + if h.blosc2_flags & B2_USEDICT != 0 { + return Err(err("dictionary-compressed chunks are not supported")); + } + if h.blosc2_flags & B2_LAZY != 0 { + return Err(err("lazy chunks are not supported")); + } + let codec = match h.flags >> 5 { + 6 => { + return Err(err(&format!( + "user-defined codec {} is not supported", + h.udcodec + ))); + } + 2 | 5 | 7 => return Err(err(&format!("unknown codec {}", h.flags >> 5))), + _ => Codec::from_flags(h.flags)?, + }; + for &f in &h.filters { + if f > FILTER_TRUNC_PREC { + return Err(err(&format!("filter {f} is not supported"))); + } + } + let blocksize = h.blocksize; + let nblocks = nbytes.div_ceil(blocksize); + let leftover = nbytes % blocksize; + let bstarts_end = nblocks + .checked_mul(4) + .and_then(|n| n.checked_add(h.overhead)) + .ok_or_else(|| err("block table overflows"))?; + if bstarts_end > src.len() { + return Err(err("block table is truncated")); + } + let dont_split = h.flags & FLAG_DONT_SPLIT != 0; + let mut tmp = vec![0u8; blocksize]; + let mut tmp2 = vec![0u8; blocksize]; + let mut zstd = None; + for j in 0..nblocks { + let is_leftover = j == nblocks - 1 && leftover > 0; + let bsize = if is_leftover { leftover } else { blocksize }; + let start = le_i32(src, h.overhead + 4 * j)?; + if start <= 0 || start as usize >= src.len() { + return Err(err("block start out of range")); + } + let mut pos = start as usize; + let nstreams = if !dont_split && !is_leftover { + h.typesize + } else { + 1 + }; + let neblock = bsize / nstreams; + if neblock == 0 || neblock * nstreams != bsize { + return Err(err("block is not a whole number of streams")); + } + let cur = &mut tmp[..bsize]; + for s in 0..nstreams { + let csize = le_i32(src, pos).map_err(|_| err("stream runs past the chunk"))?; + pos += 4; + let dst = &mut cur[s * neblock..(s + 1) * neblock]; + if csize == 0 { + dst.fill(0); + } else if csize < 0 { + let token = *src + .get(pos) + .ok_or_else(|| err("stream runs past the chunk"))?; + pos += 1; + if token & 1 == 0 || csize < -255 { + return Err(err("unsupported stream token")); + } + dst.fill((-csize) as u8); + } else { + let csize = csize as usize; + let stream = src + .get(pos..pos + csize) + .ok_or_else(|| err("stream runs past the chunk"))?; + pos += csize; + if csize == neblock { + dst.copy_from_slice(stream); + } else if codec == Codec::BloscLz { + if blosclz_decompress(stream, dst) != neblock { + return Err(err("blosclz stream is malformed")); + } + } else { + decode_stream(codec, stream, dst, &mut zstd)?; + } + } + } + let (done, rest) = out.split_at_mut(j * blocksize); + let dest = &mut rest[..bsize]; + run_filters_backward(&h, cur, &mut tmp2[..bsize], done, dest); + } + Ok(out) +} + +/// Undo the chunk's filters on one decoded block `cur`, leaving the result +/// in `dest`. `done` is the chunk decoded so far (the delta filter's +/// reference is the first block). +fn run_filters_backward( + h: &ChunkHeader, + cur: &mut [u8], + scratch: &mut [u8], + done: &[u8], + dest: &mut [u8], +) { + let bsize = cur.len(); + let ts = h.typesize; + for i in (0..6).rev() { + match h.filters[i] { + FILTER_SHUFFLE => { + let m = h.filters_meta[i] as usize; + let bytes = if m == 0 { ts } else { m }; + unshuffle(bytes, cur, scratch); + cur.copy_from_slice(scratch); + } + FILTER_BITSHUFFLE => { + let mut n = bsize / ts; + if h.version == 2 { + if !n.is_multiple_of(8) { + continue; + } + } else { + n -= n % 8; + } + let body = n * ts; + bitunshuffle_block(&cur[..body], &mut scratch[..body], n, ts); + cur[..body].copy_from_slice(&scratch[..body]); + } + FILTER_DELTA => delta_decode(done, cur, ts), + // Truncating precision is lossy and has nothing to undo. + FILTER_NONE | FILTER_TRUNC_PREC => {} + _ => unreachable!("filters are checked before decoding"), + } + } + dest.copy_from_slice(cur); +} + +/// Byte unshuffle (`unshuffle_generic`): trailing bytes that do not make a +/// whole element are copied as they are. +fn unshuffle(bytes: usize, src: &[u8], dest: &mut [u8]) { + let n = src.len() / bytes; + for i in 0..n { + for b in 0..bytes { + dest[i * bytes + b] = src[b * n + i]; + } + } + dest[n * bytes..].copy_from_slice(&src[n * bytes..]); +} + +/// `delta_decoder`: the first block is XOR-accumulated over its own +/// elements; every other block is XORed with the (decoded) first block. +/// Elements are 1, 2, 4 or 8 bytes wide (other sizes: 8 if a multiple of +/// 8, else 1); a trailing partial element is left alone. +fn delta_decode(done: &[u8], cur: &mut [u8], typesize: usize) { + let w = match typesize { + 1 | 2 | 4 | 8 => typesize, + t if t.is_multiple_of(8) => 8, + _ => 1, + }; + let n = cur.len() / w; + if done.is_empty() { + for i in 1..n { + for b in 0..w { + cur[i * w + b] ^= cur[(i - 1) * w + b]; + } + } + } else { + // The first block is at least as long as any other. + for (c, r) in cur[..n * w].iter_mut().zip(done) { + *c ^= *r; + } + } +} + +/// What the frame header says. +struct Frame<'a> { + buf: &'a [u8], + header_len: usize, + nbytes: usize, + cbytes: usize, + typesize: usize, + chunksize: usize, + nchunks: usize, + /// The decoded offsets chunk. + offsets: Vec, +} + +fn parse_frame(buf: &[u8]) -> Result, FormatError> { + if buf.len() < FRAME_HEADER_MINLEN { + return Err(err("truncated frame header")); + } + if buf[0] & 0xf0 != 0x90 || buf[1] != 0xa8 || &buf[2..10] != b"b2frame\0" { + return Err(err("not a Blosc2 frame (bad magic)")); + } + let frame_version = buf[FRAME_FLAGS] & 0x0f; + if frame_version > FRAME_VERSION_MAX { + return Err(err(&format!( + "frame format version {frame_version} is too new" + ))); + } + if buf[FRAME_FLAGS] & FRAME_VL_BLOCKS != 0 { + return Err(err("variable-length blocks are not supported")); + } + if buf[FRAME_TYPE] & 0x0f != 0 { + return Err(err("not a contiguous frame")); + } + let header_len = i32::from_be_bytes(be(buf, FRAME_HEADER_LEN)?); + let frame_len = u64::from_be_bytes(be(buf, FRAME_LEN)?); + let nbytes = i64::from_be_bytes(be(buf, FRAME_NBYTES)?); + let cbytes = i64::from_be_bytes(be(buf, FRAME_CBYTES)?); + let typesize = i32::from_be_bytes(be(buf, FRAME_TYPESIZE)?); + let chunksize = i32::from_be_bytes(be(buf, FRAME_CHUNKSIZE)?); + if header_len < FRAME_HEADER_MINLEN as i32 || header_len as u64 > frame_len { + return Err(err("bad frame header length")); + } + if frame_len > buf.len() as u64 { + return Err(err("frame is longer than the chunk")); + } + if typesize <= 0 { + return Err(err("bad type size")); + } + if nbytes < 0 || cbytes < 0 || chunksize < 0 { + return Err(err("negative size in frame header")); + } + let header_len = header_len as usize; + let buf = &buf[..frame_len as usize]; + let cbytes = usize::try_from(cbytes).map_err(|_| err("bad compressed size"))?; + let data_end = header_len + .checked_add(cbytes) + .filter(|&e| e <= buf.len()) + .ok_or_else(|| err("chunks run past the frame"))?; + let nbytes = usize::try_from(nbytes).map_err(|_| err("bad decoded size"))?; + let chunksize = chunksize as usize; + // The offsets chunk follows the data chunks. + let off_src = &buf[data_end..]; + let expected = if nbytes == 0 { + 0 + } else if chunksize > 0 { + nbytes.div_ceil(chunksize) + } else { + // Variable-size chunks: the offsets chunk tells how many. + usize::MAX + }; + let off_limit = if expected == usize::MAX { + 1 << 24 + } else { + expected + .checked_mul(8) + .ok_or_else(|| err("too many chunks"))? + }; + let offsets = if nbytes == 0 { + Vec::new() + } else { + blosc2_decompress_chunk(off_src, off_limit)? + }; + if !offsets.len().is_multiple_of(8) || (expected != usize::MAX && offsets.len() != off_limit) { + return Err(err("offsets chunk does not match the number of chunks")); + } + Ok(Frame { + buf, + header_len, + nbytes, + cbytes, + typesize: typesize as usize, + chunksize, + nchunks: offsets.len() / 8, + offsets, + }) +} + +impl Frame<'_> { + /// Decode chunk `n`, refusing more than `limit` bytes. + fn chunk(&self, n: usize, limit: usize) -> Result, FormatError> { + if n >= self.nchunks { + return Err(err("frame has no such chunk")); + } + let raw: [u8; 8] = self.offsets[8 * n..8 * n + 8].try_into().unwrap(); + let offset = i64::from_le_bytes(raw); + if offset < 0 { + // A special chunk, recorded in the offset's top byte. + if self.chunksize == 0 { + return Err(err("special chunk in a frame without a chunk size")); + } + let size = if n == self.nchunks - 1 && !self.nbytes.is_multiple_of(self.chunksize) { + self.nbytes % self.chunksize + } else { + self.chunksize + }; + if size > limit { + return Err(err("decoded size exceeds the limit")); + } + let kind = raw[7] & 7; + return match kind { + SPECIAL_ZERO | SPECIAL_UNINIT => Ok(vec![0u8; size]), + SPECIAL_NAN => { + let nan: &[u8] = match self.typesize { + 4 => &[0x00, 0x00, 0xc0, 0x7f], + 8 => &[0, 0, 0, 0, 0, 0, 0xf8, 0x7f], + _ => return Err(err("NaN chunk of a type that is not f32 or f64")), + }; + if !size.is_multiple_of(nan.len()) { + return Err(err("NaN chunk is not whole values")); + } + Ok(nan.iter().copied().cycle().take(size).collect()) + } + _ => Err(err(&format!("unknown special chunk offset {kind}"))), + }; + } + // Every chunk lies in the data section, between the header and the + // offsets chunk. + let end = self.header_len + self.cbytes; + let start = usize::try_from(offset) + .ok() + .and_then(|o| self.header_len.checked_add(o)) + .filter(|&s| s < end && end - s >= MIN_HEADER) + .ok_or_else(|| err("chunk offset out of range"))?; + blosc2_decompress_chunk(&self.buf[start..end], limit) + } + + /// The content of metalayer `name`, if the frame has it. + fn metalayer(&self, name: &[u8]) -> Result, FormatError> { + let h = &self.buf[..self.header_len]; + // [FRAME_IDX_SIZE] is the u16 index size, then a map16. + let mut p = FRAME_IDX_SIZE + 2; + if h.get(p) != Some(&0xde) { + return Err(err("bad metalayer index")); + } + let count = u16::from_be_bytes(be(h, p + 1)?) as usize; + p += 3; + if count > 16 { + return Err(err("too many metalayers")); + } + for _ in 0..count { + let tag = *h.get(p).ok_or_else(|| err("truncated metalayer index"))?; + if tag & 0xe0 != 0xa0 { + return Err(err("bad metalayer name")); + } + let len = (tag & 0x1f) as usize; + let key = h + .get(p + 1..p + 1 + len) + .ok_or_else(|| err("truncated metalayer index"))?; + p += 1 + len; + if h.get(p) != Some(&0xd2) { + return Err(err("bad metalayer offset")); + } + let off = i32::from_be_bytes(be(h, p + 1)?); + p += 5; + if key != name { + continue; + } + if off < 0 || off as usize >= h.len() { + return Err(err("metalayer offset out of range")); + } + let off = off as usize; + if h.get(off) != Some(&0xc6) { + return Err(err("bad metalayer content")); + } + let clen = u32::from_be_bytes(be(h, off + 1)?) as usize; + return h + .get(off + 5..) + .and_then(|c| c.get(..clen)) + .map(Some) + .ok_or_else(|| err("metalayer runs past the header")); + } + Ok(None) + } +} + +/// The B2ND (or Caterva) metalayer: shape, chunk shape, block shape. +#[derive(Debug, PartialEq, Eq)] +struct NdMeta { + shape: Vec, + chunkshape: Vec, + blockshape: Vec, +} + +/// A msgpack array header: fixarray or array16. +fn array_len(m: &[u8], p: &mut usize) -> Result { + let t = *m.get(*p).ok_or_else(|| err("truncated b2nd metalayer"))?; + if t & 0xf0 == 0x90 { + *p += 1; + Ok((t & 0x0f) as usize) + } else if t == 0xdc { + let n = u16::from_be_bytes(be(m, *p + 1)?) as usize; + *p += 3; + Ok(n) + } else { + Err(err("bad b2nd metalayer")) + } +} + +fn parse_nd(m: &[u8]) -> Result { + let bad = || err("bad b2nd metalayer"); + let mut p = 0; + if array_len(m, &mut p)? < 5 { + return Err(bad()); + } + // Version, then the rank: positive fixints. + let version = *m.get(p).ok_or_else(bad)?; + let ndim = *m.get(p + 1).ok_or_else(bad)? as usize; + if version > 0x7f || ndim == 0 || ndim > B2ND_MAX_DIM { + return Err(err(&format!("unsupported b2nd rank {ndim}"))); + } + p += 2; + let mut read = |wide: bool| -> Result, FormatError> { + if array_len(m, &mut p)? != ndim { + return Err(bad()); + } + (0..ndim) + .map(|_| { + let v = if wide { + if m.get(p) != Some(&0xd3) { + return Err(bad()); + } + let v = i64::from_be_bytes(be(m, p + 1)?); + p += 9; + v + } else { + if m.get(p) != Some(&0xd2) { + return Err(bad()); + } + let v = i32::from_be_bytes(be(m, p + 1)?) as i64; + p += 5; + v + }; + usize::try_from(v).map_err(|_| bad()) + }) + .collect() + }; + let shape = read(true)?; + let chunkshape = read(false)?; + let blockshape = read(false)?; + Ok(NdMeta { + shape, + chunkshape, + blockshape, + }) +} + +/// Decode a frame as the HDF5 filter does: a B2ND array into C order, or +/// else the frame's first chunk. `cd_shape` is the chunk shape recorded in +/// the filter's `cd_values` (if any), which the array's shape must match. +fn decode_frame( + input: &[u8], + limit: usize, + cd_shape: Option<&[usize]>, +) -> Result, FormatError> { + let frame = parse_frame(input)?; + let meta = match frame.metalayer(b"b2nd")? { + Some(m) => Some(m), + None => frame.metalayer(b"caterva")?, + }; + let Some(meta) = meta else { + return frame.chunk(0, limit); + }; + let nd = parse_nd(meta)?; + if let Some(cd) = cd_shape + && cd != nd.shape.as_slice() + { + return Err(err(&format!( + "array shape {:?} is not the chunk shape {cd:?}", + nd.shape + ))); + } + reassemble(&frame, &nd, limit) +} + +/// Gather a B2ND array's blocks into one C-order buffer. +fn reassemble(frame: &Frame<'_>, nd: &NdMeta, limit: usize) -> Result, FormatError> { + let ts = frame.typesize; + let ndim = nd.shape.len(); + let too_big = || err("array exceeds the chunk size"); + let mut total = ts; + let mut ext = vec![0usize; ndim]; + let mut grid = vec![0usize; ndim]; + let mut ext_bytes = ts; + for i in 0..ndim { + let (s, c, b) = (nd.shape[i], nd.chunkshape[i], nd.blockshape[i]); + if c == 0 || b == 0 || b > c { + return Err(err("bad b2nd chunk or block shape")); + } + total = total.checked_mul(s).ok_or_else(too_big)?; + ext[i] = c.div_ceil(b) * b; + ext_bytes = ext_bytes.checked_mul(ext[i]).ok_or_else(too_big)?; + grid[i] = s.div_ceil(c); + } + if total > limit { + return Err(too_big()); + } + let mut out = vec![0u8; total]; + if total == 0 { + return Ok(out); + } + // Padding to whole blocks grows a chunk by less than 2x per dimension; + // hdf5-blosc2's block shapes keep it far below 16x (a crafted frame + // could otherwise make each chunk enormous). + if ext_bytes > limit.saturating_mul(16) { + return Err(too_big()); + } + let nchunks: usize = grid.iter().product(); + if nchunks != frame.nchunks { + return Err(err("chunk count does not match the b2nd shape")); + } + let blocks_in_chunk: Vec = (0..ndim).map(|i| ext[i] / nd.blockshape[i]).collect(); + let nblocks: usize = blocks_in_chunk.iter().product(); + let block_items: usize = nd.blockshape.iter().product(); + let block_bytes = block_items * ts; + // Strides (in elements) of the output array and of a block. + let mut out_stride = vec![1usize; ndim]; + let mut blk_stride = vec![1usize; ndim]; + for i in (0..ndim.saturating_sub(1)).rev() { + out_stride[i] = out_stride[i + 1] * nd.shape[i + 1]; + blk_stride[i] = blk_stride[i + 1] * nd.blockshape[i + 1]; + } + let mut cidx = vec![0usize; ndim]; + let mut bidx = vec![0usize; ndim]; + let mut gstart = vec![0usize; ndim]; + let mut valid = vec![0usize; ndim]; + let mut pos = vec![0usize; ndim]; + for n in 0..nchunks { + unravel(n, &grid, &mut cidx); + let data = frame.chunk(n, ext_bytes)?; + if data.len() != ext_bytes { + return Err(err("chunk size does not match the b2nd shape")); + } + for b in 0..nblocks { + unravel(b, &blocks_in_chunk, &mut bidx); + let mut empty = false; + for i in 0..ndim { + let in_chunk = bidx[i] * nd.blockshape[i]; + gstart[i] = cidx[i] * nd.chunkshape[i] + in_chunk; + let lim_chunk = nd.chunkshape[i].saturating_sub(in_chunk); + let lim_shape = nd.shape[i].saturating_sub(gstart[i]); + valid[i] = nd.blockshape[i].min(lim_chunk).min(lim_shape); + empty |= valid[i] == 0; + } + if empty { + continue; + } + let block = &data[b * block_bytes..(b + 1) * block_bytes]; + // Copy row by row along the last dimension. + let row = valid[ndim - 1] * ts; + let rows: usize = valid[..ndim - 1].iter().product(); + for r in 0..rows { + unravel(r, &valid[..ndim - 1], &mut pos[..ndim - 1]); + let mut src = 0; + let mut dst = gstart[ndim - 1]; + for i in 0..ndim - 1 { + src += pos[i] * blk_stride[i]; + dst += (gstart[i] + pos[i]) * out_stride[i]; + } + out[dst * ts..dst * ts + row].copy_from_slice(&block[src * ts..src * ts + row]); + } + } + } + Ok(out) +} + +/// C-order multi-index of `n` in a grid of `dims`. +fn unravel(mut n: usize, dims: &[usize], idx: &mut [usize]) { + for i in (0..dims.len()).rev() { + idx[i] = n % dims[i]; + n /= dims[i]; + } +} + +/// Decompress one HDF5 chunk written by the Blosc2 filter (a Blosc2 frame) +/// into its data, in C order; at most `limit` bytes. +pub fn blosc2_decompress(input: &[u8], limit: usize) -> Result, FormatError> { + decode_frame(input, limit, None) +} + +/// The filter's decoder. +pub(crate) fn blosc2_decode(input: &[u8], ctx: &FilterContext<'_>) -> Result, FormatError> { + let cd = ctx.client_data(); + // cd_values[7] is the chunk rank for B2ND (> 1), and the chunk + // dimensions follow. + let cd_shape: Option> = if cd.len() >= 8 { + let rank = cd[7] as usize; + if rank < 2 || cd.len() < 8 + rank { + return Err(err("bad chunk rank in the filter parameters")); + } + Some(cd[8..8 + rank].iter().map(|&d| d as usize).collect()) + } else { + None + }; + let out = decode_frame(input, ctx.output_limit(), cd_shape.as_deref())?; + if out.is_empty() && ctx.max_output != 0 { + return Err(err("empty frame for a non-empty chunk")); + } + Ok(out) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::filter_pipeline::{FILTER_BLOSC2, FilterDescription}; + use std::path::PathBuf; + + fn fixtures() -> PathBuf { + PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("tests/fixtures/blosc2") + } + + /// Every fixture frame (see `tests/fixtures/blosc2/generate.py`), + /// sorted by name: (name, frame, expected output or error word). + #[allow(clippy::type_complexity)] + fn cases() -> Vec<(String, Vec, Result, String>)> { + let mut v = Vec::new(); + for e in std::fs::read_dir(fixtures()).unwrap() { + let p = e.unwrap().path(); + if p.extension().is_none_or(|x| x != "b2f") { + continue; + } + let name = p.file_stem().unwrap().to_string_lossy().into_owned(); + let frame = std::fs::read(&p).unwrap(); + let want = match std::fs::read(p.with_extension("out")) { + Ok(out) => Ok(out), + Err(_) => Err(std::fs::read_to_string(p.with_extension("err")).unwrap()), + }; + v.push((name, frame, want)); + } + v.sort_by(|a, b| a.0.cmp(&b.0)); + assert!(v.len() >= 20, "fixtures missing"); + v + } + + #[test] + fn fixture_frames_decode_exactly() { + for (name, frame, want) in cases() { + let got = blosc2_decompress(&frame, 1 << 20); + match want { + Ok(want) => { + let got = got.unwrap_or_else(|e| panic!("{name}: {e}")); + assert!(got == want, "{name}: decoded data differs"); + } + Err(word) => { + let e = got.expect_err(&name).to_string(); + assert!(e.contains(word.trim()), "{name}: {e}"); + } + } + } + } + + /// The output limit is enforced for every kind of frame, including the + /// padded chunks of B2ND arrays and special chunks. + #[test] + fn output_limit_is_enforced() { + for (name, frame, want) in cases() { + let Ok(want) = want else { continue }; + assert!( + blosc2_decompress(&frame, want.len() - 1).is_err(), + "{name}: decoded past the limit" + ); + assert_eq!( + blosc2_decompress(&frame, want.len()).unwrap(), + want, + "{name}" + ); + } + } + + fn desc(cd: Vec) -> FilterDescription { + FilterDescription { + filter_id: FILTER_BLOSC2, + name: None, + flags: 0, + client_data: cd, + } + } + + /// As the HDF5 filter: the B2ND array must have the chunk shape the + /// filter parameters record. + #[test] + fn b2nd_shape_must_match_the_filter_parameters() { + let frame = std::fs::read(fixtures().join("b2nd_2d.b2f")).unwrap(); + let want = std::fs::read(fixtures().join("b2nd_2d.out")).unwrap(); + let decode = |cd: Vec| { + let f = desc(cd); + blosc2_decode( + &frame, + &FilterContext { + filter: &f, + element_size: 4, + max_output: want.len(), + }, + ) + }; + assert_eq!(decode(vec![1, 0, 4, 0, 5, 1, 1, 2, 37, 29]).unwrap(), want); + assert_eq!(decode(vec![1, 0, 4, 0, 5, 1, 1]).unwrap(), want); + assert!(decode(vec![1, 0, 4, 0, 5, 1, 1, 2, 29, 37]).is_err()); + assert!(decode(vec![1, 0, 4, 0, 5, 1, 1, 3, 37, 29, 1]).is_err()); + assert!(decode(vec![1, 0, 4, 0, 5, 1, 1, 2, 37]).is_err()); + } + + /// A 32-byte extended header. + fn ext_header(flags: u8, ts: u8, nbytes: i32, blocksize: i32, cbytes: i32) -> Vec { + let mut h = vec![5u8, 1, flags | FLAG_SHUFFLE | FLAG_BITSHUFFLE, ts]; + for v in [nbytes, blocksize, cbytes] { + h.extend_from_slice(&v.to_le_bytes()); + } + h.resize(EXT_HEADER, 0); + h + } + + #[test] + fn special_chunks() { + // Zeros, uninitialised (read as zeros), NaN, one repeated value. + let mut z = ext_header(0, 4, 64, 64, 32); + z[31] = SPECIAL_ZERO << 4; + assert_eq!(blosc2_decompress_chunk(&z, 64).unwrap(), vec![0; 64]); + z[31] = SPECIAL_UNINIT << 4; + assert_eq!(blosc2_decompress_chunk(&z, 64).unwrap(), vec![0; 64]); + z[31] = SPECIAL_NAN << 4; + let nan = blosc2_decompress_chunk(&z, 64).unwrap(); + assert!( + nan.chunks(4) + .all(|c| f32::from_le_bytes(c.try_into().unwrap()).is_nan()) + ); + z[3] = 2; + assert!(blosc2_decompress_chunk(&z, 64).is_err(), "NaN of i16"); + let mut v = ext_header(0, 2, 12, 12, 35); + v[31] = SPECIAL_VALUE << 4; + v.extend_from_slice(&[1, 2, 3]); + assert_eq!( + blosc2_decompress_chunk(&v, 12).unwrap(), + [1, 2, 3].repeat(4) + ); + // The value must divide the chunk (and its blocks). + v[4] = 13; + assert!(blosc2_decompress_chunk(&v, 13).is_err()); + v[4] = 12; + v[8] = 4; + assert!(blosc2_decompress_chunk(&v, 12).is_err()); + // Special types 5 to 7 are reserved. + let mut r = ext_header(0, 4, 64, 64, 32); + r[31] = 5 << 4; + assert!(blosc2_decompress_chunk(&r, 64).is_err()); + } + + /// A regular chunk: one block of two streams (split, type size 2): a + /// run of 0x07 and a stream of zeros, then the byte shuffle undone. + #[test] + fn stream_runs_and_zero_streams() { + let mut c = ext_header(0, 2, 8, 8, 0); + c[16 + 5] = FILTER_SHUFFLE; + c.extend_from_slice(&36i32.to_le_bytes()); + c.extend_from_slice(&(-7i32).to_le_bytes()); + c.push(1); + c.extend_from_slice(&0i32.to_le_bytes()); + let n = c.len() as i32; + c[12..16].copy_from_slice(&n.to_le_bytes()); + assert_eq!( + blosc2_decompress_chunk(&c, 8).unwrap(), + [7, 0, 7, 0, 7, 0, 7, 0] + ); + // A token without the run bit is reserved. + let mut bad = c.clone(); + bad[40] = 2; + assert!(blosc2_decompress_chunk(&bad, 8).is_err()); + // Unsupported header features are errors, not data. + for (at, bit) in [(31, B2_USEDICT), (31, B2_LAZY), (30, VL_BLOCKS)] { + let mut x = c.clone(); + x[at] |= bit; + assert!( + blosc2_decompress_chunk(&x, 8).is_err(), + "byte {at} bit {bit}" + ); + } + let mut udf = c.clone(); + udf[16] = 35; + assert!(blosc2_decompress_chunk(&udf, 8).is_err()); + let mut udc = c.clone(); + udc[2] |= 6 << 5; + assert!(blosc2_decompress_chunk(&udc, 8).is_err()); + assert!(blosc2_decompress_chunk(&c, 7).is_err(), "limit"); + } + + /// Random and mutated frames: errors are fine, panics are not, and no + /// output past the limit. + #[test] + fn fuzzed_frames_never_panic() { + let limit = 20_000; + let seeds: Vec> = cases() + .into_iter() + .filter(|(_, f, w)| f.len() < 9000 && w.as_ref().is_ok_and(|w| w.len() <= limit)) + .map(|(_, f, _)| f) + .collect(); + assert!(seeds.len() >= 15); + crate::test_fuzz::fuzz_decoder(0xb2, &seeds, 20_000, limit, |s| { + blosc2_decompress(s, limit) + }); + } + + /// Blosc2 chunks on their own, mutated. + #[test] + fn fuzzed_chunks_never_panic() { + let mut seeds = Vec::new(); + for (_, f, w) in cases() { + if w.is_err() { + continue; + } + let frame = parse_frame(&f).unwrap(); + let raw: [u8; 8] = frame.offsets[..8].try_into().unwrap(); + let off = i64::from_le_bytes(raw); + if off >= 0 { + let start = frame.header_len + off as usize; + let end = frame.header_len + frame.cbytes; + let h = read_header(&f[start..end]).unwrap(); + seeds.push(f[start..start + h.cbytes].to_vec()); + } + } + assert!(seeds.len() >= 10); + crate::test_fuzz::fuzz_decoder(0xb3, &seeds, 20_000, 1 << 16, |s| { + blosc2_decompress_chunk(s, 1 << 16) + }); + } +} diff --git a/crates/clawhdf5-format/src/lib.rs b/crates/clawhdf5-format/src/lib.rs index 3e30f9d..89d324b 100644 --- a/crates/clawhdf5-format/src/lib.rs +++ b/crates/clawhdf5-format/src/lib.rs @@ -86,6 +86,8 @@ pub mod filters; mod filters_bitshuffle; #[cfg(feature = "blosc")] pub mod filters_blosc; +#[cfg(feature = "blosc2")] +pub mod filters_blosc2; #[cfg(feature = "bzip2")] mod filters_bzip2; #[cfg(feature = "lzf")] diff --git a/crates/clawhdf5-format/tests/fixtures/blosc2/always_split.b2f b/crates/clawhdf5-format/tests/fixtures/blosc2/always_split.b2f new file mode 100644 index 0000000000000000000000000000000000000000..db6322af43cc9a0dd615133bf4c3e1376a9f98fb GIT binary patch literal 1650 zcmZ8hYgkQb6n?*bI;T?(ozAHws#A(oq>FM1yF?VCLdK;+kwhiQFlI9mCheqVav6?$ zO-(Y4X+k!d+)0c}OnQtv(;Vb7G}El`V;<{yzV+;HueI0vzU%w;+EO&lD?T+gDGqf2 zVlR>-Q>5x%b@jQfyJe*JkEHvqQ)IdeAB zn*nXBM*cD3f0<4nU)^JXJs7lVfq_%P0&PnLaM&k6lE8!KC8;wE^nt(&v=t->;3Xxt zNjwCJq7qyYAV>ml3RpwO^Kz)AFqo}_C6^Ai6k4jP74&UY&82A24*784c@xA6R37{x znDa|1=1pHqZctNhAR!4hh^Dd<5wzk~z!Iy#c{8JYN(cc~x0T31Z;2OD2vE%&1xdlU za;ha~grO4zC+5K89GERUc$6|L=D>_|AS+>3sA9;3gCw;8DrPRU!&uS>>|`3jpJ6jF z3ao%i-DIHtiSU3>%q-;!$SfE$f~=a!f;lW>o;;5Xrjce4sUpVhW)2VvN$n}76Zle4 zfMYtQ7Obhydzi*3k11R^E8!}~anSHtj0xaQapqEA4oarsi-BCyMfAH0{3gaFaaySh zvzJPkg)0O#Q{=&@FEf{{8C!xObOL-W=Se40#;JK*#-cdp!*S^{wUkEd07;_rA%?;- z8DJ9ig>Box@}i+Y59SJl~u=n zJ$v~_m%?l(9%EOOF}*{KVKud>#xU6)_D zY3ugg-~Uj)zoN40xQOQ_m#@_~+`0SciQ&bozyASeq%byVr|O{A*fB>toe0=5Pj4ST zA{0Q7@{rI`VPnH1$48O)Xc8YkDd)xu=YDK2d3z`TDY|bM^O|WlD1^ zTPHWK0eo=S_$hITsTp5pX06FtSFmBz)*X9@#ets>SDg?cUAcC%;m*BBPyc%H`ke@^ zoU~h5cI?zyL`ruTH}_t>z3iBi{~!?sqr=8V5d3G8Md*^|k$_mR@3Of?lN8{hZ`~E#?&zZr1PEQ&P6?r;0;IkJp^OP{K{0yF*obixNdDBA*y(WuoR~B(g#w)xH74sHD&2=q7P2=MRaDnCH2tek zX*=t>d-(bXg+@fh#wVpOUbbQt2{YUH-L{=Yr37DjSfU@{LLI?35`5G1<~J>^XE<-A zsY-;#Mnp`v?ye+%qRfV_USv2yPZXhVyYKVfUp#jjq6y#w0s;aV?1wNo+6OUsAqVyW zIp}RB;-r0`h}Kr!n4mV3Gh-#q%C>n47h3O)CibV6&uH>1K1}`#6zqEa(+vCv_s$vv literal 0 HcmV?d00001 diff --git a/crates/clawhdf5-format/tests/fixtures/blosc2/always_split.out b/crates/clawhdf5-format/tests/fixtures/blosc2/always_split.out new file mode 100644 index 0000000000000000000000000000000000000000..a3faac59e55caf561a9670a4b6b8ef352bd0187b GIT binary patch literal 4000 zcmZwK3(!?n8He$OC?TPQ7x0Dw333Yr2{cd_@ChiC6eyUepn`~4C?Hf*t;`IKipaa&pIi^zxJHc zya_37LOZ%oOlc6>un1RS4Q|3lY{7ka80~l#yRqM-ln%t9I0F4K1S2pOC!-D1F%#$D z0$hTnSb=xqDqJ%um20sEYw;0$+H68-Hf|&kNMl!hWpJw#831) z@UZ?d_19`Uevh5zKdDcv&!~S_U%+mCYD={fI=A^hwx#?sb$_)Ry4xRs9&M@g#KEo| zir)Ie)FW_|zK?rfsrFM}Wj8<_XddL*!RBH5SF0oJN7{{2N863jkF|fTI?nDS{VDd@ zJki{yo{A|r&9!OjboC6^W~gseXSx4O``PN*=5y@NHJ_)>Ghd)yWL}`ZSbvH87VDSb z%~)!8h21j!3in;9UuBng)0Zzx8+Uf9hwxw+rXt+A8rNez?!Z?301x2_>_YO6yP^kr z;VAUSV2nh#a|))Q<<4CFg;chdLpef@*#c9lDcJLOUJag{rTJLO3;cM5mX-&F3Tf2n)9Q#!a)xl_244)i{` zQ@NA5letrSfjgNynLFudGk0n~Gj}p~%0QJnd5HZ`_YBu_r*J17kJsqAQ%W zPQ4u5Dcs2`^&Q<=&6zFO)wna753{fY*I+Gf!6w{|2k;1XB66qYPUKE+^uYi`?i`O3 z(Q@Yu{VYW8T!e7v3amuId+!)s7CJWQp26+@lX6`QmV=O+Xwrii}xmX^8R?ao;#U4 z`4u?GyKUTw+zEGT<4)mD>TB-j9_|$G)PZW`4tMe}Rqhn-)W)4EchXqbxYNp=DtC(9 z$rJ24xRbe)Pxox&PUKE{gPA*pJNYck(aW9EdAR;KjIx(IS?*N1Q@K+kcgDMRvOd1E zCg>Y?>SR0aq{f}}de<9w(o8*fDt8KZ(wi{X>^m#miQFmNDRQUfPUTMFPLVt5E$-RN zowUleaA!Stj^oUFY)4lfggfIg3rn#YH{v$jg>Yv(!kwq^JofRfyWt@8#?csn;TVna zn26I6?##ydScsN8Z_~@2B6q@_^nQE@*MmDn?vzjI*Q4dm?fN_Hzl^UTa_5_N-%{mH zxet*$KUU*Ai#zEN{0hG@NA7SZJ*9>_awk0p?v(h>dQsoWyW&ojJGqN@)>Yr+PCmfg z13m4ylNxtwPUcP;?HTSw?xYj#8+S5y z9CEx%*SS+pRVRZxkvn<1Yuu^asdA@qC!eV{?qu#%xfAXb?xgv+P|uybP+eqR44L!) z+{tn$+^H>hB6nKubmze!WX#2KtUh>rcAy zFbDH+F_vH%-tMz{x7y@RT4Qe9DIe2E?r^7Yr*NmpotMtZU-8^#^EcFcRPOA_9qzz}#m~);s*OA8cjhO}+(~~?pHaC}TDkMzNhx>ouDDYgcXH!SB!81Txp618>A91p zsHdsZT<1>aPMV?TPHx;uv;8f(6YfOrr1Kzi>U{TcCoRw~1a~rbs@$n^r^uarx$DcY z!ralFcDb{(aVO5nkvqdM72!@nxU&u$umvr59@mFEFQSWg-W}o2k?4zKF#_Svc(mM^ zsSkJNBXZ|5T#3e=@?M|aYIEaGkvn_bS!efY{mr;lzX2O@rpeq*I)?ZcfH^t-{G+R6Lt?A`4LXPwHO)D7LeOYX!uIm?|m zCs*#|#+^7PSGkirxKkT<;x|^6J86i&A$Q_8R^?6_>3+FWxRbe)PS8j0a3_9a)yADX z(afF5ozl3I8+QtK3U}h1+{&GBC!g(k?v%!z+_+P@lN)#9H&(dQ`i)h&6X)d2o%oHF KR(gI_+y4N4t(Zyx literal 0 HcmV?d00001 diff --git a/crates/clawhdf5-format/tests/fixtures/blosc2/b2nd_2d.b2f b/crates/clawhdf5-format/tests/fixtures/blosc2/b2nd_2d.b2f new file mode 100644 index 0000000000000000000000000000000000000000..a092e297936a93a2a0d0ef8263ff8a8e5adc56ab GIT binary patch literal 3950 zcma)9dtA-g7GJ+}oQiWery`}7N`;b2RIc(mxz&WoBV{;6nJ9xO-Dr9sbQ3)=k}yRj z6iLER9+L<~gmimgG8Z-8Gn#SN{{7B5bM_g3-2M5i-`?kU*4lgRwbpm7_1jkv>=?Q# zC?W(J03iPsK5&Sa=jp&~-W~3jA;=f~rVyrHOi{% zn7m-e<%~jvUPj3c5igP#8dR(8txddkmr29zD_XoOqU4JCR-VU_s?>h@8wqJ zOlGN6?kRxpW!N5I#cHCtUs53+%#Up|YtIf84y$k+sAOCNU_S=*F;4luJV*+b(7M*t z-F@7_D=a;8@u#||^@&^3caQg+I(z-RHs z@p^uiXKSY9x&p9PsB19T%I<#46JBp^a=ZjGCfRieX}7m`c76D>S8>_ds+wDJw{LeG zh%u<@Oa2$o9F1oYU+zMm-9{?Hbt{qyuF-Q4jznA7d-~jkq2anl=EEIaT7G`@<`ciD z^vs2;bqWg`oG#W=i{rsoBSwv#^vmmyp9P|_^OtIGKlpA_Y8J+x0>zQGeZ`J=Di<+R z8Tc3#yCG%AZvO>K!q==DV&mXE!QG_FSHoC&Jdq0nH%uH)1R@=M)0$iNA9uXS_ zzhwGZ)AS0;X&(D4NjdMCq{M%>2OcW+WSNCiFrXwRz$D2v>D|(!SmM-pEZO! zvJwl>G$Q?^XK#-<*fXNUW5;|S) z_vIfctE?slU=2(P9xW0Yu{t(si=*pAtV=z;MA`;sH8&d{cl=i_B4wkF_G1{{Yg3BP z%s|;U-|CqTv2}87@BH<{9|DCRrStDetMy$_W6t;3R7iR0yAWUyxHxPTfR&w-8^CJ} za#d7dI)wJ%Jbx$tR65wwv-cMsKOGg9v^8VTBro5&3l?J%%;);ojov-zkKuifr8TV3s)&NDhStt zrbG{*?(V}jwlt&OI(d>Ous}5cuAwv8p%wrN5I5qVLga}s}s<^E3QceBZ zxJ{`UdnbF%@Lw3(SI@}8#*vL}Qk%a7=6jEyzI^-XKp|H3%Mq*BC2dY~cK4X(7oa9Y zUELGl`p!ipMId5YHCH4VxAlM4JX3$D4vhtDaWwkrp1)k%aK9xvEi32X5kCaR@+h`_ zQ?z{|bG4u}vKVIX?EaM0quwPI7k|3ZxIQUm$FBX#I3_iQ&H*dH@2?zShQ{JzkK_;& z%i#`V$9HOw4uS)1mLnEj%153aAZb2C_Gc5t7Llf*#Ous5My-B(PMtM>@v`Aq!lw9e zTe0a#)se~Q6n3aNawfh{!arUkRUwh~nxH!tiL~J~ECUxUT@f8;W$T0uWA}$Y1!zN3 zYFp(NkedAyivdb0hg$R>!=@bfPT0H+>w%BoJp4Y;Gr`6PhrCHwbcuqMbrW<);69we zEEy=^OrDwjLs7|TSx5vndcbEiuxiO5x+{d&{5a8`7TBDYm7Q0#bY)Cj@-`>z*L>&w zO~6KZ!T<(pn%kM)02i**Aw;+A*qw{YaJpefG%g_Rfesbc-5dX%i8;X6pVSK@i{bWT z+<$)gwgDUv6U%8{spJFnL_wLa_0 z%E>=+q7t)RRNRIUqg*F?`I3WG+Ez-a``st(Z49ce*4=G>l9Imrz~SR_z79d-kWUp> zTSVat9_2k6TZ7?wdHk2KMKvV_k^$HekHX&K)rX!#i2Ukn_+5hr9BbYjsZX<#ua((f z^ZRJ)n_6SF?|AV>&Z^VpA~MEnbI#Al#&{9feI%cVe9f8dDU*F%Lledbn1Naokcp5C zi5&|2Dt_av_`DP*BFf5@O}TPzlpN(6cQiMNu(gj@GoKl=e>~>*`uzU-&gcD{*StTeQ&LjWlZhPUq9Da6Nja+UIQ3~l z3!bG5J?O_^MlgoSOk)lUS;-nU@(G`_mm{3wJU?)qo1{&Zl5!_mxR?7WLUA6Y64j|g zLz>ctjyz8<1~8P-yulP^GM^=Uz*;u5lP@{QG0t+4EBr#L)ZP)9$WCqwQjAiRqbfD2 zM`NC*Ezi-NJ`ChF#xRMunZrU>u$uL3<8$_Mh!dRWGS|6D>NMUFnaN3B3R9fYRHPbp zs83T`(}Aw^(4Cb+zm8@YCJK4?Ge8Xw{jFb%AMYNNfLPR^Is6cgUQIBY+ zB^`L4Ui4)!(au;VF@w1*BHH;L5kxx^na*q$k+ieH{3%I0N6e>4+PPu=o!gvUw3Cf!Cm%^WWz3bR zLA2A5=CmQ&d4XQ^XBeZowVk=1m$Hg=Y$4j&!$FR720!D^cCz?;5BVs}LzE_Ir;a)H zPE%Ubk!a^7V($zg+8Iaeo$0*GQr;)_&SrM9hXWkrG||o#ekR(9y%X(ZBNx$5QA$#d z*gG|;OJkbTmT2ci`Vj5B#_LSvEoQTjWvpTY+eq3uY(CC+e9u*)oitujv~w4KCE6)O zv{RaBrz+7-1Cn++m}BomJNWj&!3p(ax*9&UmIWlh`}U_<*%UJG+Q>4v@5y)>&mFE749~ zicp*~RHQo5PD7f}icUm3FEfClj3(N7lbOtC8LL>w7Ls-jn!n*R7rDx>q&Tbe+(iyz z?-ZmM(N0;aa%(#+JU>epx^rtgV>~DA%rZwiD~P?bo==E&_Hl%hoG13qPuwKhxr5B) zBripHkZ7kO)u>H?QWjaiX2? ziM@(9sRb!yXqrnDm3=}Hg!@d_gt$7E(Ohb6?`iFP)#gD=?6 zF-{SC=SObvU(!0SJIO}0lb^rwFlC8$YVrh)Xii%?@dCXWNVM}h<9U-=%;!B~@2qDF zpYat3IZo`IOI#&s=XPfo_vC0N4}~a7Nh(lqCC)#O7w9|oTrx*Ph!brxFv@_2f z?YvKXV{KtOyZM@E=L{G4foSJLIl*^CJJ+~LD(9AgTiYq@`5{VEo@&&l0Zn*@4m?K>`Z1W1jAJs< z&RiC=g80VT%nrU_KgT$Q*Yf8*IkUfale81xSfwdP6{4N`JjFA#BiiXf+>@i75sW2i z=Uww+R`3y<_>|r3<0vP&KL z*FL|j-_%exYst)oO*jZ(`+Iyaowj9T4_p!b(D_xYlP~>mkwP(fjNl`xJ^(&IWapu5 zyI`b)E#&dQyXbxdk&O*y*EdS>`G*nNLpgR0vK~m*hEDEFXZNM!1E~f&$Xk~EJ4;@p z_5+{E*9&Hi>JWJ_rlRbJR7z`&w`NVc~Lp5rU;Gf7a9VknS z(yoK2_U+m0v;ph4L^`^yh#bni;+IQ*iI4aZ*Z`<0{7_oMM}iQUh4PoSHjd9;80QbC z=o`RPK!1BY8k7FUp!3wxrWslFkT#OvCVSxlP@qpFoN;-`i~*`Jy` znnE7f$;YE+eiZHqm*V8i6e5phKcL)9tRTK8&gHi-Ln%uss$x^y%102%?W*&}H&&5H zVu(u0P#qdoQXV$ZyWRM!nWn9_c_c;-)3Fu!=!gpRLovP~|7zOdo=S4B?Z3sP1$_pe zzV9#jMJD$`+p(OMrp*LKmH`({XrC0EqZPmR@I*4h^lT+Q78BHk{$%SHOp`a%~@$Hci{NYVMxwR$D#!2$U|w5Xa+g-208JMy4_HdH1fYVG_!T z4C;u9sHLH|ys&F;?d0j#KKb;kZ)DQZY(Nww-Hu0aJ}$;0Qo;>fk^Xk1v?ACwyo7g1 zqIx`sx7yXQG&P9NzX0aU0}g(iIdECjrkBVsi3P+bgh`Y^{B6$iT9=Qitm49vNFavz z{FZyhy_Ic6U1ZLk< zI}h46x(rvdkt|tE8j8`y(}b)Av#A`m7EB@~?39L5LMNe>tOY`+j|ibwPwJgb{KR*^ z+r-OGylULFV*L{rR$emhrrTDp$w&|JmsT9D#w)KLI9PXO!*!(A+nIFSRz$ACMp9)0 zsr6_^da$UID35*%T)!AN@pldyNIJiF?EDwmr@QU{Djgp zqs4rYzvD)s8kgZttoJy8*>>{lik4+-f-}b_n1 zCnzf@1E#X5ptWYRdhrb-^xwYz&oA>AF29*)fGRf*13FZJrId&TcYXRzr_Q~47mV7v zqnb$=Dl$yRvP6R>;k-J4$m!O84ALdG&T4#zToiEMzdyAHkUe8qJKEL&S8-qnAwgie z#jm58&jH!brIi2R+aexD65R4TryCV;qdtP4I2RcE^a0dz$g~@%7o#5f?D+R$!x(f+ z@9Jqe`y(YBwP!_9DwdtsBM|=mQpFyl+t|0F{<0F(U)TDtrt++95kIM{9Q4c(x+GQH z2z)bi({W1pBwP|s37>>xvIdlqLt7|iYw;t|D}+-@Q;92-;$zuz6dY@d#uhJ8Ey4_E zgHcuG_kqQ-nt!5m6L--(DB(uzr4QT_W>u)KE$uk&$*sugn%B4Ag66|V-B!}_kZnlX z!LtXCTX+aRq*_VGDR#|H<>vuIY0v&7R(prYqzYN&aIu8avg3ruFHpx2b;g@p>+I*k>b9sSX+LjAda{-cTx(tSij?Fx zvl?ocInDy!hOMsXi%aoyNiKHzYdGiu%aP{t-CQ2Tt<0k(%xpt-uzK#V%=?b z3@;sh{%AFCNv3u*%PK;W+~ze9dk8bA{C(5Rx05Ep$hH|Xfj4(%4qg_ui6-)KPJ@@A z4i93hXFP+M=lJS#nXxz&b;hu>&o!s=E``~_DdrP+?n zEUopE&%Cx{{Di4jJ-qJ4ZSA#VNn=c(60-?j0B2+tvzfGw&DhEP>D3JR?HT(f#g%fj zKI93taTf44xA^8#5@(H`Sz!2x(HA5g&3#1ZwV^7*uN=m>7YqYawYVn;HD>JPSDYkF z=&@9*88bqhZz@Y!+BrbU@%L+RMXM$I(^S7$RcP|q zg0||^qo&5SmERUqaKN;MN0nwQG|cE<40z~-ilMZr;gMTLF34JFn(2VsF@sU}@mno1 zhuBPPB_xC!iK&EyZ#{80v5!cX{6n}AN6aCJn!m^mP&}EXJT682BR$O+spOlZNf#e- zTk~f^oO_(;MiR78%H*_`aGfuX_j_~5@VaYdc}aQG-{*47;*#a*Gph+CzuJA+XsQjS zf)h8NJ9`f3=DQzzoU2Do4V()}d(Yu_u6}vDoI+E_N58pVH_T<4sw5;n7O@ZF_*l|B z;@rOze@Pp*bYKB}Z2+TkjZD;kVT!Xwz0KVW)se9>$A~N!? z6O|adE;8lm-1==r%ik`$rg`$M&mkJi&gq=nHLu61`2~fsK7IQY;jc3gW$D!s{?9y{ QtKVVn68U@3AwIn6|9EUxb^rhX literal 0 HcmV?d00001 diff --git a/crates/clawhdf5-format/tests/fixtures/blosc2/b2nd_3d.out b/crates/clawhdf5-format/tests/fixtures/blosc2/b2nd_3d.out new file mode 100644 index 0000000000000000000000000000000000000000..c29f5f0ad5461edad2d9de4271b870192dbecd8c GIT binary patch literal 2520 zcmZwJY0#Eq7{>84Ygx0Ugpj40C<$4+>V1psQKS;Bk|HY07p}2n3o-U3*%FgvEMqLW z(HN2tW5_aNYqHE}LYBe!UCsT>ydO04pW{5Q>wex3_w%3QDPsM(w#4@6h|cJO9_WL_G$ zR^BLYmbb|}kDZ+DY~RIw zC%L<`&gMPMd%Npm?=AAoH;} z9w+)+C&`oL5P7O+PQ&S*8;+3}<@_x7W6WbQ-u>BfqMRfryFU+8?H6DgW;nmd*`>&R zxl+5cNUlbE9`rA*agL%uIP#4PQm~jkD(ZW zF_?t&aS1L5ck1r2xJHghN5m)yy< zJMpEtxbv+$?gV#oapxDg4lUe?4ZN$3yf^NoP0_}CSl-!Ax zk~{IW{7!z4pRmTxoha`71-(=0o#Ia7&idZrPH?C6PI0Fyy;HkW)xA^Psq{{Cl(jpt zn|XI|CvzwEk+nOSJF%agJDEF)J8_`7g*%PjNwquCfTAi zJkOoXoj3#BiP1h=yOSok*E_j|J8`~yy%QJ8>F!JKa3`1EnaiETnb~*?f1u&cFisSA zO7Gl(xoGLm=l0e36&rfr4R_j`i#vOvJ9^S>HRg zJNb6|-OjlabMY`5?ksfvG)nK(?!>F+hC6yE-o*#bxD%h?Gjr`ua3|}X%$?FZ>)rX? zGsT^M!JWjN(mQDr+1mT#PTpK{CvzupC$_cMy;Ie_QYZHoPCC-govNQZ?xX?k21@RvW1N@XIYIi(D!r3VkwZPB zcZxe{xLxNY?&Q)tqa}ALy^|-{Cra*AdM7n{r*zK$xl^@p=Y8I6#DhK22SYF&GjSK@ zVG$}U$Cvm4>rmWjgKg0PolxA_52bgCJEeDuJA+Z&8HEX$f@vtd)6$(A>^J%DO7Gku jxfAzGy%XjCI*&o`WbOoa;yE*S;$>Ofc>|?)mNxwdF;+b& literal 0 HcmV?d00001 diff --git a/crates/clawhdf5-format/tests/fixtures/blosc2/b2nd_4d.b2f b/crates/clawhdf5-format/tests/fixtures/blosc2/b2nd_4d.b2f new file mode 100644 index 0000000000000000000000000000000000000000..6fa77def7053c5fa36e16e15ab53569008c522ee GIT binary patch literal 6804 zcmbVR30zjy693l7sA-fXTL=VNcU(%pqDA(WqdtB=^|wc(bEMA7oaocWkJKx+Zgwd_t! zEs~_A=X-QW-;-K8-J?T#+Gyz^9v#y2wz<*VZWfrQ%_6fJ!V!fD4o278)&e}HrKj5_ ztE;7Z{tX=>9B9n+#n%+CtS!F!aEzO{iYPJIl4Iidyh>b7{4$wAYV^A+a-p8UV4RvJOt*@^_*0N?X+_=AEPQlG<{SAm z7U{XiJcHM=q^JFZ81b|)GR_Bv0saU|$&VnK^+L+H_$16(AiYgu@pytTH-n^wW!^Zz z1?n?#CN=L|*?2=#>_39Cp@?aPjUS+}2tjw;tN{)c(VU((DdbiUtD#zrHL z30!FWO3wtT@l2bU>J7C@ty2fpQT4f+Xr`Jkm~WXm4D`bc^poabBs09`VIh`bD+7HW zwrMXEX&S5@VI|&X&7anEf%eNbO^YU zLoEY@@`GJyg|_JIaYhj?;3t$dDj3zY1waceo#@da!%A!ErXC&AQ%i^rHO-Jn$N}LV z9l{SYMj6i->Beiu3S)<{$2efLR&7-$HCQF9v1-0rq%v%Ch(z%)P3v5(=`l^?wA>_3 z-@GjyqWQe>y>Zznr>d&EJTmXAooX+O>_zqCZEFFpX&Y+WBtFypRa;N-U*hbwCwH3G z$@jM8j%U+NTu+qb2FFNpx93w5|5hL=kh>oj|AR_lczG#4+=H(!VQ~g_?L$B*JU0hx z-b4Ngot0KHTA653Uo!fp5q<8C7&c0J%2h_|w!+kJhTa2&K@jTb+AHHM9b|4+V@<_Z zGH&LJShoeo&!SgY(EpMvH94*?pWhJ0j?Qs?#!g7h z$jWJYkGp@;@LBWHo44tZFl1!fD@#g-RB6)k{wD?{JHJKBjS47-_Me2lx=EL;eR=t% z=vprN)*UC#ULdpigWadjUm`PWe)@*3vgOKLCPkfKXkeexKt{%KW#$0%Dg$T?@EW+m zVZ7cPYSLn0G{1%8EdRbphI(leQfAK!i`um-SX*^R#jxgW+9wVfdFp&XcvPbu5($XKOLi6aIlEGFXe&MAhn;3n-xGB>!-dyJ=X*KWk$@lcip9v_lUOwU^P(u&m zaqb7r><2DuhO5a=f_94L?1U@BAA!bLtLpgJxJ;fuEe9#M$6=%6Bg7S4AOs)QVHN>A z1Uv(j1!92&t=Eq@()Bz%wy$(kUr1z!8cz>7UwXac; zoVDmlfb5T&VJ6%rIrAunVyB3$te5eNWtY_ zj2e;k__Dd%@1?duii!17zI^)J1#fTK?lX=|p8~MmeI@#c|967Xmvv(z7=W&|s_n%h};3#uFtyzbD2K|G}}h zvlBa3vroobIB*!_Cn1w(mza=4sdqb$*>3SX-6uXs&D$MjB~y4Ebm@+9lR2vge*PWd zQRwohw`~QfB~0oACb^ZEz8=R?b^7vKgj7b$w)9bW3+3>46h(Njg)1fJnA3aFgOZW9 zkRr@=hg+jP2FMG0(#`c+WOJ*V>X#A!O_V$A3_jC{P06S8DWSJ)#p;g_=6`?X_u)@Z zT(o>;?k9)GOh{R}B75(rM;}joa>m^BcQ$Qzd=C!@+}3@YDpeE$xZ?pQUvR=Wpf87} zr6}i}$hH)@9OEYPyLE(1bW3CpCB_r~E{OfjTFE7jP%;G3Erf}vhps(^`R_HjY71le zMT#`RFHEIN;!miI+p-hqzCoEgF?#{t-G)b3eu)wELs;>oB%qS(}cI-LuRbVig>~))u7Mv!N^X}IC z6GddE%t(7{?PfAwePnH5>0*a31Z5moVv7FLK^tHQ|9%Fv1%`t+Q?Rz)r#l^$yV$cK z&l$4m03Yy%rI}pdqGnzOKGcl1N^^EXxAI5e;_LOlWt58d-Nam?tVSkLRw$demFN#L zZ@+<I^Heg~dp$pY{r+v4?~Ajj zez&~%Xg|KYEX>|K{BTv6#hKW1Kp5?hn^M)9kFTxy$k?N_xErEm=y$|s^yBM%x{2N> zoniTwJvW2Qf6XWrS-)Ag-j9EJtnlg|LE&{89{K$AZ$`<;22EG4S)Z3*7*e^0V`px` z?NV$u%c0-92NQJZo5!rw>Jxw$G&2x*Q8Nz%6Eq`}(H6@bt}}lGYVvb{j0@!n3>d`2 zEx<^R)gcwa-9fMKSyqr@Dzar2a!a}jc3bzAiPN56v04J-q3(UgO`c9Bu6O^5(^%bc zkGtc*VZ8KsDurYQ8@{~oOGLi%f_*KDtDvJe&`r!Gde=bKC!RQz_;*3vQK<&*?~LTp z`1%JqMBm#EmoB49O|)w;y{5P!1G{;1Xa?pnR= x;Q<$~J;8HXP>IrIfs=>>rhg5*-DgU zjBQ3`8;;N>HKL-L`~JMAbNUxP=X>r2?m{-?Mm`il2~Rm%*%o-#?pM4)%X}2vIX0*GrO}d z2Qh_X`4XpcHWzal-{b~vk^UgVejAAcg1L0Obb`BX&3RZfqnmg=jiTB(D& zs+am{h(>FIrs!{)r`NPnYqd!`v_}VZOrPkSF6*jp=$6vOJ&`?fN5Lo>WujtKkJ`~N znn$~MI=V;i7#PE2Y)p!&F)J3u(pVGgV_WQs594s0h%<2^h>XZ?o&0zRrL6NO>oiEy z>5e`afE0|u#6)Mlb=F`VwqPgr;V}MahA1C4eK;!OY3xD z59sb(3M2zwjwgeeUUc`MX@Lym7_*H7LBaa-a0+w zFEJ=mtn+eAi#gU=VVw=JHQuw%kvM6ci;>Pc_gbf5qEiXgP#X==9BuJ5dbpi|7=afs z$vU&Jz&fk3-a5PSp>C{VfI#{Qtb%t1H0;h0>bry4_+u6i- z`M%pZ%8z-Lmw44Ww;9&SrUz6&52>^&sH*Cyp`NhLpVdvh)nCIk#yV52voK9(i*{+B z4qNB6&g%%ACnWEC9yo-u+Dbt?2n_? zIcJ^p$ZVawD1>4ti^|rigT{Cg?a>+C(Kpc+HAANu0LMSNIk` zB|2G|llfVgC0Kz~Sd;bGl&$%cb$YQMhjO%aUg1p6=j;3j*K#v=ChZ)z&Ux#6lh#fq zWmg{kK}A(Y6;)le)j-YF#_ja5&OnXOSWU9dY%S7qtR0Pz zjhvA$3P*`37Y`@xG>KNxA-cMqzA-dL#ss%BJ?6%eSQTqyW9*1MaWIZq=WJYZJDHIK z52ooSLC-& zN$WfkwcJj#XdNBnndlY$V_1x`&XkxD^Q^Nn*2SjSX`KUcJU)%{aoO!;Np$|`cFG}D z=W#SaD|EC@Pq#A|BQXJ!F#~hm&MNC{ayuWmonzKHhfDa{I=8Ho(VLtr(J94B)~Vx7 zZpOB5=Z;Q_+j)_%auydRI_naheLTeD-sB6s!fU+AUzxheIrN})id(0OYFej>TB@D8 z=vh6l!5XP?dRfyo*X^vd&PHw5`_?(ClR9IauXNp;jL4Yi+-avmRP`p;OVjCQodMqD zF)`6Pv#j&F+j%=STW4P!ayzLyS0aOTa(R;rqhz8}3-!&(P_ zEWt`Y)<$f{9vpHzAA6H8;TwE!9Wj$PIgcN!s2{7cAFDPSusPeXlOL-O2XX{o;KW2{ z5tnie-{BTN);>R0YCEZqm8$bgqI19UdXr12taTn$JwH~<$aG*^FS1cBHrYRQ7!GU+D0e0)5njM5~Hp2O8h+*#NDV;hfJQPP8Ed&uhLG05>ILsE^S#yIkfl_k-V zM{%gpY+5P1UMIAY5|Y@GZM8jCPP=S-zo9v?+urm2U+25u|Nid%-Fv^BB6nl25I28M zxD5aWw{SvXn&&IS4zhjXqzscA1#?P?#EGEP6`&39A!<7nWQqi1LLcXoEik7Sh(fbzJ~4wp3^?Q-+_a|*u?al{E|y|x(D!uipF9CQwy67TXrr$2zr&v$F+nB z07eL#qyd8s*}?`RJFzo`Fd|x%n0ESX@kMD(Lvu&>BgQO_+9G|UFIKJHyzOf@Z~x@X z+;e4>S8JQv?mm1n{Q9lBj)Ad-wVmVkJs!Tn;YVVN%ca-qZ=yGYFGj}cipr*zwwow# z0>YxA#VMz9t~XKM;0skP9SW3}U+BT;*d&ZY$;GOghFc#&7%yxnw`+9XD#!0Y0*C$* zW1=)yO47I;h~}eG&ZE=Um0cNX)<8r0Lhda@Y~3rw1Yx%qQXp$8qh&( zYu4Ae*>P4bV~JdM1VBun01gs`hBBebt(U3V$W>bi;FC{Zy!jghQ896Vh!fJj%>z}< zC5BLNK~e=9c}{!bZg>APVCh{SAlR$NT<)1E2mgFzKcOqXWf&j`aerWr0bF$uHS_Hq z5*uNrm0a^Q0UR7MDp4O{{FHYU*3}disU3ir6AenYGtM zc>TKMDPfpqNddTY5P`@`Ak|%IA!K}5AM^jCKHRVt&SPEBaA2T-s+STR)3~+Y;=Iq( zKO|BpPR_{5zff_fXMkj}=OLfQme%%;J0pdnMC7x$0u!~Ry^BiJ`C3bi%&crN5nR0k zC%ILXdwv+1L>dU7Q`}mLTKuPYm3#&B0`_?Lg%17x<{gp|KZRU1`90|XJQ<->xPb%~ zFJXpcT7a#w)&Nlr1ce`s9iuDGjZ4WmGf!PxKew>F($Lh>=6?h1`u=(r|YOkO&cqi<@_#;809_YGzJhndIsXsuW3VE*d(^Ow8A;-x3mjRFrr!JFlqX za&_I!+g+ckFIr~IwO;SIZMTO{kl=7$VVU%Yx|{9yA3hx(`HNIEFtc2P&0{a$|3GB4 zI4QmAMq_KogZ`mcqoAO~S@7AqOgoKFils&?S5;QmHce-3soaJV0TC2iKpBt~A}~RCU6lT2WcM^Iu>51K^lW00000 literal 0 HcmV?d00001 diff --git a/crates/clawhdf5-format/tests/fixtures/blosc2/b2nd_zero_chunk.out b/crates/clawhdf5-format/tests/fixtures/blosc2/b2nd_zero_chunk.out new file mode 100644 index 0000000000000000000000000000000000000000..fe7c16ec92a0ff95c712aa2320a982eea4de5db8 GIT binary patch literal 2400 zcmeI!`Ipai9L4ebx-XeQB4ppPCJJLJdyMSG6wS!KCdU?82jSSJW1TW-vaf@1?1~UI z%3(N&lc9{UP9hSHQo7%d$vOSdf8hNmJkRUg#~TPo9uz_eltmTPKux@iCU^}o=zt#R zgLg3;<1h)+kcfF$id9&Tt=NSW9KW9d~dKk6A9uXGN`)J!4fZ(&||wYhlsW z&bnG}>u*DCl#RC!Y^Kez1-9JQ+9um>d+j?rWGC#jowuLuH@j$r_O`3(>9D1YEtUg9Wi@Vs8cOL_%= z&TDvGkMiao?J?fPdwM?~;v;>WPw^R^=nH(Aul7y8-FN#wKjh#0DL>~#4&+sxk|?J- zk*JL*G)HU1qBDA-AKt?VjK^fmz--LN64lv=ZP<-(Rp+?soW*6;xsAV7=W#1w#jK1~ zvTF9C>bzpFS{r-Ix&?KH*=U=fIg=~8s&mFJ+Eu$5)Cp%^7GiOh zXB9@UHXE=>P^S}nurCL3xav&dbk63dT&z0lRc9CX@c@tU6w_7bI&bkWX7ik$&kK7g zuc$haUdJ1GOVw%TUA?yt@S#51$9udd_$R8f+}HSK)%nWztIi2O?dLrkav~oJqcqAt ztkVL~x}DB=JE${Kb*5lu*6plTo$W!L!#JTj8MuNQxC>aA<+l8)Q&x4VT1~5;Ri}M$ zI|Fn(p*qtnA*hqA+u5u-Uk7zgTDo1ZU+k9sVa^A2N(6NxRVRwg7|j@VWKY!@%#j?& zc+OxV=Wz)?3+n7vom3uY8Z(&58@$8&%*yK^15NvW1 z7cg0yyqP<=hx>V0oBR_m@+xogt~NQ`^LQa`a(S<;O|In)y{WhMH@$;4xvvlQ;Xc+U zYm-0ndA`_J`Ucz$c?8^9A!`$5fZBbnxYljpktO;!!QODF7*+3s%P(T62jNW1X$1#2R985-VO} zCE9#jW~*(3#M%>xb=-cm49k>QcO+JJ=9XAR`81zpHNM1pY|NH?Lt=GTodFVSG$*P~ z0+YCqE4Y?hB-UQlITF;lq&g483e_q2P^{YCKw`D_SnudPB-S90Q=NF9=^y)CU+SxT zy?-vTQasg&+>e%HhFs1Ca=T>)%n-+GbDaTD*|na`)&2ochKKn8 literal 0 HcmV?d00001 diff --git a/crates/clawhdf5-format/tests/fixtures/blosc2/bitshuffle_odd_blocks.b2f b/crates/clawhdf5-format/tests/fixtures/blosc2/bitshuffle_odd_blocks.b2f new file mode 100644 index 0000000000000000000000000000000000000000..6e71e429af7876af0fbed4a4adcac4b60b17c597 GIT binary patch literal 2204 zcmZA33#^WH9KiACoE%3kxs-@RF2#~XB;_(89Tg>#NUo=m2%|(pIF*@LN64)Xa~m;J zQLK~7F;WMK+zm--sK#{v{MfsldG>yu^M9T_@Af{=^WAS}!Lq)MvWNG1VL-AgNs>O_ z+Sy9XTU;UOn!fwWQD$LkaV{<@7Q6a(nT0!(WM`7Ld0+AL%U5Pxx!(Tm9ZBZlBsrKQ z8ELbM6=wQ^v*-N1`pWo<(@8mQq#Ct}>l@OHRw{a(pX--=@(3z*`O*TUq!5Ahog&E9dK1*21 zIyR6`0fp@4ASd{l^ZZGghZc{x3>B$HZR*jGrnICTkJ5#n^kpE!c!_c3FqK^9@E%L~ zfREY8mwe4`_H%?&{K_BvL;5vIQj&61q6T%S&)qbqHCc4xNuH%YgBihSCh#WHnaKhc zv4XXH!sl#d2YWcgaXgx&BxR{gv{Q#WXw3bz<{>)Ljb8L)5W|Re#_>9DF_ZZ$Vg+l7 zcJld(Z`gyhlVkiuv~!VYr??+gifE@QHK<1e?jhCABi2vwG<|rEp^T(xJJTJ{Vj5DY%R1e=DoHsiQJt7Osdk!KKR^~8=|(RGFqq*)I}?bxGmUqc zM;^;r!+P>5Alli(0giK;b6n&8PXW=+UZS0ooZ&Yvk$6|pPFX5ajoL&zjc7`=^B^7RLQi7u3?k;vC|)Ip zXeXDs}fOM5L2Wv94RElN4t{I@PI*E|C;nBRg@UL_>7SOl+r4x)jZ? zotUX8x2|9AMm5o8`4NVsG^xh)`|hMZXlQ5 z3}7&YJjiH@DWR0<%wax@si2ZoRI!<@)KJSV_Huw@{KOglq*3D}X-Z4l(2-2KkwZ`N z7|1;gWh9R=j!8VtOv+e5IZJty)x5*|Y~wS&U=RB^#0h@k5B{O?1xb=dI&JAh7qYmC z-t;4%Aw0k%jAa6)Ok*}Lv4|zS!AjP#i4XXgI=-ZygB-P&+oM@*5o#;k3HCr_}K^(au4lom2eId7_<*xR_|?Dy|`m?%YaW z1~G(TG;HT7`!hs4FOX_ynfWc&uz_f&n(ah8yNPxV6YZSlEPr#Mw~KaK(~baWJRjg+-9}?}NoS&+9Bv_xf!syh zoe_+tVLLPIb6G&#oux!OtJ%PNRPzZth<3i?Fega0^Ow1?w@M?Owqy`@C)G|b^X=qQ zKoO&OoC!=O+L=w#Sfc8`(k)wM0Ah?B^&a`IU40M-va%oJ;9IXQG`O=|LU? z7)&9aE9CecoJdJ^s2 qK>Qi{tptcgoDU$Cx;u?gT~cyR%Nv1^3}zzxTV}j&Pjk{im_VFJ6vLKF1FL zqu+VsOZREAkz794f|J~nyg_M9_KuFyU>r!qP!u>7kB&9mvp7C>B?l=d)m`AeMU7VPgnFYRrCo}^(j?Nw!N!e zUC?_}U~+X)A5c*rQb`|C$zY#Z(u?m>d;X-4s_nlu*5@ z{&#i7m4fV1fytGuEKydLX+>6P#bn#(BqwXKPHRj#*^o`zkS)r~HswvWy?@@WbJO}s zB+-un>SszKg+WSTh=wsj!zSC_KR=EM8pk9}VVb5)wtX6DWH3V+rnLEPpUVZE%OzdO zHC>r(drv&6$r;s{JgLYrRpf-Ka!OT`ZSUXHq8w0B4yhzZR5IE2{`oLED2z^uAW9LF zZSTNA3|$msau7#1#nD3v^islP+xzG3I(Ge5ez)5fWI@Vrf@%TfF%`gq*_l~+MYHmn z=477cOtyVm(vp!G$}pv6TqbB-CTU8hY06~V`%jUS3{pylXjn#Q*ks%L=d)O%ES6~n ztF&UW?Q_Us4ePYVl*0x#X#-o7$2R3nw!MGeu4C8lkx4=6Xcm0O}gv@o_u1Gtv*gJ}8TNIA646BTBA`3bYVQ1$KQzg|d5Qxo5*2 literal 0 HcmV?d00001 diff --git a/crates/clawhdf5-format/tests/fixtures/blosc2/delta_int16_blosclz.out b/crates/clawhdf5-format/tests/fixtures/blosc2/delta_int16_blosclz.out new file mode 100644 index 0000000000000000000000000000000000000000..e8918075fcd369b427c22516ac6c889feda14e8f GIT binary patch literal 2000 zcmXBUgQ8eh00q$`cc0j{ZPvDJ+qP5Nwr$&1Qrmv9t#+of{$XmWn*U7D#0rx%xxy4p ztuRf~E6mW$3bQo3!W_-5Fi-OJnys#(xnErOP66|`2Hpsm^k?bRXZs7^s=bqTtvThLuSf}ZLX^j4psulfc3H6R$M zLBU`R35IG|FkB;oks1|@)|g;9Eg!6)6@!(uauB9zJ*^*X zpbdkKv~jSBHVrn@=D`-)GT2I62is`dU^{Ie?4TWkowReXi*^lm)9%3@+B4Wodk6by z-(WxO9~__qgM)N%aEJ~K4%6Ym5jrwBN=FCB=-A*m9Uq*a6N8gbBr^-4WcWyMnuQPjIj93+~qg!Gn4zcvz1FkLt1DaXk?{si%Uc^-S=ro(rDW3&D$e zDR^101h49~;B~zbys5W>xAji&uHFmY*9XCe`Y8BVTBv0~ zE440YqqYU@)V`pDIu>+N=YlTkTF_113wo$$K`-?#=%c;`{nWo;fCd%}(%^z28d@++ Q!wW`eWWgwnE*Mk(1#5BRUjP6A literal 0 HcmV?d00001 diff --git a/crates/clawhdf5-format/tests/fixtures/blosc2/delta_int32_lz4.b2f b/crates/clawhdf5-format/tests/fixtures/blosc2/delta_int32_lz4.b2f new file mode 100644 index 0000000000000000000000000000000000000000..819f61ea83e25425fa436a5a11a9d617c67313a3 GIT binary patch literal 1584 zcma)+&r20i9L3K(=lw_?h(1#x!$mGasU$KXvDySJS{P(O(hvLLLeVHtLGj1JKR|d{ zP-svtM5$IKB-O)xiG?L)whbi{LIp+boO@>m(I_8$=AF6c`<;7c?w#o$ymq+uPEEs2 zd;m~0&I5mul7TFo4Slp{=?k6+cdN+d!BKho!UQk@XKX5xW-1+9&tvi(qBFoW5Oylu zg>(+l><$B4D+@U=?E=~KuZMtk(!jAZY$@e|kl@QGu((wq#@Nwv~x z^N1tS?jqU2(@qhGn+Fsk(RKu&*+jFMMvE|JvW~J|&~BjJNPSBLz4f-YispG57liG- zNLek|dx`dC>MO$UO&OOXsx)a=C{{h#wZ^V>o^A-gE3s;4T%@@~V_Dc4XDR0ddw^X-ri{xGRrc}O{*F{piQvYivw?ghiztf) zFL!{KJ4k&<_?Jr*+bE?pkJ2a;Hp(%|a>3pT+Q+F+h@iLJ^tz{kb4Zlk(CTR^yOGv| zN@#AgEm9OdW?SgwiHSZ*C4#3QZKDrS4hv5Il+!<>elCLPhi&v1G+)vf5jOfO%2C1I z*R!cTjf< zKcx4K@U~pb_H}DywZ04kW_#`CQRsGZJ#tLU$FnRlFKLu>{qrqdWRN9RFAJMqG} zca8Qs^@a#?+OLC`b9sBtc~l3ffyuR@I%Ih1i@4SBS`pN4Tuiw1>@cPRu2n_-y z;+2?J2(M6HNq8mYm5f(%UMYB`#d;FXbACSI9&W#N^T zS2kYRdF9}hlUFWYxq0Q`6~QYnuYA1n^NQqEfLB3Yg?JU_RfJbjUd4D7=T(ANNnWLR zmF883S6N==c$Mc>fmcOdm3UR=RfShoUe$P2=T(DOOcy)!uRgr`^6JN{Kd%A22J#xjYcQ`NyoT}`#%nmQ5xhq78pUfguQ9yF z@*2l$Jg*77Ci0rZYcj7Xyr%M+#%nsSSY9)D#qpZSYZkBByyozl%WEF5cwX~)E#S3~ z*CJkvc`f0!l-Dv|%XzKfwUXB=UaNVn;kB06I$rB}ZQ!+$*Ct+@d2QjfmDe_2+j;Ha zwUgH_Ub}hi;kB37K3@BI9pH74*CAeqc^%<(l-Ds{$9bLLb&}U9UZ;7T;dPeRIbP>^ zUEp<**Ck$;d0pXkmDe?1*LmIGb(7aEUblJO;dPhSJzn>DJ>d0_*CSq!c|GCvl-D!< EAC<{Yy#N3J literal 0 HcmV?d00001 diff --git a/crates/clawhdf5-format/tests/fixtures/blosc2/delta_shuffle_v16.b2f b/crates/clawhdf5-format/tests/fixtures/blosc2/delta_shuffle_v16.b2f new file mode 100644 index 0000000000000000000000000000000000000000..5f0cc42b8317931a2ee462d80b093cea1a592701 GIT binary patch literal 1019 zcmb7DKS%;$6n~$mUW1THS_IKmOIy_3(9#%AjSU60)zH*<1Pu*MrGla^4WXz*oq?i8 zdw(qagQl*bSd&T~=k-2MBTnn+!M*ps@B7_*-~0Xee3#qHy9Zl)JMatuTaW0_81cda z8@d-|W#W4YW2Yc9Ivs_Z_%uKo$X(vZ5DGOYc=TuYz?K1CagUsuc@u8w0PGBwRv~q` zKEbnNj4KFZG3PNZA$C=kW%5ZKZVWYT1c(2{;)!G`5sM4a8zB%3g(IOr(BX7B+#avb z?{$0rY(WdeV~JEU5swKsQ6U%zMZ%$Az~OQ_JZ`Vw=k>TX3%udrI9u1x{K!Xp`4~1? z5EdnlTSI~wATW1}L<~2}H+UUj60roz`y<1#$}WeYElM3(YKx6#t5L1h-#_ZrS|*#z z6pE#CrBou_x2JUbk8aQC_MC1n==PFsujuxgZg1%JmTvFp_MUDZ==PCr zpXfFu@P9uxly1Z5Hk@t)bQ?jpk#rkHx6yPPL$|SX8%MYCbell8iFBJpx5;#yLbs`O zn?|?kbelo9ujn?DZnNn2HQi>@Z4TY$(rq5y=F@Ef-4@dA8@er`+qZOEOt&R;TS~WO zbX!ih6?9ukw^elej&7^z_C4KxpxYX{t)<&Kx~-?%k96BWw~ch$M7Pa!+d{Xkbo+^J z+vv8PZa>p)2i2`%~SLt?*Zokv*I^Ay2?IzuB a(d{OKTKC5dM12Z1?J{6BIL`t~n^IU~s{kzaUo+-a`Zh(SY~|GB-hu4>Wp{J$MjB zb4mn79Xu+@5@loX5&J0dk&8({NwliFw$X88LqDp&ny;$9?&({1jvYF2c6j7CUI7?x z(LgOaaC;k$lwNx)&ScRu%1(v`#qM!73(P`2pD%Cno?*N;4ZQ%oCn{AlSxkTeMtgvw z=TU;zO2BUFCGxpS_BuIzr*$9obJPuCW3+dL=2OIRn0)qqVkTQ5US%uW%VaRQjbW39 zg0}HNf|G`=50%jG_G`2PMyL<`&@@u}$+J~7HdBSQEX}pTdX^~svXRcyyq@cP&?Z9) z%h@18vD|~hv3ST|Lw=6MOk{G=dqHe{oUvx;~t?a$yIVADHQ!H?tk0FjE z(e_EqG598c3NJW`my*P^XPDt6S`y~`F&(*_sDU_xRzwL8+Y+A9LAh7cQGUWHb0`p@+@XbY#iVwo8BwENK_ erH!FFx$hSh_$ey3>x!PNvVXE>2;4hg^uTX=K@=1K literal 0 HcmV?d00001 diff --git a/crates/clawhdf5-format/tests/fixtures/blosc2/delta_uint64_blosclz.out b/crates/clawhdf5-format/tests/fixtures/blosc2/delta_uint64_blosclz.out new file mode 100644 index 0000000000000000000000000000000000000000..20735223546c90661fa2654826e0ddfeddfb4f6c GIT binary patch literal 3200 zcmZ9~RZ|rJ6h&d;(p`eINFz!}3X%fS9g@-p|NAN}-OS*6TIc1Pvz~V_Jmlqq@(;iL zOSk)UdqB5`bbCa%$8>u_x2JUbk8aQC_MC1n==PFsujuxgZg1%JmTvFp_MUDZ==PCr zpXfFu@P9uxly1Z5Hk@t)bQ?jpk#rkHx6yPPL$|SX8%MYCbell8iFBJpx5;#yLbs`O zn?|?kbelo9ujn?DZnNn2HQi>@Z4TY$(rq5y=F@Ef-4@dA8@er`+qZOEOt&R;TS~WO zbX!ih6?9ukw^elej&7^z_C4KxpxYX{t)<&Kx~-?%k96BWw~ch$M7Pa!+d{Xkbo+^J z+vv8PZa>p)2i2`%~SLt?*Zokv*I^Ay2?IzuB a(d{h+SQ!sku3Nu;-P*Mv_SUW2wr}6Ib?dI( zyLau~yKn#geS7zU*f4Q)^;fT6yMF!J)vLE|-@bMC?!Ei>@7=u%V#CB=y?*`b?b~0qApv0I+#LKQn+l{$(S~ zWOSFnTu7|TVIhDkWY9wr7Q)CO4`Rc_(beNJzk-2h0>cMMVx2o8WBcqdiRlJdA{~)Q nmz9y7g98|oAglqzU;r!y(ZLi(VCl*RDP7M2C6#ZWt6%^CT@d4> literal 0 HcmV?d00001 diff --git a/crates/clawhdf5-format/tests/fixtures/blosc2/delta_uint8_lz4.out b/crates/clawhdf5-format/tests/fixtures/blosc2/delta_uint8_lz4.out new file mode 100644 index 0000000000000000000000000000000000000000..b70a6d9599651fe994a3e7abcfb7640a4284fd88 GIT binary patch literal 2000 zcmeIygH|92006*j+qP|Owr$&XTWz*&+qP}nwq5rh_xgud#>URh&cVUK$;rvZ#l_9d&BMdP%gf8h$H&jlFCZZB=g%KOK|vuQ zAz@)*5fKqlQBg55F>!Hm2?+^FNl7UwDQRhG85tQ_Sy?$bIeB?`1qB5~MMWhgC1qu0 z6%`d#RaG@LHFb4$4Gj%VO-(H=Ep2UW9UUEAU0pprJ$-$B0|Ns?Lqj7YBV%J@6B83t zQ&TfDGjnru3kwTNOG_&&D{E_O8yg#2TU$FjJ9~S32L}g7M@J_oCue787Z(>-S64ST zH+Oe;4-XGdPfsr|FK=&eA0HoIUtd2zzrTO~`uqC_1Ox;I2LAi^FDNJ|I5;>YBqTI6 zG%PGEJUl!iA|f&}GAb%6IyyQgCMGsEHZCqMK0ZDnAt5m_F)1l2IXO8cB_%aAH7zYI zJv}`mBO^02Gb<}AJ3BikCnq;IH!m+QKR>^qprEj@u&Ai0xVX5aq@=X8w5+VGyu7@k zqN1|0vZ|`8y1Kfirlz*Gwyv(OzP`Spp`o#{v8k!4xw*NerKPpCwXLnKy}iAoqocF4 zv#YDCySuxmr>D2Kx390SzrTNAU|?`?aA;^~czAeZWMp)7bZl&Fe0+RjVq$V~a%yU7 zdU|?hW@dJFc5ZHNetv#oVPSD`acOC3d3kwdWo31Bb!}~JeSLjnV`Fo3b8BmBdwY9l rXJ>bJcW-ZRe}Dhr;NbA^@aX91`1ttbs`!bW+1_+10zE8|?|7NE2aBLl-(kZv@Uvw(aZAkGA0OCZ+R zs{c1ZhoP2{A!5mrJxkWiNy({s@USon-hQfQCcimKYZal!GN*7^ty zx{M43FMz%RI*(nR|G&I_JqJUBh}_|lh7Q9}l?D6ePTlJwxKM(Tm64r;1E>vzHGmil YfV$AZ6h>eGut5Ug98gmE2D%Cc0LuthMF0Q* literal 0 HcmV?d00001 diff --git a/crates/clawhdf5-format/tests/fixtures/blosc2/delta_v3.out b/crates/clawhdf5-format/tests/fixtures/blosc2/delta_v3.out new file mode 100644 index 0000000000000000000000000000000000000000..bc826e3e48d64d9208868bccdf969ec327295470 GIT binary patch literal 900 zcmd7Q17aWs006Mswr$(CZQHi5wr$(CZQHi(?id#>URh&cVUK$;rvZ#l_9d&BMdP%gf8h$H&jlFCZWwC@3f-BqS^>EFvNz zDk>@_CMGT}E+HWyDJdx>B_%B_Eh8f%D=RA}Cnqm2ub`lysHmuC#4Z(v|xXlQ6;WMph?Y+_D3@8ICz=;-L=gwv|=H~A1?&0C# z>FMd^<>l?|?c?L)>+9?1=lAd5KYxG!fPjF&z`&rOpy1%(kdTnj(9p23u<-Ekh=_>D z$jGRusOaeEn3$N@*x0zZxcK<^goK2|#Kfeeq~zq}l$4az)YP=JwDk1!jEs!T%*?E; ztnBRUoSdB8+}ympy!`z9f`Wp=!os4WqT=G>l9H0r($ccBvhwosii(QL%F3#$s_N?M znwpy0+SS?mrsn46mX?;**4DPRw)Xb+j*gDb&d#o`uI}#co}QlG z-rl~xzW)CHfq{X+!NH-Sq2b}-k&%(n(b2K7vGMWoiHV8H$;qjysp;wInVFf{+1a_d zx%v6|g@uL1#l@wirRC-2m6es%)z!7Nwe|J&jg5`X&CRW?t?ljYot>TC-QB&tz5V_D egM)*^!^5MaqvPY_larIv)6=uFv-AJK|Nj7+jm1U) literal 0 HcmV?d00001 diff --git a/crates/clawhdf5-format/tests/fixtures/blosc2/generate.py b/crates/clawhdf5-format/tests/fixtures/blosc2/generate.py new file mode 100644 index 0000000..f79c0f7 --- /dev/null +++ b/crates/clawhdf5-format/tests/fixtures/blosc2/generate.py @@ -0,0 +1,129 @@ +"""Generate the Blosc2 frames the `filters_blosc2` unit tests decode. + +Each case is `.b2f` (a Blosc2 contiguous frame, what the HDF5 Blosc2 +filter stores per chunk) and `.out` (what decoding it must give: the +first chunk of a plain frame, or the whole array in C order for a B2ND +frame), or `.err` (a frame clawhdf5 must refuse; the file holds a word +the error message must contain). + +These cover what files written by h5py + hdf5plugin never contain, but a +Blosc2 frame may: special chunks (repeated value, NaN, uninitialised), the +delta filter over many blocks and odd type sizes, bit shuffle of blocks that +are not a multiple of 8 elements, shuffle with a byte-group size, forced +stream splitting, multi-chunk B2ND arrays with padded edge chunks and a +chunk of zeros, and features clawhdf5 refuses (dictionaries, registered +filters). + +Written with python-blosc2 4.13.1 (c-blosc2 3.3.4) in a scratch venv +(`pip install blosc2`). Re-run only to regenerate: + + python generate.py +""" +import os +import sys + +import blosc2 +import numpy as np + +out = sys.argv[1] + + +def save(name, frame, expected): + with open(os.path.join(out, name + ".b2f"), "wb") as f: + f.write(frame) + with open(os.path.join(out, name + ".out"), "wb") as f: + f.write(expected) + + +def save_err(name, frame, word): + with open(os.path.join(out, name + ".b2f"), "wb") as f: + f.write(frame) + with open(os.path.join(out, name + ".err"), "w") as f: + f.write(word) + + +def plain(data, **cparams): + """A one-chunk super-chunk frame of `data`, as hdf5-blosc2 writes.""" + data = np.ascontiguousarray(data) + cp = blosc2.CParams(typesize=data.dtype.itemsize, **cparams) + sc = blosc2.SChunk(chunksize=data.nbytes, cparams=cp) + sc.append_data(data) + return sc.to_cframe(), data.tobytes() + + +def special(nitems, dtype, kind, value=None): + dt = np.dtype(dtype) + sc = blosc2.SChunk(chunksize=nitems * dt.itemsize, + cparams=blosc2.CParams(typesize=dt.itemsize)) + sc.fill_special(nitems, kind, value) + return sc.to_cframe() + + +# Special chunks. A repeated value stays in the frame as a 33+ byte chunk; +# NaN and uninitialised chunks become special offsets. +save("value_i4", special(300, "G!2ZVfzdQDng&MGz>rP@0A(|8D*ylh literal 0 HcmV?d00001 diff --git a/crates/clawhdf5-format/tests/fixtures/blosc2/nan_f8.b2f b/crates/clawhdf5-format/tests/fixtures/blosc2/nan_f8.b2f new file mode 100644 index 0000000000000000000000000000000000000000..939321b8a1a22d1ec8d36ce0eba5c4cf325cff1e GIT binary patch literal 172 zcmbQYBFQMNC^0vc;SvJ_L*jWL0Rn552r+~*U50Qt6QB$Rup$nSN)U@P;UWXWMFz%0 zH`u@m8CfCff#l?~4D9z98167Iurji9Z~#R>SOX@{2xGwMrYVd-J!}v?=YYb>H{fzG F1^`Ikm#Z273A)J9O*?6#JvY5xEQYxcn~{O z->)Y&>hA@1_(u>LCN}ENiH;z21l2sq_%jfN{S^;$W*5RVh9Dydf<kOxRvZkJWoHb3bZ7<=_@`P&9F2cL-4Hnm-Q zjV>#gGVVB|{ zuUn<$>W8hJy+bc&7FSlWth_>EvdWr!^v$dtUA;~QpN~#V#~0i#uc~Wme>yNWg@MFH zWmPnFe&BUI=65D6Dn1QgKp<7sHn#N)jJ}#%{JO!!&Mz#bpt@7n$l{QryLUjy`IyA? z>%}GI_ZuE{J{=l+J^yhP1Bq{y-KM^0zloKd^D)1m@W{B-?7Um0l{JlR-Tf~n-zMihF`s1{IZVa;1!lsRNbX#XlCQ&ej+jB+705p>iU*;`q0a11~?R# zQ`OpQXb!A<2b?{BIVlZabi15V-_!xDzk0v?70trQzXjB^OYeZCor|YW5U`${onK76 zSM#8)vlr4_{IrHq)zCFKh_gR>{N$PQQSp#KNf~7Lq`QA?`t36C$;r26-vJ9-CwHII zp%>$lGxCZ{@76T5{M$P;J_8W9NGNUxbOE6kW0Ny;;oX$FmQDaL^B&61$|WEwt*pLB z-_+6$3I_U#OUceBlvdV0eB9MHJT<%UWdqB`BP=Pes-YzU#ef{p^ zSA+?ypa%+eIhLH6S4<>VH?(vya`1~tD#E^zMd*dtl+5b-){efx@!9#6?-*7tAu$;x zbsd9ae!*dv6RzeIfGem?Z9V-XQ*)qIEIXfwgu?cndyOn?kAOHri<;YeK@;y+zM)t- zH;c+DYiL6`4!eT&!Xx8TbMgsgjFQG~eG@Bi+m)28yjx}Dn#R`d{^7~DA9#f%6}E#K&1@XO zWf}OQ5)!2jln7MKElDe>Y40<(v~~9MNd{lFb@f88m|t1NuyI3mt?gaYs%VYv5Ie=d z$_-yvR@XK#u_BaJ)i$+t4~$OEEqz9qICg018C%#md4OYbZ-h~5qICy=PtyhWL^WCQ0iMc z=z|lp?-3LlgGONxG-|he5t=2of{FVS4YwYdoqJ%vs7jV#FHWv)2~Q3QSsjx8eprx} zf2QkkxcSa@qLFX1Pf)Itui>1T8(E;ZVtSq+8Cc@3j}Bb(qboWk2t>`Lg?fBQ*ikS$ z=ggm1{+yRB{ocda%}I*WZwf}`T1o|63PS`}lD5Zaa|jq|8Oscw^)MxA-M<``PO`zG ln3$QNeg7EpFn(UBKa0QjEEN0>7=Qf^x?x*t@bB60)qf=)>URJD literal 0 HcmV?d00001 diff --git a/crates/clawhdf5-format/tests/fixtures/blosc2/never_split.out b/crates/clawhdf5-format/tests/fixtures/blosc2/never_split.out new file mode 100644 index 0000000000000000000000000000000000000000..8a1d2f344681e2d95ae3430e9ef5b3586e7bb84c GIT binary patch literal 3000 zcmYM$hkwua7RT}PKF1cdU5OPV)YjO0tG!aS_Z~H4hN@AG*ehskp|Oh_BPdZ@?b>^* zeTy2m===N0@4ElO^YMP1*HK9>pGii^F1e+U1WQ?|BDJNFd@Ze|v-FaIGF-lu?`5jY zmIbm*Vq~LilU=f3j>#FhBsb)NJeODUK@uSa(jhYfkrzc!0u@jL_0SkC&>r3J4F+Qb z#vlUIFc*ul0&B1d+p!mia02IW4R`PuFYpc@kxWx*2F;?mwV(!T8LgzXw4pZBR@zy6 zYJVN7qjiE#(pmbWF3}kMSz~pl{;EgylwQ*7dS9REU;17XS#nEjnJtIqvqDzV%2_q5 zV@<4uwY9GHjfGm6jkR!_ZgXsrEw{C{$+p`bJ7~x3oL#Y7W|B-&OD4%Cd8D8emvT~# zI*p~dw53iT86d-CjD*V+iIgabmesOJw#shm9GBnairjKKubqw`GQb}>@i}!$QKvfU zqA6NYrw96B2*NNP6EPk0u>?P19byrOUvU^Gah^JNsq>OLs>wBtX4LE&q=obgEvuEO z(}+5)snd%(!}UArOx4-CK$q((-Kbl1x9+FTX}zd7sPkN3>ED{zQdoNPr%qlgOr7%7 zsb`I?rM0te)`vPHZJb5e44Z2UZH29~&9=kh?XaDs&NaJDofMKzGIKk5rHGW23e>41 zO{9gilP=O*25~!{&UBe83#qeKHp_O|BM0P!oRh0^M;>uIZ{?#Tg^0J)JLMNt|R zQ4R}X+liBSIYgcFa*aBV<%PTxK@y~#@bBa|V}i6Zi29ukap;sFP0pHHYTYB3i@6>b5M`zIlZiR^dYzN#_6P@PJrdI0@NvEm8nzDLa5WxdVJD} zpw4_-Y%#XMw%AVF$L*Z5i+02AS|Uj$=_SDF6qQm^i8}SAskD;z(w#a(WTcFhNp3q! zWCe9%WhZrx$Vs2h6Y9JbMRKQ;mD?%gbSmRZG^9>jbVg4M#Bh8^oynMk1>8;yHef4$ z!2xdPG<9y_4?M?T)G_trcKkJvIz_2dQLAfRZKBP!y>`_;I!H(A7!B7MPG_aAaXNeT zklW5Ry-S@J`c^+`QcGPa9xE?OU5*lc^JBOKp{{x2?8| zI=|Uz>Rh+`+>WP{RdPxJDJG@4oto0X>2#DH)EVk-@?@DM^JFP+@&@YckbQF0r*mJP z$V+ZVd6Uy16S8wVg;5;kQ4MvRPCIl#Zw%me#!zR9(~0JGyqmlm2XPE%a2dDpfZKV) zo19ovaytQr3j0 zCABn`$+BA>-sE5_%k9*%hSrQXxr_C*{?r-mbY?l77~9D0csj@Ij9s#u)JZ~}jFLri zQO7G*Wv3Iu?RYxa3FWvW45(&+VL%3v!(|`6-L_UoofSj~oc%c8a^3 zT${ye&h2#6n-OwjrL|If1jdLY+t4&TD+2j-M-5PR*~yv^0xVQyb`4 zyvd$UUlwbWj@JmCsqvFE=zorIphWU-7r zPs5%ET9E5`1*=Y-Cf3{)%j@}Y8{>55uvpQyn#J;Ze!z~Kr*qqs#Y*FJg81!};C8&8 zH}dtox3A|va67Z9vs~8j+u6!uc|AWXm)Y}&@*jD_VkJgOq-W1_u~@~ZQ^|chAuLux z&qGMWtoTK3%2Il}Fn$5j^Vk<)!(=65veV|WW&yB@O>w2Egif}s>sZ*ERNzmy_oe|WD;I|XWVnw^2$5H1ni*??v zQYR60(z54)62xM8-%fR(PJ8LfZzq&JA1f323r4!0uW>z(XU~tjp5J26|KxTANs;PP z&%fZeQ<=qTfDnE=o!qxG6r+4SpT}R&>v=4TwU^({DYu>b?D-4!9GZ;V$;6%qX<^>v zvec>Ve!({G+Zm+8*z<6mLY;-WOjq+4+{R+<=S}{d#k$F!d&PS1dY+EO%E_J=vC`D3 zNu8#=$sJtJhp^}4Y?ABwVq3{yFxKMu?RY)Ez?*!R-%d(btepII3iI2kNS!+F7wq7Q zHCV!A9D6>UH+ix9b~f=Q$ID^fU_puFblWyzn<6RO%6e8emlL~O&;ZL Y@+|&>2|bU+PVD1$PT>N3{s2$#52yv@>Hq)$ literal 0 HcmV?d00001 diff --git a/crates/clawhdf5-format/tests/fixtures/blosc2/shuffle_meta2.b2f b/crates/clawhdf5-format/tests/fixtures/blosc2/shuffle_meta2.b2f new file mode 100644 index 0000000000000000000000000000000000000000..3426ff41e2b77760cb775a10f92be50a81ce2b16 GIT binary patch literal 1003 zcmbQYBFQMNC^0vc;SvJ_L*jWL0R+simk2S0FkOc5_!mGK%=REf96*A90hqYRz;Ka) z@z4!6MzBg21}2COhRJ6c*zYkg++ko~Wh~`bz|X({!uHGz4CpFVfI7s1;uUdcucn`w zpDfY#@&5nXvr(#Bo3;yGxtFxHEc@t+8*69qno7H7E!`%vRy1VGw`D7CRi78zxU}OY z*GAQhGj|H@PR=#|_;=^z*T1JewmJ8@?dJxLX-A48I9Dgy=5(#yU|V)jPNJ{5ki{*T z=h&l|p5sqvv`80PFME_|)A;ISEy z8Ob?)xf_kY@ho;Ou;@G5X|Xn`L~`AsuRD0ZKR9Sm#kc-Z?wKR%GkV`^xy42>bhica zvn8xGkeYFaYjNyG{%`9gWZQ!atd2cQ@;UP4i5ctnM}aoI;*)P%pImG3DxvNR@AP>) zHnUwj&li1c?!t$guF@?}ToF5h(c^~sluwU*0&wElN2Cc@yI zPVdu=TGtM`Nwg^55Zs+C!FR|*Mz)-5`sPA8y`xii@NI89{kLLI(R1G#f%1Jme!bRn zcU=CGebXRnZ&5mT&h$C6>Uvk)krXaJxAK>%<-0o*=g4(g>nBt_J-7UaR!^68{leYi zi~kiae|Oz1bpN+L>-N@*oG-6SPPJ~mxOInQ`NMrXf+hBbd|my-Z1tz@`DHIWk4x9H zAH6^6a^4MY_K<&l@1DMI{eSu6ro+ae$M3(s`f>iFbM~Rff2%IP^f7cp*&R(;`JTCt z&VASsTqGr`m;Cq~E70WAx}nFV%a=Xp+c$qs+H<~tP4^JcX-tux6ti7 z(@YF^POZD!%YJCZorB?*)~jug{J7meFaGk!TlX7pF8n7yOSbPMU+d|aFMp`sT==ov zOTRp;Ub=iA?{v$l*4)V-3nC3;t{qh?O!xTLerxVTkOS3PPA_@&&d2w-w4U*!b7!7} zSp91*+vj(B+3MEIcbhUFzuuCwE6U=o?hG>&!IXcjLOtIuvNE!BaDXx$5Nkj)7c>vS Ycwl-8BQVRcL9*OAposDf6lo9#0LW;=B>(^b literal 0 HcmV?d00001 diff --git a/crates/clawhdf5-format/tests/fixtures/blosc2/shuffle_meta2.out b/crates/clawhdf5-format/tests/fixtures/blosc2/shuffle_meta2.out new file mode 100644 index 0000000000000000000000000000000000000000..da0be8caf5f846bb2d8805b1156153769c6af9e3 GIT binary patch literal 4000 zcmZ|Rdvwor9LMpYl3OlE*dca~OcY8xUEFH5%65`Dk(iS%j#A`IoE45%$(dLeCzq48 zRYO2yf*uj$$4k;3VeraX!g;T*#OC3SZ}2T*r;v%1VCC zZ@Hg8@(7Rd6wk16lZMZ!1>3R{yE2`9nZbd~WHyKMZr;O*oXp2Ki-j!Wi(JYQu3$OW zb2C3<6?gMH9$+2gYtzjAYQqlf%IMA&_SGE78yVfnwMTIrCvY|?Q|x)Gr)N!hccI=IgZhtN9<{w$>>gzUCbqXm2YqjEBFyV zVHJ0=h6i|<^*qVHc$OEqchQ~pc4wxs7yI!VMt8F89FE{vj%U1grrOgP-I;4I;3CF* zr_^4}cliOgu##VK5BIT_(H%3Tx%<_Y(Va9qoqZVHiT6&H^K8a@XN;Zf&ZEw!b0(kW ze7?k`EaNJc^IYB8?(fl^c<&^;^PB&R?wq!h-MPp;yO^DMDdW9!m3=*?! literal 0 HcmV?d00001 diff --git a/crates/clawhdf5-format/tests/fixtures/blosc2/uninit_i8.b2f b/crates/clawhdf5-format/tests/fixtures/blosc2/uninit_i8.b2f new file mode 100644 index 0000000000000000000000000000000000000000..58a2a30ca38485a13acd7aa435f63bdfd2cce945 GIT binary patch literal 172 zcmbQYBFQMNC^0vc;SvJ_L*jWL0Rn552r+~*U50R&7@!OWup$nSN)U^Q;UWVNF&?_X z23E+(3Q-RvC!b|tzsJCEhk=2Wk)4ABC<4M7FnLB815US0VFc=7gXlR26jr_gmxD0? D_(2$h literal 0 HcmV?d00001 diff --git a/crates/clawhdf5-format/tests/fixtures/blosc2/uninit_i8.out b/crates/clawhdf5-format/tests/fixtures/blosc2/uninit_i8.out new file mode 100644 index 0000000000000000000000000000000000000000..a64a5a93fb4aef4d5f63d79cb2582731b9ac5063 GIT binary patch literal 512 NcmZQz7zHCa1ONg600961 literal 0 HcmV?d00001 diff --git a/crates/clawhdf5-format/tests/fixtures/blosc2/value_f8.b2f b/crates/clawhdf5-format/tests/fixtures/blosc2/value_f8.b2f new file mode 100644 index 0000000000000000000000000000000000000000..156a4b45a5828182186c608b81325f6d55af10ca GIT binary patch literal 212 zcmbQYBFQMNC^0vc;SvJ_L*jWL0RmT+2r+~*U50SjFF+X#8X!d+AOfVE{lY~ChKmf0 zhiEY9gL=fp_C5*r)R%u literal 0 HcmV?d00001 diff --git a/crates/clawhdf5-format/tests/fixtures/blosc2/value_i4.b2f b/crates/clawhdf5-format/tests/fixtures/blosc2/value_i4.b2f new file mode 100644 index 0000000000000000000000000000000000000000..e803bc270990804c4d5d85ddf3304dd43e066e22 GIT binary patch literal 208 zcmbQYBFQMNC^0vc;SvJ_L*jWL0Rk762r+~*U50R2Hb5B+Dj-EHAOfVEWy3`VhKmf0 zhiG#K_<G!2ZVfgz9v029|$6aWAK literal 0 HcmV?d00001 diff --git a/crates/clawhdf5-format/tests/fixtures/blosc2/zero_u2.b2f b/crates/clawhdf5-format/tests/fixtures/blosc2/zero_u2.b2f new file mode 100644 index 0000000000000000000000000000000000000000..5978d316b3e51edc7adb5d00b30a9f3fe5679ecc GIT binary patch literal 172 zcmbQYBFQMNC^0vc;SvJ_L*jWL0Rn552r+~*U50S@7eE;dU`0$Il^_=Xf{P3c7a152 z-CzSNWMqY?2a=P|GO*ucV7SA;z{<$Z!2uKjVGWo(Ba8v38>cV=^{_$ooC69g-+;@( F7yz4Q8Pos( literal 0 HcmV?d00001 diff --git a/crates/clawhdf5-format/tests/fixtures/blosc2/zero_u2.out b/crates/clawhdf5-format/tests/fixtures/blosc2/zero_u2.out new file mode 100644 index 0000000000000000000000000000000000000000..1dc89f8e47e591af4c36a35046077f0ba1d1ef9d GIT binary patch literal 4000 ocmeIuF#!Mo0K%a4Pi+hzh(KY$fB^#r3>YwAz<>b*1`NCh1`nVB0RR91 literal 0 HcmV?d00001 diff --git a/crates/clawhdf5-format/tests/fixtures/blosc2/zstd_dict.b2f b/crates/clawhdf5-format/tests/fixtures/blosc2/zstd_dict.b2f new file mode 100644 index 0000000000000000000000000000000000000000..89477d8da8c9adced12097e3aca12bf7f65a733e GIT binary patch literal 4978 zcma)=c{o*T8^+f*%d8?HGG)wAQBEl&LkWpWO439ni6Y9-q)>_mO-f1w4WdpW^H@ky z5+N1MMJl1b`&nD(JLme&AK$)&uD$m1zVCBCzx#gH$@5E%R``3aUCumU7{;>(-|!mh zpCZk;@YHetXBIyAi*zYn#D{M}bh%l0EyL6@oWceH_CgLn_j-oI#8)wb&lskOVfZ;a z_`=Qbh7iM)O8>s#_mvE1fQMnCBpGHMFT+@fGfbBt!x#)p?E6Omp@V-fk!8^m% z-ZrX|2JV8NWW9?#QpCdc8~0ne{p#zNBp-bVjV>1CVD)_5oiOFq_02EdbbsdvNysRwY8yUbMLNA~7o3`$Z zjX#!r;p&at;=8qtEu9~}4DgCb$|~=POH4YOo|#ive*bavOFogIa-%i$$4|Djb)3I= zg>OJeL{!|N6KB(}XBYgw_o8J!8-gOD_8mHwd@18*eo1xR)90^0ejC7NN2-j`pD=m) zOlLRGRlXayY#+QiBOo?RUPaT;*laph6y2S?$$ne_VRtt4;~?@ z5lZTM<0qM0JGf*QmQ_D)Zs!>yH9}c^?08cPJIDDRE7k@CNA8Y0bo^{uMs`72^`oZe zuRnhqyt(^QpY=gocf`aWJ9GZ(&D_$v51zJmeE2%RFCr~Bdd%1fe^}xci+Zb1bcN&H~R>>o$f)Mjtqwl$w^6Q&?Hs(ERdkS06`6Qbtit+i;S(&1~1fn`h^j zRzG~&&Lb=(GfGW&{2vy!4)Yd!t?}O+v1{L<<2c-#d1d$No7&!d`aU2aI#f~4weR4`GilcbZ;n-2&UBjRv2tx- zXk=8}k)*SiGH(@>)zo8EpL-cW@!^VUS_YG*&9HZNM_o6D?PP?pQdMn3Q}Y@2u8WrX z_yvXU+?#Oh^o1*b-YLHK;7M!er*A*`h7294q%nB&@lcCqmi~Yjsh8nQ;4qqFbjO)m zh=ejPR~Hz0R12%tKQ|f`>)85o;f!-ZPxeX)=eS!Z@Q?lH=lD5HcRo*2OJB{$fsSuK zyF(=0IZUM>ua9Q1#?YM)Zrc6O#Rs@Aw9CaSH}Z<_J$%}R_4MhyW+s{}V{?flq-`bx>sIZv5B({X{vN?(#&+~K5@v`msT zipV=eN>&MaoNR9E*+%sKh z>-7%_-@W(Hi8B|kq35d}f#n1^QI?)2Q;42RR{3G$(Fcy4hB$8*SJXakMH~F!0aujN z^o;&M8@RYHTjL)T9t8-T!E*9Tsvb7Bzaa>SN<$3>CQ!LE%RQ(8;)EJFu{EY@TS@KP{p966bYGUG#oH~E`Ms86hq2^up59+E>?7nOq zU4a@fb{7zO=3)lKb+3;0^&R_y(SjsqmUb{zc3;u^6QOl5tNeb$vsb*LQX@eKs_RzP%^LzL9eCD zXN|~^C@rqE8Ty+*3=9lKO3%tIx=Uv(ASNxZqGdpb?=-mEJ>1SlE7hQJ-*xwK1YykT zI))&$!`y|)C&6q!NxlHq6jwcjz`2GolJW^vae>EjUm#-7zQZR|E}`YhY92MUzwP?Y z2#O7tSJNW&*}K5?A&2ecJTSssoPR@Wryvk98pnoCnaheQbX&{;RPjnyZgD023x@H7 zBP@Z~0#Qw~c643DI%U*egg7D=2o2}?jANq!IEGpd`$7nTs9=Pr&ZWZ$E0GXiy!+A% zIk16aqPeyGT=!+({=wmtyHKPX*cXhjLqJ?cQAOJj{x!?R&C7coh!vY~?9}J5D6f>${IaB(I<&dp=hT8hYs8E0WE2%e7+@n9@MlClcw+T(D#4s)>` z|4mzW#bVDC-Ak&#*N#s;{ruDdh)-y3iW$UL#3uy#>$eIksvogk3)RRdpqCJzW;wep zMSKd|9)l{twgKu|fLc%-B-hqoxP%fp62?)~IWxBqf}w-$;}w#Sg?`7Gl1)bJ+DNK4MwG$A~MGgcr}unjFN_qk?Azs*`yF~c^8zQg1`&uZmXA;^HGtrbKl}NBk|!Jt-vimq@6xEe4jj1C1an3;pld(zS&K&CrrVub0y5R65KQS+>vqRQIGEvWeq;=O{ZE_jb< zxyZ}gACnAfo_vv76GUt4`1tiFKjtfCO+8F9m=6|HMrF;JX0yUF*2iGZI9EkA9fOI} ztY$mo;L&*?#Zg2Tge1&;ZLi?oBS&k26c7bE$``R^PdrAk%QtQpg7-+e>~dt4G<1!S zq8we2sr@nTQFXAK{4zSPHwa05ARv+>d~BBUd?ek#P}1_rb7@)NeJv14w#tqx+Q{A* zO=yS$DPrMcC^H!%MgihIB0fb+XLmGs5Jk(x)2UY|;8CJ3GZTE5bO7ItB;UoX zfQ*VT@!~Z*7(q}F`xGk>&~p`dza4^3A_5j+R%oRWlY`b$Qq#dWY2!#7Wiu-FMIot- zLJ~+JGfJH@ruA&-*4r;Q3=#i$^2IAR^Ds)$tnd}1lr(&dh;3_(_WrA{8IBc)Q8nlN z8xekmA(bR6*^ig9v<-}|wUrzBxK7k&>}s2Tq!Jq^?%)!Mnfhc(+Vxu)GY2CQRZB%n z-()JQ?p1!9pe~Aj=~OL>e*Ca+01|W5OiB~$0=Go&K5zt8fGALfmRIk-^q~siwXWer zjIu5Zm-+Z%aG_>KIjI}rLqrtO!GUZ#=43>_K;j>wD)?7WK~ELlVj!bIK#c}6_|QU6 zHYHJ8jSiV2xQf-hTu`8}wH;EMP>(zTmGrV-I3E z*KV-$e_IE-0qQcKs7lWr{DP3`Ffu@X6r<3b?|TM#he+bNgDz4X21wXq0C#XX%ci8N z`lc6e$reRPUHZmTt#Hhs426Bv;XF`ElPFCMQ?^wqfKejj3*3QR-r9vR;L8U;a`LKSxjq53uMV7Ng0v b8U~NK{^ymz!ILim?vt--ysO&qKbQOmmA}MC literal 0 HcmV?d00001 diff --git a/crates/clawhdf5-format/tests/fixtures/blosc2/zstd_dict.err b/crates/clawhdf5-format/tests/fixtures/blosc2/zstd_dict.err new file mode 100644 index 0000000..658c6ba --- /dev/null +++ b/crates/clawhdf5-format/tests/fixtures/blosc2/zstd_dict.err @@ -0,0 +1 @@ +dictionar \ No newline at end of file diff --git a/crates/clawhdf5/Cargo.toml b/crates/clawhdf5/Cargo.toml index a6401fe..1eca373 100644 --- a/crates/clawhdf5/Cargo.toml +++ b/crates/clawhdf5/Cargo.toml @@ -47,8 +47,9 @@ lzf = ["clawhdf5-format/lzf"] bitshuffle = ["clawhdf5-format/bitshuffle"] bzip2 = ["clawhdf5-format/bzip2"] blosc = ["clawhdf5-format/blosc"] +blosc2 = ["clawhdf5-format/blosc2"] # Every plugin filter. -plugin-filters = ["lzf", "bitshuffle", "bzip2", "blosc"] +plugin-filters = ["lzf", "bitshuffle", "bzip2", "blosc", "blosc2"] # Dataset::verify_provenance() — recompute a dataset's SHA-256 and compare # against its stored _provenance_sha256 attribute. On by default, matching # clawhdf5-format's own default-on `provenance` feature. diff --git a/crates/clawhdf5/tests/plugin_filters_interop.rs b/crates/clawhdf5/tests/plugin_filters_interop.rs index 81cc87c..00911c6 100644 --- a/crates/clawhdf5/tests/plugin_filters_interop.rs +++ b/crates/clawhdf5/tests/plugin_filters_interop.rs @@ -77,7 +77,7 @@ except ImportError: hdf5plugin = None path = sys.argv[1] FILTERS = eval('(' + sys.argv[2] + ')') -cases = [ +cases = eval('(' + sys.argv[3] + ')') if len(sys.argv) > 3 else [ (') { let dir = tempfile::tempdir().unwrap(); let path = dir.path().join(format!("{tag}.h5")); - let n: usize = run_python(GENERATE, &[path.to_str().unwrap(), filters]) - .parse() - .unwrap(); + let mut args = vec![path.to_str().unwrap(), filters]; + args.extend(cases); + let n: usize = run_python(GENERATE, &args).parse().unwrap(); assert!(n > 0); let file = File::open(&path).unwrap(); for i in 0..n { @@ -353,6 +363,110 @@ fn blosc_written_by_clawhdf5_reads_in_hdf5plugin() { } } +#[cfg(feature = "blosc2")] +#[test] +fn blosc2_written_by_hdf5plugin_reads_exactly() { + if !have_python("h5py, hdf5plugin") { + return; + } + // Every codec hdf5plugin's Blosc2 offers, each lossless filter, and + // levels from "store" to maximum. Truncating precision is lossy, so it + // is checked separately against h5py's own reading. + check_h5py_written( + "blosc2", + r#"[(f'{c} {f} {l}', hdf5plugin.Blosc2(cname=c, clevel=l, filters=f)) + for c in ['blosclz', 'lz4', 'lz4hc', 'zlib', 'zstd'] + for f, l in [(hdf5plugin.Blosc2.NOFILTER, 5), + (hdf5plugin.Blosc2.SHUFFLE, 9), + (hdf5plugin.Blosc2.BITSHUFFLE, 1), + (hdf5plugin.Blosc2.DELTA, 5)]] + + [('blosclz level 0', hdf5plugin.Blosc2(cname='blosclz', clevel=0)), + ('zstd level 0 bitshuffle', + hdf5plugin.Blosc2(cname='zstd', clevel=0, filters=hdf5plugin.Blosc2.BITSHUFFLE))]"#, + ); +} + +/// Blosc2 over more shapes and dtypes: every integer and float width, +/// 1-D to 5-D chunks (B2ND arrays from 2-D on) with partial edge chunks and +/// block shapes that pad the chunk, datasets of zeros, of one repeated value +/// and of NaN (Blosc2's "special" chunks), and Fletcher32 before Blosc2 +/// (which makes hdf5-blosc2 fall back from B2ND to a plain frame). +#[cfg(feature = "blosc2")] +#[test] +fn blosc2_shapes_and_special_chunks_read_exactly() { + if !have_python("h5py, hdf5plugin") { + return; + } + let cases = r#"[(dt, shape, chunks, kind) + for dt in ['u2', 'f8'] + for shape, chunks in [((777,), (100,)), + ((37, 53), (10, 16)), + ((9, 10, 11), (4, 5, 3)), + ((6, 7, 5, 9), (3, 2, 5, 4))] + for kind in ['ramp', 'noise']] + + [('