diff --git a/CHANGELOG.md b/CHANGELOG.md index 3f6fbd0..80b4579 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -89,6 +89,41 @@ `names_whose_hashes_collide_are_found_by_name` in `crates/clawhdf5/tests/writer_groups_interop.rs`. +### Blosc2 (2026-09-26) +- **Blosc2 (filter 32026) reads, in pure Rust.** Files written with + hdf5plugin's `Blosc2` failed with `UnsupportedFilter`. New feature + `blosc2` (`clawhdf5-format` and `clawhdf5`, included in `plugin-filters`) + decodes the Blosc2 contiguous frame hdf5-blosc2 stores per chunk, the + B2ND arrays it uses for chunks of 2 or more dimensions (blocks gathered + back into C order), Blosc2 chunks with their special values (zeros, NaN, + uninitialised, one repeated value), and the shuffle, bit-shuffle, delta + and truncate-precision filters, over the BloscLZ, LZ4/LZ4HC, Zlib and + Zstandard codecs shared with Blosc 1. Read only: there is no Blosc2 + encoder. Dictionaries, lazy chunks, variable-length blocks, user-defined + codecs and registered Blosc2 filters (e.g. bytedelta) are errors; + uninitialised chunks read as zeros. Tested against h5py 3.16 + + hdf5plugin 7.1 (every codec, filter and level 0-9; 1- to 5-D chunks with + partial edge chunks; every integer width and f4/f8; datasets of zeros, + one value and NaN; Fletcher32 before Blosc2) and against frames from + python-blosc2 4.13.1 for what hdf5plugin never writes + (`crates/clawhdf5-format/tests/fixtures/blosc2/`); the decoder is + fuzzed. Conformance: 576 of 697 files ok (was 575) — h5ex_d_blosc2. +- **A crafted Blosc2 chunk cannot allocate more than a few times its HDF5 + chunk size.** Found in review before release: the sizes a frame declares + sized the decoder's buffers. A 173-byte frame whose offsets chunk claimed + 2 GiB was decoded in full for a 1 MiB chunk; an empty chunk allocated its + declared block size twice (about 1 GiB); a B2ND chunk was decoded whole + with its padding (up to 16x the chunk); and a Zstandard stream's declared + window (up to 100 MiB) was reserved as the decoder was reused, which also + affected Blosc 1 and bitshuffle with Zstandard. Now the offsets chunk is + capped at the chunk size, block sizes are clamped to the chunk, B2ND + blocks are placed as they are decoded (padding is never held), and the + Zstandard window is capped at twice the stream's output (at least + 128 KiB). A B2ND chunk may no longer be larger than its array, which + hdf5-blosc2 never writes. `tests/blosc2_alloc_bounds.rs` measures peak + allocation for these frames and for 45,000 fuzzed ones: at most 6x the + chunk size, twice the input and 2 MiB of Zstandard state. + ### Concurrent reads (2026-09-26) - **Full reads of chunked datasets scale with threads again when rayon's pool has one thread.** Each full read handed its chunks to rayon to @@ -459,6 +494,7 @@ missing feature ("unsupported filter: 32026 (Blosc2, not implemented by clawhdf5)"). - **Not implemented:** Blosc2 (32026) and ZFP (32013) remain a clear error. + (Blosc2 reads since the `blosc2` feature, see above.) - **Wrong data: a chunk that decodes short read as zeros** (pre-existing, every filter). HDF5 stores every chunk at the full chunk size, so a filter pipeline that decodes to fewer bytes means a corrupt chunk; every chunk diff --git a/CLAUDE.md b/CLAUDE.md index 1cb7aba..b029c5d 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -11,7 +11,7 @@ Cargo workspace with 18 crates under `crates/` (plus `libaec-sys`, an internal F |-------|------| | `clawhdf5-format` | HDF5 binary spec parser (superblock, B-tree, heap) — also holds shared type definitions and physical constants | | `clawhdf5-io` | Read/write implementation | -| `clawhdf5-filters` | Deflate backends (zlib-rs, zlib-ng, Apple Compression); the HDF5 filter pipeline, the filter registry (`clawhdf5_format::filter_registry`) and the other codecs (LZ4, Zstd, SZIP, N-Bit, scale-offset, pcodec, and the pure-Rust plugin filters LZF, bitshuffle, bzip2, Blosc 1) live in `clawhdf5-format`. No Blosc2 or ZFP. | +| `clawhdf5-filters` | Deflate backends (zlib-rs, zlib-ng, Apple Compression); the HDF5 filter pipeline, the filter registry (`clawhdf5_format::filter_registry`) and the other codecs (LZ4, Zstd, SZIP, N-Bit, scale-offset, pcodec, and the pure-Rust plugin filters LZF, bitshuffle, bzip2, Blosc 1, and Blosc2 read-only) live in `clawhdf5-format`. No ZFP. | | `clawhdf5-derive` | Proc-macro derive for HDF5-serializable structs | | `clawhdf5` | Main facade crate | | `clawhdf5-netcdf4` | NetCDF-4 compatibility layer | diff --git a/README.md b/README.md index 4ae4d1c..25f43a7 100644 --- a/README.md +++ b/README.md @@ -668,7 +668,7 @@ clawhdf5 workspace (17 crates, ~86K lines of Rust in src/, ~104K with tests ├── Core HDF5 │ ├── clawhdf5-format — Binary parser/writer (no_std-capable), shared type definitions │ ├── clawhdf5-io — I/O abstraction (file/memory readers; optional mmap, async, HSDS, MPI) -│ ├── clawhdf5-filters — Fast deflate path (zlib-ng); the filter registry and the lz4/zstd/pcodec/szip/LZF/bitshuffle/bzip2/Blosc filters live in clawhdf5-format +│ ├── clawhdf5-filters — Fast deflate path (zlib-ng); the filter registry and the lz4/zstd/pcodec/szip/LZF/bitshuffle/bzip2/Blosc/Blosc2 filters live in clawhdf5-format │ ├── clawhdf5-derive — Proc macros │ ├── clawhdf5 — High-level API │ ├── clawhdf5-netcdf4 — NetCDF-4 support @@ -779,13 +779,15 @@ stores keep their setting. Opt out with `float16 = false` or | `bitshuffle` | no | Bitshuffle filter (id 32008) with its LZ4 and Zstandard modes: read and write. Pure Rust (lz4_flex, ruzstd) | | `bzip2` | no | bzip2 filter (id 307): read and write. Pure Rust (the `bzip2` crate's libbz2-rs-sys backend compiles no C) | | `blosc` | no | Blosc 1 filter (id 32001): reads BloscLZ, LZ4/LZ4HC, Snappy, Zlib and Zstandard frames with byte or bit shuffle; writes LZ4, Snappy, Zlib or Zstandard (not BloscLZ). Pure Rust | -| `plugin-filters` | no | All four above | +| `blosc2` | no | Blosc2 filter (id 32026), read only: hdf5plugin's frames and B2ND (n-D) chunks, BloscLZ, LZ4/LZ4HC, Zlib and Zstandard, with shuffle, bit shuffle, delta or truncated precision. Pure Rust | +| `plugin-filters` | no | All five above | -Blosc2 (32026) and ZFP (32013) are not implemented: reading them fails with -`UnsupportedFilter`, whose message names the filter. Any other filter can be -supplied at run time with `filter_registry::register_filter` (a decoder -closure, or a `FilterCodec` that also encodes). The facade (`clawhdf5`) -forwards `lzf`, `bitshuffle`, `bzip2`, `blosc` and `plugin-filters`. Write +ZFP (32013) is not implemented: reading it fails with `UnsupportedFilter`, +whose message names the filter. clawhdf5 cannot write Blosc2. Any other +filter can be supplied at run time with `filter_registry::register_filter` (a +decoder closure, or a `FilterCodec` that also encodes). The facade +(`clawhdf5`) forwards `lzf`, `bitshuffle`, `bzip2`, `blosc`, `blosc2` and +`plugin-filters`. Write with `DatasetBuilder::with_lzf()`, `with_bitshuffle(..)`, `with_bzip2(..)` and `with_blosc(..)`; h5py + hdf5plugin read the result (tested both ways in `crates/clawhdf5/tests/plugin_filters_interop.rs`). The pure-Rust Zstandard diff --git a/crates/clawhdf5-format/Cargo.toml b/crates/clawhdf5-format/Cargo.toml index 1d1998c..c1ec6df 100644 --- a/crates/clawhdf5-format/Cargo.toml +++ b/crates/clawhdf5-format/Cargo.toml @@ -76,8 +76,10 @@ bitshuffle = ["lz4_flex", "ruzstd"] bzip2 = ["dep:bzip2", "std"] # Blosc 1 (32001) with its BloscLZ, LZ4, Snappy, Zlib and Zstandard codecs. blosc = ["lz4_flex", "ruzstd", "snap", "deflate", "std"] +# Blosc2 (32026), read-only: frames, B2ND arrays, and the Blosc codecs above. +blosc2 = ["blosc"] # Every plugin filter above. -plugin-filters = ["lzf", "bitshuffle", "bzip2", "blosc"] +plugin-filters = ["lzf", "bitshuffle", "bzip2", "blosc", "blosc2"] [[bench]] name = "parallel_decompress_bench" diff --git a/crates/clawhdf5-format/src/filter_registry.rs b/crates/clawhdf5-format/src/filter_registry.rs index 6d39613..442e27c 100644 --- a/crates/clawhdf5-format/src/filter_registry.rs +++ b/crates/clawhdf5-format/src/filter_registry.rs @@ -5,7 +5,8 @@ //! * **Built-in filters** — a static table of the filters compiled into this //! build: the HDF5 standard filters (deflate, shuffle, Fletcher32, szip, //! N-Bit, scale-offset) and the plugin filters whose cargo features are -//! enabled (LZ4, Zstandard, pcodec, LZF, bitshuffle, bzip2, blosc). +//! enabled (LZ4, Zstandard, pcodec, LZF, bitshuffle, bzip2, blosc, +//! blosc2). //! [`builtin_filters`] lists them. //! * **Registered filters** (`std` only) — codecs the application supplies //! for any other ID with [`register_filter`] (a [`FilterCodec`], or just a @@ -166,7 +167,7 @@ pub fn known_filter(id: u16) -> Option<(&'static str, Option<&'static str>)> { 32019 => ("JPEG", None), 32022 => ("BitGroom", None), 32023 => ("Granular BitRound", None), - 32026 => ("Blosc2", None), + 32026 => ("Blosc2", Some("blosc2")), _ => return None, }) } @@ -452,12 +453,12 @@ pub(crate) mod tests { #[test] fn unsupported_filter_error_names_the_filter() { let msg = FormatError::UnsupportedFilter(32026).to_string(); + assert!(msg.contains("Blosc2") && msg.contains("`blosc2`"), "{msg}"); + let msg = FormatError::UnsupportedFilter(32013).to_string(); assert!( - msg.contains("Blosc2") && msg.contains("not implemented"), + msg.contains("ZFP") && msg.contains("not implemented"), "{msg}" ); - let msg = FormatError::UnsupportedFilter(32013).to_string(); - assert!(msg.contains("ZFP"), "{msg}"); let msg = FormatError::UnsupportedFilter(32000).to_string(); assert!(msg.contains("LZF") && msg.contains("`lzf`"), "{msg}"); assert_eq!( diff --git a/crates/clawhdf5-format/src/filters.rs b/crates/clawhdf5-format/src/filters.rs index 266e530..42d7137 100644 --- a/crates/clawhdf5-format/src/filters.rs +++ b/crates/clawhdf5-format/src/filters.rs @@ -455,6 +455,13 @@ pub(crate) static BUILTIN_FILTERS: &[BuiltinFilter] = &[ }, encode: None, }, + #[cfg(feature = "blosc2")] + BuiltinFilter { + id: crate::filter_pipeline::FILTER_BLOSC2, + name: "blosc2", + decode: crate::filters_blosc2::blosc2_decode, + encode: None, + }, ]; /// Decode the HDF5 scale-offset filter (id 6). diff --git a/crates/clawhdf5-format/src/filters_bitshuffle.rs b/crates/clawhdf5-format/src/filters_bitshuffle.rs index d20835e..6b23291 100644 --- a/crates/clawhdf5-format/src/filters_bitshuffle.rs +++ b/crates/clawhdf5-format/src/filters_bitshuffle.rs @@ -218,12 +218,20 @@ pub(crate) fn bitshuffle_decode( } /// Decode Zstandard frames into exactly `dst`, failing if they hold more. +/// +/// ruzstd reserves a frame's declared window (by default up to 100 MiB) +/// before decoding it, so the window is capped at what the output could +/// need: twice `dst` (window sizes are rounded up), and at least 128 KiB. +/// The encoders behind these filters (c-blosc, c-blosc2, bitshuffle) +/// compress each block in one call with its size known, so libzstd's +/// window never exceeds the block. #[cfg(any(feature = "bitshuffle", feature = "blosc"))] pub(crate) fn zstd_decode_into( decoder: &mut ruzstd::decoding::FrameDecoder, frames: &[u8], dst: &mut [u8], ) -> Result { + decoder.set_max_window_size((2 * dst.len()).max(1 << 17) as u64); decoder .decode_all(frames, dst) .map_err(|e| FormatError::DecompressionError(format!("zstd: {e}"))) diff --git a/crates/clawhdf5-format/src/filters_blosc.rs b/crates/clawhdf5-format/src/filters_blosc.rs index 3edc964..fadfc1e 100644 --- a/crates/clawhdf5-format/src/filters_blosc.rs +++ b/crates/clawhdf5-format/src/filters_blosc.rs @@ -49,7 +49,7 @@ fn le32(b: &[u8], at: usize) -> Result { /// The codec inside a Blosc frame (flags bits 5-7). #[derive(Debug, Clone, Copy, PartialEq, Eq)] -enum Codec { +pub(crate) enum Codec { BloscLz, Lz4, Snappy, @@ -58,7 +58,7 @@ enum Codec { } impl Codec { - fn from_flags(flags: u8) -> Result { + pub(crate) fn from_flags(flags: u8) -> Result { match flags >> 5 { 0 => Ok(Codec::BloscLz), 1 => Ok(Codec::Lz4), @@ -71,7 +71,7 @@ impl Codec { } /// Decode one codec stream into exactly `dst`. -fn decode_stream( +pub(crate) fn decode_stream( codec: Codec, src: &[u8], dst: &mut [u8], diff --git a/crates/clawhdf5-format/src/filters_blosc2.rs b/crates/clawhdf5-format/src/filters_blosc2.rs new file mode 100644 index 0000000..f4d1690 --- /dev/null +++ b/crates/clawhdf5-format/src/filters_blosc2.rs @@ -0,0 +1,1288 @@ +//! Blosc2 (HDF5 filter 32026, `hdf5-blosc2`, hdf5plugin's `Blosc2`) in pure +//! Rust, read-only: the contiguous frame ("cframe") each chunk is stored +//! as, the Blosc2 chunks inside it, and the B2ND n-dimensional layout. +//! +//! References: c-blosc2's `README_CFRAME_FORMAT.rst`, +//! `README_CHUNK_FORMAT.rst` and `README_B2ND_METALAYER.rst`, and the +//! decoder in `blosc/blosc2.c` (`read_chunk_header`, `blosc_d`, +//! `pipeline_backward`), `blosc/frame.c` and `blosc/b2nd.c`, whose checks +//! this module mirrors. +//! +//! **The filter** (`blosc2_filter.c`). `cd_values`: `[0]` filter revision, +//! `[1]` block size, `[2]` type size, `[3]` chunk size in bytes, `[4]` +//! level, `[5]` filter (0 none, 1 shuffle, 2 bit shuffle, 3 delta, 4 +//! truncate precision), `[6]` codec, and for chunks of 2 to 8 (16 in newer +//! builds) dimensions `[7]` the rank and `[8..]` the chunk dimensions. Each +//! HDF5 chunk is one frame. A frame with a `b2nd` (or older `caterva`) +//! metalayer holds an n-D array the shape of the HDF5 chunk, whose data are +//! cut into blocks of `blockshape` (padded at the chunk's edges) stored one +//! after another, each in C order — so the decoded Blosc2 chunk is +//! reassembled into C order here. Otherwise the frame's first chunk is the +//! HDF5 chunk's data as it is. +//! +//! **Frame.** A msgpack header with fields at fixed offsets (header length, +//! frame length, sizes, type size, chunk size, codec flags) and a map of +//! metalayers; the chunks back to back; an "offsets" chunk (a Blosc2 chunk +//! of little-endian `i64` offsets, one per chunk, relative to the end of the +//! header — negative values encode a chunk of zeros, NaNs or uninitialised +//! values); and a msgpack trailer, which the decoder does not need. +//! +//! **Chunk.** A 16-byte Blosc header (as Blosc 1), extended to 32 bytes when +//! the shuffle and bit-shuffle flags are both set: six filter slots with +//! their metadata, the codec, and flags for dictionaries, lazy chunks and +//! "special" chunks (all zeros, NaN, uninitialised, or one repeated value +//! stored after the header). A regular chunk has a table of block starts; +//! each block is one stream, or `typesize` streams when the "do not split" +//! flag is clear and the block is not the short last one. A stream is an +//! `i32` length and the codec's output: 0 means zeros, negative with a token +//! byte means a run of one byte, and a length equal to the stream's size +//! means stored raw. Decoded blocks run through the filter pipeline +//! backwards (shuffle, bit shuffle, delta; truncate-precision needs no +//! decoding). The codecs are Blosc 1's: BloscLZ, LZ4 (and LZ4HC), Zlib and +//! Zstandard, shared with [`crate::filters_blosc`]. +//! +//! Chunks of uninitialised values read as zeros (libhdf5 hands back +//! whatever its buffer held). Not supported (a clear error, never data): +//! variable-length blocks, dictionaries, lazy chunks, user-defined codecs +//! and registered filters (e.g. bytedelta), sparse frames. + +use crate::error::FormatError; +use crate::filter_registry::FilterContext; +use crate::filters_bitshuffle::bitunshuffle_block; +use crate::filters_blosc::{Codec, blosclz_decompress, decode_stream}; + +const MIN_HEADER: usize = 16; +const EXT_HEADER: usize = 32; + +const FLAG_SHUFFLE: u8 = 0x01; +const FLAG_MEMCPYED: u8 = 0x02; +const FLAG_BITSHUFFLE: u8 = 0x04; +const FLAG_DELTA: u8 = 0x08; +const FLAG_DONT_SPLIT: u8 = 0x10; + +const B2_USEDICT: u8 = 0x01; +const B2_LAZY: u8 = 0x08; +const VL_BLOCKS: u8 = 0x01; + +const SPECIAL_ZERO: u8 = 1; +const SPECIAL_NAN: u8 = 2; +const SPECIAL_VALUE: u8 = 3; +const SPECIAL_UNINIT: u8 = 4; + +const FILTER_NONE: u8 = 0; +const FILTER_SHUFFLE: u8 = 1; +const FILTER_BITSHUFFLE: u8 = 2; +const FILTER_DELTA: u8 = 3; +const FILTER_TRUNC_PREC: u8 = 4; + +/// Chunk format versions: 3 is the Blosc 2 alpha series. +const VERSION_ALPHA: u8 = 3; +/// The newest chunk format c-blosc2 knows (6 added variable-length blocks). +const VERSION_MAX: u8 = 6; +/// The newest frame format c-blosc2 knows. +const FRAME_VERSION_MAX: u8 = 3; +const MAX_BLOCKSIZE: usize = 536_866_816; +/// `B2ND_MAX_DIM` in current c-blosc2 (8 in older builds). +const B2ND_MAX_DIM: usize = 16; + +// Frame header field offsets (c-blosc2 `frame.h`). +const FRAME_HEADER_LEN: usize = 11; +const FRAME_LEN: usize = 16; +const FRAME_FLAGS: usize = 25; +const FRAME_TYPE: usize = 26; +const FRAME_NBYTES: usize = 30; +const FRAME_CBYTES: usize = 39; +const FRAME_TYPESIZE: usize = 48; +const FRAME_CHUNKSIZE: usize = 58; +const FRAME_HEADER_MINLEN: usize = 87; +const FRAME_IDX_SIZE: usize = 89; +const FRAME_VL_BLOCKS: u8 = 0x80; + +fn err(msg: &str) -> FormatError { + FormatError::DecompressionError(format!("blosc2: {msg}")) +} + +fn le_i32(b: &[u8], at: usize) -> Result { + b.get(at..at + 4) + .map(|s| i32::from_le_bytes(s.try_into().unwrap())) + .ok_or_else(|| err("truncated chunk")) +} + +fn be(b: &[u8], at: usize) -> Result<[u8; N], FormatError> { + b.get(at..at + N) + .map(|s| s.try_into().unwrap()) + .ok_or_else(|| err("truncated frame header")) +} + +/// A parsed Blosc2 chunk header. +struct ChunkHeader { + version: u8, + flags: u8, + typesize: usize, + nbytes: usize, + blocksize: usize, + cbytes: usize, + overhead: usize, + filters: [u8; 6], + filters_meta: [u8; 6], + udcodec: u8, + blosc2_flags: u8, + special: u8, +} + +fn read_header(src: &[u8]) -> Result { + if src.len() < MIN_HEADER { + return Err(err("truncated chunk header")); + } + let version = src[0]; + let flags = src[2]; + let typesize = src[3] as usize; + let nbytes = le_i32(src, 4)?; + let blocksize = le_i32(src, 8)?; + let cbytes = le_i32(src, 12)?; + if cbytes < MIN_HEADER as i32 { + return Err(err("chunk size smaller than its header")); + } + if blocksize <= 0 || blocksize as usize > MAX_BLOCKSIZE { + return Err(err("bad block size")); + } + if typesize == 0 { + return Err(err("type size is zero")); + } + if nbytes < 0 { + return Err(err("negative decoded size")); + } + let (nbytes, mut blocksize, cbytes) = (nbytes as usize, blocksize as usize, cbytes as usize); + let mut h = ChunkHeader { + version, + flags, + typesize, + nbytes, + blocksize, + cbytes, + overhead: MIN_HEADER, + filters: [0; 6], + filters_meta: [0; 6], + udcodec: 0, + blosc2_flags: 0, + special: 0, + }; + let mut flags2 = 0; + if flags & (FLAG_SHUFFLE | FLAG_BITSHUFFLE) == FLAG_SHUFFLE | FLAG_BITSHUFFLE { + if cbytes < EXT_HEADER || src.len() < EXT_HEADER { + return Err(err("truncated extended chunk header")); + } + h.overhead = EXT_HEADER; + h.filters.copy_from_slice(&src[16..22]); + h.udcodec = src[22]; + h.filters_meta.copy_from_slice(&src[24..30]); + flags2 = src[30]; + h.blosc2_flags = src[31]; + h.special = (h.blosc2_flags >> 4) & 7; + if h.special == SPECIAL_VALUE { + let ts = cbytes - EXT_HEADER; + if ts == 0 || ts > nbytes || !nbytes.is_multiple_of(ts) { + return Err(err("bad repeated-value chunk")); + } + } else if h.special != 0 && h.special != SPECIAL_ZERO && !nbytes.is_multiple_of(typesize) { + return Err(err("decoded size is not a whole number of elements")); + } + if version == VERSION_ALPHA { + h.filters[5] = 0; + h.filters_meta[5] = 0; + } + } else { + // A Blosc 1 style header: the filters come from the flags. + if flags & FLAG_SHUFFLE != 0 { + h.filters[5] = FILTER_SHUFFLE; + } + if flags & FLAG_BITSHUFFLE != 0 { + h.filters[5] = FILTER_BITSHUFFLE; + } + if flags & FLAG_DELTA != 0 { + h.filters[4] = FILTER_DELTA; + } + } + if version > VERSION_MAX && flags2 & !VL_BLOCKS != 0 { + return Err(err(&format!("chunk format version {version} is too new"))); + } + if flags2 & VL_BLOCKS != 0 { + return Err(err("variable-length blocks are not supported")); + } + // c-blosc2 clamps only a non-empty chunk's block size; clamping an + // empty one too keeps its scratch blocks from being sized by the + // header (up to 512 MiB each). + if blocksize > nbytes { + blocksize = nbytes; + } + h.blocksize = blocksize; + if cbytes > src.len() { + return Err(err("chunk is longer than its buffer")); + } + Ok(h) +} + +/// Decompress one Blosc2 chunk (header included), refusing to produce more +/// than `limit` bytes. +pub fn blosc2_decompress_chunk(src: &[u8], limit: usize) -> Result, FormatError> { + let (h, src) = open_chunk(src, limit)?; + let mut out = vec![0u8; h.nbytes]; + decode_blocks(&h, src, &mut |_, _| true, &mut |at, block| { + out[at..at + block.len()].copy_from_slice(block); + })?; + Ok(out) +} + +/// Read and check a chunk's header: its decoded size must be within +/// `limit`. Returns the header and the chunk's bytes. +fn open_chunk(src: &[u8], limit: usize) -> Result<(ChunkHeader, &[u8]), FormatError> { + let h = read_header(src)?; + if h.nbytes > limit { + return Err(err("decoded size exceeds the limit")); + } + if h.special > SPECIAL_UNINIT { + return Err(err(&format!("unknown special chunk type {}", h.special))); + } + if h.flags & FLAG_MEMCPYED != 0 && h.cbytes != h.nbytes + h.overhead { + return Err(err("stored chunk has the wrong size")); + } + let src = &src[..h.cbytes]; + Ok((h, src)) +} + +/// Decode a chunk block by block into a destination the caller has +/// zeroed: `sink(offset, block)` receives each decoded block (the last may +/// be short), except blocks for which `want(offset, len)` is false and +/// the blocks of a special all-zeros chunk. Holds at most three blocks of scratch, so a +/// caller that keeps only part of each block (B2ND padding) never holds +/// the whole chunk. +fn decode_blocks( + h: &ChunkHeader, + src: &[u8], + want: &mut dyn FnMut(usize, usize) -> bool, + sink: &mut dyn FnMut(usize, &[u8]), +) -> Result<(), FormatError> { + let nbytes = h.nbytes; + // `read_header` clamps the block size to the decoded size. + let blocksize = h.blocksize; + let nblocks = if nbytes == 0 { + 0 + } else { + nbytes.div_ceil(blocksize) + }; + let leftover = nbytes % blocksize.max(1); + let block_len = |j: usize| { + if j == nblocks - 1 && leftover > 0 { + leftover + } else { + blocksize + } + }; + if h.special != 0 { + // Filled per block, as c-blosc2 does: each block must hold whole + // values. + let fill: Option<&[u8]> = match h.special { + SPECIAL_ZERO | SPECIAL_UNINIT => None, + SPECIAL_NAN => Some(match h.typesize { + 4 => &[0x00, 0x00, 0xc0, 0x7f], + 8 => &[0, 0, 0, 0, 0, 0, 0xf8, 0x7f], + _ => return Err(err("NaN chunk of a type that is not f32 or f64")), + }), + _ => Some(&src[EXT_HEADER..]), + }; + if let Some(value) = fill { + let ts = value.len(); + if !blocksize.max(1).is_multiple_of(ts) || !leftover.is_multiple_of(ts) { + return Err(err("special chunk blocks are not whole values")); + } + if nblocks > 0 { + let block: Vec = value.iter().copied().cycle().take(blocksize).collect(); + for j in 0..nblocks { + let len = block_len(j); + if want(j * blocksize, len) { + sink(j * blocksize, &block[..len]); + } + } + } + } + return Ok(()); + } + if h.flags & FLAG_MEMCPYED != 0 { + let data = &src[h.overhead..]; + for j in 0..nblocks { + let (at, len) = (j * blocksize, block_len(j)); + if want(at, len) { + sink(at, &data[at..at + len]); + } + } + return Ok(()); + } + if h.blosc2_flags & B2_USEDICT != 0 { + return Err(err("dictionary-compressed chunks are not supported")); + } + if h.blosc2_flags & B2_LAZY != 0 { + return Err(err("lazy chunks are not supported")); + } + let codec = match h.flags >> 5 { + 6 => { + return Err(err(&format!( + "user-defined codec {} is not supported", + h.udcodec + ))); + } + 2 | 5 | 7 => return Err(err(&format!("unknown codec {}", h.flags >> 5))), + _ => Codec::from_flags(h.flags)?, + }; + for &f in &h.filters { + if f > FILTER_TRUNC_PREC { + return Err(err(&format!("filter {f} is not supported"))); + } + } + if nbytes == 0 { + return Ok(()); + } + let bstarts_end = nblocks + .checked_mul(4) + .and_then(|n| n.checked_add(h.overhead)) + .ok_or_else(|| err("block table overflows"))?; + if bstarts_end > src.len() { + return Err(err("block table is truncated")); + } + let dont_split = h.flags & FLAG_DONT_SPLIT != 0; + // The delta filter's reference is the first block, decoded. + let delta = h.filters.contains(&FILTER_DELTA); + let mut first = Vec::new(); + let mut tmp = vec![0u8; blocksize]; + let mut tmp2 = vec![0u8; blocksize]; + let mut zstd = None; + for j in 0..nblocks { + let is_leftover = j == nblocks - 1 && leftover > 0; + let bsize = block_len(j); + let wanted = want(j * blocksize, bsize); + if !wanted && !(delta && j == 0) { + continue; + } + let start = le_i32(src, h.overhead + 4 * j)?; + if start <= 0 || start as usize >= src.len() { + return Err(err("block start out of range")); + } + let mut pos = start as usize; + let nstreams = if !dont_split && !is_leftover { + h.typesize + } else { + 1 + }; + let neblock = bsize / nstreams; + if neblock == 0 || neblock * nstreams != bsize { + return Err(err("block is not a whole number of streams")); + } + let cur = &mut tmp[..bsize]; + for s in 0..nstreams { + let csize = le_i32(src, pos).map_err(|_| err("stream runs past the chunk"))?; + pos += 4; + let dst = &mut cur[s * neblock..(s + 1) * neblock]; + if csize == 0 { + dst.fill(0); + } else if csize < 0 { + let token = *src + .get(pos) + .ok_or_else(|| err("stream runs past the chunk"))?; + pos += 1; + if token & 1 == 0 || csize < -255 { + return Err(err("unsupported stream token")); + } + dst.fill((-csize) as u8); + } else { + let csize = csize as usize; + let stream = src + .get(pos..pos + csize) + .ok_or_else(|| err("stream runs past the chunk"))?; + pos += csize; + if csize == neblock { + dst.copy_from_slice(stream); + } else if codec == Codec::BloscLz { + if blosclz_decompress(stream, dst) != neblock { + return Err(err("blosclz stream is malformed")); + } + } else { + decode_stream(codec, stream, dst, &mut zstd)?; + } + } + } + run_filters_backward(h, cur, &mut tmp2[..bsize], &first); + if delta && j == 0 { + first = cur.to_vec(); + } + if wanted { + sink(j * blocksize, cur); + } + } + Ok(()) +} + +/// Undo the chunk's filters on one decoded block `cur`, in place. `first` +/// is the decoded first block (empty while decoding it), the delta +/// filter's reference. +fn run_filters_backward(h: &ChunkHeader, cur: &mut [u8], scratch: &mut [u8], first: &[u8]) { + let bsize = cur.len(); + let ts = h.typesize; + for i in (0..6).rev() { + match h.filters[i] { + FILTER_SHUFFLE => { + let m = h.filters_meta[i] as usize; + let bytes = if m == 0 { ts } else { m }; + unshuffle(bytes, cur, scratch); + cur.copy_from_slice(scratch); + } + FILTER_BITSHUFFLE => { + let mut n = bsize / ts; + if h.version == 2 { + if !n.is_multiple_of(8) { + continue; + } + } else { + n -= n % 8; + } + let body = n * ts; + bitunshuffle_block(&cur[..body], &mut scratch[..body], n, ts); + cur[..body].copy_from_slice(&scratch[..body]); + } + FILTER_DELTA => delta_decode(first, cur, ts), + // Truncating precision is lossy and has nothing to undo. + FILTER_NONE | FILTER_TRUNC_PREC => {} + _ => unreachable!("filters are checked before decoding"), + } + } +} + +/// Byte unshuffle (`unshuffle_generic`): trailing bytes that do not make a +/// whole element are copied as they are. +fn unshuffle(bytes: usize, src: &[u8], dest: &mut [u8]) { + let n = src.len() / bytes; + for i in 0..n { + for b in 0..bytes { + dest[i * bytes + b] = src[b * n + i]; + } + } + dest[n * bytes..].copy_from_slice(&src[n * bytes..]); +} + +/// `delta_decoder`: the first block (`first` is empty) is XOR-accumulated +/// over its own elements; every other block is XORed with the (decoded) +/// first block. +/// Elements are 1, 2, 4 or 8 bytes wide (other sizes: 8 if a multiple of +/// 8, else 1); a trailing partial element is left alone. +fn delta_decode(first: &[u8], cur: &mut [u8], typesize: usize) { + let w = match typesize { + 1 | 2 | 4 | 8 => typesize, + t if t.is_multiple_of(8) => 8, + _ => 1, + }; + let n = cur.len() / w; + if first.is_empty() { + for i in 1..n { + for b in 0..w { + cur[i * w + b] ^= cur[(i - 1) * w + b]; + } + } + } else { + // The first block is at least as long as any other. + for (c, r) in cur[..n * w].iter_mut().zip(first) { + *c ^= *r; + } + } +} + +/// What the frame header says. +struct Frame<'a> { + buf: &'a [u8], + header_len: usize, + nbytes: usize, + cbytes: usize, + typesize: usize, + chunksize: usize, + nchunks: usize, + /// The decoded offsets chunk. + offsets: Vec, +} + +/// Parse a frame's header and decode its offsets chunk, holding no more +/// than an HDF5 chunk of `limit` bytes needs. +fn parse_frame(buf: &[u8], limit: usize) -> Result, FormatError> { + if buf.len() < FRAME_HEADER_MINLEN { + return Err(err("truncated frame header")); + } + if buf[0] & 0xf0 != 0x90 || buf[1] != 0xa8 || &buf[2..10] != b"b2frame\0" { + return Err(err("not a Blosc2 frame (bad magic)")); + } + let frame_version = buf[FRAME_FLAGS] & 0x0f; + if frame_version > FRAME_VERSION_MAX { + return Err(err(&format!( + "frame format version {frame_version} is too new" + ))); + } + if buf[FRAME_FLAGS] & FRAME_VL_BLOCKS != 0 { + return Err(err("variable-length blocks are not supported")); + } + if buf[FRAME_TYPE] & 0x0f != 0 { + return Err(err("not a contiguous frame")); + } + let header_len = i32::from_be_bytes(be(buf, FRAME_HEADER_LEN)?); + let frame_len = u64::from_be_bytes(be(buf, FRAME_LEN)?); + let nbytes = i64::from_be_bytes(be(buf, FRAME_NBYTES)?); + let cbytes = i64::from_be_bytes(be(buf, FRAME_CBYTES)?); + let typesize = i32::from_be_bytes(be(buf, FRAME_TYPESIZE)?); + let chunksize = i32::from_be_bytes(be(buf, FRAME_CHUNKSIZE)?); + if header_len < FRAME_HEADER_MINLEN as i32 || header_len as u64 > frame_len { + return Err(err("bad frame header length")); + } + if frame_len > buf.len() as u64 { + return Err(err("frame is longer than the chunk")); + } + if typesize <= 0 { + return Err(err("bad type size")); + } + if nbytes < 0 || cbytes < 0 || chunksize < 0 { + return Err(err("negative size in frame header")); + } + let header_len = header_len as usize; + let buf = &buf[..frame_len as usize]; + let cbytes = usize::try_from(cbytes).map_err(|_| err("bad compressed size"))?; + let data_end = header_len + .checked_add(cbytes) + .filter(|&e| e <= buf.len()) + .ok_or_else(|| err("chunks run past the frame"))?; + let nbytes = usize::try_from(nbytes).map_err(|_| err("bad decoded size"))?; + let chunksize = chunksize as usize; + // The offsets chunk follows the data chunks: one `i64` per chunk. The + // frame header's sizes are the file's word, so they must not size it: + // it may be no larger than the HDF5 chunk (`limit`, at least 128 + // bytes), i.e. one Blosc2 chunk per 8 bytes of output. hdf5-blosc2 + // writes one chunk per frame; only python-blosc2 arrays of tiny + // chunks come near the cap. + let off_src = &buf[data_end..]; + let max_offsets = limit.max(128); + let off_len = if nbytes == 0 { + 0 + } else if chunksize > 0 { + nbytes + .div_ceil(chunksize) + .checked_mul(8) + .filter(|&n| n <= max_offsets) + .ok_or_else(|| err("frame has more chunks than the HDF5 chunk can hold"))? + } else { + // Variable-size chunks: the offsets chunk tells how many. + max_offsets + }; + let offsets = if nbytes == 0 { + Vec::new() + } else { + blosc2_decompress_chunk(off_src, off_len)? + }; + let expected = if chunksize > 0 { + off_len + } else { + offsets.len() + }; + if !offsets.len().is_multiple_of(8) || offsets.len() != expected { + return Err(err("offsets chunk does not match the number of chunks")); + } + Ok(Frame { + buf, + header_len, + nbytes, + cbytes, + typesize: typesize as usize, + chunksize, + nchunks: offsets.len() / 8, + offsets, + }) +} + +impl Frame<'_> { + /// Where chunk `n` is: its size from the frame header (every chunk but + /// the last holds `chunksize` bytes; unknown if that is 0), and either + /// the special value its offset records or its bytes. + fn locate(&self, n: usize) -> Result<(Option, Located<'_>), FormatError> { + if n >= self.nchunks { + return Err(err("frame has no such chunk")); + } + let raw: [u8; 8] = self.offsets[8 * n..8 * n + 8].try_into().unwrap(); + let offset = i64::from_le_bytes(raw); + let size = if self.chunksize == 0 { + None + } else if n == self.nchunks - 1 && !self.nbytes.is_multiple_of(self.chunksize) { + Some(self.nbytes % self.chunksize) + } else { + Some(self.chunksize) + }; + if offset < 0 { + // A special chunk, recorded in the offset's top byte. + if size.is_none() { + return Err(err("special chunk in a frame without a chunk size")); + } + let kind = raw[7] & 7; + return match kind { + SPECIAL_ZERO | SPECIAL_UNINIT => Ok((size, Located::Zeros)), + SPECIAL_NAN => Ok(( + size, + Located::Fill(match self.typesize { + 4 => &[0x00, 0x00, 0xc0, 0x7f], + 8 => &[0, 0, 0, 0, 0, 0, 0xf8, 0x7f], + _ => return Err(err("NaN chunk of a type that is not f32 or f64")), + }), + )), + _ => Err(err(&format!("unknown special chunk offset {kind}"))), + }; + } + // Every chunk lies in the data section, between the header and the + // offsets chunk. + let end = self.header_len + self.cbytes; + let start = usize::try_from(offset) + .ok() + .and_then(|o| self.header_len.checked_add(o)) + .filter(|&s| s < end && end - s >= MIN_HEADER) + .ok_or_else(|| err("chunk offset out of range"))?; + Ok((size, Located::Data(&self.buf[start..end]))) + } + + /// Open chunk `n` (a regular one), which must decode to the frame's + /// size for it, and at most `limit` bytes. + fn open<'s>( + &self, + size: Option, + src: &'s [u8], + limit: usize, + ) -> Result<(ChunkHeader, &'s [u8]), FormatError> { + let (h, src) = open_chunk(src, limit.min(size.unwrap_or(limit)))?; + if size.is_some_and(|s| s != h.nbytes) { + return Err(err("chunk size does not match the frame's chunk size")); + } + Ok((h, src)) + } + + /// Decode chunk `n`, refusing more than `limit` bytes. + fn chunk(&self, n: usize, limit: usize) -> Result, FormatError> { + let (size, at) = self.locate(n)?; + let src = match at { + Located::Data(src) => src, + Located::Zeros | Located::Fill(_) => { + let size = size.unwrap_or(0); + if size > limit { + return Err(err("decoded size exceeds the limit")); + } + return match at { + Located::Fill(value) if !size.is_multiple_of(value.len()) => { + Err(err("NaN chunk is not whole values")) + } + Located::Fill(value) => Ok(value.iter().copied().cycle().take(size).collect()), + _ => Ok(vec![0u8; size]), + }; + } + }; + let (h, src) = self.open(size, src, limit)?; + let mut out = vec![0u8; h.nbytes]; + decode_blocks(&h, src, &mut |_, _| true, &mut |at, block| { + out[at..at + block.len()].copy_from_slice(block); + })?; + Ok(out) + } + + /// Decode chunk `n` (exactly `size` bytes) block by block, as + /// [`decode_blocks`], into a destination the caller has zeroed. Its + /// blocks must be whole multiples of `unit` bytes and at most + /// `max_block`; special chunks are handed over `unit` bytes at a time. + fn chunk_blocks( + &self, + n: usize, + size: usize, + unit: usize, + max_block: usize, + want: &mut dyn FnMut(usize, usize) -> bool, + sink: &mut dyn FnMut(usize, &[u8]), + ) -> Result<(), FormatError> { + let (frame_size, at) = self.locate(n)?; + if frame_size.is_some_and(|s| s != size) { + return Err(err("chunk size does not match the b2nd shape")); + } + match at { + Located::Zeros => Ok(()), + Located::Fill(value) => { + if !unit.is_multiple_of(value.len()) || !size.is_multiple_of(unit) { + return Err(err("NaN chunk is not whole values")); + } + let block: Vec = value.iter().copied().cycle().take(unit).collect(); + for k in 0..size / unit { + if want(k * unit, unit) { + sink(k * unit, &block); + } + } + Ok(()) + } + Located::Data(src) => { + let (h, src) = self.open(Some(size), src, size)?; + if h.nbytes > 0 && (!h.blocksize.is_multiple_of(unit) || h.blocksize > max_block) { + return Err(err("chunk blocks do not match the b2nd blocks")); + } + decode_blocks(&h, src, want, sink) + } + } + } + + /// The content of metalayer `name`, if the frame has it. + fn metalayer(&self, name: &[u8]) -> Result, FormatError> { + let h = &self.buf[..self.header_len]; + // [FRAME_IDX_SIZE] is the u16 index size, then a map16. + let mut p = FRAME_IDX_SIZE + 2; + if h.get(p) != Some(&0xde) { + return Err(err("bad metalayer index")); + } + let count = u16::from_be_bytes(be(h, p + 1)?) as usize; + p += 3; + if count > 16 { + return Err(err("too many metalayers")); + } + for _ in 0..count { + let tag = *h.get(p).ok_or_else(|| err("truncated metalayer index"))?; + if tag & 0xe0 != 0xa0 { + return Err(err("bad metalayer name")); + } + let len = (tag & 0x1f) as usize; + let key = h + .get(p + 1..p + 1 + len) + .ok_or_else(|| err("truncated metalayer index"))?; + p += 1 + len; + if h.get(p) != Some(&0xd2) { + return Err(err("bad metalayer offset")); + } + let off = i32::from_be_bytes(be(h, p + 1)?); + p += 5; + if key != name { + continue; + } + if off < 0 || off as usize >= h.len() { + return Err(err("metalayer offset out of range")); + } + let off = off as usize; + if h.get(off) != Some(&0xc6) { + return Err(err("bad metalayer content")); + } + let clen = u32::from_be_bytes(be(h, off + 1)?) as usize; + return h + .get(off + 5..) + .and_then(|c| c.get(..clen)) + .map(Some) + .ok_or_else(|| err("metalayer runs past the header")); + } + Ok(None) + } +} + +/// Where a frame's chunk is. +enum Located<'a> { + /// All zeros (or uninitialised, read as zeros). + Zeros, + /// One value repeated (NaN). + Fill(&'static [u8]), + /// A Blosc2 chunk, running to the end of the data section. + Data(&'a [u8]), +} + +/// The B2ND (or Caterva) metalayer: shape, chunk shape, block shape. +#[derive(Debug, PartialEq, Eq)] +struct NdMeta { + shape: Vec, + chunkshape: Vec, + blockshape: Vec, +} + +/// A msgpack array header: fixarray or array16. +fn array_len(m: &[u8], p: &mut usize) -> Result { + let t = *m.get(*p).ok_or_else(|| err("truncated b2nd metalayer"))?; + if t & 0xf0 == 0x90 { + *p += 1; + Ok((t & 0x0f) as usize) + } else if t == 0xdc { + let n = u16::from_be_bytes(be(m, *p + 1)?) as usize; + *p += 3; + Ok(n) + } else { + Err(err("bad b2nd metalayer")) + } +} + +fn parse_nd(m: &[u8]) -> Result { + let bad = || err("bad b2nd metalayer"); + let mut p = 0; + if array_len(m, &mut p)? < 5 { + return Err(bad()); + } + // Version, then the rank: positive fixints. + let version = *m.get(p).ok_or_else(bad)?; + let ndim = *m.get(p + 1).ok_or_else(bad)? as usize; + if version > 0x7f || ndim == 0 || ndim > B2ND_MAX_DIM { + return Err(err(&format!("unsupported b2nd rank {ndim}"))); + } + p += 2; + let mut read = |wide: bool| -> Result, FormatError> { + if array_len(m, &mut p)? != ndim { + return Err(bad()); + } + (0..ndim) + .map(|_| { + let v = if wide { + if m.get(p) != Some(&0xd3) { + return Err(bad()); + } + let v = i64::from_be_bytes(be(m, p + 1)?); + p += 9; + v + } else { + if m.get(p) != Some(&0xd2) { + return Err(bad()); + } + let v = i32::from_be_bytes(be(m, p + 1)?) as i64; + p += 5; + v + }; + usize::try_from(v).map_err(|_| bad()) + }) + .collect() + }; + let shape = read(true)?; + let chunkshape = read(false)?; + let blockshape = read(false)?; + Ok(NdMeta { + shape, + chunkshape, + blockshape, + }) +} + +/// Decode a frame as the HDF5 filter does: a B2ND array into C order, or +/// else the frame's first chunk. `cd_shape` is the chunk shape recorded in +/// the filter's `cd_values` (if any), which the array's shape must match. +fn decode_frame( + input: &[u8], + limit: usize, + cd_shape: Option<&[usize]>, +) -> Result, FormatError> { + let frame = parse_frame(input, limit)?; + let meta = match frame.metalayer(b"b2nd")? { + Some(m) => Some(m), + None => frame.metalayer(b"caterva")?, + }; + let Some(meta) = meta else { + return frame.chunk(0, limit); + }; + let nd = parse_nd(meta)?; + if let Some(cd) = cd_shape + && cd != nd.shape.as_slice() + { + return Err(err(&format!( + "array shape {:?} is not the chunk shape {cd:?}", + nd.shape + ))); + } + reassemble(&frame, &nd, limit) +} + +/// Gather a B2ND array's blocks into one C-order buffer. +/// +/// Each Blosc2 chunk holds one B2ND chunk padded to whole blocks, which can +/// be many times the array (a crafted frame's shapes are the file's word). +/// So the chunks are never decoded whole: each block is placed in the +/// output as it is decoded, blocks that are all padding are skipped, and +/// no more than a few blocks (each within the array) are held at once. +fn reassemble(frame: &Frame<'_>, nd: &NdMeta, limit: usize) -> Result, FormatError> { + let ts = frame.typesize; + let ndim = nd.shape.len(); + let too_big = || err("array exceeds the chunk size"); + let mut total = ts; + let mut ext = vec![0usize; ndim]; + let mut grid = vec![0usize; ndim]; + let mut ext_bytes = ts; + for i in 0..ndim { + let (s, c, b) = (nd.shape[i], nd.chunkshape[i], nd.blockshape[i]); + if c == 0 || b == 0 || b > c { + return Err(err("bad b2nd chunk or block shape")); + } + total = total.checked_mul(s).ok_or_else(too_big)?; + ext[i] = c.div_ceil(b) * b; + ext_bytes = ext_bytes.checked_mul(ext[i]).ok_or_else(too_big)?; + grid[i] = s.div_ceil(c); + } + if total > limit { + return Err(too_big()); + } + let mut out = vec![0u8; total]; + if total == 0 { + return Ok(out); + } + // hdf5-blosc2's chunk is the array. A chunk larger than the array + // would be padding decoded for nothing; with chunks and blocks within + // the array, a block is never larger than the output, and a chunk + // pads each dimension by less than one block. + if (0..ndim).any(|i| nd.chunkshape[i] > nd.shape[i]) { + return Err(err("b2nd chunk is larger than the array")); + } + // A Blosc2 chunk of `ext_bytes`, exactly the padded chunk. + if ext_bytes > i32::MAX as usize { + return Err(too_big()); + } + let nchunks: usize = grid.iter().product(); + if nchunks != frame.nchunks { + return Err(err("chunk count does not match the b2nd shape")); + } + let blocks_in_chunk: Vec = (0..ndim).map(|i| ext[i] / nd.blockshape[i]).collect(); + let block_items: usize = nd.blockshape.iter().product(); + let block_bytes = block_items * ts; + // Strides (in elements) of the output array and of a block. + let mut out_stride = vec![1usize; ndim]; + let mut blk_stride = vec![1usize; ndim]; + for i in (0..ndim.saturating_sub(1)).rev() { + out_stride[i] = out_stride[i + 1] * nd.shape[i + 1]; + blk_stride[i] = blk_stride[i + 1] * nd.blockshape[i + 1]; + } + let geometry = BlockGeometry { + nd, + blocks_in_chunk: &blocks_in_chunk, + }; + let mut cidx = vec![0usize; ndim]; + let mut w = BlockPlace::new(ndim); + let mut p = BlockPlace::new(ndim); + let mut pos = vec![0usize; ndim]; + for n in 0..nchunks { + unravel(n, &grid, &mut cidx); + let blocks = |at: usize, len: usize| at / block_bytes..(at + len).div_ceil(block_bytes); + let mut want = + |at: usize, len: usize| blocks(at, len).any(|b| geometry.place(b, &cidx, &mut w)); + let mut sink = |at: usize, data: &[u8]| { + for b in blocks(at, data.len()) { + if !geometry.place(b, &cidx, &mut p) { + continue; + } + let block = &data[b * block_bytes - at..][..block_bytes]; + // Copy row by row along the last dimension. + let row = p.valid[ndim - 1] * ts; + let rows: usize = p.valid[..ndim - 1].iter().product(); + for r in 0..rows { + unravel(r, &p.valid[..ndim - 1], &mut pos[..ndim - 1]); + let mut src = 0; + let mut dst = p.gstart[ndim - 1]; + for i in 0..ndim - 1 { + src += pos[i] * blk_stride[i]; + dst += (p.gstart[i] + pos[i]) * out_stride[i]; + } + out[dst * ts..dst * ts + row].copy_from_slice(&block[src * ts..src * ts + row]); + } + } + }; + frame.chunk_blocks(n, ext_bytes, block_bytes, limit, &mut want, &mut sink)?; + } + Ok(out) +} + +/// The block layout of a B2ND array's chunks. +struct BlockGeometry<'a> { + nd: &'a NdMeta, + blocks_in_chunk: &'a [usize], +} + +/// Where one block of a chunk lands: its first element in the array and +/// how much of it (per dimension) lies within the chunk and the array. +struct BlockPlace { + bidx: Vec, + gstart: Vec, + valid: Vec, +} + +impl BlockPlace { + fn new(ndim: usize) -> BlockPlace { + BlockPlace { + bidx: vec![0; ndim], + gstart: vec![0; ndim], + valid: vec![0; ndim], + } + } +} + +impl BlockGeometry<'_> { + /// Place block `b` of the chunk at grid index `cidx`; false if it is + /// all padding. + fn place(&self, b: usize, cidx: &[usize], p: &mut BlockPlace) -> bool { + let nd = self.nd; + unravel(b, self.blocks_in_chunk, &mut p.bidx); + let mut empty = false; + for (i, &c) in cidx.iter().enumerate() { + let in_chunk = p.bidx[i] * nd.blockshape[i]; + p.gstart[i] = c * nd.chunkshape[i] + in_chunk; + let lim_chunk = nd.chunkshape[i].saturating_sub(in_chunk); + let lim_shape = nd.shape[i].saturating_sub(p.gstart[i]); + p.valid[i] = nd.blockshape[i].min(lim_chunk).min(lim_shape); + empty |= p.valid[i] == 0; + } + !empty + } +} + +/// C-order multi-index of `n` in a grid of `dims`. +fn unravel(mut n: usize, dims: &[usize], idx: &mut [usize]) { + for i in (0..dims.len()).rev() { + idx[i] = n % dims[i]; + n /= dims[i]; + } +} + +/// Decompress one HDF5 chunk written by the Blosc2 filter (a Blosc2 frame) +/// into its data, in C order; at most `limit` bytes. +pub fn blosc2_decompress(input: &[u8], limit: usize) -> Result, FormatError> { + decode_frame(input, limit, None) +} + +/// The filter's decoder. +pub(crate) fn blosc2_decode(input: &[u8], ctx: &FilterContext<'_>) -> Result, FormatError> { + let cd = ctx.client_data(); + // cd_values[7] is the chunk rank for B2ND (> 1), and the chunk + // dimensions follow. + let cd_shape: Option> = if cd.len() >= 8 { + let rank = cd[7] as usize; + if rank < 2 || cd.len() < 8 + rank { + return Err(err("bad chunk rank in the filter parameters")); + } + Some(cd[8..8 + rank].iter().map(|&d| d as usize).collect()) + } else { + None + }; + let out = decode_frame(input, ctx.output_limit(), cd_shape.as_deref())?; + if out.is_empty() && ctx.max_output != 0 { + return Err(err("empty frame for a non-empty chunk")); + } + Ok(out) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::filter_pipeline::{FILTER_BLOSC2, FilterDescription}; + use std::path::PathBuf; + + fn fixtures() -> PathBuf { + PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("tests/fixtures/blosc2") + } + + /// Every fixture frame (see `tests/fixtures/blosc2/generate.py`), + /// sorted by name: (name, frame, expected output or error word). + #[allow(clippy::type_complexity)] + fn cases() -> Vec<(String, Vec, Result, String>)> { + let mut v = Vec::new(); + for e in std::fs::read_dir(fixtures()).unwrap() { + let p = e.unwrap().path(); + if p.extension().is_none_or(|x| x != "b2f") { + continue; + } + let name = p.file_stem().unwrap().to_string_lossy().into_owned(); + let frame = std::fs::read(&p).unwrap(); + let want = match std::fs::read(p.with_extension("out")) { + Ok(out) => Ok(out), + Err(_) => Err(std::fs::read_to_string(p.with_extension("err")).unwrap()), + }; + v.push((name, frame, want)); + } + v.sort_by(|a, b| a.0.cmp(&b.0)); + assert!(v.len() >= 20, "fixtures missing"); + v + } + + #[test] + fn fixture_frames_decode_exactly() { + for (name, frame, want) in cases() { + let got = blosc2_decompress(&frame, 1 << 20); + match want { + Ok(want) => { + let got = got.unwrap_or_else(|e| panic!("{name}: {e}")); + assert!(got == want, "{name}: decoded data differs"); + } + Err(word) => { + let e = got.expect_err(&name).to_string(); + assert!(e.contains(word.trim()), "{name}: {e}"); + } + } + } + } + + /// The output limit is enforced for every kind of frame, including the + /// padded chunks of B2ND arrays and special chunks. + #[test] + fn output_limit_is_enforced() { + for (name, frame, want) in cases() { + let Ok(want) = want else { continue }; + assert!( + blosc2_decompress(&frame, want.len() - 1).is_err(), + "{name}: decoded past the limit" + ); + assert_eq!( + blosc2_decompress(&frame, want.len()).unwrap(), + want, + "{name}" + ); + } + } + + fn desc(cd: Vec) -> FilterDescription { + FilterDescription { + filter_id: FILTER_BLOSC2, + name: None, + flags: 0, + client_data: cd, + } + } + + /// As the HDF5 filter: the B2ND array must have the chunk shape the + /// filter parameters record. + #[test] + fn b2nd_shape_must_match_the_filter_parameters() { + let frame = std::fs::read(fixtures().join("b2nd_2d.b2f")).unwrap(); + let want = std::fs::read(fixtures().join("b2nd_2d.out")).unwrap(); + let decode = |cd: Vec| { + let f = desc(cd); + blosc2_decode( + &frame, + &FilterContext { + filter: &f, + element_size: 4, + max_output: want.len(), + }, + ) + }; + assert_eq!(decode(vec![1, 0, 4, 0, 5, 1, 1, 2, 37, 29]).unwrap(), want); + assert_eq!(decode(vec![1, 0, 4, 0, 5, 1, 1]).unwrap(), want); + assert!(decode(vec![1, 0, 4, 0, 5, 1, 1, 2, 29, 37]).is_err()); + assert!(decode(vec![1, 0, 4, 0, 5, 1, 1, 3, 37, 29, 1]).is_err()); + assert!(decode(vec![1, 0, 4, 0, 5, 1, 1, 2, 37]).is_err()); + } + + /// A 32-byte extended header. + fn ext_header(flags: u8, ts: u8, nbytes: i32, blocksize: i32, cbytes: i32) -> Vec { + let mut h = vec![5u8, 1, flags | FLAG_SHUFFLE | FLAG_BITSHUFFLE, ts]; + for v in [nbytes, blocksize, cbytes] { + h.extend_from_slice(&v.to_le_bytes()); + } + h.resize(EXT_HEADER, 0); + h + } + + #[test] + fn special_chunks() { + // Zeros, uninitialised (read as zeros), NaN, one repeated value. + let mut z = ext_header(0, 4, 64, 64, 32); + z[31] = SPECIAL_ZERO << 4; + assert_eq!(blosc2_decompress_chunk(&z, 64).unwrap(), vec![0; 64]); + z[31] = SPECIAL_UNINIT << 4; + assert_eq!(blosc2_decompress_chunk(&z, 64).unwrap(), vec![0; 64]); + z[31] = SPECIAL_NAN << 4; + let nan = blosc2_decompress_chunk(&z, 64).unwrap(); + assert!( + nan.chunks(4) + .all(|c| f32::from_le_bytes(c.try_into().unwrap()).is_nan()) + ); + z[3] = 2; + assert!(blosc2_decompress_chunk(&z, 64).is_err(), "NaN of i16"); + let mut v = ext_header(0, 2, 12, 12, 35); + v[31] = SPECIAL_VALUE << 4; + v.extend_from_slice(&[1, 2, 3]); + assert_eq!( + blosc2_decompress_chunk(&v, 12).unwrap(), + [1, 2, 3].repeat(4) + ); + // The value must divide the chunk (and its blocks). + v[4] = 13; + assert!(blosc2_decompress_chunk(&v, 13).is_err()); + v[4] = 12; + v[8] = 4; + assert!(blosc2_decompress_chunk(&v, 12).is_err()); + // Special types 5 to 7 are reserved. + let mut r = ext_header(0, 4, 64, 64, 32); + r[31] = 5 << 4; + assert!(blosc2_decompress_chunk(&r, 64).is_err()); + } + + /// A regular chunk: one block of two streams (split, type size 2): a + /// run of 0x07 and a stream of zeros, then the byte shuffle undone. + #[test] + fn stream_runs_and_zero_streams() { + let mut c = ext_header(0, 2, 8, 8, 0); + c[16 + 5] = FILTER_SHUFFLE; + c.extend_from_slice(&36i32.to_le_bytes()); + c.extend_from_slice(&(-7i32).to_le_bytes()); + c.push(1); + c.extend_from_slice(&0i32.to_le_bytes()); + let n = c.len() as i32; + c[12..16].copy_from_slice(&n.to_le_bytes()); + assert_eq!( + blosc2_decompress_chunk(&c, 8).unwrap(), + [7, 0, 7, 0, 7, 0, 7, 0] + ); + // A token without the run bit is reserved. + let mut bad = c.clone(); + bad[40] = 2; + assert!(blosc2_decompress_chunk(&bad, 8).is_err()); + // Unsupported header features are errors, not data. + for (at, bit) in [(31, B2_USEDICT), (31, B2_LAZY), (30, VL_BLOCKS)] { + let mut x = c.clone(); + x[at] |= bit; + assert!( + blosc2_decompress_chunk(&x, 8).is_err(), + "byte {at} bit {bit}" + ); + } + let mut udf = c.clone(); + udf[16] = 35; + assert!(blosc2_decompress_chunk(&udf, 8).is_err()); + let mut udc = c.clone(); + udc[2] |= 6 << 5; + assert!(blosc2_decompress_chunk(&udc, 8).is_err()); + assert!(blosc2_decompress_chunk(&c, 7).is_err(), "limit"); + } + + /// Random and mutated frames: errors are fine, panics are not, and no + /// output past the limit. Peak allocation is fuzzed separately, with a + /// counting allocator, in `tests/blosc2_alloc_bounds.rs`. + #[test] + fn fuzzed_frames_never_panic() { + let limit = 20_000; + let seeds: Vec> = cases() + .into_iter() + .filter(|(_, f, w)| f.len() < 9000 && w.as_ref().is_ok_and(|w| w.len() <= limit)) + .map(|(_, f, _)| f) + .collect(); + assert!(seeds.len() >= 15); + crate::test_fuzz::fuzz_decoder(0xb2, &seeds, 20_000, limit, |s| { + blosc2_decompress(s, limit) + }); + } + + /// Blosc2 chunks on their own, mutated. + #[test] + fn fuzzed_chunks_never_panic() { + let mut seeds = Vec::new(); + for (_, f, w) in cases() { + if w.is_err() { + continue; + } + let frame = parse_frame(&f, 1 << 20).unwrap(); + let raw: [u8; 8] = frame.offsets[..8].try_into().unwrap(); + let off = i64::from_le_bytes(raw); + if off >= 0 { + let start = frame.header_len + off as usize; + let end = frame.header_len + frame.cbytes; + let h = read_header(&f[start..end]).unwrap(); + seeds.push(f[start..start + h.cbytes].to_vec()); + } + } + assert!(seeds.len() >= 10); + crate::test_fuzz::fuzz_decoder(0xb3, &seeds, 20_000, 1 << 16, |s| { + blosc2_decompress_chunk(s, 1 << 16) + }); + } +} diff --git a/crates/clawhdf5-format/src/lib.rs b/crates/clawhdf5-format/src/lib.rs index 92954bc..d538665 100644 --- a/crates/clawhdf5-format/src/lib.rs +++ b/crates/clawhdf5-format/src/lib.rs @@ -87,6 +87,8 @@ pub mod filters; mod filters_bitshuffle; #[cfg(feature = "blosc")] pub mod filters_blosc; +#[cfg(feature = "blosc2")] +pub mod filters_blosc2; #[cfg(feature = "bzip2")] mod filters_bzip2; #[cfg(feature = "lzf")] diff --git a/crates/clawhdf5-format/tests/blosc2_alloc_bounds.rs b/crates/clawhdf5-format/tests/blosc2_alloc_bounds.rs new file mode 100644 index 0000000..1667186 --- /dev/null +++ b/crates/clawhdf5-format/tests/blosc2_alloc_bounds.rs @@ -0,0 +1,641 @@ +//! Crafted Blosc2 frames and chunks cannot make the decoder allocate out of +//! proportion to the HDF5 chunk it decodes. +//! +//! A frame's header, its offsets chunk and its chunk headers all declare +//! sizes, and the decoder used to allocate what they declared: a 173-byte +//! frame whose offsets chunk claimed 2 GiB was decoded in full before any +//! check failed. Every allocation is now bounded by the output limit (the +//! HDF5 chunk's size) and the input's length. +//! +//! Peak heap use is measured with a counting global allocator; the tests +//! share it, so each holds `SERIAL` for its whole run. +#![cfg(feature = "blosc2")] + +use std::alloc::{GlobalAlloc, Layout, System}; +use std::sync::Mutex; +use std::sync::atomic::{AtomicUsize, Ordering}; + +use clawhdf5_format::filters_blosc2::{blosc2_decompress, blosc2_decompress_chunk}; + +struct Counting; + +static CURRENT: AtomicUsize = AtomicUsize::new(0); +static PEAK: AtomicUsize = AtomicUsize::new(0); +static SERIAL: Mutex<()> = Mutex::new(()); + +unsafe impl GlobalAlloc for Counting { + unsafe fn alloc(&self, layout: Layout) -> *mut u8 { + let p = unsafe { System.alloc(layout) }; + if !p.is_null() { + let now = CURRENT.fetch_add(layout.size(), Ordering::Relaxed) + layout.size(); + PEAK.fetch_max(now, Ordering::Relaxed); + } + p + } + + unsafe fn alloc_zeroed(&self, layout: Layout) -> *mut u8 { + let p = unsafe { System.alloc_zeroed(layout) }; + if !p.is_null() { + let now = CURRENT.fetch_add(layout.size(), Ordering::Relaxed) + layout.size(); + PEAK.fetch_max(now, Ordering::Relaxed); + } + p + } + + unsafe fn dealloc(&self, ptr: *mut u8, layout: Layout) { + unsafe { System.dealloc(ptr, layout) }; + CURRENT.fetch_sub(layout.size(), Ordering::Relaxed); + } +} + +#[global_allocator] +static ALLOC: Counting = Counting; + +/// Bytes allocated at the peak of `f`, above what was live when it started. +fn peak_during(f: impl FnOnce() -> T) -> (T, usize) { + let base = CURRENT.load(Ordering::Relaxed); + PEAK.store(base, Ordering::Relaxed); + let out = f(); + (out, PEAK.load(Ordering::Relaxed).saturating_sub(base)) +} + +/// What decoding one HDF5 chunk of `limit` bytes from `input` may hold at +/// once: the output, a few blocks of scratch (each no larger than the +/// output), the offsets table, and the Zstandard decoder's state, which has +/// a fixed ceiling: a window of at most 128 KiB (or twice the stream) and a +/// block's table of sequences (up to 98,303 of 12 bytes, 1.2 MB). +fn bound(limit: usize, input: &[u8]) -> usize { + 6 * limit + 2 * input.len() + (2 << 20) +} + +fn lock() -> std::sync::MutexGuard<'static, ()> { + SERIAL.lock().unwrap_or_else(|e| e.into_inner()) +} + +/// A 32-byte (extended) Blosc2 chunk header. +fn chunk_header(ts: u8, nbytes: i32, blocksize: i32, cbytes: i32, special: u8) -> Vec { + let mut c = vec![5u8, 1, 0x05, ts]; + for v in [nbytes, blocksize, cbytes] { + c.extend_from_slice(&v.to_le_bytes()); + } + c.resize(32, 0); + c[31] = special << 4; + c +} + +/// A chunk of `nbytes` bytes that repeats one value (special type 3). +fn repeated(value: &[u8], nbytes: i32, blocksize: i32) -> Vec { + let mut c = chunk_header( + value.len() as u8, + nbytes, + blocksize, + 32 + value.len() as i32, + 3, + ); + c.extend_from_slice(value); + c +} + +/// A frame offset recording a special chunk of `kind` (1 zeros, 2 NaN). +fn special_offset(kind: u8) -> [u8; 8] { + (((0x80 | kind) as i64) << 56).to_le_bytes() +} + +/// A B2ND metalayer. +fn nd_meta(shape: &[i64], chunks: &[i32], blocks: &[i32]) -> Vec { + let n = shape.len() as u8; + let mut m = vec![0x95, 0, n, 0x90 | n]; + for s in shape { + m.push(0xd3); + m.extend_from_slice(&s.to_be_bytes()); + } + for dims in [chunks, blocks] { + m.push(0x90 | n); + for d in dims { + m.push(0xd2); + m.extend_from_slice(&d.to_be_bytes()); + } + } + m +} + +/// A contiguous frame: header (with a `b2nd` metalayer if given), the data +/// chunks, then the offsets chunk. +fn frame( + meta: Option<&[u8]>, + nbytes: i64, + typesize: i32, + chunksize: i32, + data: &[u8], + offsets: &[u8], +) -> Vec { + let mut h = vec![0u8; 91]; + h[0] = 0x9e; + h[1] = 0xa8; + h[2..10].copy_from_slice(b"b2frame\0"); + h[25] = 2; + match meta { + Some(m) => { + h.extend_from_slice(&[0xde, 0, 1, 0xa4]); + h.extend_from_slice(b"b2nd"); + let at = h.len() as i32 + 5; + h.push(0xd2); + h.extend_from_slice(&at.to_be_bytes()); + h.push(0xc6); + h.extend_from_slice(&(m.len() as u32).to_be_bytes()); + h.extend_from_slice(m); + } + None => h.extend_from_slice(&[0xde, 0, 0]), + } + let header_len = h.len() as i32; + h[11..15].copy_from_slice(&header_len.to_be_bytes()); + h[30..38].copy_from_slice(&nbytes.to_be_bytes()); + h[39..47].copy_from_slice(&(data.len() as i64).to_be_bytes()); + h[48..52].copy_from_slice(&typesize.to_be_bytes()); + h[58..62].copy_from_slice(&chunksize.to_be_bytes()); + h.extend_from_slice(data); + h.extend_from_slice(offsets); + let len = h.len() as u64; + h[16..24].copy_from_slice(&len.to_be_bytes()); + h +} + +/// The frame header's own sizes must not size the offsets chunk: a frame +/// declaring 32 Mi chunks of 4 bytes, whose offsets chunk (40 bytes) says +/// "one repeated offset, 256 MiB of them", made the decoder build all +/// 256 MiB of offsets for a 1 MiB HDF5 chunk and then return 4 bytes. +#[test] +fn offsets_chunk_is_bounded_by_the_output_limit() { + let _g = lock(); + let limit = 1 << 20; + let offsets_len: i32 = 256 << 20; + let nchunks = offsets_len as i64 / 8; + let offsets = repeated(&special_offset(1), offsets_len, 64 << 20); + let f = frame(None, nchunks * 4, 4, 4, &[], &offsets); + let (r, peak) = peak_during(|| blosc2_decompress(&f, limit)); + assert!(r.is_err(), "decoded {:?} bytes", r.map(|v| v.len())); + assert!( + peak <= bound(limit, &f), + "peak {peak} bytes for a {}-byte frame", + f.len() + ); + // The same frame with a variable chunk size (0): the offsets chunk + // alone says how many chunks there are. + let f = frame(None, nchunks * 4, 4, 0, &[], &offsets); + let (r, peak) = peak_during(|| blosc2_decompress(&f, limit)); + assert!(r.is_err()); + assert!(peak <= bound(limit, &f), "chunksize 0: peak {peak} bytes"); +} + +/// A legitimate frame of this shape (one chunk, its offset special) still +/// decodes. +#[test] +fn small_frames_still_decode() { + let _g = lock(); + let offsets = repeated(&special_offset(1), 8, 8); + let f = frame(None, 64, 4, 64, &[], &offsets); + assert_eq!(blosc2_decompress(&f, 64).unwrap(), vec![0; 64]); + let _ = blosc2_decompress_chunk; +} + +/// A chunk that decodes to nothing kept its declared block size (up to +/// 512 MiB) and allocated two scratch blocks of it: about 1 GiB for a +/// 20-byte chunk. +#[test] +fn empty_chunk_does_not_allocate_its_block_size() { + let _g = lock(); + let mut c = vec![5u8, 1, 0x01, 1]; + for v in [0i32, 0x1FFF_F000, 20] { + c.extend_from_slice(&v.to_le_bytes()); + } + c.resize(20, 0); + let (r, peak) = peak_during(|| blosc2_decompress_chunk(&c, 1 << 20)); + assert_eq!(r.map(|v| v.len()).unwrap_or(0), 0); + assert!( + peak <= bound(0, &c), + "peak {peak} bytes for a 20-byte chunk" + ); + // Inside a frame for a non-empty HDF5 chunk it is an error, not data. + let offsets = repeated(&0i64.to_le_bytes(), 8, 8); + let f = frame(None, 64, 4, 64, &c, &offsets); + let (r, peak) = peak_during(|| blosc2_decompress(&f, 64)); + assert!(r.is_err(), "decoded {:?}", r.map(|v| v.len())); + assert!(peak <= bound(64, &f), "in a frame: peak {peak} bytes"); +} + +/// B2ND chunks were decoded whole, padding included, with up to 16x the +/// HDF5 chunk size as their limit. Blocks are now placed as they are +/// decoded, so the padding is never held. +/// +/// Ten dimensions: nine of 3 split into blocks of 2 (padded to 4) and one +/// of 4, so each chunk is 13x the array. One chunk, stored three ways: as a +/// NaN chunk in the frame's offsets, as a repeated-value chunk, and as a +/// chunk of stored (uncompressed) blocks. +#[test] +fn b2nd_padding_is_never_held() { + let _g = lock(); + let ts = 4usize; + let mut shape = vec![3i64; 9]; + shape.push(4); + let chunks: Vec = shape.iter().map(|&s| s as i32).collect(); + let mut blocks = vec![2i32; 9]; + blocks.push(4); + let meta = nd_meta(&shape, &chunks, &blocks); + let items: usize = shape.iter().product::() as usize; + let limit = items * ts; + let block_bytes = ts * blocks.iter().product::() as usize; + let ext_bytes = ts * 4usize.pow(9) * 4; + assert!(ext_bytes > 13 * limit); + let offsets = |off: [u8; 8]| repeated(&off, 8, 8); + + let value = 1.5f32.to_le_bytes(); + let stored = { + // Every block stored raw: block k holds the value k. + let mut c = chunk_header(4, ext_bytes as i32, block_bytes as i32, 0, 0); + c[2] = 0x02 | 0x10; // memcpyed, not split + c.truncate(16); + for k in 0..ext_bytes / block_bytes { + c.extend((k as f32).to_le_bytes().repeat(block_bytes / 4)); + } + let n = c.len() as i32; + c[12..16].copy_from_slice(&n.to_le_bytes()); + c + }; + let cases: Vec<(&str, Vec)> = vec![ + ( + "NaN offset", + frame( + Some(&meta), + ext_bytes as i64, + 4, + ext_bytes as i32, + &[], + &offsets(special_offset(2)), + ), + ), + ( + "repeated value", + frame( + Some(&meta), + ext_bytes as i64, + 4, + ext_bytes as i32, + &repeated(&value, ext_bytes as i32, block_bytes as i32), + &offsets(0i64.to_le_bytes()), + ), + ), + ( + "stored blocks", + frame( + Some(&meta), + ext_bytes as i64, + 4, + ext_bytes as i32, + &stored, + &offsets(0i64.to_le_bytes()), + ), + ), + ]; + for (name, f) in cases { + let (r, peak) = peak_during(|| blosc2_decompress(&f, limit)); + let out = r.unwrap_or_else(|e| panic!("{name}: {e}")); + assert_eq!(out.len(), limit, "{name}"); + match name { + "NaN offset" => assert!( + out.chunks(4) + .all(|v| f32::from_le_bytes(v.try_into().unwrap()).is_nan()) + ), + "repeated value" => assert!(out.chunks(4).all(|v| v == value)), + _ => { + // Element (i0..i9) lies in block (i0/2, .., i8/2), numbered + // in C order over a 2x..x2x1 grid of blocks. + let mut idx = [0usize; 10]; + for (e, v) in out.chunks(4).enumerate() { + let mut n = e; + for d in (0..10).rev() { + idx[d] = n % shape[d] as usize; + n /= shape[d] as usize; + } + let k = idx[..9].iter().fold(0, |k, &i| k * 2 + i / 2); + assert_eq!( + f32::from_le_bytes(v.try_into().unwrap()), + k as f32, + "{name} {e}" + ); + } + } + } + assert!( + peak <= bound(limit, &f), + "{name}: peak {peak} bytes for a {limit}-byte chunk ({}-byte frame)", + f.len() + ); + } +} + +/// A B2ND chunk larger than the array (here 16x, the old cap) is refused, +/// or at least never allocated. +#[test] +fn b2nd_chunk_larger_than_the_array_is_not_allocated() { + let _g = lock(); + let limit = 1 << 20; + let c = 16 * limit as i32; + let meta = nd_meta(&[limit as i64], &[c], &[c]); + let offsets = repeated(&special_offset(1), 8, 8); + let f = frame(Some(&meta), c as i64, 1, c, &[], &offsets); + let (r, peak) = peak_during(|| blosc2_decompress(&f, limit)); + assert!( + peak <= bound(limit, &f), + "peak {peak} bytes ({:?})", + r.map(|v| v.len()) + ); +} + +/// ruzstd reserves a frame's declared window (up to 100 MiB) before it +/// decodes a frame with a decoder it has used before: a Blosc2 chunk of +/// two 16-byte Zstandard streams, each declaring a 96 MiB window, +/// allocated 96 MiB. c-blosc2 compresses each block with its size known, +/// so its windows never exceed the block. +#[test] +fn zstd_window_is_bounded_by_the_output() { + let _g = lock(); + let mut z = 0xfd2f_b528u32.to_le_bytes().to_vec(); + // No single segment, no checksum; window 2^26 + 4/8 of it = 96 MiB. + z.extend_from_slice(&[0x00, (16 << 3) | 4]); + // One raw block, last, of 16 bytes. + let h = 1 | (16 << 3); + z.extend_from_slice(&[h as u8, (h >> 8) as u8, 0]); + z.extend_from_slice(&[7; 16]); + // Two blocks of 16 bytes, one stream each (not split), Zstandard + // (codec 4). + let chunk = |z: &[u8]| { + let mut c = vec![5u8, 1, 0x10 | (4 << 5), 1]; + for v in [32i32, 16, 0] { + c.extend_from_slice(&v.to_le_bytes()); + } + let first = 24 + 4 + z.len(); + c.extend_from_slice(&24i32.to_le_bytes()); + c.extend_from_slice(&(first as i32).to_le_bytes()); + for _ in 0..2 { + c.extend_from_slice(&(z.len() as i32).to_le_bytes()); + c.extend_from_slice(z); + } + let n = c.len() as i32; + c[12..16].copy_from_slice(&n.to_le_bytes()); + c + }; + let c = chunk(&z); + let (r, peak) = peak_during(|| blosc2_decompress_chunk(&c, 32)); + assert!(peak <= bound(32, &c), "peak {peak} bytes ({r:?})"); + assert!(r.is_err(), "{r:?}"); + // The same streams with a window they can use read. + z[5] = 0; + assert_eq!( + blosc2_decompress_chunk(&chunk(&z), 32).unwrap(), + vec![7; 32] + ); +} + +/// xorshift64*: deterministic, so a failure reproduces. +struct Rng(u64); + +impl Rng { + fn next(&mut self) -> u64 { + let mut x = self.0; + x ^= x >> 12; + x ^= x << 25; + x ^= x >> 27; + self.0 = x; + x.wrapping_mul(0x2545_F491_4F6C_DD1D) + } + + fn below(&mut self, n: usize) -> usize { + (self.next() % n.max(1) as u64) as usize + } + + /// A size that tends to the edges: small, a power of two, huge. + fn size(&mut self) -> i64 { + match self.below(6) { + 0 => self.below(64) as i64, + 1 => 1 << self.below(31), + 2 => i32::MAX as i64 - self.below(4096) as i64, + 3 => (1i64 << self.below(62)) + self.below(8) as i64, + 4 => MAX_BLOCK - self.below(3) as i64, + _ => self.next() as i32 as i64, + } + } +} + +const MAX_BLOCK: i64 = 536_866_816; + +/// One to four edits: bytes, or a size field written little-endian (chunk +/// headers) or big-endian (frame headers), most often at a header's size +/// fields. +fn mutate(rng: &mut Rng, seed: &[u8], data_at: usize) -> Vec { + let mut v = seed.to_vec(); + for _ in 0..1 + rng.below(4) { + let len = v.len(); + if len < 16 { + v.push(rng.next() as u8); + continue; + } + match rng.below(8) { + 0 => { + let i = rng.below(len); + v[i] ^= 1 << rng.below(8); + } + 1 => { + let i = rng.below(len); + v[i] = rng.next() as u8; + } + 2 => { + // Frame header: nbytes, cbytes (i64), typesize, chunksize. + let x = rng.size(); + match rng.below(4) { + 0 if len >= 38 => v[30..38].copy_from_slice(&x.to_be_bytes()), + 1 if len >= 47 => v[39..47].copy_from_slice(&x.to_be_bytes()), + 2 if len >= 52 => v[48..52].copy_from_slice(&(x as i32).to_be_bytes()), + _ if len >= 62 => v[58..62].copy_from_slice(&(x as i32).to_be_bytes()), + _ => {} + } + } + 3 | 4 => { + // A chunk header's nbytes, blocksize or cbytes: in the first + // data chunk, or anywhere (the offsets chunk comes last). + let at = if rng.below(2) == 0 && data_at + 16 <= len { + data_at + 4 * (1 + rng.below(3)) + } else { + rng.below(len - 3) + }; + let x = rng.size() as i32; + v[at..at + 4].copy_from_slice(&x.to_le_bytes()); + } + 5 => v.truncate(rng.below(len)), + 6 => { + let at = rng.below(len); + v[at] = [0x10, 0x20, 0x30, 0x40, 0x05, 0x07, 0x02][rng.below(7)]; + } + _ => { + let i = rng.below(len - 3); + let x = rng.size() as i32; + v[i..i + 4].copy_from_slice(&x.to_be_bytes()); + } + } + } + v +} + +/// Every fixture frame that decodes, with its decoded size. +fn seeds() -> Vec<(Vec, usize)> { + let dir = std::path::Path::new(env!("CARGO_MANIFEST_DIR")).join("tests/fixtures/blosc2"); + let mut v = Vec::new(); + for e in std::fs::read_dir(dir).unwrap() { + let p = e.unwrap().path(); + if p.extension().is_some_and(|x| x == "b2f") + && let Ok(out) = std::fs::read(p.with_extension("out")) + { + v.push((std::fs::read(&p).unwrap(), out.len())); + } + } + v.sort(); + assert!(v.len() >= 20, "fixtures missing"); + v +} + +fn header_len(frame: &[u8]) -> usize { + i32::from_be_bytes(frame[11..15].try_into().unwrap()) as usize +} + +/// Mutated fixture frames, decoded with their HDF5 chunk size as the +/// limit, and their first chunks on their own: whatever they declare, no +/// decode holds more than a small multiple of the output and the input. +#[test] +fn fuzzed_frames_and_chunks_stay_within_the_allocation_bound() { + let _g = lock(); + let seeds = seeds(); + let mut rng = Rng(0xb2a1); + let mut worst = (0.0f64, String::new()); + for i in 0..20_000 { + let (seed, limit) = &seeds[rng.below(seeds.len())]; + let f = mutate(&mut rng, seed, header_len(seed)); + let (r, peak) = peak_during(|| blosc2_decompress(&f, *limit)); + if let Ok(out) = &r { + assert!(out.len() <= *limit, "iteration {i}: output past the limit"); + } + assert!( + peak <= bound(*limit, &f), + "iteration {i}: peak {peak} bytes for a {limit}-byte chunk from {} bytes ({:?})", + f.len(), + r.map(|v| v.len()) + ); + let ratio = peak as f64 / bound(*limit, &f) as f64; + if ratio > worst.0 { + worst = ( + ratio, + format!( + "frame iteration {i}: peak {peak}, limit {limit}, input {}", + f.len() + ), + ); + } + } + for i in 0..20_000 { + let (seed, _) = &seeds[rng.below(seeds.len())]; + let at = header_len(seed); + let chunk = &seed[at..]; + let c = mutate(&mut rng, chunk, 0); + let limit = 1 << 16; + let (r, peak) = peak_during(|| blosc2_decompress_chunk(&c, limit)); + assert!( + peak <= bound(limit, &c), + "chunk iteration {i}: peak {peak} bytes from {} bytes ({:?})", + c.len(), + r.map(|v| v.len()) + ); + } + eprintln!("worst peak / bound: {:.2} ({})", worst.0, worst.1); +} + +/// Frames built from random header sizes, offsets chunks and B2ND shapes +/// (chunk and block shapes that pad, special and repeated-value chunks). +#[test] +fn random_frames_stay_within_the_allocation_bound() { + let _g = lock(); + let mut rng = Rng(0xb2a2); + for i in 0..5_000 { + let ts = [1usize, 2, 4, 8][rng.below(4)]; + let ndim = 1 + rng.below(8); + let mut shape = Vec::new(); + let mut chunks = Vec::new(); + let mut blocks = Vec::new(); + for _ in 0..ndim { + let s = 1 + rng.below(if ndim > 3 { 4 } else { 40 }); + let c = if rng.below(8) == 0 { + s * (1 + rng.below(4)) + } else { + 1 + rng.below(s) + }; + let b = 1 + rng.below(c); + shape.push(s as i64); + chunks.push(c as i32); + blocks.push(b as i32); + } + let items: usize = shape.iter().product::() as usize; + let limit = items * ts; + let meta = nd_meta(&shape, &chunks, &blocks); + let ext: usize = ts + * chunks + .iter() + .zip(&blocks) + .map(|(&c, &b)| (c as usize).div_ceil(b as usize) * b as usize) + .product::(); + let nchunks: usize = shape + .iter() + .zip(&chunks) + .map(|(&s, &c)| (s as usize).div_ceil(c as usize)) + .product(); + let block_bytes = ts * blocks.iter().product::() as usize; + let chunksize = if rng.below(4) == 0 { + rng.size() + } else { + ext as i64 + }; + let nbytes = if rng.below(4) == 0 { + rng.size() + } else { + (nchunks * ext) as i64 + }; + let off_n = if rng.below(4) == 0 { + rng.size() as i32 + } else { + 8 * nchunks as i32 + }; + let (data, off) = match rng.below(3) { + 0 => (Vec::new(), special_offset(1 + rng.below(2) as u8)), + _ => { + let bs = if rng.below(4) == 0 { + rng.size() as i32 + } else { + block_bytes as i32 + }; + let value: Vec = (0..ts).map(|_| rng.next() as u8).collect(); + let n = if rng.below(4) == 0 { + rng.size() as i32 + } else { + ext as i32 + }; + (repeated(&value, n, bs), 0i64.to_le_bytes()) + } + }; + let offsets = repeated(&off, off_n, off_n.clamp(1, 8)); + let meta = (rng.below(4) != 0).then_some(meta.as_slice()); + let f = frame(meta, nbytes, ts as i32, chunksize as i32, &data, &offsets); + let (r, peak) = peak_during(|| blosc2_decompress(&f, limit)); + assert!( + peak <= bound(limit, &f), + "iteration {i}: peak {peak} bytes for a {limit}-byte chunk ({:?}, shape {shape:?} \ + chunks {chunks:?} blocks {blocks:?})", + r.map(|v| v.len()) + ); + } +} diff --git a/crates/clawhdf5-format/tests/fixtures/blosc2/always_split.b2f b/crates/clawhdf5-format/tests/fixtures/blosc2/always_split.b2f new file mode 100644 index 0000000..db6322a Binary files /dev/null and b/crates/clawhdf5-format/tests/fixtures/blosc2/always_split.b2f differ diff --git a/crates/clawhdf5-format/tests/fixtures/blosc2/always_split.out b/crates/clawhdf5-format/tests/fixtures/blosc2/always_split.out new file mode 100644 index 0000000..a3faac5 Binary files /dev/null and b/crates/clawhdf5-format/tests/fixtures/blosc2/always_split.out differ diff --git a/crates/clawhdf5-format/tests/fixtures/blosc2/b2nd_2d.b2f b/crates/clawhdf5-format/tests/fixtures/blosc2/b2nd_2d.b2f new file mode 100644 index 0000000..a092e29 Binary files /dev/null and b/crates/clawhdf5-format/tests/fixtures/blosc2/b2nd_2d.b2f differ diff --git a/crates/clawhdf5-format/tests/fixtures/blosc2/b2nd_2d.out b/crates/clawhdf5-format/tests/fixtures/blosc2/b2nd_2d.out new file mode 100644 index 0000000..4e6ff85 Binary files /dev/null and b/crates/clawhdf5-format/tests/fixtures/blosc2/b2nd_2d.out differ diff --git a/crates/clawhdf5-format/tests/fixtures/blosc2/b2nd_3d.b2f b/crates/clawhdf5-format/tests/fixtures/blosc2/b2nd_3d.b2f new file mode 100644 index 0000000..edaf272 Binary files /dev/null and b/crates/clawhdf5-format/tests/fixtures/blosc2/b2nd_3d.b2f differ diff --git a/crates/clawhdf5-format/tests/fixtures/blosc2/b2nd_3d.out b/crates/clawhdf5-format/tests/fixtures/blosc2/b2nd_3d.out new file mode 100644 index 0000000..c29f5f0 Binary files /dev/null and b/crates/clawhdf5-format/tests/fixtures/blosc2/b2nd_3d.out differ diff --git a/crates/clawhdf5-format/tests/fixtures/blosc2/b2nd_4d.b2f b/crates/clawhdf5-format/tests/fixtures/blosc2/b2nd_4d.b2f new file mode 100644 index 0000000..6fa77de Binary files /dev/null and b/crates/clawhdf5-format/tests/fixtures/blosc2/b2nd_4d.b2f differ diff --git a/crates/clawhdf5-format/tests/fixtures/blosc2/b2nd_4d.out b/crates/clawhdf5-format/tests/fixtures/blosc2/b2nd_4d.out new file mode 100644 index 0000000..bcd534f Binary files /dev/null and b/crates/clawhdf5-format/tests/fixtures/blosc2/b2nd_4d.out differ diff --git a/crates/clawhdf5-format/tests/fixtures/blosc2/b2nd_zero_chunk.b2f b/crates/clawhdf5-format/tests/fixtures/blosc2/b2nd_zero_chunk.b2f new file mode 100644 index 0000000..029f33d Binary files /dev/null and b/crates/clawhdf5-format/tests/fixtures/blosc2/b2nd_zero_chunk.b2f differ diff --git a/crates/clawhdf5-format/tests/fixtures/blosc2/b2nd_zero_chunk.out b/crates/clawhdf5-format/tests/fixtures/blosc2/b2nd_zero_chunk.out new file mode 100644 index 0000000..fe7c16e Binary files /dev/null and b/crates/clawhdf5-format/tests/fixtures/blosc2/b2nd_zero_chunk.out differ diff --git a/crates/clawhdf5-format/tests/fixtures/blosc2/bitshuffle_odd_blocks.b2f b/crates/clawhdf5-format/tests/fixtures/blosc2/bitshuffle_odd_blocks.b2f new file mode 100644 index 0000000..6e71e42 Binary files /dev/null and b/crates/clawhdf5-format/tests/fixtures/blosc2/bitshuffle_odd_blocks.b2f differ diff --git a/crates/clawhdf5-format/tests/fixtures/blosc2/bitshuffle_odd_blocks.out b/crates/clawhdf5-format/tests/fixtures/blosc2/bitshuffle_odd_blocks.out new file mode 100644 index 0000000..83e486e Binary files /dev/null and b/crates/clawhdf5-format/tests/fixtures/blosc2/bitshuffle_odd_blocks.out differ diff --git a/crates/clawhdf5-format/tests/fixtures/blosc2/bytedelta.b2f b/crates/clawhdf5-format/tests/fixtures/blosc2/bytedelta.b2f new file mode 100644 index 0000000..2995a86 Binary files /dev/null and b/crates/clawhdf5-format/tests/fixtures/blosc2/bytedelta.b2f differ diff --git a/crates/clawhdf5-format/tests/fixtures/blosc2/bytedelta.err b/crates/clawhdf5-format/tests/fixtures/blosc2/bytedelta.err new file mode 100644 index 0000000..f69e717 --- /dev/null +++ b/crates/clawhdf5-format/tests/fixtures/blosc2/bytedelta.err @@ -0,0 +1 @@ +filter 35 \ No newline at end of file diff --git a/crates/clawhdf5-format/tests/fixtures/blosc2/delta_int16_blosclz.b2f b/crates/clawhdf5-format/tests/fixtures/blosc2/delta_int16_blosclz.b2f new file mode 100644 index 0000000..7face64 Binary files /dev/null and b/crates/clawhdf5-format/tests/fixtures/blosc2/delta_int16_blosclz.b2f differ diff --git a/crates/clawhdf5-format/tests/fixtures/blosc2/delta_int16_blosclz.out b/crates/clawhdf5-format/tests/fixtures/blosc2/delta_int16_blosclz.out new file mode 100644 index 0000000..e891807 Binary files /dev/null and b/crates/clawhdf5-format/tests/fixtures/blosc2/delta_int16_blosclz.out differ diff --git a/crates/clawhdf5-format/tests/fixtures/blosc2/delta_int32_lz4.b2f b/crates/clawhdf5-format/tests/fixtures/blosc2/delta_int32_lz4.b2f new file mode 100644 index 0000000..819f61e Binary files /dev/null and b/crates/clawhdf5-format/tests/fixtures/blosc2/delta_int32_lz4.b2f differ diff --git a/crates/clawhdf5-format/tests/fixtures/blosc2/delta_int32_lz4.out b/crates/clawhdf5-format/tests/fixtures/blosc2/delta_int32_lz4.out new file mode 100644 index 0000000..97e148c Binary files /dev/null and b/crates/clawhdf5-format/tests/fixtures/blosc2/delta_int32_lz4.out differ diff --git a/crates/clawhdf5-format/tests/fixtures/blosc2/delta_shuffle_v16.b2f b/crates/clawhdf5-format/tests/fixtures/blosc2/delta_shuffle_v16.b2f new file mode 100644 index 0000000..5f0cc42 Binary files /dev/null and b/crates/clawhdf5-format/tests/fixtures/blosc2/delta_shuffle_v16.b2f differ diff --git a/crates/clawhdf5-format/tests/fixtures/blosc2/delta_shuffle_v16.out b/crates/clawhdf5-format/tests/fixtures/blosc2/delta_shuffle_v16.out new file mode 100644 index 0000000..2073522 Binary files /dev/null and b/crates/clawhdf5-format/tests/fixtures/blosc2/delta_shuffle_v16.out differ diff --git a/crates/clawhdf5-format/tests/fixtures/blosc2/delta_uint64_blosclz.b2f b/crates/clawhdf5-format/tests/fixtures/blosc2/delta_uint64_blosclz.b2f new file mode 100644 index 0000000..f052059 Binary files /dev/null and b/crates/clawhdf5-format/tests/fixtures/blosc2/delta_uint64_blosclz.b2f differ diff --git a/crates/clawhdf5-format/tests/fixtures/blosc2/delta_uint64_blosclz.out b/crates/clawhdf5-format/tests/fixtures/blosc2/delta_uint64_blosclz.out new file mode 100644 index 0000000..2073522 Binary files /dev/null and b/crates/clawhdf5-format/tests/fixtures/blosc2/delta_uint64_blosclz.out differ diff --git a/crates/clawhdf5-format/tests/fixtures/blosc2/delta_uint8_lz4.b2f b/crates/clawhdf5-format/tests/fixtures/blosc2/delta_uint8_lz4.b2f new file mode 100644 index 0000000..8ecfdd8 Binary files /dev/null and b/crates/clawhdf5-format/tests/fixtures/blosc2/delta_uint8_lz4.b2f differ diff --git a/crates/clawhdf5-format/tests/fixtures/blosc2/delta_uint8_lz4.out b/crates/clawhdf5-format/tests/fixtures/blosc2/delta_uint8_lz4.out new file mode 100644 index 0000000..b70a6d9 Binary files /dev/null and b/crates/clawhdf5-format/tests/fixtures/blosc2/delta_uint8_lz4.out differ diff --git a/crates/clawhdf5-format/tests/fixtures/blosc2/delta_v3.b2f b/crates/clawhdf5-format/tests/fixtures/blosc2/delta_v3.b2f new file mode 100644 index 0000000..cda8773 Binary files /dev/null and b/crates/clawhdf5-format/tests/fixtures/blosc2/delta_v3.b2f differ diff --git a/crates/clawhdf5-format/tests/fixtures/blosc2/delta_v3.out b/crates/clawhdf5-format/tests/fixtures/blosc2/delta_v3.out new file mode 100644 index 0000000..bc826e3 Binary files /dev/null and b/crates/clawhdf5-format/tests/fixtures/blosc2/delta_v3.out differ diff --git a/crates/clawhdf5-format/tests/fixtures/blosc2/generate.py b/crates/clawhdf5-format/tests/fixtures/blosc2/generate.py new file mode 100644 index 0000000..f79c0f7 --- /dev/null +++ b/crates/clawhdf5-format/tests/fixtures/blosc2/generate.py @@ -0,0 +1,129 @@ +"""Generate the Blosc2 frames the `filters_blosc2` unit tests decode. + +Each case is `.b2f` (a Blosc2 contiguous frame, what the HDF5 Blosc2 +filter stores per chunk) and `.out` (what decoding it must give: the +first chunk of a plain frame, or the whole array in C order for a B2ND +frame), or `.err` (a frame clawhdf5 must refuse; the file holds a word +the error message must contain). + +These cover what files written by h5py + hdf5plugin never contain, but a +Blosc2 frame may: special chunks (repeated value, NaN, uninitialised), the +delta filter over many blocks and odd type sizes, bit shuffle of blocks that +are not a multiple of 8 elements, shuffle with a byte-group size, forced +stream splitting, multi-chunk B2ND arrays with padded edge chunks and a +chunk of zeros, and features clawhdf5 refuses (dictionaries, registered +filters). + +Written with python-blosc2 4.13.1 (c-blosc2 3.3.4) in a scratch venv +(`pip install blosc2`). Re-run only to regenerate: + + python generate.py +""" +import os +import sys + +import blosc2 +import numpy as np + +out = sys.argv[1] + + +def save(name, frame, expected): + with open(os.path.join(out, name + ".b2f"), "wb") as f: + f.write(frame) + with open(os.path.join(out, name + ".out"), "wb") as f: + f.write(expected) + + +def save_err(name, frame, word): + with open(os.path.join(out, name + ".b2f"), "wb") as f: + f.write(frame) + with open(os.path.join(out, name + ".err"), "w") as f: + f.write(word) + + +def plain(data, **cparams): + """A one-chunk super-chunk frame of `data`, as hdf5-blosc2 writes.""" + data = np.ascontiguousarray(data) + cp = blosc2.CParams(typesize=data.dtype.itemsize, **cparams) + sc = blosc2.SChunk(chunksize=data.nbytes, cparams=cp) + sc.append_data(data) + return sc.to_cframe(), data.tobytes() + + +def special(nitems, dtype, kind, value=None): + dt = np.dtype(dtype) + sc = blosc2.SChunk(chunksize=nitems * dt.itemsize, + cparams=blosc2.CParams(typesize=dt.itemsize)) + sc.fill_special(nitems, kind, value) + return sc.to_cframe() + + +# Special chunks. A repeated value stays in the frame as a 33+ byte chunk; +# NaN and uninitialised chunks become special offsets. +save("value_i4", special(300, " 3 else [ (') { let dir = tempfile::tempdir().unwrap(); let path = dir.path().join(format!("{tag}.h5")); - let n: usize = run_python(GENERATE, &[path.to_str().unwrap(), filters]) - .parse() - .unwrap(); + let mut args = vec![path.to_str().unwrap(), filters]; + args.extend(cases); + let n: usize = run_python(GENERATE, &args).parse().unwrap(); assert!(n > 0); let file = File::open(&path).unwrap(); for i in 0..n { @@ -353,6 +363,110 @@ fn blosc_written_by_clawhdf5_reads_in_hdf5plugin() { } } +#[cfg(feature = "blosc2")] +#[test] +fn blosc2_written_by_hdf5plugin_reads_exactly() { + if !have_python("h5py, hdf5plugin") { + return; + } + // Every codec hdf5plugin's Blosc2 offers, each lossless filter, and + // levels from "store" to maximum. Truncating precision is lossy, so it + // is checked separately against h5py's own reading. + check_h5py_written( + "blosc2", + r#"[(f'{c} {f} {l}', hdf5plugin.Blosc2(cname=c, clevel=l, filters=f)) + for c in ['blosclz', 'lz4', 'lz4hc', 'zlib', 'zstd'] + for f, l in [(hdf5plugin.Blosc2.NOFILTER, 5), + (hdf5plugin.Blosc2.SHUFFLE, 9), + (hdf5plugin.Blosc2.BITSHUFFLE, 1), + (hdf5plugin.Blosc2.DELTA, 5)]] + + [('blosclz level 0', hdf5plugin.Blosc2(cname='blosclz', clevel=0)), + ('zstd level 0 bitshuffle', + hdf5plugin.Blosc2(cname='zstd', clevel=0, filters=hdf5plugin.Blosc2.BITSHUFFLE))]"#, + ); +} + +/// Blosc2 over more shapes and dtypes: every integer and float width, +/// 1-D to 5-D chunks (B2ND arrays from 2-D on) with partial edge chunks and +/// block shapes that pad the chunk, datasets of zeros, of one repeated value +/// and of NaN (Blosc2's "special" chunks), and Fletcher32 before Blosc2 +/// (which makes hdf5-blosc2 fall back from B2ND to a plain frame). +#[cfg(feature = "blosc2")] +#[test] +fn blosc2_shapes_and_special_chunks_read_exactly() { + if !have_python("h5py, hdf5plugin") { + return; + } + let cases = r#"[(dt, shape, chunks, kind) + for dt in ['u2', 'f8'] + for shape, chunks in [((777,), (100,)), + ((37, 53), (10, 16)), + ((9, 10, 11), (4, 5, 3)), + ((6, 7, 5, 9), (3, 2, 5, 4))] + for kind in ['ramp', 'noise']] + + [('