diff --git a/crates/clawhdf5-format/Cargo.toml b/crates/clawhdf5-format/Cargo.toml index 1d1998c..c1ec6df 100644 --- a/crates/clawhdf5-format/Cargo.toml +++ b/crates/clawhdf5-format/Cargo.toml @@ -76,8 +76,10 @@ bitshuffle = ["lz4_flex", "ruzstd"] bzip2 = ["dep:bzip2", "std"] # Blosc 1 (32001) with its BloscLZ, LZ4, Snappy, Zlib and Zstandard codecs. blosc = ["lz4_flex", "ruzstd", "snap", "deflate", "std"] +# Blosc2 (32026), read-only: frames, B2ND arrays, and the Blosc codecs above. +blosc2 = ["blosc"] # Every plugin filter above. -plugin-filters = ["lzf", "bitshuffle", "bzip2", "blosc"] +plugin-filters = ["lzf", "bitshuffle", "bzip2", "blosc", "blosc2"] [[bench]] name = "parallel_decompress_bench" diff --git a/crates/clawhdf5-format/src/filter_registry.rs b/crates/clawhdf5-format/src/filter_registry.rs index 6d39613..442e27c 100644 --- a/crates/clawhdf5-format/src/filter_registry.rs +++ b/crates/clawhdf5-format/src/filter_registry.rs @@ -5,7 +5,8 @@ //! * **Built-in filters** — a static table of the filters compiled into this //! build: the HDF5 standard filters (deflate, shuffle, Fletcher32, szip, //! N-Bit, scale-offset) and the plugin filters whose cargo features are -//! enabled (LZ4, Zstandard, pcodec, LZF, bitshuffle, bzip2, blosc). +//! enabled (LZ4, Zstandard, pcodec, LZF, bitshuffle, bzip2, blosc, +//! blosc2). //! [`builtin_filters`] lists them. //! * **Registered filters** (`std` only) — codecs the application supplies //! for any other ID with [`register_filter`] (a [`FilterCodec`], or just a @@ -166,7 +167,7 @@ pub fn known_filter(id: u16) -> Option<(&'static str, Option<&'static str>)> { 32019 => ("JPEG", None), 32022 => ("BitGroom", None), 32023 => ("Granular BitRound", None), - 32026 => ("Blosc2", None), + 32026 => ("Blosc2", Some("blosc2")), _ => return None, }) } @@ -452,12 +453,12 @@ pub(crate) mod tests { #[test] fn unsupported_filter_error_names_the_filter() { let msg = FormatError::UnsupportedFilter(32026).to_string(); + assert!(msg.contains("Blosc2") && msg.contains("`blosc2`"), "{msg}"); + let msg = FormatError::UnsupportedFilter(32013).to_string(); assert!( - msg.contains("Blosc2") && msg.contains("not implemented"), + msg.contains("ZFP") && msg.contains("not implemented"), "{msg}" ); - let msg = FormatError::UnsupportedFilter(32013).to_string(); - assert!(msg.contains("ZFP"), "{msg}"); let msg = FormatError::UnsupportedFilter(32000).to_string(); assert!(msg.contains("LZF") && msg.contains("`lzf`"), "{msg}"); assert_eq!( diff --git a/crates/clawhdf5-format/src/filters.rs b/crates/clawhdf5-format/src/filters.rs index 36f2e96..d577eac 100644 --- a/crates/clawhdf5-format/src/filters.rs +++ b/crates/clawhdf5-format/src/filters.rs @@ -278,6 +278,13 @@ pub(crate) static BUILTIN_FILTERS: &[BuiltinFilter] = &[ }, encode: None, }, + #[cfg(feature = "blosc2")] + BuiltinFilter { + id: crate::filter_pipeline::FILTER_BLOSC2, + name: "blosc2", + decode: crate::filters_blosc2::blosc2_decode, + encode: None, + }, ]; /// Decode the HDF5 scale-offset filter (id 6). diff --git a/crates/clawhdf5-format/src/filters_blosc.rs b/crates/clawhdf5-format/src/filters_blosc.rs index 3edc964..fadfc1e 100644 --- a/crates/clawhdf5-format/src/filters_blosc.rs +++ b/crates/clawhdf5-format/src/filters_blosc.rs @@ -49,7 +49,7 @@ fn le32(b: &[u8], at: usize) -> Result { /// The codec inside a Blosc frame (flags bits 5-7). #[derive(Debug, Clone, Copy, PartialEq, Eq)] -enum Codec { +pub(crate) enum Codec { BloscLz, Lz4, Snappy, @@ -58,7 +58,7 @@ enum Codec { } impl Codec { - fn from_flags(flags: u8) -> Result { + pub(crate) fn from_flags(flags: u8) -> Result { match flags >> 5 { 0 => Ok(Codec::BloscLz), 1 => Ok(Codec::Lz4), @@ -71,7 +71,7 @@ impl Codec { } /// Decode one codec stream into exactly `dst`. -fn decode_stream( +pub(crate) fn decode_stream( codec: Codec, src: &[u8], dst: &mut [u8], diff --git a/crates/clawhdf5-format/src/filters_blosc2.rs b/crates/clawhdf5-format/src/filters_blosc2.rs new file mode 100644 index 0000000..5f332f8 --- /dev/null +++ b/crates/clawhdf5-format/src/filters_blosc2.rs @@ -0,0 +1,1083 @@ +//! Blosc2 (HDF5 filter 32026, `hdf5-blosc2`, hdf5plugin's `Blosc2`) in pure +//! Rust, read-only: the contiguous frame ("cframe") each chunk is stored +//! as, the Blosc2 chunks inside it, and the B2ND n-dimensional layout. +//! +//! References: c-blosc2's `README_CFRAME_FORMAT.rst`, +//! `README_CHUNK_FORMAT.rst` and `README_B2ND_METALAYER.rst`, and the +//! decoder in `blosc/blosc2.c` (`read_chunk_header`, `blosc_d`, +//! `pipeline_backward`), `blosc/frame.c` and `blosc/b2nd.c`, whose checks +//! this module mirrors. +//! +//! **The filter** (`blosc2_filter.c`). `cd_values`: `[0]` filter revision, +//! `[1]` block size, `[2]` type size, `[3]` chunk size in bytes, `[4]` +//! level, `[5]` filter (0 none, 1 shuffle, 2 bit shuffle, 3 delta, 4 +//! truncate precision), `[6]` codec, and for chunks of 2 to 8 (16 in newer +//! builds) dimensions `[7]` the rank and `[8..]` the chunk dimensions. Each +//! HDF5 chunk is one frame. A frame with a `b2nd` (or older `caterva`) +//! metalayer holds an n-D array the shape of the HDF5 chunk, whose data are +//! cut into blocks of `blockshape` (padded at the chunk's edges) stored one +//! after another, each in C order — so the decoded Blosc2 chunk is +//! reassembled into C order here. Otherwise the frame's first chunk is the +//! HDF5 chunk's data as it is. +//! +//! **Frame.** A msgpack header with fields at fixed offsets (header length, +//! frame length, sizes, type size, chunk size, codec flags) and a map of +//! metalayers; the chunks back to back; an "offsets" chunk (a Blosc2 chunk +//! of little-endian `i64` offsets, one per chunk, relative to the end of the +//! header — negative values encode a chunk of zeros, NaNs or uninitialised +//! values); and a msgpack trailer, which the decoder does not need. +//! +//! **Chunk.** A 16-byte Blosc header (as Blosc 1), extended to 32 bytes when +//! the shuffle and bit-shuffle flags are both set: six filter slots with +//! their metadata, the codec, and flags for dictionaries, lazy chunks and +//! "special" chunks (all zeros, NaN, uninitialised, or one repeated value +//! stored after the header). A regular chunk has a table of block starts; +//! each block is one stream, or `typesize` streams when the "do not split" +//! flag is clear and the block is not the short last one. A stream is an +//! `i32` length and the codec's output: 0 means zeros, negative with a token +//! byte means a run of one byte, and a length equal to the stream's size +//! means stored raw. Decoded blocks run through the filter pipeline +//! backwards (shuffle, bit shuffle, delta; truncate-precision needs no +//! decoding). The codecs are Blosc 1's: BloscLZ, LZ4 (and LZ4HC), Zlib and +//! Zstandard, shared with [`crate::filters_blosc`]. +//! +//! Chunks of uninitialised values read as zeros (libhdf5 hands back +//! whatever its buffer held). Not supported (a clear error, never data): +//! variable-length blocks, dictionaries, lazy chunks, user-defined codecs +//! and registered filters (e.g. bytedelta), sparse frames. + +use crate::error::FormatError; +use crate::filter_registry::FilterContext; +use crate::filters_bitshuffle::bitunshuffle_block; +use crate::filters_blosc::{Codec, blosclz_decompress, decode_stream}; + +const MIN_HEADER: usize = 16; +const EXT_HEADER: usize = 32; + +const FLAG_SHUFFLE: u8 = 0x01; +const FLAG_MEMCPYED: u8 = 0x02; +const FLAG_BITSHUFFLE: u8 = 0x04; +const FLAG_DELTA: u8 = 0x08; +const FLAG_DONT_SPLIT: u8 = 0x10; + +const B2_USEDICT: u8 = 0x01; +const B2_LAZY: u8 = 0x08; +const VL_BLOCKS: u8 = 0x01; + +const SPECIAL_ZERO: u8 = 1; +const SPECIAL_NAN: u8 = 2; +const SPECIAL_VALUE: u8 = 3; +const SPECIAL_UNINIT: u8 = 4; + +const FILTER_NONE: u8 = 0; +const FILTER_SHUFFLE: u8 = 1; +const FILTER_BITSHUFFLE: u8 = 2; +const FILTER_DELTA: u8 = 3; +const FILTER_TRUNC_PREC: u8 = 4; + +/// Chunk format versions: 3 is the Blosc 2 alpha series. +const VERSION_ALPHA: u8 = 3; +/// The newest chunk format c-blosc2 knows (6 added variable-length blocks). +const VERSION_MAX: u8 = 6; +/// The newest frame format c-blosc2 knows. +const FRAME_VERSION_MAX: u8 = 3; +const MAX_BLOCKSIZE: usize = 536_866_816; +/// `B2ND_MAX_DIM` in current c-blosc2 (8 in older builds). +const B2ND_MAX_DIM: usize = 16; + +// Frame header field offsets (c-blosc2 `frame.h`). +const FRAME_HEADER_LEN: usize = 11; +const FRAME_LEN: usize = 16; +const FRAME_FLAGS: usize = 25; +const FRAME_TYPE: usize = 26; +const FRAME_NBYTES: usize = 30; +const FRAME_CBYTES: usize = 39; +const FRAME_TYPESIZE: usize = 48; +const FRAME_CHUNKSIZE: usize = 58; +const FRAME_HEADER_MINLEN: usize = 87; +const FRAME_IDX_SIZE: usize = 89; +const FRAME_VL_BLOCKS: u8 = 0x80; + +fn err(msg: &str) -> FormatError { + FormatError::DecompressionError(format!("blosc2: {msg}")) +} + +fn le_i32(b: &[u8], at: usize) -> Result { + b.get(at..at + 4) + .map(|s| i32::from_le_bytes(s.try_into().unwrap())) + .ok_or_else(|| err("truncated chunk")) +} + +fn be(b: &[u8], at: usize) -> Result<[u8; N], FormatError> { + b.get(at..at + N) + .map(|s| s.try_into().unwrap()) + .ok_or_else(|| err("truncated frame header")) +} + +/// A parsed Blosc2 chunk header. +struct ChunkHeader { + version: u8, + flags: u8, + typesize: usize, + nbytes: usize, + blocksize: usize, + cbytes: usize, + overhead: usize, + filters: [u8; 6], + filters_meta: [u8; 6], + udcodec: u8, + blosc2_flags: u8, + special: u8, +} + +fn read_header(src: &[u8]) -> Result { + if src.len() < MIN_HEADER { + return Err(err("truncated chunk header")); + } + let version = src[0]; + let flags = src[2]; + let typesize = src[3] as usize; + let nbytes = le_i32(src, 4)?; + let blocksize = le_i32(src, 8)?; + let cbytes = le_i32(src, 12)?; + if cbytes < MIN_HEADER as i32 { + return Err(err("chunk size smaller than its header")); + } + if blocksize <= 0 || blocksize as usize > MAX_BLOCKSIZE { + return Err(err("bad block size")); + } + if typesize == 0 { + return Err(err("type size is zero")); + } + if nbytes < 0 { + return Err(err("negative decoded size")); + } + let (nbytes, mut blocksize, cbytes) = (nbytes as usize, blocksize as usize, cbytes as usize); + let mut h = ChunkHeader { + version, + flags, + typesize, + nbytes, + blocksize, + cbytes, + overhead: MIN_HEADER, + filters: [0; 6], + filters_meta: [0; 6], + udcodec: 0, + blosc2_flags: 0, + special: 0, + }; + let mut flags2 = 0; + if flags & (FLAG_SHUFFLE | FLAG_BITSHUFFLE) == FLAG_SHUFFLE | FLAG_BITSHUFFLE { + if cbytes < EXT_HEADER || src.len() < EXT_HEADER { + return Err(err("truncated extended chunk header")); + } + h.overhead = EXT_HEADER; + h.filters.copy_from_slice(&src[16..22]); + h.udcodec = src[22]; + h.filters_meta.copy_from_slice(&src[24..30]); + flags2 = src[30]; + h.blosc2_flags = src[31]; + h.special = (h.blosc2_flags >> 4) & 7; + if h.special == SPECIAL_VALUE { + let ts = cbytes - EXT_HEADER; + if ts == 0 || ts > nbytes || !nbytes.is_multiple_of(ts) { + return Err(err("bad repeated-value chunk")); + } + } else if h.special != 0 && h.special != SPECIAL_ZERO && !nbytes.is_multiple_of(typesize) { + return Err(err("decoded size is not a whole number of elements")); + } + if version == VERSION_ALPHA { + h.filters[5] = 0; + h.filters_meta[5] = 0; + } + } else { + // A Blosc 1 style header: the filters come from the flags. + if flags & FLAG_SHUFFLE != 0 { + h.filters[5] = FILTER_SHUFFLE; + } + if flags & FLAG_BITSHUFFLE != 0 { + h.filters[5] = FILTER_BITSHUFFLE; + } + if flags & FLAG_DELTA != 0 { + h.filters[4] = FILTER_DELTA; + } + } + if version > VERSION_MAX && flags2 & !VL_BLOCKS != 0 { + return Err(err(&format!("chunk format version {version} is too new"))); + } + if flags2 & VL_BLOCKS != 0 { + return Err(err("variable-length blocks are not supported")); + } + if nbytes > 0 && blocksize > nbytes { + blocksize = nbytes; + } + h.blocksize = blocksize; + if cbytes > src.len() { + return Err(err("chunk is longer than its buffer")); + } + Ok(h) +} + +/// Decompress one Blosc2 chunk (header included), refusing to produce more +/// than `limit` bytes. +pub fn blosc2_decompress_chunk(src: &[u8], limit: usize) -> Result, FormatError> { + let h = read_header(src)?; + if h.nbytes > limit { + return Err(err("decoded size exceeds the limit")); + } + let src = &src[..h.cbytes]; + if h.special > SPECIAL_UNINIT { + return Err(err(&format!("unknown special chunk type {}", h.special))); + } + let memcpyed = h.flags & FLAG_MEMCPYED != 0; + if memcpyed && h.cbytes != h.nbytes + h.overhead { + return Err(err("stored chunk has the wrong size")); + } + if h.nbytes == 0 && h.cbytes == h.overhead && h.special == 0 { + return Ok(Vec::new()); + } + let nbytes = h.nbytes; + let mut out = vec![0u8; nbytes]; + if h.special != 0 { + // Filled per block, as c-blosc2 does: each block must hold whole + // values. + let fill: Option<&[u8]> = match h.special { + SPECIAL_ZERO | SPECIAL_UNINIT => None, + SPECIAL_NAN => Some(match h.typesize { + 4 => &[0x00, 0x00, 0xc0, 0x7f], + 8 => &[0, 0, 0, 0, 0, 0, 0xf8, 0x7f], + _ => return Err(err("NaN chunk of a type that is not f32 or f64")), + }), + _ => Some(&src[EXT_HEADER..]), + }; + if let Some(value) = fill { + let ts = value.len(); + let blocksize = h.blocksize.max(1); + if !blocksize.is_multiple_of(ts) || !(nbytes % blocksize).is_multiple_of(ts) { + return Err(err("special chunk blocks are not whole values")); + } + for v in out.chunks_exact_mut(ts) { + v.copy_from_slice(value); + } + } + return Ok(out); + } + if memcpyed { + out.copy_from_slice(&src[h.overhead..]); + return Ok(out); + } + if h.blosc2_flags & B2_USEDICT != 0 { + return Err(err("dictionary-compressed chunks are not supported")); + } + if h.blosc2_flags & B2_LAZY != 0 { + return Err(err("lazy chunks are not supported")); + } + let codec = match h.flags >> 5 { + 6 => { + return Err(err(&format!( + "user-defined codec {} is not supported", + h.udcodec + ))); + } + 2 | 5 | 7 => return Err(err(&format!("unknown codec {}", h.flags >> 5))), + _ => Codec::from_flags(h.flags)?, + }; + for &f in &h.filters { + if f > FILTER_TRUNC_PREC { + return Err(err(&format!("filter {f} is not supported"))); + } + } + let blocksize = h.blocksize; + let nblocks = nbytes.div_ceil(blocksize); + let leftover = nbytes % blocksize; + let bstarts_end = nblocks + .checked_mul(4) + .and_then(|n| n.checked_add(h.overhead)) + .ok_or_else(|| err("block table overflows"))?; + if bstarts_end > src.len() { + return Err(err("block table is truncated")); + } + let dont_split = h.flags & FLAG_DONT_SPLIT != 0; + let mut tmp = vec![0u8; blocksize]; + let mut tmp2 = vec![0u8; blocksize]; + let mut zstd = None; + for j in 0..nblocks { + let is_leftover = j == nblocks - 1 && leftover > 0; + let bsize = if is_leftover { leftover } else { blocksize }; + let start = le_i32(src, h.overhead + 4 * j)?; + if start <= 0 || start as usize >= src.len() { + return Err(err("block start out of range")); + } + let mut pos = start as usize; + let nstreams = if !dont_split && !is_leftover { + h.typesize + } else { + 1 + }; + let neblock = bsize / nstreams; + if neblock == 0 || neblock * nstreams != bsize { + return Err(err("block is not a whole number of streams")); + } + let cur = &mut tmp[..bsize]; + for s in 0..nstreams { + let csize = le_i32(src, pos).map_err(|_| err("stream runs past the chunk"))?; + pos += 4; + let dst = &mut cur[s * neblock..(s + 1) * neblock]; + if csize == 0 { + dst.fill(0); + } else if csize < 0 { + let token = *src + .get(pos) + .ok_or_else(|| err("stream runs past the chunk"))?; + pos += 1; + if token & 1 == 0 || csize < -255 { + return Err(err("unsupported stream token")); + } + dst.fill((-csize) as u8); + } else { + let csize = csize as usize; + let stream = src + .get(pos..pos + csize) + .ok_or_else(|| err("stream runs past the chunk"))?; + pos += csize; + if csize == neblock { + dst.copy_from_slice(stream); + } else if codec == Codec::BloscLz { + if blosclz_decompress(stream, dst) != neblock { + return Err(err("blosclz stream is malformed")); + } + } else { + decode_stream(codec, stream, dst, &mut zstd)?; + } + } + } + let (done, rest) = out.split_at_mut(j * blocksize); + let dest = &mut rest[..bsize]; + run_filters_backward(&h, cur, &mut tmp2[..bsize], done, dest); + } + Ok(out) +} + +/// Undo the chunk's filters on one decoded block `cur`, leaving the result +/// in `dest`. `done` is the chunk decoded so far (the delta filter's +/// reference is the first block). +fn run_filters_backward( + h: &ChunkHeader, + cur: &mut [u8], + scratch: &mut [u8], + done: &[u8], + dest: &mut [u8], +) { + let bsize = cur.len(); + let ts = h.typesize; + for i in (0..6).rev() { + match h.filters[i] { + FILTER_SHUFFLE => { + let m = h.filters_meta[i] as usize; + let bytes = if m == 0 { ts } else { m }; + unshuffle(bytes, cur, scratch); + cur.copy_from_slice(scratch); + } + FILTER_BITSHUFFLE => { + let mut n = bsize / ts; + if h.version == 2 { + if !n.is_multiple_of(8) { + continue; + } + } else { + n -= n % 8; + } + let body = n * ts; + bitunshuffle_block(&cur[..body], &mut scratch[..body], n, ts); + cur[..body].copy_from_slice(&scratch[..body]); + } + FILTER_DELTA => delta_decode(done, cur, ts), + // Truncating precision is lossy and has nothing to undo. + FILTER_NONE | FILTER_TRUNC_PREC => {} + _ => unreachable!("filters are checked before decoding"), + } + } + dest.copy_from_slice(cur); +} + +/// Byte unshuffle (`unshuffle_generic`): trailing bytes that do not make a +/// whole element are copied as they are. +fn unshuffle(bytes: usize, src: &[u8], dest: &mut [u8]) { + let n = src.len() / bytes; + for i in 0..n { + for b in 0..bytes { + dest[i * bytes + b] = src[b * n + i]; + } + } + dest[n * bytes..].copy_from_slice(&src[n * bytes..]); +} + +/// `delta_decoder`: the first block is XOR-accumulated over its own +/// elements; every other block is XORed with the (decoded) first block. +/// Elements are 1, 2, 4 or 8 bytes wide (other sizes: 8 if a multiple of +/// 8, else 1); a trailing partial element is left alone. +fn delta_decode(done: &[u8], cur: &mut [u8], typesize: usize) { + let w = match typesize { + 1 | 2 | 4 | 8 => typesize, + t if t.is_multiple_of(8) => 8, + _ => 1, + }; + let n = cur.len() / w; + if done.is_empty() { + for i in 1..n { + for b in 0..w { + cur[i * w + b] ^= cur[(i - 1) * w + b]; + } + } + } else { + // The first block is at least as long as any other. + for (c, r) in cur[..n * w].iter_mut().zip(done) { + *c ^= *r; + } + } +} + +/// What the frame header says. +struct Frame<'a> { + buf: &'a [u8], + header_len: usize, + nbytes: usize, + cbytes: usize, + typesize: usize, + chunksize: usize, + nchunks: usize, + /// The decoded offsets chunk. + offsets: Vec, +} + +fn parse_frame(buf: &[u8]) -> Result, FormatError> { + if buf.len() < FRAME_HEADER_MINLEN { + return Err(err("truncated frame header")); + } + if buf[0] & 0xf0 != 0x90 || buf[1] != 0xa8 || &buf[2..10] != b"b2frame\0" { + return Err(err("not a Blosc2 frame (bad magic)")); + } + let frame_version = buf[FRAME_FLAGS] & 0x0f; + if frame_version > FRAME_VERSION_MAX { + return Err(err(&format!( + "frame format version {frame_version} is too new" + ))); + } + if buf[FRAME_FLAGS] & FRAME_VL_BLOCKS != 0 { + return Err(err("variable-length blocks are not supported")); + } + if buf[FRAME_TYPE] & 0x0f != 0 { + return Err(err("not a contiguous frame")); + } + let header_len = i32::from_be_bytes(be(buf, FRAME_HEADER_LEN)?); + let frame_len = u64::from_be_bytes(be(buf, FRAME_LEN)?); + let nbytes = i64::from_be_bytes(be(buf, FRAME_NBYTES)?); + let cbytes = i64::from_be_bytes(be(buf, FRAME_CBYTES)?); + let typesize = i32::from_be_bytes(be(buf, FRAME_TYPESIZE)?); + let chunksize = i32::from_be_bytes(be(buf, FRAME_CHUNKSIZE)?); + if header_len < FRAME_HEADER_MINLEN as i32 || header_len as u64 > frame_len { + return Err(err("bad frame header length")); + } + if frame_len > buf.len() as u64 { + return Err(err("frame is longer than the chunk")); + } + if typesize <= 0 { + return Err(err("bad type size")); + } + if nbytes < 0 || cbytes < 0 || chunksize < 0 { + return Err(err("negative size in frame header")); + } + let header_len = header_len as usize; + let buf = &buf[..frame_len as usize]; + let cbytes = usize::try_from(cbytes).map_err(|_| err("bad compressed size"))?; + let data_end = header_len + .checked_add(cbytes) + .filter(|&e| e <= buf.len()) + .ok_or_else(|| err("chunks run past the frame"))?; + let nbytes = usize::try_from(nbytes).map_err(|_| err("bad decoded size"))?; + let chunksize = chunksize as usize; + // The offsets chunk follows the data chunks. + let off_src = &buf[data_end..]; + let expected = if nbytes == 0 { + 0 + } else if chunksize > 0 { + nbytes.div_ceil(chunksize) + } else { + // Variable-size chunks: the offsets chunk tells how many. + usize::MAX + }; + let off_limit = if expected == usize::MAX { + 1 << 24 + } else { + expected + .checked_mul(8) + .ok_or_else(|| err("too many chunks"))? + }; + let offsets = if nbytes == 0 { + Vec::new() + } else { + blosc2_decompress_chunk(off_src, off_limit)? + }; + if !offsets.len().is_multiple_of(8) || (expected != usize::MAX && offsets.len() != off_limit) { + return Err(err("offsets chunk does not match the number of chunks")); + } + Ok(Frame { + buf, + header_len, + nbytes, + cbytes, + typesize: typesize as usize, + chunksize, + nchunks: offsets.len() / 8, + offsets, + }) +} + +impl Frame<'_> { + /// Decode chunk `n`, refusing more than `limit` bytes. + fn chunk(&self, n: usize, limit: usize) -> Result, FormatError> { + if n >= self.nchunks { + return Err(err("frame has no such chunk")); + } + let raw: [u8; 8] = self.offsets[8 * n..8 * n + 8].try_into().unwrap(); + let offset = i64::from_le_bytes(raw); + if offset < 0 { + // A special chunk, recorded in the offset's top byte. + if self.chunksize == 0 { + return Err(err("special chunk in a frame without a chunk size")); + } + let size = if n == self.nchunks - 1 && !self.nbytes.is_multiple_of(self.chunksize) { + self.nbytes % self.chunksize + } else { + self.chunksize + }; + if size > limit { + return Err(err("decoded size exceeds the limit")); + } + let kind = raw[7] & 7; + return match kind { + SPECIAL_ZERO | SPECIAL_UNINIT => Ok(vec![0u8; size]), + SPECIAL_NAN => { + let nan: &[u8] = match self.typesize { + 4 => &[0x00, 0x00, 0xc0, 0x7f], + 8 => &[0, 0, 0, 0, 0, 0, 0xf8, 0x7f], + _ => return Err(err("NaN chunk of a type that is not f32 or f64")), + }; + if !size.is_multiple_of(nan.len()) { + return Err(err("NaN chunk is not whole values")); + } + Ok(nan.iter().copied().cycle().take(size).collect()) + } + _ => Err(err(&format!("unknown special chunk offset {kind}"))), + }; + } + // Every chunk lies in the data section, between the header and the + // offsets chunk. + let end = self.header_len + self.cbytes; + let start = usize::try_from(offset) + .ok() + .and_then(|o| self.header_len.checked_add(o)) + .filter(|&s| s < end && end - s >= MIN_HEADER) + .ok_or_else(|| err("chunk offset out of range"))?; + blosc2_decompress_chunk(&self.buf[start..end], limit) + } + + /// The content of metalayer `name`, if the frame has it. + fn metalayer(&self, name: &[u8]) -> Result, FormatError> { + let h = &self.buf[..self.header_len]; + // [FRAME_IDX_SIZE] is the u16 index size, then a map16. + let mut p = FRAME_IDX_SIZE + 2; + if h.get(p) != Some(&0xde) { + return Err(err("bad metalayer index")); + } + let count = u16::from_be_bytes(be(h, p + 1)?) as usize; + p += 3; + if count > 16 { + return Err(err("too many metalayers")); + } + for _ in 0..count { + let tag = *h.get(p).ok_or_else(|| err("truncated metalayer index"))?; + if tag & 0xe0 != 0xa0 { + return Err(err("bad metalayer name")); + } + let len = (tag & 0x1f) as usize; + let key = h + .get(p + 1..p + 1 + len) + .ok_or_else(|| err("truncated metalayer index"))?; + p += 1 + len; + if h.get(p) != Some(&0xd2) { + return Err(err("bad metalayer offset")); + } + let off = i32::from_be_bytes(be(h, p + 1)?); + p += 5; + if key != name { + continue; + } + if off < 0 || off as usize >= h.len() { + return Err(err("metalayer offset out of range")); + } + let off = off as usize; + if h.get(off) != Some(&0xc6) { + return Err(err("bad metalayer content")); + } + let clen = u32::from_be_bytes(be(h, off + 1)?) as usize; + return h + .get(off + 5..) + .and_then(|c| c.get(..clen)) + .map(Some) + .ok_or_else(|| err("metalayer runs past the header")); + } + Ok(None) + } +} + +/// The B2ND (or Caterva) metalayer: shape, chunk shape, block shape. +#[derive(Debug, PartialEq, Eq)] +struct NdMeta { + shape: Vec, + chunkshape: Vec, + blockshape: Vec, +} + +/// A msgpack array header: fixarray or array16. +fn array_len(m: &[u8], p: &mut usize) -> Result { + let t = *m.get(*p).ok_or_else(|| err("truncated b2nd metalayer"))?; + if t & 0xf0 == 0x90 { + *p += 1; + Ok((t & 0x0f) as usize) + } else if t == 0xdc { + let n = u16::from_be_bytes(be(m, *p + 1)?) as usize; + *p += 3; + Ok(n) + } else { + Err(err("bad b2nd metalayer")) + } +} + +fn parse_nd(m: &[u8]) -> Result { + let bad = || err("bad b2nd metalayer"); + let mut p = 0; + if array_len(m, &mut p)? < 5 { + return Err(bad()); + } + // Version, then the rank: positive fixints. + let version = *m.get(p).ok_or_else(bad)?; + let ndim = *m.get(p + 1).ok_or_else(bad)? as usize; + if version > 0x7f || ndim == 0 || ndim > B2ND_MAX_DIM { + return Err(err(&format!("unsupported b2nd rank {ndim}"))); + } + p += 2; + let mut read = |wide: bool| -> Result, FormatError> { + if array_len(m, &mut p)? != ndim { + return Err(bad()); + } + (0..ndim) + .map(|_| { + let v = if wide { + if m.get(p) != Some(&0xd3) { + return Err(bad()); + } + let v = i64::from_be_bytes(be(m, p + 1)?); + p += 9; + v + } else { + if m.get(p) != Some(&0xd2) { + return Err(bad()); + } + let v = i32::from_be_bytes(be(m, p + 1)?) as i64; + p += 5; + v + }; + usize::try_from(v).map_err(|_| bad()) + }) + .collect() + }; + let shape = read(true)?; + let chunkshape = read(false)?; + let blockshape = read(false)?; + Ok(NdMeta { + shape, + chunkshape, + blockshape, + }) +} + +/// Decode a frame as the HDF5 filter does: a B2ND array into C order, or +/// else the frame's first chunk. `cd_shape` is the chunk shape recorded in +/// the filter's `cd_values` (if any), which the array's shape must match. +fn decode_frame( + input: &[u8], + limit: usize, + cd_shape: Option<&[usize]>, +) -> Result, FormatError> { + let frame = parse_frame(input)?; + let meta = match frame.metalayer(b"b2nd")? { + Some(m) => Some(m), + None => frame.metalayer(b"caterva")?, + }; + let Some(meta) = meta else { + return frame.chunk(0, limit); + }; + let nd = parse_nd(meta)?; + if let Some(cd) = cd_shape + && cd != nd.shape.as_slice() + { + return Err(err(&format!( + "array shape {:?} is not the chunk shape {cd:?}", + nd.shape + ))); + } + reassemble(&frame, &nd, limit) +} + +/// Gather a B2ND array's blocks into one C-order buffer. +fn reassemble(frame: &Frame<'_>, nd: &NdMeta, limit: usize) -> Result, FormatError> { + let ts = frame.typesize; + let ndim = nd.shape.len(); + let too_big = || err("array exceeds the chunk size"); + let mut total = ts; + let mut ext = vec![0usize; ndim]; + let mut grid = vec![0usize; ndim]; + let mut ext_bytes = ts; + for i in 0..ndim { + let (s, c, b) = (nd.shape[i], nd.chunkshape[i], nd.blockshape[i]); + if c == 0 || b == 0 || b > c { + return Err(err("bad b2nd chunk or block shape")); + } + total = total.checked_mul(s).ok_or_else(too_big)?; + ext[i] = c.div_ceil(b) * b; + ext_bytes = ext_bytes.checked_mul(ext[i]).ok_or_else(too_big)?; + grid[i] = s.div_ceil(c); + } + if total > limit { + return Err(too_big()); + } + let mut out = vec![0u8; total]; + if total == 0 { + return Ok(out); + } + // Padding to whole blocks grows a chunk by less than 2x per dimension; + // hdf5-blosc2's block shapes keep it far below 16x (a crafted frame + // could otherwise make each chunk enormous). + if ext_bytes > limit.saturating_mul(16) { + return Err(too_big()); + } + let nchunks: usize = grid.iter().product(); + if nchunks != frame.nchunks { + return Err(err("chunk count does not match the b2nd shape")); + } + let blocks_in_chunk: Vec = (0..ndim).map(|i| ext[i] / nd.blockshape[i]).collect(); + let nblocks: usize = blocks_in_chunk.iter().product(); + let block_items: usize = nd.blockshape.iter().product(); + let block_bytes = block_items * ts; + // Strides (in elements) of the output array and of a block. + let mut out_stride = vec![1usize; ndim]; + let mut blk_stride = vec![1usize; ndim]; + for i in (0..ndim.saturating_sub(1)).rev() { + out_stride[i] = out_stride[i + 1] * nd.shape[i + 1]; + blk_stride[i] = blk_stride[i + 1] * nd.blockshape[i + 1]; + } + let mut cidx = vec![0usize; ndim]; + let mut bidx = vec![0usize; ndim]; + let mut gstart = vec![0usize; ndim]; + let mut valid = vec![0usize; ndim]; + let mut pos = vec![0usize; ndim]; + for n in 0..nchunks { + unravel(n, &grid, &mut cidx); + let data = frame.chunk(n, ext_bytes)?; + if data.len() != ext_bytes { + return Err(err("chunk size does not match the b2nd shape")); + } + for b in 0..nblocks { + unravel(b, &blocks_in_chunk, &mut bidx); + let mut empty = false; + for i in 0..ndim { + let in_chunk = bidx[i] * nd.blockshape[i]; + gstart[i] = cidx[i] * nd.chunkshape[i] + in_chunk; + let lim_chunk = nd.chunkshape[i].saturating_sub(in_chunk); + let lim_shape = nd.shape[i].saturating_sub(gstart[i]); + valid[i] = nd.blockshape[i].min(lim_chunk).min(lim_shape); + empty |= valid[i] == 0; + } + if empty { + continue; + } + let block = &data[b * block_bytes..(b + 1) * block_bytes]; + // Copy row by row along the last dimension. + let row = valid[ndim - 1] * ts; + let rows: usize = valid[..ndim - 1].iter().product(); + for r in 0..rows { + unravel(r, &valid[..ndim - 1], &mut pos[..ndim - 1]); + let mut src = 0; + let mut dst = gstart[ndim - 1]; + for i in 0..ndim - 1 { + src += pos[i] * blk_stride[i]; + dst += (gstart[i] + pos[i]) * out_stride[i]; + } + out[dst * ts..dst * ts + row].copy_from_slice(&block[src * ts..src * ts + row]); + } + } + } + Ok(out) +} + +/// C-order multi-index of `n` in a grid of `dims`. +fn unravel(mut n: usize, dims: &[usize], idx: &mut [usize]) { + for i in (0..dims.len()).rev() { + idx[i] = n % dims[i]; + n /= dims[i]; + } +} + +/// Decompress one HDF5 chunk written by the Blosc2 filter (a Blosc2 frame) +/// into its data, in C order; at most `limit` bytes. +pub fn blosc2_decompress(input: &[u8], limit: usize) -> Result, FormatError> { + decode_frame(input, limit, None) +} + +/// The filter's decoder. +pub(crate) fn blosc2_decode(input: &[u8], ctx: &FilterContext<'_>) -> Result, FormatError> { + let cd = ctx.client_data(); + // cd_values[7] is the chunk rank for B2ND (> 1), and the chunk + // dimensions follow. + let cd_shape: Option> = if cd.len() >= 8 { + let rank = cd[7] as usize; + if rank < 2 || cd.len() < 8 + rank { + return Err(err("bad chunk rank in the filter parameters")); + } + Some(cd[8..8 + rank].iter().map(|&d| d as usize).collect()) + } else { + None + }; + let out = decode_frame(input, ctx.output_limit(), cd_shape.as_deref())?; + if out.is_empty() && ctx.max_output != 0 { + return Err(err("empty frame for a non-empty chunk")); + } + Ok(out) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::filter_pipeline::{FILTER_BLOSC2, FilterDescription}; + use std::path::PathBuf; + + fn fixtures() -> PathBuf { + PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("tests/fixtures/blosc2") + } + + /// Every fixture frame (see `tests/fixtures/blosc2/generate.py`), + /// sorted by name: (name, frame, expected output or error word). + #[allow(clippy::type_complexity)] + fn cases() -> Vec<(String, Vec, Result, String>)> { + let mut v = Vec::new(); + for e in std::fs::read_dir(fixtures()).unwrap() { + let p = e.unwrap().path(); + if p.extension().is_none_or(|x| x != "b2f") { + continue; + } + let name = p.file_stem().unwrap().to_string_lossy().into_owned(); + let frame = std::fs::read(&p).unwrap(); + let want = match std::fs::read(p.with_extension("out")) { + Ok(out) => Ok(out), + Err(_) => Err(std::fs::read_to_string(p.with_extension("err")).unwrap()), + }; + v.push((name, frame, want)); + } + v.sort_by(|a, b| a.0.cmp(&b.0)); + assert!(v.len() >= 20, "fixtures missing"); + v + } + + #[test] + fn fixture_frames_decode_exactly() { + for (name, frame, want) in cases() { + let got = blosc2_decompress(&frame, 1 << 20); + match want { + Ok(want) => { + let got = got.unwrap_or_else(|e| panic!("{name}: {e}")); + assert!(got == want, "{name}: decoded data differs"); + } + Err(word) => { + let e = got.expect_err(&name).to_string(); + assert!(e.contains(word.trim()), "{name}: {e}"); + } + } + } + } + + /// The output limit is enforced for every kind of frame, including the + /// padded chunks of B2ND arrays and special chunks. + #[test] + fn output_limit_is_enforced() { + for (name, frame, want) in cases() { + let Ok(want) = want else { continue }; + assert!( + blosc2_decompress(&frame, want.len() - 1).is_err(), + "{name}: decoded past the limit" + ); + assert_eq!( + blosc2_decompress(&frame, want.len()).unwrap(), + want, + "{name}" + ); + } + } + + fn desc(cd: Vec) -> FilterDescription { + FilterDescription { + filter_id: FILTER_BLOSC2, + name: None, + flags: 0, + client_data: cd, + } + } + + /// As the HDF5 filter: the B2ND array must have the chunk shape the + /// filter parameters record. + #[test] + fn b2nd_shape_must_match_the_filter_parameters() { + let frame = std::fs::read(fixtures().join("b2nd_2d.b2f")).unwrap(); + let want = std::fs::read(fixtures().join("b2nd_2d.out")).unwrap(); + let decode = |cd: Vec| { + let f = desc(cd); + blosc2_decode( + &frame, + &FilterContext { + filter: &f, + element_size: 4, + max_output: want.len(), + }, + ) + }; + assert_eq!(decode(vec![1, 0, 4, 0, 5, 1, 1, 2, 37, 29]).unwrap(), want); + assert_eq!(decode(vec![1, 0, 4, 0, 5, 1, 1]).unwrap(), want); + assert!(decode(vec![1, 0, 4, 0, 5, 1, 1, 2, 29, 37]).is_err()); + assert!(decode(vec![1, 0, 4, 0, 5, 1, 1, 3, 37, 29, 1]).is_err()); + assert!(decode(vec![1, 0, 4, 0, 5, 1, 1, 2, 37]).is_err()); + } + + /// A 32-byte extended header. + fn ext_header(flags: u8, ts: u8, nbytes: i32, blocksize: i32, cbytes: i32) -> Vec { + let mut h = vec![5u8, 1, flags | FLAG_SHUFFLE | FLAG_BITSHUFFLE, ts]; + for v in [nbytes, blocksize, cbytes] { + h.extend_from_slice(&v.to_le_bytes()); + } + h.resize(EXT_HEADER, 0); + h + } + + #[test] + fn special_chunks() { + // Zeros, uninitialised (read as zeros), NaN, one repeated value. + let mut z = ext_header(0, 4, 64, 64, 32); + z[31] = SPECIAL_ZERO << 4; + assert_eq!(blosc2_decompress_chunk(&z, 64).unwrap(), vec![0; 64]); + z[31] = SPECIAL_UNINIT << 4; + assert_eq!(blosc2_decompress_chunk(&z, 64).unwrap(), vec![0; 64]); + z[31] = SPECIAL_NAN << 4; + let nan = blosc2_decompress_chunk(&z, 64).unwrap(); + assert!( + nan.chunks(4) + .all(|c| f32::from_le_bytes(c.try_into().unwrap()).is_nan()) + ); + z[3] = 2; + assert!(blosc2_decompress_chunk(&z, 64).is_err(), "NaN of i16"); + let mut v = ext_header(0, 2, 12, 12, 35); + v[31] = SPECIAL_VALUE << 4; + v.extend_from_slice(&[1, 2, 3]); + assert_eq!( + blosc2_decompress_chunk(&v, 12).unwrap(), + [1, 2, 3].repeat(4) + ); + // The value must divide the chunk (and its blocks). + v[4] = 13; + assert!(blosc2_decompress_chunk(&v, 13).is_err()); + v[4] = 12; + v[8] = 4; + assert!(blosc2_decompress_chunk(&v, 12).is_err()); + // Special types 5 to 7 are reserved. + let mut r = ext_header(0, 4, 64, 64, 32); + r[31] = 5 << 4; + assert!(blosc2_decompress_chunk(&r, 64).is_err()); + } + + /// A regular chunk: one block of two streams (split, type size 2): a + /// run of 0x07 and a stream of zeros, then the byte shuffle undone. + #[test] + fn stream_runs_and_zero_streams() { + let mut c = ext_header(0, 2, 8, 8, 0); + c[16 + 5] = FILTER_SHUFFLE; + c.extend_from_slice(&36i32.to_le_bytes()); + c.extend_from_slice(&(-7i32).to_le_bytes()); + c.push(1); + c.extend_from_slice(&0i32.to_le_bytes()); + let n = c.len() as i32; + c[12..16].copy_from_slice(&n.to_le_bytes()); + assert_eq!( + blosc2_decompress_chunk(&c, 8).unwrap(), + [7, 0, 7, 0, 7, 0, 7, 0] + ); + // A token without the run bit is reserved. + let mut bad = c.clone(); + bad[40] = 2; + assert!(blosc2_decompress_chunk(&bad, 8).is_err()); + // Unsupported header features are errors, not data. + for (at, bit) in [(31, B2_USEDICT), (31, B2_LAZY), (30, VL_BLOCKS)] { + let mut x = c.clone(); + x[at] |= bit; + assert!( + blosc2_decompress_chunk(&x, 8).is_err(), + "byte {at} bit {bit}" + ); + } + let mut udf = c.clone(); + udf[16] = 35; + assert!(blosc2_decompress_chunk(&udf, 8).is_err()); + let mut udc = c.clone(); + udc[2] |= 6 << 5; + assert!(blosc2_decompress_chunk(&udc, 8).is_err()); + assert!(blosc2_decompress_chunk(&c, 7).is_err(), "limit"); + } + + /// Random and mutated frames: errors are fine, panics are not, and no + /// output past the limit. + #[test] + fn fuzzed_frames_never_panic() { + let limit = 20_000; + let seeds: Vec> = cases() + .into_iter() + .filter(|(_, f, w)| f.len() < 9000 && w.as_ref().is_ok_and(|w| w.len() <= limit)) + .map(|(_, f, _)| f) + .collect(); + assert!(seeds.len() >= 15); + crate::test_fuzz::fuzz_decoder(0xb2, &seeds, 20_000, limit, |s| { + blosc2_decompress(s, limit) + }); + } + + /// Blosc2 chunks on their own, mutated. + #[test] + fn fuzzed_chunks_never_panic() { + let mut seeds = Vec::new(); + for (_, f, w) in cases() { + if w.is_err() { + continue; + } + let frame = parse_frame(&f).unwrap(); + let raw: [u8; 8] = frame.offsets[..8].try_into().unwrap(); + let off = i64::from_le_bytes(raw); + if off >= 0 { + let start = frame.header_len + off as usize; + let end = frame.header_len + frame.cbytes; + let h = read_header(&f[start..end]).unwrap(); + seeds.push(f[start..start + h.cbytes].to_vec()); + } + } + assert!(seeds.len() >= 10); + crate::test_fuzz::fuzz_decoder(0xb3, &seeds, 20_000, 1 << 16, |s| { + blosc2_decompress_chunk(s, 1 << 16) + }); + } +} diff --git a/crates/clawhdf5-format/src/lib.rs b/crates/clawhdf5-format/src/lib.rs index 3e30f9d..89d324b 100644 --- a/crates/clawhdf5-format/src/lib.rs +++ b/crates/clawhdf5-format/src/lib.rs @@ -86,6 +86,8 @@ pub mod filters; mod filters_bitshuffle; #[cfg(feature = "blosc")] pub mod filters_blosc; +#[cfg(feature = "blosc2")] +pub mod filters_blosc2; #[cfg(feature = "bzip2")] mod filters_bzip2; #[cfg(feature = "lzf")] diff --git a/crates/clawhdf5-format/tests/fixtures/blosc2/always_split.b2f b/crates/clawhdf5-format/tests/fixtures/blosc2/always_split.b2f new file mode 100644 index 0000000..db6322a Binary files /dev/null and b/crates/clawhdf5-format/tests/fixtures/blosc2/always_split.b2f differ diff --git a/crates/clawhdf5-format/tests/fixtures/blosc2/always_split.out b/crates/clawhdf5-format/tests/fixtures/blosc2/always_split.out new file mode 100644 index 0000000..a3faac5 Binary files /dev/null and b/crates/clawhdf5-format/tests/fixtures/blosc2/always_split.out differ diff --git a/crates/clawhdf5-format/tests/fixtures/blosc2/b2nd_2d.b2f b/crates/clawhdf5-format/tests/fixtures/blosc2/b2nd_2d.b2f new file mode 100644 index 0000000..a092e29 Binary files /dev/null and b/crates/clawhdf5-format/tests/fixtures/blosc2/b2nd_2d.b2f differ diff --git a/crates/clawhdf5-format/tests/fixtures/blosc2/b2nd_2d.out b/crates/clawhdf5-format/tests/fixtures/blosc2/b2nd_2d.out new file mode 100644 index 0000000..4e6ff85 Binary files /dev/null and b/crates/clawhdf5-format/tests/fixtures/blosc2/b2nd_2d.out differ diff --git a/crates/clawhdf5-format/tests/fixtures/blosc2/b2nd_3d.b2f b/crates/clawhdf5-format/tests/fixtures/blosc2/b2nd_3d.b2f new file mode 100644 index 0000000..edaf272 Binary files /dev/null and b/crates/clawhdf5-format/tests/fixtures/blosc2/b2nd_3d.b2f differ diff --git a/crates/clawhdf5-format/tests/fixtures/blosc2/b2nd_3d.out b/crates/clawhdf5-format/tests/fixtures/blosc2/b2nd_3d.out new file mode 100644 index 0000000..c29f5f0 Binary files /dev/null and b/crates/clawhdf5-format/tests/fixtures/blosc2/b2nd_3d.out differ diff --git a/crates/clawhdf5-format/tests/fixtures/blosc2/b2nd_4d.b2f b/crates/clawhdf5-format/tests/fixtures/blosc2/b2nd_4d.b2f new file mode 100644 index 0000000..6fa77de Binary files /dev/null and b/crates/clawhdf5-format/tests/fixtures/blosc2/b2nd_4d.b2f differ diff --git a/crates/clawhdf5-format/tests/fixtures/blosc2/b2nd_4d.out b/crates/clawhdf5-format/tests/fixtures/blosc2/b2nd_4d.out new file mode 100644 index 0000000..bcd534f Binary files /dev/null and b/crates/clawhdf5-format/tests/fixtures/blosc2/b2nd_4d.out differ diff --git a/crates/clawhdf5-format/tests/fixtures/blosc2/b2nd_zero_chunk.b2f b/crates/clawhdf5-format/tests/fixtures/blosc2/b2nd_zero_chunk.b2f new file mode 100644 index 0000000..029f33d Binary files /dev/null and b/crates/clawhdf5-format/tests/fixtures/blosc2/b2nd_zero_chunk.b2f differ diff --git a/crates/clawhdf5-format/tests/fixtures/blosc2/b2nd_zero_chunk.out b/crates/clawhdf5-format/tests/fixtures/blosc2/b2nd_zero_chunk.out new file mode 100644 index 0000000..fe7c16e Binary files /dev/null and b/crates/clawhdf5-format/tests/fixtures/blosc2/b2nd_zero_chunk.out differ diff --git a/crates/clawhdf5-format/tests/fixtures/blosc2/bitshuffle_odd_blocks.b2f b/crates/clawhdf5-format/tests/fixtures/blosc2/bitshuffle_odd_blocks.b2f new file mode 100644 index 0000000..6e71e42 Binary files /dev/null and b/crates/clawhdf5-format/tests/fixtures/blosc2/bitshuffle_odd_blocks.b2f differ diff --git a/crates/clawhdf5-format/tests/fixtures/blosc2/bitshuffle_odd_blocks.out b/crates/clawhdf5-format/tests/fixtures/blosc2/bitshuffle_odd_blocks.out new file mode 100644 index 0000000..83e486e Binary files /dev/null and b/crates/clawhdf5-format/tests/fixtures/blosc2/bitshuffle_odd_blocks.out differ diff --git a/crates/clawhdf5-format/tests/fixtures/blosc2/bytedelta.b2f b/crates/clawhdf5-format/tests/fixtures/blosc2/bytedelta.b2f new file mode 100644 index 0000000..2995a86 Binary files /dev/null and b/crates/clawhdf5-format/tests/fixtures/blosc2/bytedelta.b2f differ diff --git a/crates/clawhdf5-format/tests/fixtures/blosc2/bytedelta.err b/crates/clawhdf5-format/tests/fixtures/blosc2/bytedelta.err new file mode 100644 index 0000000..f69e717 --- /dev/null +++ b/crates/clawhdf5-format/tests/fixtures/blosc2/bytedelta.err @@ -0,0 +1 @@ +filter 35 \ No newline at end of file diff --git a/crates/clawhdf5-format/tests/fixtures/blosc2/delta_int16_blosclz.b2f b/crates/clawhdf5-format/tests/fixtures/blosc2/delta_int16_blosclz.b2f new file mode 100644 index 0000000..7face64 Binary files /dev/null and b/crates/clawhdf5-format/tests/fixtures/blosc2/delta_int16_blosclz.b2f differ diff --git a/crates/clawhdf5-format/tests/fixtures/blosc2/delta_int16_blosclz.out b/crates/clawhdf5-format/tests/fixtures/blosc2/delta_int16_blosclz.out new file mode 100644 index 0000000..e891807 Binary files /dev/null and b/crates/clawhdf5-format/tests/fixtures/blosc2/delta_int16_blosclz.out differ diff --git a/crates/clawhdf5-format/tests/fixtures/blosc2/delta_int32_lz4.b2f b/crates/clawhdf5-format/tests/fixtures/blosc2/delta_int32_lz4.b2f new file mode 100644 index 0000000..819f61e Binary files /dev/null and b/crates/clawhdf5-format/tests/fixtures/blosc2/delta_int32_lz4.b2f differ diff --git a/crates/clawhdf5-format/tests/fixtures/blosc2/delta_int32_lz4.out b/crates/clawhdf5-format/tests/fixtures/blosc2/delta_int32_lz4.out new file mode 100644 index 0000000..97e148c Binary files /dev/null and b/crates/clawhdf5-format/tests/fixtures/blosc2/delta_int32_lz4.out differ diff --git a/crates/clawhdf5-format/tests/fixtures/blosc2/delta_shuffle_v16.b2f b/crates/clawhdf5-format/tests/fixtures/blosc2/delta_shuffle_v16.b2f new file mode 100644 index 0000000..5f0cc42 Binary files /dev/null and b/crates/clawhdf5-format/tests/fixtures/blosc2/delta_shuffle_v16.b2f differ diff --git a/crates/clawhdf5-format/tests/fixtures/blosc2/delta_shuffle_v16.out b/crates/clawhdf5-format/tests/fixtures/blosc2/delta_shuffle_v16.out new file mode 100644 index 0000000..2073522 Binary files /dev/null and b/crates/clawhdf5-format/tests/fixtures/blosc2/delta_shuffle_v16.out differ diff --git a/crates/clawhdf5-format/tests/fixtures/blosc2/delta_uint64_blosclz.b2f b/crates/clawhdf5-format/tests/fixtures/blosc2/delta_uint64_blosclz.b2f new file mode 100644 index 0000000..f052059 Binary files /dev/null and b/crates/clawhdf5-format/tests/fixtures/blosc2/delta_uint64_blosclz.b2f differ diff --git a/crates/clawhdf5-format/tests/fixtures/blosc2/delta_uint64_blosclz.out b/crates/clawhdf5-format/tests/fixtures/blosc2/delta_uint64_blosclz.out new file mode 100644 index 0000000..2073522 Binary files /dev/null and b/crates/clawhdf5-format/tests/fixtures/blosc2/delta_uint64_blosclz.out differ diff --git a/crates/clawhdf5-format/tests/fixtures/blosc2/delta_uint8_lz4.b2f b/crates/clawhdf5-format/tests/fixtures/blosc2/delta_uint8_lz4.b2f new file mode 100644 index 0000000..8ecfdd8 Binary files /dev/null and b/crates/clawhdf5-format/tests/fixtures/blosc2/delta_uint8_lz4.b2f differ diff --git a/crates/clawhdf5-format/tests/fixtures/blosc2/delta_uint8_lz4.out b/crates/clawhdf5-format/tests/fixtures/blosc2/delta_uint8_lz4.out new file mode 100644 index 0000000..b70a6d9 Binary files /dev/null and b/crates/clawhdf5-format/tests/fixtures/blosc2/delta_uint8_lz4.out differ diff --git a/crates/clawhdf5-format/tests/fixtures/blosc2/delta_v3.b2f b/crates/clawhdf5-format/tests/fixtures/blosc2/delta_v3.b2f new file mode 100644 index 0000000..cda8773 Binary files /dev/null and b/crates/clawhdf5-format/tests/fixtures/blosc2/delta_v3.b2f differ diff --git a/crates/clawhdf5-format/tests/fixtures/blosc2/delta_v3.out b/crates/clawhdf5-format/tests/fixtures/blosc2/delta_v3.out new file mode 100644 index 0000000..bc826e3 Binary files /dev/null and b/crates/clawhdf5-format/tests/fixtures/blosc2/delta_v3.out differ diff --git a/crates/clawhdf5-format/tests/fixtures/blosc2/generate.py b/crates/clawhdf5-format/tests/fixtures/blosc2/generate.py new file mode 100644 index 0000000..f79c0f7 --- /dev/null +++ b/crates/clawhdf5-format/tests/fixtures/blosc2/generate.py @@ -0,0 +1,129 @@ +"""Generate the Blosc2 frames the `filters_blosc2` unit tests decode. + +Each case is `.b2f` (a Blosc2 contiguous frame, what the HDF5 Blosc2 +filter stores per chunk) and `.out` (what decoding it must give: the +first chunk of a plain frame, or the whole array in C order for a B2ND +frame), or `.err` (a frame clawhdf5 must refuse; the file holds a word +the error message must contain). + +These cover what files written by h5py + hdf5plugin never contain, but a +Blosc2 frame may: special chunks (repeated value, NaN, uninitialised), the +delta filter over many blocks and odd type sizes, bit shuffle of blocks that +are not a multiple of 8 elements, shuffle with a byte-group size, forced +stream splitting, multi-chunk B2ND arrays with padded edge chunks and a +chunk of zeros, and features clawhdf5 refuses (dictionaries, registered +filters). + +Written with python-blosc2 4.13.1 (c-blosc2 3.3.4) in a scratch venv +(`pip install blosc2`). Re-run only to regenerate: + + python generate.py +""" +import os +import sys + +import blosc2 +import numpy as np + +out = sys.argv[1] + + +def save(name, frame, expected): + with open(os.path.join(out, name + ".b2f"), "wb") as f: + f.write(frame) + with open(os.path.join(out, name + ".out"), "wb") as f: + f.write(expected) + + +def save_err(name, frame, word): + with open(os.path.join(out, name + ".b2f"), "wb") as f: + f.write(frame) + with open(os.path.join(out, name + ".err"), "w") as f: + f.write(word) + + +def plain(data, **cparams): + """A one-chunk super-chunk frame of `data`, as hdf5-blosc2 writes.""" + data = np.ascontiguousarray(data) + cp = blosc2.CParams(typesize=data.dtype.itemsize, **cparams) + sc = blosc2.SChunk(chunksize=data.nbytes, cparams=cp) + sc.append_data(data) + return sc.to_cframe(), data.tobytes() + + +def special(nitems, dtype, kind, value=None): + dt = np.dtype(dtype) + sc = blosc2.SChunk(chunksize=nitems * dt.itemsize, + cparams=blosc2.CParams(typesize=dt.itemsize)) + sc.fill_special(nitems, kind, value) + return sc.to_cframe() + + +# Special chunks. A repeated value stays in the frame as a 33+ byte chunk; +# NaN and uninitialised chunks become special offsets. +save("value_i4", special(300, " 3 else [ (') { let dir = tempfile::tempdir().unwrap(); let path = dir.path().join(format!("{tag}.h5")); - let n: usize = run_python(GENERATE, &[path.to_str().unwrap(), filters]) - .parse() - .unwrap(); + let mut args = vec![path.to_str().unwrap(), filters]; + args.extend(cases); + let n: usize = run_python(GENERATE, &args).parse().unwrap(); assert!(n > 0); let file = File::open(&path).unwrap(); for i in 0..n { @@ -353,6 +363,110 @@ fn blosc_written_by_clawhdf5_reads_in_hdf5plugin() { } } +#[cfg(feature = "blosc2")] +#[test] +fn blosc2_written_by_hdf5plugin_reads_exactly() { + if !have_python("h5py, hdf5plugin") { + return; + } + // Every codec hdf5plugin's Blosc2 offers, each lossless filter, and + // levels from "store" to maximum. Truncating precision is lossy, so it + // is checked separately against h5py's own reading. + check_h5py_written( + "blosc2", + r#"[(f'{c} {f} {l}', hdf5plugin.Blosc2(cname=c, clevel=l, filters=f)) + for c in ['blosclz', 'lz4', 'lz4hc', 'zlib', 'zstd'] + for f, l in [(hdf5plugin.Blosc2.NOFILTER, 5), + (hdf5plugin.Blosc2.SHUFFLE, 9), + (hdf5plugin.Blosc2.BITSHUFFLE, 1), + (hdf5plugin.Blosc2.DELTA, 5)]] + + [('blosclz level 0', hdf5plugin.Blosc2(cname='blosclz', clevel=0)), + ('zstd level 0 bitshuffle', + hdf5plugin.Blosc2(cname='zstd', clevel=0, filters=hdf5plugin.Blosc2.BITSHUFFLE))]"#, + ); +} + +/// Blosc2 over more shapes and dtypes: every integer and float width, +/// 1-D to 5-D chunks (B2ND arrays from 2-D on) with partial edge chunks and +/// block shapes that pad the chunk, datasets of zeros, of one repeated value +/// and of NaN (Blosc2's "special" chunks), and Fletcher32 before Blosc2 +/// (which makes hdf5-blosc2 fall back from B2ND to a plain frame). +#[cfg(feature = "blosc2")] +#[test] +fn blosc2_shapes_and_special_chunks_read_exactly() { + if !have_python("h5py, hdf5plugin") { + return; + } + let cases = r#"[(dt, shape, chunks, kind) + for dt in ['u2', 'f8'] + for shape, chunks in [((777,), (100,)), + ((37, 53), (10, 16)), + ((9, 10, 11), (4, 5, 3)), + ((6, 7, 5, 9), (3, 2, 5, 4))] + for kind in ['ramp', 'noise']] + + [('