From 22dc87b07c810665a7825a6b2a14488b89e3d54d Mon Sep 17 00:00:00 2001 From: osobh Date: Sat, 26 Sep 2026 11:30:39 -0500 Subject: [PATCH] fix(format): never hold a B2ND chunk's padding B2ND chunks were decoded whole, padding included, with up to 16x the HDF5 chunk size as their limit, so a crafted frame made each chunk allocate and fill up to 16x the output (4 GiB for a 256 MiB HDF5 chunk). The padding is the real bound (prod(ceil(c/b)*b) per chunk), but that can be 2^ndim times the array, so it is no longer held at all: - A Blosc2 chunk is now decoded block by block (decode_blocks), each block handed to a sink as it is ready, with at most three blocks of scratch. blosc2_decompress_chunk and plain frames still collect every block. - reassemble places each B2ND block straight into the output and skips blocks that are all padding (they are not decoded unless the delta filter needs the first block). A frame's NaN chunks are handed over one B2ND block at a time and its zero chunks cost nothing. - A B2ND chunk must decode to exactly its padded size, its Blosc2 blocks must be whole B2ND blocks no larger than the output, and a chunk may not be larger than the array (hdf5-blosc2's chunk is the array), so a block is never larger than the output. Peak allocation for a 10-D array padded to 13x (NaN, repeated-value and stored-block chunks) and for a 16x chunk was 4.5 MB and 17.8 MB for 315 KB and 1 MiB outputs before, and is now within the tests' bound. Blosc2 files written by hdf5plugin in 9-D and 12-D, an 8 MiB single chunk, 1x1x1 and edge-chunk shapes still read exactly. Co-Authored-By: Claude Opus 5.5 (1M context) --- crates/clawhdf5-format/src/filters_blosc2.rs | 377 +++++++++++++----- .../tests/blosc2_alloc_bounds.rs | 128 ++++++ 2 files changed, 408 insertions(+), 97 deletions(-) diff --git a/crates/clawhdf5-format/src/filters_blosc2.rs b/crates/clawhdf5-format/src/filters_blosc2.rs index 812ff24..71ad4b6 100644 --- a/crates/clawhdf5-format/src/filters_blosc2.rs +++ b/crates/clawhdf5-format/src/filters_blosc2.rs @@ -225,20 +225,59 @@ fn read_header(src: &[u8]) -> Result { /// Decompress one Blosc2 chunk (header included), refusing to produce more /// than `limit` bytes. pub fn blosc2_decompress_chunk(src: &[u8], limit: usize) -> Result, FormatError> { + let (h, src) = open_chunk(src, limit)?; + let mut out = vec![0u8; h.nbytes]; + decode_blocks(&h, src, &mut |_, _| true, &mut |at, block| { + out[at..at + block.len()].copy_from_slice(block); + })?; + Ok(out) +} + +/// Read and check a chunk's header: its decoded size must be within +/// `limit`. Returns the header and the chunk's bytes. +fn open_chunk(src: &[u8], limit: usize) -> Result<(ChunkHeader, &[u8]), FormatError> { let h = read_header(src)?; if h.nbytes > limit { return Err(err("decoded size exceeds the limit")); } - let src = &src[..h.cbytes]; if h.special > SPECIAL_UNINIT { return Err(err(&format!("unknown special chunk type {}", h.special))); } - let memcpyed = h.flags & FLAG_MEMCPYED != 0; - if memcpyed && h.cbytes != h.nbytes + h.overhead { + if h.flags & FLAG_MEMCPYED != 0 && h.cbytes != h.nbytes + h.overhead { return Err(err("stored chunk has the wrong size")); } + let src = &src[..h.cbytes]; + Ok((h, src)) +} + +/// Decode a chunk block by block into a destination the caller has +/// zeroed: `sink(offset, block)` receives each decoded block (the last may +/// be short), except blocks for which `want(offset, len)` is false and +/// the blocks of a special all-zeros chunk. Holds at most three blocks of scratch, so a +/// caller that keeps only part of each block (B2ND padding) never holds +/// the whole chunk. +fn decode_blocks( + h: &ChunkHeader, + src: &[u8], + want: &mut dyn FnMut(usize, usize) -> bool, + sink: &mut dyn FnMut(usize, &[u8]), +) -> Result<(), FormatError> { let nbytes = h.nbytes; - let mut out = vec![0u8; nbytes]; + // `read_header` clamps the block size to the decoded size. + let blocksize = h.blocksize; + let nblocks = if nbytes == 0 { + 0 + } else { + nbytes.div_ceil(blocksize) + }; + let leftover = nbytes % blocksize.max(1); + let block_len = |j: usize| { + if j == nblocks - 1 && leftover > 0 { + leftover + } else { + blocksize + } + }; if h.special != 0 { // Filled per block, as c-blosc2 does: each block must hold whole // values. @@ -253,19 +292,30 @@ pub fn blosc2_decompress_chunk(src: &[u8], limit: usize) -> Result, Form }; if let Some(value) = fill { let ts = value.len(); - let blocksize = h.blocksize.max(1); - if !blocksize.is_multiple_of(ts) || !(nbytes % blocksize).is_multiple_of(ts) { + if !blocksize.max(1).is_multiple_of(ts) || !leftover.is_multiple_of(ts) { return Err(err("special chunk blocks are not whole values")); } - for v in out.chunks_exact_mut(ts) { - v.copy_from_slice(value); + if nblocks > 0 { + let block: Vec = value.iter().copied().cycle().take(blocksize).collect(); + for j in 0..nblocks { + let len = block_len(j); + if want(j * blocksize, len) { + sink(j * blocksize, &block[..len]); + } + } } } - return Ok(out); + return Ok(()); } - if memcpyed { - out.copy_from_slice(&src[h.overhead..]); - return Ok(out); + if h.flags & FLAG_MEMCPYED != 0 { + let data = &src[h.overhead..]; + for j in 0..nblocks { + let (at, len) = (j * blocksize, block_len(j)); + if want(at, len) { + sink(at, &data[at..at + len]); + } + } + return Ok(()); } if h.blosc2_flags & B2_USEDICT != 0 { return Err(err("dictionary-compressed chunks are not supported")); @@ -289,11 +339,8 @@ pub fn blosc2_decompress_chunk(src: &[u8], limit: usize) -> Result, Form } } if nbytes == 0 { - return Ok(out); + return Ok(()); } - let blocksize = h.blocksize; - let nblocks = nbytes.div_ceil(blocksize); - let leftover = nbytes % blocksize; let bstarts_end = nblocks .checked_mul(4) .and_then(|n| n.checked_add(h.overhead)) @@ -302,12 +349,19 @@ pub fn blosc2_decompress_chunk(src: &[u8], limit: usize) -> Result, Form return Err(err("block table is truncated")); } let dont_split = h.flags & FLAG_DONT_SPLIT != 0; + // The delta filter's reference is the first block, decoded. + let delta = h.filters.contains(&FILTER_DELTA); + let mut first = Vec::new(); let mut tmp = vec![0u8; blocksize]; let mut tmp2 = vec![0u8; blocksize]; let mut zstd = None; for j in 0..nblocks { let is_leftover = j == nblocks - 1 && leftover > 0; - let bsize = if is_leftover { leftover } else { blocksize }; + let bsize = block_len(j); + let wanted = want(j * blocksize, bsize); + if !wanted && !(delta && j == 0) { + continue; + } let start = le_i32(src, h.overhead + 4 * j)?; if start <= 0 || start as usize >= src.len() { return Err(err("block start out of range")); @@ -355,23 +409,21 @@ pub fn blosc2_decompress_chunk(src: &[u8], limit: usize) -> Result, Form } } } - let (done, rest) = out.split_at_mut(j * blocksize); - let dest = &mut rest[..bsize]; - run_filters_backward(&h, cur, &mut tmp2[..bsize], done, dest); + run_filters_backward(h, cur, &mut tmp2[..bsize], &first); + if delta && j == 0 { + first = cur.to_vec(); + } + if wanted { + sink(j * blocksize, cur); + } } - Ok(out) + Ok(()) } -/// Undo the chunk's filters on one decoded block `cur`, leaving the result -/// in `dest`. `done` is the chunk decoded so far (the delta filter's -/// reference is the first block). -fn run_filters_backward( - h: &ChunkHeader, - cur: &mut [u8], - scratch: &mut [u8], - done: &[u8], - dest: &mut [u8], -) { +/// Undo the chunk's filters on one decoded block `cur`, in place. `first` +/// is the decoded first block (empty while decoding it), the delta +/// filter's reference. +fn run_filters_backward(h: &ChunkHeader, cur: &mut [u8], scratch: &mut [u8], first: &[u8]) { let bsize = cur.len(); let ts = h.typesize; for i in (0..6).rev() { @@ -395,13 +447,12 @@ fn run_filters_backward( bitunshuffle_block(&cur[..body], &mut scratch[..body], n, ts); cur[..body].copy_from_slice(&scratch[..body]); } - FILTER_DELTA => delta_decode(done, cur, ts), + FILTER_DELTA => delta_decode(first, cur, ts), // Truncating precision is lossy and has nothing to undo. FILTER_NONE | FILTER_TRUNC_PREC => {} _ => unreachable!("filters are checked before decoding"), } } - dest.copy_from_slice(cur); } /// Byte unshuffle (`unshuffle_generic`): trailing bytes that do not make a @@ -416,18 +467,19 @@ fn unshuffle(bytes: usize, src: &[u8], dest: &mut [u8]) { dest[n * bytes..].copy_from_slice(&src[n * bytes..]); } -/// `delta_decoder`: the first block is XOR-accumulated over its own -/// elements; every other block is XORed with the (decoded) first block. +/// `delta_decoder`: the first block (`first` is empty) is XOR-accumulated +/// over its own elements; every other block is XORed with the (decoded) +/// first block. /// Elements are 1, 2, 4 or 8 bytes wide (other sizes: 8 if a multiple of /// 8, else 1); a trailing partial element is left alone. -fn delta_decode(done: &[u8], cur: &mut [u8], typesize: usize) { +fn delta_decode(first: &[u8], cur: &mut [u8], typesize: usize) { let w = match typesize { 1 | 2 | 4 | 8 => typesize, t if t.is_multiple_of(8) => 8, _ => 1, }; let n = cur.len() / w; - if done.is_empty() { + if first.is_empty() { for i in 1..n { for b in 0..w { cur[i * w + b] ^= cur[(i - 1) * w + b]; @@ -435,7 +487,7 @@ fn delta_decode(done: &[u8], cur: &mut [u8], typesize: usize) { } } else { // The first block is at least as long as any other. - for (c, r) in cur[..n * w].iter_mut().zip(done) { + for (c, r) in cur[..n * w].iter_mut().zip(first) { *c ^= *r; } } @@ -548,14 +600,15 @@ fn parse_frame(buf: &[u8], limit: usize) -> Result, FormatError> { } impl Frame<'_> { - /// Decode chunk `n`, refusing more than `limit` bytes. - fn chunk(&self, n: usize, limit: usize) -> Result, FormatError> { + /// Where chunk `n` is: its size from the frame header (every chunk but + /// the last holds `chunksize` bytes; unknown if that is 0), and either + /// the special value its offset records or its bytes. + fn locate(&self, n: usize) -> Result<(Option, Located<'_>), FormatError> { if n >= self.nchunks { return Err(err("frame has no such chunk")); } let raw: [u8; 8] = self.offsets[8 * n..8 * n + 8].try_into().unwrap(); let offset = i64::from_le_bytes(raw); - // Every chunk but the last holds `chunksize` bytes. let size = if self.chunksize == 0 { None } else if n == self.nchunks - 1 && !self.nbytes.is_multiple_of(self.chunksize) { @@ -565,26 +618,20 @@ impl Frame<'_> { }; if offset < 0 { // A special chunk, recorded in the offset's top byte. - let Some(size) = size else { + if size.is_none() { return Err(err("special chunk in a frame without a chunk size")); - }; - if size > limit { - return Err(err("decoded size exceeds the limit")); } let kind = raw[7] & 7; return match kind { - SPECIAL_ZERO | SPECIAL_UNINIT => Ok(vec![0u8; size]), - SPECIAL_NAN => { - let nan: &[u8] = match self.typesize { + SPECIAL_ZERO | SPECIAL_UNINIT => Ok((size, Located::Zeros)), + SPECIAL_NAN => Ok(( + size, + Located::Fill(match self.typesize { 4 => &[0x00, 0x00, 0xc0, 0x7f], 8 => &[0, 0, 0, 0, 0, 0, 0xf8, 0x7f], _ => return Err(err("NaN chunk of a type that is not f32 or f64")), - }; - if !size.is_multiple_of(nan.len()) { - return Err(err("NaN chunk is not whole values")); - } - Ok(nan.iter().copied().cycle().take(size).collect()) - } + }), + )), _ => Err(err(&format!("unknown special chunk offset {kind}"))), }; } @@ -596,12 +643,90 @@ impl Frame<'_> { .and_then(|o| self.header_len.checked_add(o)) .filter(|&s| s < end && end - s >= MIN_HEADER) .ok_or_else(|| err("chunk offset out of range"))?; - let data = - blosc2_decompress_chunk(&self.buf[start..end], limit.min(size.unwrap_or(limit)))?; - if size.is_some_and(|s| s != data.len()) { + Ok((size, Located::Data(&self.buf[start..end]))) + } + + /// Open chunk `n` (a regular one), which must decode to the frame's + /// size for it, and at most `limit` bytes. + fn open<'s>( + &self, + size: Option, + src: &'s [u8], + limit: usize, + ) -> Result<(ChunkHeader, &'s [u8]), FormatError> { + let (h, src) = open_chunk(src, limit.min(size.unwrap_or(limit)))?; + if size.is_some_and(|s| s != h.nbytes) { return Err(err("chunk size does not match the frame's chunk size")); } - Ok(data) + Ok((h, src)) + } + + /// Decode chunk `n`, refusing more than `limit` bytes. + fn chunk(&self, n: usize, limit: usize) -> Result, FormatError> { + let (size, at) = self.locate(n)?; + let src = match at { + Located::Data(src) => src, + Located::Zeros | Located::Fill(_) => { + let size = size.unwrap_or(0); + if size > limit { + return Err(err("decoded size exceeds the limit")); + } + return match at { + Located::Fill(value) if !size.is_multiple_of(value.len()) => { + Err(err("NaN chunk is not whole values")) + } + Located::Fill(value) => Ok(value.iter().copied().cycle().take(size).collect()), + _ => Ok(vec![0u8; size]), + }; + } + }; + let (h, src) = self.open(size, src, limit)?; + let mut out = vec![0u8; h.nbytes]; + decode_blocks(&h, src, &mut |_, _| true, &mut |at, block| { + out[at..at + block.len()].copy_from_slice(block); + })?; + Ok(out) + } + + /// Decode chunk `n` (exactly `size` bytes) block by block, as + /// [`decode_blocks`], into a destination the caller has zeroed. Its + /// blocks must be whole multiples of `unit` bytes and at most + /// `max_block`; special chunks are handed over `unit` bytes at a time. + fn chunk_blocks( + &self, + n: usize, + size: usize, + unit: usize, + max_block: usize, + want: &mut dyn FnMut(usize, usize) -> bool, + sink: &mut dyn FnMut(usize, &[u8]), + ) -> Result<(), FormatError> { + let (frame_size, at) = self.locate(n)?; + if frame_size.is_some_and(|s| s != size) { + return Err(err("chunk size does not match the b2nd shape")); + } + match at { + Located::Zeros => Ok(()), + Located::Fill(value) => { + if !unit.is_multiple_of(value.len()) || !size.is_multiple_of(unit) { + return Err(err("NaN chunk is not whole values")); + } + let block: Vec = value.iter().copied().cycle().take(unit).collect(); + for k in 0..size / unit { + if want(k * unit, unit) { + sink(k * unit, &block); + } + } + Ok(()) + } + Located::Data(src) => { + let (h, src) = self.open(Some(size), src, size)?; + if h.nbytes > 0 && (!h.blocksize.is_multiple_of(unit) || h.blocksize > max_block) { + return Err(err("chunk blocks do not match the b2nd blocks")); + } + decode_blocks(&h, src, want, sink) + } + } } /// The content of metalayer `name`, if the frame has it. @@ -653,6 +778,16 @@ impl Frame<'_> { } } +/// Where a frame's chunk is. +enum Located<'a> { + /// All zeros (or uninitialised, read as zeros). + Zeros, + /// One value repeated (NaN). + Fill(&'static [u8]), + /// A Blosc2 chunk, running to the end of the data section. + Data(&'a [u8]), +} + /// The B2ND (or Caterva) metalayer: shape, chunk shape, block shape. #[derive(Debug, PartialEq, Eq)] struct NdMeta { @@ -753,6 +888,12 @@ fn decode_frame( } /// Gather a B2ND array's blocks into one C-order buffer. +/// +/// Each Blosc2 chunk holds one B2ND chunk padded to whole blocks, which can +/// be many times the array (a crafted frame's shapes are the file's word). +/// So the chunks are never decoded whole: each block is placed in the +/// output as it is decoded, blocks that are all padding are skipped, and +/// no more than a few blocks (each within the array) are held at once. fn reassemble(frame: &Frame<'_>, nd: &NdMeta, limit: usize) -> Result, FormatError> { let ts = frame.typesize; let ndim = nd.shape.len(); @@ -778,10 +919,15 @@ fn reassemble(frame: &Frame<'_>, nd: &NdMeta, limit: usize) -> Result, F if total == 0 { return Ok(out); } - // Padding to whole blocks grows a chunk by less than 2x per dimension; - // hdf5-blosc2's block shapes keep it far below 16x (a crafted frame - // could otherwise make each chunk enormous). - if ext_bytes > limit.saturating_mul(16) { + // hdf5-blosc2's chunk is the array. A chunk larger than the array + // would be padding decoded for nothing; with chunks and blocks within + // the array, a block is never larger than the output, and a chunk + // pads each dimension by less than one block. + if (0..ndim).any(|i| nd.chunkshape[i] > nd.shape[i]) { + return Err(err("b2nd chunk is larger than the array")); + } + // A Blosc2 chunk of `ext_bytes`, exactly the padded chunk. + if ext_bytes > i32::MAX as usize { return Err(too_big()); } let nchunks: usize = grid.iter().product(); @@ -789,7 +935,6 @@ fn reassemble(frame: &Frame<'_>, nd: &NdMeta, limit: usize) -> Result, F return Err(err("chunk count does not match the b2nd shape")); } let blocks_in_chunk: Vec = (0..ndim).map(|i| ext[i] / nd.blockshape[i]).collect(); - let nblocks: usize = blocks_in_chunk.iter().product(); let block_items: usize = nd.blockshape.iter().product(); let block_bytes = block_items * ts; // Strides (in elements) of the output array and of a block. @@ -799,50 +944,88 @@ fn reassemble(frame: &Frame<'_>, nd: &NdMeta, limit: usize) -> Result, F out_stride[i] = out_stride[i + 1] * nd.shape[i + 1]; blk_stride[i] = blk_stride[i + 1] * nd.blockshape[i + 1]; } + let geometry = BlockGeometry { + nd, + blocks_in_chunk: &blocks_in_chunk, + }; let mut cidx = vec![0usize; ndim]; - let mut bidx = vec![0usize; ndim]; - let mut gstart = vec![0usize; ndim]; - let mut valid = vec![0usize; ndim]; + let mut w = BlockPlace::new(ndim); + let mut p = BlockPlace::new(ndim); let mut pos = vec![0usize; ndim]; for n in 0..nchunks { unravel(n, &grid, &mut cidx); - let data = frame.chunk(n, ext_bytes)?; - if data.len() != ext_bytes { - return Err(err("chunk size does not match the b2nd shape")); - } - for b in 0..nblocks { - unravel(b, &blocks_in_chunk, &mut bidx); - let mut empty = false; - for i in 0..ndim { - let in_chunk = bidx[i] * nd.blockshape[i]; - gstart[i] = cidx[i] * nd.chunkshape[i] + in_chunk; - let lim_chunk = nd.chunkshape[i].saturating_sub(in_chunk); - let lim_shape = nd.shape[i].saturating_sub(gstart[i]); - valid[i] = nd.blockshape[i].min(lim_chunk).min(lim_shape); - empty |= valid[i] == 0; - } - if empty { - continue; - } - let block = &data[b * block_bytes..(b + 1) * block_bytes]; - // Copy row by row along the last dimension. - let row = valid[ndim - 1] * ts; - let rows: usize = valid[..ndim - 1].iter().product(); - for r in 0..rows { - unravel(r, &valid[..ndim - 1], &mut pos[..ndim - 1]); - let mut src = 0; - let mut dst = gstart[ndim - 1]; - for i in 0..ndim - 1 { - src += pos[i] * blk_stride[i]; - dst += (gstart[i] + pos[i]) * out_stride[i]; + let blocks = |at: usize, len: usize| at / block_bytes..(at + len).div_ceil(block_bytes); + let mut want = + |at: usize, len: usize| blocks(at, len).any(|b| geometry.place(b, &cidx, &mut w)); + let mut sink = |at: usize, data: &[u8]| { + for b in blocks(at, data.len()) { + if !geometry.place(b, &cidx, &mut p) { + continue; + } + let block = &data[b * block_bytes - at..][..block_bytes]; + // Copy row by row along the last dimension. + let row = p.valid[ndim - 1] * ts; + let rows: usize = p.valid[..ndim - 1].iter().product(); + for r in 0..rows { + unravel(r, &p.valid[..ndim - 1], &mut pos[..ndim - 1]); + let mut src = 0; + let mut dst = p.gstart[ndim - 1]; + for i in 0..ndim - 1 { + src += pos[i] * blk_stride[i]; + dst += (p.gstart[i] + pos[i]) * out_stride[i]; + } + out[dst * ts..dst * ts + row].copy_from_slice(&block[src * ts..src * ts + row]); } - out[dst * ts..dst * ts + row].copy_from_slice(&block[src * ts..src * ts + row]); } - } + }; + frame.chunk_blocks(n, ext_bytes, block_bytes, limit, &mut want, &mut sink)?; } Ok(out) } +/// The block layout of a B2ND array's chunks. +struct BlockGeometry<'a> { + nd: &'a NdMeta, + blocks_in_chunk: &'a [usize], +} + +/// Where one block of a chunk lands: its first element in the array and +/// how much of it (per dimension) lies within the chunk and the array. +struct BlockPlace { + bidx: Vec, + gstart: Vec, + valid: Vec, +} + +impl BlockPlace { + fn new(ndim: usize) -> BlockPlace { + BlockPlace { + bidx: vec![0; ndim], + gstart: vec![0; ndim], + valid: vec![0; ndim], + } + } +} + +impl BlockGeometry<'_> { + /// Place block `b` of the chunk at grid index `cidx`; false if it is + /// all padding. + fn place(&self, b: usize, cidx: &[usize], p: &mut BlockPlace) -> bool { + let nd = self.nd; + unravel(b, self.blocks_in_chunk, &mut p.bidx); + let mut empty = false; + for i in 0..nd.shape.len() { + let in_chunk = p.bidx[i] * nd.blockshape[i]; + p.gstart[i] = cidx[i] * nd.chunkshape[i] + in_chunk; + let lim_chunk = nd.chunkshape[i].saturating_sub(in_chunk); + let lim_shape = nd.shape[i].saturating_sub(p.gstart[i]); + p.valid[i] = nd.blockshape[i].min(lim_chunk).min(lim_shape); + empty |= p.valid[i] == 0; + } + !empty + } +} + /// C-order multi-index of `n` in a grid of `dims`. fn unravel(mut n: usize, dims: &[usize], idx: &mut [usize]) { for i in (0..dims.len()).rev() { diff --git a/crates/clawhdf5-format/tests/blosc2_alloc_bounds.rs b/crates/clawhdf5-format/tests/blosc2_alloc_bounds.rs index 833cd53..a6ba114 100644 --- a/crates/clawhdf5-format/tests/blosc2_alloc_bounds.rs +++ b/crates/clawhdf5-format/tests/blosc2_alloc_bounds.rs @@ -220,3 +220,131 @@ fn empty_chunk_does_not_allocate_its_block_size() { assert!(r.is_err(), "decoded {:?}", r.map(|v| v.len())); assert!(peak <= bound(64, &f), "in a frame: peak {peak} bytes"); } + +/// B2ND chunks were decoded whole, padding included, with up to 16x the +/// HDF5 chunk size as their limit. Blocks are now placed as they are +/// decoded, so the padding is never held. +/// +/// Ten dimensions: nine of 3 split into blocks of 2 (padded to 4) and one +/// of 4, so each chunk is 13x the array. One chunk, stored three ways: as a +/// NaN chunk in the frame's offsets, as a repeated-value chunk, and as a +/// chunk of stored (uncompressed) blocks. +#[test] +fn b2nd_padding_is_never_held() { + let _g = lock(); + let ts = 4usize; + let mut shape = vec![3i64; 9]; + shape.push(4); + let chunks: Vec = shape.iter().map(|&s| s as i32).collect(); + let mut blocks = vec![2i32; 9]; + blocks.push(4); + let meta = nd_meta(&shape, &chunks, &blocks); + let items: usize = shape.iter().product::() as usize; + let limit = items * ts; + let block_bytes = ts * blocks.iter().product::() as usize; + let ext_bytes = ts * 4usize.pow(9) * 4; + assert!(ext_bytes > 13 * limit); + let offsets = |off: [u8; 8]| repeated(&off, 8, 8); + + let value = 1.5f32.to_le_bytes(); + let stored = { + // Every block stored raw: block k holds the value k. + let mut c = chunk_header(4, ext_bytes as i32, block_bytes as i32, 0, 0); + c[2] = 0x02 | 0x10; // memcpyed, not split + c.truncate(16); + for k in 0..ext_bytes / block_bytes { + c.extend((k as f32).to_le_bytes().repeat(block_bytes / 4)); + } + let n = c.len() as i32; + c[12..16].copy_from_slice(&n.to_le_bytes()); + c + }; + let cases: Vec<(&str, Vec)> = vec![ + ( + "NaN offset", + frame( + Some(&meta), + ext_bytes as i64, + 4, + ext_bytes as i32, + &[], + &offsets(special_offset(2)), + ), + ), + ( + "repeated value", + frame( + Some(&meta), + ext_bytes as i64, + 4, + ext_bytes as i32, + &repeated(&value, ext_bytes as i32, block_bytes as i32), + &offsets(0i64.to_le_bytes()), + ), + ), + ( + "stored blocks", + frame( + Some(&meta), + ext_bytes as i64, + 4, + ext_bytes as i32, + &stored, + &offsets(0i64.to_le_bytes()), + ), + ), + ]; + for (name, f) in cases { + let (r, peak) = peak_during(|| blosc2_decompress(&f, limit)); + let out = r.unwrap_or_else(|e| panic!("{name}: {e}")); + assert_eq!(out.len(), limit, "{name}"); + match name { + "NaN offset" => assert!( + out.chunks(4) + .all(|v| f32::from_le_bytes(v.try_into().unwrap()).is_nan()) + ), + "repeated value" => assert!(out.chunks(4).all(|v| v == value)), + _ => { + // Element (i0..i9) lies in block (i0/2, .., i8/2), numbered + // in C order over a 2x..x2x1 grid of blocks. + let mut idx = [0usize; 10]; + for (e, v) in out.chunks(4).enumerate() { + let mut n = e; + for d in (0..10).rev() { + idx[d] = n % shape[d] as usize; + n /= shape[d] as usize; + } + let k = idx[..9].iter().fold(0, |k, &i| k * 2 + i / 2); + assert_eq!( + f32::from_le_bytes(v.try_into().unwrap()), + k as f32, + "{name} {e}" + ); + } + } + } + assert!( + peak <= bound(limit, &f), + "{name}: peak {peak} bytes for a {limit}-byte chunk ({}-byte frame)", + f.len() + ); + } +} + +/// A B2ND chunk larger than the array (here 16x, the old cap) is refused, +/// or at least never allocated. +#[test] +fn b2nd_chunk_larger_than_the_array_is_not_allocated() { + let _g = lock(); + let limit = 1 << 20; + let c = 16 * limit as i32; + let meta = nd_meta(&[limit as i64], &[c], &[c]); + let offsets = repeated(&special_offset(1), 8, 8); + let f = frame(Some(&meta), c as i64, 1, c, &[], &offsets); + let (r, peak) = peak_during(|| blosc2_decompress(&f, limit)); + assert!( + peak <= bound(limit, &f), + "peak {peak} bytes ({:?})", + r.map(|v| v.len()) + ); +}