quantumclaw/clawhdf5
3
1
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 1 Wiki Activity

fix(format): never hold a B2ND chunk's padding

Browse Source 
B2ND chunks were decoded whole, padding included, with up to 16x the HDF5
chunk size as their limit, so a crafted frame made each chunk allocate and
fill up to 16x the output (4 GiB for a 256 MiB HDF5 chunk). The padding is
the real bound (prod(ceil(c/b)*b) per chunk), but that can be 2^ndim times
the array, so it is no longer held at all:

- A Blosc2 chunk is now decoded block by block (decode_blocks), each block
  handed to a sink as it is ready, with at most three blocks of scratch.
  blosc2_decompress_chunk and plain frames still collect every block.
- reassemble places each B2ND block straight into the output and skips
  blocks that are all padding (they are not decoded unless the delta
  filter needs the first block). A frame's NaN chunks are handed over one
  B2ND block at a time and its zero chunks cost nothing.
- A B2ND chunk must decode to exactly its padded size, its Blosc2 blocks
  must be whole B2ND blocks no larger than the output, and a chunk may not
  be larger than the array (hdf5-blosc2's chunk is the array), so a block
  is never larger than the output.

Peak allocation for a 10-D array padded to 13x (NaN, repeated-value and
stored-block chunks) and for a 16x chunk was 4.5 MB and 17.8 MB for
315 KB and 1 MiB outputs before, and is now within the tests' bound.
Blosc2 files written by hdf5plugin in 9-D and 12-D, an 8 MiB single chunk,
1x1x1 and edge-chunk shapes still read exactly.

Co-Authored-By: Claude Opus 5.5 (1M context) <[email protected]>
This commit is contained in:
osobh
2026-09-26 11:30:39 -05:00
co-authored by Claude Opus 5.5
parent e05530a805
commit 22dc87b07c
2 changed files with 408 additions and 97 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/clawhdf5-format/src/filters_blosc2.rs
+271 -88
View File
@@ -225,20 +225,59 @@ fn read_header(src: &[u8]) -> Result<ChunkHeader, FormatError> {
/// Decompress one Blosc2 chunk (header included), refusing to produce more
/// than `limit` bytes.
pub fn blosc2_decompress_chunk(src: &[u8], limit: usize) -> Result<Vec<u8>, FormatError> {
let (h, src) = open_chunk(src, limit)?;
let mut out = vec![0u8; h.nbytes];
decode_blocks(&h, src, &mut |_, _| true, &mut |at, block| {
out[at..at + block.len()].copy_from_slice(block);
})?;
Ok(out)
}
/// Read and check a chunk's header: its decoded size must be within
/// `limit`. Returns the header and the chunk's bytes.
fn open_chunk(src: &[u8], limit: usize) -> Result<(ChunkHeader, &[u8]), FormatError> {
let h = read_header(src)?;
if h.nbytes > limit {
return Err(err("decoded size exceeds the limit"));
}
let src = &src[..h.cbytes];
if h.special > SPECIAL_UNINIT {
return Err(err(&format!("unknown special chunk type {}", h.special)));
}
let memcpyed = h.flags & FLAG_MEMCPYED != 0;
if memcpyed && h.cbytes != h.nbytes + h.overhead {
if h.flags & FLAG_MEMCPYED != 0 && h.cbytes != h.nbytes + h.overhead {
return Err(err("stored chunk has the wrong size"));
}
let src = &src[..h.cbytes];
Ok((h, src))
}
/// Decode a chunk block by block into a destination the caller has
/// zeroed: `sink(offset, block)` receives each decoded block (the last may
/// be short), except blocks for which `want(offset, len)` is false and
/// the blocks of a special all-zeros chunk. Holds at most three blocks of scratch, so a
/// caller that keeps only part of each block (B2ND padding) never holds
/// the whole chunk.
fn decode_blocks(
h: &ChunkHeader,
src: &[u8],
want: &mut dyn FnMut(usize, usize) -> bool,
sink: &mut dyn FnMut(usize, &[u8]),
) -> Result<(), FormatError> {
let nbytes = h.nbytes;
let mut out = vec![0u8; nbytes];
// `read_header` clamps the block size to the decoded size.
let blocksize = h.blocksize;
let nblocks = if nbytes == 0 {
0
} else {
nbytes.div_ceil(blocksize)
};
let leftover = nbytes % blocksize.max(1);
let block_len = |j: usize| {
if j == nblocks - 1 && leftover > 0 {
leftover
} else {
blocksize
}
};
if h.special != 0 {
// Filled per block, as c-blosc2 does: each block must hold whole
// values.
@@ -253,19 +292,30 @@ pub fn blosc2_decompress_chunk(src: &[u8], limit: usize) -> Result<Vec<u8>, Form
};
if let Some(value) = fill {
let ts = value.len();
let blocksize = h.blocksize.max(1);
if !blocksize.is_multiple_of(ts) || !(nbytes % blocksize).is_multiple_of(ts) {
if !blocksize.max(1).is_multiple_of(ts) || !leftover.is_multiple_of(ts) {
return Err(err("special chunk blocks are not whole values"));
}
for v in out.chunks_exact_mut(ts) {
v.copy_from_slice(value);
if nblocks > 0 {
let block: Vec<u8> = value.iter().copied().cycle().take(blocksize).collect();
for j in 0..nblocks {
let len = block_len(j);
if want(j * blocksize, len) {
sink(j * blocksize, &block[..len]);
}
}
return Ok(out);
}
if memcpyed {
out.copy_from_slice(&src[h.overhead..]);
return Ok(out);
}
return Ok(());
}
if h.flags & FLAG_MEMCPYED != 0 {
let data = &src[h.overhead..];
for j in 0..nblocks {
let (at, len) = (j * blocksize, block_len(j));
if want(at, len) {
sink(at, &data[at..at + len]);
}
}
return Ok(());
}
if h.blosc2_flags & B2_USEDICT != 0 {
return Err(err("dictionary-compressed chunks are not supported"));
@@ -289,11 +339,8 @@ pub fn blosc2_decompress_chunk(src: &[u8], limit: usize) -> Result<Vec<u8>, Form
}
}
if nbytes == 0 {
return Ok(out);
return Ok(());
}
let blocksize = h.blocksize;
let nblocks = nbytes.div_ceil(blocksize);
let leftover = nbytes % blocksize;
let bstarts_end = nblocks
.checked_mul(4)
.and_then(|n| n.checked_add(h.overhead))
@@ -302,12 +349,19 @@ pub fn blosc2_decompress_chunk(src: &[u8], limit: usize) -> Result<Vec<u8>, Form
return Err(err("block table is truncated"));
}
let dont_split = h.flags & FLAG_DONT_SPLIT != 0;
// The delta filter's reference is the first block, decoded.
let delta = h.filters.contains(&FILTER_DELTA);
let mut first = Vec::new();
let mut tmp = vec![0u8; blocksize];
let mut tmp2 = vec![0u8; blocksize];
let mut zstd = None;
for j in 0..nblocks {
let is_leftover = j == nblocks - 1 && leftover > 0;
let bsize = if is_leftover { leftover } else { blocksize };
let bsize = block_len(j);
let wanted = want(j * blocksize, bsize);
if !wanted && !(delta && j == 0) {
continue;
}
let start = le_i32(src, h.overhead + 4 * j)?;
if start <= 0 || start as usize >= src.len() {
return Err(err("block start out of range"));
@@ -355,23 +409,21 @@ pub fn blosc2_decompress_chunk(src: &[u8], limit: usize) -> Result<Vec<u8>, Form
}
}
}
let (done, rest) = out.split_at_mut(j * blocksize);
let dest = &mut rest[..bsize];
run_filters_backward(&h, cur, &mut tmp2[..bsize], done, dest);
run_filters_backward(h, cur, &mut tmp2[..bsize], &first);
if delta && j == 0 {
first = cur.to_vec();
}
Ok(out)
if wanted {
sink(j * blocksize, cur);
}
}
Ok(())
}
/// Undo the chunk's filters on one decoded block `cur`, leaving the result
/// in `dest`. `done` is the chunk decoded so far (the delta filter's
/// reference is the first block).
fn run_filters_backward(
h: &ChunkHeader,
cur: &mut [u8],
scratch: &mut [u8],
done: &[u8],
dest: &mut [u8],
) {
/// Undo the chunk's filters on one decoded block `cur`, in place. `first`
/// is the decoded first block (empty while decoding it), the delta
/// filter's reference.
fn run_filters_backward(h: &ChunkHeader, cur: &mut [u8], scratch: &mut [u8], first: &[u8]) {
let bsize = cur.len();
let ts = h.typesize;
for i in (0..6).rev() {
@@ -395,13 +447,12 @@ fn run_filters_backward(
bitunshuffle_block(&cur[..body], &mut scratch[..body], n, ts);
cur[..body].copy_from_slice(&scratch[..body]);
}
FILTER_DELTA => delta_decode(done, cur, ts),
FILTER_DELTA => delta_decode(first, cur, ts),
// Truncating precision is lossy and has nothing to undo.
FILTER_NONE | FILTER_TRUNC_PREC => {}
_ => unreachable!("filters are checked before decoding"),
}
}
dest.copy_from_slice(cur);
}
/// Byte unshuffle (`unshuffle_generic`): trailing bytes that do not make a
@@ -416,18 +467,19 @@ fn unshuffle(bytes: usize, src: &[u8], dest: &mut [u8]) {
dest[n * bytes..].copy_from_slice(&src[n * bytes..]);
}
/// `delta_decoder`: the first block is XOR-accumulated over its own
/// elements; every other block is XORed with the (decoded) first block.
/// `delta_decoder`: the first block (`first` is empty) is XOR-accumulated
/// over its own elements; every other block is XORed with the (decoded)
/// first block.
/// Elements are 1, 2, 4 or 8 bytes wide (other sizes: 8 if a multiple of
/// 8, else 1); a trailing partial element is left alone.
fn delta_decode(done: &[u8], cur: &mut [u8], typesize: usize) {
fn delta_decode(first: &[u8], cur: &mut [u8], typesize: usize) {
let w = match typesize {
1 | 2 | 4 | 8 => typesize,
t if t.is_multiple_of(8) => 8,
_ => 1,
};
let n = cur.len() / w;
if done.is_empty() {
if first.is_empty() {
for i in 1..n {
for b in 0..w {
cur[i * w + b] ^= cur[(i - 1) * w + b];
@@ -435,7 +487,7 @@ fn delta_decode(done: &[u8], cur: &mut [u8], typesize: usize) {
}
} else {
// The first block is at least as long as any other.
for (c, r) in cur[..n * w].iter_mut().zip(done) {
for (c, r) in cur[..n * w].iter_mut().zip(first) {
*c ^= *r;
}
}
@@ -548,14 +600,15 @@ fn parse_frame(buf: &[u8], limit: usize) -> Result<Frame<'_>, FormatError> {
}
impl Frame<'_> {
/// Decode chunk `n`, refusing more than `limit` bytes.
fn chunk(&self, n: usize, limit: usize) -> Result<Vec<u8>, FormatError> {
/// Where chunk `n` is: its size from the frame header (every chunk but
/// the last holds `chunksize` bytes; unknown if that is 0), and either
/// the special value its offset records or its bytes.
fn locate(&self, n: usize) -> Result<(Option<usize>, Located<'_>), FormatError> {
if n >= self.nchunks {
return Err(err("frame has no such chunk"));
}
let raw: [u8; 8] = self.offsets[8 * n..8 * n + 8].try_into().unwrap();
let offset = i64::from_le_bytes(raw);
// Every chunk but the last holds `chunksize` bytes.
let size = if self.chunksize == 0 {
None
} else if n == self.nchunks - 1 && !self.nbytes.is_multiple_of(self.chunksize) {
@@ -565,26 +618,20 @@ impl Frame<'_> {
};
if offset < 0 {
// A special chunk, recorded in the offset's top byte.
let Some(size) = size else {
if size.is_none() {
return Err(err("special chunk in a frame without a chunk size"));
};
if size > limit {
return Err(err("decoded size exceeds the limit"));
}
let kind = raw[7] & 7;
return match kind {
SPECIAL_ZERO | SPECIAL_UNINIT => Ok(vec![0u8; size]),
SPECIAL_NAN => {
let nan: &[u8] = match self.typesize {
SPECIAL_ZERO | SPECIAL_UNINIT => Ok((size, Located::Zeros)),
SPECIAL_NAN => Ok((
size,
Located::Fill(match self.typesize {
4 => &[0x00, 0x00, 0xc0, 0x7f],
8 => &[0, 0, 0, 0, 0, 0, 0xf8, 0x7f],
_ => return Err(err("NaN chunk of a type that is not f32 or f64")),
};
if !size.is_multiple_of(nan.len()) {
return Err(err("NaN chunk is not whole values"));
}
Ok(nan.iter().copied().cycle().take(size).collect())
}
}),
)),
_ => Err(err(&format!("unknown special chunk offset {kind}"))),
};
}
@@ -596,12 +643,90 @@ impl Frame<'_> {
.and_then(|o| self.header_len.checked_add(o))
.filter(|&s| s < end && end - s >= MIN_HEADER)
.ok_or_else(|| err("chunk offset out of range"))?;
let data =
blosc2_decompress_chunk(&self.buf[start..end], limit.min(size.unwrap_or(limit)))?;
if size.is_some_and(|s| s != data.len()) {
Ok((size, Located::Data(&self.buf[start..end])))
}
/// Open chunk `n` (a regular one), which must decode to the frame's
/// size for it, and at most `limit` bytes.
fn open<'s>(
&self,
size: Option<usize>,
src: &'s [u8],
limit: usize,
) -> Result<(ChunkHeader, &'s [u8]), FormatError> {
let (h, src) = open_chunk(src, limit.min(size.unwrap_or(limit)))?;
if size.is_some_and(|s| s != h.nbytes) {
return Err(err("chunk size does not match the frame's chunk size"));
}
Ok(data)
Ok((h, src))
}
/// Decode chunk `n`, refusing more than `limit` bytes.
fn chunk(&self, n: usize, limit: usize) -> Result<Vec<u8>, FormatError> {
let (size, at) = self.locate(n)?;
let src = match at {
Located::Data(src) => src,
Located::Zeros | Located::Fill(_) => {
let size = size.unwrap_or(0);
if size > limit {
return Err(err("decoded size exceeds the limit"));
}
return match at {
Located::Fill(value) if !size.is_multiple_of(value.len()) => {
Err(err("NaN chunk is not whole values"))
}
Located::Fill(value) => Ok(value.iter().copied().cycle().take(size).collect()),
_ => Ok(vec![0u8; size]),
};
}
};
let (h, src) = self.open(size, src, limit)?;
let mut out = vec![0u8; h.nbytes];
decode_blocks(&h, src, &mut |_, _| true, &mut |at, block| {
out[at..at + block.len()].copy_from_slice(block);
})?;
Ok(out)
}
/// Decode chunk `n` (exactly `size` bytes) block by block, as
/// [`decode_blocks`], into a destination the caller has zeroed. Its
/// blocks must be whole multiples of `unit` bytes and at most
/// `max_block`; special chunks are handed over `unit` bytes at a time.
fn chunk_blocks(
&self,
n: usize,
size: usize,
unit: usize,
max_block: usize,
want: &mut dyn FnMut(usize, usize) -> bool,
sink: &mut dyn FnMut(usize, &[u8]),
) -> Result<(), FormatError> {
let (frame_size, at) = self.locate(n)?;
if frame_size.is_some_and(|s| s != size) {
return Err(err("chunk size does not match the b2nd shape"));
}
match at {
Located::Zeros => Ok(()),
Located::Fill(value) => {
if !unit.is_multiple_of(value.len()) || !size.is_multiple_of(unit) {
return Err(err("NaN chunk is not whole values"));
}
let block: Vec<u8> = value.iter().copied().cycle().take(unit).collect();
for k in 0..size / unit {
if want(k * unit, unit) {
sink(k * unit, &block);
}
}
Ok(())
}
Located::Data(src) => {
let (h, src) = self.open(Some(size), src, size)?;
if h.nbytes > 0 && (!h.blocksize.is_multiple_of(unit) || h.blocksize > max_block) {
return Err(err("chunk blocks do not match the b2nd blocks"));
}
decode_blocks(&h, src, want, sink)
}
}
}
/// The content of metalayer `name`, if the frame has it.
@@ -653,6 +778,16 @@ impl Frame<'_> {
}
}
/// Where a frame's chunk is.
enum Located<'a> {
/// All zeros (or uninitialised, read as zeros).
Zeros,
/// One value repeated (NaN).
Fill(&'static [u8]),
/// A Blosc2 chunk, running to the end of the data section.
Data(&'a [u8]),
}
/// The B2ND (or Caterva) metalayer: shape, chunk shape, block shape.
#[derive(Debug, PartialEq, Eq)]
struct NdMeta {
@@ -753,6 +888,12 @@ fn decode_frame(
}
/// Gather a B2ND array's blocks into one C-order buffer.
///
/// Each Blosc2 chunk holds one B2ND chunk padded to whole blocks, which can
/// be many times the array (a crafted frame's shapes are the file's word).
/// So the chunks are never decoded whole: each block is placed in the
/// output as it is decoded, blocks that are all padding are skipped, and
/// no more than a few blocks (each within the array) are held at once.
fn reassemble(frame: &Frame<'_>, nd: &NdMeta, limit: usize) -> Result<Vec<u8>, FormatError> {
let ts = frame.typesize;
let ndim = nd.shape.len();
@@ -778,10 +919,15 @@ fn reassemble(frame: &Frame<'_>, nd: &NdMeta, limit: usize) -> Result<Vec<u8>, F
if total == 0 {
return Ok(out);
}
// Padding to whole blocks grows a chunk by less than 2x per dimension;
// hdf5-blosc2's block shapes keep it far below 16x (a crafted frame
// could otherwise make each chunk enormous).
if ext_bytes > limit.saturating_mul(16) {
// hdf5-blosc2's chunk is the array. A chunk larger than the array
// would be padding decoded for nothing; with chunks and blocks within
// the array, a block is never larger than the output, and a chunk
// pads each dimension by less than one block.
if (0..ndim).any(|i| nd.chunkshape[i] > nd.shape[i]) {
return Err(err("b2nd chunk is larger than the array"));
}
// A Blosc2 chunk of `ext_bytes`, exactly the padded chunk.
if ext_bytes > i32::MAX as usize {
return Err(too_big());
}
let nchunks: usize = grid.iter().product();
@@ -789,7 +935,6 @@ fn reassemble(frame: &Frame<'_>, nd: &NdMeta, limit: usize) -> Result<Vec<u8>, F
return Err(err("chunk count does not match the b2nd shape"));
}
let blocks_in_chunk: Vec<usize> = (0..ndim).map(|i| ext[i] / nd.blockshape[i]).collect();
let nblocks: usize = blocks_in_chunk.iter().product();
let block_items: usize = nd.blockshape.iter().product();
let block_bytes = block_items * ts;
// Strides (in elements) of the output array and of a block.
@@ -799,50 +944,88 @@ fn reassemble(frame: &Frame<'_>, nd: &NdMeta, limit: usize) -> Result<Vec<u8>, F
out_stride[i] = out_stride[i + 1] * nd.shape[i + 1];
blk_stride[i] = blk_stride[i + 1] * nd.blockshape[i + 1];
}
let geometry = BlockGeometry {
nd,
blocks_in_chunk: &blocks_in_chunk,
};
let mut cidx = vec![0usize; ndim];
let mut bidx = vec![0usize; ndim];
let mut gstart = vec![0usize; ndim];
let mut valid = vec![0usize; ndim];
let mut w = BlockPlace::new(ndim);
let mut p = BlockPlace::new(ndim);
let mut pos = vec![0usize; ndim];
for n in 0..nchunks {
unravel(n, &grid, &mut cidx);
let data = frame.chunk(n, ext_bytes)?;
if data.len() != ext_bytes {
return Err(err("chunk size does not match the b2nd shape"));
}
for b in 0..nblocks {
unravel(b, &blocks_in_chunk, &mut bidx);
let mut empty = false;
for i in 0..ndim {
let in_chunk = bidx[i] * nd.blockshape[i];
gstart[i] = cidx[i] * nd.chunkshape[i] + in_chunk;
let lim_chunk = nd.chunkshape[i].saturating_sub(in_chunk);
let lim_shape = nd.shape[i].saturating_sub(gstart[i]);
valid[i] = nd.blockshape[i].min(lim_chunk).min(lim_shape);
empty |= valid[i] == 0;
}
if empty {
let blocks = |at: usize, len: usize| at / block_bytes..(at + len).div_ceil(block_bytes);
let mut want =
|at: usize, len: usize| blocks(at, len).any(|b| geometry.place(b, &cidx, &mut w));
let mut sink = |at: usize, data: &[u8]| {
for b in blocks(at, data.len()) {
if !geometry.place(b, &cidx, &mut p) {
continue;
}
let block = &data[b * block_bytes..(b + 1) * block_bytes];
let block = &data[b * block_bytes - at..][..block_bytes];
// Copy row by row along the last dimension.
let row = valid[ndim - 1] * ts;
let rows: usize = valid[..ndim - 1].iter().product();
let row = p.valid[ndim - 1] * ts;
let rows: usize = p.valid[..ndim - 1].iter().product();
for r in 0..rows {
unravel(r, &valid[..ndim - 1], &mut pos[..ndim - 1]);
unravel(r, &p.valid[..ndim - 1], &mut pos[..ndim - 1]);
let mut src = 0;
let mut dst = gstart[ndim - 1];
let mut dst = p.gstart[ndim - 1];
for i in 0..ndim - 1 {
src += pos[i] * blk_stride[i];
dst += (gstart[i] + pos[i]) * out_stride[i];
dst += (p.gstart[i] + pos[i]) * out_stride[i];
}
out[dst * ts..dst * ts + row].copy_from_slice(&block[src * ts..src * ts + row]);
}
}
};
frame.chunk_blocks(n, ext_bytes, block_bytes, limit, &mut want, &mut sink)?;
}
Ok(out)
}
/// The block layout of a B2ND array's chunks.
struct BlockGeometry<'a> {
nd: &'a NdMeta,
blocks_in_chunk: &'a [usize],
}
/// Where one block of a chunk lands: its first element in the array and
/// how much of it (per dimension) lies within the chunk and the array.
struct BlockPlace {
bidx: Vec<usize>,
gstart: Vec<usize>,
valid: Vec<usize>,
}
impl BlockPlace {
fn new(ndim: usize) -> BlockPlace {
BlockPlace {
bidx: vec![0; ndim],
gstart: vec![0; ndim],
valid: vec![0; ndim],
}
}
}
impl BlockGeometry<'_> {
/// Place block `b` of the chunk at grid index `cidx`; false if it is
/// all padding.
fn place(&self, b: usize, cidx: &[usize], p: &mut BlockPlace) -> bool {
let nd = self.nd;
unravel(b, self.blocks_in_chunk, &mut p.bidx);
let mut empty = false;
for i in 0..nd.shape.len() {
let in_chunk = p.bidx[i] * nd.blockshape[i];
p.gstart[i] = cidx[i] * nd.chunkshape[i] + in_chunk;
let lim_chunk = nd.chunkshape[i].saturating_sub(in_chunk);
let lim_shape = nd.shape[i].saturating_sub(p.gstart[i]);
p.valid[i] = nd.blockshape[i].min(lim_chunk).min(lim_shape);
empty |= p.valid[i] == 0;
}
!empty
}
}
/// C-order multi-index of `n` in a grid of `dims`.
fn unravel(mut n: usize, dims: &[usize], idx: &mut [usize]) {
for i in (0..dims.len()).rev() {
crates/clawhdf5-format/tests/blosc2_alloc_bounds.rs
+128
View File
@@ -220,3 +220,131 @@ fn empty_chunk_does_not_allocate_its_block_size() {
assert!(r.is_err(), "decoded {:?}", r.map(|v| v.len()));
assert!(peak <= bound(64, &f), "in a frame: peak {peak} bytes");
}
/// B2ND chunks were decoded whole, padding included, with up to 16x the
/// HDF5 chunk size as their limit. Blocks are now placed as they are
/// decoded, so the padding is never held.
///
/// Ten dimensions: nine of 3 split into blocks of 2 (padded to 4) and one
/// of 4, so each chunk is 13x the array. One chunk, stored three ways: as a
/// NaN chunk in the frame's offsets, as a repeated-value chunk, and as a
/// chunk of stored (uncompressed) blocks.
#[test]
fn b2nd_padding_is_never_held() {
let _g = lock();
let ts = 4usize;
let mut shape = vec![3i64; 9];
shape.push(4);
let chunks: Vec<i32> = shape.iter().map(|&s| s as i32).collect();
let mut blocks = vec![2i32; 9];
blocks.push(4);
let meta = nd_meta(&shape, &chunks, &blocks);
let items: usize = shape.iter().product::<i64>() as usize;
let limit = items * ts;
let block_bytes = ts * blocks.iter().product::<i32>() as usize;
let ext_bytes = ts * 4usize.pow(9) * 4;
assert!(ext_bytes > 13 * limit);
let offsets = |off: [u8; 8]| repeated(&off, 8, 8);
let value = 1.5f32.to_le_bytes();
let stored = {
// Every block stored raw: block k holds the value k.
let mut c = chunk_header(4, ext_bytes as i32, block_bytes as i32, 0, 0);
c[2] = 0x02 | 0x10; // memcpyed, not split
c.truncate(16);
for k in 0..ext_bytes / block_bytes {
c.extend((k as f32).to_le_bytes().repeat(block_bytes / 4));
}
let n = c.len() as i32;
c[12..16].copy_from_slice(&n.to_le_bytes());
c
};
let cases: Vec<(&str, Vec<u8>)> = vec![
(
"NaN offset",
frame(
Some(&meta),
ext_bytes as i64,
4,
ext_bytes as i32,
&[],
&offsets(special_offset(2)),
),
),
(
"repeated value",
frame(
Some(&meta),
ext_bytes as i64,
4,
ext_bytes as i32,
&repeated(&value, ext_bytes as i32, block_bytes as i32),
&offsets(0i64.to_le_bytes()),
),
),
(
"stored blocks",
frame(
Some(&meta),
ext_bytes as i64,
4,
ext_bytes as i32,
&stored,
&offsets(0i64.to_le_bytes()),
),
),
];
for (name, f) in cases {
let (r, peak) = peak_during(|| blosc2_decompress(&f, limit));
let out = r.unwrap_or_else(|e| panic!("{name}: {e}"));
assert_eq!(out.len(), limit, "{name}");
match name {
"NaN offset" => assert!(
out.chunks(4)
.all(|v| f32::from_le_bytes(v.try_into().unwrap()).is_nan())
),
"repeated value" => assert!(out.chunks(4).all(|v| v == value)),
_ => {
// Element (i0..i9) lies in block (i0/2, .., i8/2), numbered
// in C order over a 2x..x2x1 grid of blocks.
let mut idx = [0usize; 10];
for (e, v) in out.chunks(4).enumerate() {
let mut n = e;
for d in (0..10).rev() {
idx[d] = n % shape[d] as usize;
n /= shape[d] as usize;
}
let k = idx[..9].iter().fold(0, |k, &i| k * 2 + i / 2);
assert_eq!(
f32::from_le_bytes(v.try_into().unwrap()),
k as f32,
"{name} {e}"
);
}
}
}
assert!(
peak <= bound(limit, &f),
"{name}: peak {peak} bytes for a {limit}-byte chunk ({}-byte frame)",
f.len()
);
}
}
/// A B2ND chunk larger than the array (here 16x, the old cap) is refused,
/// or at least never allocated.
#[test]
fn b2nd_chunk_larger_than_the_array_is_not_allocated() {
let _g = lock();
let limit = 1 << 20;
let c = 16 * limit as i32;
let meta = nd_meta(&[limit as i64], &[c], &[c]);
let offsets = repeated(&special_offset(1), 8, 8);
let f = frame(Some(&meta), c as i64, 1, c, &[], &offsets);
let (r, peak) = peak_during(|| blosc2_decompress(&f, limit));
assert!(
peak <= bound(limit, &f),
"peak {peak} bytes ({:?})",
r.map(|v| v.len())
);
}