fix(format): bound a Blosc2 frame's offsets chunk by the HDF5 chunk size

parse_frame sized the offsets chunk from the frame header's own nbytes and
chunksize, so a 173-byte frame declaring 32 Mi chunks, with a 40-byte
repeated-value offsets chunk, built 256 MiB (up to 2 GiB) of offsets for a
1 MiB HDF5 chunk and then returned 4 bytes. The offsets chunk is now capped
at the output limit (at least 128 bytes); a frame whose nbytes/chunksize
imply more chunks than that is refused before anything is allocated.

tests/blosc2_alloc_bounds.rs measures peak allocation with a counting
global allocator; the reviewer's frame failed it (decoded Ok(4)) before.

Co-Authored-By: Claude Opus 5.5 (1M context) <[email protected]>
This commit is contained in:
osobh
2026-09-26 11:26:28 -05:00
co-authored by Claude Opus 5.5
parent 56abaec75e
commit 989335b67b
2 changed files with 213 additions and 16 deletions
+22 -16
View File
@@ -451,7 +451,9 @@ struct Frame<'a> {
offsets: Vec<u8>,
}
fn parse_frame(buf: &[u8]) -> Result<Frame<'_>, FormatError> {
/// Parse a frame's header and decode its offsets chunk, holding no more
/// than an HDF5 chunk of `limit` bytes needs.
fn parse_frame(buf: &[u8], limit: usize) -> Result<Frame<'_>, FormatError> {
if buf.len() < FRAME_HEADER_MINLEN {
return Err(err("truncated frame header"));
}
@@ -497,29 +499,33 @@ fn parse_frame(buf: &[u8]) -> Result<Frame<'_>, FormatError> {
.ok_or_else(|| err("chunks run past the frame"))?;
let nbytes = usize::try_from(nbytes).map_err(|_| err("bad decoded size"))?;
let chunksize = chunksize as usize;
// The offsets chunk follows the data chunks.
// The offsets chunk follows the data chunks: one `i64` per chunk. The
// frame header's sizes are the file's word, so they must not size it:
// it may be no larger than the HDF5 chunk (`limit`, at least 128
// bytes), i.e. one Blosc2 chunk per 8 bytes of output. hdf5-blosc2
// writes one chunk per frame; only python-blosc2 arrays of tiny
// chunks come near the cap.
let off_src = &buf[data_end..];
let expected = if nbytes == 0 {
let max_offsets = limit.max(128);
let off_len = if nbytes == 0 {
0
} else if chunksize > 0 {
nbytes.div_ceil(chunksize)
nbytes
.div_ceil(chunksize)
.checked_mul(8)
.filter(|&n| n <= max_offsets)
.ok_or_else(|| err("frame has more chunks than the HDF5 chunk can hold"))?
} else {
// Variable-size chunks: the offsets chunk tells how many.
usize::MAX
};
let off_limit = if expected == usize::MAX {
1 << 24
} else {
expected
.checked_mul(8)
.ok_or_else(|| err("too many chunks"))?
max_offsets
};
let offsets = if nbytes == 0 {
Vec::new()
} else {
blosc2_decompress_chunk(off_src, off_limit)?
blosc2_decompress_chunk(off_src, off_len)?
};
if !offsets.len().is_multiple_of(8) || (expected != usize::MAX && offsets.len() != off_limit) {
let expected = if chunksize > 0 { off_len } else { offsets.len() };
if !offsets.len().is_multiple_of(8) || offsets.len() != expected {
return Err(err("offsets chunk does not match the number of chunks"));
}
Ok(Frame {
@@ -711,7 +717,7 @@ fn decode_frame(
limit: usize,
cd_shape: Option<&[usize]>,
) -> Result<Vec<u8>, FormatError> {
let frame = parse_frame(input)?;
let frame = parse_frame(input, limit)?;
let meta = match frame.metalayer(b"b2nd")? {
Some(m) => Some(m),
None => frame.metalayer(b"caterva")?,
@@ -1065,7 +1071,7 @@ mod tests {
if w.is_err() {
continue;
}
let frame = parse_frame(&f).unwrap();
let frame = parse_frame(&f, 1 << 20).unwrap();
let raw: [u8; 8] = frame.offsets[..8].try_into().unwrap();
let off = i64::from_le_bytes(raw);
if off >= 0 {