fix(format): bound a Blosc2 frame's offsets chunk by the HDF5 chunk size
parse_frame sized the offsets chunk from the frame header's own nbytes and chunksize, so a 173-byte frame declaring 32 Mi chunks, with a 40-byte repeated-value offsets chunk, built 256 MiB (up to 2 GiB) of offsets for a 1 MiB HDF5 chunk and then returned 4 bytes. The offsets chunk is now capped at the output limit (at least 128 bytes); a frame whose nbytes/chunksize imply more chunks than that is refused before anything is allocated. tests/blosc2_alloc_bounds.rs measures peak allocation with a counting global allocator; the reviewer's frame failed it (decoded Ok(4)) before. Co-Authored-By: Claude Opus 5.5 (1M context) <[email protected]>
This commit is contained in:
@@ -451,7 +451,9 @@ struct Frame<'a> {
|
||||
offsets: Vec<u8>,
|
||||
}
|
||||
|
||||
fn parse_frame(buf: &[u8]) -> Result<Frame<'_>, FormatError> {
|
||||
/// Parse a frame's header and decode its offsets chunk, holding no more
|
||||
/// than an HDF5 chunk of `limit` bytes needs.
|
||||
fn parse_frame(buf: &[u8], limit: usize) -> Result<Frame<'_>, FormatError> {
|
||||
if buf.len() < FRAME_HEADER_MINLEN {
|
||||
return Err(err("truncated frame header"));
|
||||
}
|
||||
@@ -497,29 +499,33 @@ fn parse_frame(buf: &[u8]) -> Result<Frame<'_>, FormatError> {
|
||||
.ok_or_else(|| err("chunks run past the frame"))?;
|
||||
let nbytes = usize::try_from(nbytes).map_err(|_| err("bad decoded size"))?;
|
||||
let chunksize = chunksize as usize;
|
||||
// The offsets chunk follows the data chunks.
|
||||
// The offsets chunk follows the data chunks: one `i64` per chunk. The
|
||||
// frame header's sizes are the file's word, so they must not size it:
|
||||
// it may be no larger than the HDF5 chunk (`limit`, at least 128
|
||||
// bytes), i.e. one Blosc2 chunk per 8 bytes of output. hdf5-blosc2
|
||||
// writes one chunk per frame; only python-blosc2 arrays of tiny
|
||||
// chunks come near the cap.
|
||||
let off_src = &buf[data_end..];
|
||||
let expected = if nbytes == 0 {
|
||||
let max_offsets = limit.max(128);
|
||||
let off_len = if nbytes == 0 {
|
||||
0
|
||||
} else if chunksize > 0 {
|
||||
nbytes.div_ceil(chunksize)
|
||||
nbytes
|
||||
.div_ceil(chunksize)
|
||||
.checked_mul(8)
|
||||
.filter(|&n| n <= max_offsets)
|
||||
.ok_or_else(|| err("frame has more chunks than the HDF5 chunk can hold"))?
|
||||
} else {
|
||||
// Variable-size chunks: the offsets chunk tells how many.
|
||||
usize::MAX
|
||||
};
|
||||
let off_limit = if expected == usize::MAX {
|
||||
1 << 24
|
||||
} else {
|
||||
expected
|
||||
.checked_mul(8)
|
||||
.ok_or_else(|| err("too many chunks"))?
|
||||
max_offsets
|
||||
};
|
||||
let offsets = if nbytes == 0 {
|
||||
Vec::new()
|
||||
} else {
|
||||
blosc2_decompress_chunk(off_src, off_limit)?
|
||||
blosc2_decompress_chunk(off_src, off_len)?
|
||||
};
|
||||
if !offsets.len().is_multiple_of(8) || (expected != usize::MAX && offsets.len() != off_limit) {
|
||||
let expected = if chunksize > 0 { off_len } else { offsets.len() };
|
||||
if !offsets.len().is_multiple_of(8) || offsets.len() != expected {
|
||||
return Err(err("offsets chunk does not match the number of chunks"));
|
||||
}
|
||||
Ok(Frame {
|
||||
@@ -711,7 +717,7 @@ fn decode_frame(
|
||||
limit: usize,
|
||||
cd_shape: Option<&[usize]>,
|
||||
) -> Result<Vec<u8>, FormatError> {
|
||||
let frame = parse_frame(input)?;
|
||||
let frame = parse_frame(input, limit)?;
|
||||
let meta = match frame.metalayer(b"b2nd")? {
|
||||
Some(m) => Some(m),
|
||||
None => frame.metalayer(b"caterva")?,
|
||||
@@ -1065,7 +1071,7 @@ mod tests {
|
||||
if w.is_err() {
|
||||
continue;
|
||||
}
|
||||
let frame = parse_frame(&f).unwrap();
|
||||
let frame = parse_frame(&f, 1 << 20).unwrap();
|
||||
let raw: [u8; 8] = frame.offsets[..8].try_into().unwrap();
|
||||
let off = i64::from_le_bytes(raw);
|
||||
if off >= 0 {
|
||||
|
||||
Reference in New Issue
Block a user