fix(format): an empty Blosc2 chunk no longer allocates its block size
A chunk header with nbytes 0 and no special type kept its declared block size (up to 512 MiB): the block size was clamped to nbytes only when nbytes was positive, and the scratch blocks were allocated before the (empty) block loop, so a 20-byte chunk allocated about 1 GiB. The block size is now clamped to nbytes always, and an empty chunk returns before any scratch is allocated. A frame chunk must also decode to the size the frame header gives it (chunksize, or the remainder for the last chunk), and is decoded with that as its limit, so an empty chunk in a frame for a non-empty HDF5 chunk is an error rather than an empty result. Co-Authored-By: Claude Opus 5.5 (1M context) <[email protected]>
This commit is contained in:
@@ -83,7 +83,13 @@ fn chunk_header(ts: u8, nbytes: i32, blocksize: i32, cbytes: i32, special: u8) -
|
||||
|
||||
/// A chunk of `nbytes` bytes that repeats one value (special type 3).
|
||||
fn repeated(value: &[u8], nbytes: i32, blocksize: i32) -> Vec<u8> {
|
||||
let mut c = chunk_header(value.len() as u8, nbytes, blocksize, 32 + value.len() as i32, 3);
|
||||
let mut c = chunk_header(
|
||||
value.len() as u8,
|
||||
nbytes,
|
||||
blocksize,
|
||||
32 + value.len() as i32,
|
||||
3,
|
||||
);
|
||||
c.extend_from_slice(value);
|
||||
c
|
||||
}
|
||||
@@ -189,3 +195,28 @@ fn small_frames_still_decode() {
|
||||
assert_eq!(blosc2_decompress(&f, 64).unwrap(), vec![0; 64]);
|
||||
let _ = blosc2_decompress_chunk;
|
||||
}
|
||||
|
||||
/// A chunk that decodes to nothing kept its declared block size (up to
|
||||
/// 512 MiB) and allocated two scratch blocks of it: about 1 GiB for a
|
||||
/// 20-byte chunk.
|
||||
#[test]
|
||||
fn empty_chunk_does_not_allocate_its_block_size() {
|
||||
let _g = lock();
|
||||
let mut c = vec![5u8, 1, 0x01, 1];
|
||||
for v in [0i32, 0x1FFF_F000, 20] {
|
||||
c.extend_from_slice(&v.to_le_bytes());
|
||||
}
|
||||
c.resize(20, 0);
|
||||
let (r, peak) = peak_during(|| blosc2_decompress_chunk(&c, 1 << 20));
|
||||
assert_eq!(r.map(|v| v.len()).unwrap_or(0), 0);
|
||||
assert!(
|
||||
peak <= bound(0, &c),
|
||||
"peak {peak} bytes for a 20-byte chunk"
|
||||
);
|
||||
// Inside a frame for a non-empty HDF5 chunk it is an error, not data.
|
||||
let offsets = repeated(&0i64.to_le_bytes(), 8, 8);
|
||||
let f = frame(None, 64, 4, 64, &c, &offsets);
|
||||
let (r, peak) = peak_during(|| blosc2_decompress(&f, 64));
|
||||
assert!(r.is_err(), "decoded {:?}", r.map(|v| v.len()));
|
||||
assert!(peak <= bound(64, &f), "in a frame: peak {peak} bytes");
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user