diff --git a/crates/clawhdf5-format/src/filters_bitshuffle.rs b/crates/clawhdf5-format/src/filters_bitshuffle.rs index d20835e..6b23291 100644 --- a/crates/clawhdf5-format/src/filters_bitshuffle.rs +++ b/crates/clawhdf5-format/src/filters_bitshuffle.rs @@ -218,12 +218,20 @@ pub(crate) fn bitshuffle_decode( } /// Decode Zstandard frames into exactly `dst`, failing if they hold more. +/// +/// ruzstd reserves a frame's declared window (by default up to 100 MiB) +/// before decoding it, so the window is capped at what the output could +/// need: twice `dst` (window sizes are rounded up), and at least 128 KiB. +/// The encoders behind these filters (c-blosc, c-blosc2, bitshuffle) +/// compress each block in one call with its size known, so libzstd's +/// window never exceeds the block. #[cfg(any(feature = "bitshuffle", feature = "blosc"))] pub(crate) fn zstd_decode_into( decoder: &mut ruzstd::decoding::FrameDecoder, frames: &[u8], dst: &mut [u8], ) -> Result { + decoder.set_max_window_size((2 * dst.len()).max(1 << 17) as u64); decoder .decode_all(frames, dst) .map_err(|e| FormatError::DecompressionError(format!("zstd: {e}"))) diff --git a/crates/clawhdf5-format/tests/blosc2_alloc_bounds.rs b/crates/clawhdf5-format/tests/blosc2_alloc_bounds.rs index a6ba114..5dc71b0 100644 --- a/crates/clawhdf5-format/tests/blosc2_alloc_bounds.rs +++ b/crates/clawhdf5-format/tests/blosc2_alloc_bounds.rs @@ -61,9 +61,11 @@ fn peak_during(f: impl FnOnce() -> T) -> (T, usize) { /// What decoding one HDF5 chunk of `limit` bytes from `input` may hold at /// once: the output, a few blocks of scratch (each no larger than the -/// output), the offsets table, and the Zstandard decoder's state. +/// output), the offsets table, and the Zstandard decoder's state, which has +/// a fixed ceiling: a window of at most 128 KiB (or twice the stream) and a +/// block's table of sequences (up to 98,303 of 12 bytes, 1.2 MB). fn bound(limit: usize, input: &[u8]) -> usize { - 6 * limit + 2 * input.len() + (1 << 20) + 6 * limit + 2 * input.len() + (2 << 20) } fn lock() -> std::sync::MutexGuard<'static, ()> { @@ -348,3 +350,48 @@ fn b2nd_chunk_larger_than_the_array_is_not_allocated() { r.map(|v| v.len()) ); } + +/// ruzstd reserves a frame's declared window (up to 100 MiB) before it +/// decodes a frame with a decoder it has used before: a Blosc2 chunk of +/// two 16-byte Zstandard streams, each declaring a 96 MiB window, +/// allocated 96 MiB. c-blosc2 compresses each block with its size known, +/// so its windows never exceed the block. +#[test] +fn zstd_window_is_bounded_by_the_output() { + let _g = lock(); + let mut z = 0xfd2f_b528u32.to_le_bytes().to_vec(); + // No single segment, no checksum; window 2^26 + 4/8 of it = 96 MiB. + z.extend_from_slice(&[0x00, (16 << 3) | 4]); + // One raw block, last, of 16 bytes. + let h = 1 | (16 << 3); + z.extend_from_slice(&[h as u8, (h >> 8) as u8, 0]); + z.extend_from_slice(&[7; 16]); + // Two blocks of 16 bytes, one stream each (not split), Zstandard + // (codec 4). + let chunk = |z: &[u8]| { + let mut c = vec![5u8, 1, 0x10 | (4 << 5), 1]; + for v in [32i32, 16, 0] { + c.extend_from_slice(&v.to_le_bytes()); + } + let first = 24 + 4 + z.len(); + c.extend_from_slice(&24i32.to_le_bytes()); + c.extend_from_slice(&(first as i32).to_le_bytes()); + for _ in 0..2 { + c.extend_from_slice(&(z.len() as i32).to_le_bytes()); + c.extend_from_slice(z); + } + let n = c.len() as i32; + c[12..16].copy_from_slice(&n.to_le_bytes()); + c + }; + let c = chunk(&z); + let (r, peak) = peak_during(|| blosc2_decompress_chunk(&c, 32)); + assert!(peak <= bound(32, &c), "peak {peak} bytes ({r:?})"); + assert!(r.is_err(), "{r:?}"); + // The same streams with a window they can use read. + z[5] = 0; + assert_eq!( + blosc2_decompress_chunk(&chunk(&z), 32).unwrap(), + vec![7; 32] + ); +}