diff --git a/crates/clawhdf5-format/src/filters_blosc2.rs b/crates/clawhdf5-format/src/filters_blosc2.rs index 71ad4b6..a70a906 100644 --- a/crates/clawhdf5-format/src/filters_blosc2.rs +++ b/crates/clawhdf5-format/src/filters_blosc2.rs @@ -1246,7 +1246,8 @@ mod tests { } /// Random and mutated frames: errors are fine, panics are not, and no - /// output past the limit. + /// output past the limit. Peak allocation is fuzzed separately, with a + /// counting allocator, in `tests/blosc2_alloc_bounds.rs`. #[test] fn fuzzed_frames_never_panic() { let limit = 20_000; diff --git a/crates/clawhdf5-format/tests/blosc2_alloc_bounds.rs b/crates/clawhdf5-format/tests/blosc2_alloc_bounds.rs index 5dc71b0..1667186 100644 --- a/crates/clawhdf5-format/tests/blosc2_alloc_bounds.rs +++ b/crates/clawhdf5-format/tests/blosc2_alloc_bounds.rs @@ -395,3 +395,247 @@ fn zstd_window_is_bounded_by_the_output() { vec![7; 32] ); } + +/// xorshift64*: deterministic, so a failure reproduces. +struct Rng(u64); + +impl Rng { + fn next(&mut self) -> u64 { + let mut x = self.0; + x ^= x >> 12; + x ^= x << 25; + x ^= x >> 27; + self.0 = x; + x.wrapping_mul(0x2545_F491_4F6C_DD1D) + } + + fn below(&mut self, n: usize) -> usize { + (self.next() % n.max(1) as u64) as usize + } + + /// A size that tends to the edges: small, a power of two, huge. + fn size(&mut self) -> i64 { + match self.below(6) { + 0 => self.below(64) as i64, + 1 => 1 << self.below(31), + 2 => i32::MAX as i64 - self.below(4096) as i64, + 3 => (1i64 << self.below(62)) + self.below(8) as i64, + 4 => MAX_BLOCK - self.below(3) as i64, + _ => self.next() as i32 as i64, + } + } +} + +const MAX_BLOCK: i64 = 536_866_816; + +/// One to four edits: bytes, or a size field written little-endian (chunk +/// headers) or big-endian (frame headers), most often at a header's size +/// fields. +fn mutate(rng: &mut Rng, seed: &[u8], data_at: usize) -> Vec { + let mut v = seed.to_vec(); + for _ in 0..1 + rng.below(4) { + let len = v.len(); + if len < 16 { + v.push(rng.next() as u8); + continue; + } + match rng.below(8) { + 0 => { + let i = rng.below(len); + v[i] ^= 1 << rng.below(8); + } + 1 => { + let i = rng.below(len); + v[i] = rng.next() as u8; + } + 2 => { + // Frame header: nbytes, cbytes (i64), typesize, chunksize. + let x = rng.size(); + match rng.below(4) { + 0 if len >= 38 => v[30..38].copy_from_slice(&x.to_be_bytes()), + 1 if len >= 47 => v[39..47].copy_from_slice(&x.to_be_bytes()), + 2 if len >= 52 => v[48..52].copy_from_slice(&(x as i32).to_be_bytes()), + _ if len >= 62 => v[58..62].copy_from_slice(&(x as i32).to_be_bytes()), + _ => {} + } + } + 3 | 4 => { + // A chunk header's nbytes, blocksize or cbytes: in the first + // data chunk, or anywhere (the offsets chunk comes last). + let at = if rng.below(2) == 0 && data_at + 16 <= len { + data_at + 4 * (1 + rng.below(3)) + } else { + rng.below(len - 3) + }; + let x = rng.size() as i32; + v[at..at + 4].copy_from_slice(&x.to_le_bytes()); + } + 5 => v.truncate(rng.below(len)), + 6 => { + let at = rng.below(len); + v[at] = [0x10, 0x20, 0x30, 0x40, 0x05, 0x07, 0x02][rng.below(7)]; + } + _ => { + let i = rng.below(len - 3); + let x = rng.size() as i32; + v[i..i + 4].copy_from_slice(&x.to_be_bytes()); + } + } + } + v +} + +/// Every fixture frame that decodes, with its decoded size. +fn seeds() -> Vec<(Vec, usize)> { + let dir = std::path::Path::new(env!("CARGO_MANIFEST_DIR")).join("tests/fixtures/blosc2"); + let mut v = Vec::new(); + for e in std::fs::read_dir(dir).unwrap() { + let p = e.unwrap().path(); + if p.extension().is_some_and(|x| x == "b2f") + && let Ok(out) = std::fs::read(p.with_extension("out")) + { + v.push((std::fs::read(&p).unwrap(), out.len())); + } + } + v.sort(); + assert!(v.len() >= 20, "fixtures missing"); + v +} + +fn header_len(frame: &[u8]) -> usize { + i32::from_be_bytes(frame[11..15].try_into().unwrap()) as usize +} + +/// Mutated fixture frames, decoded with their HDF5 chunk size as the +/// limit, and their first chunks on their own: whatever they declare, no +/// decode holds more than a small multiple of the output and the input. +#[test] +fn fuzzed_frames_and_chunks_stay_within_the_allocation_bound() { + let _g = lock(); + let seeds = seeds(); + let mut rng = Rng(0xb2a1); + let mut worst = (0.0f64, String::new()); + for i in 0..20_000 { + let (seed, limit) = &seeds[rng.below(seeds.len())]; + let f = mutate(&mut rng, seed, header_len(seed)); + let (r, peak) = peak_during(|| blosc2_decompress(&f, *limit)); + if let Ok(out) = &r { + assert!(out.len() <= *limit, "iteration {i}: output past the limit"); + } + assert!( + peak <= bound(*limit, &f), + "iteration {i}: peak {peak} bytes for a {limit}-byte chunk from {} bytes ({:?})", + f.len(), + r.map(|v| v.len()) + ); + let ratio = peak as f64 / bound(*limit, &f) as f64; + if ratio > worst.0 { + worst = ( + ratio, + format!( + "frame iteration {i}: peak {peak}, limit {limit}, input {}", + f.len() + ), + ); + } + } + for i in 0..20_000 { + let (seed, _) = &seeds[rng.below(seeds.len())]; + let at = header_len(seed); + let chunk = &seed[at..]; + let c = mutate(&mut rng, chunk, 0); + let limit = 1 << 16; + let (r, peak) = peak_during(|| blosc2_decompress_chunk(&c, limit)); + assert!( + peak <= bound(limit, &c), + "chunk iteration {i}: peak {peak} bytes from {} bytes ({:?})", + c.len(), + r.map(|v| v.len()) + ); + } + eprintln!("worst peak / bound: {:.2} ({})", worst.0, worst.1); +} + +/// Frames built from random header sizes, offsets chunks and B2ND shapes +/// (chunk and block shapes that pad, special and repeated-value chunks). +#[test] +fn random_frames_stay_within_the_allocation_bound() { + let _g = lock(); + let mut rng = Rng(0xb2a2); + for i in 0..5_000 { + let ts = [1usize, 2, 4, 8][rng.below(4)]; + let ndim = 1 + rng.below(8); + let mut shape = Vec::new(); + let mut chunks = Vec::new(); + let mut blocks = Vec::new(); + for _ in 0..ndim { + let s = 1 + rng.below(if ndim > 3 { 4 } else { 40 }); + let c = if rng.below(8) == 0 { + s * (1 + rng.below(4)) + } else { + 1 + rng.below(s) + }; + let b = 1 + rng.below(c); + shape.push(s as i64); + chunks.push(c as i32); + blocks.push(b as i32); + } + let items: usize = shape.iter().product::() as usize; + let limit = items * ts; + let meta = nd_meta(&shape, &chunks, &blocks); + let ext: usize = ts + * chunks + .iter() + .zip(&blocks) + .map(|(&c, &b)| (c as usize).div_ceil(b as usize) * b as usize) + .product::(); + let nchunks: usize = shape + .iter() + .zip(&chunks) + .map(|(&s, &c)| (s as usize).div_ceil(c as usize)) + .product(); + let block_bytes = ts * blocks.iter().product::() as usize; + let chunksize = if rng.below(4) == 0 { + rng.size() + } else { + ext as i64 + }; + let nbytes = if rng.below(4) == 0 { + rng.size() + } else { + (nchunks * ext) as i64 + }; + let off_n = if rng.below(4) == 0 { + rng.size() as i32 + } else { + 8 * nchunks as i32 + }; + let (data, off) = match rng.below(3) { + 0 => (Vec::new(), special_offset(1 + rng.below(2) as u8)), + _ => { + let bs = if rng.below(4) == 0 { + rng.size() as i32 + } else { + block_bytes as i32 + }; + let value: Vec = (0..ts).map(|_| rng.next() as u8).collect(); + let n = if rng.below(4) == 0 { + rng.size() as i32 + } else { + ext as i32 + }; + (repeated(&value, n, bs), 0i64.to_le_bytes()) + } + }; + let offsets = repeated(&off, off_n, off_n.clamp(1, 8)); + let meta = (rng.below(4) != 0).then_some(meta.as_slice()); + let f = frame(meta, nbytes, ts as i32, chunksize as i32, &data, &offsets); + let (r, peak) = peak_during(|| blosc2_decompress(&f, limit)); + assert!( + peak <= bound(limit, &f), + "iteration {i}: peak {peak} bytes for a {limit}-byte chunk ({:?}, shape {shape:?} \ + chunks {chunks:?} blocks {blocks:?})", + r.map(|v| v.len()) + ); + } +}