From 7acfb795848704db88c171a6f3110752ce83e668 Mon Sep 17 00:00:00 2001 From: osobh Date: Sat, 26 Sep 2026 10:03:22 -0500 Subject: [PATCH 01/43] writer: order dense name indexes by hash, then name libhdf5 compares the name when two hashes are equal; the writer broke ties by insertion order, and libhdf5 could not find one of two names whose lookup3 hashes collide (k69209 / k155448). Test fails before the fix with h5py's KeyError. Co-Authored-By: Claude Opus 5.5 (1M context) --- CHANGELOG.md | 11 +++++ crates/clawhdf5-format/src/file_writer.rs | 43 ++++++++++++------- .../clawhdf5/tests/writer_groups_interop.rs | 39 +++++++++++++++++ 3 files changed, 77 insertions(+), 16 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 29804fc..0e50755 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,17 @@ ## Unreleased +### Writer: large dense indexes (2026-09-26) +- **Links and attributes whose name hashes collide are found by name.** The + dense name indexes (a group's links: B-tree v2 type 5; an object's + attributes: type 8) are ordered by the name's lookup3 hash and, when two + hashes are equal, by the name itself, as libhdf5 compares them. The writer + broke ties by insertion order, so libhdf5 could not open one of two + colliding names (`"k69209"` and `"k155448"` share hash `0x3a0b13e6`; + collisions are likely from about 77 000 names). Regression test + `names_whose_hashes_collide_are_found_by_name` in + `crates/clawhdf5/tests/writer_groups_interop.rs`. + ### Concurrent reads (2026-09-26) - **Full reads of chunked datasets scale with threads again when rayon's pool has one thread.** Each full read handed its chunks to rayon to diff --git a/crates/clawhdf5-format/src/file_writer.rs b/crates/clawhdf5-format/src/file_writer.rs index 11345a1..80a296b 100644 --- a/crates/clawhdf5-format/src/file_writer.rs +++ b/crates/clawhdf5-format/src/file_writer.rs @@ -910,20 +910,27 @@ pub(crate) fn build_dense_attrs( let heap_id_length = heap.heap_id_length; let heap_ids = &heap.heap_ids; - // Build B-tree v2 type 8 records (17 bytes each) + // Build B-tree v2 type 8 records (17 bytes each), in the index's key + // order: libhdf5 compares the name hash, then — for names whose hashes + // collide — the names themselves (`strcmp`). let record_size: u16 = heap_id_length + 1 + 4 + 4; - let mut records: Vec<(u32, u32, Vec)> = Vec::with_capacity(attrs.len()); - for (i, heap_id) in heap_ids.iter().enumerate() { - let mut rec = Vec::with_capacity(record_size as usize); - rec.extend_from_slice(heap_id); - rec.push(0); // msg_flags - rec.extend_from_slice(&(i as u32).to_le_bytes()); // creation_order - rec.extend_from_slice(&name_hashes[i].to_le_bytes()); // hash - records.push((name_hashes[i], i as u32, rec)); - } - records.sort_by(|a, b| a.0.cmp(&b.0).then(a.1.cmp(&b.1))); - - let records: Vec> = records.into_iter().map(|(_, _, rec)| rec).collect(); + let mut order: Vec = (0..attrs.len()).collect(); + order.sort_by(|&a, &b| { + name_hashes[a] + .cmp(&name_hashes[b]) + .then_with(|| attrs[a].name.as_bytes().cmp(attrs[b].name.as_bytes())) + }); + let records: Vec> = order + .into_iter() + .map(|i| { + let mut rec = Vec::with_capacity(record_size as usize); + rec.extend_from_slice(&heap_ids[i]); + rec.push(0); // msg_flags + rec.extend_from_slice(&(i as u32).to_le_bytes()); // creation_order + rec.extend_from_slice(&name_hashes[i].to_le_bytes()); // hash + rec + }) + .collect(); let bthd_addr = btree_addr; let mut blob = heap.blob; blob.extend_from_slice(&single_leaf_v2_btree( @@ -1039,14 +1046,18 @@ pub(crate) fn build_dense_links( let heap = build_single_block_fractal_heap(&serialized, base_address, 32, 7)?; let heap_id_length = heap.heap_id_length; - // Type 5 records: hash(4) + heap_id. The B-tree's key is the name hash, - // so records are sorted by (hash, order). + // Type 5 records: hash(4) + heap_id. The B-tree's key is the name hash + // and, for names whose hashes collide, the name (libhdf5 compares them + // with `strcmp`): records out of that order are not found by name. let mut by_name: Vec<(u32, usize)> = links .iter() .enumerate() .map(|(i, l)| (crate::checksum::jenkins_lookup3(l.name.as_bytes()), i)) .collect(); - by_name.sort_unstable(); + by_name.sort_unstable_by(|&(ha, a), &(hb, b)| { + ha.cmp(&hb) + .then_with(|| links[a].name.as_bytes().cmp(links[b].name.as_bytes())) + }); let name_records: Vec> = by_name .iter() .map(|&(hash, i)| { diff --git a/crates/clawhdf5/tests/writer_groups_interop.rs b/crates/clawhdf5/tests/writer_groups_interop.rs index 97e3544..46d155b 100644 --- a/crates/clawhdf5/tests/writer_groups_interop.rs +++ b/crates/clawhdf5/tests/writer_groups_interop.rs @@ -627,6 +627,45 @@ fn non_ascii_names_are_utf8() { assert_eq!(f.dataset("größe/wert").unwrap().read_i32().unwrap(), [1]); } +#[test] +fn names_whose_hashes_collide_are_found_by_name() { + skip_if_no_python!(); + // "k69209" and "k155448" have the same lookup3 hash (0x3a0b13e6). The + // dense name indexes (links: type 5, attributes: type 8) are ordered by + // hash and then by name, and libhdf5's lookup relies on it. The writer + // broke ties by insertion order, so with "k69209" added first libhdf5 + // could not open "k155448" by name. + let dir = tempfile::tempdir().unwrap(); + let mut b = FileBuilder::new(); + let mut g = b.create_group("g"); + for (i, n) in ["k69209", "k155448"] + .into_iter() + .chain((0..10).map(|_| "")) + .enumerate() + { + let name = if n.is_empty() { format!("d{i}") } else { n.into() }; + g.create_dataset(&name).with_i32_data(&[i as i32]); + } + b.add_group(g.finish()); + let x = b.create_dataset("x"); + x.with_i32_data(&[0]); + x.set_attr("k69209", AttrValue::I64(1)); + x.set_attr("k155448", AttrValue::I64(2)); + for i in 0..10 { + x.set_attr(&format!("a{i}"), AttrValue::I64(10 + i)); + } + let path = write(&dir, "collide.h5", b); + let out = h5py( + &path, + "with h5py.File(path, 'r') as f:\n\ + \x20 g, a = f['g'], f['x'].attrs\n\ + \x20 print(json.dumps([int(g['k69209'][0]), int(g['k155448'][0]), 'k155448' in g,\n\ + \x20 int(a['k69209']), int(a['k155448']), 'k155448' in a]))", + ); + assert_eq!(out, "[0, 1, true, 1, 2, true]"); + h5dump_ok(&path); +} + #[test] fn a_group_attribute_set_again_takes_the_new_value() { skip_if_no_python!(); From cc1c872a93b03db0046e5e5fe48d1afa5fc6b58a Mon Sep 17 00:00:00 2001 From: osobh Date: Sat, 26 Sep 2026 10:05:31 -0500 Subject: [PATCH 02/43] format: decode chunks into reusable scratch buffers decompress_chunk_exact_with decodes a chunk into a DecodeScratch the caller keeps between chunks, instead of a new Vec per chunk and per filter stage. Deflate inflates into a kept buffer with a reset (not rebuilt) inflater, shuffle interleaves into the other buffer, and Fletcher32 checks and drops its checksum in place (on the stored bytes when it is the first filter undone). Other filters go through the registry as before. Output and errors are those of decompress_chunk_exact; a unit test checks that for every pipeline shape and filter mask with one reused scratch. Co-Authored-By: Claude Opus 5.5 (1M context) --- crates/clawhdf5-format/src/filters.rs | 363 ++++++++++++++++++++++++-- 1 file changed, 342 insertions(+), 21 deletions(-) diff --git a/crates/clawhdf5-format/src/filters.rs b/crates/clawhdf5-format/src/filters.rs index 36f2e96..872b8d8 100644 --- a/crates/clawhdf5-format/src/filters.rs +++ b/crates/clawhdf5-format/src/filters.rs @@ -148,6 +148,180 @@ pub fn decompress_chunk_exact( Ok(data) } +/// Buffers a chunk decoder keeps between chunks, so decoding a dataset's +/// chunks one after another reuses the same memory instead of allocating +/// (and faulting in) fresh buffers for every chunk and every filter stage. +/// +/// Use one per thread with [`decompress_chunk_exact_with`]. Buffers larger +/// than [`DecodeScratch::RETAIN_BYTES`] are released by +/// [`DecodeScratch::trim`], so a scratch kept for a long time (a +/// thread-local, say) does not hold on to a huge chunk's memory. +#[derive(Default)] +pub struct DecodeScratch { + a: Vec, + b: Vec, + #[cfg(feature = "deflate")] + inflater: Option, +} + +impl core::fmt::Debug for DecodeScratch { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + f.debug_struct("DecodeScratch") + .field("a_capacity", &self.a.capacity()) + .field("b_capacity", &self.b.capacity()) + .finish_non_exhaustive() + } +} + +/// Which buffer holds the data between two filter stages. +#[derive(Clone, Copy)] +enum Stage { + /// `compressed[..len]`: still (a prefix of) the stored bytes. + Stored(usize), + A, + B, +} + +impl DecodeScratch { + /// Largest buffer [`trim`](Self::trim) keeps (4 MiB). + pub const RETAIN_BYTES: usize = 4 << 20; + + /// An empty scratch; buffers are allocated on first use. + pub fn new() -> Self { + Self::default() + } + + /// Release any buffer larger than [`Self::RETAIN_BYTES`]. + pub fn trim(&mut self) { + for buf in [&mut self.a, &mut self.b] { + if buf.capacity() > Self::RETAIN_BYTES { + *buf = Vec::new(); + } + } + } +} + +/// [`decompress_chunk_exact`] into reusable buffers: the decoded chunk is +/// returned as a slice of `scratch` (or of `compressed`, when every filter +/// that was applied only appended a checksum), valid until `scratch` is used +/// again. +/// +/// Deflate, shuffle and Fletcher32 — h5py's and libhdf5's usual pipeline — +/// decode without allocating once `scratch` has grown to the chunk size: the +/// inflater writes into a kept buffer (and its state is reset, not +/// rebuilt), shuffle interleaves into the other buffer, and Fletcher32 checks +/// the checksum and drops it in place. Every other filter goes through the +/// filter registry as [`decompress_chunk_masked`] does, and its output +/// replaces a scratch buffer. The result is byte for byte what +/// [`decompress_chunk_exact`] returns, with the same errors. +pub fn decompress_chunk_exact_with<'s>( + compressed: &'s [u8], + pipeline: &FilterPipeline, + chunk_size: usize, + element_size: u32, + filter_mask: u32, + coords: &[u64], + scratch: &'s mut DecodeScratch, +) -> Result<&'s [u8], FormatError> { + // Same per-stage bounds as `decompress_chunk_masked`. + let mut bounds = [0usize; 32]; + let mut bounds_vec = Vec::new(); + let bounds: &mut [usize] = if pipeline.filters.len() <= bounds.len() { + &mut bounds[..pipeline.filters.len()] + } else { + bounds_vec.resize(pipeline.filters.len(), 0); + &mut bounds_vec + }; + let mut size = chunk_size; + for (i, filter) in pipeline.filters.iter().enumerate() { + bounds[i] = size; + if !filter_skipped(filter_mask, i) { + size = filter_output_bound(filter.filter_id, size); + } + } + + let mut stage = Stage::Stored(compressed.len()); + for (i, filter) in pipeline.filters.iter().enumerate().rev() { + if filter_skipped(filter_mask, i) { + continue; + } + let ctx = FilterContext { + filter, + element_size: element_size as usize, + max_output: bounds[i], + }; + // The stage's input, and the buffer its output goes to (the one + // not holding the input). + let (input, out): (&[u8], &mut Vec) = match stage { + Stage::Stored(len) => (&compressed[..len], &mut scratch.a), + Stage::A => (&scratch.a, &mut scratch.b), + Stage::B => (&scratch.b, &mut scratch.a), + }; + let next = match stage { + Stage::Stored(_) | Stage::B => Stage::A, + Stage::A => Stage::B, + }; + match filter.filter_id { + // Built in and never overridable (`register_filter` refuses + // built-in IDs), so the registry would pick exactly these. + FILTER_FLETCHER32 => { + // Check and drop the checksum where the data is. + let payload = fletcher32_payload(input)?; + stage = match stage { + Stage::Stored(_) => Stage::Stored(payload), + Stage::A => { + scratch.a.truncate(payload); + Stage::A + } + Stage::B => { + scratch.b.truncate(payload); + Stage::B + } + }; + continue; + } + FILTER_SHUFFLE => shuffle_decompress_into(input, ctx.element_size, out), + #[cfg(all( + feature = "deflate", + not(all(target_os = "macos", feature = "system-zlib-decompress")) + ))] + FILTER_DEFLATE => { + let limit = if ctx.max_output != 0 { + ctx.max_output + } else { + MAX_DECOMPRESS_SIZE + }; + let size_hint = if ctx.max_output != 0 { + ctx.max_output + } else { + input.len().saturating_mul(4).min(1 << 20) + }; + let inflater = scratch + .inflater + .get_or_insert_with(|| flate2::Decompress::new(true)); + inflater.reset(true); + inflate_bounded_into(inflater, input, size_hint, limit, out) + .map_err(FormatError::DecompressionError)?; + } + _ => *out = filter_registry::decode(input, &ctx)?, + } + stage = next; + } + + let data: &[u8] = match stage { + Stage::Stored(len) => &compressed[..len], + Stage::A => &scratch.a, + Stage::B => &scratch.b, + }; + if chunk_size != 0 && data.len() != chunk_size { + return Err(FormatError::ChunkedReadError(format!( + "chunk at {coords:?} decoded to {} bytes, expected {chunk_size}", + data.len() + ))); + } + Ok(data) +} + /// Apply a filter pipeline to compress a chunk. /// Filters are applied in FORWARD order for compression. pub fn compress_chunk( @@ -892,33 +1066,55 @@ pub(crate) fn inflate_bounded( size_hint: usize, limit: usize, ) -> Result, String> { - use flate2::{Decompress, FlushDecompress, Status}; + let mut out = Vec::new(); + inflate_bounded_into( + &mut flate2::Decompress::new(true), + data, + size_hint, + limit, + &mut out, + )?; + Ok(out) +} + +/// [`inflate_bounded`] with a fresh or reset `inflater`, into `out`: its +/// contents are replaced and its allocation reused. +#[cfg(feature = "deflate")] +fn inflate_bounded_into( + inflater: &mut flate2::Decompress, + data: &[u8], + size_hint: usize, + limit: usize, + out: &mut Vec, +) -> Result<(), String> { + use flate2::{FlushDecompress, Status}; // One byte of headroom past the limit distinguishes an over-size stream // from one that legitimately ends exactly at the limit. let max_capacity = limit.saturating_add(1); - let mut out = Vec::new(); - out.try_reserve_exact(size_hint.clamp(1, max_capacity)) + // A kept buffer may already be larger than `max_capacity`; the decoder + // can then write past the limit, which the check below still refuses. + out.clear(); + let want = size_hint.clamp(1, max_capacity); + out.try_reserve_exact(want) .map_err(|e| format!("deflate: cannot allocate output: {e}"))?; - let mut inflater = Decompress::new(true); loop { let (in_before, out_before) = (inflater.total_in(), inflater.total_out()); let status = inflater - .decompress_vec( - &data[in_before as usize..], - &mut out, - FlushDecompress::Finish, - ) + .decompress_vec(&data[in_before as usize..], out, FlushDecompress::Finish) .map_err(|e| format!("deflate: {e}"))?; if out.len() > limit { return Err("deflate: output exceeds size limit".into()); } match status { - Status::StreamEnd => return Ok(out), + Status::StreamEnd => return Ok(()), Status::Ok | Status::BufError if out.len() == out.capacity() => { // Out of room: double, up to the limit. - let grow = out.capacity().min(max_capacity - out.capacity()).max(1); + let grow = out + .capacity() + .min(max_capacity.saturating_sub(out.capacity())) + .max(1); out.try_reserve_exact(grow) .map_err(|e| format!("deflate: cannot allocate output: {e}"))?; } @@ -1217,16 +1413,30 @@ fn zstd_compress(data: &[u8], level: u32) -> Result, FormatError> { /// On disk: all byte-0s of each element together, then all byte-1s, etc. /// Output: elements in natural order. fn shuffle_decompress(data: &[u8], element_size: usize) -> Result, FormatError> { + let mut result = Vec::new(); + shuffle_decompress_into(data, element_size, &mut result); + Ok(result) +} + +/// [`shuffle_decompress`] into `result`, replacing its contents and reusing +/// its allocation. +fn shuffle_decompress_into(data: &[u8], element_size: usize, result: &mut Vec) { if element_size <= 1 { - return Ok(data.to_vec()); + result.clear(); + result.extend_from_slice(data); + return; } // Like libhdf5, only whole elements are shuffled; trailing bytes (e.g. a // Fletcher32 checksum appended before the shuffle) are stored as-is. let whole = data.len() - data.len() % element_size; let (data, tail) = data.split_at(whole); let num_elements = data.len() / element_size; - let mut result = vec![0u8; whole]; - result.reserve_exact(tail.len()); + // Every byte of `result[..whole]` is overwritten below, so a reused + // buffer keeps its old bytes instead of being zeroed first; only growth + // is zero-filled. + result.truncate(whole); + result.reserve_exact(whole + tail.len() - result.len()); + result.resize(whole, 0); // The shuffled stream is `element_size` byte planes of `num_elements` // bytes each; un-shuffling interleaves them. This is on the read path of @@ -1245,10 +1455,10 @@ fn shuffle_decompress(data: &[u8], element_size: usize) -> Result, Forma } } match element_size { - 2 => interleave::<2>(data, num_elements, &mut result), - 4 => interleave::<4>(data, num_elements, &mut result), - 8 => interleave::<8>(data, num_elements, &mut result), - 16 => interleave::<16>(data, num_elements, &mut result), + 2 => interleave::<2>(data, num_elements, result), + 4 => interleave::<4>(data, num_elements, result), + 8 => interleave::<8>(data, num_elements, result), + 16 => interleave::<16>(data, num_elements, result), _ => { for (i, element) in result.chunks_exact_mut(element_size).enumerate() { for (j, byte) in element.iter_mut().enumerate() { @@ -1258,8 +1468,6 @@ fn shuffle_decompress(data: &[u8], element_size: usize) -> Result, Forma } } result.extend_from_slice(tail); - - Ok(result) } /// Shuffle (compress direction): group bytes by position within each element. @@ -1411,6 +1619,12 @@ fn fletcher32_compute(data: &[u8]) -> u32 { /// Verify Fletcher32 checksum and strip it from the data. /// The last 4 bytes are the stored checksum. fn fletcher32_verify(data: &[u8]) -> Result, FormatError> { + fletcher32_payload(data).map(|len| data[..len].to_vec()) +} + +/// Verify the Fletcher32 checksum that ends `data`; the length of the data +/// before it. +fn fletcher32_payload(data: &[u8]) -> Result { if data.len() < 4 { return Err(FormatError::FilterError( "fletcher32: data too short for checksum".into(), @@ -1430,7 +1644,7 @@ fn fletcher32_verify(data: &[u8]) -> Result, FormatError> { computed, }); } - Ok(payload.to_vec()) + Ok(payload.len()) } /// Append Fletcher32 checksum to data. @@ -1545,6 +1759,113 @@ fn pcodec_decompress( #[cfg(test)] mod tests { + /// `decompress_chunk_exact_with` returns exactly what + /// `decompress_chunk_exact` returns — data or error — for every pipeline + /// shape, filter mask and chunk size, with one scratch reused across all + /// of them in an order that grows, shrinks and swaps its buffers. + #[test] + fn decode_with_scratch_matches_the_allocating_decoder() { + let f = |filter_id: u16, client_data: Vec| FilterDescription { + filter_id, + name: None, + flags: 0, + client_data, + }; + let mut pipelines = vec![ + vec![f(FILTER_SHUFFLE, vec![4])], + vec![f(FILTER_FLETCHER32, vec![])], + // NetCDF-4's order: the checksum is taken before shuffle. + vec![f(FILTER_FLETCHER32, vec![]), f(FILTER_SHUFFLE, vec![4])], + ]; + #[cfg(feature = "deflate")] + pipelines.extend([ + vec![f(FILTER_DEFLATE, vec![4])], + vec![f(FILTER_SHUFFLE, vec![4]), f(FILTER_DEFLATE, vec![4])], + // h5py's order with `fletcher32=True`: checksum last. + vec![ + f(FILTER_SHUFFLE, vec![4]), + f(FILTER_DEFLATE, vec![4]), + f(FILTER_FLETCHER32, vec![]), + ], + vec![ + f(FILTER_FLETCHER32, vec![]), + f(FILTER_SHUFFLE, vec![4]), + f(FILTER_DEFLATE, vec![1]), + ], + ]); + #[cfg(feature = "lzf")] + pipelines.push(vec![ + f(FILTER_SHUFFLE, vec![4]), + f(crate::filter_pipeline::FILTER_LZF, vec![]), + ]); + + let mut scratch = DecodeScratch::new(); + for elements in [1usize, 7, 4096, 3, 65536, 100] { + let data: Vec = (0..elements as u32) + .flat_map(|i| (i.wrapping_mul(2654435761) >> (i % 13)).to_le_bytes()) + .collect(); + for filters in &pipelines { + let pipeline = FilterPipeline { + version: 2, + filters: filters.clone(), + }; + let n = filters.len() as u32; + for mask in 0..(1u32 << n) { + // Encode only the filters the mask says were applied. + let mut stored = data.clone(); + for (i, filter) in filters.iter().enumerate() { + if mask & (1 << i) == 0 { + let ctx = FilterContext { + filter, + element_size: 4, + max_output: 0, + }; + stored = filter_registry::encode(&stored, &ctx).unwrap(); + } + } + let mut cases = vec![(stored.clone(), data.len())]; + // Corrupt: last byte flipped, truncated, wrong size. + let mut flipped = stored.clone(); + *flipped.last_mut().unwrap() ^= 0x5a; + cases.push((flipped, data.len())); + cases.push((stored[..stored.len() / 2].to_vec(), data.len())); + cases.push((stored.clone(), data.len() + 4)); + cases.push((stored.clone(), 0)); + for (bytes, size) in cases { + let want = decompress_chunk_exact(&bytes, &pipeline, size, 4, mask, &[3]); + let got = decompress_chunk_exact_with( + &bytes, + &pipeline, + size, + 4, + mask, + &[3], + &mut scratch, + ) + .map(<[u8]>::to_vec); + match (&want, &got) { + (Ok(w), Ok(g)) => assert_eq!(w, g, "{filters:?} mask {mask}"), + (Err(w), Err(g)) => { + assert_eq!(w.to_string(), g.to_string(), "{filters:?}") + } + _ => panic!("{filters:?} mask {mask} size {size}: {want:?} vs {got:?}"), + } + } + } + } + } + // Long-lived scratch gives back a huge chunk's buffers. + let big = vec![0u8; DecodeScratch::RETAIN_BYTES + 8]; + let shuffle = FilterPipeline { + version: 2, + filters: vec![f(FILTER_SHUFFLE, vec![4])], + }; + decompress_chunk_exact_with(&big, &shuffle, big.len(), 4, 0, &[0], &mut scratch).unwrap(); + scratch.trim(); + assert!(scratch.a.capacity() <= DecodeScratch::RETAIN_BYTES); + assert!(scratch.b.capacity() <= DecodeScratch::RETAIN_BYTES); + } + /// A chunk whose pipeline decodes to fewer bytes than the chunk holds is /// an error naming the chunk, never a short buffer the reader pads. #[test] From d63c76e7ab42f20f3b41b2f409969db5e595133f Mon Sep 17 00:00:00 2001 From: osobh Date: Sat, 26 Sep 2026 10:12:07 -0500 Subject: [PATCH 03/43] writer: v2 B-trees with internal nodes (no 65 535-record limit) Dense link and attribute indexes and the chunk index for several unlimited dimensions were single leaves, capping them at 65 535 records. btree_v2_write builds trees of any depth, with node capacities and pointer widths from libhdf5's H5B2__hdr_init arithmetic (now shared with the reader as btree_v2::node_info) and libhdf5's node sizes (512 dense, 2048 chunks). Indexes that fit the old one-leaf layout are written byte for byte as before (compared for 10..65 535 links, attrs and chunks, tracked and filtered). Tests: 100 000 links (short names; long names with creation order), 70 000 attributes, 200 000 chunks (and 80 000 deflated), read by h5py, h5dump and clawhdf5 and edited by h5py r+; h5rs check on the same shapes, asserting depths 2-3. Co-Authored-By: Claude Opus 5.5 (1M context) --- CHANGELOG.md | 14 + crates/clawhdf5-format/src/btree_v2.rs | 81 +++- crates/clawhdf5-format/src/btree_v2_write.rs | 396 ++++++++++++++++++ crates/clawhdf5-format/src/chunked_write.rs | 90 ++-- crates/clawhdf5-format/src/file_writer.rs | 119 +++--- crates/clawhdf5-format/src/lib.rs | 1 + crates/clawhdf5-tools/tests/h5rs_interop.rs | 72 ++++ crates/clawhdf5/tests/chunk_index_interop.rs | 14 - crates/clawhdf5/tests/deep_btree_interop.rs | 343 +++++++++++++++ .../clawhdf5/tests/writer_groups_interop.rs | 32 +- docs/known-issues.md | 28 +- 11 files changed, 1005 insertions(+), 185 deletions(-) create mode 100644 crates/clawhdf5-format/src/btree_v2_write.rs create mode 100644 crates/clawhdf5/tests/deep_btree_interop.rs diff --git a/CHANGELOG.md b/CHANGELOG.md index 0e50755..448d2a0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,20 @@ ## Unreleased ### Writer: large dense indexes (2026-09-26) +- **No more 65 535-record limit on the writer's v2 B-trees.** Dense link + storage (name index and creation-order index), dense attribute storage + and the chunk index of datasets with more than one unlimited dimension + were written as a single leaf, so a group with more than 65 535 links, an + object with more than 65 535 dense attributes, or such a dataset with more + than 65 535 chunks was an error. The writer now builds internal nodes to + any depth (`clawhdf5_format::btree_v2_write`), with node capacities and + child-pointer widths from the same arithmetic as libhdf5's + `H5B2__hdr_init` (shared with the reader, `btree_v2::node_info`) and + libhdf5's node sizes (512 bytes for dense storage, 2048 for chunks). + Indexes that fit the old one-leaf layout are written byte for byte as + before. New tests `crates/clawhdf5/tests/deep_btree_interop.rs` (100 000 + links, 70 000 attributes, 200 000 chunks; h5py, h5dump, clawhdf5, and + h5py "r+" edits) and `check_files_with_deep_btrees` (`h5rs check`). - **Links and attributes whose name hashes collide are found by name.** The dense name indexes (a group's links: B-tree v2 type 5; an object's attributes: type 8) are ordered by the name's lookup3 hash and, when two diff --git a/crates/clawhdf5-format/src/btree_v2.rs b/crates/clawhdf5-format/src/btree_v2.rs index 0b70981..9159253 100644 --- a/crates/clawhdf5-format/src/btree_v2.rs +++ b/crates/clawhdf5-format/src/btree_v2.rs @@ -71,7 +71,7 @@ fn ensure_len(data: &[u8], pos: usize, needed: usize) -> Result<(), FormatError> /// Compute the number of bytes needed to represent a count, using variable-width encoding. /// B-tree v2 uses this for the number of records fields in internal nodes. -fn bytes_for_max_records(max_nrec: u64) -> usize { +pub(crate) fn bytes_for_max_records(max_nrec: u64) -> usize { if max_nrec == 0 { return 1; } @@ -163,7 +163,7 @@ impl BTreeV2Header { /// Compute maximum records per node for a given depth level. /// leaf: (node_size - overhead) / record_size /// internal: depends on pointers -fn max_records_leaf(node_size: u32, record_size: u16) -> u64 { +pub(crate) fn max_records_leaf(node_size: u32, record_size: u16) -> u64 { // Leaf overhead: signature(4) + version(1) + type(1) + checksum(4) = 10 let overhead = 10u32; if node_size <= overhead || record_size == 0 { @@ -418,10 +418,7 @@ fn collect_internal_records( } /// Most records a subtree whose root is at `depth` can hold (libhdf5's -/// `cum_max_nrec`): a leaf holds `max_leaf_nrec`; an internal node at depth -/// `d` holds `max_nrec(d)` records and `max_nrec(d) + 1` subtrees of depth -/// `d - 1`, where `max_nrec(d)` is what fits in a node once each record is -/// paired with a child pointer of the width depth `d` needs. +/// `cum_max_nrec`). See [`node_info`]. fn cum_max_records( node_size: u32, record_size: u16, @@ -429,24 +426,82 @@ fn cum_max_records( max_leaf_nrec: u64, depth: u16, ) -> u64 { + node_info_from_leaf(node_size, record_size, offset_size, max_leaf_nrec, depth) + .last() + .map_or(max_leaf_nrec, |n| n.cum_max_nrec) +} + +/// Capacity of a B-tree v2 node at one depth, as libhdf5 computes it +/// (`H5B2__hdr_init`'s `node_info`). +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) struct NodeInfo { + /// Most records one node at this depth holds. + pub(crate) max_nrec: u64, + /// Most records a subtree rooted at this depth holds. + pub(crate) cum_max_nrec: u64, + /// Bytes a subtree's total record count takes in a pointer to a node + /// at this depth (0 for a leaf, whose count is its own). + pub(crate) cum_max_nrec_size: usize, +} + +/// Node capacities for depths `0..=depth` (entry `d` for depth `d`): a leaf +/// holds `max_nrec(0)` records; an internal node at depth `d` holds +/// `max_nrec(d)` records and `max_nrec(d) + 1` subtrees of depth `d - 1`, +/// where `max_nrec(d)` is what fits in a node once each record is paired +/// with a child pointer of the width depth `d` needs (address, the child's +/// record count in the width a *leaf's* maximum needs, and below the first +/// internal level the child subtree's total in the width its maximum +/// needs), with one pointer more than records. +pub(crate) fn node_info( + node_size: u32, + record_size: u16, + offset_size: u8, + depth: u16, +) -> Vec { + let max_leaf = max_records_leaf(node_size, record_size); + node_info_from_leaf(node_size, record_size, offset_size, max_leaf, depth) +} + +fn node_info_from_leaf( + node_size: u32, + record_size: u16, + offset_size: u8, + max_leaf_nrec: u64, + depth: u16, +) -> Vec { // Internal node overhead: signature(4) + version(1) + type(1) + checksum(4). const PREFIX: u64 = 10; let nrec_width = bytes_for_max_records(max_leaf_nrec) as u64; - let mut cum = max_leaf_nrec; - let mut cum_width = 0u64; + let mut info = Vec::with_capacity(usize::from(depth) + 1); + info.push(NodeInfo { + max_nrec: max_leaf_nrec, + cum_max_nrec: max_leaf_nrec, + cum_max_nrec_size: 0, + }); for d in 1..=depth { - let ptr = u64::from(offset_size) + nrec_width + if d > 1 { cum_width } else { 0 }; + let below = info[usize::from(d) - 1]; + let ptr = u64::from(offset_size) + + nrec_width + + if d > 1 { + below.cum_max_nrec_size as u64 + } else { + 0 + }; let max_nrec = u64::from(node_size) .saturating_sub(PREFIX) .saturating_sub(ptr) / (u64::from(record_size) + ptr).max(1); - cum = max_nrec + let cum = max_nrec .saturating_add(1) - .saturating_mul(cum) + .saturating_mul(below.cum_max_nrec) .saturating_add(max_nrec); - cum_width = bytes_for_max_records(cum) as u64; + info.push(NodeInfo { + max_nrec, + cum_max_nrec: cum, + cum_max_nrec_size: bytes_for_max_records(cum), + }); } - cum + info } #[cfg(test)] diff --git a/crates/clawhdf5-format/src/btree_v2_write.rs b/crates/clawhdf5-format/src/btree_v2_write.rs new file mode 100644 index 0000000..1a18984 --- /dev/null +++ b/crates/clawhdf5-format/src/btree_v2_write.rs @@ -0,0 +1,396 @@ +//! Writing version-2 B-trees: a header (`BTHD`) and its nodes, leaves +//! (`BTLF`) and, for more records than one leaf holds, internal nodes +//! (`BTIN`) to any depth. +//! +//! Node capacities come from [`crate::btree_v2::node_info`], the arithmetic +//! libhdf5 uses (`H5B2__hdr_init`) and the reader decodes pointers with, so +//! the pointer widths the writer encodes are the ones every reader expects. + +#[cfg(not(feature = "std"))] +use alloc::{format, vec, vec::Vec}; + +use crate::btree_v2::{NodeInfo, bytes_for_max_records, node_info}; +use crate::checksum::jenkins_lookup3; +use crate::error::FormatError; + +/// How a B-tree is laid out: its record type and node geometry, as the +/// header records them. +#[derive(Debug, Clone, Copy)] +pub(crate) struct BTreeV2Params { + /// Record type (5: link names, 6: link creation order, 8: attribute + /// names, 9: attribute creation order, 10/11: chunks). + pub(crate) tree_type: u8, + /// Bytes per node. + pub(crate) node_size: u32, + /// Bytes per record. + pub(crate) record_size: u16, + /// Split and merge percentages. The writer fills nodes itself; these + /// only tell libhdf5 when to split and merge as it modifies the tree. + pub(crate) split_percent: u8, + pub(crate) merge_percent: u8, +} + +/// Size of a B-tree v2 header. +pub(crate) fn header_size(offset_size: u8, length_size: u8) -> usize { + 4 + 1 + 1 + 4 + 2 + 2 + 1 + 1 + offset_size as usize + 2 + length_size as usize + 4 +} + +/// Deepest tree the writer builds. Even at the smallest fan-out libhdf5's +/// arithmetic allows, a few levels hold more records than any file could. +const MAX_WRITE_DEPTH: u16 = 32; + +/// Write a B-tree v2 holding `records` (`record_size` bytes each, +/// concatenated, already in the tree's key order) at `addr`: the header, +/// then its nodes, each `node_size` bytes. No records gives a header with +/// an undefined root. +/// +/// The tree is as shallow as the node size allows: a single leaf when the +/// records fit one, otherwise internal nodes above leaves. Records are +/// spread evenly over each node's children, so every node but the root is +/// at least about half full (above libhdf5's merge threshold, which is below +/// half), and each node holds at most its depth's maximum. +pub(crate) fn build_btree_v2( + p: BTreeV2Params, + records: &[u8], + addr: u64, + offset_size: u8, + length_size: u8, +) -> Result, FormatError> { + let rs = usize::from(p.record_size); + if rs == 0 || !records.len().is_multiple_of(rs) { + return Err(FormatError::SerializationError(format!( + "B-tree v2 records are {} bytes, not a multiple of the record size {rs}", + records.len() + ))); + } + let n = (records.len() / rs) as u64; + let hdr_len = header_size(offset_size, length_size); + + // The shallowest depth whose subtree can hold every record. + let mut info = node_info(p.node_size, p.record_size, offset_size, 0); + let max_leaf = info[0].max_nrec; + if max_leaf == 0 || max_leaf > u64::from(u16::MAX) { + return Err(FormatError::SerializationError(format!( + "a {}-byte B-tree v2 node holds {max_leaf} {}-byte records; \ + a node holds 1 to 65535", + p.node_size, p.record_size + ))); + } + let mut depth = 0u16; + while info[usize::from(depth)].cum_max_nrec < n { + depth += 1; + if depth > MAX_WRITE_DEPTH { + return Err(FormatError::SerializationError(format!( + "{n} records do not fit a B-tree v2 of {}-byte nodes", + p.node_size + ))); + } + info = node_info(p.node_size, p.record_size, offset_size, depth); + let max = info[usize::from(depth)].max_nrec; + if max == 0 || max > u64::from(u16::MAX) { + return Err(FormatError::SerializationError(format!( + "a {}-byte B-tree v2 internal node holds {max} records; \ + a node holds 1 to 65535", + p.node_size + ))); + } + } + + let mut w = TreeWriter { + p, + records, + info: &info, + nrec_width: bytes_for_max_records(max_leaf), + offset_size, + first_node: addr + hdr_len as u64, + nodes: Vec::new(), + }; + let root = (n > 0).then(|| w.node(depth, 0, n as usize)).transpose()?; + + let mut out = Vec::with_capacity(hdr_len + w.nodes.len() * p.node_size as usize); + out.extend_from_slice(b"BTHD"); + out.push(0); // version + out.push(p.tree_type); + out.extend_from_slice(&p.node_size.to_le_bytes()); + out.extend_from_slice(&p.record_size.to_le_bytes()); + out.extend_from_slice(&depth.to_le_bytes()); + out.push(p.split_percent); + out.push(p.merge_percent); + match root { + Some(r) => push_uint(&mut out, r.addr, offset_size as usize), + None => out.extend(core::iter::repeat_n(0xFF, offset_size as usize)), + } + let root_nrec = root.map_or(0, |r| r.nrec); + out.extend_from_slice(&(root_nrec as u16).to_le_bytes()); + push_uint(&mut out, n, length_size as usize); + let sum = jenkins_lookup3(&out); + out.extend_from_slice(&sum.to_le_bytes()); + debug_assert_eq!(out.len(), hdr_len); + for node in &w.nodes { + out.extend_from_slice(node); + } + Ok(out) +} + +/// A written node, as its parent points at it. +#[derive(Debug, Clone, Copy)] +struct NodeRef { + addr: u64, + /// Records in the node itself. + nrec: u64, + /// Records in the subtree it roots. + all_nrec: u64, +} + +struct TreeWriter<'a> { + p: BTreeV2Params, + records: &'a [u8], + info: &'a [NodeInfo], + /// Width of a child's record count: what a leaf's maximum needs. + nrec_width: usize, + offset_size: u8, + /// Address of the first node (right after the header). + first_node: u64, + /// Nodes in file order (children before their parent). + nodes: Vec>, +} + +impl TreeWriter<'_> { + fn record(&self, i: usize) -> &[u8] { + let rs = usize::from(self.p.record_size); + &self.records[i * rs..(i + 1) * rs] + } + + fn push_node(&mut self, mut node: Vec) -> u64 { + // The checksum covers the node up to it, not the padding after. + let sum = jenkins_lookup3(&node); + node.extend_from_slice(&sum.to_le_bytes()); + debug_assert!(node.len() <= self.p.node_size as usize); + node.resize(self.p.node_size as usize, 0); + let addr = self.first_node + self.nodes.len() as u64 * u64::from(self.p.node_size); + self.nodes.push(node); + addr + } + + /// Write the subtree of `depth` holding records `first..first + n`. + fn node(&mut self, depth: u16, first: usize, n: usize) -> Result { + let rs = usize::from(self.p.record_size); + let mut node = Vec::with_capacity(self.p.node_size as usize); + if depth == 0 { + debug_assert!(n as u64 <= self.info[0].max_nrec); + node.extend_from_slice(b"BTLF"); + node.push(0); // version + node.push(self.p.tree_type); + node.extend_from_slice(&self.records[first * rs..(first + n) * rs]); + let addr = self.push_node(node); + return Ok(NodeRef { + addr, + nrec: n as u64, + all_nrec: n as u64, + }); + } + + // As few children as hold the records, at least two, with the + // records spread evenly: `k` children and `k - 1` records between + // them. + let below = self.info[usize::from(depth) - 1].cum_max_nrec; + let k = (n as u64 + 1).div_ceil(below + 1).max(2); + let max = self.info[usize::from(depth)].max_nrec; + if k - 1 > max || (n as u64) < k - 1 + k { + return Err(FormatError::SerializationError(format!( + "cannot spread {n} B-tree v2 records over {k} children at depth {depth}" + ))); + } + let k = k as usize; + let in_children = n - (k - 1); + let (base, extra) = (in_children / k, in_children % k); + + let mut children = Vec::with_capacity(k); + let mut separators = Vec::with_capacity(k - 1); + let mut next = first; + for c in 0..k { + let m = base + usize::from(c < extra); + children.push(self.node(depth - 1, next, m)?); + next += m; + if c + 1 < k { + separators.push(next); + next += 1; + } + } + debug_assert_eq!(next, first + n); + + node.extend_from_slice(b"BTIN"); + node.push(0); // version + node.push(self.p.tree_type); + for &s in &separators { + node.extend_from_slice(self.record(s)); + } + let total_width = if depth > 1 { + self.info[usize::from(depth) - 1].cum_max_nrec_size + } else { + 0 + }; + for c in &children { + push_uint(&mut node, c.addr, self.offset_size as usize); + push_uint(&mut node, c.nrec, self.nrec_width); + if depth > 1 { + push_uint(&mut node, c.all_nrec, total_width); + } + } + let addr = self.push_node(node); + Ok(NodeRef { + addr, + nrec: (k - 1) as u64, + all_nrec: n as u64, + }) + } +} + +/// Append `v` as a `width`-byte little-endian integer. +fn push_uint(buf: &mut Vec, v: u64, width: usize) { + let bytes = v.to_le_bytes(); + buf.extend_from_slice(&bytes[..width.min(8)]); + buf.extend(vec![0u8; width.saturating_sub(8)]); +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::btree_v2::{BTreeV2Header, collect_btree_v2_records}; + + fn params(node_size: u32, record_size: u16) -> BTreeV2Params { + BTreeV2Params { + tree_type: 5, + node_size, + record_size, + split_percent: 100, + merge_percent: 40, + } + } + + /// `n` 11-byte records: a big-endian counter, so byte order is key order. + fn records(n: usize, rs: usize) -> Vec { + let mut out = Vec::with_capacity(n * rs); + for i in 0..n { + let mut r = vec![0u8; rs]; + r[..8].copy_from_slice(&(i as u64).to_be_bytes()); + out.extend_from_slice(&r); + } + out + } + + fn roundtrip(node_size: u32, rs: u16, n: usize, os: u8, ls: u8) -> BTreeV2Header { + let recs = records(n, usize::from(rs)); + let base = 4096u64; + let tree = build_btree_v2(params(node_size, rs), &recs, base, os, ls).unwrap(); + let mut file = vec![0u8; base as usize]; + file.extend_from_slice(&tree); + let hdr = BTreeV2Header::parse(&file, base as usize, os, ls).unwrap(); + assert_eq!(hdr.total_records, n as u64); + let got = collect_btree_v2_records(&file, &hdr, os, ls).unwrap(); + assert_eq!(got.len(), n); + let flat: Vec = got.into_iter().flat_map(|r| r.data).collect(); + assert_eq!(flat, recs, "node {node_size} rs {rs} n {n}"); + hdr + } + + #[test] + fn one_leaf_then_deeper_trees_read_back_in_order() { + // 512-byte nodes of 11-byte records: 45 per leaf, 1149 at depth 1, + // 26 449 at depth 2. + let info = node_info(512, 11, 8, 3); + assert_eq!( + info.iter().map(|i| i.cum_max_nrec).collect::>(), + [45, 1149, 26_449, 608_349] + ); + for (n, depth) in [ + (0, 0), + (1, 0), + (45, 0), + (46, 1), + (1149, 1), + (1150, 2), + (26_449, 2), + (26_450, 3), + (100_000, 3), + ] { + let hdr = roundtrip(512, 11, n, 8, 8); + assert_eq!(hdr.depth, depth, "{n} records"); + } + } + + #[test] + fn pointer_widths_follow_the_offset_and_length_sizes() { + for (os, ls) in [(4, 4), (8, 4), (4, 8), (2, 2)] { + roundtrip(512, 11, 5000, os, ls); + } + // Wide counts: a leaf of 2048 bytes / 9-byte records (226, one byte) + // and deeper subtree totals of three bytes. + roundtrip(2048, 9, 300_000, 8, 8); + } + + #[test] + fn every_node_is_within_its_capacity_and_above_the_merge_threshold() { + let rs = 17u16; + let n = 70_000usize; + let info = node_info(512, rs, 8, 3); + let recs = records(n, usize::from(rs)); + let tree = build_btree_v2(params(512, rs), &recs, 0, 8, 8).unwrap(); + let hdr_len = header_size(8, 8); + let nodes = (tree.len() - hdr_len) / 512; + for i in 0..nodes { + let node = &tree[hdr_len + i * 512..hdr_len + (i + 1) * 512]; + let sig = &node[..4]; + if sig == b"BTLF" { + continue; // counts checked through the parents below + } + assert_eq!(sig, b"BTIN"); + } + // Walk from the header: each child's count within [40%, 100%]. + let hdr = BTreeV2Header::parse(&tree, 0, 8, 8).unwrap(); + assert_eq!(hdr.depth, 3); + assert!(u64::from(hdr.num_records_in_root) <= info[3].max_nrec); + fn walk(tree: &[u8], addr: usize, nrec: usize, depth: usize, info: &[NodeInfo], rs: usize) { + if depth == 0 { + return; + } + let nrec_w = bytes_for_max_records(info[0].max_nrec); + let tot_w = if depth > 1 { + info[depth - 1].cum_max_nrec_size + } else { + 0 + }; + let mut pos = addr + 6 + nrec * rs; + for _ in 0..=nrec { + let a = u64::from_le_bytes(tree[pos..pos + 8].try_into().unwrap()) as usize; + pos += 8; + let mut c = 0usize; + for b in 0..nrec_w { + c |= usize::from(tree[pos + b]) << (8 * b); + } + pos += nrec_w + tot_w; + let max = info[depth - 1].max_nrec as usize; + assert!(c <= max && c * 100 > max * 40, "{c} of {max}"); + walk(tree, a, c, depth - 1, info, rs); + } + } + walk( + &tree, + hdr.root_node_address as usize, + usize::from(hdr.num_records_in_root), + 3, + &info, + usize::from(rs), + ); + assert!(nodes > 0); + } + + #[test] + fn a_node_too_small_or_too_big_is_an_error() { + assert!(build_btree_v2(params(16, 11), &records(1, 11), 0, 8, 8).is_err()); + // A leaf with room for more than 65 535 records. + assert!(build_btree_v2(params(1 << 20, 11), &records(1, 11), 0, 8, 8).is_err()); + // Records that are not whole. + assert!(build_btree_v2(params(512, 11), &[0u8; 12], 0, 8, 8).is_err()); + } +} diff --git a/crates/clawhdf5-format/src/chunked_write.rs b/crates/clawhdf5-format/src/chunked_write.rs index 05383a6..2a8bd09 100644 --- a/crates/clawhdf5-format/src/chunked_write.rs +++ b/crates/clawhdf5-format/src/chunked_write.rs @@ -6,6 +6,7 @@ extern crate alloc; #[cfg(not(feature = "std"))] use alloc::{format, vec, vec::Vec}; +use crate::btree_v2_write::{BTreeV2Params, build_btree_v2}; use crate::checksum::jenkins_lookup3; use crate::chunk_cache::{CACHE_LINE_SIZE, align_to_cache_line}; use crate::chunk_grid::ChunkGrid; @@ -1105,11 +1106,12 @@ const BT2_CHUNK_FILTERED: u8 = 11; /// /// `records` are `(scaled coordinates, chunk)` in lexicographic order of the /// coordinates, which is the order the library's comparator -/// (`H5VM_vector_cmp_u`) keeps them in. The tree is a single leaf: the -/// library's 2048-byte node when the records fit, otherwise a leaf node -/// sized to hold them all (the root's record count is 16-bit, so at most -/// 65535 chunks). Returns the bytes and the node size the layout message -/// must record. +/// (`H5VM_vector_cmp_u`) keeps them in. Up to 65 535 chunks go in a single +/// leaf: the library's 2048-byte node when the records fit, otherwise a leaf +/// node sized to hold them all (the layout the writer has always used, kept +/// so those files do not change). More chunks get the library's 2048-byte +/// nodes with internal nodes above the leaves. Returns the bytes and the +/// node size the layout message must record. fn build_btree_v2_chunk_index_at( rank: usize, records: &[(Vec, &WrittenChunk)], @@ -1119,73 +1121,49 @@ fn build_btree_v2_chunk_index_at( base_address: u64, ) -> Result<(Vec, u32), FormatError> { let os = offset_size as usize; - let nrec = u16::try_from(records.len()).map_err(|_| { - FormatError::ChunkedReadError( - "more than 65535 chunks with more than one unlimited dimension: \ - use larger chunks" - .into(), - ) - })?; let chunk_size_bytes = has_filters.then(|| { let slots: Vec> = records.iter().map(|(_, c)| Some((*c).clone())).collect(); filtered_chunk_size_len(&slots) }); let record_size = os + chunk_size_bytes.map_or(0, |n| n + 4) + 8 * rank; - // Leaf: signature, version, type, records, checksum. - let leaf_len = 4 + 1 + 1 + records.len() * record_size + 4; - let node_size = u32::try_from(leaf_len) - .map_err(|_| FormatError::Overflow("B-tree v2 leaf size".into()))? - .max(BT2_NODE_SIZE); + let record_size_u16 = u16::try_from(record_size) + .map_err(|_| FormatError::Overflow("B-tree v2 record size".into()))?; + let node_size = if records.len() <= usize::from(u16::MAX) { + // Leaf: signature, version, type, records, checksum. + let leaf_len = 4 + 1 + 1 + records.len() * record_size + 4; + u32::try_from(leaf_len) + .map_err(|_| FormatError::Overflow("B-tree v2 leaf size".into()))? + .max(BT2_NODE_SIZE) + } else { + BT2_NODE_SIZE + }; let tree_type = if has_filters { BT2_CHUNK_FILTERED } else { BT2_CHUNK_UNFILTERED }; - let hdr_len = 4 + 1 + 1 + 4 + 2 + 2 + 1 + 1 + os + 2 + length_size as usize + 4; - let leaf_address = base_address + hdr_len as u64; - - let mut out = Vec::with_capacity(hdr_len + node_size as usize); - out.extend_from_slice(b"BTHD"); - out.push(0); // version - out.push(tree_type); - out.extend_from_slice(&node_size.to_le_bytes()); - out.extend_from_slice(&(record_size as u16).to_le_bytes()); - out.extend_from_slice(&0u16.to_le_bytes()); // depth - out.push(BT2_SPLIT_PERCENT); - out.push(BT2_MERGE_PERCENT); - if records.is_empty() { - out.extend(core::iter::repeat_n(0xFF, os)); - } else { - push_addr(&mut out, leaf_address, offset_size); - } - out.extend_from_slice(&nrec.to_le_bytes()); - match length_size { - 4 => out.extend_from_slice(&(records.len() as u32).to_le_bytes()), - _ => out.extend_from_slice(&(records.len() as u64).to_le_bytes()), - } - let sum = jenkins_lookup3(&out); - out.extend_from_slice(&sum.to_le_bytes()); - debug_assert_eq!(out.len(), hdr_len); - if records.is_empty() { - return Ok((out, node_size)); - } - - let leaf_start = out.len(); - out.extend_from_slice(b"BTLF"); - out.push(0); // version - out.push(tree_type); + let mut flat = Vec::with_capacity(records.len() * record_size); for (scaled, chunk) in records { - push_index_element(&mut out, Some(chunk), offset_size, chunk_size_bytes); + push_index_element(&mut flat, Some(chunk), offset_size, chunk_size_bytes); for &c in scaled { - out.extend_from_slice(&c.to_le_bytes()); + flat.extend_from_slice(&c.to_le_bytes()); } } - let sum = jenkins_lookup3(&out[leaf_start..]); - out.extend_from_slice(&sum.to_le_bytes()); - // The library reads whole nodes; pad the leaf out to the node size. - out.resize(leaf_start + node_size as usize, 0); + let out = build_btree_v2( + BTreeV2Params { + tree_type, + node_size, + record_size: record_size_u16, + split_percent: BT2_SPLIT_PERCENT, + merge_percent: BT2_MERGE_PERCENT, + }, + &flat, + base_address, + offset_size, + length_size, + )?; Ok((out, node_size)) } diff --git a/crates/clawhdf5-format/src/file_writer.rs b/crates/clawhdf5-format/src/file_writer.rs index 80a296b..a9b3af5 100644 --- a/crates/clawhdf5-format/src/file_writer.rs +++ b/crates/clawhdf5-format/src/file_writer.rs @@ -7,6 +7,7 @@ use alloc::{format, vec, vec::Vec}; use crate::attribute::AttributeMessage; +use crate::btree_v2_write::{BTreeV2Params, build_btree_v2}; use crate::chunked_write::{ ChunkOptions, PrecompressedChunks, build_chunked_data_from_precompressed, precompress_chunks, }; @@ -933,13 +934,7 @@ pub(crate) fn build_dense_attrs( .collect(); let bthd_addr = btree_addr; let mut blob = heap.blob; - blob.extend_from_slice(&single_leaf_v2_btree( - 8, - record_size, - &records, - bthd_addr, - "attributes on one object", - )?); + blob.extend_from_slice(&dense_v2_btree(8, record_size, &records, bthd_addr)?); let attr_info = serialize_attribute_info(frhp_addr, bthd_addr); @@ -960,69 +955,57 @@ pub(crate) struct DenseLinkBlob { pub(crate) blob: Vec, } -/// A v2 B-tree of `btree_type` holding `records` (already in key order) in a -/// single leaf, laid out at `addr`: the header, then the leaf. `what` names -/// the records in the error for too many ("links in one group"). -fn single_leaf_v2_btree( +/// libhdf5's node size for the dense link and attribute indexes it creates +/// (`H5G_NAME_BT2_NODE_SIZE`, `H5A_NAME_BT2_NODE_SIZE`, and the +/// creation-order indexes'), with their split and merge percentages. +const DENSE_BT2_NODE_SIZE: u32 = 512; +const DENSE_BT2_SPLIT_PERCENT: u8 = 100; +const DENSE_BT2_MERGE_PERCENT: u8 = 40; + +/// A dense-storage v2 B-tree of `btree_type` holding `records` (already in +/// key order), laid out at `addr`: the header, then its nodes. +/// +/// Up to 65 535 records go in one leaf node sized to hold them (the layout +/// the writer has always used, kept so those files do not change). A leaf's +/// record count is a 2-byte field, and libhdf5 sizes a leaf's capacity from +/// the node size: a node with room for more than 65 535 records makes it +/// overflow that count when it adds one, so the node is capped at a full +/// leaf. More records get libhdf5's own 512-byte nodes, with internal nodes +/// above the leaves. +fn dense_v2_btree( btree_type: u8, record_size: u16, records: &[Vec], addr: u64, - what: &str, ) -> Result, FormatError> { - let os = OFFSET_SIZE as usize; - let ls = LENGTH_SIZE as usize; - // The root node's record count is a 2-byte field; more records need - // internal nodes, which the writer does not build. - let num_records = u16::try_from(records.len()).map_err(|_| { - FormatError::SerializationError(format!( - "{} {what}: at most {} can be written \ - (a deeper B-tree index is not implemented)", - records.len(), - u16::MAX - )) - })?; - let bthd_size = 4 + 1 + 1 + 4 + 2 + 2 + 1 + 1 + os + 2 + ls + 4; - let btlf_size = 4 + 1 + 1 + (records.len() * record_size as usize) + 4; - // libhdf5 sizes a leaf's capacity from the node size, and a leaf's - // record count is a 2-byte field: a node with room for more than - // 65 535 records makes it overflow that count when it adds one (the - // group can then no longer be listed). Cap the node at a full leaf. - let max_node = btlf_size - records.len() * record_size as usize - + usize::from(u16::MAX) * record_size as usize; - let node_size = btlf_size.next_power_of_two().max(512).min(max_node) as u32; - let btlf_addr = addr + bthd_size as u64; - - let mut out = Vec::with_capacity(bthd_size + node_size as usize); - out.extend_from_slice(b"BTHD"); - out.push(0); // version - out.push(btree_type); - out.extend_from_slice(&node_size.to_le_bytes()); - out.extend_from_slice(&record_size.to_le_bytes()); - out.extend_from_slice(&0u16.to_le_bytes()); // depth = 0 (single leaf) - out.push(100); // split_percent - out.push(40); // merge_percent - write_offset(&mut out, btlf_addr, OFFSET_SIZE); - out.extend_from_slice(&num_records.to_le_bytes()); - write_length(&mut out, records.len() as u64, LENGTH_SIZE); - let checksum = crate::checksum::jenkins_lookup3(&out); - out.extend_from_slice(&checksum.to_le_bytes()); - debug_assert_eq!(out.len(), bthd_size); - - let mut btlf = Vec::with_capacity(node_size as usize); - btlf.extend_from_slice(b"BTLF"); - btlf.push(0); // version - btlf.push(btree_type); - for rec in records { - debug_assert_eq!(rec.len(), record_size as usize); - btlf.extend_from_slice(rec); - } - // The checksum follows the records, not the end of the node. - let checksum = crate::checksum::jenkins_lookup3(&btlf); - btlf.extend_from_slice(&checksum.to_le_bytes()); - btlf.resize(node_size as usize, 0); - out.extend_from_slice(&btlf); - Ok(out) + let rs = usize::from(record_size); + let n = records.len(); + let node_size = if n <= usize::from(u16::MAX) { + let btlf_size = 4 + 1 + 1 + n * rs + 4; + let max_node = 4 + 1 + 1 + usize::from(u16::MAX) * rs + 4; + u32::try_from(btlf_size.next_power_of_two().max(512).min(max_node)) + .map_err(|_| FormatError::Overflow("B-tree v2 node size".into()))? + } else { + DENSE_BT2_NODE_SIZE + }; + let flat: Vec = records + .iter() + .inspect(|r| debug_assert_eq!(r.len(), rs)) + .flat_map(|r| r.iter().copied()) + .collect(); + build_btree_v2( + BTreeV2Params { + tree_type: btree_type, + node_size, + record_size, + split_percent: DENSE_BT2_SPLIT_PERCENT, + merge_percent: DENSE_BT2_MERGE_PERCENT, + }, + &flat, + addr, + OFFSET_SIZE, + LENGTH_SIZE, + ) } /// Build dense link storage for a group's links, laid out at `base_address`. @@ -1068,12 +1051,11 @@ pub(crate) fn build_dense_links( .collect(); let name_bt_addr = heap.btree_addr; let mut blob = heap.blob; - blob.extend_from_slice(&single_leaf_v2_btree( + blob.extend_from_slice(&dense_v2_btree( 5, 4 + heap_id_length, &name_records, name_bt_addr, - "links in one group", )?); let link_info_message = if track_order { @@ -1093,12 +1075,11 @@ pub(crate) fn build_dense_links( }) .collect(); let order_bt_addr = base_address + blob.len() as u64; - blob.extend_from_slice(&single_leaf_v2_btree( + blob.extend_from_slice(&dense_v2_btree( 6, 8 + heap_id_length, &order_records, order_bt_addr, - "links in one group", )?); let next_order = by_order.last().map_or(0, |&(o, _)| o + 1); serialize_link_info( diff --git a/crates/clawhdf5-format/src/lib.rs b/crates/clawhdf5-format/src/lib.rs index 3e30f9d..92954bc 100644 --- a/crates/clawhdf5-format/src/lib.rs +++ b/crates/clawhdf5-format/src/lib.rs @@ -61,6 +61,7 @@ pub mod attribute; pub mod attribute_info; pub mod btree_v1; pub mod btree_v2; +mod btree_v2_write; mod bulk_alloc; pub mod checksum; pub mod chunk_cache; diff --git a/crates/clawhdf5-tools/tests/h5rs_interop.rs b/crates/clawhdf5-tools/tests/h5rs_interop.rs index fcd205a..32f6fb2 100644 --- a/crates/clawhdf5-tools/tests/h5rs_interop.rs +++ b/crates/clawhdf5-tools/tests/h5rs_interop.rs @@ -1024,3 +1024,75 @@ fn check_files_with_big_dense_storage() { assert_eq!(code(&o), 0, "{p}:\n{}", stdout(&o)); assert!(stdout(&o).contains("no problems found"), "{}", stdout(&o)); } + +/// `(type, depth)` of every v2 B-tree header in a file written with 8-byte +/// offsets and lengths (found by signature and checksum). +fn btree_v2_depths(data: &[u8]) -> Vec<(u8, u16)> { + const LEN: usize = 4 + 1 + 1 + 4 + 2 + 2 + 1 + 1 + 8 + 2 + 8; + let mut out = Vec::new(); + for at in 0..data.len().saturating_sub(LEN + 4) { + if &data[at..at + 4] != b"BTHD" { + continue; + } + let stored = u32::from_le_bytes(data[at + LEN..at + LEN + 4].try_into().unwrap()); + if jenkins_lookup3(&data[at..at + LEN]) == stored { + out.push(( + data[at + 5], + u16::from_le_bytes([data[at + 12], data[at + 13]]), + )); + } + } + out +} + +#[test] +fn check_files_with_deep_btrees() { + // Dense indexes and a chunk index too big for one leaf: the writer then + // builds internal nodes, whose child pointers carry record counts in + // widths derived from the node size. `check` reads every record through + // them and compares the count with the header's. + use clawhdf5::{AttrValue, FileBuilder}; + const U: u64 = u64::MAX; + let dir = tempfile::tempdir().unwrap(); + let mut b = FileBuilder::new(); + let x = b.create_dataset("x"); + x.with_i32_data(&[7]); + for i in 0..70_000 { + x.set_attr(&format!("attr_{i}"), AttrValue::I64(i)); + } + let mut g = b.create_group("g"); + g.track_order(true); + for i in 0..100_000 { + g.add_hard_link(&format!("k{i}"), "/x"); + } + b.add_group(g.finish()); + let p = dir.path().join("deep.h5").to_string_lossy().into_owned(); + b.write(&p).unwrap(); + let o = h5rs(&["check", &p]); + assert_eq!(code(&o), 0, "{p}:\n{}", stdout(&o)); + assert!(stdout(&o).contains("no problems found"), "{}", stdout(&o)); + let mut depths = btree_v2_depths(&std::fs::read(&p).unwrap()); + depths.sort(); + assert_eq!(depths, [(5, 3), (6, 3), (8, 3)]); + + let mut b = FileBuilder::new(); + b.create_dataset("d") + .with_i32_data(&(0..200_000).collect::>()) + .with_shape(&[400, 500]) + .with_chunks(&[1, 1]) + .with_maxshape(&[U, U]); + b.create_dataset("z") + .with_i32_data(&(0..70_000).collect::>()) + .with_shape(&[70, 1000]) + .with_chunks(&[1, 1]) + .with_maxshape(&[U, U]) + .with_deflate(1); + let p = dir.path().join("chunks.h5").to_string_lossy().into_owned(); + b.write(&p).unwrap(); + let o = h5rs(&["check", "--data", &p]); + assert_eq!(code(&o), 0, "{p}:\n{}", stdout(&o)); + assert!(stdout(&o).contains("no problems found"), "{}", stdout(&o)); + let mut depths = btree_v2_depths(&std::fs::read(&p).unwrap()); + depths.sort(); + assert_eq!(depths, [(10, 2), (11, 2)]); +} diff --git a/crates/clawhdf5/tests/chunk_index_interop.rs b/crates/clawhdf5/tests/chunk_index_interop.rs index 3ff0420..96ecffb 100644 --- a/crates/clawhdf5/tests/chunk_index_interop.rs +++ b/crates/clawhdf5/tests/chunk_index_interop.rs @@ -559,20 +559,6 @@ fn we_write_btree_v2_for_several_unlimited_dims() { check_we_write(&cases); } -/// A single-leaf B-tree has a 16-bit record count; beyond it the writer -/// refuses rather than writing a tree libhdf5 would misread. -#[test] -fn btree_v2_index_past_one_leaf_is_refused() { - let mut b = FileBuilder::new(); - b.create_dataset("d") - .with_i32_data(&vec![0i32; 70_000]) - .with_shape(&[70_000, 1]) - .with_chunks(&[1, 1]) - .with_maxshape(&[u64::MAX, u64::MAX]); - let dir = tempfile::tempdir().unwrap(); - assert!(b.write(dir.path().join("too_many.h5")).is_err()); -} - /// A maxshape equal to the shape cannot grow, so it needs no chunks: the /// dataset stays contiguous (as h5py makes it) unless chunks are requested. #[test] diff --git a/crates/clawhdf5/tests/deep_btree_interop.rs b/crates/clawhdf5/tests/deep_btree_interop.rs new file mode 100644 index 0000000..64b6e9d --- /dev/null +++ b/crates/clawhdf5/tests/deep_btree_interop.rs @@ -0,0 +1,343 @@ +//! Version-2 B-trees deeper than one leaf, as `FileBuilder` writes them for +//! big dense indexes: a group's links (name index, type 5, and creation +//! order index, type 6), an object's attributes (name index, type 8) and +//! the chunk index of a dataset with two unlimited dimensions (type 10 and, +//! with a filter, 11). Read back by h5py (libhdf5), h5dump and clawhdf5, +//! then modified by h5py in "r+" mode, which splits, merges and +//! redistributes the nodes the writer built. +//! +//! Skipped when python3 with h5py is unavailable, unless +//! `CLAWHDF5_REQUIRE_INTEROP=1`. + +use std::process::Command; + +use clawhdf5::{AttrValue, File, FileBuilder}; + +fn python() -> String { + std::env::var("CLAWHDF5_PYTHON").unwrap_or_else(|_| "python3".to_string()) +} + +fn interop_required() -> bool { + std::env::var("CLAWHDF5_REQUIRE_INTEROP").is_ok_and(|v| v == "1") +} + +fn python_available() -> bool { + Command::new(python()) + .args(["-c", "import h5py"]) + .output() + .map(|o| o.status.success()) + .unwrap_or(false) +} + +macro_rules! skip_if_no_python { + () => { + if !python_available() { + assert!( + !interop_required(), + "CLAWHDF5_REQUIRE_INTEROP=1 but python3 with h5py is not available" + ); + eprintln!("SKIP: python3 with h5py not available"); + return; + } + }; +} + +/// Run `body` under h5py with `path` bound to the file's path. +fn h5py(path: &str, body: &str) -> String { + let script = format!("import h5py, numpy as np, json\npath = r'{path}'\n{body}"); + let output = Command::new(python()) + .args(["-c", &script]) + .output() + .expect("failed to run python"); + if !output.status.success() { + panic!( + "Python script failed:\nSTDOUT: {}\nSTDERR: {}", + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr) + ); + } + String::from_utf8_lossy(&output.stdout).trim().to_string() +} + +/// h5dump of `args` must succeed; returns its output. +fn h5dump(args: &[&str]) -> String { + let ok = Command::new("h5dump") + .arg("--version") + .output() + .is_ok_and(|o| o.status.success()); + if !ok { + assert!(!interop_required(), "h5dump is not available"); + return String::new(); + } + let o = Command::new("h5dump").args(args).output().unwrap(); + let out = String::from_utf8_lossy(&o.stdout).to_string(); + assert!( + o.status.success(), + "h5dump {args:?} failed:\n{out}{}", + String::from_utf8_lossy(&o.stderr) + ); + out +} + +// ---- 100 000 links in one group ---- + +const NLINKS: usize = 100_000; + +/// The compact names: `k0`..`k99998` and `k155448`, whose name hash equals +/// that of `k69209` — a collision inside a many-level name index. +fn compact_name(i: usize) -> String { + if i == NLINKS - 1 { + "k155448".into() + } else { + format!("k{i}") + } +} + +/// The long names (111 bytes), added in a scrambled order so creation order +/// is not name order: the `i`th created is `long_name(scramble(i))`. +fn long_name(j: usize) -> String { + format!("link_{j:06}_{}", "x".repeat(100)) +} + +fn scramble(i: usize) -> usize { + i * 7919 % NLINKS +} + +const PY_NAMES: &str = "\ +N = 100000\n\ +compact = ['k%d' % i for i in range(N - 1)] + ['k155448']\n\ +def long_name(j): return 'link_%06d_' % j + 'x' * 100\n\ +created = [long_name(i * 7919 % N) for i in range(N)]\n"; + +#[test] +fn a_hundred_thousand_links_in_one_group() { + skip_if_no_python!(); + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("links.h5").display().to_string(); + let mut b = FileBuilder::new(); + for v in 0..10 { + b.create_dataset(&format!("v{v}")).with_i32_data(&[v]); + } + // Name index only (type 5), 11-byte records: depth 3 in 512-byte nodes. + let mut g = b.create_group("compact"); + for i in 0..NLINKS { + g.add_hard_link(&compact_name(i), &format!("/v{}", i % 10)); + } + b.add_group(g.finish()); + // Creation order tracked and indexed: a type-6 index as well. + let mut g = b.create_group("long"); + g.track_order(true); + for i in 0..NLINKS { + let j = scramble(i); + g.add_hard_link(&long_name(j), &format!("/v{}", j % 10)); + } + b.add_group(g.finish()); + b.write(&path).unwrap(); + + let check = format!( + "{PY_NAMES}\ + with h5py.File(path, 'r') as f:\n\ + \x20 g, l = f['compact'], f['long']\n\ + \x20 out = [len(g), list(g) == sorted(compact), len(l), list(l) == created]\n\ + \x20 out.append(all(int(g[compact[i]][0]) == i % 10 for i in range(0, N, 997)))\n\ + \x20 out.append([int(g['k69209'][0]), int(g['k155448'][0]), 'k155448' in g, 'k100000' in g])\n\ + \x20 out.append(all(int(l[long_name(j)][0]) == j % 10 for j in range(0, N, 1009)))\n\ + \x20 out.append(h5py.h5o.get_info(f['v3'].id).rc)\n\ + \x20 print(json.dumps(out))" + ); + assert_eq!( + h5py(&path, &check), + "[100000, true, 100000, true, true, [9, 9, true, false], true, 20001]" + ); + let d = h5dump(&["-d", "/compact/k155448", &path]); + assert!(d.is_empty() || d.contains("(0): 9"), "{d}"); + let d = h5dump(&["-d", &format!("/long/{}", long_name(99_999)), &path]); + assert!(d.is_empty() || d.contains("(0): 9"), "{d}"); + + // clawhdf5 reads both indexes back. + let f = File::open(&path).unwrap(); + let g = f.group("compact").unwrap(); + let mut names = g.datasets().unwrap(); + names.sort(); + let mut want: Vec = (0..NLINKS).map(compact_name).collect(); + want.sort(); + assert_eq!(names, want); + assert_eq!(g.dataset("k155448").unwrap().read_i32().unwrap(), [9]); + let l = f.group("long").unwrap(); + assert_eq!(l.datasets().unwrap().len(), NLINKS); + assert_eq!( + l.dataset(&long_name(12_345)).unwrap().read_i32().unwrap(), + [5] + ); + drop(f); + + // libhdf5 inserts into and removes from the trees we wrote. + let modify = format!( + "{PY_NAMES}\ + with h5py.File(path, 'r+') as f:\n\ + \x20 g, l = f['compact'], f['long']\n\ + \x20 for i in range(3000):\n\ + \x20 g['new%d' % i] = f['v1']\n\ + \x20 for i in range(0, N, 7):\n\ + \x20 del g[compact[i]]\n\ + \x20 l['zz_new'] = f['v2']\n\ + \x20 for j in range(0, N, 3):\n\ + \x20 del l[long_name(j)]\n\ + with h5py.File(path, 'r') as f:\n\ + \x20 g, l = f['compact'], f['long']\n\ + \x20 left = sorted([n for i, n in enumerate(compact) if i % 7] + ['new%d' % i for i in range(3000)])\n\ + \x20 kept = [n for n in created if int(n[5:11]) % 3] + ['zz_new']\n\ + \x20 print(json.dumps([len(g), list(g) == left, int(g['new2999'][0]),\n\ + \x20 int(g['k155448'][0]), len(l), list(l) == kept, int(l['zz_new'][0])]))" + ); + assert_eq!(h5py(&path, &modify), "[88714, true, 1, 9, 66667, true, 2]"); + h5dump(&["-d", "/compact/new0", &path]); + let f = File::open(&path).unwrap(); + assert_eq!( + f.group("compact").unwrap().datasets().unwrap().len(), + 88_714 + ); + assert_eq!(f.group("long").unwrap().datasets().unwrap().len(), 66_667); +} + +// ---- 70 000 attributes on one object ---- + +#[test] +fn seventy_thousand_attributes_on_one_object() { + skip_if_no_python!(); + const N: i64 = 70_000; + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("attrs.h5").display().to_string(); + let mut b = FileBuilder::new(); + let x = b.create_dataset("x"); + x.with_i32_data(&[1]); + for i in 0..N { + x.set_attr(&format!("attr_{i}"), AttrValue::I64(i * 3)); + } + let mut g = b.create_group("g"); + for i in 0..N { + g.set_attr(&format!("s{i:05}"), AttrValue::String(format!("value {i}"))); + } + b.add_group(g.finish()); + b.write(&path).unwrap(); + + let check = "\ + N = 70000\n\ + with h5py.File(path, 'r') as f:\n\ + \x20 a, s = f['x'].attrs, f['g'].attrs\n\ + \x20 names = list(a)\n\ + \x20 out = [len(a), names == sorted('attr_%d' % i for i in range(N))]\n\ + \x20 out.append(all(int(a['attr_%d' % i]) == 3 * i for i in range(0, N, 331)))\n\ + \x20 out.append(len(s))\n\ + \x20 v = dict(s.items())\n\ + \x20 out.append(all(v['s%05d' % i].decode() == 'value %d' % i for i in range(N)))\n\ + \x20 print(json.dumps(out))"; + assert_eq!(h5py(&path, check), "[70000, true, true, 70000, true]"); + let d = h5dump(&["-a", "/x/attr_69999", &path]); + assert!(d.is_empty() || d.contains("(0): 209997"), "{d}"); + + let f = File::open(&path).unwrap(); + let attrs = f.dataset("x").unwrap().attrs().unwrap(); + assert_eq!(attrs.len(), N as usize); + for i in [0, 1, 35_000, N - 1] { + assert!( + matches!(attrs[&format!("attr_{i}")], AttrValue::I64(v) if v == 3 * i), + "attr_{i}" + ); + } + let attrs = f.group("g").unwrap().attrs().unwrap(); + assert_eq!(attrs.len(), N as usize); + drop(f); + + let modify = "\ + N = 70000\n\ + with h5py.File(path, 'r+') as f:\n\ + \x20 a = f['x'].attrs\n\ + \x20 for i in range(2000):\n\ + \x20 a['new_%d' % i] = i\n\ + \x20 for i in range(0, N, 5):\n\ + \x20 del a['attr_%d' % i]\n\ + \x20 f['g'].attrs['s00000'] = 'changed'\n\ + with h5py.File(path, 'r') as f:\n\ + \x20 a = f['x'].attrs\n\ + \x20 want = sorted(['attr_%d' % i for i in range(N) if i % 5] + ['new_%d' % i for i in range(2000)])\n\ + \x20 print(json.dumps([len(a), list(a) == want, int(a['attr_69999']), int(a['new_1999']),\n\ + \x20 'attr_5' in a, f['g'].attrs['s00000'], len(f['g'].attrs)]))"; + assert_eq!( + h5py(&path, modify), + r#"[58000, true, 209997, 1999, false, "changed", 70000]"# + ); + let f = File::open(&path).unwrap(); + assert_eq!(f.dataset("x").unwrap().attrs().unwrap().len(), 58_000); +} + +// ---- 200 000 chunks with two unlimited dimensions ---- + +#[test] +fn two_hundred_thousand_chunks_with_two_unlimited_dims() { + skip_if_no_python!(); + const U: u64 = u64::MAX; + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("chunks.h5").display().to_string(); + let data: Vec = (0..200_000).collect(); + let small: Vec = (0..80_000).map(|v| v * 2).collect(); + let mut b = FileBuilder::new(); + // Type 10 (unfiltered): 24-byte records, depth 2 in 2048-byte nodes. + b.create_dataset("d") + .with_i32_data(&data) + .with_shape(&[400, 500]) + .with_chunks(&[1, 1]) + .with_maxshape(&[U, U]); + // Type 11 (filtered): each record also holds a size and filter mask. + b.create_dataset("z") + .with_i32_data(&small) + .with_shape(&[200, 400]) + .with_chunks(&[1, 1]) + .with_maxshape(&[U, U]) + .with_deflate(1); + b.write(&path).unwrap(); + + let check = "\ + with h5py.File(path, 'r') as f:\n\ + \x20 d, z = f['d'], f['z']\n\ + \x20 print(json.dumps([d.shape, d.chunks, d.id.get_num_chunks(),\n\ + \x20 bool(np.array_equal(d[()], np.arange(200000).reshape(400, 500))),\n\ + \x20 int(d[399, 499]), z.id.get_num_chunks(),\n\ + \x20 bool(np.array_equal(z[()], 2 * np.arange(80000).reshape(200, 400)))]))"; + assert_eq!( + h5py(&path, check), + "[[400, 500], [1, 1], 200000, true, 199999, 80000, true]" + ); + let d = h5dump(&["-d", "/d", "-s", "399,498", "-c", "1,2", &path]); + assert!(d.is_empty() || d.contains("199998, 199999"), "{d}"); + + let f = File::open(&path).unwrap(); + assert_eq!(f.dataset("d").unwrap().read_i32().unwrap(), data); + assert_eq!(f.dataset("z").unwrap().read_i32().unwrap(), small); + drop(f); + + // libhdf5 adds chunks to both trees. + let modify = "\ + with h5py.File(path, 'r+') as f:\n\ + \x20 d, z = f['d'], f['z']\n\ + \x20 d.resize((401, 510))\n\ + \x20 d[400, :] = -1\n\ + \x20 d[:, 500:] = -2\n\ + \x20 z.resize((201, 400))\n\ + \x20 z[200, :] = 7\n\ + with h5py.File(path, 'r') as f:\n\ + \x20 d, z = f['d'][()], f['z'][()]\n\ + \x20 print(json.dumps([f['d'].id.get_num_chunks(),\n\ + \x20 bool(np.array_equal(d[:400, :500], np.arange(200000).reshape(400, 500))),\n\ + \x20 int(d[400, 3]), int(d[5, 505]), f['z'].id.get_num_chunks(),\n\ + \x20 bool(np.array_equal(z[:200], 2 * np.arange(80000).reshape(200, 400))), int(z[200, 9])]))"; + assert_eq!( + h5py(&path, modify), + "[204510, true, -1, -2, 80400, true, 7]" + ); + let f = File::open(&path).unwrap(); + let d = f.dataset("d").unwrap().read_i32().unwrap(); + assert_eq!(d.len(), 401 * 510); + assert_eq!(d[499], 499); + assert_eq!(d[400 * 510 + 3], -1); +} diff --git a/crates/clawhdf5/tests/writer_groups_interop.rs b/crates/clawhdf5/tests/writer_groups_interop.rs index 46d155b..d16567c 100644 --- a/crates/clawhdf5/tests/writer_groups_interop.rs +++ b/crates/clawhdf5/tests/writer_groups_interop.rs @@ -531,32 +531,6 @@ fn ten_thousand_links_in_one_group() { ); } -#[test] -fn more_links_than_one_index_leaf_holds_is_an_error() { - let mut b = FileBuilder::new(); - for i in 0..70_000 { - b.add_soft_link(&format!("s{i}"), "/x"); - } - let err = b.finish().unwrap_err().to_string(); - assert!( - err.contains("70000 links in one group: at most 65535"), - "{err}" - ); - // Dense attributes have the same one-leaf index. Their count used to - // be written modulo 65 536. - let mut b = FileBuilder::new(); - let x = b.create_dataset("x"); - x.with_i32_data(&[1]); - for i in 0..70_000 { - x.set_attr(&format!("a{i}"), AttrValue::I64(i)); - } - let err = b.finish().unwrap_err().to_string(); - assert!( - err.contains("70000 attributes on one object: at most 65535"), - "{err}" - ); -} - #[test] fn track_order_lists_members_in_creation_order() { skip_if_no_python!(); @@ -643,7 +617,11 @@ fn names_whose_hashes_collide_are_found_by_name() { .chain((0..10).map(|_| "")) .enumerate() { - let name = if n.is_empty() { format!("d{i}") } else { n.into() }; + let name = if n.is_empty() { + format!("d{i}") + } else { + n.into() + }; g.create_dataset(&name).with_i32_data(&[i as i32]); } b.add_group(g.finish()); diff --git a/docs/known-issues.md b/docs/known-issues.md index 2657cdf..865102b 100644 --- a/docs/known-issues.md +++ b/docs/known-issues.md @@ -264,10 +264,21 @@ fill-value item that did is fixed). external links at any depth and optional creation-order tracking; h5py, h5dump and `h5rs check --data` read them (`crates/clawhdf5/tests/writer_groups_interop.rs`, - `crates/clawhdf5-tools/tests/h5rs_interop.rs`). Still missing: a group - with more than 65 535 links, or an object with more than 65 535 dense - attributes, is an error (the index is one B-tree leaf), and attribute - creation order is not tracked. + `crates/clawhdf5-tools/tests/h5rs_interop.rs`). Still missing: + attribute creation order is not tracked. + - ~~A group with more than 65 535 links, or an object with more than + 65 535 dense attributes, is an error (the index is one B-tree leaf).~~ + **Fixed 2026-09-26:** the dense indexes are v2 B-trees of any depth + (libhdf5's 512-byte nodes once the records outgrow the one-leaf layout, + which smaller indexes keep byte for byte). Tested on tank with 100 000 + links in one group (short names, and 111-byte names with creation order + tracked) and 70 000 attributes on one object: h5py lists them in order + and reads the values, h5dump reads spot checks, `h5rs check` reads every + record, and h5py in "r+" mode adds and deletes thousands of links and + attributes in those trees (`cargo test -p clawhdf5 --test + deep_btree_interop`; `cargo test -p clawhdf5-tools --test h5rs_interop + check_files_with_deep_btrees`). The name indexes are ordered by hash and + then name, as libhdf5 needs for names whose hashes collide. - ~~Dense link or attribute storage past 512 KiB of messages was written unreadable (child indirect blocks of the fractal heap written as direct blocks).~~ **Fixed 2026-09-26** (it affected 2.7.0 too): tested with @@ -280,8 +291,13 @@ fill-value item that did is fixed). an object, or more than 8 links in a group) one attribute or link message over 65 515 bytes is an error. - Output that HDF5 1.8 can read. - - A B-tree v2 chunk index larger than one leaf, so datasets with several - unlimited dimensions are limited to 65 535 chunks. + - ~~A B-tree v2 chunk index larger than one leaf, so datasets with + several unlimited dimensions are limited to 65 535 chunks.~~ **Fixed + 2026-09-26:** more chunks get libhdf5's 2048-byte nodes with internal + nodes above the leaves. Tested on tank with 200 000 chunks (and 80 000 + deflated): h5py and clawhdf5 read every value, and h5py resizes the + dataset and writes 4 510 new chunks into the tree (same commands as + above). --- From 9e59499c56535a8d6252ee06b26658e57374785d Mon Sep 17 00:00:00 2001 From: osobh Date: Sat, 26 Sep 2026 10:13:11 -0500 Subject: [PATCH 04/43] conformance: fix three reference-probe artefacts ref.py and compare.py reported 13 files as mismatches or our-errors that were artefacts of the harness, not differences between the readers: - User-defined links (tall.h5, tudlink.h5, twithub*.h5, tmany.h5, ...): h5py's `get(name, getlink=True)` reports a user-defined link as a HardLink, so ref.py listed it as an object. Read the link type from H5Lget_info instead. - Objects h5py cannot open (cve-2019-8397/8398, cve-2021-46243, cve-2024-32618): the probe deduplicates by header address, ref.py by ObjectID, which an unopenable object does not have, so each extra hard link to it was listed again. Deduplicate those by link address. - Nested array types (tarray3.h5): h5py expands them into trailing dims; hash_values stripped one level and numpy broadcast every element into a whole subarray. Strip every level. compare.py no longer compares the attributes or links of an object h5py could not open at all (cve-2018-17438/17439, cve-2019-9151): h5py read none, so ours are neither extra nor errors against it. Co-Authored-By: Claude Opus 5.5 (1M context) --- conformance/compare.py | 8 +++++++- conformance/ref.py | 34 ++++++++++++++++++++++++---------- 2 files changed, 31 insertions(+), 11 deletions(-) diff --git a/conformance/compare.py b/conformance/compare.py index 8c14353..c1d65b2 100755 --- a/conformance/compare.py +++ b/conformance/compare.py @@ -146,7 +146,13 @@ for rel in files: if a.get("kind") != b.get("kind") and "error" not in b and "error" not in a: issues.append(("mismatch", f"{p}: kind {a.get('kind')} vs ours {b.get('kind')}", "kind", b)) ok = False + # h5py could not open the object at all: it read none of its + # attributes or links, so there is nothing to compare ours with + # (the object's own error is compared above and below). + ref_unopened = a.get("kind") == "unknown" and "error" in a for k in ("error", "list_error", "attrs_error"): + if ref_unopened and k != "error": + continue if k in b and k not in a: issues.append(("our-error", f"{p}: {k}: {b[k]}", b[k], b)) ok = False @@ -164,7 +170,7 @@ for rel in files: issues.append(("mismatch", f"{p}: values differ (h5py {a.get('dtype')} vs ours {b.get('dtype')})", "values", b | {"ref_head": a.get("head"), "ref_dtype": a.get("dtype")})) ok = False ra, oa = a.get("attrs") or {}, b.get("attrs") or {} - if "attrs_error" not in b and "attrs_error" not in a: + if "attrs_error" not in b and "attrs_error" not in a and not ref_unopened: for an in sorted(set(ra) | set(oa)): x, y = ra.get(an), oa.get(an) if x is None: diff --git a/conformance/ref.py b/conformance/ref.py index 573deeb..428b6a5 100755 --- a/conformance/ref.py +++ b/conformance/ref.py @@ -111,8 +111,11 @@ def note_conversion(tid, dt, rec): def hash_values(arr, dt, rec): - if dt.subdtype is not None: - # h5py expands an HDF5 array element type into trailing array dims + # h5py expands an HDF5 array element type into trailing array dims, a + # nested array type (an array of arrays) into all of them. Converting the + # expanded array back to the inner subarray type would broadcast every + # element into a whole subarray, so strip every level. + while dt.subdtype is not None: dt = dt.subdtype[0] arr = np.asarray(arr, dtype=dt) if simple(dt): @@ -173,9 +176,13 @@ def main(path): return objects = [] seen = set() - stack = [("/", None)] + # Objects h5py cannot open have no ObjectID to deduplicate by; they are + # deduplicated by the address their hard link points at instead, as the + # probe deduplicates every object by header address. + seen_unopenable = set() + stack = [("/", None, None)] while stack: - p, obj = stack.pop() + p, obj, link_addr = stack.pop() if len(objects) >= MAX_OBJECTS: top["truncated"] = True break @@ -185,6 +192,10 @@ def main(path): obj = f[p] key = hash(obj.id) # h5py ObjectID hash = (fileno, object address/token) except Exception as e: # noqa: BLE001 + if link_addr is not None: + if link_addr in seen_unopenable: + continue + seen_unopenable.add(link_addr) rec["kind"] = "unknown" rec["error"] = err(e) objects.append(rec) @@ -232,15 +243,18 @@ def main(path): base = "" if p == "/" else p kids = [] for n in names: + # The link's own type: `obj.get(n, getlink=True)` reports + # a user-defined link (type 64-255) as a HardLink. try: - link = obj.get(n, getlink=True) + info = obj.id.links.get_info(n.encode("utf-8", "surrogateescape")) except Exception: # noqa: BLE001 - link = None - if link is not None and not isinstance(link, h5py.HardLink): + info = None + if info is not None and info.type != h5py.h5l.TYPE_HARD: continue - kids.append(f"{base}/{n}") - for k in reversed(kids): - stack.append((k, None)) + addr = info.u if info is not None else None + kids.append((f"{base}/{n}", addr)) + for k, addr in reversed(kids): + stack.append((k, None, addr)) except Exception as e: # noqa: BLE001 rec["list_error"] = err(e) objects.append(rec) From f0db8176781192715a05658c191a00b60f924e47 Mon Sep 17 00:00:00 2001 From: osobh Date: Sat, 26 Sep 2026 10:12:38 -0500 Subject: [PATCH 05/43] format: decode full chunked reads straight into the output Full reads of a chunked dataset (the cached reader behind the facade's read_* and the uncached one behind mmap/lazy files and verify_provenance) now decode each chunk into this thread's reusable scratch buffers and copy it straight to its place in the output. Before, the cached reader decoded batches of 128 chunks into fresh Vecs and the uncached one decoded every chunk of the dataset into its own buffer before assembling any: a new 256 KiB allocation (and its page faults) per chunk and per filter stage. Chunks are still inserted into the file's chunk cache when the whole dataset fits in it. With the parallel feature the calling thread now decodes too, sharing the chunks with whichever rayon workers are free (run_with_helpers): a helper the busy pool only starts after the read is done returns at once. Before, the caller handed every chunk to the pool and slept, so readers outside a small pool (2-4 threads) queued behind its workers; with a one-thread pool the reads went sequential. Chunks are placed concurrently only when the index puts them on the chunk grid at distinct places (a corrupt index is read one chunk at a time), and the error returned is still the first failing chunk's. Fix: a chunk stored unfiltered in a filtered dataset (every filter-mask bit set) that is shorter than a chunk read as zeros where its data was missing through the cached reader (the facade's read_*); it is now an error naming the chunk, as the uncached reader already made it. Regression tests, both failing before this change: tests/busy_decode_pool.rs (both workers of a two-thread pool busy, four readers) and short_unfiltered_chunk_of_a_filtered_dataset_is_an_error. Co-Authored-By: Claude Opus 5.5 (1M context) --- crates/clawhdf5-format/src/chunked_read.rs | 1054 +++++++++++++------ crates/clawhdf5-format/src/parallel_read.rs | 174 +++ crates/clawhdf5/tests/busy_decode_pool.rs | 112 ++ 3 files changed, 1027 insertions(+), 313 deletions(-) create mode 100644 crates/clawhdf5/tests/busy_decode_pool.rs diff --git a/crates/clawhdf5-format/src/chunked_read.rs b/crates/clawhdf5-format/src/chunked_read.rs index 70176e2..7d3efd0 100644 --- a/crates/clawhdf5-format/src/chunked_read.rs +++ b/crates/clawhdf5-format/src/chunked_read.rs @@ -6,15 +6,16 @@ extern crate alloc; #[cfg(not(feature = "std"))] use alloc::{format, vec, vec::Vec}; -use crate::chunk_cache::CacheAlignedBuffer; #[cfg(feature = "std")] -use crate::chunk_cache::ChunkCache; +use crate::chunk_cache::{CacheAlignedBuffer, ChunkCache}; use crate::data_layout::DataLayout; use crate::dataspace::Dataspace; use crate::datatype::Datatype; use crate::error::FormatError; use crate::extensible_array::{ExtensibleArrayHeader, read_extensible_array_chunks}; use crate::filter_pipeline::FilterPipeline; +use crate::filters::{DecodeScratch, decompress_chunk_exact_with}; +#[cfg(feature = "std")] use crate::filters::{all_filters_skipped, decompress_chunk_exact}; use crate::fixed_array::{FixedArrayHeader, read_fixed_array_chunks}; #[cfg(feature = "std")] @@ -26,60 +27,329 @@ use crate::parallel_read; #[cfg(feature = "parallel")] use crate::lane_partition::PartitionStats; -/// Decompress all chunks into cache-line-aligned buffers, using lane-partitioned -/// parallel decompression when the `parallel` feature is enabled and the chunk -/// count exceeds the threshold. -fn decompress_all_chunks( - file_data: &[u8], - chunks: &[ChunkInfo], - pipeline: Option<&FilterPipeline>, - chunk_total_bytes: usize, - element_size: u32, -) -> Result, FormatError> { - #[cfg(feature = "parallel")] +/// Run `f` with this thread's chunk-decoding scratch buffers (see +/// [`DecodeScratch`]), kept between reads so decoding reuses memory instead +/// of faulting in fresh pages for every chunk. A re-entrant call (a +/// registered filter codec that itself reads a file) gets a fresh scratch. +fn with_scratch(f: impl FnOnce(&mut DecodeScratch) -> R) -> R { + #[cfg(feature = "std")] { - if let Some(pl) = pipeline - && parallel_read::should_use_parallel(chunks.len()) - && parallel_read::pool_can_parallelise() - { - // Seed from the first chunk's address and count for determinism. - let seed = chunks.first().map(|c| c.address).unwrap_or(0) ^ (chunks.len() as u64); - let (data, _stats) = parallel_read::decompress_chunks_lane_partitioned( - file_data, - chunks, - pl, - chunk_total_bytes, - element_size, - seed, - None, // auto-detect lane count - )?; - return Ok(data.into_iter().map(CacheAlignedBuffer::from_vec).collect()); + use std::cell::RefCell; + std::thread_local! { + static SCRATCH: RefCell = RefCell::new(DecodeScratch::new()); + } + let mut f = Some(f); + let kept = SCRATCH.try_with(|cell| { + let mut scratch = cell.try_borrow_mut().ok()?; + let f = f.take()?; + let r = f(&mut scratch); + scratch.trim(); + Some(r) + }); + if let Ok(Some(r)) = kept { + return r; + } + let f = f.expect("with_scratch: closure already run"); + f(&mut DecodeScratch::new()) + } + #[cfg(not(feature = "std"))] + f(&mut DecodeScratch::new()) +} + +/// A full read's output buffer, written through a raw pointer so that +/// several threads can place chunks into it at once. +struct OutBuf<'a> { + ptr: *mut u8, + len: usize, + _borrow: core::marker::PhantomData<&'a mut [u8]>, +} + +// SAFETY: `OutBuf` is a `&mut [u8]` that hands out writes; sharing it across +// threads is sound as long as concurrent writes do not overlap, which +// `write`'s contract requires. +unsafe impl Send for OutBuf<'_> {} +unsafe impl Sync for OutBuf<'_> {} + +impl<'a> OutBuf<'a> { + fn new(out: &'a mut [u8]) -> Self { + Self { + ptr: out.as_mut_ptr(), + len: out.len(), + _borrow: core::marker::PhantomData, } } - // Sequential fallback — allocate into aligned buffers - let mut result = Vec::with_capacity(chunks.len()); - for chunk_info in chunks { - let c_addr = chunk_info.address as usize; - let size = chunk_info.chunk_size as usize; - ensure_len(file_data, c_addr, size)?; - let raw_chunk = &file_data[c_addr..c_addr + size]; - - let decompressed = if let Some(pl) = pipeline { - decompress_chunk_exact( - raw_chunk, - pl, - chunk_total_bytes, - element_size, - chunk_info.filter_mask, - &chunk_info.offsets, - )? - } else { - raw_chunk.to_vec() - }; - result.push(CacheAlignedBuffer::from_vec(decompressed)); + fn len(&self) -> usize { + self.len } - Ok(result) + + /// Copy `src` to `[at, at + src.len())`. Out of range is a no-op (the + /// callers check first). + /// + /// # Safety + /// + /// No other thread may be writing an overlapping range at the same time. + unsafe fn write(&self, at: usize, src: &[u8]) { + if at.checked_add(src.len()).is_none_or(|end| end > self.len) { + debug_assert!(false, "OutBuf::write out of range"); + return; + } + // SAFETY: in range (checked above) of a live `&'a mut [u8]`; `src` + // cannot overlap it (the output is exclusively borrowed); no + // concurrent overlapping write (the caller's contract). + unsafe { core::ptr::copy_nonoverlapping(src.as_ptr(), self.ptr.add(at), src.len()) } + } +} + +/// How a chunked dataset's chunks map into its row-major output. +struct ChunkPlacer { + rank: usize, + chunk_dims: Vec, + ds_dims: Vec, + ds_strides: Vec, + chunk_strides: Vec, + elem_size: usize, +} + +impl ChunkPlacer { + /// `chunk_dims` and `ds_dims` have one entry per dimension; the dataset + /// must not be empty (so the stride products stay in range). + fn new(chunk_dims: &[usize], ds_dims: &[usize], elem_size: usize) -> Self { + let rank = chunk_dims.len(); + let mut ds_strides = vec![1usize; rank]; + let mut chunk_strides = vec![1usize; rank]; + for i in (0..rank.saturating_sub(1)).rev() { + ds_strides[i] = ds_strides[i + 1] * ds_dims[i + 1]; + chunk_strides[i] = chunk_strides[i + 1] * chunk_dims[i + 1]; + } + Self { + rank, + chunk_dims: chunk_dims.to_vec(), + ds_dims: ds_dims.to_vec(), + ds_strides, + chunk_strides, + elem_size, + } + } + + /// Copy the decoded chunk `data`, whose first element is at `offsets` + /// (one per dimension), to its place in `out`. + /// + /// # Safety + /// + /// No other thread may be placing a chunk whose region overlaps this + /// one's (see [`Self::regions_disjoint`]). + unsafe fn place(&self, data: &[u8], offsets: &[u64], out: &OutBuf<'_>) { + if self.rank == 0 { + let copy_len = data.len().min(out.len()); + // SAFETY: in range; exclusivity is the caller's contract. + unsafe { out.write(0, &data[..copy_len]) }; + return; + } + let mut short = [0usize; 8]; + let mut long = Vec::new(); + let chunk_offsets: &mut [usize] = if self.rank <= short.len() { + &mut short[..self.rank] + } else { + long.resize(self.rank, 0); + &mut long + }; + for (o, &off) in chunk_offsets.iter_mut().zip(offsets) { + *o = off as usize; + } + // SAFETY: the caller's contract. + unsafe { + copy_chunk_into( + data, + out, + chunk_offsets, + &self.chunk_dims, + &self.ds_dims, + &self.ds_strides, + &self.chunk_strides, + self.elem_size, + self.rank, + ) + }; + } + + /// Whether placing `chunks` writes pairwise disjoint regions of the + /// output, so they can be placed concurrently: every chunk starts on the + /// chunk grid and no two start at the same place (a chunk wholly outside + /// the dataset writes nothing and is ignored). A corrupt index that + /// breaks this is read one chunk at a time instead. + #[cfg(feature = "parallel")] + fn regions_disjoint(&self, chunks: &[ChunkInfo]) -> bool { + if self.rank == 0 { + return chunks.len() <= 1; + } + let mut keys = Vec::with_capacity(chunks.len()); + 'chunks: for c in chunks { + if c.offsets.len() < self.rank { + return false; + } + let mut key = 0u64; + for d in 0..self.rank { + let (off, cd, dd) = ( + c.offsets[d], + self.chunk_dims[d] as u64, + self.ds_dims[d] as u64, + ); + if off >= dd { + continue 'chunks; + } + if off % cd != 0 { + return false; + } + let Some(k) = key + .checked_mul(dd.div_ceil(cd)) + .and_then(|k| k.checked_add(off / cd)) + else { + return false; + }; + key = k; + } + keys.push(key); + } + keys.sort_unstable(); + keys.windows(2).all(|w| w[0] != w[1]) + } +} + +/// The per-file chunk cache a full read uses, if any: the cache, this +/// dataset's key in it (its chunk-index address), and whether the dataset +/// fits in it — only then are its chunks looked up and inserted. +#[cfg(feature = "std")] +type CacheUse<'a> = Option<(&'a ChunkCache, u64, bool)>; +#[cfg(not(feature = "std"))] +type CacheUse<'a> = Option<&'a core::convert::Infallible>; + +/// Decode every chunk in `chunks` and place it in `output` (the dataset's +/// whole row-major extent, zeroed). +/// +/// Each chunk goes straight from its decoder to its place in the output: +/// decoded into this thread's reusable scratch (or, when the cache is to +/// keep it, into a buffer the cache takes), then copied. With the +/// `parallel` feature and a filter pipeline, the chunks are shared out +/// between the calling thread and idle rayon workers +/// ([`parallel_read::run_with_helpers`]), so the caller never waits on a +/// busy pool. The error returned is the first failing chunk's, in `chunks` +/// order. +#[allow(clippy::too_many_arguments)] +fn fill_from_chunks( + file_data: &[u8], + chunks: &[ChunkInfo], + pipeline: Option<&FilterPipeline>, + placer: &ChunkPlacer, + chunk_total_bytes: usize, + cache: CacheUse<'_>, + output: &mut [u8], +) -> Result<(), FormatError> { + let rank = placer.rank; + let elem_size = placer.elem_size as u32; + let out = OutBuf::new(output); + #[cfg(not(feature = "std"))] + let _ = cache; + + // Decode chunk `i` and place it. Its callers below run it either on one + // thread, or on several for chunks whose regions are pairwise disjoint, + // each chunk once: no two threads ever write the same bytes. + let work = |i: usize, scratch: &mut DecodeScratch| -> Result<(), FormatError> { + let c = &chunks[i]; + if c.offsets.len() < rank { + return Err(FormatError::ChunkedReadError(format!( + "chunk index entry has {} offsets for a rank-{rank} dataset", + c.offsets.len() + ))); + } + let offsets = &c.offsets[..rank]; + let c_addr = c.address as usize; + let size = c.chunk_size as usize; + ensure_len(file_data, c_addr, size)?; + let raw = &file_data[c_addr..c_addr + size]; + let Some(pl) = pipeline else { + // SAFETY: see above. + unsafe { placer.place(raw, offsets, &out) }; + return Ok(()); + }; + // A chunk stored as-is (every filter skipped) is checked and placed + // straight from the file bytes, never cached. + #[cfg(feature = "std")] + if let Some((cache, key, true)) = cache + && !all_filters_skipped(pl, c.filter_mask) + { + let cached = match cache.get_decompressed_in(key, offsets) { + Some(hit) => hit, + None => { + let data = decompress_chunk_exact( + raw, + pl, + chunk_total_bytes, + elem_size, + c.filter_mask, + &c.offsets, + )?; + cache.put_decompressed_in(key, offsets.to_vec(), data) + } + }; + // SAFETY: see above. + unsafe { placer.place(&cached, offsets, &out) }; + return Ok(()); + } + let data = decompress_chunk_exact_with( + raw, + pl, + chunk_total_bytes, + elem_size, + c.filter_mask, + &c.offsets, + scratch, + )?; + // SAFETY: see above. + unsafe { placer.place(data, offsets, &out) }; + Ok(()) + }; + + #[cfg(feature = "parallel")] + if pipeline.is_some() + && parallel_read::should_use_parallel(chunks.len()) + && placer.regions_disjoint(chunks) + { + use core::sync::atomic::{AtomicUsize, Ordering}; + use std::sync::{Mutex, PoisonError}; + + let n = chunks.len(); + let next = AtomicUsize::new(0); + let failed: Mutex> = Mutex::new(None); + let body = || { + with_scratch(|scratch| { + loop { + let i = next.fetch_add(1, Ordering::Relaxed); + if i >= n { + break; + } + if let Err(e) = work(i, scratch) { + let mut failed = failed.lock().unwrap_or_else(PoisonError::into_inner); + if failed.as_ref().is_none_or(|(at, _)| i < *at) { + *failed = Some((i, e)); + } + // Stop handing out chunks. Every chunk before `i` was + // claimed already and finishes, so the error kept is + // the first in order. + next.store(n, Ordering::Relaxed); + break; + } + } + }) + }; + parallel_read::run_with_helpers(parallel_read::helper_count(n), &body); + return match failed.into_inner().unwrap_or_else(PoisonError::into_inner) { + Some((_, e)) => Err(e), + None => Ok(()), + }; + } + + with_scratch(|scratch| (0..chunks.len()).try_for_each(|i| work(i, scratch))) } /// Decompress all chunks with lane-partitioned parallelism and return @@ -557,11 +827,6 @@ pub fn generate_implicit_chunks( chunks } -/// Read a chunked dataset, decompressing chunks as needed. -/// Chunks decompressed together before being copied out, bounding the extra -/// memory a parallel full read holds at once. -const DECODE_BATCH: usize = 128; - /// B-tree v2 record types used for chunk indexing. const BT2_CHUNK_UNFILTERED: u8 = 10; const BT2_CHUNK_FILTERED: u8 = 11; @@ -820,6 +1085,111 @@ pub fn list_chunks( Ok((chunks, chunk_dims)) } +/// The chunk cache a full read may use (`None` without `std`). +#[cfg(feature = "std")] +pub(crate) type CacheRef<'a> = Option<&'a ChunkCache>; +/// The chunk cache a full read may use (`None` without `std`). +#[cfg(not(feature = "std"))] +pub(crate) type CacheRef<'a> = Option<&'a core::convert::Infallible>; + +/// The body of every full chunked read: list the chunks (through the +/// cache's index for this dataset when there is a cache), allocate the +/// output with `alloc` (zeroed, `total_bytes` long, as bytes through +/// `bytes`), and decode every chunk straight into it. +#[allow(clippy::too_many_arguments)] +pub(crate) fn read_chunked_full( + file_data: &[u8], + layout: &DataLayout, + dataspace: &Dataspace, + datatype: &Datatype, + pipeline: Option<&FilterPipeline>, + offset_size: u8, + length_size: u8, + cache: CacheRef<'_>, + alloc: impl FnOnce(usize) -> Result, + bytes: impl FnOnce(&mut O) -> &mut [u8], +) -> Result { + check_chunk_element_size(layout, datatype, offset_size)?; + let elem_size = datatype.type_size() as usize; + let list = || { + list_chunks( + file_data, + layout, + dataspace, + elem_size, + offset_size, + length_size, + ) + }; + + #[cfg(feature = "std")] + let (chunks, chunk_dims, cache_key) = match cache { + Some(cache) => { + let (chunk_dimensions, version, addr_opt) = match layout { + DataLayout::Chunked { + chunk_dimensions, + version, + btree_address, + .. + } => (chunk_dimensions, *version, *btree_address), + _ => { + return Err(FormatError::ChunkedReadError( + "expected chunked layout".into(), + )); + } + }; + let addr = addr_opt.ok_or_else(|| { + FormatError::ChunkedReadError("no address for chunked layout".into()) + })?; + let (rank, chunk_dims) = + chunk_geometry(chunk_dimensions, version, dataspace, elem_size)?; + // The per-file cache is shared across datasets (and threads); + // every lookup is keyed by this dataset's chunk-index address, so + // another dataset's index or chunks are never used for this read. + let chunks = cache.chunks_for(addr, rank, || list().map(|(chunks, _)| chunks))?; + (chunks, chunk_dims, Some((cache, addr))) + } + None => { + let (chunks, chunk_dims) = list()?; + (chunks, chunk_dims, None) + } + }; + #[cfg(not(feature = "std"))] + let (chunks, chunk_dims) = { + let _ = cache; + list()? + }; + + let total_bytes = checked_byte_len(dataspace.checked_num_elements()?, elem_size)?; + let mut output = alloc(total_bytes)?; + if total_bytes == 0 { + // Also keeps the stride products in range: with a zero-sized + // dimension the total is 0 even if other dimensions are huge. + return Ok(output); + } + let ds_dims: Vec = dataspace.dimensions.iter().map(|&d| d as usize).collect(); + let placer = ChunkPlacer::new(&chunk_dims, &ds_dims, elem_size); + let chunk_total_bytes = checked_chunk_byte_len(&chunk_dims, elem_size)?; + // Chunks are cached only when the whole dataset fits: pushing a larger + // dataset through the cache just evicts each chunk moments after + // inserting it. + #[cfg(feature = "std")] + let cache_use = cache_key.map(|(cache, key)| (cache, key, total_bytes <= cache.max_bytes())); + #[cfg(not(feature = "std"))] + let cache_use = None; + fill_from_chunks( + file_data, + &chunks, + pipeline, + &placer, + chunk_total_bytes, + cache_use, + bytes(&mut output), + )?; + Ok(output) +} + +/// Read a chunked dataset, decompressing chunks as needed. pub fn read_chunked_data( file_data: &[u8], layout: &DataLayout, @@ -829,118 +1199,26 @@ pub fn read_chunked_data( offset_size: u8, length_size: u8, ) -> Result, FormatError> { - check_chunk_element_size(layout, datatype, offset_size)?; - let elem_size = datatype.type_size() as usize; - let (chunks, chunk_dims) = list_chunks( + read_chunked_full( file_data, layout, dataspace, - elem_size, + datatype, + pipeline, offset_size, length_size, - )?; - let rank = chunk_dims.len(); - let ds_dims: Vec = dataspace.dimensions.iter().map(|&d| d as usize).collect(); - - // Assemble output - let total_bytes = checked_byte_len(dataspace.checked_num_elements()?, elem_size)?; - if total_bytes == 0 { - // Also keeps the stride products below in range: with a zero-sized - // dimension the total is 0 even if other dimensions are huge. - return Ok(Vec::new()); - } - let mut output = alloc_output(total_bytes)?; - - let mut ds_strides = vec![1usize; rank]; - for i in (0..rank.saturating_sub(1)).rev() { - ds_strides[i] = ds_strides[i + 1] * ds_dims[i + 1]; - } - - let mut chunk_strides = vec![1usize; rank]; - for i in (0..rank.saturating_sub(1)).rev() { - chunk_strides[i] = chunk_strides[i + 1] * chunk_dims[i + 1]; - } - - let chunk_total_bytes = checked_chunk_byte_len(&chunk_dims, elem_size)?; - - // Fast path: no filters — copy directly from file_data without intermediate alloc - if pipeline.is_none() { - for chunk_info in &chunks { - let chunk_offsets: Vec = chunk_info - .offsets - .iter() - .take(rank) - .map(|&o| o as usize) - .collect(); - - let c_addr = chunk_info.address as usize; - let size = chunk_info.chunk_size as usize; - ensure_len(file_data, c_addr, size)?; - let chunk_data = &file_data[c_addr..c_addr + size]; - - if rank == 0 { - let copy_len = chunk_data.len().min(output.len()); - output[..copy_len].copy_from_slice(&chunk_data[..copy_len]); - } else { - copy_chunk_to_output( - chunk_data, - &mut output, - &chunk_offsets, - &chunk_dims, - &ds_dims, - &ds_strides, - &chunk_strides, - elem_size, - rank, - ); - } - } - return Ok(output); - } - - // Filtered path: decompress all chunks then assemble - let decompressed_chunks = decompress_all_chunks( - file_data, - &chunks, - pipeline, - chunk_total_bytes, - elem_size as u32, - )?; - - for (chunk_info, decompressed) in chunks.iter().zip(decompressed_chunks.iter()) { - let chunk_offsets: Vec = chunk_info - .offsets - .iter() - .take(rank) - .map(|&o| o as usize) - .collect(); - - if rank == 0 { - let copy_len = decompressed.len().min(output.len()); - output[..copy_len].copy_from_slice(&decompressed[..copy_len]); - } else { - copy_chunk_to_output( - decompressed, - &mut output, - &chunk_offsets, - &chunk_dims, - &ds_dims, - &ds_strides, - &chunk_strides, - elem_size, - rank, - ); - } - } - - Ok(output) + None, + alloc_output, + |out| out.as_mut_slice(), + ) } /// Read a chunked dataset with caching support. /// /// On the first call, scans the chunk index (B-tree / fixed array / etc.) once /// and populates the cache's hash index. Subsequent calls skip the index scan -/// entirely. Decompressed chunk data is also cached with LRU eviction. +/// entirely. Decompressed chunk data is also cached with LRU eviction, when +/// the whole dataset fits in the cache. #[cfg(feature = "std")] #[allow(clippy::too_many_arguments)] pub fn read_chunked_data_cached( @@ -953,159 +1231,18 @@ pub fn read_chunked_data_cached( length_size: u8, cache: &ChunkCache, ) -> Result, FormatError> { - let (chunk_dimensions, version, addr_opt) = match layout { - DataLayout::Chunked { - chunk_dimensions, - version, - btree_address, - .. - } => (chunk_dimensions, *version, *btree_address), - _ => { - return Err(FormatError::ChunkedReadError( - "expected chunked layout".into(), - )); - } - }; - - let addr = addr_opt - .ok_or_else(|| FormatError::ChunkedReadError("no address for chunked layout".into()))?; - - check_chunk_element_size(layout, datatype, offset_size)?; - let elem_size = datatype.type_size() as usize; - let (rank, chunk_dims) = chunk_geometry(chunk_dimensions, version, dataspace, elem_size)?; - let ds_dims: Vec = dataspace.dimensions.iter().map(|&d| d as usize).collect(); - - // The per-file cache is shared across datasets (and threads); every - // lookup is keyed by this dataset's chunk-index address, so another - // dataset's index or chunks are never used for this read. - let chunks = cache.chunks_for(addr, rank, || { - list_chunks( - file_data, - layout, - dataspace, - elem_size, - offset_size, - length_size, - ) - .map(|(chunks, _)| chunks) - })?; - - // Assemble output - let total_bytes = checked_byte_len(dataspace.checked_num_elements()?, elem_size)?; - if total_bytes == 0 { - // Also keeps the stride products below in range: with a zero-sized - // dimension the total is 0 even if other dimensions are huge. - return Ok(Vec::new()); - } - let mut output = alloc_output(total_bytes)?; - - let mut ds_strides = vec![1usize; rank]; - for i in (0..rank.saturating_sub(1)).rev() { - ds_strides[i] = ds_strides[i + 1] * ds_dims[i + 1]; - } - - let mut chunk_strides = vec![1usize; rank]; - for i in (0..rank.saturating_sub(1)).rev() { - chunk_strides[i] = chunk_strides[i + 1] * chunk_dims[i + 1]; - } - - let chunk_total_bytes = checked_chunk_byte_len(&chunk_dims, elem_size)?; - - let mut place = |data: &[u8], chunk_info: &ChunkInfo| { - if rank == 0 { - let copy_len = data.len().min(output.len()); - output[..copy_len].copy_from_slice(&data[..copy_len]); - return; - } - let chunk_offsets: Vec = chunk_info - .offsets - .iter() - .take(rank) - .map(|&o| o as usize) - .collect(); - copy_chunk_to_output( - data, - &mut output, - &chunk_offsets, - &chunk_dims, - &ds_dims, - &ds_strides, - &chunk_strides, - elem_size, - rank, - ); - }; - let raw_bytes = |chunk_info: &ChunkInfo| -> Result<&[u8], FormatError> { - let c_addr = chunk_info.address as usize; - let size = chunk_info.chunk_size as usize; - ensure_len(file_data, c_addr, size)?; - Ok(&file_data[c_addr..c_addr + size]) - }; - - // Chunks stored as-is (no pipeline, or the filter mask says this chunk - // skipped every filter) are copied straight from the file bytes: they are - // already in memory, so routing them through a Vec and then an aligned - // cache buffer was two extra copies of the whole dataset for nothing. - let stored_raw = - |c: &ChunkInfo| pipeline.is_none_or(|pl| all_filters_skipped(pl, c.filter_mask)); - let mut misses: Vec<&ChunkInfo> = Vec::new(); - for chunk_info in &chunks { - if stored_raw(chunk_info) { - place(raw_bytes(chunk_info)?, chunk_info); - continue; - } - let coord: Vec = chunk_info.offsets.iter().take(rank).copied().collect(); - match cache.get_decompressed_in(addr, &coord) { - Some(cached) => place(&cached, chunk_info), - None => misses.push(chunk_info), - } - } - - // Decompress what the cache didn't have, a bounded batch at a time — in - // parallel with the `parallel` feature (this path, the one the facade - // uses, was sequential; only the uncached reader was parallel), unless the - // pool has one thread: then every reading thread would queue behind that - // one worker, so each decodes its own chunks instead. Chunks are - // cached only when the whole dataset fits: pushing a larger dataset - // through the cache just evicts each chunk moments after inserting it. - let cache_them = total_bytes <= cache.max_bytes(); - if let Some(pl) = pipeline { - let decode = |c: &&ChunkInfo| -> Result, FormatError> { - decompress_chunk_exact( - raw_bytes(c)?, - pl, - chunk_total_bytes, - elem_size as u32, - c.filter_mask, - &c.offsets, - ) - }; - for batch in misses.chunks(DECODE_BATCH) { - #[cfg(feature = "parallel")] - let decoded: Vec, FormatError>> = - if batch.len() >= 4 && parallel_read::pool_can_parallelise() { - use rayon::prelude::*; - batch.par_iter().map(decode).collect() - } else { - batch.iter().map(decode).collect() - }; - #[cfg(not(feature = "parallel"))] - let decoded: Vec, FormatError>> = batch.iter().map(decode).collect(); - - for (chunk_info, data) in batch.iter().zip(decoded) { - let data = data?; - if cache_them { - let coord: Vec = chunk_info.offsets.iter().take(rank).copied().collect(); - let cached = cache.put_decompressed_in(addr, coord, data); - place(&cached, chunk_info); - } else { - place(&data, chunk_info); - } - } - } - } - - Ok(output) + read_chunked_full( + file_data, + layout, + dataspace, + datatype, + pipeline, + offset_size, + length_size, + Some(cache), + alloc_output, + |out| out.as_mut_slice(), + ) } /// Sweep context passed into `read_chunked_data_sweep` to enable adaptive @@ -1486,6 +1623,7 @@ pub fn read_chunked_data_indexed( } /// Copy chunk data into the output buffer at the correct N-D position. +#[cfg(any(feature = "std", test))] #[allow(clippy::too_many_arguments)] fn copy_chunk_to_output( chunk_data: &[u8], @@ -1497,6 +1635,41 @@ fn copy_chunk_to_output( chunk_strides: &[usize], elem_size: usize, rank: usize, +) { + // SAFETY: `output` is exclusively borrowed, so no other write can + // overlap this one. + unsafe { + copy_chunk_into( + chunk_data, + &OutBuf::new(output), + chunk_offsets, + chunk_dims, + ds_dims, + ds_strides, + chunk_strides, + elem_size, + rank, + ) + } +} + +/// [`copy_chunk_to_output`] through an [`OutBuf`]: only the bytes of the +/// chunk's region (clipped to the dataset) are written. +/// +/// # Safety +/// +/// No other thread may be writing an overlapping region of `output`. +#[allow(clippy::too_many_arguments)] +unsafe fn copy_chunk_into( + chunk_data: &[u8], + output: &OutBuf<'_>, + chunk_offsets: &[usize], + chunk_dims: &[usize], + ds_dims: &[usize], + ds_strides: &[usize], + chunk_strides: &[usize], + elem_size: usize, + rank: usize, ) { // Row-copy approach: iterate over outer dimensions, memcpy the innermost // dimension in bulk. For 1-D data this is a single memcpy per chunk. @@ -1518,7 +1691,8 @@ fn copy_chunk_to_output( .is_some_and(|end| end <= output.len()) && src_bytes <= chunk_data.len() { - output[dst_start..dst_start + src_bytes].copy_from_slice(&chunk_data[..src_bytes]); + // SAFETY: in range (checked); exclusivity is the caller's contract. + unsafe { output.write(dst_start, &chunk_data[..src_bytes]) }; } return; } @@ -1620,8 +1794,8 @@ fn copy_chunk_to_output( .checked_add(row_bytes) .is_some_and(|end| end <= output.len()); if fits { - output[dst_start..dst_start + row_bytes] - .copy_from_slice(&chunk_data[src_start..src_start + row_bytes]); + // SAFETY: in range (checked); exclusivity is the caller's contract. + unsafe { output.write(dst_start, &chunk_data[src_start..src_start + row_bytes]) }; } } } @@ -2071,6 +2245,198 @@ mod tests { ); } + /// A 2-D `f32` dataset of `rows x cols` in `cr x cc` chunks, deflated + /// behind shuffle: (file bytes, chunk list, pipeline, expected output). + #[cfg(feature = "deflate")] + fn deflated_grid( + rows: usize, + cols: usize, + cr: usize, + cc: usize, + ) -> (Vec, Vec, FilterPipeline, Vec) { + use crate::filter_pipeline::{FILTER_DEFLATE, FILTER_SHUFFLE, FilterDescription}; + let pipeline = FilterPipeline { + version: 2, + filters: vec![ + FilterDescription { + filter_id: FILTER_SHUFFLE, + name: None, + flags: 0, + client_data: vec![4], + }, + FilterDescription { + filter_id: FILTER_DEFLATE, + name: None, + flags: 0, + client_data: vec![1], + }, + ], + }; + let value = |r: usize, c: usize| (r * cols + c) as f32 * 0.5; + let expected: Vec = (0..rows * cols) + .flat_map(|i| value(i / cols, i % cols).to_le_bytes()) + .collect(); + let (mut file, mut chunks) = (vec![0u8; 8], Vec::new()); + for r0 in (0..rows).step_by(cr) { + for c0 in (0..cols).step_by(cc) { + // Full-size chunks; the edge padding is zeros. + let chunk: Vec = (0..cr * cc) + .flat_map(|i| { + let (r, c) = (r0 + i / cc, c0 + i % cc); + let v = if r < rows && c < cols { + value(r, c) + } else { + 0.0 + }; + v.to_le_bytes() + }) + .collect(); + let stored = crate::filters::compress_chunk(&chunk, &pipeline, 4).unwrap(); + chunks.push(ChunkInfo { + chunk_size: stored.len() as u32, + filter_mask: 0, + offsets: vec![r0 as u64, c0 as u64, 0], + address: file.len() as u64, + }); + file.extend(stored); + } + } + (file, chunks, pipeline, expected) + } + + #[cfg(feature = "deflate")] + fn fill( + file: &[u8], + chunks: &[ChunkInfo], + pipeline: &FilterPipeline, + dims: [usize; 2], + chunk: [usize; 2], + ) -> Result, FormatError> { + let placer = ChunkPlacer::new(&chunk, &dims, 4); + let mut out = vec![0u8; dims[0] * dims[1] * 4]; + fill_from_chunks( + file, + chunks, + Some(pipeline), + &placer, + chunk[0] * chunk[1] * 4, + None, + &mut out, + )?; + Ok(out) + } + + /// Chunks decoded straight into the output (in parallel with the + /// `parallel` feature, partial edge chunks included) land where the + /// row-by-row reference puts them. + #[test] + #[cfg(feature = "deflate")] + fn chunks_fill_the_output_in_place() { + for (dims, chunk) in [ + ([100, 70], [16, 32]), + ([64, 64], [16, 16]), + ([5, 3], [8, 8]), + ] { + let (file, chunks, pipeline, expected) = + deflated_grid(dims[0], dims[1], chunk[0], chunk[1]); + for _ in 0..4 { + assert_eq!( + fill(&file, &chunks, &pipeline, dims, chunk).unwrap(), + expected + ); + } + // Any chunk order gives the same output. + let mut reversed = chunks.clone(); + reversed.reverse(); + assert_eq!( + fill(&file, &reversed, &pipeline, dims, chunk).unwrap(), + expected + ); + } + } + + /// Of several corrupt chunks, the error names the first in chunk order, + /// however the chunks were shared out between threads. + #[test] + #[cfg(feature = "deflate")] + fn first_corrupt_chunk_is_the_error() { + let (dims, chunk) = ([128, 64], [8, 16]); + let (mut file, mut chunks, pipeline, _) = + deflated_grid(dims[0], dims[1], chunk[0], chunk[1]); + // Valid streams that decode short: an error naming the chunk. + let short = crate::filters::compress_chunk(&[1u8; 64], &pipeline, 4).unwrap(); + for bad in [5usize, 11, 40] { + chunks[bad].address = file.len() as u64; + chunks[bad].chunk_size = short.len() as u32; + file.extend_from_slice(&short); + } + for _ in 0..20 { + let err = fill(&file, &chunks, &pipeline, dims, chunk).unwrap_err(); + let want = format!("{:?}", chunks[5].offsets); + assert!(err.to_string().contains(&want), "{err} (want {want})"); + } + } + + /// Only chunks on the grid, each at a distinct place, may be placed + /// concurrently; anything else is read one chunk at a time. + #[test] + #[cfg(feature = "parallel")] + fn regions_disjoint_only_for_distinct_grid_chunks() { + let placer = ChunkPlacer::new(&[10, 10], &[25, 30], 4); + let chunk = |r: u64, c: u64| ChunkInfo { + chunk_size: 0, + filter_mask: 0, + offsets: vec![r, c, 0], + address: 0, + }; + let grid: Vec = (0..3) + .flat_map(|r| (0..3).map(move |c| chunk(r * 10, c * 10))) + .collect(); + assert!(placer.regions_disjoint(&grid)); + // Chunks wholly outside the dataset write nothing. + let mut outside = grid.clone(); + outside.push(chunk(30, 0)); + outside.push(chunk(30, 0)); + assert!(placer.regions_disjoint(&outside)); + let mut duplicate = grid.clone(); + duplicate.push(chunk(10, 20)); + assert!(!placer.regions_disjoint(&duplicate)); + let mut off_grid = grid.clone(); + off_grid[4] = chunk(15, 10); + assert!(!placer.regions_disjoint(&off_grid)); + let mut short = grid; + short[0].offsets.truncate(1); + assert!(!placer.regions_disjoint(&short)); + } + + /// A duplicated chunk in a corrupt index is not placed from two threads + /// at once: the read goes one chunk at a time, as before. + #[test] + #[cfg(feature = "deflate")] + fn duplicate_chunks_are_read_in_order() { + let (dims, chunk) = ([64, 64], [16, 16]); + let (file, mut chunks, pipeline, expected) = + deflated_grid(dims[0], dims[1], chunk[0], chunk[1]); + // A second entry at chunk 3's place, holding chunk 7's data: the + // later entry wins, as a sequential read has it. + let mut dup = chunks[7].clone(); + dup.offsets = chunks[3].offsets.clone(); + chunks.push(dup); + let got = fill(&file, &chunks, &pipeline, dims, chunk).unwrap(); + let chunk7_in_3: Vec = { + let mut e = expected.clone(); + let (r3, c3) = (chunks[3].offsets[0] as usize, chunks[3].offsets[1] as usize); + let (r7, c7) = (chunks[7].offsets[0] as usize, chunks[7].offsets[1] as usize); + for r in 0..16 { + let src = ((r7 + r) * 64 + c7) * 4; + let dst = ((r3 + r) * 64 + c3) * 4; + e[dst..dst + 64].copy_from_slice(&expected[src..src + 64]); + } + e + }; + assert_eq!(got, chunk7_in_3); + } + #[test] fn copy_chunk_to_output_1d_rejects_overflowing_offset_without_panicking() { // Found by fuzzing: `global_start * elem_size` overflowed for a @@ -2481,6 +2847,68 @@ mod tests { use crate::chunk_cache::ChunkCache; + /// A chunk stored unfiltered in a filtered dataset (every filter-mask + /// bit set) must still hold the whole chunk. The cached reader placed a + /// short one and read its missing rows as zeros; both readers refuse it. + #[test] + fn short_unfiltered_chunk_of_a_filtered_dataset_is_an_error() { + use crate::filter_pipeline::{FILTER_SHUFFLE, FilterDescription}; + let mut file_data = vec![0u8; 0x2000]; + let mut infos = Vec::new(); + for k in 0..3u64 { + let address = 0x1000 + k as usize * 80; + for i in 0..10u64 { + let at = address + i as usize * 8; + file_data[at..at + 8].copy_from_slice(&((k * 10 + i) as f64).to_le_bytes()); + } + infos.push(ChunkInfo { + chunk_size: if k == 1 { 40 } else { 80 }, + filter_mask: 1, + offsets: vec![k * 10, 0], + address: address as u64, + }); + } + let btree = build_chunk_btree_leaf(&infos, 2, 8); + file_data[0x100..0x100 + btree.len()].copy_from_slice(&btree); + let layout = DataLayout::Chunked { + chunk_dimensions: vec![10, 8], + btree_address: Some(0x100), + version: 3, + chunk_index_type: None, + single_chunk_filtered_size: None, + single_chunk_filter_mask: None, + dont_filter_partial_edge_chunks: false, + }; + let dataspace = simple_space(vec![30]); + let pipeline = FilterPipeline { + version: 2, + filters: vec![FilterDescription { + filter_id: FILTER_SHUFFLE, + name: None, + flags: 0, + client_data: vec![8], + }], + }; + let dt = make_f64_type(); + let cache = ChunkCache::new(); + for result in [ + read_chunked_data(&file_data, &layout, &dataspace, &dt, Some(&pipeline), 8, 8), + read_chunked_data_cached( + &file_data, + &layout, + &dataspace, + &dt, + Some(&pipeline), + 8, + 8, + &cache, + ), + ] { + let err = result.unwrap_err().to_string(); + assert!(err.contains("[10, 0]") && err.contains("40"), "{err}"); + } + } + #[test] fn cached_read_populates_index_and_returns_correct_data() { let values: Vec = (0..20).map(|i| i as f64).collect(); diff --git a/crates/clawhdf5-format/src/parallel_read.rs b/crates/clawhdf5-format/src/parallel_read.rs index 1b940c6..61d06cd 100644 --- a/crates/clawhdf5-format/src/parallel_read.rs +++ b/crates/clawhdf5-format/src/parallel_read.rs @@ -41,6 +41,126 @@ pub fn pool_can_parallelise() -> bool { rayon::current_num_threads() > 1 } +/// How many rayon workers [`run_with_helpers`] should ask to help with +/// `items` work items, given that the calling thread works too: the pool's +/// other threads (all of them when the caller is not one), at most one per +/// item beyond the caller's first. +pub(crate) fn helper_count(items: usize) -> usize { + let pool = rayon::current_num_threads(); + let others = if rayon::current_thread_index().is_some() { + pool.saturating_sub(1) + } else { + pool + }; + others.min(items.saturating_sub(1)) +} + +/// Run `body` on the calling thread and on up to `helpers` rayon workers at +/// once, returning when the caller's call has finished and every worker that +/// started one has too. `body` shares its work out itself (typically by +/// claiming items from an atomic counter until none are left). +/// +/// The caller never waits for a worker to *become* free: helpers are queued +/// on the pool, and one that only gets to run after the caller has finished +/// returns without calling `body`. So a busy or small pool can only fail to +/// speed a read up, never hold it back — with `par_iter`, the calling thread +/// (not a pool worker) handed all the work to the pool and slept, and N +/// threads reading through a 2-worker pool decoded on 2 cores. +/// +/// A panic in `body`, on any thread, is resumed on the caller once every +/// helper that started has stopped. +pub(crate) fn run_with_helpers(helpers: usize, body: &(dyn Fn() + Sync)) { + use std::panic::{AssertUnwindSafe, catch_unwind, resume_unwind}; + use std::sync::{Arc, Condvar, Mutex, PoisonError}; + + if helpers == 0 { + body(); + return; + } + + type Body = dyn Fn() + Sync + 'static; + struct Shared { + /// `body`, its lifetime erased. Only dereferenced by a helper that + /// registered in `state` while it was open (see below). + body: *const Body, + /// (closed, helpers inside `body`). + state: Mutex<(bool, usize)>, + idle: Condvar, + panic: Mutex>>, + } + // SAFETY: `body` points to a `Sync` closure, so calling it from other + // threads is allowed; the pointer is only used under the protocol below, + // which keeps it from outliving the closure. + unsafe impl Send for Shared {} + unsafe impl Sync for Shared {} + + fn help(shared: &Shared) { + { + let mut state = shared.state.lock().unwrap_or_else(PoisonError::into_inner); + if state.0 { + return; + } + state.1 += 1; + } + // SAFETY: registered while open, so the caller of `run_with_helpers` + // is still inside it (it closes, then waits until no helper is + // registered, before returning), and `body` is alive. + let body = unsafe { &*shared.body }; + if let Err(payload) = catch_unwind(AssertUnwindSafe(body)) { + shared + .panic + .lock() + .unwrap_or_else(PoisonError::into_inner) + .get_or_insert(payload); + } + let mut state = shared.state.lock().unwrap_or_else(PoisonError::into_inner); + state.1 -= 1; + if state.1 == 0 { + shared.idle.notify_all(); + } + } + + let body_ptr: *const (dyn Fn() + Sync + '_) = body; + // SAFETY: only the lifetime changes (same fat-pointer layout). The + // pointer is dereferenced only while this function is running: see + // `help` and the wait below. + let body_ptr: *const Body = unsafe { core::mem::transmute(body_ptr) }; + let shared = Arc::new(Shared { + body: body_ptr, + state: Mutex::new((false, 0)), + idle: Condvar::new(), + panic: Mutex::new(None), + }); + for _ in 0..helpers { + let shared = Arc::clone(&shared); + rayon::spawn(move || help(&shared)); + } + let caller = catch_unwind(AssertUnwindSafe(body)); + { + // Close, then wait for the helpers inside `body`; later ones return + // at once. This must happen even if `body` panicked on this thread. + let mut state = shared.state.lock().unwrap_or_else(PoisonError::into_inner); + state.0 = true; + while state.1 > 0 { + state = shared + .idle + .wait(state) + .unwrap_or_else(PoisonError::into_inner); + } + } + if let Err(payload) = caller { + resume_unwind(payload); + } + let helper_panic = shared + .panic + .lock() + .unwrap_or_else(PoisonError::into_inner) + .take(); + if let Some(payload) = helper_panic { + resume_unwind(payload); + } +} + /// Decompress chunks in parallel using lane-partitioned assignment. /// /// Instead of naive `par_iter`, chunks are deterministically assigned to lanes @@ -258,6 +378,60 @@ mod tests { (file, infos) } + /// Every item is processed exactly once, whatever mix of caller and + /// helpers ends up doing it. + #[test] + fn run_with_helpers_shares_all_work() { + use core::sync::atomic::{AtomicUsize, Ordering}; + for helpers in [0, 1, 3, 16] { + let n = 1000; + let next = AtomicUsize::new(0); + let done: Vec = (0..n).map(|_| AtomicUsize::new(0)).collect(); + run_with_helpers(helpers, &|| { + loop { + let i = next.fetch_add(1, Ordering::Relaxed); + if i >= n { + break; + } + done[i].fetch_add(1, Ordering::Relaxed); + } + }); + assert!(done.iter().all(|d| d.load(Ordering::Relaxed) == 1)); + } + } + + /// A panic in the shared body reaches the caller whichever thread it + /// happened on, and only after the helpers inside the body have left it + /// (they borrow the caller's stack). + #[test] + fn run_with_helpers_propagates_panics() { + use core::sync::atomic::{AtomicUsize, Ordering}; + use std::panic::{AssertUnwindSafe, catch_unwind}; + let caller = std::thread::current().id(); + for panic_on_caller in [true, false] { + let inside = AtomicUsize::new(0); + let calls = AtomicUsize::new(0); + let result = catch_unwind(AssertUnwindSafe(|| { + run_with_helpers(4, &|| { + inside.fetch_add(1, Ordering::SeqCst); + calls.fetch_add(1, Ordering::SeqCst); + let on_caller = std::thread::current().id() == caller; + std::thread::sleep(std::time::Duration::from_millis(20)); + inside.fetch_sub(1, Ordering::SeqCst); + if on_caller == panic_on_caller { + panic!("boom"); + } + }); + })); + // A helper may never have run (the pool was slow to start it), + // in which case nothing panicked when `panic_on_caller` is false. + if panic_on_caller || calls.load(Ordering::SeqCst) > 1 { + assert!(result.is_err()); + } + assert_eq!(inside.load(Ordering::SeqCst), 0); + } + } + /// Every parallel decoder refuses a chunk that decodes short, naming it. #[test] fn short_decoded_chunk_is_an_error() { diff --git a/crates/clawhdf5/tests/busy_decode_pool.rs b/crates/clawhdf5/tests/busy_decode_pool.rs new file mode 100644 index 0000000..d53eeab --- /dev/null +++ b/crates/clawhdf5/tests/busy_decode_pool.rs @@ -0,0 +1,112 @@ +//! Full reads of chunked datasets must not wait for a busy rayon pool of +//! any size. +//! +//! A full read handed its chunks to the rayon pool (`par_iter`) and the +//! calling thread — not a pool worker — slept until the pool had decoded +//! them. With a small pool (2-4 threads) and more reading threads than +//! workers, every reader queued behind the same few workers +//! (`docs/known-issues.md`, "Concurrent and contiguous read performance"). +//! Now the calling thread decodes too, and pool workers only help when they +//! are free. The test keeps both workers of a two-thread pool busy and +//! requires reads to finish anyway, with the right values. +//! +//! One test in its own binary: it configures the process-wide rayon pool. + +#![cfg(feature = "parallel")] + +use std::sync::mpsc; +use std::time::Duration; + +use clawhdf5::{File, FileBuilder}; + +const N: usize = 4096; // 64 chunks of 64 elements + +fn values() -> Vec { + (0..N).map(|i| i as f64 * 0.25 - 7.0).collect() +} + +fn build() -> File { + let mut b = FileBuilder::new(); + b.create_dataset("data") + .with_f64_data(&values()) + .with_shape(&[N as u64]) + .with_chunks(&[64]) + .with_deflate(1) + .with_provenance("test-suite", "2026-09-26T00:00:00Z", None); + File::from_bytes(b.finish().unwrap()).unwrap() +} + +/// Run `f` on a fresh thread; `None` if it has not finished within `limit`. +fn finishes_within( + limit: Duration, + f: impl FnOnce() -> T + Send + 'static, +) -> Option { + let (tx, rx) = mpsc::channel(); + std::thread::spawn(move || { + let _ = tx.send(f()); + }); + rx.recv_timeout(limit).ok() +} + +#[test] +fn full_reads_do_not_wait_for_a_busy_small_pool() { + const WORKERS: usize = 2; + rayon::ThreadPoolBuilder::new() + .num_threads(WORKERS) + .build_global() + .expect("this test binary configures the global pool first"); + + // Built first: the writer compresses on the pool too. + let file = std::sync::Arc::new(build()); + + // Occupy every worker of the pool until the reads are done. + let (started_tx, started_rx) = mpsc::channel(); + let (release_tx, release_rx) = mpsc::channel::<()>(); + let release_rx = std::sync::Arc::new(std::sync::Mutex::new(release_rx)); + for _ in 0..WORKERS { + let (started_tx, release_rx) = (started_tx.clone(), release_rx.clone()); + rayon::spawn(move || { + started_tx.send(()).unwrap(); + let _ = release_rx.lock().unwrap().recv(); + }); + } + for _ in 0..WORKERS { + started_rx.recv().unwrap(); + } + + let limit = Duration::from_secs(20); + // Several readers at once, as in the `concurrent_read` benchmark: the + // cached full read (`read_*`), the typed one and the uncached reader + // behind `verify_provenance`. + let readers: Vec<_> = (0..4) + .map(|_| { + let file = std::sync::Arc::clone(&file); + std::thread::spawn(move || { + finishes_within(limit, move || { + let ds = file.dataset("data").unwrap(); + ( + ds.read_f64().unwrap(), + ds.read_f32().unwrap(), + ds.verify_provenance().unwrap(), + ) + }) + }) + }) + .collect(); + let results: Vec<_> = readers.into_iter().map(|h| h.join().unwrap()).collect(); + // Free the workers before asserting, so a failure does not hang the + // blocked reader threads forever. + for _ in 0..WORKERS { + release_tx.send(()).unwrap(); + } + + let want = values(); + let want_f32: Vec = want.iter().map(|&v| v as f32).collect(); + for result in results { + let (f64s, f32s, verified) = + result.expect("a full read waited for the busy two-thread rayon pool"); + assert_eq!(f64s, want); + assert_eq!(f32s, want_f32); + assert_eq!(verified, clawhdf5::provenance::VerifyResult::Ok); + } +} From 1207df51895c311261e7ceba6413e91e3099fb26 Mon Sep 17 00:00:00 2001 From: osobh Date: Sat, 26 Sep 2026 10:14:31 -0500 Subject: [PATCH 06/43] feat(format): ObjectHeader::object_class, libhdf5's object classification libhdf5 decides what an object header is in a fixed order (H5O__obj_class_real): a group if it has a Symbol Table or Link Info message, a dataset if it has a Datatype *and* a Dataspace message, a named datatype if it has a Datatype message. The conformance probe called any header with a Data Layout message a dataset, so cve-2024-33874's /Dset1 (a datatype and a layout, no dataspace), which h5py opens as a named datatype, was reported as a dataset we failed to read (MissingMessage(Dataspace)). The probe now classifies with object_class(). Co-Authored-By: Claude Opus 5.5 (1M context) --- conformance/probe/src/main.rs | 70 +++++++++++++----- crates/clawhdf5-format/src/object_header.rs | 81 +++++++++++++++++++++ 2 files changed, 133 insertions(+), 18 deletions(-) diff --git a/conformance/probe/src/main.rs b/conformance/probe/src/main.rs index 72f02a1..aa3d6a1 100644 --- a/conformance/probe/src/main.rs +++ b/conformance/probe/src/main.rs @@ -31,7 +31,7 @@ use clawhdf5_format::filter_pipeline::FilterPipeline; use clawhdf5_format::group_v1::{self, GroupEntry}; use clawhdf5_format::group_v2; use clawhdf5_format::message_type::MessageType; -use clawhdf5_format::object_header::ObjectHeader; +use clawhdf5_format::object_header::{ObjectClass, ObjectHeader}; use clawhdf5_format::signature; use clawhdf5_format::superblock::Superblock; use clawhdf5_format::symbol_table::SymbolTableMessage; @@ -657,6 +657,20 @@ fn is_group(h: &ObjectHeader) -> bool { }) } +/// The probe's kind for an object header: libhdf5's object class +/// ([`ObjectHeader::object_class`]: group, then dataset — a datatype *and* a +/// dataspace — then named datatype), which is what h5py opens the object as. +/// The root group, and a header with only link messages, count as groups. +fn kind_of(h: &ObjectHeader, is_root: bool) -> &'static str { + match h.object_class() { + Some(ObjectClass::Group) => "group", + Some(ObjectClass::Dataset) => "dataset", + _ if is_root || is_group(h) => "group", + Some(ObjectClass::NamedDatatype) => "datatype", + None => "unknown", + } +} + fn main() { install_hook(); let path = std::env::args().nth(1).expect("usage: probe "); @@ -735,23 +749,7 @@ fn main() { continue; } }; - let is_ds = h - .messages - .iter() - .any(|m| m.msg_type == MessageType::DataLayout); - let kind = if is_ds { - "dataset" - } else if is_group(&h) || addr == sb.root_group_address { - "group" - } else if h - .messages - .iter() - .any(|m| m.msg_type == MessageType::Datatype) - { - "datatype" - } else { - "unknown" - }; + let kind = kind_of(&h, addr == sb.root_group_address); rec.insert("kind".into(), Value::String(kind.into())); if kind == "dataset" && let Err(msg) = guarded(|| ctx.read_dataset(&h, &mut rec)) @@ -867,6 +865,42 @@ mod tests { assert!(ieee_layout(&f32le)); } + #[test] + fn kind_follows_libhdf5_object_class() { + use clawhdf5_format::object_header::HeaderMessage; + let header = |types: &[MessageType]| ObjectHeader { + version: 2, + messages: types + .iter() + .map(|&msg_type| HeaderMessage { + msg_type, + size: 0, + flags: 0, + creation_order: None, + data: Vec::new(), + }) + .collect(), + reference_count: None, + flags: 0, + access_time: None, + modification_time: None, + change_time: None, + birth_time: None, + }; + use MessageType::*; + // cve-2024-33874 `/Dset1`: a datatype and a layout but no dataspace + // is a named datatype to libhdf5 (h5py opens it as one). + assert_eq!(kind_of(&header(&[Datatype, DataLayout]), false), "datatype"); + assert_eq!( + kind_of(&header(&[Datatype, Dataspace, DataLayout]), false), + "dataset" + ); + assert_eq!(kind_of(&header(&[SymbolTable]), false), "group"); + assert_eq!(kind_of(&header(&[Link]), false), "group"); + assert_eq!(kind_of(&header(&[]), true), "group"); + assert_eq!(kind_of(&header(&[]), false), "unknown"); + } + #[test] fn partial_precision_int_is_shifted_and_sign_extended() { let dt = Datatype::FixedPoint { diff --git a/crates/clawhdf5-format/src/object_header.rs b/crates/clawhdf5-format/src/object_header.rs index 2c3c819..1fc8696 100644 --- a/crates/clawhdf5-format/src/object_header.rs +++ b/crates/clawhdf5-format/src/object_header.rs @@ -75,7 +75,40 @@ fn read_offset(data: &[u8], pos: usize, size: u8) -> Result { }) } +/// The kind of object an object header describes. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum ObjectClass { + /// A group: the header has a Symbol Table or a Link Info message. + Group, + /// A dataset: the header has a Datatype and a Dataspace message. + Dataset, + /// A committed (named) datatype: a Datatype message, no Dataspace. + NamedDatatype, +} + impl ObjectHeader { + /// The kind of object this header describes, decided as libhdf5 decides + /// it (`H5O__obj_class_real`): group first (a Symbol Table or Link Info + /// message), then dataset (a Datatype *and* a Dataspace message — not a + /// Data Layout message), then named datatype (a Datatype message). + /// `None` when none applies; libhdf5 then cannot open the object + /// ("unable to determine object type"). + /// + /// A header with a Datatype and a Data Layout message but no Dataspace + /// is a named datatype to libhdf5, not a dataset. + pub fn object_class(&self) -> Option { + let has = |t: MessageType| self.messages.iter().any(|m| m.msg_type == t); + if has(MessageType::SymbolTable) || has(MessageType::LinkInfo) { + Some(ObjectClass::Group) + } else if has(MessageType::Datatype) && has(MessageType::Dataspace) { + Some(ObjectClass::Dataset) + } else if has(MessageType::Datatype) { + Some(ObjectClass::NamedDatatype) + } else { + None + } + } + /// Parse an object header at the given offset in the data buffer. /// /// `offset_size` and `length_size` come from the superblock. @@ -662,6 +695,54 @@ fn check_message( mod tests { use super::*; + fn header_with(types: &[MessageType]) -> ObjectHeader { + ObjectHeader { + version: 2, + messages: types + .iter() + .map(|&msg_type| HeaderMessage { + msg_type, + size: 0, + flags: 0, + creation_order: None, + data: Vec::new(), + }) + .collect(), + reference_count: None, + flags: 0, + access_time: None, + modification_time: None, + change_time: None, + birth_time: None, + } + } + + #[test] + fn object_class_follows_libhdf5() { + use MessageType::*; + let class = |t: &[MessageType]| header_with(t).object_class(); + assert_eq!( + class(&[Datatype, Dataspace, DataLayout]), + Some(ObjectClass::Dataset) + ); + // A Data Layout message does not make a dataset without a dataspace + // (cve-2024-33874 `/Dset1`: h5py opens it as a named datatype). + assert_eq!( + class(&[Datatype, DataLayout]), + Some(ObjectClass::NamedDatatype) + ); + assert_eq!(class(&[Datatype]), Some(ObjectClass::NamedDatatype)); + // Group messages win over dataset messages. + assert_eq!( + class(&[Datatype, Dataspace, SymbolTable]), + Some(ObjectClass::Group) + ); + assert_eq!(class(&[LinkInfo]), Some(ObjectClass::Group)); + // Link messages alone are not a group; nothing is not an object. + assert_eq!(class(&[Link]), None); + assert_eq!(class(&[]), None); + } + // Helper: build a v1 object header with given messages fn build_v1_header( messages: &[(u16, &[u8], u8)], // (type, data, flags) From 16b7359485713a00ad54f5478941f7f06031aa85 Mon Sep 17 00:00:00 2001 From: osobh Date: Sat, 26 Sep 2026 10:15:50 -0500 Subject: [PATCH 07/43] fix(format): a v1 group with an empty link name fails its listing MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit libhdf5 refuses to list a symbol-table group that has an entry with an empty name (H5G__ent_to_link: "invalid link name"), so h5py cannot list cve-2021-46244's /BAG_root. We listed it, with an object at "/BAG_root/" (the empty name, pointing at address 0). resolve_v1_group_entries — the listing — now fails with the new FormatError::InvalidLinkName; path lookups still find the group's other names, as libhdf5's by-name lookup does. Co-Authored-By: Claude Opus 5.5 (1M context) --- crates/clawhdf5-format/src/error.rs | 6 ++++ crates/clawhdf5-format/src/group_v1.rs | 44 ++++++++++++++++++++++++-- crates/clawhdf5-format/src/group_v2.rs | 4 ++- 3 files changed, 50 insertions(+), 4 deletions(-) diff --git a/crates/clawhdf5-format/src/error.rs b/crates/clawhdf5-format/src/error.rs index 0056b14..d285cbf 100644 --- a/crates/clawhdf5-format/src/error.rs +++ b/crates/clawhdf5-format/src/error.rs @@ -223,6 +223,9 @@ pub enum FormatError { /// The file's actual length in bytes. actual_len: u64, }, + /// A link libhdf5 refuses to list: a symbol-table entry with an empty + /// name ("invalid link name"). Listing the group fails, as in libhdf5. + InvalidLinkName, } impl fmt::Display for FormatError { @@ -494,6 +497,9 @@ impl fmt::Display for FormatError { but the file is {actual_len} bytes" ) } + FormatError::InvalidLinkName => { + write!(f, "invalid link name: a group entry has an empty name") + } } } } diff --git a/crates/clawhdf5-format/src/group_v1.rs b/crates/clawhdf5-format/src/group_v1.rs index 81c149c..a4f97c8 100644 --- a/crates/clawhdf5-format/src/group_v1.rs +++ b/crates/clawhdf5-format/src/group_v1.rs @@ -21,12 +21,35 @@ pub struct GroupEntry { pub cache_type: u32, } -/// Given a SymbolTableMessage, resolve all group children. +/// Given a SymbolTableMessage, resolve all group children: the group's +/// listing. +/// +/// An entry with an empty name fails the listing with +/// [`FormatError::InvalidLinkName`], as it fails libhdf5's link iteration +/// (`H5G__ent_to_link`: "invalid link name"). Looking a name up +/// ([`resolve_path`], and the path resolution in +/// [`crate::group_v2::resolve_path_any`]) still works in such a group, as it +/// does in libhdf5. pub fn resolve_v1_group_entries( file_data: &[u8], sym_table_msg: &SymbolTableMessage, offset_size: u8, length_size: u8, +) -> Result, FormatError> { + let entries = v1_group_entries(file_data, sym_table_msg, offset_size, length_size)?; + if entries.iter().any(|e| e.name.is_empty()) { + return Err(FormatError::InvalidLinkName); + } + Ok(entries) +} + +/// Every entry of a v1 group, empty names included — for looking a name up, +/// which never matches an empty name. +pub(crate) fn v1_group_entries( + file_data: &[u8], + sym_table_msg: &SymbolTableMessage, + offset_size: u8, + length_size: u8, ) -> Result, FormatError> { // Parse local heap let heap = LocalHeap::parse( @@ -207,8 +230,7 @@ pub fn resolve_path( let mut current_sym_table = root_sym_table.clone(); for (i, component) in components.iter().enumerate() { - let entries = - resolve_v1_group_entries(file_data, ¤t_sym_table, offset_size, length_size)?; + let entries = v1_group_entries(file_data, ¤t_sym_table, offset_size, length_size)?; let found = entries.iter().find(|e| e.name == *component); match found { @@ -425,6 +447,22 @@ mod tests { assert_eq!(entries[1].object_header_address, 0x2000); } + /// cve-2021-46244 `/BAG_root`: a symbol-table entry with an empty name. + /// libhdf5 fails the group's listing ("invalid link name"); a lookup of + /// the other names still works. + #[test] + fn empty_entry_name_fails_the_listing_not_a_lookup() { + let (file, msg) = build_synthetic_group(&[("", 0x1000, 0), ("elevation", 0x2000, 0)], 8, 8); + assert_eq!( + resolve_v1_group_entries(&file, &msg, 8, 8).unwrap_err(), + FormatError::InvalidLinkName + ); + assert_eq!( + resolve_path(&file, &msg, "elevation", 8, 8).unwrap(), + 0x2000 + ); + } + #[test] fn resolve_path_single_level() { let (file, msg) = diff --git a/crates/clawhdf5-format/src/group_v2.rs b/crates/clawhdf5-format/src/group_v2.rs index 57119c9..b645bd6 100644 --- a/crates/clawhdf5-format/src/group_v2.rs +++ b/crates/clawhdf5-format/src/group_v2.rs @@ -450,7 +450,9 @@ fn resolve_group_entries( .find(|m| m.msg_type == MessageType::SymbolTable) .ok_or_else(|| FormatError::PathNotFound(String::from("no symbol table message")))?; let stm = SymbolTableMessage::parse(&sym_msg.data, offset_size)?; - group_v1::resolve_v1_group_entries(file_data, &stm, offset_size, length_size) + // A lookup: an entry with an empty name (which fails a listing) is + // skipped by the name comparison, as in libhdf5. + group_v1::v1_group_entries(file_data, &stm, offset_size, length_size) } else if is_v2_group(object_header) { resolve_v2_group_entries(file_data, object_header, offset_size, length_size) } else { From c5cd14c2b203d8f93bdafd2b6278db579955c40b Mon Sep 17 00:00:00 2001 From: osobh Date: Sat, 26 Sep 2026 10:17:02 -0500 Subject: [PATCH 08/43] docs: tools to measure how a range reader would read HDF5 inventory.py counts the functions and call sites that take the whole file as &[u8]; range-trace (standalone crate, x86-64 Linux) records every load clawhdf5 makes from a file by mprotect + single-step, unchanged library code; libhdf5_reads.py counts libhdf5's reads through h5py's fileobj driver and prints a dataset's chunk extents. Co-Authored-By: Claude Opus 5.5 (1M context) --- docs/design/tools/inventory.py | 131 ++++++++ docs/design/tools/libhdf5_reads.py | 98 ++++++ docs/design/tools/range-trace/Cargo.toml | 17 + docs/design/tools/range-trace/src/main.rs | 375 ++++++++++++++++++++++ 4 files changed, 621 insertions(+) create mode 100644 docs/design/tools/inventory.py create mode 100644 docs/design/tools/libhdf5_reads.py create mode 100644 docs/design/tools/range-trace/Cargo.toml create mode 100644 docs/design/tools/range-trace/src/main.rs diff --git a/docs/design/tools/inventory.py b/docs/design/tools/inventory.py new file mode 100644 index 0000000..ce405af --- /dev/null +++ b/docs/design/tools/inventory.py @@ -0,0 +1,131 @@ +#!/usr/bin/env python3 +"""Inventory of whole-file `&[u8]` parameters in the clawhdf5 workspace. + +Used by docs/design/range-reads.md. Run from the repository root: + + python3 docs/design/tools/inventory.py # per-file table + python3 docs/design/tools/inventory.py --list # every signature + python3 docs/design/tools/inventory.py --patterns # read patterns per crate + +A function counts as taking "the whole file" when it has a parameter named +`file_data: &[u8]` (the repository convention), or a `data` / `file` / `buf` / +`bytes` / `mmap` parameter of type `&[u8]` *together with* a parameter whose +name says it is a file address (`*address*`, `*addr*`, `*offset*` of an +integer type). The second rule is a heuristic; `--list` prints every match so +it can be checked by eye. Code after the first `#[cfg(test)] mod ... {` in a +file is excluded (the repository keeps unit tests at the end of each file), +as are the tests/ and benches/ directories. +""" +import os +import re +import sys +from collections import defaultdict + +ROOT = os.getcwd() +FN_RE = re.compile(r"\bfn\s+([A-Za-z_][A-Za-z0-9_]*)\s*(<[^()]*?>)?\s*\(", re.S) +PARAM_RE = re.compile(r"([A-Za-z_][A-Za-z0-9_]*)\s*:\s*&(?:'[a-z_]+\s+)?\[u8\]") +ADDR_RE = re.compile(r"\b([a-z_]*(?:address|addr|offset)[a-z_]*)\s*:\s*(?:u64|usize|u32)") +WHOLE_NAMES = {"data", "file", "buf", "bytes", "file_bytes", "mmap"} + + +def signature(src, start): + depth, i = 0, start + while i < len(src): + c = src[i] + if c == "(": + depth += 1 + elif c == ")": + depth -= 1 + if depth == 0: + return src[start + 1 : i] + i += 1 + return "" + + +def non_test_source(src): + m = re.search(r"#\[cfg\(test\)\]\s*mod\s+\w+\s*\{", src) + return src[: m.start()] if m else src + + +PATTERNS = [ + ("`file_data` passed on (call sites)", re.compile(r"[(,]\s*&?file_data\s*[,)]")), + ("`file_data[..]` slicing", re.compile(r"\bfile_data\s*\[")), + ("open-ended `file_data[x..]`", re.compile(r"\bfile_data\s*\[[^\]]*\.\.\s*\]")), + ("`file_data.get(..)`", re.compile(r"\bfile_data\s*\.\s*get\s*\(")), + ("`file_data.len()`", re.compile(r"\bfile_data\s*\.\s*len\s*\(\)")), + ("address/offset `as usize` casts", re.compile(r"\b[a-z_]*(?:addr|address|offset)[a-z_]*\s+as\s+usize")), + ("`ObjectHeader::parse(` calls", re.compile(r"ObjectHeader::parse\s*\(")), + ("`.as_bytes()` on a file/reader", re.compile(r"\b(?:file|reader|data|self\.file|self\.data|self\.file\.data|root\.file)\s*\.\s*as_bytes\s*\(\)")), +] + + +def patterns(): + """Per-crate counts of the read patterns (non-test code only).""" + per_crate = defaultdict(lambda: [0] * len(PATTERNS)) + for crate in sorted(os.listdir(os.path.join(ROOT, "crates"))): + srcdir = os.path.join(ROOT, "crates", crate, "src") + for dp, _, fns in os.walk(srcdir): + for fn in fns: + if not fn.endswith(".rs"): + continue + with open(os.path.join(dp, fn), encoding="utf-8") as fh: + src = non_test_source(fh.read()) + for i, (_, rx) in enumerate(PATTERNS): + per_crate[crate][i] += len(rx.findall(src)) + print("| crate | " + " | ".join(n for n, _ in PATTERNS) + " |") + print("|---|" + "---:|" * len(PATTERNS)) + tot = [0] * len(PATTERNS) + for c, v in sorted(per_crate.items()): + if any(v): + print("| %s | %s |" % (c, " | ".join(str(x) for x in v))) + tot = [a + b for a, b in zip(tot, v)] + print("| **total** | %s |" % " | ".join("**%d**" % x for x in tot)) + + +def main(): + if "--patterns" in sys.argv: + patterns() + return + per_file = defaultdict(lambda: [0, 0, 0]) # [file_data, heuristic, pub] + rows = [] + for crate in sorted(os.listdir(os.path.join(ROOT, "crates"))): + srcdir = os.path.join(ROOT, "crates", crate, "src") + for dp, _, fns in os.walk(srcdir): + for fn in sorted(fns): + if not fn.endswith(".rs"): + continue + path = os.path.join(dp, fn) + with open(path, encoding="utf-8") as fh: + src = non_test_source(fh.read()) + for m in FN_RE.finditer(src): + sig = signature(src, m.end() - 1) + params = PARAM_RE.findall(sig) + kind = None + if "file_data" in params: + kind = "file_data" + elif any(p in WHOLE_NAMES for p in params) and ADDR_RE.search(sig): + kind = "heuristic" + if not kind: + continue + rel = os.path.relpath(path, ROOT) + line_start = src.rfind("\n", 0, m.start()) + 1 + is_pub = src[line_start : m.start()].strip().startswith("pub") + per_file[rel][0 if kind == "file_data" else 1] += 1 + per_file[rel][2] += int(is_pub) + line = src.count("\n", 0, m.start()) + 1 + rows.append((rel, line, m.group(1), kind, is_pub)) + if "--list" in sys.argv: + for r in rows: + print("%s:%d %s [%s%s]" % (r[0], r[1], r[2], r[3], ", pub" if r[4] else "")) + return + print("| file | `file_data` fns | other whole-slice fns (heuristic) | of which `pub` |") + print("|---|---:|---:|---:|") + tot = [0, 0, 0] + for f, (a, b, p) in sorted(per_file.items(), key=lambda kv: (-(kv[1][0] + kv[1][1]), kv[0])): + print("| %s | %d | %d | %d |" % (f, a, b, p)) + tot = [tot[0] + a, tot[1] + b, tot[2] + p] + print("| **total** | **%d** | **%d** | **%d** |" % tuple(tot)) + + +if __name__ == "__main__": + main() diff --git a/docs/design/tools/libhdf5_reads.py b/docs/design/tools/libhdf5_reads.py new file mode 100644 index 0000000..5d6053e --- /dev/null +++ b/docs/design/tools/libhdf5_reads.py @@ -0,0 +1,98 @@ +#!/usr/bin/env python3 +"""The libhdf5 side of the range-read measurement (docs/design/range-reads.md). + + libhdf5_reads.py FILE DATASET # count libhdf5's reads + libhdf5_reads.py FILE DATASET --extents # print DATASET's stored extents + +Counting: the file is opened through h5py's `fileobj` driver with a Python +file-like object that logs every `readinto`/`read` libhdf5 makes (this is how +h5py + fsspec reads remote files: each call becomes a range request unless +fsspec's own block cache absorbs it). The phases mirror range-trace: open, +list (visit every object; shape and dtype of every dataset), read (the whole +dataset). Default h5py settings (libhdf5's metadata cache, 64 KiB sieve +buffer, 1 MiB raw chunk cache) apply. + +--extents prints `offset size` lines for the dataset's stored data (the +contiguous block, or every allocated chunk), the input range-trace uses to +split its read phase into metadata and raw data. +""" +import sys + +import h5py + + +class LoggingFile: + def __init__(self, path): + self.f = open(path, "rb") + self.pos = 0 + self.log = [] + + def seek(self, off, whence=0): + self.pos = self.f.seek(off, whence) + return self.pos + + def tell(self): + return self.pos + + def readinto(self, b): + n = self.f.readinto(b) + self.log.append((self.pos, n)) + self.pos += n + return n + + def read(self, size=-1): + data = self.f.read(size) + self.log.append((self.pos, len(data))) + self.pos += len(data) + return data + + +def extents(path, name): + with h5py.File(path, "r") as f: + ds = f[name] + if ds.chunks is None: + off = ds.id.get_offset() + if off is not None: + print(off, ds.id.get_storage_size()) + return + for i in range(ds.id.get_num_chunks()): + info = ds.id.get_chunk_info(i) + print(info.byte_offset, info.size) + + +def summarise(label, log): + n = len(log) + total = sum(s for _, s in log) + distinct = len(set(log)) + print("| %s | %d | %d | %d |" % (label, n, distinct, total)) + + +def count(path, name): + lf = LoggingFile(path) + f = h5py.File(lf, "r") + opened = list(lf.log) + lf.log.clear() + + def visit(_n, obj): + if isinstance(obj, h5py.Dataset): + obj.shape, obj.dtype + + f.visititems(visit) + listed = list(lf.log) + lf.log.clear() + f[name][()] + readlog = list(lf.log) + f.close() + print("| phase | read calls | distinct (offset, len) | bytes |") + print("|---|---:|---:|---:|") + summarise("open", opened) + summarise("list", listed) + summarise("read", readlog) + summarise("open+list+read", opened + listed + readlog) + + +if __name__ == "__main__": + if len(sys.argv) >= 4 and sys.argv[3] == "--extents": + extents(sys.argv[1], sys.argv[2]) + else: + count(sys.argv[1], sys.argv[2]) diff --git a/docs/design/tools/range-trace/Cargo.toml b/docs/design/tools/range-trace/Cargo.toml new file mode 100644 index 0000000..5b0d8cd --- /dev/null +++ b/docs/design/tools/range-trace/Cargo.toml @@ -0,0 +1,17 @@ +[package] +name = "range-trace" +version = "0.1.0" +edition = "2024" +rust-version = "1.92" +publish = false +description = "Records which byte ranges of an HDF5 file the clawhdf5 facade reads (docs/design/range-reads.md)" + +# Outside the main workspace on purpose: a measurement tool for a design +# document, never built by `cargo test --workspace`. x86-64 Linux only. +[workspace] + +[dependencies] +# Default features minus nothing: `parallel` is off by default, and the tracer +# relies on every read happening on the main thread. +clawhdf5 = { path = "../../../../crates/clawhdf5" } +libc = "0.2" diff --git a/docs/design/tools/range-trace/src/main.rs b/docs/design/tools/range-trace/src/main.rs new file mode 100644 index 0000000..24b7ce8 --- /dev/null +++ b/docs/design/tools/range-trace/src/main.rs @@ -0,0 +1,375 @@ +//! range-trace: which bytes of an HDF5 file does clawhdf5 read? +//! +//! Measurement tool for `docs/design/range-reads.md` (x86-64 Linux only). +//! +//! The file is loaded into a page-aligned buffer and handed to +//! `clawhdf5::File::from_bytes`, so the library parses it exactly as it does +//! today. The buffer is then `mprotect`ed to `PROT_NONE`. Every load from it +//! faults; the SIGSEGV handler logs the exact faulting address, makes that +//! page readable and sets the x86 trap flag, so the CPU single-steps the one +//! instruction and the SIGTRAP handler re-protects the page. Every load +//! instruction that touches the file is therefore recorded (with its first +//! byte; widths are not decoded, see `ACCESS_WIDTH`). +//! +//! Bulk copies (raw data) would fault once per load; a page that faults more +//! than `BULK_THRESHOLD` times in one phase is left readable for the rest of +//! that phase and counted as wholly read ("bulk page"). +//! +//! Phases: `open` (superblock), `list` (walk every group; for every dataset +//! its shape and dtype, i.e. what `h5ls -r -v` or a tree view needs), and +//! `read` (read the named dataset in full through `read_selection(All)`). +//! +//! Usage: range-trace FILE DATASET_PATH [RAW_EXTENTS] +//! +//! RAW_EXTENTS (optional) lists the dataset's stored data as `offset size` +//! lines (absolute file offsets; `libhdf5_reads.py --extents` writes it from +//! h5py). With it the `read` phase is split into `read (metadata)`, the +//! loads outside those extents, and `read (raw data)`, the loads inside. +//! +//! Every read must happen on this thread: build without the facade's +//! `parallel` feature (off by default). + +use std::alloc::{Layout, alloc_zeroed}; +use std::collections::BTreeSet; +use std::sync::atomic::{AtomicPtr, AtomicUsize, Ordering}; + +use clawhdf5::{File, Group, Selection}; + +const PAGE: usize = 4096; +const BULK_THRESHOLD: u32 = 4096; +/// Bytes assumed read by one logged load (the widest scalar load the parsers +/// issue; SIMD copies are bulk anyway). +const ACCESS_WIDTH: u64 = 8; +/// Two accesses closer than this belong to the same structure, i.e. would be +/// one range request. +const MERGE_GAP: u64 = 64; +const LOG_CAP: usize = 64 << 20; + +static BUF_START: AtomicUsize = AtomicUsize::new(0); +static BUF_LEN: AtomicUsize = AtomicUsize::new(0); +static LOG: AtomicPtr = AtomicPtr::new(std::ptr::null_mut()); +static LOG_LEN: AtomicUsize = AtomicUsize::new(0); +static HITS: AtomicPtr = AtomicPtr::new(std::ptr::null_mut()); +static BULK: AtomicPtr = AtomicPtr::new(std::ptr::null_mut()); +static PENDING: [AtomicUsize; 8] = [const { AtomicUsize::new(0) }; 8]; +static PENDING_N: AtomicUsize = AtomicUsize::new(0); + +const TF: i64 = 0x100; + +extern "C" fn on_segv(_sig: libc::c_int, info: *mut libc::siginfo_t, ctx: *mut libc::c_void) { + unsafe { + let addr = (*info).si_addr() as usize; + let start = BUF_START.load(Ordering::Relaxed); + let len = BUF_LEN.load(Ordering::Relaxed); + if addr < start || addr >= start + len { + // A genuine crash: restore the default action and re-fault. + libc::signal(libc::SIGSEGV, libc::SIG_DFL); + return; + } + let off = addr - start; + let n = LOG_LEN.load(Ordering::Relaxed); + if n < LOG_CAP { + *LOG.load(Ordering::Relaxed).add(n) = off as u64; + LOG_LEN.store(n + 1, Ordering::Relaxed); + } + let page = off / PAGE; + let hits = HITS.load(Ordering::Relaxed).add(page); + *hits += 1; + libc::mprotect( + (start + page * PAGE) as *mut libc::c_void, + PAGE, + libc::PROT_READ | libc::PROT_WRITE, + ); + if *hits >= BULK_THRESHOLD { + *BULK.load(Ordering::Relaxed).add(page) = 1; + } else { + let p = PENDING_N.load(Ordering::Relaxed); + if p < PENDING.len() { + PENDING[p].store(page, Ordering::Relaxed); + PENDING_N.store(p + 1, Ordering::Relaxed); + } + } + let uc = ctx as *mut libc::ucontext_t; + (*uc).uc_mcontext.gregs[libc::REG_EFL as usize] |= TF; + } +} + +extern "C" fn on_trap(_sig: libc::c_int, _info: *mut libc::siginfo_t, ctx: *mut libc::c_void) { + unsafe { + let start = BUF_START.load(Ordering::Relaxed); + let n = PENDING_N.load(Ordering::Relaxed); + for p in PENDING.iter().take(n) { + let page = p.load(Ordering::Relaxed); + libc::mprotect( + (start + page * PAGE) as *mut libc::c_void, + PAGE, + libc::PROT_NONE, + ); + } + PENDING_N.store(0, Ordering::Relaxed); + let uc = ctx as *mut libc::ucontext_t; + (*uc).uc_mcontext.gregs[libc::REG_EFL as usize] &= !TF; + } +} + +fn install( + sig: libc::c_int, + h: extern "C" fn(libc::c_int, *mut libc::siginfo_t, *mut libc::c_void), +) { + unsafe { + let mut sa: libc::sigaction = std::mem::zeroed(); + sa.sa_sigaction = h as usize; + sa.sa_flags = libc::SA_SIGINFO | libc::SA_NODEFER; + libc::sigemptyset(&mut sa.sa_mask); + assert_eq!(libc::sigaction(sig, &sa, std::ptr::null_mut()), 0); + } +} + +fn protect(prot: libc::c_int) { + let start = BUF_START.load(Ordering::Relaxed); + let len = BUF_LEN.load(Ordering::Relaxed); + unsafe { + assert_eq!(libc::mprotect(start as *mut libc::c_void, len, prot), 0); + } +} + +struct Phase { + name: &'static str, + log: Vec, + bulk_pages: Vec, +} + +/// Start a phase: clear the per-page state and protect the buffer. +fn begin() { + let pages = BUF_LEN.load(Ordering::Relaxed) / PAGE; + unsafe { + std::ptr::write_bytes(HITS.load(Ordering::Relaxed), 0, pages); + std::ptr::write_bytes(BULK.load(Ordering::Relaxed), 0, pages); + } + LOG_LEN.store(0, Ordering::Relaxed); + protect(libc::PROT_NONE); +} + +fn end(name: &'static str) -> Phase { + protect(libc::PROT_READ | libc::PROT_WRITE); + let n = LOG_LEN.load(Ordering::Relaxed); + let log = unsafe { std::slice::from_raw_parts(LOG.load(Ordering::Relaxed), n) }.to_vec(); + let pages = BUF_LEN.load(Ordering::Relaxed) / PAGE; + let bulk = unsafe { std::slice::from_raw_parts(BULK.load(Ordering::Relaxed), pages) }; + let bulk_pages = (0..pages) + .filter(|&p| bulk[p] != 0) + .map(|p| p as u64) + .collect(); + if n == LOG_CAP { + eprintln!("warning: access log full in phase {name}"); + } + Phase { + name, + log, + bulk_pages, + } +} + +/// Byte intervals [lo, hi) read in a phase, merged when closer than `gap`. +fn ranges(ph: &[&Phase], gap: u64) -> Vec<(u64, u64)> { + let mut iv: Vec<(u64, u64)> = Vec::new(); + for p in ph { + iv.extend(p.log.iter().map(|&o| (o, o + ACCESS_WIDTH))); + iv.extend( + p.bulk_pages + .iter() + .map(|&pg| (pg * PAGE as u64, (pg + 1) * PAGE as u64)), + ); + } + iv.sort_unstable(); + let mut out: Vec<(u64, u64)> = Vec::new(); + for (lo, hi) in iv { + match out.last_mut() { + Some(last) if lo <= last.1 + gap => last.1 = last.1.max(hi), + _ => out.push((lo, hi)), + } + } + out +} + +/// Requests a reader with no cache at all would make: a new request each +/// time the access stream leaves the neighbourhood of the current run. +fn uncached_requests(p: &Phase) -> usize { + let mut n = 0; + let (mut lo, mut hi) = (u64::MAX, 0u64); + for &o in &p.log { + if lo != u64::MAX && o + MERGE_GAP >= lo && o <= hi + MERGE_GAP { + lo = lo.min(o); + hi = hi.max(o + ACCESS_WIDTH); + } else { + n += 1; + lo = o; + hi = o + ACCESS_WIDTH; + } + } + n + p.bulk_pages.len() +} + +fn blocks(ph: &[&Phase], block: u64) -> usize { + let mut set = BTreeSet::new(); + for (lo, hi) in ranges(ph, 0) { + for b in lo / block..=(hi - 1) / block { + set.insert(b); + } + } + set.len() +} + +fn read_extents(path: &str) -> Vec<(u64, u64)> { + let text = std::fs::read_to_string(path).expect("read extents"); + let mut v: Vec<(u64, u64)> = text + .lines() + .filter_map(|l| { + let mut it = l.split_whitespace().map(|t| t.parse::()); + match (it.next(), it.next()) { + (Some(Ok(o)), Some(Ok(n))) => Some((o, o + n)), + _ => None, + } + }) + .collect(); + v.sort_unstable(); + v +} + +fn in_extents(ext: &[(u64, u64)], lo: u64, hi: u64) -> bool { + let i = ext.partition_point(|e| e.1 <= lo); + i < ext.len() && ext[i].0 < hi +} + +/// Split a phase into loads outside and inside the raw-data extents. A bulk +/// page counts as raw data when it overlaps an extent. +fn split_raw(p: &Phase, ext: &[(u64, u64)]) -> (Phase, Phase) { + let (mut m, mut r) = (Vec::new(), Vec::new()); + for &o in &p.log { + if in_extents(ext, o, o + 1) { + r.push(o) + } else { + m.push(o) + } + } + let pg = PAGE as u64; + let (bm, br): (Vec, Vec) = p + .bulk_pages + .iter() + .partition(|&&b| !in_extents(ext, b * pg, (b + 1) * pg)); + ( + Phase { + name: "read (metadata)", + log: m, + bulk_pages: bm, + }, + Phase { + name: "read (raw data)", + log: r, + bulk_pages: br, + }, + ) +} + +fn walk(g: &Group<'_>, path: &str, objs: &mut usize) { + for name in g.datasets().unwrap_or_default() { + *objs += 1; + if let Ok(ds) = g.dataset(&name) { + let _ = ds.shape(); + let _ = ds.dtype(); + } + } + for name in g.groups().unwrap_or_default() { + *objs += 1; + if let Ok(sub) = g.group(&name) { + walk(&sub, &format!("{path}/{name}"), objs); + } + } +} + +fn main() { + let args: Vec = std::env::args().collect(); + if args.len() != 3 && args.len() != 4 { + eprintln!("usage: range-trace FILE DATASET_PATH [RAW_EXTENTS]"); + std::process::exit(2); + } + let bytes = std::fs::read(&args[1]).expect("read file"); + let len = bytes.len(); + let cap = len.div_ceil(PAGE).max(1) * PAGE; + let pages = cap / PAGE; + // Page-aligned buffer so that protection covers exactly the file. The + // Vec is never dropped (its layout differs from Vec's own), see the end. + let ptr = unsafe { alloc_zeroed(Layout::from_size_align(cap, PAGE).unwrap()) }; + unsafe { std::ptr::copy_nonoverlapping(bytes.as_ptr(), ptr, len) }; + drop(bytes); + let buf = unsafe { Vec::from_raw_parts(ptr, len, cap) }; + BUF_START.store(ptr as usize, Ordering::Relaxed); + BUF_LEN.store(cap, Ordering::Relaxed); + let mut log = vec![0u64; LOG_CAP]; + LOG.store(log.as_mut_ptr(), Ordering::Relaxed); + let mut hits = vec![0u32; pages]; + HITS.store(hits.as_mut_ptr(), Ordering::Relaxed); + let mut bulk = vec![0u8; pages]; + BULK.store(bulk.as_mut_ptr(), Ordering::Relaxed); + install(libc::SIGSEGV, on_segv); + install(libc::SIGTRAP, on_trap); + + begin(); + let file = File::from_bytes(buf).expect("open"); + let p_open = end("open"); + + begin(); + let mut objs = 0; + walk(&file.root(), "", &mut objs); + let p_list = end("list"); + + begin(); + let out = file + .dataset(&args[2]) + .and_then(|d| d.read_selection(&Selection::All)) + .expect("read dataset"); + let p_read = end("read"); + + println!("file: {} ({} bytes), objects listed: {objs}", args[1], len); + println!("dataset: {} ({} bytes decoded)", args[2], out.len()); + println!( + "| phase | loads logged | bulk 4K pages | uncached requests | distinct ranges (gap<{MERGE_GAP}B) | bytes in ranges | 4 KiB blocks | 64 KiB blocks | 1 MiB blocks |" + ); + println!("|---|---:|---:|---:|---:|---:|---:|---:|---:|"); + let row = |label: String, ph: &[&Phase], uncached: usize| { + let r = ranges(ph, MERGE_GAP); + let bytes: u64 = r.iter().map(|(a, b)| b - a).sum(); + let loads: usize = ph.iter().map(|p| p.log.len()).sum(); + let bulk: usize = ph.iter().map(|p| p.bulk_pages.len()).sum(); + println!( + "| {label} | {loads} | {bulk} | {uncached} | {} | {bytes} | {} | {} | {} |", + r.len(), + blocks(ph, 4 << 10), + blocks(ph, 64 << 10), + blocks(ph, 1 << 20) + ); + }; + for p in [&p_open, &p_list, &p_read] { + row(p.name.to_string(), &[p], uncached_requests(p)); + } + if let Some(path) = args.get(3) { + let (meta, raw) = split_raw(&p_read, &read_extents(path)); + for p in [&meta, &raw] { + row(p.name.to_string(), &[p], uncached_requests(p)); + } + let all = [&p_open, &p_list, &meta]; + let unc: usize = all.iter().map(|p| uncached_requests(p)).sum(); + row("all metadata".into(), &all, unc); + } + let all = [&p_open, &p_list, &p_read]; + let unc: usize = all.iter().map(|p| uncached_requests(p)).sum(); + row("open+list+read".into(), &all, unc); + let meta = [&p_open, &p_list]; + let unc: usize = meta.iter().map(|p| uncached_requests(p)).sum(); + row("open+list".into(), &meta, unc); + // The File owns a buffer whose layout Vec does not know; never drop it. + std::mem::forget(file); + std::mem::forget(log); + std::mem::forget(hits); + std::mem::forget(bulk); +} From 1c1af460b6830851adf032ff064d56a621ac6798 Mon Sep 17 00:00:00 2001 From: osobh Date: Sat, 26 Sep 2026 10:17:02 -0500 Subject: [PATCH 09/43] docs: design for range reads (S3/HTTP, wasm lazy loading, SWMR, 32-bit) Inventory of the 109 whole-file parser functions, a measurement of the metadata ranges clawhdf5 touches on three corpus files against libhdf5, options (storage trait, virtual slice, metadata prefetch, userfaultfd) with how ros3, h5py+fsspec, pyfive, h5wasm and object_store do it, and an incremental plan: M0 name-index lookups, M1 metadata over a Storage trait, M2 raw data with batched ranges, M3 object_store backend, M4 wasm fetch-driven lazy loading. Co-Authored-By: Claude Opus 5.5 (1M context) --- docs/design/range-reads.md | 499 +++++++++++++++++++++++++++++++++++++ 1 file changed, 499 insertions(+) create mode 100644 docs/design/range-reads.md diff --git a/docs/design/range-reads.md b/docs/design/range-reads.md new file mode 100644 index 0000000..2e82a6c --- /dev/null +++ b/docs/design/range-reads.md @@ -0,0 +1,499 @@ +# Design: range reads (reading HDF5 without holding the whole file) + +Status: proposal, 2026-09-26. No library code has changed; this document is +the plan for Phase 3's largest architectural change. Every count below was +taken on `tank` on 2026-09-26 at commit `de2a53f`, with the commands given +next to it. No timing numbers appear here on purpose: the machine was shared +with other build jobs when this was written. + +## The problem + +Every reader in clawhdf5 parses the file through one `&[u8]` that covers the +whole file: + +- `clawhdf5_io::HDF5Read::as_bytes(&self) -> &[u8]` is the only read method of + the I/O trait (`crates/clawhdf5-io/src/lib.rs`). `FileReader` reads the + whole file into a `Vec`, `MmapReader` maps it. +- The facade's `File` keeps a `Backing` (`Vec` or mmap) and hands + `FileData::as_bytes()` — the file from the superblock to the recorded end of + file — to every `clawhdf5-format` call (`crates/clawhdf5/src/reader.rs`). +- The format crate's parsers take `file_data: &[u8]` plus a `usize` address and + slice into it. +- `LazyFile` (facade) parses metadata lazily, but still over + `HDF5Read::as_bytes`. `AsyncHDF5Read` (`clawhdf5-io`, `async` feature) + already has `read_at(offset, len)`, but `AsyncHDF5File::open` calls + `read_all()` and parses the result. +- The browser reader (`clawhdf5-wasm`) is `open(bytes: Vec)`: the page + must download the whole file before it can list a group. + +That rules out four things users of other HDF5 readers have: + +1. **Remote files by range request** — libhdf5's `ros3` driver, h5py over + `fsspec`/`s3fs`, pyfive over `fsspec`. Today a 30 GB file on S3 has to be + downloaded before its tree can be listed. +2. **Lazy loading in the browser** — h5wasm can back a file with HTTP range + requests; our wasm reader holds the whole file. +3. **SWMR readers of a growing file** — the slice is fixed at open; a reader + cannot see the file grow (and an mmap of a growing file is fragile). +4. **Files larger than the address space on 32-bit targets** — including + `wasm32`, where `usize` is 32 bits and a file above 4 GiB cannot even be put + in a slice; browsers cap `ArrayBuffer`s lower than that. + +## 1. Inventory + +### 1.1 Functions that take the whole file + +`python3 docs/design/tools/inventory.py` scans non-test source (code before a +file's `#[cfg(test)] mod`, no `tests/`) for functions with a +`file_data: &[u8]` parameter (the repository's convention), and — as a +heuristic — functions with a `data`/`buf`/`file`/`bytes` slice *and* an +integer address/offset parameter. `--list` prints every hit. + +- **103 functions take `file_data: &[u8]`** (73 of them `pub`), in 25 files. +- The heuristic adds 27, of which only **6 really take the whole file under + another name**: `ObjectHeader::parse` and its internal `parse_v1`, + `parse_v1_chunk`, `parse_v2`, `parse_v2_continuation` (`data: &[u8]`), and + `Superblock::parse`. The other 21 are helpers (`ensure_len`, + `read_length`, `read_uint`, …) that bound-check whatever slice they are + given, or writer-side `write_at`s in `clawhdf5-io`. + +So **109 functions** have to change signature. Grouped by what they read: + +| Read pattern | Modules (whole-file functions) | Count | +|---|---|---:| +| **Small metadata reads at addresses** — a structure whose size is usually only known after its prefix is read | `group_v2` 9, `shared_message` 9, `fractal_heap` 7, `attribute` 6, `object_header` 5, `group_v1` 5, `extensible_array` 4, `btree_v2` 4, `btree_v1` 3, `fixed_array` 3, `local_heap` 3, `fill_value` 3, `superblock` 2, `data_layout` 1, `symbol_table` 1 | 65 | +| **Bulk raw data** — a contiguous extent, or a list of chunk extents known once the chunk index has been walked | `chunked_read` 12, `data_read` 9, `vds` 6, `parallel_read` 3, `partial_read` 1, `provenance` 1 | 32 | +| **Heap-resident data** — VL strings and sequences: many small reads into global-heap collections | `vl_data` 3, `global_heap` 2, facade `vlen.rs` 3, facade `types.rs` 3 | 11 | +| Bindings | `clawhdf5-py/src/convert.rs` | 1 | + +### 1.2 How they read + +`python3 docs/design/tools/inventory.py --patterns` (non-test code): + +| Pattern | Where | Count | +|---|---|---:| +| `file_data` passed on to another function (call sites) | format 262, facade 7, py 2 | 271 | +| `file_data[..]` slicing | format | 137 | +| of which open-ended `file_data[x..]` (slice to end of file) | format | 5 | +| `file_data.len()` (end-of-file bound checks) | format | 38 | +| address/offset `as usize` casts | format 120, facade 5, io 5, ann 3 | 133 | +| `.as_bytes()` on a file or reader, feeding format calls | facade 26, py 8, tools 1, wasm 1 | 36 | + +Three things in this table shape the design: + +- **Open-ended slices and `len()` checks** assume the whole file is present. + With a range reader "the rest of the file" is a request for gigabytes; each + of these 43 sites must become a bounded read. +- **The 133 `as usize` casts** are where a 64-bit HDF5 address is squeezed + into a pointer-sized index. On 32-bit targets they truncate or must fail. + A storage trait that takes `u64` offsets removes most of them. +- **The bytes escape through the public API.** `File::as_bytes()`, + `LazyFile::as_bytes()`, `MmapFile::as_bytes()` return the whole file, and 15 + facade methods return `&'f [u8]`/`&'f [T]` borrowed from it + (`Dataset::read_raw_ref`, `read_as_slice`, `read_*_zerocopy`). The Python + bindings (`clawhdf5-py/src/node.rs`, `attrs.rs`, …) call + `clawhdf5_format` directly with `file.as_bytes()`, and so do `h5rs` and the + wasm reader. These must keep working for local files (they are the + zero-copy fast path) and fail cleanly for remote ones. + +### 1.3 Pieces that already exist + +- `clawhdf5_format::metadata_cache::MetadataCache` — LRU keyed by file offset + with a byte budget (2 MiB default). Not currently in the read path. +- `clawhdf5_io::prefetch::PrefetchReader` — ring buffer of `(offset, len)` + entries over an `HDF5Read`, plus a sweep detector for chunk access patterns. +- `clawhdf5_io::async_read::AsyncHDF5Read::read_at` — the right shape of + trait, but async and only used to read everything. +- `clawhdf5_io::hsds` — a client for the HSDS REST service. Different + protocol (the server parses HDF5), not a range reader. + +## 2. Measurement: how chatty is a naive range reader? + +`docs/design/tools/range-trace/` loads a file into a page-aligned buffer, +opens it with `clawhdf5::File::from_bytes` (today's code, unchanged), +`mprotect`s the buffer and single-steps every load that faults (SIGSEGV logs +the exact address and unprotects the page; the x86 trap flag re-protects it +after one instruction). So every load instruction that touches the file is +recorded, with no change to the library. A page loaded more than 4096 times in +a phase is left readable and counted as wholly read. Phases: **open** +(superblock), **list** (walk every group; shape and dtype of every dataset — +what a tree view or `h5ls -r -v` needs), **read** (one dataset, whole). With +the dataset's chunk extents (from h5py, `libhdf5_reads.py --extents`) the read +phase is split into the chunk index and the raw data. + +Columns: *uncached requests* — a new request every time the access stream +leaves the neighbourhood (±64 B) of the current run, i.e. a reader that turns +each parse into a read with no cache; *ranges* — distinct byte ranges after +merging accesses closer than 64 B; *N KiB blocks* — distinct aligned blocks, +i.e. the requests a reader with a block cache of that size would make. For +comparison, `libhdf5_reads.py` opens the same file in h5py through a Python +file object (the `fileobj` driver — exactly how h5py + fsspec read remote +files) and counts libhdf5's `read` calls, default h5py settings. + +```bash +CARGO_TARGET_DIR=$PWD/target cargo build --release \ + --manifest-path docs/design/tools/range-trace/Cargo.toml +PY=/home/osobh/projects/clawhdf5/.venv/bin/python +$PY docs/design/tools/libhdf5_reads.py FILE DSET --extents > ext.txt +target/release/range-trace FILE DSET ext.txt +$PY docs/design/tools/libhdf5_reads.py FILE DSET +``` + +Files (from `conformance/.cache/corpus`), 2026-09-26 on tank: + +- **A** `xarray-data/imerghh_730.hdf5` — NASA IMERG, 7.7 MB, superblock v0, + 21 objects; read `/Grid/precipitation` (25 deflated chunks). +- **B** `NCAS-CMS_pyfive/tests/data/cmip_bad_eg.nc` — netCDF-4, 48 MB, + superblock v2, 9 objects; read `/tas` (780 deflated chunks, 43.7 MB stored). +- **C** `hdf5/tools/test/testfiles/h5stat_newgrat.h5` — 6.4 MB, superblock + v3, 35 001 groups in one dense (fractal heap + v2 B-tree) root group; read + `/DATASET_NAME` (a scalar with no storage). +- **B′** B repacked with paged aggregation: + `h5repack -S PAGE -G 65536` (55.5 MB). + +clawhdf5 (range-trace): + +| File | Phase | Uncached requests | Ranges | Bytes in ranges | 4 KiB blocks | 64 KiB blocks | 1 MiB blocks | +|---|---|---:|---:|---:|---:|---:|---:| +| A | open + list | 410 | 5 | 25 285 | 8 | 3 | 2 | +| A | read: chunk index | 33 | 8 | 4 608 | 7 | 3 | 2 | +| A | read: raw data | (25 chunks) | 2 | 1 496 533 | 368 | 25 | 3 | +| A | all metadata | 443 | 6 | 26 549 | 10 | 3 | 2 | +| B | open + list | 143 | 5 | 17 951 | 7 | 1 | 1 | +| B | read: chunk index | 46 | 19 | 49 583 | 28 | 15 | 14 | +| B | read: raw data | (780 chunks) | 15 | 43 656 557 | 10 671 | 668 | 43 | +| B | all metadata | 189 | 20 | 57 023 | 30 | 15 | 14 | +| B′ | all metadata | 413 | 24 | 71 083 | 25 | 3 | 2 | +| B′ | read: raw data | (780 chunks) | 780 | 43 613 717 | 10 920 | 780 | 49 | +| C | open + list | 455 779 | 3 | 6 357 627 | 1 553 | 98 | 7 | +| C | read: resolve `/DATASET_NAME` | 38 525 | 936 | 1 137 116 | 948 | 98 | 7 | + +libhdf5 through h5py's `fileobj` driver (read calls, bytes): + +| File | open | list | read | total calls | total bytes | +|---|---:|---:|---:|---:|---:| +| A | 3 | 93 | 25 | 121 | 1 563 559 | +| B | 2 | 78 | 780 | 860 | 43 769 952 | +| C | 4 | 36 105 | 1 | 36 110 | 19 725 823 | + +What this says: + +1. **A reader with no cache is hopeless.** clawhdf5's parsers revisit the same + structures many times (A: 26 000 loads to list 21 objects: + `Group::dataset(name)` and `Group::group(name)` re-read the group's whole + link list for every lookup). Mapped + one-to-one onto requests that is 410 round trips to list 21 objects, and + 455 779 to list C. **A cache is not an optimisation, it is the design.** +2. **With a block cache, metadata is cheap on typical files.** Metadata of A + and B touches 5–20 ranges and 2–15 blocks of 1 MiB; a 1 MiB-block cache + lists A in 2 requests and B in 1, where libhdf5 issues 96 and 80 calls + (fsspec's block cache absorbs those for h5py). +3. **"Prefetch the metadata region" alone does not work.** In B the chunk + index (v1 B-tree nodes) is interleaved with the raw data: the metadata + touches 14 distinct 1 MiB blocks spread over 48 MB. Only 4 of the 611 + corpus files h5py can open use paged aggregation (all four are libhdf5 + test files; 502 have a v0 superblock). Repacked as B′, the same metadata + fits in 3 blocks of 64 KiB — paging helps a lot when present, but a reader + cannot count on it. +4. **Raw data is naturally a batch.** Once the chunk index is walked, all + chunk extents are known: B's 780 chunks are 15 byte ranges after merging + neighbours, so a coalescing `get_ranges` call reads the dataset in a + handful of parallel requests. libhdf5 issues one read per chunk (780). +5. **An existing inefficiency becomes a blocker.** Resolving `/DATASET_NAME` + in C reads 1.1 MB (936 ranges) because + `group_v2::resolve_path_following_links` enumerates every link of the + group (`resolve_group_entries`) and compares names, instead of hashing + the name and descending the v2 B-tree name index. On an mmap this is + merely slow; over HTTP it is 98 requests of 64 KiB for one lookup. + libhdf5 reads one 512-byte block. Listing C is inherently whole-file (35 001 + object headers spread over the file), and libhdf5 reads 19.7 MB — three + times the file — to do it. + +Caveats: loads are logged with their first byte and a nominal width of 8 B, +so byte totals are approximate (±64 B per range); "uncached requests" is a +model, not a measured reader. These are counts, not timings. + +## 3. Options + +### (a) A storage trait threaded through the format crate + +```rust +// clawhdf5-format (no_std + alloc) +pub trait Storage { + /// Bytes [offset, offset + len). Short only at end of file. + fn read_at(&self, offset: u64, len: usize) -> Result, FormatError>; + /// Current length; may grow between calls (SWMR). + fn len(&self) -> u64; + /// Batch read; backends coalesce and parallelise. Default: loop. + fn read_ranges(&self, ranges: &[Range]) -> Result>, FormatError> { .. } + /// The whole file as one slice, when the backend has it (Vec, mmap). + fn as_contiguous(&self) -> Option<&[u8]> { None } +} +impl Storage for [u8] { /* Cow::Borrowed, as_contiguous = Some(self) */ } +``` + +Every `file_data: &[u8]` becomes `file: &dyn Storage` (or `&S` generic), and +every `file_data[a..b]` becomes `file.read_at(a, b - a)?`. + +- **For:** explicit, sound, errors are values (a network error is an `Err`, + not a panic); works for any backend; `u64` offsets fix 32-bit; `len()` can + grow, which is what SWMR needs; `read_ranges` gives raw data its natural + batch shape; `impl Storage for [u8]` lets the migration go one module at a + time with `&[u8]` callers unchanged. +- **Against:** touches all 109 functions and ~400 slice/len/call sites + (§1.2); every parse that learns a structure's size from its prefix needs two + reads (prefix, then body) — cheap against a cache, but it must be written + that way. `Cow::Owned` results cost a copy per read for remote backends. +- **Sync vs async:** the format crate is `no_std` and synchronous and should + stay so (parsing is CPU work; an async parser would colour 109 functions and + their callers). Remote backends are async. Two bridges, both needed: + - native: a backend that blocks on its own runtime (`object_store` + + a private tokio runtime), used from ordinary threads; + - wasm (no blocking on the main thread): a **restartable** mode where the + cache returns `FormatError::NeedBytes { offset, len }` on a miss; an async + driver fetches the block and re-runs the (pure, idempotent) operation. + parquet-rs's async reader uses the same "fetch, then parse synchronously" + split. The measurement bounds the retries: one per missed block, i.e. 1–15 + for A and B's metadata at 1 MiB blocks. +- **`&dyn` vs generic:** generics monomorphise 109 functions per backend + (binary size matters for wasm); `&dyn Storage` costs one indirect call per + structure read, negligible next to parsing. Hot raw-data loops keep their + speed through `as_contiguous()`. + +### (b) A page-cache "virtual slice" + +A type that implements `Index, Output = [u8]>` and faults pages +in on demand from interior-mutable storage, so parsers keep their slicing +syntax. + +- `Index::index` returns `&[u8]` borrowed from `&self`. A cache that evicts + pages invalidates references that are still alive — unsound — so pages can + never be evicted while the object lives (unbounded memory, which is the + problem we are solving). +- A range that crosses a page boundary has to be copied into a contiguous + buffer that also must outlive the borrow: an append-only arena that only + grows. +- `Index` cannot fail: a network error or truncated response becomes a + panic, which breaks "never return wrong data; unsupported is a clean + error". +- Open-ended slices (`file_data[x..]`) and `file_data.len()` still mean "the + whole rest of the file". +- It hides the cost model: a `for` loop over `file_data[i]` becomes thousands + of cache lookups, and nothing in the signature says the call can block. + +Rejected. It saves the signature churn of (a) but moves every failure into +panics and unsoundness. + +### (c) Keep `&[u8]` for metadata by prefetching it; range-read raw data only + +Fetch "the metadata" up front into a buffer, parse it with today's code, and +range-read only chunk data. + +- HDF5 has no pointer to a metadata region. libhdf5's paged aggregation + (`H5Pset_file_space_strategy(H5F_FSPACE_STRATEGY_PAGE)`, + `h5repack -S PAGE -G `) keeps metadata and raw data in separate pages, + and "setting an appropriate page size can have all internal file metadata + in just one page" ([Cloud-Optimized HDF/NetCDF guide][cog]) — but only + when the writer chose it: 4 of 611 corpus files (§2). +- Without paging, metadata is scattered: in B the chunk index is spread over + 14 MiB-blocks of a 48 MB file. Prefetching the first N MiB (what earthaccess' + block cache effectively does for "shared metadata at the file beginning" + ([earthaccess][ea])) is a good *heuristic*, not a correctness basis. +- A sparse buffer still presents as a `&[u8]` of the whole file's length, so + every metadata read outside the prefetched part must be caught — back to (b). + +Useful as a **policy** on top of (a) (prefetch the first block, use the page +size of paged files as the block size), not as the architecture. + +### (d) mmap a userfaultfd- or FUSE-backed file + +Keep `&[u8]` everywhere and let the kernel fault in remote pages +(userfaultfd handler, or a FUSE file system doing range GETs). + +- Linux-only (userfaultfd) or needs a FUSE mount and privileges; nothing on + macOS/Windows without a kernel extension; impossible in wasm. +- Faults are synchronous and uninterruptible from the parser's view: a network + error becomes `SIGBUS`, a stalled request hangs the thread. +- No batching: raw data arrives one fault (4 KiB, or the FUSE read size) at a + time unless the handler guesses read-ahead. +- 32-bit address space is still exhausted by mapping a large file. + +Rejected. It is how one would retrofit a C library that cannot change; we can. + +### How other readers do it + +- **libhdf5 `ros3`** replaces each POSIX read with an S3 range GET + ([h5py docs][h5py-file], [HDF Group, cloud storage options][hdfg-cloud]). + libhdf5 has a real metadata cache above the driver, and HDF5 2.2.0 added an + I/O block cache to ros3 "to reduce the number of requests to S3 for files + not using paged allocation", while paged files use the page buffer + ([HDF5 2.2.0 release][hdf5-220], [ros3 issue #4700][ros3-4700]). The page + buffer (`page_buf_size`) is "only allowed for HDF5 files created with + fs_strategy='page'" ([h5py docs][h5py-file]). PyPI h5py wheels do not + include ros3. +- **h5py + fsspec** hands libhdf5 a Python file object; each libhdf5 read + becomes a `read()` on an fsspec file, whose cache decides the requests. + fsspec's old default `readahead` cache requested 16x more data than + `blockcache` when opening HDF5 files; earthaccess now defaults to + `blockcache` with 4–16 MiB blocks by file size ([earthaccess][ea]); the + Cloud-Optimized guide recommends `blockcache` plus h5py's `page_buf_size` + and `rdcc_nbytes` ([guide][cog]). Our §2 comparison uses exactly this + path, without fsspec's cache. +- **pyfive** (pure Python, like us) supports lazy loading "on both Posix and + S3 filesystems" through fsspec, reads a variable's attributes and chunk + B-tree when it is accessed, and can merge chunk range requests with + fsspec's `merge_range_requests` ([pyfive][pyfive], [pyfive #257][pyfive-257]). +- **h5wasm** (libhdf5 compiled with Emscripten) backs a file with + `FS.createLazyFile`, which "can use range requests to incrementally access + the h5 file over the wire" ([h5wasm][h5wasm]); Emscripten's lazy files use + synchronous XHR, which browsers only allow in Web Workers + ([Emscripten FS API][emfs]). Files larger than memory remain an open issue + ([h5wasm #40][h5wasm-40]). +- **jsfive** (pure JS port of pyfive) reads from an `ArrayBuffer` of the whole + file ([jsfive][jsfive]); no lazy loading. +- **Rust `object_store`** (0.14): one `ObjectStore` trait over S3, GCS, Azure, + HTTP/WebDAV, local files and memory; `get_ranges` "will automatically + coalesce adjacent ranges into an appropriate number of parallel requests", + with `OBJECT_STORE_COALESCE_DEFAULT` as the gap below which ranges merge + ([docs.rs][os]). It builds for wasm32 except the local-filesystem and + some chunked-upload parts. + +## 4. Recommendation and migration plan + +Adopt **(a)**, with a block cache as a required part of every non-local +backend, **(c)** as a cache policy, and the wasm path through the restartable +`NeedBytes` mode. Every milestone keeps `main` green: `cargo test +--workspace`, clippy, the conformance gate at 575/697 unchanged, and the mmap +fast path within benchmark noise. + +**M0 — prerequisites (≈1 week).** +- Dense-group name lookup through the v2 B-tree name index (Jenkins hash, + record type 5), instead of enumerating all links; same for creation-order + and attribute name lookups. §2 shows 936 ranges → a handful. Regression + test: count links decoded for one lookup in a 35 001-link group. +- One checked helper for address → index conversion; replace the 133 + `as usize` casts. On 32-bit an address past `usize::MAX` is a clean error + (today it truncates or panics). +- `Group::dataset(name)`/`Group::group(name)` call `children()`, which + re-reads the group's whole link list on every lookup, so walking a group of + n children decodes its links O(n) times. Look names up through the index + (above) and let a listing hand out its entries, so the cache has less to + absorb. + +**M1 — metadata over the trait, in-memory impl identical to today (2–3 weeks).** +- Add `Storage` (above) to `clawhdf5-format`, `no_std`-compatible, with + `impl Storage for [u8]`. Add a storage error variant — `FormatError` and the + facade `Error` are not `#[non_exhaustive]`, so this is a breaking change for + exhaustive matches: bundle it with the next major version, or mark both + enums `#[non_exhaustive]` first. +- Convert modules bottom-up — superblock, object header, local/global heap, + B-tree v1/v2, fractal heap, fixed/extensible array, symbol table, group + v1/v2, shared messages, attributes, fill value, data layout — one commit + each. The old `&[u8]` signature stays as a thin wrapper over the new one + (`fn parse(data: &[u8], ..) { parse_in(data as &dyn Storage, ..) }`), so + callers and the other crates don't move yet. +- Replace the 5 open-ended slices and 38 `len()` checks with bounded reads. + +**M2 — raw data over the trait (1–2 weeks).** +- `data_read`, `chunked_read`, `parallel_read`, `partial_read`, `vds`, + VL/global heap. Chunked reads first collect the chunk extents the selection + needs, then call `read_ranges` once and decompress in parallel as today. +- Zero-copy stays: when `as_contiguous()` is `Some`, contiguous reads return + borrowed slices; `read_raw_ref`/`read_*_zerocopy`/`File::as_bytes` keep + their signatures and return a clear "not available for this storage" error + on other backends (they already return `Option`/`Result`). +- Facade: `File::open_storage(Box)`; `File::open` + keeps mmap and `from_bytes` keeps `Vec`, both through `impl Storage for [u8]`. + +**M3 — HTTP/S3 backend (1–2 weeks).** +- `clawhdf5-io`, feature `remote` (off by default, so the default tree stays + free of C and TLS stacks): `RangeStorage` over `object_store` (HTTP, S3, GCS, + Azure), with a `BlockCache` (LRU, block size configurable, default 1 MiB per + §2; the page size for paged files; the first block prefetched on open) and + a request counter exposed for tests and users. +- Python bindings: `clawhdf5.File("s3://…")` / `https://` through it. + +**M4 — wasm lazy loading (1–2 weeks).** +- `clawhdf5-wasm`: `openUrl(url) -> Promise` backed by `fetch` with a + `Range` header, on the main thread, via the restartable `NeedBytes` loop (no + Worker, no synchronous XHR — the thing h5wasm's lazy files need). Falls back + to a whole download when the server does not answer 206. +- `examples/wasm-viewer`: open by URL. + +**M5 — SWMR and growth (later, separate design).** `Storage::len()` may grow; +add `File::refresh()` that re-reads the superblock/EOF and invalidates cached +blocks past the old end. Needs libhdf5 SWMR semantics research first. + +Total: roughly 6–10 engineer-weeks for M0–M4 (estimate, not measured). + +### Keeping the local fast path + +- `impl Storage for [u8]` returns `Cow::Borrowed` — no copy, no allocation. +- Hot loops (raw-data copies, contiguous typed reads, `read_selection_native`) + branch once on `as_contiguous()` and then run today's code. +- `&dyn` dispatch is per structure, not per byte; parse code keeps working on + the returned slice. +- Gate: `crates/clawhdf5/benches/mmap_bench.rs`, the concurrent-read benches in + `clawhdf5-bench`, and the conformance run time, before and after each M1/M2 + commit, on an otherwise idle machine. Anything outside noise blocks the + commit. + +### Risks + +- **Silent regressions on local files** — mitigated by the bench gate above + and by M1 being a pure refactor (every conformance hash identical). +- **Two reads per structure** (prefix, then body) could double requests on a + cold cache. Block-aligned caching makes the second read a hit; the measured + block counts already include this pattern. +- **API break** — new error variant (see M1); `as_bytes()`-style APIs become + fallible for non-local storage. ClawBrainHub uses `File`, `FileBuilder`, + `AttrValue`, `Selection` on local files only, so it is unaffected by + behaviour, only by exhaustive matches on `Error`. +- **Restartable parsing** assumes operations are pure over the storage. The + chunk cache and metadata cache must only be filled by completed reads. +- **Cache memory** — the block cache needs a byte budget and eviction, which + (unlike option (b)) is sound because parsers hold `Cow`s, not borrows into + the cache. + +### Testing + +- **In-memory equivalence:** a `CountingStorage` wrapper over `[u8]` that + records every `read_at`; run the conformance probe through it and require + identical results. It also produces the request counts of §2 without the + trap-flag tracer. +- **Adversarial storage:** a wrapper that returns short reads, errors on the + Nth request, or serves blocks of 1 byte, to prove every miss is an `Err` + and never wrong data. +- **HTTP:** an in-process server on `127.0.0.1` (std `TcpListener`) that + honours `Range`, answers 206/416, can refuse ranges (200), and counts + requests. Tests assert both correctness against h5py and **request budgets** + (e.g. listing file A with a 1 MiB block cache takes ≤ 3 requests) so a + change that makes the reader chattier fails CI. +- **S3:** `object_store`'s in-memory store in unit tests; a MinIO or real + bucket only in an opt-in job. +- **wasm:** the existing Node/Chromium harness in `examples/wasm-viewer/test` + with a local range-capable server. +- **32-bit:** a `wasm32` or `i686` build that opens a sparse > 4 GiB file + through the counting storage. + +## Tools + +- `docs/design/tools/inventory.py` — §1 tables (`--list`, `--patterns`). +- `docs/design/tools/range-trace/` — §2 tracer (standalone crate, x86-64 + Linux, not part of the workspace). +- `docs/design/tools/libhdf5_reads.py` — §2 libhdf5 comparison and chunk + extents. + +[cog]: https://guide.cloudnativegeo.org/cloud-optimized-netcdf4-hdf5/ +[ea]: https://earthaccess.readthedocs.io/en/latest/user/explanation/fsspec/ +[h5py-file]: https://docs.h5py.org/en/stable/high/file.html +[hdfg-cloud]: https://www.hdfgroup.org/2022/08/08/cloud-storage-options-for-hdf5/ +[hdf5-220]: https://www.hdfgroup.org/2026/07/30/release-of-hdf5-2-2-0-and-two-august-events-newsletter-210/ +[ros3-4700]: https://github.com/HDFGroup/hdf5/issues/4700 +[pyfive]: https://pyfive.readthedocs.io/en/latest/quickstart/usage.html +[pyfive-257]: https://github.com/NCAS-CMS/pyfive/issues/257 +[h5wasm]: https://github.com/usnistgov/h5wasm +[h5wasm-40]: https://github.com/usnistgov/h5wasm/issues/40 +[emfs]: https://emscripten.org/docs/api_reference/Filesystem-API.html +[jsfive]: https://github.com/usnistgov/jsfive +[os]: https://docs.rs/object_store/latest/object_store/ From 3aab433edb0e857abfd667c82eed2fff95986ad4 Mon Sep 17 00:00:00 2001 From: osobh Date: Sat, 26 Sep 2026 10:17:15 -0500 Subject: [PATCH 10/43] fix(format): dataspaces and contiguous storage as libhdf5 reads them - A simple dataspace of rank 0 holds one element in libhdf5 (the product of no dimensions; h5py reads it as shape ()). num_elements() said 0, so cve-2020-18494's /dset1 failed with DataSizeMismatch { expected: 0 }. - A contiguous dataset whose storage is larger than its elements reads: libhdf5 reads the elements from the start of the storage and ignores the rest (H5D__contig_check checks only that they fit in the file). We required the sizes to be equal, so the scalar /Dset1 of cve-2024-32623 and cve-2025-2309 (240 bytes of storage for one int) failed. Storage too small for the elements is still an error. data_read::contiguous_read_len is the rule, used by every contiguous read path. - Dataspace::parse refuses what H5O__sdspace_decode refuses: more than 32 dimensions, a rank on a scalar or null dataspace, a dimension larger than its maximum (new FormatError::InvalidDataspace). Co-Authored-By: Claude Opus 5.5 (1M context) --- crates/clawhdf5-format/src/data_read.rs | 62 ++++++++++++++----- crates/clawhdf5-format/src/dataspace.rs | 82 ++++++++++++++++++++----- crates/clawhdf5-format/src/error.rs | 7 +++ crates/clawhdf5/src/mmap_file.rs | 8 +-- 4 files changed, 124 insertions(+), 35 deletions(-) diff --git a/crates/clawhdf5-format/src/data_read.rs b/crates/clawhdf5-format/src/data_read.rs index 8aec190..d1766a9 100644 --- a/crates/clawhdf5-format/src/data_read.rs +++ b/crates/clawhdf5-format/src/data_read.rs @@ -32,6 +32,22 @@ fn ensure_len(data: &[u8], offset: usize, needed: usize) -> Result<(), FormatErr Ok(()) } +/// How many bytes to read from a contiguous dataset's storage of +/// `storage_size` bytes (the layout message's size) holding `needed` bytes +/// of elements. libhdf5 reads the elements' bytes from the start of the +/// storage and ignores storage past them (`H5D__contig_check` checks only +/// that the elements fit in the file), so a larger storage reads; one too +/// small to hold the elements is an error. +pub fn contiguous_read_len(storage_size: u64, needed: usize) -> Result { + if storage_size < needed as u64 { + return Err(FormatError::DataSizeMismatch { + expected: needed, + actual: usize::try_from(storage_size).unwrap_or(usize::MAX), + }); + } + Ok(needed) +} + /// Zero-copy read of contiguous raw data, returning a borrowed slice. /// /// For contiguous layouts, returns a direct `&[u8]` slice into `file_data`. @@ -55,13 +71,7 @@ pub fn read_raw_data_zerocopy<'a>( DataLayout::Contiguous { address, size } => { let addr = address.ok_or(FormatError::NoDataAllocated)?; let addr = addr as usize; - let sz = *size as usize; - if sz != expected_size { - return Err(FormatError::DataSizeMismatch { - expected: expected_size, - actual: sz, - }); - } + let sz = contiguous_read_len(*size, expected_size)?; ensure_len(file_data, addr, sz)?; Ok(Some(&file_data[addr..addr + sz])) } @@ -172,13 +182,7 @@ fn read_raw_data_full_impl( DataLayout::Contiguous { address, size } => { let addr = address.ok_or(FormatError::NoDataAllocated)?; let addr = addr as usize; - let sz = *size as usize; - if sz != expected_size { - return Err(FormatError::DataSizeMismatch { - expected: expected_size, - actual: sz, - }); - } + let sz = contiguous_read_len(*size, expected_size)?; ensure_len(file_data, addr, sz)?; let mut out = crate::bulk_alloc::vec_for_bulk(sz); out.extend_from_slice(&file_data[addr..addr + sz]); @@ -2632,6 +2636,36 @@ mod tests { assert_eq!(result.unwrap(), &[1.5f32, 2.5, 3.5]); } + /// libhdf5 reads a contiguous dataset's elements from the start of its + /// storage and ignores storage past them (cve-2024-32623's scalar + /// `/Dset1` has 240 bytes of storage for one 4-byte element). Storage too + /// small for the elements is still an error. + #[test] + fn contiguous_storage_larger_than_the_elements_reads() { + let dt = make_f64_le_type(); + let ds = make_simple_dataspace(&[2]); + let mut file_data = vec![0u8; 64]; + file_data[..8].copy_from_slice(&1.5f64.to_le_bytes()); + file_data[8..16].copy_from_slice(&2.5f64.to_le_bytes()); + file_data[16..24].copy_from_slice(&9.0f64.to_le_bytes()); + let layout = DataLayout::Contiguous { + address: Some(0), + size: 40, + }; + let raw = read_raw_data(&file_data, &layout, &ds, &dt).unwrap(); + assert_eq!(raw, file_data[..16]); + let zc = read_raw_data_zerocopy(&file_data, &layout, &ds, &dt).unwrap(); + assert_eq!(zc, Some(&file_data[..16])); + let small = DataLayout::Contiguous { + address: Some(0), + size: 8, + }; + assert!(matches!( + read_raw_data(&file_data, &small, &ds, &dt), + Err(FormatError::DataSizeMismatch { .. }) + )); + } + #[test] fn zerocopy_size_mismatch() { let dt = make_f64_le_type(); diff --git a/crates/clawhdf5-format/src/dataspace.rs b/crates/clawhdf5-format/src/dataspace.rs index a08ca9d..8eb37e1 100644 --- a/crates/clawhdf5-format/src/dataspace.rs +++ b/crates/clawhdf5-format/src/dataspace.rs @@ -7,6 +7,9 @@ use alloc::vec::Vec; use crate::error::FormatError; +/// Most dimensions a dataspace can have (`H5S_MAX_RANK`). +pub const MAX_RANK: u8 = 32; + /// Type of dataspace. #[derive(Debug, Clone, PartialEq)] pub enum DataspaceType { @@ -67,6 +70,12 @@ impl Dataspace { let version = data[0]; let rank = data[1]; let flags = data[2]; + // H5O__sdspace_decode's checks. + if rank > MAX_RANK { + return Err(FormatError::InvalidDataspace( + "simple dataspace dimensionality is too large", + )); + } let (space_type, header_size) = match version { 1 => { @@ -88,6 +97,11 @@ impl Dataspace { 2 => DataspaceType::Null, _ => return Err(FormatError::InvalidDataspaceType(type_byte)), }; + if st != DataspaceType::Simple && rank > 0 { + return Err(FormatError::InvalidDataspace( + "invalid rank for scalar or NULL dataspace", + )); + } (st, 4usize) } _ => return Err(FormatError::InvalidDataspaceVersion(version)), @@ -107,8 +121,13 @@ impl Dataspace { // Read max dimensions if flags bit 0 is set let max_dimensions = if flags & 0x01 != 0 { let mut max_dims = Vec::with_capacity(rank as usize); - for _ in 0..rank { + for i in 0..rank as usize { let val = read_length(data, pos, length_size)?; + if dimensions[i] > val { + return Err(FormatError::InvalidDataspace( + "dataspace dimension size is greater than its maximum size", + )); + } max_dims.push(val); pos += ls; } @@ -176,7 +195,6 @@ impl Dataspace { match self.space_type { DataspaceType::Null => Ok(0), DataspaceType::Scalar => Ok(1), - DataspaceType::Simple if self.dimensions.is_empty() => Ok(0), DataspaceType::Simple => self .dimensions .iter() @@ -195,18 +213,14 @@ impl Dataspace { match self.space_type { DataspaceType::Null => 0, DataspaceType::Scalar => 1, - DataspaceType::Simple => { - if self.dimensions.is_empty() { - 0 - } else { - // Saturate rather than wrap: a wrapped product could - // under-size a buffer. Size-critical callers use - // `checked_num_elements`. - self.dimensions - .iter() - .fold(1u64, |acc, &d| acc.saturating_mul(d)) - } - } + // A simple dataspace of rank 0 holds one element, as in libhdf5 + // (the product of no dimensions). Saturate rather than wrap: a + // wrapped product could under-size a buffer. Size-critical + // callers use `checked_num_elements`. + DataspaceType::Simple => self + .dimensions + .iter() + .fold(1u64, |acc, &d| acc.saturating_mul(d)), } } } @@ -352,4 +366,44 @@ mod tests { let ds = Dataspace::parse(&data, 8).unwrap(); assert_eq!(ds.max_dimensions, Some(vec![10])); } + + /// A simple dataspace of rank 0 (cve-2020-18494's `/dset1`) holds one + /// element in libhdf5, which h5py reads as shape `()`. It was 0. + #[test] + fn simple_rank_zero_holds_one_element() { + let data = build_v2_dataspace(0, 0, 1, &[], None); + let ds = Dataspace::parse(&data, 8).unwrap(); + assert_eq!(ds.space_type, DataspaceType::Simple); + assert_eq!(ds.num_elements(), 1); + assert_eq!(ds.checked_num_elements().unwrap(), 1); + } + + /// `H5O__sdspace_decode`'s checks. + #[test] + fn refuses_what_libhdf5_refuses() { + let too_many = build_v2_dataspace(33, 0, 1, &[1; 33], None); + assert!(matches!( + Dataspace::parse(&too_many, 8), + Err(FormatError::InvalidDataspace(_)) + )); + let scalar_with_rank = build_v2_dataspace(1, 0, 0, &[4], None); + assert!(matches!( + Dataspace::parse(&scalar_with_rank, 8), + Err(FormatError::InvalidDataspace(_)) + )); + let null_with_rank = build_v2_dataspace(1, 0, 2, &[4], None); + assert!(matches!( + Dataspace::parse(&null_with_rank, 8), + Err(FormatError::InvalidDataspace(_)) + )); + let over_max = build_v1_dataspace(2, 0x01, &[5, 20], Some(&[10, 10])); + assert!(matches!( + Dataspace::parse(&over_max, 8), + Err(FormatError::InvalidDataspace(_)) + )); + // 32 dimensions, and a size equal to the maximum or unlimited, are fine. + assert!(Dataspace::parse(&build_v2_dataspace(32, 0, 1, &[1; 32], None), 8).is_ok()); + let at_max = build_v1_dataspace(2, 0x01, &[10, 20], Some(&[10, u64::MAX])); + assert!(Dataspace::parse(&at_max, 8).is_ok()); + } } diff --git a/crates/clawhdf5-format/src/error.rs b/crates/clawhdf5-format/src/error.rs index d285cbf..b365794 100644 --- a/crates/clawhdf5-format/src/error.rs +++ b/crates/clawhdf5-format/src/error.rs @@ -226,6 +226,10 @@ pub enum FormatError { /// A link libhdf5 refuses to list: a symbol-table entry with an empty /// name ("invalid link name"). Listing the group fails, as in libhdf5. InvalidLinkName, + /// A dataspace message libhdf5 refuses to decode (the reason is + /// libhdf5's own error text): more than 32 dimensions, a rank on a + /// scalar or null dataspace, a dimension larger than its maximum. + InvalidDataspace(&'static str), } impl fmt::Display for FormatError { @@ -500,6 +504,9 @@ impl fmt::Display for FormatError { FormatError::InvalidLinkName => { write!(f, "invalid link name: a group entry has an empty name") } + FormatError::InvalidDataspace(why) => { + write!(f, "invalid dataspace: {why}") + } } } } diff --git a/crates/clawhdf5/src/mmap_file.rs b/crates/clawhdf5/src/mmap_file.rs index 76d9ea1..59adee9 100644 --- a/crates/clawhdf5/src/mmap_file.rs +++ b/crates/clawhdf5/src/mmap_file.rs @@ -390,13 +390,7 @@ impl<'f> MmapDataset<'f> { match &dl { DataLayout::Contiguous { address, size } => { let addr = address.ok_or(Error::Format(FormatError::NoDataAllocated))?; - let sz = *size as usize; - if sz != expected { - return Err(Error::Format(FormatError::DataSizeMismatch { - expected, - actual: sz, - })); - } + let sz = clawhdf5_format::data_read::contiguous_read_len(*size, expected)?; let data = self.file.hdf5_bytes(); let a = addr as usize; if a + sz > data.len() { From 193a5f8a8267337c8534bb1f6d1b55a575fb3bce Mon Sep 17 00:00:00 2001 From: osobh Date: Sat, 26 Sep 2026 10:17:37 -0500 Subject: [PATCH 11/43] writer: track attribute creation order with track_order h5py's track_order=True orders attributes as well as links; the writer tracked links only. A tracking object's header now sets the attribute creation order tracked/indexed flags and carries per-message creation orders, an Attribute Info message holds the next order (inline too), and dense storage gets a type-9 creation-order index. The file default applies to datasets, with DatasetBuilder::track_order per dataset; more than 65 535 attributes on a tracking object is an error (libhdf5's counter is 2 bytes). The reader lists such attributes in creation order. h5py lists them in order (inline, dense, 20 000 on one dataset) and keeps numbering in r+ mode, including its inline-to-dense move. Co-Authored-By: Claude Opus 5.5 (1M context) --- CHANGELOG.md | 16 + crates/clawhdf5-format/src/attribute.rs | 35 +- crates/clawhdf5-format/src/file_writer.rs | 360 ++++++++++++++---- .../src/object_header_writer.rs | 65 +++- crates/clawhdf5-format/src/type_builders.rs | 23 +- crates/clawhdf5-tools/tests/h5rs_interop.rs | 6 + crates/clawhdf5/src/writer.rs | 8 +- .../clawhdf5/tests/writer_groups_interop.rs | 91 +++++ docs/known-issues.md | 12 +- 9 files changed, 515 insertions(+), 101 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 448d2a0..dbecdc4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,22 @@ ## Unreleased ### Writer: large dense indexes (2026-09-26) +- **`track_order` orders attributes too, as h5py's `track_order=True` + does.** It tracked link creation order only, so h5py listed a tracked + object's attributes by name. A tracking object's header now has the + attribute creation order tracked and indexed flags, an Attribute Info + message with the next creation order (also for inline attributes), and a + creation order on each inline attribute message; dense attribute storage + gets a creation-order index (B-tree type 9). `FileWriter::track_order` / + `FileBuilder::track_order` now apply to datasets' attributes as well, and + `DatasetBuilder::track_order` sets it per dataset. Groups and datasets + that track order are written differently from before; others are + unchanged. More than 65 535 attributes on a tracking object is an error + (libhdf5's creation order counter is 2 bytes). The reader + (`attribute::extract_attributes*`) lists a tracking object's attributes + in creation order. Test `track_order_lists_attributes_in_creation_order` + (h5py lists, reads and extends them in "r+" mode, including libhdf5's + move from inline to dense storage). - **No more 65 535-record limit on the writer's v2 B-trees.** Dense link storage (name index and creation-order index), dense attribute storage and the chunk index of datasets with more than one unlimited dimension diff --git a/crates/clawhdf5-format/src/attribute.rs b/crates/clawhdf5-format/src/attribute.rs index 11bde3e..6c4e9ae 100644 --- a/crates/clawhdf5-format/src/attribute.rs +++ b/crates/clawhdf5-format/src/attribute.rs @@ -450,6 +450,8 @@ fn extract_attributes_with( on_error: &mut dyn FnMut(FormatError) -> Result<(), FormatError>, ) -> Result, FormatError> { let mut attrs = Vec::new(); + // Each attribute's creation order, where the file records one. + let mut orders: Vec = Vec::new(); // Collect compact attributes (inline in OH) for msg in &header.messages { @@ -479,7 +481,10 @@ fn extract_attributes_with( }; let attr = attr.and_then(|a| check_in_header(a, header)); match attr { - Ok(attr) => attrs.push(attr), + Ok(attr) => { + attrs.push(attr); + orders.push(msg.creation_order.map_or(0, u32::from)); + } Err(e) => on_error(e)?, } } @@ -487,20 +492,30 @@ fn extract_attributes_with( // Check for dense attributes via AttributeInfo message let attr_info = find_attribute_info(header, offset_size)?; - if let Some(info) = attr_info + if let Some(info) = &attr_info && let Some(fh_addr) = info.fractal_heap_address { extract_dense_attributes( file_data, - &info, + info, fh_addr, offset_size, length_size, &mut attrs, + &mut orders, on_error, )?; } + // An object that tracks attribute creation order lists its attributes + // in that order (h5py's `track_order=True`), as libhdf5 does; otherwise + // they come in storage order. + if attr_info.is_some_and(|i| i.max_creation_index.is_some()) { + let mut paired: Vec<(u32, AttributeMessage)> = orders.into_iter().zip(attrs).collect(); + paired.sort_by_key(|(o, _)| *o); + attrs = paired.into_iter().map(|(_, a)| a).collect(); + } + Ok(attrs) } @@ -518,7 +533,9 @@ fn find_attribute_info( Ok(None) } -/// Extract attributes from dense storage (fractal heap + B-tree v2). +/// Extract attributes from dense storage (fractal heap + B-tree v2), and +/// each one's creation order into `orders`. +#[allow(clippy::too_many_arguments)] fn extract_dense_attributes( file_data: &[u8], attr_info: &AttributeInfoMessage, @@ -526,6 +543,7 @@ fn extract_dense_attributes( offset_size: u8, length_size: u8, attrs: &mut Vec, + orders: &mut Vec, on_error: &mut dyn FnMut(FormatError) -> Result<(), FormatError>, ) -> Result<(), FormatError> { // Parse fractal heap @@ -561,7 +579,14 @@ fn extract_dense_attributes( AttributeMessage::parse_in_file(&attr_data, file_data, offset_size, length_size) }); match attr { - Ok(attr) => attrs.push(attr), + Ok(attr) => { + attrs.push(attr); + let order = record + .data + .get(id_len + 1..id_len + 5) + .map_or(0, |b| u32::from_le_bytes([b[0], b[1], b[2], b[3]])); + orders.push(order); + } Err(e) => on_error(e)?, } } diff --git a/crates/clawhdf5-format/src/file_writer.rs b/crates/clawhdf5-format/src/file_writer.rs index a9b3af5..2568582 100644 --- a/crates/clawhdf5-format/src/file_writer.rs +++ b/crates/clawhdf5-format/src/file_writer.rs @@ -75,14 +75,57 @@ const DENSE_LINK_THRESHOLD: usize = 8; // ---- OH builders ---- +/// An object's attributes as its header stores them: inline Attribute +/// messages, or (`dense`) the Attribute Info message of dense storage; with +/// `track_order`, their creation order tracked and indexed. +#[derive(Clone, Copy)] +pub(crate) struct AttrStorage<'a> { + pub(crate) attrs: &'a [AttributeMessage], + pub(crate) dense: Option<&'a DenseAttrBlob>, + pub(crate) track_order: bool, +} + +impl AttrStorage<'_> { + /// Add the attribute messages to the header being built. Tracking + /// creation order, as libhdf5 does it: the header's flags say so, an + /// Attribute Info message is written even for inline attributes (it + /// holds the next creation order), and each inline attribute's message + /// carries its creation order. + fn add_to(&self, w: &mut ObjectHeaderWriter) { + if self.track_order { + w.track_attr_order(); + } + if let Some(blob) = self.dense { + w.add_message(MessageType::AttributeInfo, blob.attr_info_message.clone()); + return; + } + if self.track_order { + w.add_message( + MessageType::AttributeInfo, + serialize_attribute_info( + u64::MAX, + u64::MAX, + Some((self.attrs.len() as u16, u64::MAX)), + ), + ); + } + for (i, attr) in self.attrs.iter().enumerate() { + w.add_message_with_order( + MessageType::Attribute, + attr.serialize(LENGTH_SIZE), + i as u16, + ); + } + } +} + #[allow(clippy::too_many_arguments)] pub(crate) fn build_chunked_dataset_oh( dt: &Datatype, ds: &Dataspace, layout_message: &[u8], pipeline_message: Option<&[u8]>, - attrs: &[AttributeMessage], - dense_blob: Option<&DenseAttrBlob>, + attrs: AttrStorage<'_>, fill_message: &[u8], refcount: u32, ) -> Result, FormatError> { @@ -94,13 +137,7 @@ pub(crate) fn build_chunked_dataset_oh( if let Some(pm) = pipeline_message { w.add_message(MessageType::FilterPipeline, pm.to_vec()); } - if let Some(blob) = dense_blob { - w.add_message(MessageType::AttributeInfo, blob.attr_info_message.clone()); - } else { - for attr in attrs { - w.add_message(MessageType::Attribute, attr.serialize(LENGTH_SIZE)); - } - } + attrs.add_to(&mut w); add_refcount(&mut w, refcount); w.serialize() } @@ -111,8 +148,7 @@ pub(crate) fn build_dataset_oh( ds: &Dataspace, data_addr: u64, data_size: u64, - attrs: &[AttributeMessage], - dense_blob: Option<&DenseAttrBlob>, + attrs: AttrStorage<'_>, fill_message: &[u8], refcount: u32, ) -> Result, FormatError> { @@ -132,13 +168,7 @@ pub(crate) fn build_dataset_oh( dl.extend_from_slice(&data_addr.to_le_bytes()); dl.extend_from_slice(&data_size.to_le_bytes()); w.add_message(MessageType::DataLayout, dl); - if let Some(blob) = dense_blob { - w.add_message(MessageType::AttributeInfo, blob.attr_info_message.clone()); - } else { - for attr in attrs { - w.add_message(MessageType::Attribute, attr.serialize(LENGTH_SIZE)); - } - } + attrs.add_to(&mut w); add_refcount(&mut w, refcount); w.serialize() } @@ -148,8 +178,7 @@ pub(crate) fn build_compact_dataset_oh( dt: &Datatype, ds: &Dataspace, data: &[u8], - attrs: &[AttributeMessage], - dense_blob: Option<&DenseAttrBlob>, + attrs: AttrStorage<'_>, fill_message: &[u8], refcount: u32, ) -> Result, FormatError> { @@ -164,13 +193,7 @@ pub(crate) fn build_compact_dataset_oh( dl.extend_from_slice(&(data.len() as u16).to_le_bytes()); dl.extend_from_slice(data); w.add_message(MessageType::DataLayout, dl); - if let Some(blob) = dense_blob { - w.add_message(MessageType::AttributeInfo, blob.attr_info_message.clone()); - } else { - for attr in attrs { - w.add_message(MessageType::Attribute, attr.serialize(LENGTH_SIZE)); - } - } + attrs.add_to(&mut w); add_refcount(&mut w, refcount); w.serialize() } @@ -182,8 +205,7 @@ pub(crate) fn build_group_oh( links: &[LinkMessage], link_info: &[u8], dense_links: bool, - attrs: &[AttributeMessage], - dense_blob: Option<&DenseAttrBlob>, + attrs: AttrStorage<'_>, refcount: u32, ) -> Result, FormatError> { let mut w = ObjectHeaderWriter::new(); @@ -198,13 +220,7 @@ pub(crate) fn build_group_oh( w.add_message(MessageType::Link, link.serialize(OFFSET_SIZE)); } } - if let Some(blob) = dense_blob { - w.add_message(MessageType::AttributeInfo, blob.attr_info_message.clone()); - } else { - for attr in attrs { - w.add_message(MessageType::Attribute, attr.serialize(LENGTH_SIZE)); - } - } + attrs.add_to(&mut w); add_refcount(&mut w, refcount); w.serialize() } @@ -891,11 +907,31 @@ fn write_frhp(p: WriteFrhp) -> Vec { frhp } +/// libhdf5 numbers the attributes of an object that tracks their creation +/// order with a 2-byte counter. +fn check_tracked_attr_count(track_order: bool, n: usize) -> Result<(), FormatError> { + if track_order && n > usize::from(u16::MAX) { + return Err(FormatError::SerializationError(format!( + "{n} attributes on one object with creation order tracked: libhdf5 \ + numbers at most {} (set fewer, or turn off track_order)", + u16::MAX + ))); + } + Ok(()) +} + /// Build dense attribute storage for a set of attributes. +/// +/// With `track_order` the Attribute Info message tracks creation order (an +/// attribute's creation order is its position in `attrs`) and a type-9 +/// creation-order index follows the name index, as libhdf5 writes for h5py's +/// `track_order=True`. libhdf5 numbers at most 65 535 attributes. pub(crate) fn build_dense_attrs( attrs: &[AttributeMessage], base_address: u64, + track_order: bool, ) -> Result { + check_tracked_attr_count(track_order, attrs.len())?; // Dense attrs use v3 attribute messages (adds character set encoding byte). let serialized: Vec> = attrs.iter().map(|a| a.serialize_v3(LENGTH_SIZE)).collect(); @@ -936,7 +972,30 @@ pub(crate) fn build_dense_attrs( let mut blob = heap.blob; blob.extend_from_slice(&dense_v2_btree(8, record_size, &records, bthd_addr)?); - let attr_info = serialize_attribute_info(frhp_addr, bthd_addr); + let order = if track_order { + // Type 9 records: heap ID, message flags, creation order (the key). + let records: Vec> = heap_ids + .iter() + .enumerate() + .map(|(i, heap_id)| { + let mut rec = heap_id.clone(); + rec.push(0); // msg_flags + rec.extend_from_slice(&(i as u32).to_le_bytes()); + rec + }) + .collect(); + let corder_addr = base_address + blob.len() as u64; + blob.extend_from_slice(&dense_v2_btree( + 9, + heap_id_length + 1 + 4, + &records, + corder_addr, + )?); + Some((attrs.len() as u16, corder_addr)) + } else { + None + }; + let attr_info = serialize_attribute_info(frhp_addr, bthd_addr, order); Ok(DenseAttrBlob { attr_info_message: attr_info, @@ -1139,12 +1198,25 @@ fn encode_managed_id(offset: u64, length: u64, max_heap_size: u16, id_length: u1 id } -fn serialize_attribute_info(fh_addr: u64, btree_name_addr: u64) -> Vec { +/// Serialize an Attribute Info message (version 0). `order` — the next +/// creation order to assign and the creation-order index's address — is +/// present when creation order is tracked and indexed. +fn serialize_attribute_info( + fh_addr: u64, + btree_name_addr: u64, + order: Option<(u16, u64)>, +) -> Vec { let mut data = Vec::new(); data.push(0); // version - data.push(0x00); // flags + data.push(if order.is_some() { 0x03 } else { 0x00 }); // flags: tracked, indexed + if let Some((next, _)) = order { + data.extend_from_slice(&next.to_le_bytes()); + } data.extend_from_slice(&fh_addr.to_le_bytes()); data.extend_from_slice(&btree_name_addr.to_le_bytes()); + if let Some((_, corder_addr)) = order { + data.extend_from_slice(&corder_addr.to_le_bytes()); + } data } @@ -1219,8 +1291,7 @@ pub(crate) fn build_vds_dataset_oh( dt: &Datatype, ds: &Dataspace, global_heap_addr: u64, - attrs: &[AttributeMessage], - dense_blob: Option<&DenseAttrBlob>, + attrs: AttrStorage<'_>, fill_message: &[u8], refcount: u32, ) -> Result, FormatError> { @@ -1235,13 +1306,7 @@ pub(crate) fn build_vds_dataset_oh( dl.extend_from_slice(&global_heap_addr.to_le_bytes()); dl.extend_from_slice(&1u32.to_le_bytes()); // object index 1 in the collection w.add_message(MessageType::DataLayout, dl); - if let Some(blob) = dense_blob { - w.add_message(MessageType::AttributeInfo, blob.attr_info_message.clone()); - } else { - for attr in attrs { - w.add_message(MessageType::Attribute, attr.serialize(LENGTH_SIZE)); - } - } + attrs.add_to(&mut w); add_refcount(&mut w, refcount); w.serialize() } @@ -1276,7 +1341,8 @@ fn write_undef_offset(buf: &mut Vec, offset_size: u8) { pub struct FileWriter { /// The root group's contents (its name is unused). root: GroupBuilder, - /// Default for groups that do not call [`GroupBuilder::track_order`]. + /// Default for groups and datasets that do not set their own + /// `track_order`. track_order: bool, /// Global alignment threshold: datasets with raw data >= this many bytes /// will have their data aligned to `alignment_bytes`. @@ -1311,11 +1377,18 @@ struct DsFlat { virtual_sources: Option>, /// Number of hard links to the dataset. refcount: u32, + /// Track (and index) attribute creation order. + track_order: bool, } /// Convert a DatasetBuilder into a DsFlat, handling VDS (which does not /// require a `data` field). -fn flatten_ds(db: DatasetBuilder, refcount: u32) -> Result { +fn flatten_ds( + db: DatasetBuilder, + refcount: u32, + default_track_order: bool, +) -> Result { + let track_order = db.track_order.unwrap_or(default_track_order); let dt = db.datatype.ok_or(FormatError::DatasetMissingData)?; let shape = db.shape.ok_or(FormatError::DatasetMissingShape)?; let is_vds = db.virtual_sources.is_some(); @@ -1365,6 +1438,7 @@ fn flatten_ds(db: DatasetBuilder, refcount: u32) -> Result alignment: db.alignment, virtual_sources: db.virtual_sources, refcount, + track_order, }) } @@ -1421,10 +1495,12 @@ impl FileWriter { self } - /// Track (and index) link creation order in every group that does not - /// set its own [`GroupBuilder::track_order`], the root included — as - /// h5py's `track_order=True`: libhdf5 then lists members in the order - /// they were added. Off by default (members are listed by name). + /// Track (and index) creation order — of links and attributes in every + /// group that does not set its own [`GroupBuilder::track_order`], the + /// root included, and of attributes on every dataset that does not set + /// its own [`DatasetBuilder::track_order`] — as h5py's + /// `track_order=True`: libhdf5 then lists members and attributes in the + /// order they were added. Off by default (they are listed by name). pub fn track_order(&mut self, track: bool) -> &mut Self { self.track_order = track; self @@ -1495,7 +1571,7 @@ impl FileWriter { let all_ds: Vec = tree .datasets .into_iter() - .map(|(db, refcount)| flatten_ds(db, refcount)) + .map(|(db, refcount)| flatten_ds(db, refcount, self.track_order)) .collect::>()?; let groups: Vec = tree .groups @@ -1512,6 +1588,15 @@ impl FileWriter { }) .collect(); + // Refuse up front what dense storage would refuse after the work. + let tracked = groups + .iter() + .map(|g| (g.track_order, g.attrs.len())) + .chain(all_ds.iter().map(|d| (d.track_order, d.attrs.len()))); + for (track, n) in tracked { + check_tracked_attr_count(track, n)?; + } + // Every datatype must have an on-disk encoding before anything is laid // out: `Datatype::serialize` itself cannot report a failure. let group_attrs = groups.iter().flat_map(|g| &g.attrs); @@ -1566,7 +1651,7 @@ impl FileWriter { .map(|(gi, g)| { let dummy_links = g.link_messages(&[], &[]); let attr_blob = group_dense[gi] - .then(|| build_dense_attrs(&g.attrs, 0)) + .then(|| build_dense_attrs(&g.attrs, 0, g.track_order)) .transpose()?; let li = if group_links_dense[gi] { serialize_link_info( @@ -1582,8 +1667,11 @@ impl FileWriter { &dummy_links, &li, group_links_dense[gi], - &g.attrs, - attr_blob.as_ref(), + AttrStorage { + attrs: &g.attrs, + dense: attr_blob.as_ref(), + track_order: g.track_order, + }, g.refcount, ) .map(|oh| oh.len()) @@ -1602,7 +1690,7 @@ impl FileWriter { let mut dummy_cursor = 0u64; for (i, d) in all_ds.iter().enumerate() { let dense_blob = ds_dense[i] - .then(|| build_dense_attrs(&d.attrs, 0)) + .then(|| build_dense_attrs(&d.attrs, 0, d.track_order)) .transpose()?; if is_vds[i] { // VDS: dummy OH with address 0 to get the OH size. The global @@ -1611,8 +1699,11 @@ impl FileWriter { &d.dt, &d.ds, 0, // dummy address - &d.attrs, - dense_blob.as_ref(), + AttrStorage { + attrs: &d.attrs, + dense: dense_blob.as_ref(), + track_order: d.track_order, + }, &d.fill_message, d.refcount, )?; @@ -1651,8 +1742,11 @@ impl FileWriter { &d.ds, &result.layout_message, result.pipeline_message.as_deref(), - &d.attrs, - dense_blob.as_ref(), + AttrStorage { + attrs: &d.attrs, + dense: dense_blob.as_ref(), + track_order: d.track_order, + }, &d.fill_message, d.refcount, )?; @@ -1666,8 +1760,11 @@ impl FileWriter { &d.dt, &d.ds, &d.raw, - &d.attrs, - dense_blob.as_ref(), + AttrStorage { + attrs: &d.attrs, + dense: dense_blob.as_ref(), + track_order: d.track_order, + }, &d.fill_message, d.refcount, )?; @@ -1682,8 +1779,11 @@ impl FileWriter { &d.ds, 0, d.raw.len() as u64, - &d.attrs, - dense_blob.as_ref(), + AttrStorage { + attrs: &d.attrs, + dense: dense_blob.as_ref(), + track_order: d.track_order, + }, &d.fill_message, d.refcount, )?; @@ -1727,7 +1827,7 @@ impl FileWriter { group_link_blob_addrs.push(None); } if group_dense[gi] { - let blob = build_dense_attrs(&g.attrs, cursor2 as u64)?; + let blob = build_dense_attrs(&g.attrs, cursor2 as u64, g.track_order)?; cursor2 += blob.blob.len(); group_dense_blobs.push(Some(blob)); } else { @@ -1744,7 +1844,8 @@ impl FileWriter { let addr = cursor2 as u64; cursor2 += sz; if ds_dense[i] { - let blob = build_dense_attrs(&all_ds[i].attrs, cursor2 as u64)?; + let blob = + build_dense_attrs(&all_ds[i].attrs, cursor2 as u64, all_ds[i].track_order)?; cursor2 += blob.blob.len(); ds_dense_blobs.push(Some(blob)); } else { @@ -1768,8 +1869,11 @@ impl FileWriter { &d.dt, &d.ds, heap_addr, - &d.attrs, - ds_dense_blobs[i].as_ref(), + AttrStorage { + attrs: &d.attrs, + dense: ds_dense_blobs[i].as_ref(), + track_order: d.track_order, + }, &d.fill_message, d.refcount, )?; @@ -1796,8 +1900,11 @@ impl FileWriter { &d.ds, &result.layout_message, result.pipeline_message.as_deref(), - &d.attrs, - ds_dense_blobs[i].as_ref(), + AttrStorage { + attrs: &d.attrs, + dense: ds_dense_blobs[i].as_ref(), + track_order: d.track_order, + }, &d.fill_message, d.refcount, )?; @@ -1812,8 +1919,11 @@ impl FileWriter { &d.dt, &d.ds, &d.raw, - &d.attrs, - ds_dense_blobs[i].as_ref(), + AttrStorage { + attrs: &d.attrs, + dense: ds_dense_blobs[i].as_ref(), + track_order: d.track_order, + }, &d.fill_message, d.refcount, )?; @@ -1838,8 +1948,11 @@ impl FileWriter { &d.ds, cursor2 as u64, d.raw.len() as u64, - &d.attrs, - ds_dense_blobs[i].as_ref(), + AttrStorage { + attrs: &d.attrs, + dense: ds_dense_blobs[i].as_ref(), + track_order: d.track_order, + }, &d.fill_message, d.refcount, )?; @@ -1907,8 +2020,11 @@ impl FileWriter { &links, &li, link_blob.is_some(), - &g.attrs, - group_dense_blobs[gi].as_ref(), + AttrStorage { + attrs: &g.attrs, + dense: group_dense_blobs[gi].as_ref(), + track_order: g.track_order, + }, g.refcount, )?; debug_assert_eq!(oh.len(), group_oh_sizes[gi]); @@ -2139,6 +2255,92 @@ mod tests { assert_eq!(read_dataset_f64(&bytes, "data"), vec![1.0, 2.0, 3.0]); } + /// Attribute names of the object at `path`, in the order the reader + /// lists them. + fn attr_names(bytes: &[u8], path: &str) -> Vec { + let sig = signature::find_signature(bytes).unwrap(); + let sb = Superblock::parse(bytes, sig).unwrap(); + let addr = if path == "/" { + sb.root_group_address + } else { + resolve_path_any(bytes, &sb, path).unwrap() + }; + let hdr = + ObjectHeader::parse(bytes, addr as usize, sb.offset_size, sb.length_size).unwrap(); + crate::attribute::extract_attributes_full(bytes, &hdr, sb.offset_size, sb.length_size) + .unwrap() + .into_iter() + .map(|a| a.name) + .collect() + } + + #[test] + fn tracked_attributes_are_read_in_creation_order() { + let set = |names: &[String]| -> Vec<(String, AttrValue)> { + names + .iter() + .enumerate() + .map(|(i, n)| (n.clone(), AttrValue::I64(i as i64))) + .collect() + }; + let compact: Vec = ["zeta", "alpha", "mid"].map(String::from).to_vec(); + let dense: Vec = (0..30).rev().map(|i| format!("a{i:02}")).collect(); + let mut fw = FileWriter::new(); + fw.track_order(true); + for (n, v) in set(&compact) { + fw.set_root_attr(&n, v); + } + let ds = fw.create_dataset("dense"); + ds.with_i32_data(&[1]); + for (n, v) in set(&dense) { + ds.set_attr(&n, v); + } + let ds = fw.create_dataset("untracked"); + ds.with_i32_data(&[1]).track_order(false); + for (n, v) in set(&dense) { + ds.set_attr(&n, v); + } + let mut g = fw.create_group("g"); + g.track_order(false); + for (n, v) in set(&compact) { + g.set_attr(&n, v); + } + fw.add_group(g.finish()); + let bytes = fw.finish().unwrap(); + assert_eq!(attr_names(&bytes, "/"), compact); + assert_eq!(attr_names(&bytes, "dense"), dense); + // Without tracking: storage order (inline: as added; dense: hash). + assert_eq!(attr_names(&bytes, "g"), compact); + let mut by_hash = dense.clone(); + by_hash.sort_by_key(|n| crate::checksum::jenkins_lookup3(n.as_bytes())); + assert_eq!(attr_names(&bytes, "untracked"), by_hash); + } + + #[test] + fn too_many_tracked_attributes_is_an_error() { + // libhdf5 numbers at most 65 535 attributes on an object that + // tracks their creation order (a 2-byte field). (`set_attr` looks + // for an earlier value, so 65 536 of them through the builder take + // a while; build the messages directly.) + let attrs: Vec = (0..65_536) + .map(|i| build_attr_message(&format!("a{i}"), &AttrValue::I64(i))) + .collect(); + let err = build_dense_attrs(&attrs, 0, true) + .err() + .unwrap() + .to_string(); + assert!(err.contains("65536 attributes on one object"), "{err}"); + assert!(build_dense_attrs(&attrs[1..], 0, true).is_ok()); + assert!(build_dense_attrs(&attrs, 0, false).is_ok()); + let mut fw = FileWriter::new(); + let ds = fw.create_dataset("x"); + ds.with_i32_data(&[1]).track_order(true); + for i in 0..20 { + ds.set_attr(&format!("a{i}"), AttrValue::I64(i)); + } + assert!(fw.finish().is_ok()); + } + #[test] fn dense_attrs_root_group_self_roundtrip() { let mut fw = FileWriter::new(); diff --git a/crates/clawhdf5-format/src/object_header_writer.rs b/crates/clawhdf5-format/src/object_header_writer.rs index 8d52ad4..bbb3c0a 100644 --- a/crates/clawhdf5-format/src/object_header_writer.rs +++ b/crates/clawhdf5-format/src/object_header_writer.rs @@ -12,9 +12,16 @@ use crate::message_type::MessageType; /// its size truncated to 16 bits produced files libhdf5 refuses. pub const MAX_MESSAGE_SIZE: usize = u16::MAX as usize; +/// Object header flags: attribute creation order tracked (each message +/// then carries a 2-byte creation order) and indexed. +const OHDR_ATTR_CRT_ORDER_TRACKED: u8 = 0x04; +const OHDR_ATTR_CRT_ORDER_INDEXED: u8 = 0x08; + /// Writer for v2 object headers with proper checksums. pub struct ObjectHeaderWriter { - messages: Vec<(MessageType, Vec, u8)>, // (type, data, msg_flags) + messages: Vec<(MessageType, Vec, u8, u16)>, // (type, data, msg_flags, creation order) + /// Attribute creation order tracked and indexed. + attr_order: bool, } impl ObjectHeaderWriter { @@ -22,17 +29,33 @@ impl ObjectHeaderWriter { pub fn new() -> Self { Self { messages: Vec::new(), + attr_order: false, } } + /// Track and index attribute creation order, as libhdf5 does for an + /// object created with `H5P_CRT_ORDER_TRACKED | H5P_CRT_ORDER_INDEXED` + /// (h5py's `track_order=True`): the header's flags say so, and every + /// message carries a creation order (an attribute's own; 0 for the + /// others). libhdf5 reads the setting back from these flags. + pub fn track_attr_order(&mut self) { + self.attr_order = true; + } + /// Add a message to the header with default flags (0). pub fn add_message(&mut self, msg_type: MessageType, data: Vec) { - self.messages.push((msg_type, data, 0)); + self.messages.push((msg_type, data, 0, 0)); } /// Add a message with specific flags. pub fn add_message_with_flags(&mut self, msg_type: MessageType, data: Vec, flags: u8) { - self.messages.push((msg_type, data, flags)); + self.messages.push((msg_type, data, flags, 0)); + } + + /// Add a message with its creation order, which is written only when + /// attribute creation order is tracked ([`Self::track_attr_order`]). + pub fn add_message_with_order(&mut self, msg_type: MessageType, data: Vec, order: u16) { + self.messages.push((msg_type, data, 0, order)); } /// Serialize the complete v2 object header (OHDR + messages + checksum). @@ -41,10 +64,10 @@ impl ObjectHeaderWriter { /// than [`MAX_MESSAGE_SIZE`] (e.g. an attribute over ~64 KiB, which would /// need dense attribute storage), rather than writing a corrupt header. pub fn serialize(&self) -> Result, FormatError> { - if let Some((msg_type, data, _)) = self + if let Some((msg_type, data, _, _)) = self .messages .iter() - .find(|(_, data, _)| data.len() > MAX_MESSAGE_SIZE) + .find(|(_, data, _, _)| data.len() > MAX_MESSAGE_SIZE) { return Err(FormatError::SerializationError(format!( "{msg_type:?} message is {} bytes; an object header message holds at most \ @@ -52,11 +75,13 @@ impl ObjectHeaderWriter { data.len() ))); } - // Calculate total message bytes: each message has type(1) + size(2) + flags(1) + data + // Calculate total message bytes: each message has type(1) + size(2) + + // flags(1) [+ creation order(2)] + data + let msg_header = if self.attr_order { 6 } else { 4 }; let msg_bytes_total: usize = self .messages .iter() - .map(|(_, data, _)| 4 + data.len()) + .map(|(_, data, _, _)| msg_header + data.len()) .sum(); // Determine chunk size field width based on msg_bytes_total @@ -68,6 +93,12 @@ impl ObjectHeaderWriter { (0x02u8, 4) }; + let flags = if self.attr_order { + flags | OHDR_ATTR_CRT_ORDER_TRACKED | OHDR_ATTR_CRT_ORDER_INDEXED + } else { + flags + }; + let mut buf = Vec::new(); // OHDR signature @@ -85,7 +116,7 @@ impl ObjectHeaderWriter { } // Messages - for (msg_type, data, msg_flags) in &self.messages { + for (msg_type, data, msg_flags, order) in &self.messages { let type_id = msg_type.to_u16(); assert!( type_id <= 255, @@ -94,6 +125,9 @@ impl ObjectHeaderWriter { buf.push(type_id as u8); // type (1 byte in v2) buf.extend_from_slice(&(data.len() as u16).to_le_bytes()); // size (2 bytes) buf.push(*msg_flags); // flags + if self.attr_order { + buf.extend_from_slice(&order.to_le_bytes()); // creation order + } buf.extend_from_slice(data); } @@ -193,6 +227,21 @@ mod tests { assert_eq!(hdr.messages.len(), 0); } + #[test] + fn tracked_attribute_order_is_in_the_flags_and_every_message() { + let mut writer = ObjectHeaderWriter::new(); + writer.track_attr_order(); + writer.add_message(MessageType::Dataspace, vec![1, 2, 3, 4]); + writer.add_message_with_order(MessageType::Attribute, vec![5, 6], 7); + let bytes = writer.serialize().unwrap(); + assert_eq!(bytes[5] & 0x0C, 0x0C); + let hdr = ObjectHeader::parse(&bytes, 0, 8, 8).unwrap(); + assert_eq!(hdr.messages.len(), 2); + assert_eq!(hdr.messages[0].creation_order, Some(0)); + assert_eq!(hdr.messages[1].creation_order, Some(7)); + assert_eq!(hdr.messages[1].data, vec![5, 6]); + } + #[test] fn two_messages_roundtrip() { let mut writer = ObjectHeaderWriter::new(); diff --git a/crates/clawhdf5-format/src/type_builders.rs b/crates/clawhdf5-format/src/type_builders.rs index ce43f64..3019daa 100644 --- a/crates/clawhdf5-format/src/type_builders.rs +++ b/crates/clawhdf5-format/src/type_builders.rs @@ -503,6 +503,9 @@ pub struct DatasetBuilder { /// `data` field is ignored; instead the global heap blob is built from /// these mappings and a VDS layout message is emitted. pub(crate) virtual_sources: Option>, + /// Track (and index) attribute creation order; `None` follows the + /// file's default (`FileWriter::track_order`). + pub(crate) track_order: Option, #[cfg(feature = "provenance")] pub(crate) provenance: Option, } @@ -522,11 +525,22 @@ impl DatasetBuilder { compact: false, alignment: 0, virtual_sources: None, + track_order: None, #[cfg(feature = "provenance")] provenance: None, } } + /// Track the creation order of this dataset's attributes, and index it, + /// as h5py's `create_dataset(..., track_order=True)` does: libhdf5 (and + /// h5py) then list the attributes in the order they were set rather + /// than by name. libhdf5 numbers at most 65 535 attributes on an object + /// that tracks their order; more is an error when the file is written. + pub fn track_order(&mut self, track: bool) -> &mut Self { + self.track_order = Some(track); + self + } + pub fn with_f64_data(&mut self, data: &[f64]) -> &mut Self { self.datatype = Some(make_f64_type()); let mut b = Vec::with_capacity(data.len() * 8); @@ -986,10 +1000,11 @@ impl GroupBuilder { self.attrs.push((name.to_string(), value)); } - /// Track the creation order of this group's links, and index it, as - /// h5py's `track_order=True` does: libhdf5 (and h5py) then list the - /// group's members in the order they were added rather than by name. - /// Applies to links only, not to attributes. + /// Track the creation order of this group's links and attributes, and + /// index it, as h5py's `track_order=True` does: libhdf5 (and h5py) then + /// list the group's members, and its attributes, in the order they were + /// added rather than by name. libhdf5 numbers at most 65 535 attributes + /// on an object that tracks their order. pub fn track_order(&mut self, track: bool) -> &mut Self { self.track_order = Some(track); self diff --git a/crates/clawhdf5-tools/tests/h5rs_interop.rs b/crates/clawhdf5-tools/tests/h5rs_interop.rs index 32f6fb2..f7b0f74 100644 --- a/crates/clawhdf5-tools/tests/h5rs_interop.rs +++ b/crates/clawhdf5-tools/tests/h5rs_interop.rs @@ -940,11 +940,17 @@ fn write_nested_links(dir: &Path) -> Vec { g.create_dataset(&format!("n{i:02}")).with_i32_data(&[i]); } g.add_hard_link("back", "/a/b/c"); + // Attribute creation order tracked too: dense, with a type-9 index. + for i in (0..12).rev() { + g.set_attr(&format!("attr{i:02}"), AttrValue::I64(i)); + } b.add_group(g.finish()); let mut g = b.create_group("compact_ordered"); g.track_order(true); g.create_dataset("z").with_i32_data(&[1]); g.create_dataset("a").with_i32_data(&[2]); + g.set_attr("zz", AttrValue::I64(1)); + g.set_attr("aa", AttrValue::I64(2)); b.add_group(g.finish()); let nested = dir.join("nested.h5"); b.write(&nested).unwrap(); diff --git a/crates/clawhdf5/src/writer.rs b/crates/clawhdf5/src/writer.rs index bfa9393..ea48bc9 100644 --- a/crates/clawhdf5/src/writer.rs +++ b/crates/clawhdf5/src/writer.rs @@ -88,9 +88,11 @@ impl FileBuilder { self } - /// Track link creation order in every group that does not set its own - /// (`GroupBuilder::track_order`), as h5py's `track_order=True`: libhdf5 - /// then lists members in the order they were added. + /// Track the creation order of links and attributes in every group, and + /// of attributes on every dataset, that does not set its own + /// (`GroupBuilder::track_order`, `DatasetBuilder::track_order`), as + /// h5py's `track_order=True`: libhdf5 then lists members and attributes + /// in the order they were added. pub fn track_order(&mut self, track: bool) -> &mut Self { self.writer.track_order(track); self diff --git a/crates/clawhdf5/tests/writer_groups_interop.rs b/crates/clawhdf5/tests/writer_groups_interop.rs index d16567c..777ec07 100644 --- a/crates/clawhdf5/tests/writer_groups_interop.rs +++ b/crates/clawhdf5/tests/writer_groups_interop.rs @@ -531,6 +531,97 @@ fn ten_thousand_links_in_one_group() { ); } +#[test] +fn track_order_lists_attributes_in_creation_order() { + skip_if_no_python!(); + // h5py's track_order=True orders an object's attributes as well as a + // group's links; the writer tracked links only, so h5py listed the + // attributes by name. Now the object header's flags say attribute + // creation order is tracked and indexed, an Attribute Info message + // holds the next order, inline attributes carry theirs, and dense + // storage gets a creation-order index (B-tree type 9). + let dir = tempfile::tempdir().unwrap(); + let small = ["zeta", "alpha", "mid"]; + let mut b = FileBuilder::new(); + b.track_order(true); // the root, and every group and dataset by default + for (i, n) in small.iter().enumerate() { + b.set_attr(n, AttrValue::I64(i as i64)); + } + let mut g = b.create_group("g"); // dense: 30 attributes + for i in (0..30).rev() { + g.set_attr(&format!("a{i:02}"), AttrValue::I64(i)); + } + b.add_group(g.finish()); + let d = b.create_dataset("d"); + d.with_i32_data(&[1]); + for (i, n) in small.iter().enumerate() { + d.set_attr(n, AttrValue::I64(i as i64)); + } + // 20 000 attributes: a one-leaf creation-order index of 20 000 records. + let big = b.create_dataset("big"); + big.with_i32_data(&[2]); + for i in (0..20_000).rev() { + big.set_attr(&format!("b{i:05}"), AttrValue::I64(i)); + } + let plain = b.create_dataset("plain"); + plain.with_i32_data(&[3]).track_order(false); + for (i, n) in small.iter().enumerate() { + plain.set_attr(n, AttrValue::I64(i as i64)); + } + let path = write(&dir, "attr_order.h5", b); + + let out = h5py( + &path, + "def order(o):\n\ + \x20 return o.id.get_create_plist().get_attr_creation_order()\n\ + with h5py.File(path, 'r') as f:\n\ + \x20 big = list(f['big'].attrs)\n\ + \x20 print(json.dumps([list(f.attrs), [int(v) for v in f.attrs.values()],\n\ + \x20 list(f['g'].attrs)[:3], len(f['g'].attrs), list(f['d'].attrs),\n\ + \x20 big[:2], big == ['b%05d' % i for i in range(19999, -1, -1)],\n\ + \x20 int(f['big'].attrs['b00007']), list(f['plain'].attrs),\n\ + \x20 [order(f['/']), order(f['g']), order(f['d']), order(f['plain'])]]))", + ); + assert_eq!( + out, + r#"[["zeta", "alpha", "mid"], [0, 1, 2], ["a29", "a28", "a27"], 30, ["zeta", "alpha", "mid"], ["b19999", "b19998"], true, 7, ["alpha", "mid", "zeta"], [3, 3, 3, 0]]"# + ); + h5dump_ok(&path); + let f = File::open(&path).unwrap(); + assert_eq!(f.dataset("big").unwrap().attrs().unwrap().len(), 20_000); + assert!(matches!( + f.group("g").unwrap().attrs().unwrap()["a07"], + AttrValue::I64(7) + )); + drop(f); + + // libhdf5 continues the numbering: new attributes come last, also when + // it moves the inline ones of `d` to dense storage. + let out = h5py( + &path, + "with h5py.File(path, 'r+') as f:\n\ + \x20 f.attrs['new'] = 9\n\ + \x20 del f.attrs['alpha']\n\ + \x20 f['g'].attrs['new'] = 9\n\ + \x20 del f['g'].attrs['a15']\n\ + \x20 for i in range(8):\n\ + \x20 f['d'].attrs['x%d' % i] = i\n\ + \x20 f['big'].attrs['new'] = 9\n\ + \x20 for i in range(0, 20000, 2):\n\ + \x20 del f['big'].attrs['b%05d' % i]\n\ + with h5py.File(path, 'r') as f:\n\ + \x20 big = list(f['big'].attrs)\n\ + \x20 print(json.dumps([list(f.attrs), list(f['g'].attrs)[-2:], len(f['g'].attrs),\n\ + \x20 list(f['d'].attrs)[:4], len(f['d'].attrs),\n\ + \x20 big == ['b%05d' % i for i in range(19999, -1, -2)] + ['new']]))", + ); + assert_eq!( + out, + r#"[["zeta", "mid", "new"], ["a00", "new"], 30, ["zeta", "alpha", "mid", "x0"], 11, true]"# + ); + h5dump_ok(&path); +} + #[test] fn track_order_lists_members_in_creation_order() { skip_if_no_python!(); diff --git a/docs/known-issues.md b/docs/known-issues.md index 865102b..fca0df6 100644 --- a/docs/known-issues.md +++ b/docs/known-issues.md @@ -264,8 +264,16 @@ fill-value item that did is fixed). external links at any depth and optional creation-order tracking; h5py, h5dump and `h5rs check --data` read them (`crates/clawhdf5/tests/writer_groups_interop.rs`, - `crates/clawhdf5-tools/tests/h5rs_interop.rs`). Still missing: - attribute creation order is not tracked. + `crates/clawhdf5-tools/tests/h5rs_interop.rs`). ~~Still missing: + attribute creation order is not tracked.~~ **Fixed 2026-09-26:** + `track_order` (file default, `GroupBuilder`, and the new + `DatasetBuilder::track_order`) tracks and indexes attribute creation + order as h5py's `track_order=True` does; h5py lists the attributes in + the order they were set, inline and dense (20 000 on one dataset), and + keeps numbering them in "r+" mode (tank, + `cargo test -p clawhdf5 --test writer_groups_interop + track_order_lists_attributes_in_creation_order`). libhdf5 numbers at + most 65 535 attributes on such an object, so more is an error. - ~~A group with more than 65 535 links, or an object with more than 65 535 dense attributes, is an error (the index is one B-tree leaf).~~ **Fixed 2026-09-26:** the dense indexes are v2 B-trees of any depth From 9e9b849dd7cf6bd9de4e7ca72a4abdf42c3f4c59 Mon Sep 17 00:00:00 2001 From: osobh Date: Sat, 26 Sep 2026 10:18:29 -0500 Subject: [PATCH 12/43] read chunked datasets straight into the typed output read_f32/read_f64/read_i32/read_i64/read_u64 of a chunked dataset that stores exactly that type in native byte order now decode every chunk straight into the Vec they return (data_read::read_chunked_native), on File (through its chunk cache), MmapFile and LazyFile. Before, the chunks went into a byte buffer that read_as_* then copied into a second, typed one: two dataset-sized allocations and a full extra copy per read. The output is zeroed pages from the allocator, backed by transparent huge pages when large, like the byte reader's. Other types and byte orders, and datasets with no storage or external data, keep converting through the byte readers; unallocated chunks read as the fill value as before. tests/chunked_read_paths_interop.rs checks every chunked read path (File twice, so cached; from_bytes; MmapFile; LazyFile; small, strided and point selections; with and without the parallel feature) against h5py for 1-8 byte integers and 2-8 byte floats in both byte orders, through deflate, shuffle, Fletcher32, LZF, SZIP and Blosc, with partial edge chunks, sparse datasets with default and non-default fill values, and datasets larger than the chunk cache. A filter this build lacks must be an error (or, when an optional filter declined every chunk, the right data). Co-Authored-By: Claude Opus 5.5 (1M context) --- crates/clawhdf5-format/src/data_read.rs | 114 ++++++ crates/clawhdf5/src/lazy.rs | 42 ++ crates/clawhdf5/src/mmap_file.rs | 42 ++ crates/clawhdf5/src/reader.rs | 43 +++ .../tests/chunked_read_paths_interop.rs | 361 ++++++++++++++++++ 5 files changed, 602 insertions(+) create mode 100644 crates/clawhdf5/tests/chunked_read_paths_interop.rs diff --git a/crates/clawhdf5-format/src/data_read.rs b/crates/clawhdf5-format/src/data_read.rs index 8aec190..fd7054c 100644 --- a/crates/clawhdf5-format/src/data_read.rs +++ b/crates/clawhdf5-format/src/data_read.rs @@ -756,6 +756,120 @@ pub fn read_selection_native( crate::gather::gather::(raw, dims, elem_size, selection).map(Some) } +/// The bytes of a slice of [`NativeElement`]s. +#[cfg(feature = "std")] +fn bytes_of_mut(values: &mut [T]) -> &mut [u8] { + // SAFETY: `T: NativeElement` has no padding and every bit pattern is a + // valid value, so its storage may be viewed, and written, as bytes; the + // byte slice covers exactly the values' storage and borrows it + // exclusively for its lifetime. + unsafe { + core::slice::from_raw_parts_mut( + values.as_mut_ptr().cast::(), + core::mem::size_of_val(values), + ) + } +} + +/// `count` zeroed values of `T`, from zeroed pages where the allocator can +/// (see [`crate::chunked_read::alloc_output`]) and backed by huge pages when +/// large. A size taken from the file surfaces as an error, not an abort. +#[cfg(feature = "std")] +fn alloc_zeroed_values(count: usize) -> Result, FormatError> { + if count == 0 || core::mem::size_of::() == 0 { + return Ok(Vec::new()); + } + let failed = || { + FormatError::Overflow(format!( + "cannot allocate {count} values of {} bytes for dataset output", + core::mem::size_of::() + )) + }; + let layout = core::alloc::Layout::array::(count).map_err(|_| failed())?; + // SAFETY: `layout` has non-zero size (count > 0, T not zero-sized). + let ptr = unsafe { std::alloc::alloc_zeroed(layout) }; + if ptr.is_null() { + return Err(failed()); + } + crate::bulk_alloc::advise_huge_pages(ptr, layout.size()); + // SAFETY: allocated by the global allocator with the layout of + // `[T; count]`, which is what `Vec` with capacity `count` frees; all + // bytes are zero, a valid `T` (`NativeElement`: any bit pattern is). + Ok(unsafe { Vec::from_raw_parts(ptr.cast::(), count, count) }) +} + +/// Read a whole chunked dataset that stores `T` natively +/// ([`NativeElement::is_native`]) straight into a `Vec`: each chunk is +/// decoded and copied to its place in the typed output, with no byte buffer +/// to convert from afterwards. Unallocated chunks read as the dataset's fill +/// value, as [`crate::fill_value::read_full_with_fill`] makes them. +/// +/// `Ok(None)` when this does not apply — the datatype is not `T`'s native +/// representation (another type, another byte order: the caller converts +/// through the byte readers and the `read_as_*` functions), the layout is +/// not chunked, no storage is allocated, or the data lives in external +/// files. `cache` is the file's chunk cache, used as +/// [`crate::chunked_read::read_chunked_data_cached`] uses it. +#[cfg(feature = "std")] +#[allow(clippy::too_many_arguments)] +pub fn read_chunked_native( + messages: &[crate::object_header::HeaderMessage], + file_data: &[u8], + layout: &DataLayout, + dataspace: &Dataspace, + datatype: &Datatype, + pipeline: Option<&FilterPipeline>, + offset_size: u8, + length_size: u8, + cache: Option<&ChunkCache>, +) -> Result>, FormatError> { + use crate::fill_value; + use crate::message_type::MessageType; + + if !T::is_native(datatype) + || !matches!(layout, DataLayout::Chunked { .. }) + || !fill_value::has_storage(layout) + || messages + .iter() + .any(|m| m.msg_type == MessageType::ExternalDataFiles) + { + return Ok(None); + } + let size = core::mem::size_of::(); + let mut values = crate::chunked_read::read_chunked_full( + file_data, + layout, + dataspace, + datatype, + pipeline, + offset_size, + length_size, + cache, + |total_bytes| { + if !total_bytes.is_multiple_of(size) { + return Err(FormatError::DataSizeMismatch { + expected: total_bytes.next_multiple_of(size), + actual: total_bytes, + }); + } + alloc_zeroed_values::(total_bytes / size) + }, + |values| bytes_of_mut(values), + )?; + let fill = fill_value::dataset_fill_value_in(file_data, messages, offset_size, length_size)?; + fill_value::apply_to_unallocated_chunks( + bytes_of_mut(&mut values), + file_data, + layout, + dataspace, + size, + fill.as_deref(), + offset_size, + length_size, + )?; + Ok(Some(values)) +} + /// Convert raw bytes to `f64` values. pub fn read_as_f64(raw: &[u8], datatype: &Datatype) -> Result, FormatError> { // Array datatypes read as a flat sequence of their base elements, and diff --git a/crates/clawhdf5/src/lazy.rs b/crates/clawhdf5/src/lazy.rs index a33b9bb..c3b779e 100644 --- a/crates/clawhdf5/src/lazy.rs +++ b/crates/clawhdf5/src/lazy.rs @@ -349,6 +349,9 @@ impl<'f, R: HDF5Read> LazyDataset<'f, R> { /// Read all data as `f64` values. pub fn read_f64(&self) -> Result, Error> { + if let Some(values) = self.read_chunked_native::()? { + return Ok(values); + } let raw = self.read_raw()?; let dt = self.datatype()?; Ok(data_read::read_as_f64(&raw, &dt)?) @@ -396,6 +399,9 @@ impl<'f, R: HDF5Read> LazyDataset<'f, R> { /// Read all data as `f32` values. pub fn read_f32(&self) -> Result, Error> { + if let Some(values) = self.read_chunked_native::()? { + return Ok(values); + } let raw = self.read_raw()?; let dt = self.datatype()?; Ok(data_read::read_as_f32(&raw, &dt)?) @@ -403,6 +409,9 @@ impl<'f, R: HDF5Read> LazyDataset<'f, R> { /// Read all data as `i32` values. pub fn read_i32(&self) -> Result, Error> { + if let Some(values) = self.read_chunked_native::()? { + return Ok(values); + } let raw = self.read_raw()?; let dt = self.datatype()?; Ok(data_read::read_as_i32(&raw, &dt)?) @@ -410,6 +419,9 @@ impl<'f, R: HDF5Read> LazyDataset<'f, R> { /// Read all data as `i64` values. pub fn read_i64(&self) -> Result, Error> { + if let Some(values) = self.read_chunked_native::()? { + return Ok(values); + } let raw = self.read_raw()?; let dt = self.datatype()?; Ok(data_read::read_as_i64(&raw, &dt)?) @@ -417,6 +429,9 @@ impl<'f, R: HDF5Read> LazyDataset<'f, R> { /// Read all data as `u64` values. pub fn read_u64(&self) -> Result, Error> { + if let Some(values) = self.read_chunked_native::()? { + return Ok(values); + } let raw = self.read_raw()?; let dt = self.datatype()?; Ok(data_read::read_as_u64(&raw, &dt)?) @@ -550,6 +565,33 @@ impl<'f, R: HDF5Read> LazyDataset<'f, R> { .transpose() } + /// A chunked dataset that stores `T` natively, decoded straight into a + /// `Vec` (no byte buffer to convert); `None` for any other dataset + /// (see [`data_read::read_chunked_native`]). + fn read_chunked_native(&self) -> Result>, Error> { + let dl = self.data_layout()?; + if !matches!(dl, DataLayout::Chunked { .. }) { + return Ok(None); + } + let dt = self.datatype()?; + if !T::is_native(&dt) { + return Ok(None); + } + let ds = self.dataspace()?; + let pipeline = self.filter_pipeline()?; + Ok(data_read::read_chunked_native::( + &self.header.messages, + self.file.hdf5_bytes(), + &dl, + &ds, + &dt, + pipeline.as_ref(), + self.file.offset_size(), + self.file.length_size(), + None, + )?) + } + fn read_raw(&self) -> Result, Error> { let dt = self.datatype()?; let ds = self.dataspace()?; diff --git a/crates/clawhdf5/src/mmap_file.rs b/crates/clawhdf5/src/mmap_file.rs index 76d9ea1..7813b9c 100644 --- a/crates/clawhdf5/src/mmap_file.rs +++ b/crates/clawhdf5/src/mmap_file.rs @@ -276,6 +276,9 @@ impl<'f> MmapDataset<'f> { /// Read all data as `f64` values. pub fn read_f64(&self) -> Result, Error> { + if let Some(values) = self.read_chunked_native::()? { + return Ok(values); + } let raw = self.read_raw()?; let dt = self.datatype()?; Ok(data_read::read_as_f64(&raw, &dt)?) @@ -310,6 +313,9 @@ impl<'f> MmapDataset<'f> { /// Read all data as `f32` values. pub fn read_f32(&self) -> Result, Error> { + if let Some(values) = self.read_chunked_native::()? { + return Ok(values); + } let raw = self.read_raw()?; let dt = self.datatype()?; Ok(data_read::read_as_f32(&raw, &dt)?) @@ -317,6 +323,9 @@ impl<'f> MmapDataset<'f> { /// Read all data as `i32` values. pub fn read_i32(&self) -> Result, Error> { + if let Some(values) = self.read_chunked_native::()? { + return Ok(values); + } let raw = self.read_raw()?; let dt = self.datatype()?; Ok(data_read::read_as_i32(&raw, &dt)?) @@ -324,6 +333,9 @@ impl<'f> MmapDataset<'f> { /// Read all data as `i64` values. pub fn read_i64(&self) -> Result, Error> { + if let Some(values) = self.read_chunked_native::()? { + return Ok(values); + } let raw = self.read_raw()?; let dt = self.datatype()?; Ok(data_read::read_as_i64(&raw, &dt)?) @@ -331,6 +343,9 @@ impl<'f> MmapDataset<'f> { /// Read all data as `u64` values. pub fn read_u64(&self) -> Result, Error> { + if let Some(values) = self.read_chunked_native::()? { + return Ok(values); + } let raw = self.read_raw()?; let dt = self.datatype()?; Ok(data_read::read_as_u64(&raw, &dt)?) @@ -496,6 +511,33 @@ impl<'f> MmapDataset<'f> { .transpose() } + /// A chunked dataset that stores `T` natively, decoded straight into a + /// `Vec` (no byte buffer to convert); `None` for any other dataset + /// (see [`data_read::read_chunked_native`]). + fn read_chunked_native(&self) -> Result>, Error> { + let dl = self.data_layout()?; + if !matches!(dl, DataLayout::Chunked { .. }) { + return Ok(None); + } + let dt = self.datatype()?; + if !T::is_native(&dt) { + return Ok(None); + } + let ds = self.dataspace()?; + let pipeline = self.filter_pipeline()?; + Ok(data_read::read_chunked_native::( + &self.header.messages, + self.file.hdf5_bytes(), + &dl, + &ds, + &dt, + pipeline.as_ref(), + self.file.offset_size(), + self.file.length_size(), + None, + )?) + } + fn read_raw(&self) -> Result, Error> { let dt = self.datatype()?; let ds = self.dataspace()?; diff --git a/crates/clawhdf5/src/reader.rs b/crates/clawhdf5/src/reader.rs index 90faf11..446a5a7 100644 --- a/crates/clawhdf5/src/reader.rs +++ b/crates/clawhdf5/src/reader.rs @@ -507,6 +507,9 @@ impl<'f> Dataset<'f> { if let Ok(Some(bytes)) = self.read_raw_ref() { return Ok(data_read::read_as_f64(bytes, &dt)?); } + if let Some(values) = self.read_chunked_native::()? { + return Ok(values); + } let raw = self.read_raw()?; Ok(data_read::read_as_f64(&raw, &dt)?) } @@ -524,6 +527,9 @@ impl<'f> Dataset<'f> { if let Ok(Some(bytes)) = self.read_raw_ref() { return Ok(data_read::read_as_f32(bytes, &dt)?); } + if let Some(values) = self.read_chunked_native::()? { + return Ok(values); + } let raw = self.read_raw()?; Ok(data_read::read_as_f32(&raw, &dt)?) } @@ -536,6 +542,9 @@ impl<'f> Dataset<'f> { if let Ok(Some(bytes)) = self.read_raw_ref() { return Ok(data_read::read_as_i32(bytes, &dt)?); } + if let Some(values) = self.read_chunked_native::()? { + return Ok(values); + } let raw = self.read_raw()?; Ok(data_read::read_as_i32(&raw, &dt)?) } @@ -548,6 +557,9 @@ impl<'f> Dataset<'f> { if let Ok(Some(bytes)) = self.read_raw_ref() { return Ok(data_read::read_as_i64(bytes, &dt)?); } + if let Some(values) = self.read_chunked_native::()? { + return Ok(values); + } let raw = self.read_raw()?; Ok(data_read::read_as_i64(&raw, &dt)?) } @@ -560,6 +572,9 @@ impl<'f> Dataset<'f> { if let Ok(Some(bytes)) = self.read_raw_ref() { return Ok(data_read::read_as_u64(bytes, &dt)?); } + if let Some(values) = self.read_chunked_native::()? { + return Ok(values); + } let raw = self.read_raw()?; Ok(data_read::read_as_u64(&raw, &dt)?) } @@ -1050,6 +1065,34 @@ impl<'f> Dataset<'f> { .transpose() } + /// A chunked dataset that stores `T` natively, decoded straight into a + /// `Vec` through the file's chunk cache (no byte buffer to convert); + /// `None` for any other dataset (see + /// [`data_read::read_chunked_native`]). + fn read_chunked_native(&self) -> Result>, Error> { + let dl = self.data_layout()?; + if !matches!(dl, DataLayout::Chunked { .. }) { + return Ok(None); + } + let dt = self.datatype()?; + if !T::is_native(&dt) { + return Ok(None); + } + let ds = self.dataspace()?; + let pipeline = self.filter_pipeline()?; + Ok(data_read::read_chunked_native::( + &self.header.messages, + self.file.data.as_bytes(), + &dl, + &ds, + &dt, + pipeline.as_ref(), + self.file.offset_size(), + self.file.length_size(), + Some(&self.file.chunk_cache), + )?) + } + fn read_raw(&self) -> Result, Error> { let dt = self.datatype()?; let ds = self.dataspace()?; diff --git a/crates/clawhdf5/tests/chunked_read_paths_interop.rs b/crates/clawhdf5/tests/chunked_read_paths_interop.rs new file mode 100644 index 0000000..2865a22 --- /dev/null +++ b/crates/clawhdf5/tests/chunked_read_paths_interop.rs @@ -0,0 +1,361 @@ +//! Every full and selection read path of chunked datasets against h5py. +//! +//! h5py (libhdf5) writes chunked datasets of every numeric type the typed +//! readers cover, in both byte orders, through deflate, shuffle, +//! Fletcher32, LZF, SZIP and Blosc, in 1-3 dimensional shapes whose chunks +//! do not divide them (partial edge chunks), plus sparse datasets whose +//! unwritten chunks read as a fill value. Next to each it stores the values +//! as contiguous `f64`, and checks that h5py reads the chunked dataset back +//! as those values. +//! +//! clawhdf5 must read every chunked dataset as those values through every +//! reader: `File` (the chunk-cached reader, twice so the second read can hit +//! the cache, and the typed readers that decode straight into their output), +//! `File::from_bytes`, `MmapFile`, `LazyFile`, and the selection readers +//! (a small hyperslab, a strided one covering most of the dataset, points). +//! With `--features parallel` the same reads decode chunks on several +//! threads. A filter this build does not include must be an error, never +//! data. +//! +//! Skipped when python3 with h5py is unavailable, unless +//! `CLAWHDF5_REQUIRE_INTEROP=1`. + +use std::process::Command; + +use clawhdf5::{File, LazyFile, MmapFile, Selection}; + +fn python() -> String { + std::env::var("CLAWHDF5_PYTHON").unwrap_or_else(|_| "python3".to_string()) +} + +fn interop_required() -> bool { + std::env::var("CLAWHDF5_REQUIRE_INTEROP").is_ok_and(|v| v == "1") +} + +/// Whether the interop test can run; panics instead of skipping when +/// `CLAWHDF5_REQUIRE_INTEROP=1`. +fn have_python() -> bool { + let ok = Command::new(python()) + .args(["-c", "import h5py, numpy"]) + .output() + .map(|o| o.status.success()) + .unwrap_or(false); + if ok { + return true; + } + assert!( + !interop_required(), + "CLAWHDF5_REQUIRE_INTEROP=1 but python3 with h5py is not available" + ); + eprintln!("SKIP: python3 with h5py not available"); + false +} + +/// Writes `c/` (chunked) and `e/` (the expected values, `f64`, +/// contiguous) for every case; prints one line per case: +/// `name filter_ids(comma-separated or -)`. +const GENERATE: &str = r#" +import sys +import numpy as np, h5py +try: + import hdf5plugin +except ImportError: + hdf5plugin = None +path = sys.argv[1] + +dtypes = [] +for code in ['i1', 'u1', 'i2', 'u2', 'i4', 'u4', 'i8', 'u8', 'f2', 'f4', 'f8']: + orders = ['|'] if code[1] == '1' else ['<', '>'] + dtypes += [np.dtype(o + code) for o in orders] + +filters = { + 'none': {}, + 'gzip': dict(compression='gzip', compression_opts=4), + 'shuffle_gzip': dict(shuffle=True, compression='gzip', compression_opts=1), + # h5py puts the checksum last (applied after deflate). + 'shuffle_gzip_fletcher': dict(shuffle=True, compression='gzip', fletcher32=True), + 'fletcher': dict(fletcher32=True), + 'shuffle_lzf': dict(shuffle=True, compression='lzf'), +} +if h5py.h5z.filter_avail(h5py.h5z.FILTER_SZIP): + filters['szip'] = dict(compression='szip', compression_opts=('nn', 8)) +if hdf5plugin is not None: + filters['blosc'] = dict(**hdf5plugin.Blosc(cname='lz4', clevel=5, + shuffle=hdf5plugin.Blosc.SHUFFLE)) + +shapes = [((37, 23), (8, 5)), ((101,), (16,)), ((9, 10, 11), (4, 3, 5))] + +def values(n, dt): + i = np.arange(n, dtype=np.int64) + if dt.kind == 'f': + v = ((i * 7 + 3) % 1000) / 8.0 - 60.0 + elif dt.kind == 'i': + v = (i * 7 + 3) % 200 - 100 + else: + v = (i * 7 + 3) % 250 + return v.astype(dt) + +def filter_ids(dset): + plist = dset.id.get_create_plist() + ids = [str(plist.get_filter(k)[0]) for k in range(plist.get_nfilters())] + return ','.join(ids) or '-' + +with h5py.File(path, 'w') as f: + n = 0 + for dt in dtypes: + for fname, fopts in filters.items(): + for s, (shape, chunks) in enumerate(shapes): + name = f"{dt.str.replace('|', 'x').replace('<', 'le').replace('>', 'be')}_{fname}_{s}" + data = values(int(np.prod(shape)), dt).reshape(shape) + try: + d = f.create_dataset('c/' + name, data=data, chunks=chunks, **fopts) + except (ValueError, TypeError) as e: + continue # a filter that refuses this type + f.create_dataset('e/' + name, data=data.astype('i4'): + name = f"{dt.str.replace('<', 'le').replace('>', 'be')}_large" + shape = (2600, 2048) + data = values(shape[0] * shape[1], dt).reshape(shape) + d = f.create_dataset('c/' + name, data=data, chunks=(256, 256), + shuffle=True, compression='gzip', compression_opts=1) + f.create_dataset('e/' + name, data=data.astype('', 'be')}_sparse_{fname}_{fill}" + shape, chunks = (37, 23), (8, 5) + kw = dict(filters[fname]) + if fill is not None: + kw['fillvalue'] = np.array(fill, dtype=dt) + d = f.create_dataset('c/' + name, shape=shape, dtype=dt, chunks=chunks, **kw) + full = values(37 * 23, dt).reshape(shape) + d[3:17, 4:12] = full[3:17, 4:12] + d[30:, 20:] = full[30:, 20:] + expect = d[()] + want = np.full(shape, 0 if fill is None else fill, dtype=dt) + want[3:17, 4:12] = full[3:17, 4:12] + want[30:, 20:] = full[30:, 20:] + assert np.array_equal(expect, want), name + f.create_dataset('e/' + name, data=want.astype(' bool { + ids == "-" + || ids.split(',').all(|id| { + clawhdf5_format::filter_registry::is_filter_available(id.parse().expect("filter id")) + }) +} + +/// The expected values converted as libhdf5 converts them for each typed +/// reader (every case's values are exact in `f32` and within `i32`). +struct Expected { + f64s: Vec, +} + +impl Expected { + fn f32s(&self) -> Vec { + self.f64s.iter().map(|&v| v as f32).collect() + } + /// Truncation toward zero; negative values read as unsigned are 0. + fn i32s(&self) -> Vec { + self.f64s.iter().map(|&v| v as i32).collect() + } + fn i64s(&self) -> Vec { + self.f64s.iter().map(|&v| v as i64).collect() + } + fn u64s(&self) -> Vec { + self.f64s.iter().map(|&v| v as u64).collect() + } + fn select(&self, idx: &[usize]) -> Expected { + Expected { + f64s: idx.iter().map(|&i| self.f64s[i]).collect(), + } + } +} + +/// The typed readers' results for one dataset, compared with `want`. +macro_rules! check_typed { + ($ds:expr, $want:expr, $name:expr, $path:expr) => {{ + let (ds, want, name, path) = (&$ds, &$want, $name, $path); + assert_eq!(ds.read_f64().unwrap(), want.f64s, "{name} {path} f64"); + assert_eq!(ds.read_f32().unwrap(), want.f32s(), "{name} {path} f32"); + assert_eq!(ds.read_i32().unwrap(), want.i32s(), "{name} {path} i32"); + assert_eq!(ds.read_i64().unwrap(), want.i64s(), "{name} {path} i64"); + assert_eq!(ds.read_u64().unwrap(), want.u64s(), "{name} {path} u64"); + }}; +} + +/// Row-major indices of the elements `sel` picks from a dataset of `dims`. +fn selected(sel: &Selection, dims: &[u64]) -> Vec { + let strides: Vec = (0..dims.len()) + .map(|d| dims[d + 1..].iter().product::() as usize) + .collect(); + match sel { + Selection::Hyperslab { + start, + stride, + count, + block, + } => { + // Per dimension, the selected coordinates in order. + let axes: Vec> = (0..dims.len()) + .map(|d| { + (0..count[d]) + .flat_map(|c| (0..block[d]).map(move |b| start[d] + c * stride[d] + b)) + .collect() + }) + .collect(); + let mut out = vec![0usize]; + for (d, axis) in axes.iter().enumerate() { + let stride = strides[d]; + out = out + .iter() + .flat_map(|&base| axis.iter().map(move |&x| base + x as usize * stride)) + .collect(); + } + out + } + Selection::Points(points) => points + .iter() + .map(|p| p.iter().zip(&strides).map(|(&x, &s)| x as usize * s).sum()) + .collect(), + _ => unreachable!(), + } +} + +/// A small box (read through the partial-read path), a strided selection +/// covering most of the dataset (read in full, then selected), and points. +fn selections(dims: &[u64]) -> Vec { + let rank = dims.len(); + let small = Selection::Hyperslab { + start: dims.iter().map(|&d| d / 3).collect(), + stride: vec![1; rank], + count: dims.iter().map(|&d| (d / 4).max(1)).collect(), + block: vec![1; rank], + }; + let strided = Selection::Hyperslab { + start: vec![0; rank], + stride: vec![2; rank], + count: dims.iter().map(|&d| d.div_ceil(2)).collect(), + block: vec![1; rank], + }; + let points = Selection::Points(vec![ + vec![0; rank], + dims.iter().map(|&d| d - 1).collect(), + dims.iter().map(|&d| d / 2).collect(), + dims.iter().map(|&d| (d * 2) / 3).collect(), + ]); + vec![small, strided, points] +} + +#[test] +fn chunked_reads_match_h5py_on_every_path() { + if !have_python() { + return; + } + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("chunked_paths.h5"); + let out = Command::new(python()) + .arg("-c") + .arg(GENERATE) + .arg(&path) + .output() + .expect("run python"); + assert!( + out.status.success(), + "generator failed: {}", + String::from_utf8_lossy(&out.stderr) + ); + let cases: Vec<(String, String)> = String::from_utf8(out.stdout) + .unwrap() + .lines() + .map(|l| { + let (name, ids) = l.split_once(' ').unwrap(); + (name.to_string(), ids.to_string()) + }) + .collect(); + assert!(cases.len() > 300, "only {} cases", cases.len()); + if interop_required() { + // The generator must have covered the plugin filters too. + for f in ["_szip_", "_blosc_", "_shuffle_lzf_", "_sparse_"] { + assert!(cases.iter().any(|(n, _)| n.contains(f)), "no {f} case"); + } + } + + let file = File::open(&path).unwrap(); + let owned = File::from_bytes(std::fs::read(&path).unwrap()).unwrap(); + let mmap = MmapFile::open(&path).unwrap(); + let lazy = LazyFile::open_mmap(&path).unwrap(); + let (mut checked, mut refused) = (0, 0); + for (name, ids) in &cases { + let chunked = format!("c/{name}"); + let want = Expected { + f64s: file + .dataset(&format!("e/{name}")) + .unwrap() + .read_f64() + .unwrap(), + }; + let ds = file.dataset(&chunked).unwrap(); + if !filters_available(ids) { + // Unsupported filter: an error from every reader, never wrong + // data. (An optional filter, as Blosc is, may have declined every + // chunk — they are then stored as-is and read fine.) + let results = [ + ds.read_f64(), + owned.dataset(&chunked).unwrap().read_f64(), + mmap.dataset(&chunked).unwrap().read_f64(), + lazy.dataset(&chunked).unwrap().read_f64(), + ]; + let errors = results.iter().filter(|r| r.is_err()).count(); + assert!(errors == 0 || errors == results.len(), "{name}"); + for values in results.into_iter().flatten() { + assert_eq!(values, want.f64s, "{name}"); + } + if errors > 0 { + assert!(ds.read_f32().is_err(), "{name}"); + refused += 1; + continue; + } + } + // Twice: the second read of a small dataset comes from the cache. + check_typed!(ds, want, name, "File"); + check_typed!(ds, want, name, "File (cached)"); + check_typed!(owned.dataset(&chunked).unwrap(), want, name, "from_bytes"); + check_typed!(mmap.dataset(&chunked).unwrap(), want, name, "MmapFile"); + check_typed!(lazy.dataset(&chunked).unwrap(), want, name, "LazyFile"); + + let dims = ds.shape().unwrap(); + for sel in selections(&dims) { + let want = want.select(&selected(&sel, &dims)); + assert_eq!( + ds.read_f64_selection(&sel).unwrap(), + want.f64s, + "{name} {sel:?}" + ); + assert_eq!( + ds.read_f32_selection(&sel).unwrap(), + want.f32s(), + "{name} {sel:?}" + ); + assert_eq!( + ds.read_i64_selection(&sel).unwrap(), + want.i64s(), + "{name} {sel:?}" + ); + } + checked += 1; + } + eprintln!("{checked} datasets read on every path, {refused} refused (filter not built in)"); + assert!(checked > 300); +} From 9e608b975c7deb219dcdb8f34307fa29683088b4 Mon Sep 17 00:00:00 2001 From: osobh Date: Sat, 26 Sep 2026 10:20:07 -0500 Subject: [PATCH 13/43] format: decode selected chunks into reusable buffers A selection read (a hyperslab or points covering at most half the dataset) decoded each chunk it overlaps into fresh buffers, one per filter stage; it now uses the thread's chunk-decoding scratch like the full readers. Covered by tests/chunked_read_paths_interop.rs (small hyperslabs and points over every filter and type) and the partial-read equivalence tests. Co-Authored-By: Claude Opus 5.5 (1M context) --- crates/clawhdf5-format/src/chunked_read.rs | 2 +- crates/clawhdf5-format/src/partial_read.rs | 99 +++++++++++----------- 2 files changed, 52 insertions(+), 49 deletions(-) diff --git a/crates/clawhdf5-format/src/chunked_read.rs b/crates/clawhdf5-format/src/chunked_read.rs index 7d3efd0..712aab8 100644 --- a/crates/clawhdf5-format/src/chunked_read.rs +++ b/crates/clawhdf5-format/src/chunked_read.rs @@ -31,7 +31,7 @@ use crate::lane_partition::PartitionStats; /// [`DecodeScratch`]), kept between reads so decoding reuses memory instead /// of faulting in fresh pages for every chunk. A re-entrant call (a /// registered filter codec that itself reads a file) gets a fresh scratch. -fn with_scratch(f: impl FnOnce(&mut DecodeScratch) -> R) -> R { +pub(crate) fn with_scratch(f: impl FnOnce(&mut DecodeScratch) -> R) -> R { #[cfg(feature = "std")] { use std::cell::RefCell; diff --git a/crates/clawhdf5-format/src/partial_read.rs b/crates/clawhdf5-format/src/partial_read.rs index 9865c46..0151255 100644 --- a/crates/clawhdf5-format/src/partial_read.rs +++ b/crates/clawhdf5-format/src/partial_read.rs @@ -24,7 +24,7 @@ use crate::data_read::extract_selection_from_buffer; use crate::dataspace::Dataspace; use crate::error::FormatError; use crate::filter_pipeline::FilterPipeline; -use crate::filters::{all_filters_skipped, decompress_chunk_exact}; +use crate::filters::{all_filters_skipped, decompress_chunk_exact_with}; use crate::selection::Selection; /// The smallest axis-aligned box containing every selected element, as @@ -305,54 +305,57 @@ pub fn read_selection( let rank = dims.len(); let chunk_shape: Vec = chunk_dims.iter().map(|&d| d as u64).collect(); let chunk_bytes = crate::chunked_read::checked_chunk_byte_len(&chunk_dims, elem_size)?; - for chunk in &chunks { - if chunk.offsets.len() < rank || chunk.address == u64::MAX { - continue; - } - let origin = &chunk.offsets[..rank]; - let overlaps = (0..rank).all(|d| { - origin[d] < box_start[d] + box_extent[d] - && origin[d].saturating_add(chunk_shape[d]) > box_start[d] - }); - if !overlaps { - continue; - } - let at = usize::try_from(chunk.address) - .map_err(|_| FormatError::Overflow("chunk address exceeds usize".into()))?; - let raw = at - .checked_add(chunk.chunk_size as usize) - .and_then(|end| file_data.get(at..end)) - .ok_or(FormatError::UnexpectedEof { - expected: at.saturating_add(chunk.chunk_size as usize), - available: file_data.len(), - })?; - // Mirrors the full-read path: filter-mask bit i set means - // filter i was not applied to this chunk. - let decoded; - let data: &[u8] = match pipeline { - Some(pl) if !all_filters_skipped(pl, chunk.filter_mask) => { - decoded = decompress_chunk_exact( - raw, - pl, - chunk_bytes, - elem_size as u32, - chunk.filter_mask, - &chunk.offsets[..rank], - )?; - &decoded + // Chunks are decoded into this thread's reusable buffers. + crate::chunked_read::with_scratch(|scratch| -> Result<(), FormatError> { + for chunk in &chunks { + if chunk.offsets.len() < rank || chunk.address == u64::MAX { + continue; } - _ => raw, - }; - copy_overlap( - data, - origin, - &chunk_shape, - &mut boxed, - &box_start, - &box_extent, - elem_size, - ); - } + let origin = &chunk.offsets[..rank]; + let overlaps = (0..rank).all(|d| { + origin[d] < box_start[d] + box_extent[d] + && origin[d].saturating_add(chunk_shape[d]) > box_start[d] + }); + if !overlaps { + continue; + } + let at = usize::try_from(chunk.address) + .map_err(|_| FormatError::Overflow("chunk address exceeds usize".into()))?; + let raw = at + .checked_add(chunk.chunk_size as usize) + .and_then(|end| file_data.get(at..end)) + .ok_or(FormatError::UnexpectedEof { + expected: at.saturating_add(chunk.chunk_size as usize), + available: file_data.len(), + })?; + // Mirrors the full-read path: filter-mask bit i set means + // filter i was not applied to this chunk. + let data: &[u8] = match pipeline { + Some(pl) if !all_filters_skipped(pl, chunk.filter_mask) => { + decompress_chunk_exact_with( + raw, + pl, + chunk_bytes, + elem_size as u32, + chunk.filter_mask, + &chunk.offsets[..rank], + scratch, + )? + } + _ => raw, + }; + copy_overlap( + data, + origin, + &chunk_shape, + &mut boxed, + &box_start, + &box_extent, + elem_size, + ); + } + Ok(()) + })?; } _ => return Ok(None), } From b22b15f00a868e3e4fa48c70ebc636eaac3944b5 Mon Sep 17 00:00:00 2001 From: osobh Date: Sat, 26 Sep 2026 10:22:04 -0500 Subject: [PATCH 14/43] fix: refuse at open the dataset storage libhdf5 refuses at open libhdf5 checks a dataset's storage when it opens the dataset (H5D__contig_check, H5D__compact_init): the element count times the element size must not overflow, contiguous storage must end inside the file, compact data must be the dataset's size. File::dataset opened cve-2024-32624's /Dset_OBJREF (2^62 + 2 references of 8 bytes) and reported its shape; only reading failed. data_read::check_dataset_storage makes those checks (new FormatError::InvalidDatasetStorage), and File, MmapFile and LazyFile run it whenever they open a dataset (by path, by address, from a group), as does the conformance probe. As before, a datatype, dataspace or layout that does not decode is left for the read to report, so such a dataset still opens and its attributes still read. An empty contiguous dataset at a defined address, which libhdf5 refuses, is still accepted: clawhdf5 up to v2.7.0 wrote them. Co-Authored-By: Claude Opus 5.5 (1M context) --- conformance/probe/src/main.rs | 14 +-- crates/clawhdf5-format/src/data_read.rs | 93 +++++++++++++++++++ crates/clawhdf5-format/src/error.rs | 8 ++ crates/clawhdf5/src/lazy.rs | 28 +++++- crates/clawhdf5/src/mmap_file.rs | 28 +++++- crates/clawhdf5/src/reader.rs | 33 +++++-- .../tests/header_validation_interop.rs | 49 ++++++++++ 7 files changed, 233 insertions(+), 20 deletions(-) diff --git a/conformance/probe/src/main.rs b/conformance/probe/src/main.rs index aa3d6a1..844f343 100644 --- a/conformance/probe/src/main.rs +++ b/conformance/probe/src/main.rs @@ -308,18 +308,20 @@ impl<'a> Ctx<'a> { ) .map_err(e)?; } - let (shape, n) = Self::shape(&ds); - rec.insert("shape".into(), shape); - if n.saturating_mul(dt.type_size() as u64) > MAX_BYTES { - rec.insert("skipped".into(), Value::String("too large".into())); - return Ok(()); - } let lm = h .messages .iter() .find(|m| m.msg_type == MessageType::DataLayout) .ok_or("MissingMessage(DataLayout)")?; let dl = DataLayout::parse(&lm.data, self.os, self.ls).map_err(e)?; + // What libhdf5 checks when it opens the dataset (as File::dataset). + data_read::check_dataset_storage(&dl, &ds, &dt, self.data.len() as u64).map_err(e)?; + let (shape, n) = Self::shape(&ds); + rec.insert("shape".into(), shape); + if n.saturating_mul(dt.type_size() as u64) > MAX_BYTES { + rec.insert("skipped".into(), Value::String("too large".into())); + return Ok(()); + } rec.insert( "layout".into(), Value::String( diff --git a/crates/clawhdf5-format/src/data_read.rs b/crates/clawhdf5-format/src/data_read.rs index d1766a9..591142c 100644 --- a/crates/clawhdf5-format/src/data_read.rs +++ b/crates/clawhdf5-format/src/data_read.rs @@ -32,6 +32,64 @@ fn ensure_len(data: &[u8], offset: usize, needed: usize) -> Result<(), FormatErr Ok(()) } +/// The storage checks libhdf5 makes when it opens a dataset, before any +/// data is read (`H5D__contig_check`, `H5D__compact_init`), so a dataset +/// they refuse fails to open, as in libhdf5, instead of opening and +/// reporting a shape nothing can be read from: +/// +/// - the element count times the element size must not overflow 64 bits +/// ("size of dataset's storage overflowed" — `cve-2024-32624` +/// `/Dset_OBJREF`, 2^62 references of 8 bytes); +/// - contiguous storage at a defined address must end within the file's +/// `file_len` bytes (the HDF5 data up to the end of file the superblock +/// records); +/// - compact data must be exactly the dataset's size. +/// +/// Deliberately not refused, unlike libhdf5: an empty contiguous dataset at +/// a defined address (libhdf5's overflow test `addr + 0 <= addr` refuses +/// it), which clawhdf5 up to v2.7.0 wrote. Chunked and virtual layouts are +/// checked when their data is read. +pub fn check_dataset_storage( + layout: &DataLayout, + dataspace: &Dataspace, + datatype: &Datatype, + file_len: u64, +) -> Result<(), FormatError> { + if !matches!( + layout, + DataLayout::Contiguous { .. } | DataLayout::Compact { .. } + ) { + return Ok(()); + } + const OVERFLOWED: &str = "size of dataset's storage overflowed"; + let n = dataspace + .checked_num_elements() + .map_err(|_| FormatError::InvalidDatasetStorage(OVERFLOWED))?; + let data_size = n + .checked_mul(u64::from(datatype.type_size())) + .ok_or(FormatError::InvalidDatasetStorage(OVERFLOWED))?; + match layout { + DataLayout::Contiguous { + address: Some(address), + .. + } if address + .checked_add(data_size) + .is_none_or(|end| end > file_len) => + { + Err(FormatError::InvalidDatasetStorage( + "invalid dataset size, likely file corruption", + )) + } + DataLayout::Compact { data } if data.len() as u64 != data_size => { + Err(FormatError::InvalidDatasetStorage( + "bad value from dataset header - size of compact dataset's data buffer \ + doesn't match size of dataset data", + )) + } + _ => Ok(()), + } +} + /// How many bytes to read from a contiguous dataset's storage of /// `storage_size` bytes (the layout message's size) holding `needed` bytes /// of elements. libhdf5 reads the elements' bytes from the start of the @@ -2666,6 +2724,41 @@ mod tests { )); } + /// `H5D__contig_check` / `H5D__compact_init`, run when a dataset opens. + #[test] + fn dataset_storage_checks_at_open() { + let dt = make_f64_le_type(); + let contiguous = |address| DataLayout::Contiguous { address, size: 0 }; + // cve-2024-32624 `/Dset_OBJREF`: 2^62 + 2 elements of 8 bytes. + let huge = make_simple_dataspace(&[(1 << 62) + 2]); + assert_eq!( + check_dataset_storage(&contiguous(None), &huge, &dt, 1 << 20), + Err(FormatError::InvalidDatasetStorage( + "size of dataset's storage overflowed" + )) + ); + let ds = make_simple_dataspace(&[4]); + assert!(check_dataset_storage(&contiguous(Some(100)), &ds, &dt, 132).is_ok()); + assert!(matches!( + check_dataset_storage(&contiguous(Some(100)), &ds, &dt, 131), + Err(FormatError::InvalidDatasetStorage(_)) + )); + assert!(matches!( + check_dataset_storage(&contiguous(Some(u64::MAX - 8)), &ds, &dt, u64::MAX), + Err(FormatError::InvalidDatasetStorage(_)) + )); + // Not allocated, and (unlike libhdf5) empty at a defined address. + assert!(check_dataset_storage(&contiguous(None), &ds, &dt, 0).is_ok()); + let empty = make_simple_dataspace(&[0]); + assert!(check_dataset_storage(&contiguous(Some(64)), &empty, &dt, 64).is_ok()); + let compact = |n: usize| DataLayout::Compact { data: vec![0; n] }; + assert!(check_dataset_storage(&compact(32), &ds, &dt, 0).is_ok()); + assert!(matches!( + check_dataset_storage(&compact(24), &ds, &dt, 0), + Err(FormatError::InvalidDatasetStorage(_)) + )); + } + #[test] fn zerocopy_size_mismatch() { let dt = make_f64_le_type(); diff --git a/crates/clawhdf5-format/src/error.rs b/crates/clawhdf5-format/src/error.rs index b365794..ddd9ba7 100644 --- a/crates/clawhdf5-format/src/error.rs +++ b/crates/clawhdf5-format/src/error.rs @@ -230,6 +230,11 @@ pub enum FormatError { /// libhdf5's own error text): more than 32 dimensions, a rank on a /// scalar or null dataspace, a dimension larger than its maximum. InvalidDataspace(&'static str), + /// A dataset whose storage libhdf5 refuses when it opens the dataset + /// (the reason is libhdf5's own error text): an element count times + /// element size that overflows, contiguous storage past the end of the + /// file, compact data of the wrong size. + InvalidDatasetStorage(&'static str), } impl fmt::Display for FormatError { @@ -507,6 +512,9 @@ impl fmt::Display for FormatError { FormatError::InvalidDataspace(why) => { write!(f, "invalid dataspace: {why}") } + FormatError::InvalidDatasetStorage(why) => { + write!(f, "invalid dataset storage: {why}") + } } } } diff --git a/crates/clawhdf5/src/lazy.rs b/crates/clawhdf5/src/lazy.rs index a33b9bb..3232ee6 100644 --- a/crates/clawhdf5/src/lazy.rs +++ b/crates/clawhdf5/src/lazy.rs @@ -135,10 +135,11 @@ impl LazyFile { if !has_message(&hdr, MessageType::DataLayout) { return Err(Error::NotADataset(path.to_string())); } - Ok(LazyDataset { + LazyDataset { file: self, header: hdr, - }) + } + .check_open() } /// Resolve a path and return a `LazyGroup` handle. @@ -284,10 +285,11 @@ impl<'f, R: HDF5Read> LazyGroup<'f, R> { if !has_message(&hdr, MessageType::DataLayout) { return Err(Error::NotADataset(name.to_string())); } - Ok(LazyDataset { + LazyDataset { file: self.file, header: hdr, - }) + } + .check_open() } /// Get a subgroup within this group by name. @@ -326,6 +328,24 @@ pub struct LazyDataset<'f, R: HDF5Read> { } impl<'f, R: HDF5Read> LazyDataset<'f, R> { + /// libhdf5's storage checks when it opens a dataset + /// ([`data_read::check_dataset_storage`]): a dataset whose element count + /// times element size overflows, or whose contiguous storage runs past + /// the end of the file, fails to open. A datatype, dataspace or layout + /// that does not decode is left for the read to report, as before (the + /// dataset still opens, and its attributes can be read). + fn check_open(self) -> Result { + let decoded = (|| -> Result<_, Error> { + let data = self.required_payload(MessageType::Dataspace)?; + let ds = Dataspace::parse(&data, self.file.length_size())?; + Ok((self.datatype()?, ds, self.data_layout()?)) + })(); + if let Ok((dt, ds, dl)) = decoded { + data_read::check_dataset_storage(&dl, &ds, &dt, self.file.hdf5_bytes().len() as u64)?; + } + Ok(self) + } + /// Returns the shape (dimensions) of the dataset. pub fn shape(&self) -> Result, Error> { let ds = self.dataspace()?; diff --git a/crates/clawhdf5/src/mmap_file.rs b/crates/clawhdf5/src/mmap_file.rs index 59adee9..2a8400b 100644 --- a/crates/clawhdf5/src/mmap_file.rs +++ b/crates/clawhdf5/src/mmap_file.rs @@ -85,10 +85,11 @@ impl MmapFile { if !has_message(&hdr, MessageType::DataLayout) { return Err(Error::NotADataset(path.to_string())); } - Ok(MmapDataset { + MmapDataset { file: self, header: hdr, - }) + } + .check_open() } /// Resolve a path and return a `MmapGroup` handle. @@ -211,10 +212,11 @@ impl<'f> MmapGroup<'f> { if !has_message(&hdr, MessageType::DataLayout) { return Err(Error::NotADataset(name.to_string())); } - Ok(MmapDataset { + MmapDataset { file: self.file, header: hdr, - }) + } + .check_open() } /// Get a subgroup within this group by name. @@ -253,6 +255,24 @@ pub struct MmapDataset<'f> { } impl<'f> MmapDataset<'f> { + /// libhdf5's storage checks when it opens a dataset + /// ([`data_read::check_dataset_storage`]): a dataset whose element count + /// times element size overflows, or whose contiguous storage runs past + /// the end of the file, fails to open. A datatype, dataspace or layout + /// that does not decode is left for the read to report, as before (the + /// dataset still opens, and its attributes can be read). + fn check_open(self) -> Result { + let decoded = (|| -> Result<_, Error> { + let data = self.required_payload(MessageType::Dataspace)?; + let ds = Dataspace::parse(&data, self.file.length_size())?; + Ok((self.datatype()?, ds, self.data_layout()?)) + })(); + if let Ok((dt, ds, dl)) = decoded { + data_read::check_dataset_storage(&dl, &ds, &dt, self.file.hdf5_bytes().len() as u64)?; + } + Ok(self) + } + /// Returns the shape (dimensions) of the dataset. pub fn shape(&self) -> Result, Error> { let ds = self.dataspace()?; diff --git a/crates/clawhdf5/src/reader.rs b/crates/clawhdf5/src/reader.rs index 90faf11..40ac08b 100644 --- a/crates/clawhdf5/src/reader.rs +++ b/crates/clawhdf5/src/reader.rs @@ -170,10 +170,11 @@ impl File { if !has_message(&hdr, MessageType::DataLayout) { return Err(Error::NotADataset(path.to_string())); } - Ok(Dataset { + Dataset { file: self, header: hdr, - }) + } + .check_open() } /// A `Dataset` handle for the object header at `address` (an address @@ -186,10 +187,11 @@ impl File { if !has_message(&hdr, MessageType::DataLayout) { return Err(Error::NotADataset(format!("object at address {address}"))); } - Ok(Dataset { + Dataset { file: self, header: hdr, - }) + } + .check_open() } /// Resolve a path and return a `Group` handle. @@ -427,10 +429,11 @@ impl<'f> Group<'f> { if !has_message(&hdr, MessageType::DataLayout) { return Err(Error::NotADataset(name.to_string())); } - Ok(Dataset { + Dataset { file: self.file, header: hdr, - }) + } + .check_open() } /// Get a subgroup within this group by name. @@ -469,6 +472,24 @@ pub struct Dataset<'f> { } impl<'f> Dataset<'f> { + /// libhdf5's storage checks when it opens a dataset + /// ([`data_read::check_dataset_storage`]): a dataset whose element count + /// times element size overflows, or whose contiguous storage runs past + /// the end of the file, fails to open. A datatype, dataspace or layout + /// that does not decode is left for the read to report, as before (the + /// dataset still opens, and its attributes can be read). + fn check_open(self) -> Result { + let decoded = (|| -> Result<_, Error> { + let data = self.required_payload(MessageType::Dataspace)?; + let ds = Dataspace::parse(&data, self.file.length_size())?; + Ok((self.datatype()?, ds, self.data_layout()?)) + })(); + if let Ok((dt, ds, dl)) = decoded { + data_read::check_dataset_storage(&dl, &ds, &dt, self.file.data.len() as u64)?; + } + Ok(self) + } + /// Returns the shape (dimensions) of the dataset. pub fn shape(&self) -> Result, Error> { let ds = self.dataspace()?; diff --git a/crates/clawhdf5/tests/header_validation_interop.rs b/crates/clawhdf5/tests/header_validation_interop.rs index 2f52d27..0bfd30e 100644 --- a/crates/clawhdf5/tests/header_validation_interop.rs +++ b/crates/clawhdf5/tests/header_validation_interop.rs @@ -508,3 +508,52 @@ for libver in ("earliest", "latest"): ); } } + +/// cve-2024-32624 `/Dset_OBJREF`: a dataspace whose element count times the +/// element size overflows 64 bits. libhdf5 refuses to open the dataset +/// ("size of dataset's storage overflowed"); `File::dataset` used to open it +/// and report its shape, and only reading failed. The same for storage that +/// runs past the end of the file ("invalid dataset size, likely file +/// corruption"). +#[test] +fn dataset_storage_libhdf5_refuses_at_open_is_refused_at_open() { + skip_if_no_python!(); + let dir = tempfile::tempdir().unwrap(); + // A contiguous int64 dataset of 3 elements, libver earliest (a version 1 + // dataspace holding the size and the maximum size). "overflow" makes + // both 2^62 + 2 (x 8 bytes overflows); "pasteof" makes both 1000 (the + // storage runs past the end of the file). + let verdicts = h5py_verdicts( + dir.path(), + r#" +good = os.path.join(d, "good.h5") +with h5py.File(good, "w", libver="earliest") as f: + f.create_dataset("d", data=np.array([7, 8, 9], dtype=" 0 and data.find(three, at + 1) < 0 +for name, n in (("overflow", (1 << 62) + 2), ("pasteof", 1000)): + bad = bytearray(data) + bad[at:at + 16] = struct.pack(" Date: Sat, 26 Sep 2026 10:22:22 -0500 Subject: [PATCH 15/43] feat(format): read Blosc2 (filter 32026) in pure Rust hdf5plugin's Blosc2 was a clear "not implemented" error. It stores each HDF5 chunk as a Blosc2 contiguous frame; for chunks of 2+ dimensions the frame holds a B2ND array whose data are cut into (padded) blocks stored one after another. filters_blosc2 (feature `blosc2`, in `plugin-filters`; the facade forwards both) decodes, following c-blosc2's decoder and hdf5-blosc2's blosc2_filter.c: - the frame: the msgpack header's fixed fields, the metalayer index, the offsets chunk (with the special zero/NaN/uninitialised offsets) and chunk lookup; - Blosc2 chunks: 16- and 32-byte headers, special chunks (zeros, NaN, uninitialised, one repeated value), split and unsplit streams, zero and run-length streams, and the filter pipeline run backwards (shuffle, shuffle with a byte-group size, bit shuffle including the version-2 and later handling of a partial group of 8, delta against the first block, truncated precision); - the codecs, shared with Blosc 1: BloscLZ, LZ4/LZ4HC, Zlib, Zstandard; - B2ND arrays: blocks gathered into C order, padding dropped, several chunks per array, and the array shape checked against the chunk shape in cd_values as the HDF5 filter does. Dictionaries, lazy chunks, variable-length blocks, user-defined codecs and registered filters (e.g. bytedelta) are errors. Uninitialised chunks read as zeros. No encoder. Tests: h5py + hdf5plugin write every codec x filter (none, shuffle, bitshuffle, delta) and levels 0-9 over the plugin-filter cases, then i1..u8/f4/f8 in 1-D to 5-D chunks with partial edge chunks, datasets of zeros, one value and NaN, and Fletcher32 before Blosc2 (plain frames for n-D chunks); clawhdf5 reads each exactly as its unfiltered twin, and truncated precision exactly as h5py reads it. Fixture frames from python-blosc2 (tests/fixtures/blosc2/generate.py) cover what hdf5plugin never writes: special chunks, delta over many blocks and odd type sizes, odd bit-shuffle blocks, forced splitting, multi-chunk B2ND arrays with a zero chunk, and the refused features. The decoder is fuzzed (random and mutated frames and chunks: no panic, output within the limit). Conformance: h5ex_d_blosc2.h5 now reads (576 of 697 ok, baseline 575). Co-Authored-By: Claude Opus 5.5 (1M context) --- crates/clawhdf5-format/Cargo.toml | 4 +- crates/clawhdf5-format/src/filter_registry.rs | 11 +- crates/clawhdf5-format/src/filters.rs | 7 + crates/clawhdf5-format/src/filters_blosc.rs | 6 +- crates/clawhdf5-format/src/filters_blosc2.rs | 1083 +++++++++++++++++ crates/clawhdf5-format/src/lib.rs | 2 + .../tests/fixtures/blosc2/always_split.b2f | Bin 0 -> 1650 bytes .../tests/fixtures/blosc2/always_split.out | Bin 0 -> 4000 bytes .../tests/fixtures/blosc2/b2nd_2d.b2f | Bin 0 -> 3950 bytes .../tests/fixtures/blosc2/b2nd_2d.out | Bin 0 -> 4292 bytes .../tests/fixtures/blosc2/b2nd_3d.b2f | Bin 0 -> 8108 bytes .../tests/fixtures/blosc2/b2nd_3d.out | Bin 0 -> 2520 bytes .../tests/fixtures/blosc2/b2nd_4d.b2f | Bin 0 -> 6804 bytes .../tests/fixtures/blosc2/b2nd_4d.out | Bin 0 -> 2100 bytes .../tests/fixtures/blosc2/b2nd_zero_chunk.b2f | Bin 0 -> 1849 bytes .../tests/fixtures/blosc2/b2nd_zero_chunk.out | Bin 0 -> 2400 bytes .../fixtures/blosc2/bitshuffle_odd_blocks.b2f | Bin 0 -> 2204 bytes .../fixtures/blosc2/bitshuffle_odd_blocks.out | Bin 0 -> 2000 bytes .../tests/fixtures/blosc2/bytedelta.b2f | Bin 0 -> 458 bytes .../tests/fixtures/blosc2/bytedelta.err | 1 + .../fixtures/blosc2/delta_int16_blosclz.b2f | Bin 0 -> 1601 bytes .../fixtures/blosc2/delta_int16_blosclz.out | Bin 0 -> 2000 bytes .../tests/fixtures/blosc2/delta_int32_lz4.b2f | Bin 0 -> 1584 bytes .../tests/fixtures/blosc2/delta_int32_lz4.out | Bin 0 -> 2800 bytes .../fixtures/blosc2/delta_shuffle_v16.b2f | Bin 0 -> 1019 bytes .../fixtures/blosc2/delta_shuffle_v16.out | Bin 0 -> 3200 bytes .../fixtures/blosc2/delta_uint64_blosclz.b2f | Bin 0 -> 1323 bytes .../fixtures/blosc2/delta_uint64_blosclz.out | Bin 0 -> 3200 bytes .../tests/fixtures/blosc2/delta_uint8_lz4.b2f | Bin 0 -> 1494 bytes .../tests/fixtures/blosc2/delta_uint8_lz4.out | Bin 0 -> 2000 bytes .../tests/fixtures/blosc2/delta_v3.b2f | Bin 0 -> 377 bytes .../tests/fixtures/blosc2/delta_v3.out | Bin 0 -> 900 bytes .../tests/fixtures/blosc2/generate.py | 129 ++ .../tests/fixtures/blosc2/nan_f4.b2f | Bin 0 -> 172 bytes .../tests/fixtures/blosc2/nan_f4.out | Bin 0 -> 2000 bytes .../tests/fixtures/blosc2/nan_f8.b2f | Bin 0 -> 172 bytes .../tests/fixtures/blosc2/nan_f8.out | Bin 0 -> 2400 bytes .../tests/fixtures/blosc2/never_split.b2f | Bin 0 -> 1671 bytes .../tests/fixtures/blosc2/never_split.out | Bin 0 -> 3000 bytes .../tests/fixtures/blosc2/shuffle_meta2.b2f | Bin 0 -> 1003 bytes .../tests/fixtures/blosc2/shuffle_meta2.out | Bin 0 -> 4000 bytes .../tests/fixtures/blosc2/uninit_i8.b2f | Bin 0 -> 172 bytes .../tests/fixtures/blosc2/uninit_i8.out | Bin 0 -> 512 bytes .../tests/fixtures/blosc2/value_f8.b2f | Bin 0 -> 212 bytes .../tests/fixtures/blosc2/value_f8.out | Bin 0 -> 2000 bytes .../tests/fixtures/blosc2/value_i4.b2f | Bin 0 -> 208 bytes .../tests/fixtures/blosc2/value_i4.out | Bin 0 -> 1200 bytes .../tests/fixtures/blosc2/zero_u2.b2f | Bin 0 -> 172 bytes .../tests/fixtures/blosc2/zero_u2.out | Bin 0 -> 4000 bytes .../tests/fixtures/blosc2/zstd_dict.b2f | Bin 0 -> 4978 bytes .../tests/fixtures/blosc2/zstd_dict.err | 1 + crates/clawhdf5/Cargo.toml | 3 +- .../clawhdf5/tests/plugin_filters_interop.rs | 135 +- scripts/ci-test.sh | 4 +- 54 files changed, 1366 insertions(+), 20 deletions(-) create mode 100644 crates/clawhdf5-format/src/filters_blosc2.rs create mode 100644 crates/clawhdf5-format/tests/fixtures/blosc2/always_split.b2f create mode 100644 crates/clawhdf5-format/tests/fixtures/blosc2/always_split.out create mode 100644 crates/clawhdf5-format/tests/fixtures/blosc2/b2nd_2d.b2f create mode 100644 crates/clawhdf5-format/tests/fixtures/blosc2/b2nd_2d.out create mode 100644 crates/clawhdf5-format/tests/fixtures/blosc2/b2nd_3d.b2f create mode 100644 crates/clawhdf5-format/tests/fixtures/blosc2/b2nd_3d.out create mode 100644 crates/clawhdf5-format/tests/fixtures/blosc2/b2nd_4d.b2f create mode 100644 crates/clawhdf5-format/tests/fixtures/blosc2/b2nd_4d.out create mode 100644 crates/clawhdf5-format/tests/fixtures/blosc2/b2nd_zero_chunk.b2f create mode 100644 crates/clawhdf5-format/tests/fixtures/blosc2/b2nd_zero_chunk.out create mode 100644 crates/clawhdf5-format/tests/fixtures/blosc2/bitshuffle_odd_blocks.b2f create mode 100644 crates/clawhdf5-format/tests/fixtures/blosc2/bitshuffle_odd_blocks.out create mode 100644 crates/clawhdf5-format/tests/fixtures/blosc2/bytedelta.b2f create mode 100644 crates/clawhdf5-format/tests/fixtures/blosc2/bytedelta.err create mode 100644 crates/clawhdf5-format/tests/fixtures/blosc2/delta_int16_blosclz.b2f create mode 100644 crates/clawhdf5-format/tests/fixtures/blosc2/delta_int16_blosclz.out create mode 100644 crates/clawhdf5-format/tests/fixtures/blosc2/delta_int32_lz4.b2f create mode 100644 crates/clawhdf5-format/tests/fixtures/blosc2/delta_int32_lz4.out create mode 100644 crates/clawhdf5-format/tests/fixtures/blosc2/delta_shuffle_v16.b2f create mode 100644 crates/clawhdf5-format/tests/fixtures/blosc2/delta_shuffle_v16.out create mode 100644 crates/clawhdf5-format/tests/fixtures/blosc2/delta_uint64_blosclz.b2f create mode 100644 crates/clawhdf5-format/tests/fixtures/blosc2/delta_uint64_blosclz.out create mode 100644 crates/clawhdf5-format/tests/fixtures/blosc2/delta_uint8_lz4.b2f create mode 100644 crates/clawhdf5-format/tests/fixtures/blosc2/delta_uint8_lz4.out create mode 100644 crates/clawhdf5-format/tests/fixtures/blosc2/delta_v3.b2f create mode 100644 crates/clawhdf5-format/tests/fixtures/blosc2/delta_v3.out create mode 100644 crates/clawhdf5-format/tests/fixtures/blosc2/generate.py create mode 100644 crates/clawhdf5-format/tests/fixtures/blosc2/nan_f4.b2f create mode 100644 crates/clawhdf5-format/tests/fixtures/blosc2/nan_f4.out create mode 100644 crates/clawhdf5-format/tests/fixtures/blosc2/nan_f8.b2f create mode 100644 crates/clawhdf5-format/tests/fixtures/blosc2/nan_f8.out create mode 100644 crates/clawhdf5-format/tests/fixtures/blosc2/never_split.b2f create mode 100644 crates/clawhdf5-format/tests/fixtures/blosc2/never_split.out create mode 100644 crates/clawhdf5-format/tests/fixtures/blosc2/shuffle_meta2.b2f create mode 100644 crates/clawhdf5-format/tests/fixtures/blosc2/shuffle_meta2.out create mode 100644 crates/clawhdf5-format/tests/fixtures/blosc2/uninit_i8.b2f create mode 100644 crates/clawhdf5-format/tests/fixtures/blosc2/uninit_i8.out create mode 100644 crates/clawhdf5-format/tests/fixtures/blosc2/value_f8.b2f create mode 100644 crates/clawhdf5-format/tests/fixtures/blosc2/value_f8.out create mode 100644 crates/clawhdf5-format/tests/fixtures/blosc2/value_i4.b2f create mode 100644 crates/clawhdf5-format/tests/fixtures/blosc2/value_i4.out create mode 100644 crates/clawhdf5-format/tests/fixtures/blosc2/zero_u2.b2f create mode 100644 crates/clawhdf5-format/tests/fixtures/blosc2/zero_u2.out create mode 100644 crates/clawhdf5-format/tests/fixtures/blosc2/zstd_dict.b2f create mode 100644 crates/clawhdf5-format/tests/fixtures/blosc2/zstd_dict.err diff --git a/crates/clawhdf5-format/Cargo.toml b/crates/clawhdf5-format/Cargo.toml index 1d1998c..c1ec6df 100644 --- a/crates/clawhdf5-format/Cargo.toml +++ b/crates/clawhdf5-format/Cargo.toml @@ -76,8 +76,10 @@ bitshuffle = ["lz4_flex", "ruzstd"] bzip2 = ["dep:bzip2", "std"] # Blosc 1 (32001) with its BloscLZ, LZ4, Snappy, Zlib and Zstandard codecs. blosc = ["lz4_flex", "ruzstd", "snap", "deflate", "std"] +# Blosc2 (32026), read-only: frames, B2ND arrays, and the Blosc codecs above. +blosc2 = ["blosc"] # Every plugin filter above. -plugin-filters = ["lzf", "bitshuffle", "bzip2", "blosc"] +plugin-filters = ["lzf", "bitshuffle", "bzip2", "blosc", "blosc2"] [[bench]] name = "parallel_decompress_bench" diff --git a/crates/clawhdf5-format/src/filter_registry.rs b/crates/clawhdf5-format/src/filter_registry.rs index 6d39613..442e27c 100644 --- a/crates/clawhdf5-format/src/filter_registry.rs +++ b/crates/clawhdf5-format/src/filter_registry.rs @@ -5,7 +5,8 @@ //! * **Built-in filters** — a static table of the filters compiled into this //! build: the HDF5 standard filters (deflate, shuffle, Fletcher32, szip, //! N-Bit, scale-offset) and the plugin filters whose cargo features are -//! enabled (LZ4, Zstandard, pcodec, LZF, bitshuffle, bzip2, blosc). +//! enabled (LZ4, Zstandard, pcodec, LZF, bitshuffle, bzip2, blosc, +//! blosc2). //! [`builtin_filters`] lists them. //! * **Registered filters** (`std` only) — codecs the application supplies //! for any other ID with [`register_filter`] (a [`FilterCodec`], or just a @@ -166,7 +167,7 @@ pub fn known_filter(id: u16) -> Option<(&'static str, Option<&'static str>)> { 32019 => ("JPEG", None), 32022 => ("BitGroom", None), 32023 => ("Granular BitRound", None), - 32026 => ("Blosc2", None), + 32026 => ("Blosc2", Some("blosc2")), _ => return None, }) } @@ -452,12 +453,12 @@ pub(crate) mod tests { #[test] fn unsupported_filter_error_names_the_filter() { let msg = FormatError::UnsupportedFilter(32026).to_string(); + assert!(msg.contains("Blosc2") && msg.contains("`blosc2`"), "{msg}"); + let msg = FormatError::UnsupportedFilter(32013).to_string(); assert!( - msg.contains("Blosc2") && msg.contains("not implemented"), + msg.contains("ZFP") && msg.contains("not implemented"), "{msg}" ); - let msg = FormatError::UnsupportedFilter(32013).to_string(); - assert!(msg.contains("ZFP"), "{msg}"); let msg = FormatError::UnsupportedFilter(32000).to_string(); assert!(msg.contains("LZF") && msg.contains("`lzf`"), "{msg}"); assert_eq!( diff --git a/crates/clawhdf5-format/src/filters.rs b/crates/clawhdf5-format/src/filters.rs index 36f2e96..d577eac 100644 --- a/crates/clawhdf5-format/src/filters.rs +++ b/crates/clawhdf5-format/src/filters.rs @@ -278,6 +278,13 @@ pub(crate) static BUILTIN_FILTERS: &[BuiltinFilter] = &[ }, encode: None, }, + #[cfg(feature = "blosc2")] + BuiltinFilter { + id: crate::filter_pipeline::FILTER_BLOSC2, + name: "blosc2", + decode: crate::filters_blosc2::blosc2_decode, + encode: None, + }, ]; /// Decode the HDF5 scale-offset filter (id 6). diff --git a/crates/clawhdf5-format/src/filters_blosc.rs b/crates/clawhdf5-format/src/filters_blosc.rs index 3edc964..fadfc1e 100644 --- a/crates/clawhdf5-format/src/filters_blosc.rs +++ b/crates/clawhdf5-format/src/filters_blosc.rs @@ -49,7 +49,7 @@ fn le32(b: &[u8], at: usize) -> Result { /// The codec inside a Blosc frame (flags bits 5-7). #[derive(Debug, Clone, Copy, PartialEq, Eq)] -enum Codec { +pub(crate) enum Codec { BloscLz, Lz4, Snappy, @@ -58,7 +58,7 @@ enum Codec { } impl Codec { - fn from_flags(flags: u8) -> Result { + pub(crate) fn from_flags(flags: u8) -> Result { match flags >> 5 { 0 => Ok(Codec::BloscLz), 1 => Ok(Codec::Lz4), @@ -71,7 +71,7 @@ impl Codec { } /// Decode one codec stream into exactly `dst`. -fn decode_stream( +pub(crate) fn decode_stream( codec: Codec, src: &[u8], dst: &mut [u8], diff --git a/crates/clawhdf5-format/src/filters_blosc2.rs b/crates/clawhdf5-format/src/filters_blosc2.rs new file mode 100644 index 0000000..5f332f8 --- /dev/null +++ b/crates/clawhdf5-format/src/filters_blosc2.rs @@ -0,0 +1,1083 @@ +//! Blosc2 (HDF5 filter 32026, `hdf5-blosc2`, hdf5plugin's `Blosc2`) in pure +//! Rust, read-only: the contiguous frame ("cframe") each chunk is stored +//! as, the Blosc2 chunks inside it, and the B2ND n-dimensional layout. +//! +//! References: c-blosc2's `README_CFRAME_FORMAT.rst`, +//! `README_CHUNK_FORMAT.rst` and `README_B2ND_METALAYER.rst`, and the +//! decoder in `blosc/blosc2.c` (`read_chunk_header`, `blosc_d`, +//! `pipeline_backward`), `blosc/frame.c` and `blosc/b2nd.c`, whose checks +//! this module mirrors. +//! +//! **The filter** (`blosc2_filter.c`). `cd_values`: `[0]` filter revision, +//! `[1]` block size, `[2]` type size, `[3]` chunk size in bytes, `[4]` +//! level, `[5]` filter (0 none, 1 shuffle, 2 bit shuffle, 3 delta, 4 +//! truncate precision), `[6]` codec, and for chunks of 2 to 8 (16 in newer +//! builds) dimensions `[7]` the rank and `[8..]` the chunk dimensions. Each +//! HDF5 chunk is one frame. A frame with a `b2nd` (or older `caterva`) +//! metalayer holds an n-D array the shape of the HDF5 chunk, whose data are +//! cut into blocks of `blockshape` (padded at the chunk's edges) stored one +//! after another, each in C order — so the decoded Blosc2 chunk is +//! reassembled into C order here. Otherwise the frame's first chunk is the +//! HDF5 chunk's data as it is. +//! +//! **Frame.** A msgpack header with fields at fixed offsets (header length, +//! frame length, sizes, type size, chunk size, codec flags) and a map of +//! metalayers; the chunks back to back; an "offsets" chunk (a Blosc2 chunk +//! of little-endian `i64` offsets, one per chunk, relative to the end of the +//! header — negative values encode a chunk of zeros, NaNs or uninitialised +//! values); and a msgpack trailer, which the decoder does not need. +//! +//! **Chunk.** A 16-byte Blosc header (as Blosc 1), extended to 32 bytes when +//! the shuffle and bit-shuffle flags are both set: six filter slots with +//! their metadata, the codec, and flags for dictionaries, lazy chunks and +//! "special" chunks (all zeros, NaN, uninitialised, or one repeated value +//! stored after the header). A regular chunk has a table of block starts; +//! each block is one stream, or `typesize` streams when the "do not split" +//! flag is clear and the block is not the short last one. A stream is an +//! `i32` length and the codec's output: 0 means zeros, negative with a token +//! byte means a run of one byte, and a length equal to the stream's size +//! means stored raw. Decoded blocks run through the filter pipeline +//! backwards (shuffle, bit shuffle, delta; truncate-precision needs no +//! decoding). The codecs are Blosc 1's: BloscLZ, LZ4 (and LZ4HC), Zlib and +//! Zstandard, shared with [`crate::filters_blosc`]. +//! +//! Chunks of uninitialised values read as zeros (libhdf5 hands back +//! whatever its buffer held). Not supported (a clear error, never data): +//! variable-length blocks, dictionaries, lazy chunks, user-defined codecs +//! and registered filters (e.g. bytedelta), sparse frames. + +use crate::error::FormatError; +use crate::filter_registry::FilterContext; +use crate::filters_bitshuffle::bitunshuffle_block; +use crate::filters_blosc::{Codec, blosclz_decompress, decode_stream}; + +const MIN_HEADER: usize = 16; +const EXT_HEADER: usize = 32; + +const FLAG_SHUFFLE: u8 = 0x01; +const FLAG_MEMCPYED: u8 = 0x02; +const FLAG_BITSHUFFLE: u8 = 0x04; +const FLAG_DELTA: u8 = 0x08; +const FLAG_DONT_SPLIT: u8 = 0x10; + +const B2_USEDICT: u8 = 0x01; +const B2_LAZY: u8 = 0x08; +const VL_BLOCKS: u8 = 0x01; + +const SPECIAL_ZERO: u8 = 1; +const SPECIAL_NAN: u8 = 2; +const SPECIAL_VALUE: u8 = 3; +const SPECIAL_UNINIT: u8 = 4; + +const FILTER_NONE: u8 = 0; +const FILTER_SHUFFLE: u8 = 1; +const FILTER_BITSHUFFLE: u8 = 2; +const FILTER_DELTA: u8 = 3; +const FILTER_TRUNC_PREC: u8 = 4; + +/// Chunk format versions: 3 is the Blosc 2 alpha series. +const VERSION_ALPHA: u8 = 3; +/// The newest chunk format c-blosc2 knows (6 added variable-length blocks). +const VERSION_MAX: u8 = 6; +/// The newest frame format c-blosc2 knows. +const FRAME_VERSION_MAX: u8 = 3; +const MAX_BLOCKSIZE: usize = 536_866_816; +/// `B2ND_MAX_DIM` in current c-blosc2 (8 in older builds). +const B2ND_MAX_DIM: usize = 16; + +// Frame header field offsets (c-blosc2 `frame.h`). +const FRAME_HEADER_LEN: usize = 11; +const FRAME_LEN: usize = 16; +const FRAME_FLAGS: usize = 25; +const FRAME_TYPE: usize = 26; +const FRAME_NBYTES: usize = 30; +const FRAME_CBYTES: usize = 39; +const FRAME_TYPESIZE: usize = 48; +const FRAME_CHUNKSIZE: usize = 58; +const FRAME_HEADER_MINLEN: usize = 87; +const FRAME_IDX_SIZE: usize = 89; +const FRAME_VL_BLOCKS: u8 = 0x80; + +fn err(msg: &str) -> FormatError { + FormatError::DecompressionError(format!("blosc2: {msg}")) +} + +fn le_i32(b: &[u8], at: usize) -> Result { + b.get(at..at + 4) + .map(|s| i32::from_le_bytes(s.try_into().unwrap())) + .ok_or_else(|| err("truncated chunk")) +} + +fn be(b: &[u8], at: usize) -> Result<[u8; N], FormatError> { + b.get(at..at + N) + .map(|s| s.try_into().unwrap()) + .ok_or_else(|| err("truncated frame header")) +} + +/// A parsed Blosc2 chunk header. +struct ChunkHeader { + version: u8, + flags: u8, + typesize: usize, + nbytes: usize, + blocksize: usize, + cbytes: usize, + overhead: usize, + filters: [u8; 6], + filters_meta: [u8; 6], + udcodec: u8, + blosc2_flags: u8, + special: u8, +} + +fn read_header(src: &[u8]) -> Result { + if src.len() < MIN_HEADER { + return Err(err("truncated chunk header")); + } + let version = src[0]; + let flags = src[2]; + let typesize = src[3] as usize; + let nbytes = le_i32(src, 4)?; + let blocksize = le_i32(src, 8)?; + let cbytes = le_i32(src, 12)?; + if cbytes < MIN_HEADER as i32 { + return Err(err("chunk size smaller than its header")); + } + if blocksize <= 0 || blocksize as usize > MAX_BLOCKSIZE { + return Err(err("bad block size")); + } + if typesize == 0 { + return Err(err("type size is zero")); + } + if nbytes < 0 { + return Err(err("negative decoded size")); + } + let (nbytes, mut blocksize, cbytes) = (nbytes as usize, blocksize as usize, cbytes as usize); + let mut h = ChunkHeader { + version, + flags, + typesize, + nbytes, + blocksize, + cbytes, + overhead: MIN_HEADER, + filters: [0; 6], + filters_meta: [0; 6], + udcodec: 0, + blosc2_flags: 0, + special: 0, + }; + let mut flags2 = 0; + if flags & (FLAG_SHUFFLE | FLAG_BITSHUFFLE) == FLAG_SHUFFLE | FLAG_BITSHUFFLE { + if cbytes < EXT_HEADER || src.len() < EXT_HEADER { + return Err(err("truncated extended chunk header")); + } + h.overhead = EXT_HEADER; + h.filters.copy_from_slice(&src[16..22]); + h.udcodec = src[22]; + h.filters_meta.copy_from_slice(&src[24..30]); + flags2 = src[30]; + h.blosc2_flags = src[31]; + h.special = (h.blosc2_flags >> 4) & 7; + if h.special == SPECIAL_VALUE { + let ts = cbytes - EXT_HEADER; + if ts == 0 || ts > nbytes || !nbytes.is_multiple_of(ts) { + return Err(err("bad repeated-value chunk")); + } + } else if h.special != 0 && h.special != SPECIAL_ZERO && !nbytes.is_multiple_of(typesize) { + return Err(err("decoded size is not a whole number of elements")); + } + if version == VERSION_ALPHA { + h.filters[5] = 0; + h.filters_meta[5] = 0; + } + } else { + // A Blosc 1 style header: the filters come from the flags. + if flags & FLAG_SHUFFLE != 0 { + h.filters[5] = FILTER_SHUFFLE; + } + if flags & FLAG_BITSHUFFLE != 0 { + h.filters[5] = FILTER_BITSHUFFLE; + } + if flags & FLAG_DELTA != 0 { + h.filters[4] = FILTER_DELTA; + } + } + if version > VERSION_MAX && flags2 & !VL_BLOCKS != 0 { + return Err(err(&format!("chunk format version {version} is too new"))); + } + if flags2 & VL_BLOCKS != 0 { + return Err(err("variable-length blocks are not supported")); + } + if nbytes > 0 && blocksize > nbytes { + blocksize = nbytes; + } + h.blocksize = blocksize; + if cbytes > src.len() { + return Err(err("chunk is longer than its buffer")); + } + Ok(h) +} + +/// Decompress one Blosc2 chunk (header included), refusing to produce more +/// than `limit` bytes. +pub fn blosc2_decompress_chunk(src: &[u8], limit: usize) -> Result, FormatError> { + let h = read_header(src)?; + if h.nbytes > limit { + return Err(err("decoded size exceeds the limit")); + } + let src = &src[..h.cbytes]; + if h.special > SPECIAL_UNINIT { + return Err(err(&format!("unknown special chunk type {}", h.special))); + } + let memcpyed = h.flags & FLAG_MEMCPYED != 0; + if memcpyed && h.cbytes != h.nbytes + h.overhead { + return Err(err("stored chunk has the wrong size")); + } + if h.nbytes == 0 && h.cbytes == h.overhead && h.special == 0 { + return Ok(Vec::new()); + } + let nbytes = h.nbytes; + let mut out = vec![0u8; nbytes]; + if h.special != 0 { + // Filled per block, as c-blosc2 does: each block must hold whole + // values. + let fill: Option<&[u8]> = match h.special { + SPECIAL_ZERO | SPECIAL_UNINIT => None, + SPECIAL_NAN => Some(match h.typesize { + 4 => &[0x00, 0x00, 0xc0, 0x7f], + 8 => &[0, 0, 0, 0, 0, 0, 0xf8, 0x7f], + _ => return Err(err("NaN chunk of a type that is not f32 or f64")), + }), + _ => Some(&src[EXT_HEADER..]), + }; + if let Some(value) = fill { + let ts = value.len(); + let blocksize = h.blocksize.max(1); + if !blocksize.is_multiple_of(ts) || !(nbytes % blocksize).is_multiple_of(ts) { + return Err(err("special chunk blocks are not whole values")); + } + for v in out.chunks_exact_mut(ts) { + v.copy_from_slice(value); + } + } + return Ok(out); + } + if memcpyed { + out.copy_from_slice(&src[h.overhead..]); + return Ok(out); + } + if h.blosc2_flags & B2_USEDICT != 0 { + return Err(err("dictionary-compressed chunks are not supported")); + } + if h.blosc2_flags & B2_LAZY != 0 { + return Err(err("lazy chunks are not supported")); + } + let codec = match h.flags >> 5 { + 6 => { + return Err(err(&format!( + "user-defined codec {} is not supported", + h.udcodec + ))); + } + 2 | 5 | 7 => return Err(err(&format!("unknown codec {}", h.flags >> 5))), + _ => Codec::from_flags(h.flags)?, + }; + for &f in &h.filters { + if f > FILTER_TRUNC_PREC { + return Err(err(&format!("filter {f} is not supported"))); + } + } + let blocksize = h.blocksize; + let nblocks = nbytes.div_ceil(blocksize); + let leftover = nbytes % blocksize; + let bstarts_end = nblocks + .checked_mul(4) + .and_then(|n| n.checked_add(h.overhead)) + .ok_or_else(|| err("block table overflows"))?; + if bstarts_end > src.len() { + return Err(err("block table is truncated")); + } + let dont_split = h.flags & FLAG_DONT_SPLIT != 0; + let mut tmp = vec![0u8; blocksize]; + let mut tmp2 = vec![0u8; blocksize]; + let mut zstd = None; + for j in 0..nblocks { + let is_leftover = j == nblocks - 1 && leftover > 0; + let bsize = if is_leftover { leftover } else { blocksize }; + let start = le_i32(src, h.overhead + 4 * j)?; + if start <= 0 || start as usize >= src.len() { + return Err(err("block start out of range")); + } + let mut pos = start as usize; + let nstreams = if !dont_split && !is_leftover { + h.typesize + } else { + 1 + }; + let neblock = bsize / nstreams; + if neblock == 0 || neblock * nstreams != bsize { + return Err(err("block is not a whole number of streams")); + } + let cur = &mut tmp[..bsize]; + for s in 0..nstreams { + let csize = le_i32(src, pos).map_err(|_| err("stream runs past the chunk"))?; + pos += 4; + let dst = &mut cur[s * neblock..(s + 1) * neblock]; + if csize == 0 { + dst.fill(0); + } else if csize < 0 { + let token = *src + .get(pos) + .ok_or_else(|| err("stream runs past the chunk"))?; + pos += 1; + if token & 1 == 0 || csize < -255 { + return Err(err("unsupported stream token")); + } + dst.fill((-csize) as u8); + } else { + let csize = csize as usize; + let stream = src + .get(pos..pos + csize) + .ok_or_else(|| err("stream runs past the chunk"))?; + pos += csize; + if csize == neblock { + dst.copy_from_slice(stream); + } else if codec == Codec::BloscLz { + if blosclz_decompress(stream, dst) != neblock { + return Err(err("blosclz stream is malformed")); + } + } else { + decode_stream(codec, stream, dst, &mut zstd)?; + } + } + } + let (done, rest) = out.split_at_mut(j * blocksize); + let dest = &mut rest[..bsize]; + run_filters_backward(&h, cur, &mut tmp2[..bsize], done, dest); + } + Ok(out) +} + +/// Undo the chunk's filters on one decoded block `cur`, leaving the result +/// in `dest`. `done` is the chunk decoded so far (the delta filter's +/// reference is the first block). +fn run_filters_backward( + h: &ChunkHeader, + cur: &mut [u8], + scratch: &mut [u8], + done: &[u8], + dest: &mut [u8], +) { + let bsize = cur.len(); + let ts = h.typesize; + for i in (0..6).rev() { + match h.filters[i] { + FILTER_SHUFFLE => { + let m = h.filters_meta[i] as usize; + let bytes = if m == 0 { ts } else { m }; + unshuffle(bytes, cur, scratch); + cur.copy_from_slice(scratch); + } + FILTER_BITSHUFFLE => { + let mut n = bsize / ts; + if h.version == 2 { + if !n.is_multiple_of(8) { + continue; + } + } else { + n -= n % 8; + } + let body = n * ts; + bitunshuffle_block(&cur[..body], &mut scratch[..body], n, ts); + cur[..body].copy_from_slice(&scratch[..body]); + } + FILTER_DELTA => delta_decode(done, cur, ts), + // Truncating precision is lossy and has nothing to undo. + FILTER_NONE | FILTER_TRUNC_PREC => {} + _ => unreachable!("filters are checked before decoding"), + } + } + dest.copy_from_slice(cur); +} + +/// Byte unshuffle (`unshuffle_generic`): trailing bytes that do not make a +/// whole element are copied as they are. +fn unshuffle(bytes: usize, src: &[u8], dest: &mut [u8]) { + let n = src.len() / bytes; + for i in 0..n { + for b in 0..bytes { + dest[i * bytes + b] = src[b * n + i]; + } + } + dest[n * bytes..].copy_from_slice(&src[n * bytes..]); +} + +/// `delta_decoder`: the first block is XOR-accumulated over its own +/// elements; every other block is XORed with the (decoded) first block. +/// Elements are 1, 2, 4 or 8 bytes wide (other sizes: 8 if a multiple of +/// 8, else 1); a trailing partial element is left alone. +fn delta_decode(done: &[u8], cur: &mut [u8], typesize: usize) { + let w = match typesize { + 1 | 2 | 4 | 8 => typesize, + t if t.is_multiple_of(8) => 8, + _ => 1, + }; + let n = cur.len() / w; + if done.is_empty() { + for i in 1..n { + for b in 0..w { + cur[i * w + b] ^= cur[(i - 1) * w + b]; + } + } + } else { + // The first block is at least as long as any other. + for (c, r) in cur[..n * w].iter_mut().zip(done) { + *c ^= *r; + } + } +} + +/// What the frame header says. +struct Frame<'a> { + buf: &'a [u8], + header_len: usize, + nbytes: usize, + cbytes: usize, + typesize: usize, + chunksize: usize, + nchunks: usize, + /// The decoded offsets chunk. + offsets: Vec, +} + +fn parse_frame(buf: &[u8]) -> Result, FormatError> { + if buf.len() < FRAME_HEADER_MINLEN { + return Err(err("truncated frame header")); + } + if buf[0] & 0xf0 != 0x90 || buf[1] != 0xa8 || &buf[2..10] != b"b2frame\0" { + return Err(err("not a Blosc2 frame (bad magic)")); + } + let frame_version = buf[FRAME_FLAGS] & 0x0f; + if frame_version > FRAME_VERSION_MAX { + return Err(err(&format!( + "frame format version {frame_version} is too new" + ))); + } + if buf[FRAME_FLAGS] & FRAME_VL_BLOCKS != 0 { + return Err(err("variable-length blocks are not supported")); + } + if buf[FRAME_TYPE] & 0x0f != 0 { + return Err(err("not a contiguous frame")); + } + let header_len = i32::from_be_bytes(be(buf, FRAME_HEADER_LEN)?); + let frame_len = u64::from_be_bytes(be(buf, FRAME_LEN)?); + let nbytes = i64::from_be_bytes(be(buf, FRAME_NBYTES)?); + let cbytes = i64::from_be_bytes(be(buf, FRAME_CBYTES)?); + let typesize = i32::from_be_bytes(be(buf, FRAME_TYPESIZE)?); + let chunksize = i32::from_be_bytes(be(buf, FRAME_CHUNKSIZE)?); + if header_len < FRAME_HEADER_MINLEN as i32 || header_len as u64 > frame_len { + return Err(err("bad frame header length")); + } + if frame_len > buf.len() as u64 { + return Err(err("frame is longer than the chunk")); + } + if typesize <= 0 { + return Err(err("bad type size")); + } + if nbytes < 0 || cbytes < 0 || chunksize < 0 { + return Err(err("negative size in frame header")); + } + let header_len = header_len as usize; + let buf = &buf[..frame_len as usize]; + let cbytes = usize::try_from(cbytes).map_err(|_| err("bad compressed size"))?; + let data_end = header_len + .checked_add(cbytes) + .filter(|&e| e <= buf.len()) + .ok_or_else(|| err("chunks run past the frame"))?; + let nbytes = usize::try_from(nbytes).map_err(|_| err("bad decoded size"))?; + let chunksize = chunksize as usize; + // The offsets chunk follows the data chunks. + let off_src = &buf[data_end..]; + let expected = if nbytes == 0 { + 0 + } else if chunksize > 0 { + nbytes.div_ceil(chunksize) + } else { + // Variable-size chunks: the offsets chunk tells how many. + usize::MAX + }; + let off_limit = if expected == usize::MAX { + 1 << 24 + } else { + expected + .checked_mul(8) + .ok_or_else(|| err("too many chunks"))? + }; + let offsets = if nbytes == 0 { + Vec::new() + } else { + blosc2_decompress_chunk(off_src, off_limit)? + }; + if !offsets.len().is_multiple_of(8) || (expected != usize::MAX && offsets.len() != off_limit) { + return Err(err("offsets chunk does not match the number of chunks")); + } + Ok(Frame { + buf, + header_len, + nbytes, + cbytes, + typesize: typesize as usize, + chunksize, + nchunks: offsets.len() / 8, + offsets, + }) +} + +impl Frame<'_> { + /// Decode chunk `n`, refusing more than `limit` bytes. + fn chunk(&self, n: usize, limit: usize) -> Result, FormatError> { + if n >= self.nchunks { + return Err(err("frame has no such chunk")); + } + let raw: [u8; 8] = self.offsets[8 * n..8 * n + 8].try_into().unwrap(); + let offset = i64::from_le_bytes(raw); + if offset < 0 { + // A special chunk, recorded in the offset's top byte. + if self.chunksize == 0 { + return Err(err("special chunk in a frame without a chunk size")); + } + let size = if n == self.nchunks - 1 && !self.nbytes.is_multiple_of(self.chunksize) { + self.nbytes % self.chunksize + } else { + self.chunksize + }; + if size > limit { + return Err(err("decoded size exceeds the limit")); + } + let kind = raw[7] & 7; + return match kind { + SPECIAL_ZERO | SPECIAL_UNINIT => Ok(vec![0u8; size]), + SPECIAL_NAN => { + let nan: &[u8] = match self.typesize { + 4 => &[0x00, 0x00, 0xc0, 0x7f], + 8 => &[0, 0, 0, 0, 0, 0, 0xf8, 0x7f], + _ => return Err(err("NaN chunk of a type that is not f32 or f64")), + }; + if !size.is_multiple_of(nan.len()) { + return Err(err("NaN chunk is not whole values")); + } + Ok(nan.iter().copied().cycle().take(size).collect()) + } + _ => Err(err(&format!("unknown special chunk offset {kind}"))), + }; + } + // Every chunk lies in the data section, between the header and the + // offsets chunk. + let end = self.header_len + self.cbytes; + let start = usize::try_from(offset) + .ok() + .and_then(|o| self.header_len.checked_add(o)) + .filter(|&s| s < end && end - s >= MIN_HEADER) + .ok_or_else(|| err("chunk offset out of range"))?; + blosc2_decompress_chunk(&self.buf[start..end], limit) + } + + /// The content of metalayer `name`, if the frame has it. + fn metalayer(&self, name: &[u8]) -> Result, FormatError> { + let h = &self.buf[..self.header_len]; + // [FRAME_IDX_SIZE] is the u16 index size, then a map16. + let mut p = FRAME_IDX_SIZE + 2; + if h.get(p) != Some(&0xde) { + return Err(err("bad metalayer index")); + } + let count = u16::from_be_bytes(be(h, p + 1)?) as usize; + p += 3; + if count > 16 { + return Err(err("too many metalayers")); + } + for _ in 0..count { + let tag = *h.get(p).ok_or_else(|| err("truncated metalayer index"))?; + if tag & 0xe0 != 0xa0 { + return Err(err("bad metalayer name")); + } + let len = (tag & 0x1f) as usize; + let key = h + .get(p + 1..p + 1 + len) + .ok_or_else(|| err("truncated metalayer index"))?; + p += 1 + len; + if h.get(p) != Some(&0xd2) { + return Err(err("bad metalayer offset")); + } + let off = i32::from_be_bytes(be(h, p + 1)?); + p += 5; + if key != name { + continue; + } + if off < 0 || off as usize >= h.len() { + return Err(err("metalayer offset out of range")); + } + let off = off as usize; + if h.get(off) != Some(&0xc6) { + return Err(err("bad metalayer content")); + } + let clen = u32::from_be_bytes(be(h, off + 1)?) as usize; + return h + .get(off + 5..) + .and_then(|c| c.get(..clen)) + .map(Some) + .ok_or_else(|| err("metalayer runs past the header")); + } + Ok(None) + } +} + +/// The B2ND (or Caterva) metalayer: shape, chunk shape, block shape. +#[derive(Debug, PartialEq, Eq)] +struct NdMeta { + shape: Vec, + chunkshape: Vec, + blockshape: Vec, +} + +/// A msgpack array header: fixarray or array16. +fn array_len(m: &[u8], p: &mut usize) -> Result { + let t = *m.get(*p).ok_or_else(|| err("truncated b2nd metalayer"))?; + if t & 0xf0 == 0x90 { + *p += 1; + Ok((t & 0x0f) as usize) + } else if t == 0xdc { + let n = u16::from_be_bytes(be(m, *p + 1)?) as usize; + *p += 3; + Ok(n) + } else { + Err(err("bad b2nd metalayer")) + } +} + +fn parse_nd(m: &[u8]) -> Result { + let bad = || err("bad b2nd metalayer"); + let mut p = 0; + if array_len(m, &mut p)? < 5 { + return Err(bad()); + } + // Version, then the rank: positive fixints. + let version = *m.get(p).ok_or_else(bad)?; + let ndim = *m.get(p + 1).ok_or_else(bad)? as usize; + if version > 0x7f || ndim == 0 || ndim > B2ND_MAX_DIM { + return Err(err(&format!("unsupported b2nd rank {ndim}"))); + } + p += 2; + let mut read = |wide: bool| -> Result, FormatError> { + if array_len(m, &mut p)? != ndim { + return Err(bad()); + } + (0..ndim) + .map(|_| { + let v = if wide { + if m.get(p) != Some(&0xd3) { + return Err(bad()); + } + let v = i64::from_be_bytes(be(m, p + 1)?); + p += 9; + v + } else { + if m.get(p) != Some(&0xd2) { + return Err(bad()); + } + let v = i32::from_be_bytes(be(m, p + 1)?) as i64; + p += 5; + v + }; + usize::try_from(v).map_err(|_| bad()) + }) + .collect() + }; + let shape = read(true)?; + let chunkshape = read(false)?; + let blockshape = read(false)?; + Ok(NdMeta { + shape, + chunkshape, + blockshape, + }) +} + +/// Decode a frame as the HDF5 filter does: a B2ND array into C order, or +/// else the frame's first chunk. `cd_shape` is the chunk shape recorded in +/// the filter's `cd_values` (if any), which the array's shape must match. +fn decode_frame( + input: &[u8], + limit: usize, + cd_shape: Option<&[usize]>, +) -> Result, FormatError> { + let frame = parse_frame(input)?; + let meta = match frame.metalayer(b"b2nd")? { + Some(m) => Some(m), + None => frame.metalayer(b"caterva")?, + }; + let Some(meta) = meta else { + return frame.chunk(0, limit); + }; + let nd = parse_nd(meta)?; + if let Some(cd) = cd_shape + && cd != nd.shape.as_slice() + { + return Err(err(&format!( + "array shape {:?} is not the chunk shape {cd:?}", + nd.shape + ))); + } + reassemble(&frame, &nd, limit) +} + +/// Gather a B2ND array's blocks into one C-order buffer. +fn reassemble(frame: &Frame<'_>, nd: &NdMeta, limit: usize) -> Result, FormatError> { + let ts = frame.typesize; + let ndim = nd.shape.len(); + let too_big = || err("array exceeds the chunk size"); + let mut total = ts; + let mut ext = vec![0usize; ndim]; + let mut grid = vec![0usize; ndim]; + let mut ext_bytes = ts; + for i in 0..ndim { + let (s, c, b) = (nd.shape[i], nd.chunkshape[i], nd.blockshape[i]); + if c == 0 || b == 0 || b > c { + return Err(err("bad b2nd chunk or block shape")); + } + total = total.checked_mul(s).ok_or_else(too_big)?; + ext[i] = c.div_ceil(b) * b; + ext_bytes = ext_bytes.checked_mul(ext[i]).ok_or_else(too_big)?; + grid[i] = s.div_ceil(c); + } + if total > limit { + return Err(too_big()); + } + let mut out = vec![0u8; total]; + if total == 0 { + return Ok(out); + } + // Padding to whole blocks grows a chunk by less than 2x per dimension; + // hdf5-blosc2's block shapes keep it far below 16x (a crafted frame + // could otherwise make each chunk enormous). + if ext_bytes > limit.saturating_mul(16) { + return Err(too_big()); + } + let nchunks: usize = grid.iter().product(); + if nchunks != frame.nchunks { + return Err(err("chunk count does not match the b2nd shape")); + } + let blocks_in_chunk: Vec = (0..ndim).map(|i| ext[i] / nd.blockshape[i]).collect(); + let nblocks: usize = blocks_in_chunk.iter().product(); + let block_items: usize = nd.blockshape.iter().product(); + let block_bytes = block_items * ts; + // Strides (in elements) of the output array and of a block. + let mut out_stride = vec![1usize; ndim]; + let mut blk_stride = vec![1usize; ndim]; + for i in (0..ndim.saturating_sub(1)).rev() { + out_stride[i] = out_stride[i + 1] * nd.shape[i + 1]; + blk_stride[i] = blk_stride[i + 1] * nd.blockshape[i + 1]; + } + let mut cidx = vec![0usize; ndim]; + let mut bidx = vec![0usize; ndim]; + let mut gstart = vec![0usize; ndim]; + let mut valid = vec![0usize; ndim]; + let mut pos = vec![0usize; ndim]; + for n in 0..nchunks { + unravel(n, &grid, &mut cidx); + let data = frame.chunk(n, ext_bytes)?; + if data.len() != ext_bytes { + return Err(err("chunk size does not match the b2nd shape")); + } + for b in 0..nblocks { + unravel(b, &blocks_in_chunk, &mut bidx); + let mut empty = false; + for i in 0..ndim { + let in_chunk = bidx[i] * nd.blockshape[i]; + gstart[i] = cidx[i] * nd.chunkshape[i] + in_chunk; + let lim_chunk = nd.chunkshape[i].saturating_sub(in_chunk); + let lim_shape = nd.shape[i].saturating_sub(gstart[i]); + valid[i] = nd.blockshape[i].min(lim_chunk).min(lim_shape); + empty |= valid[i] == 0; + } + if empty { + continue; + } + let block = &data[b * block_bytes..(b + 1) * block_bytes]; + // Copy row by row along the last dimension. + let row = valid[ndim - 1] * ts; + let rows: usize = valid[..ndim - 1].iter().product(); + for r in 0..rows { + unravel(r, &valid[..ndim - 1], &mut pos[..ndim - 1]); + let mut src = 0; + let mut dst = gstart[ndim - 1]; + for i in 0..ndim - 1 { + src += pos[i] * blk_stride[i]; + dst += (gstart[i] + pos[i]) * out_stride[i]; + } + out[dst * ts..dst * ts + row].copy_from_slice(&block[src * ts..src * ts + row]); + } + } + } + Ok(out) +} + +/// C-order multi-index of `n` in a grid of `dims`. +fn unravel(mut n: usize, dims: &[usize], idx: &mut [usize]) { + for i in (0..dims.len()).rev() { + idx[i] = n % dims[i]; + n /= dims[i]; + } +} + +/// Decompress one HDF5 chunk written by the Blosc2 filter (a Blosc2 frame) +/// into its data, in C order; at most `limit` bytes. +pub fn blosc2_decompress(input: &[u8], limit: usize) -> Result, FormatError> { + decode_frame(input, limit, None) +} + +/// The filter's decoder. +pub(crate) fn blosc2_decode(input: &[u8], ctx: &FilterContext<'_>) -> Result, FormatError> { + let cd = ctx.client_data(); + // cd_values[7] is the chunk rank for B2ND (> 1), and the chunk + // dimensions follow. + let cd_shape: Option> = if cd.len() >= 8 { + let rank = cd[7] as usize; + if rank < 2 || cd.len() < 8 + rank { + return Err(err("bad chunk rank in the filter parameters")); + } + Some(cd[8..8 + rank].iter().map(|&d| d as usize).collect()) + } else { + None + }; + let out = decode_frame(input, ctx.output_limit(), cd_shape.as_deref())?; + if out.is_empty() && ctx.max_output != 0 { + return Err(err("empty frame for a non-empty chunk")); + } + Ok(out) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::filter_pipeline::{FILTER_BLOSC2, FilterDescription}; + use std::path::PathBuf; + + fn fixtures() -> PathBuf { + PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("tests/fixtures/blosc2") + } + + /// Every fixture frame (see `tests/fixtures/blosc2/generate.py`), + /// sorted by name: (name, frame, expected output or error word). + #[allow(clippy::type_complexity)] + fn cases() -> Vec<(String, Vec, Result, String>)> { + let mut v = Vec::new(); + for e in std::fs::read_dir(fixtures()).unwrap() { + let p = e.unwrap().path(); + if p.extension().is_none_or(|x| x != "b2f") { + continue; + } + let name = p.file_stem().unwrap().to_string_lossy().into_owned(); + let frame = std::fs::read(&p).unwrap(); + let want = match std::fs::read(p.with_extension("out")) { + Ok(out) => Ok(out), + Err(_) => Err(std::fs::read_to_string(p.with_extension("err")).unwrap()), + }; + v.push((name, frame, want)); + } + v.sort_by(|a, b| a.0.cmp(&b.0)); + assert!(v.len() >= 20, "fixtures missing"); + v + } + + #[test] + fn fixture_frames_decode_exactly() { + for (name, frame, want) in cases() { + let got = blosc2_decompress(&frame, 1 << 20); + match want { + Ok(want) => { + let got = got.unwrap_or_else(|e| panic!("{name}: {e}")); + assert!(got == want, "{name}: decoded data differs"); + } + Err(word) => { + let e = got.expect_err(&name).to_string(); + assert!(e.contains(word.trim()), "{name}: {e}"); + } + } + } + } + + /// The output limit is enforced for every kind of frame, including the + /// padded chunks of B2ND arrays and special chunks. + #[test] + fn output_limit_is_enforced() { + for (name, frame, want) in cases() { + let Ok(want) = want else { continue }; + assert!( + blosc2_decompress(&frame, want.len() - 1).is_err(), + "{name}: decoded past the limit" + ); + assert_eq!( + blosc2_decompress(&frame, want.len()).unwrap(), + want, + "{name}" + ); + } + } + + fn desc(cd: Vec) -> FilterDescription { + FilterDescription { + filter_id: FILTER_BLOSC2, + name: None, + flags: 0, + client_data: cd, + } + } + + /// As the HDF5 filter: the B2ND array must have the chunk shape the + /// filter parameters record. + #[test] + fn b2nd_shape_must_match_the_filter_parameters() { + let frame = std::fs::read(fixtures().join("b2nd_2d.b2f")).unwrap(); + let want = std::fs::read(fixtures().join("b2nd_2d.out")).unwrap(); + let decode = |cd: Vec| { + let f = desc(cd); + blosc2_decode( + &frame, + &FilterContext { + filter: &f, + element_size: 4, + max_output: want.len(), + }, + ) + }; + assert_eq!(decode(vec![1, 0, 4, 0, 5, 1, 1, 2, 37, 29]).unwrap(), want); + assert_eq!(decode(vec![1, 0, 4, 0, 5, 1, 1]).unwrap(), want); + assert!(decode(vec![1, 0, 4, 0, 5, 1, 1, 2, 29, 37]).is_err()); + assert!(decode(vec![1, 0, 4, 0, 5, 1, 1, 3, 37, 29, 1]).is_err()); + assert!(decode(vec![1, 0, 4, 0, 5, 1, 1, 2, 37]).is_err()); + } + + /// A 32-byte extended header. + fn ext_header(flags: u8, ts: u8, nbytes: i32, blocksize: i32, cbytes: i32) -> Vec { + let mut h = vec![5u8, 1, flags | FLAG_SHUFFLE | FLAG_BITSHUFFLE, ts]; + for v in [nbytes, blocksize, cbytes] { + h.extend_from_slice(&v.to_le_bytes()); + } + h.resize(EXT_HEADER, 0); + h + } + + #[test] + fn special_chunks() { + // Zeros, uninitialised (read as zeros), NaN, one repeated value. + let mut z = ext_header(0, 4, 64, 64, 32); + z[31] = SPECIAL_ZERO << 4; + assert_eq!(blosc2_decompress_chunk(&z, 64).unwrap(), vec![0; 64]); + z[31] = SPECIAL_UNINIT << 4; + assert_eq!(blosc2_decompress_chunk(&z, 64).unwrap(), vec![0; 64]); + z[31] = SPECIAL_NAN << 4; + let nan = blosc2_decompress_chunk(&z, 64).unwrap(); + assert!( + nan.chunks(4) + .all(|c| f32::from_le_bytes(c.try_into().unwrap()).is_nan()) + ); + z[3] = 2; + assert!(blosc2_decompress_chunk(&z, 64).is_err(), "NaN of i16"); + let mut v = ext_header(0, 2, 12, 12, 35); + v[31] = SPECIAL_VALUE << 4; + v.extend_from_slice(&[1, 2, 3]); + assert_eq!( + blosc2_decompress_chunk(&v, 12).unwrap(), + [1, 2, 3].repeat(4) + ); + // The value must divide the chunk (and its blocks). + v[4] = 13; + assert!(blosc2_decompress_chunk(&v, 13).is_err()); + v[4] = 12; + v[8] = 4; + assert!(blosc2_decompress_chunk(&v, 12).is_err()); + // Special types 5 to 7 are reserved. + let mut r = ext_header(0, 4, 64, 64, 32); + r[31] = 5 << 4; + assert!(blosc2_decompress_chunk(&r, 64).is_err()); + } + + /// A regular chunk: one block of two streams (split, type size 2): a + /// run of 0x07 and a stream of zeros, then the byte shuffle undone. + #[test] + fn stream_runs_and_zero_streams() { + let mut c = ext_header(0, 2, 8, 8, 0); + c[16 + 5] = FILTER_SHUFFLE; + c.extend_from_slice(&36i32.to_le_bytes()); + c.extend_from_slice(&(-7i32).to_le_bytes()); + c.push(1); + c.extend_from_slice(&0i32.to_le_bytes()); + let n = c.len() as i32; + c[12..16].copy_from_slice(&n.to_le_bytes()); + assert_eq!( + blosc2_decompress_chunk(&c, 8).unwrap(), + [7, 0, 7, 0, 7, 0, 7, 0] + ); + // A token without the run bit is reserved. + let mut bad = c.clone(); + bad[40] = 2; + assert!(blosc2_decompress_chunk(&bad, 8).is_err()); + // Unsupported header features are errors, not data. + for (at, bit) in [(31, B2_USEDICT), (31, B2_LAZY), (30, VL_BLOCKS)] { + let mut x = c.clone(); + x[at] |= bit; + assert!( + blosc2_decompress_chunk(&x, 8).is_err(), + "byte {at} bit {bit}" + ); + } + let mut udf = c.clone(); + udf[16] = 35; + assert!(blosc2_decompress_chunk(&udf, 8).is_err()); + let mut udc = c.clone(); + udc[2] |= 6 << 5; + assert!(blosc2_decompress_chunk(&udc, 8).is_err()); + assert!(blosc2_decompress_chunk(&c, 7).is_err(), "limit"); + } + + /// Random and mutated frames: errors are fine, panics are not, and no + /// output past the limit. + #[test] + fn fuzzed_frames_never_panic() { + let limit = 20_000; + let seeds: Vec> = cases() + .into_iter() + .filter(|(_, f, w)| f.len() < 9000 && w.as_ref().is_ok_and(|w| w.len() <= limit)) + .map(|(_, f, _)| f) + .collect(); + assert!(seeds.len() >= 15); + crate::test_fuzz::fuzz_decoder(0xb2, &seeds, 20_000, limit, |s| { + blosc2_decompress(s, limit) + }); + } + + /// Blosc2 chunks on their own, mutated. + #[test] + fn fuzzed_chunks_never_panic() { + let mut seeds = Vec::new(); + for (_, f, w) in cases() { + if w.is_err() { + continue; + } + let frame = parse_frame(&f).unwrap(); + let raw: [u8; 8] = frame.offsets[..8].try_into().unwrap(); + let off = i64::from_le_bytes(raw); + if off >= 0 { + let start = frame.header_len + off as usize; + let end = frame.header_len + frame.cbytes; + let h = read_header(&f[start..end]).unwrap(); + seeds.push(f[start..start + h.cbytes].to_vec()); + } + } + assert!(seeds.len() >= 10); + crate::test_fuzz::fuzz_decoder(0xb3, &seeds, 20_000, 1 << 16, |s| { + blosc2_decompress_chunk(s, 1 << 16) + }); + } +} diff --git a/crates/clawhdf5-format/src/lib.rs b/crates/clawhdf5-format/src/lib.rs index 3e30f9d..89d324b 100644 --- a/crates/clawhdf5-format/src/lib.rs +++ b/crates/clawhdf5-format/src/lib.rs @@ -86,6 +86,8 @@ pub mod filters; mod filters_bitshuffle; #[cfg(feature = "blosc")] pub mod filters_blosc; +#[cfg(feature = "blosc2")] +pub mod filters_blosc2; #[cfg(feature = "bzip2")] mod filters_bzip2; #[cfg(feature = "lzf")] diff --git a/crates/clawhdf5-format/tests/fixtures/blosc2/always_split.b2f b/crates/clawhdf5-format/tests/fixtures/blosc2/always_split.b2f new file mode 100644 index 0000000000000000000000000000000000000000..db6322af43cc9a0dd615133bf4c3e1376a9f98fb GIT binary patch literal 1650 zcmZ8hYgkQb6n?*bI;T?(ozAHws#A(oq>FM1yF?VCLdK;+kwhiQFlI9mCheqVav6?$ zO-(Y4X+k!d+)0c}OnQtv(;Vb7G}El`V;<{yzV+;HueI0vzU%w;+EO&lD?T+gDGqf2 zVlR>-Q>5x%b@jQfyJe*JkEHvqQ)IdeAB zn*nXBM*cD3f0<4nU)^JXJs7lVfq_%P0&PnLaM&k6lE8!KC8;wE^nt(&v=t->;3Xxt zNjwCJq7qyYAV>ml3RpwO^Kz)AFqo}_C6^Ai6k4jP74&UY&82A24*784c@xA6R37{x znDa|1=1pHqZctNhAR!4hh^Dd<5wzk~z!Iy#c{8JYN(cc~x0T31Z;2OD2vE%&1xdlU za;ha~grO4zC+5K89GERUc$6|L=D>_|AS+>3sA9;3gCw;8DrPRU!&uS>>|`3jpJ6jF z3ao%i-DIHtiSU3>%q-;!$SfE$f~=a!f;lW>o;;5Xrjce4sUpVhW)2VvN$n}76Zle4 zfMYtQ7Obhydzi*3k11R^E8!}~anSHtj0xaQapqEA4oarsi-BCyMfAH0{3gaFaaySh zvzJPkg)0O#Q{=&@FEf{{8C!xObOL-W=Se40#;JK*#-cdp!*S^{wUkEd07;_rA%?;- z8DJ9ig>Box@}i+Y59SJl~u=n zJ$v~_m%?l(9%EOOF}*{KVKud>#xU6)_D zY3ugg-~Uj)zoN40xQOQ_m#@_~+`0SciQ&bozyASeq%byVr|O{A*fB>toe0=5Pj4ST zA{0Q7@{rI`VPnH1$48O)Xc8YkDd)xu=YDK2d3z`TDY|bM^O|WlD1^ zTPHWK0eo=S_$hITsTp5pX06FtSFmBz)*X9@#ets>SDg?cUAcC%;m*BBPyc%H`ke@^ zoU~h5cI?zyL`ruTH}_t>z3iBi{~!?sqr=8V5d3G8Md*^|k$_mR@3Of?lN8{hZ`~E#?&zZr1PEQ&P6?r;0;IkJp^OP{K{0yF*obixNdDBA*y(WuoR~B(g#w)xH74sHD&2=q7P2=MRaDnCH2tek zX*=t>d-(bXg+@fh#wVpOUbbQt2{YUH-L{=Yr37DjSfU@{LLI?35`5G1<~J>^XE<-A zsY-;#Mnp`v?ye+%qRfV_USv2yPZXhVyYKVfUp#jjq6y#w0s;aV?1wNo+6OUsAqVyW zIp}RB;-r0`h}Kr!n4mV3Gh-#q%C>n47h3O)CibV6&uH>1K1}`#6zqEa(+vCv_s$vv literal 0 HcmV?d00001 diff --git a/crates/clawhdf5-format/tests/fixtures/blosc2/always_split.out b/crates/clawhdf5-format/tests/fixtures/blosc2/always_split.out new file mode 100644 index 0000000000000000000000000000000000000000..a3faac59e55caf561a9670a4b6b8ef352bd0187b GIT binary patch literal 4000 zcmZwK3(!?n8He$OC?TPQ7x0Dw333Yr2{cd_@ChiC6eyUepn`~4C?Hf*t;`IKipaa&pIi^zxJHc zya_37LOZ%oOlc6>un1RS4Q|3lY{7ka80~l#yRqM-ln%t9I0F4K1S2pOC!-D1F%#$D z0$hTnSb=xqDqJ%um20sEYw;0$+H68-Hf|&kNMl!hWpJw#831) z@UZ?d_19`Uevh5zKdDcv&!~S_U%+mCYD={fI=A^hwx#?sb$_)Ry4xRs9&M@g#KEo| zir)Ie)FW_|zK?rfsrFM}Wj8<_XddL*!RBH5SF0oJN7{{2N863jkF|fTI?nDS{VDd@ zJki{yo{A|r&9!OjboC6^W~gseXSx4O``PN*=5y@NHJ_)>Ghd)yWL}`ZSbvH87VDSb z%~)!8h21j!3in;9UuBng)0Zzx8+Uf9hwxw+rXt+A8rNez?!Z?301x2_>_YO6yP^kr z;VAUSV2nh#a|))Q<<4CFg;chdLpef@*#c9lDcJLOUJag{rTJLO3;cM5mX-&F3Tf2n)9Q#!a)xl_244)i{` zQ@NA5letrSfjgNynLFudGk0n~Gj}p~%0QJnd5HZ`_YBu_r*J17kJsqAQ%W zPQ4u5Dcs2`^&Q<=&6zFO)wna753{fY*I+Gf!6w{|2k;1XB66qYPUKE+^uYi`?i`O3 z(Q@Yu{VYW8T!e7v3amuId+!)s7CJWQp26+@lX6`QmV=O+Xwrii}xmX^8R?ao;#U4 z`4u?GyKUTw+zEGT<4)mD>TB-j9_|$G)PZW`4tMe}Rqhn-)W)4EchXqbxYNp=DtC(9 z$rJ24xRbe)Pxox&PUKE{gPA*pJNYck(aW9EdAR;KjIx(IS?*N1Q@K+kcgDMRvOd1E zCg>Y?>SR0aq{f}}de<9w(o8*fDt8KZ(wi{X>^m#miQFmNDRQUfPUTMFPLVt5E$-RN zowUleaA!Stj^oUFY)4lfggfIg3rn#YH{v$jg>Yv(!kwq^JofRfyWt@8#?csn;TVna zn26I6?##ydScsN8Z_~@2B6q@_^nQE@*MmDn?vzjI*Q4dm?fN_Hzl^UTa_5_N-%{mH zxet*$KUU*Ai#zEN{0hG@NA7SZJ*9>_awk0p?v(h>dQsoWyW&ojJGqN@)>Yr+PCmfg z13m4ylNxtwPUcP;?HTSw?xYj#8+S5y z9CEx%*SS+pRVRZxkvn<1Yuu^asdA@qC!eV{?qu#%xfAXb?xgv+P|uybP+eqR44L!) z+{tn$+^H>hB6nKubmze!WX#2KtUh>rcAy zFbDH+F_vH%-tMz{x7y@RT4Qe9DIe2E?r^7Yr*NmpotMtZU-8^#^EcFcRPOA_9qzz}#m~);s*OA8cjhO}+(~~?pHaC}TDkMzNhx>ouDDYgcXH!SB!81Txp618>A91p zsHdsZT<1>aPMV?TPHx;uv;8f(6YfOrr1Kzi>U{TcCoRw~1a~rbs@$n^r^uarx$DcY z!ralFcDb{(aVO5nkvqdM72!@nxU&u$umvr59@mFEFQSWg-W}o2k?4zKF#_Svc(mM^ zsSkJNBXZ|5T#3e=@?M|aYIEaGkvn_bS!efY{mr;lzX2O@rpeq*I)?ZcfH^t-{G+R6Lt?A`4LXPwHO)D7LeOYX!uIm?|m zCs*#|#+^7PSGkirxKkT<;x|^6J86i&A$Q_8R^?6_>3+FWxRbe)PS8j0a3_9a)yADX z(afF5ozl3I8+QtK3U}h1+{&GBC!g(k?v%!z+_+P@lN)#9H&(dQ`i)h&6X)d2o%oHF KR(gI_+y4N4t(Zyx literal 0 HcmV?d00001 diff --git a/crates/clawhdf5-format/tests/fixtures/blosc2/b2nd_2d.b2f b/crates/clawhdf5-format/tests/fixtures/blosc2/b2nd_2d.b2f new file mode 100644 index 0000000000000000000000000000000000000000..a092e297936a93a2a0d0ef8263ff8a8e5adc56ab GIT binary patch literal 3950 zcma)9dtA-g7GJ+}oQiWery`}7N`;b2RIc(mxz&WoBV{;6nJ9xO-Dr9sbQ3)=k}yRj z6iLER9+L<~gmimgG8Z-8Gn#SN{{7B5bM_g3-2M5i-`?kU*4lgRwbpm7_1jkv>=?Q# zC?W(J03iPsK5&Sa=jp&~-W~3jA;=f~rVyrHOi{% zn7m-e<%~jvUPj3c5igP#8dR(8txddkmr29zD_XoOqU4JCR-VU_s?>h@8wqJ zOlGN6?kRxpW!N5I#cHCtUs53+%#Up|YtIf84y$k+sAOCNU_S=*F;4luJV*+b(7M*t z-F@7_D=a;8@u#||^@&^3caQg+I(z-RHs z@p^uiXKSY9x&p9PsB19T%I<#46JBp^a=ZjGCfRieX}7m`c76D>S8>_ds+wDJw{LeG zh%u<@Oa2$o9F1oYU+zMm-9{?Hbt{qyuF-Q4jznA7d-~jkq2anl=EEIaT7G`@<`ciD z^vs2;bqWg`oG#W=i{rsoBSwv#^vmmyp9P|_^OtIGKlpA_Y8J+x0>zQGeZ`J=Di<+R z8Tc3#yCG%AZvO>K!q==DV&mXE!QG_FSHoC&Jdq0nH%uH)1R@=M)0$iNA9uXS_ zzhwGZ)AS0;X&(D4NjdMCq{M%>2OcW+WSNCiFrXwRz$D2v>D|(!SmM-pEZO! zvJwl>G$Q?^XK#-<*fXNUW5;|S) z_vIfctE?slU=2(P9xW0Yu{t(si=*pAtV=z;MA`;sH8&d{cl=i_B4wkF_G1{{Yg3BP z%s|;U-|CqTv2}87@BH<{9|DCRrStDetMy$_W6t;3R7iR0yAWUyxHxPTfR&w-8^CJ} za#d7dI)wJ%Jbx$tR65wwv-cMsKOGg9v^8VTBro5&3l?J%%;);ojov-zkKuifr8TV3s)&NDhStt zrbG{*?(V}jwlt&OI(d>Ous}5cuAwv8p%wrN5I5qVLga}s}s<^E3QceBZ zxJ{`UdnbF%@Lw3(SI@}8#*vL}Qk%a7=6jEyzI^-XKp|H3%Mq*BC2dY~cK4X(7oa9Y zUELGl`p!ipMId5YHCH4VxAlM4JX3$D4vhtDaWwkrp1)k%aK9xvEi32X5kCaR@+h`_ zQ?z{|bG4u}vKVIX?EaM0quwPI7k|3ZxIQUm$FBX#I3_iQ&H*dH@2?zShQ{JzkK_;& z%i#`V$9HOw4uS)1mLnEj%153aAZb2C_Gc5t7Llf*#Ous5My-B(PMtM>@v`Aq!lw9e zTe0a#)se~Q6n3aNawfh{!arUkRUwh~nxH!tiL~J~ECUxUT@f8;W$T0uWA}$Y1!zN3 zYFp(NkedAyivdb0hg$R>!=@bfPT0H+>w%BoJp4Y;Gr`6PhrCHwbcuqMbrW<);69we zEEy=^OrDwjLs7|TSx5vndcbEiuxiO5x+{d&{5a8`7TBDYm7Q0#bY)Cj@-`>z*L>&w zO~6KZ!T<(pn%kM)02i**Aw;+A*qw{YaJpefG%g_Rfesbc-5dX%i8;X6pVSK@i{bWT z+<$)gwgDUv6U%8{spJFnL_wLa_0 z%E>=+q7t)RRNRIUqg*F?`I3WG+Ez-a``st(Z49ce*4=G>l9Imrz~SR_z79d-kWUp> zTSVat9_2k6TZ7?wdHk2KMKvV_k^$HekHX&K)rX!#i2Ukn_+5hr9BbYjsZX<#ua((f z^ZRJ)n_6SF?|AV>&Z^VpA~MEnbI#Al#&{9feI%cVe9f8dDU*F%Lledbn1Naokcp5C zi5&|2Dt_av_`DP*BFf5@O}TPzlpN(6cQiMNu(gj@GoKl=e>~>*`uzU-&gcD{*StTeQ&LjWlZhPUq9Da6Nja+UIQ3~l z3!bG5J?O_^MlgoSOk)lUS;-nU@(G`_mm{3wJU?)qo1{&Zl5!_mxR?7WLUA6Y64j|g zLz>ctjyz8<1~8P-yulP^GM^=Uz*;u5lP@{QG0t+4EBr#L)ZP)9$WCqwQjAiRqbfD2 zM`NC*Ezi-NJ`ChF#xRMunZrU>u$uL3<8$_Mh!dRWGS|6D>NMUFnaN3B3R9fYRHPbp zs83T`(}Aw^(4Cb+zm8@YCJK4?Ge8Xw{jFb%AMYNNfLPR^Is6cgUQIBY+ zB^`L4Ui4)!(au;VF@w1*BHH;L5kxx^na*q$k+ieH{3%I0N6e>4+PPu=o!gvUw3Cf!Cm%^WWz3bR zLA2A5=CmQ&d4XQ^XBeZowVk=1m$Hg=Y$4j&!$FR720!D^cCz?;5BVs}LzE_Ir;a)H zPE%Ubk!a^7V($zg+8Iaeo$0*GQr;)_&SrM9hXWkrG||o#ekR(9y%X(ZBNx$5QA$#d z*gG|;OJkbTmT2ci`Vj5B#_LSvEoQTjWvpTY+eq3uY(CC+e9u*)oitujv~w4KCE6)O zv{RaBrz+7-1Cn++m}BomJNWj&!3p(ax*9&UmIWlh`}U_<*%UJG+Q>4v@5y)>&mFE749~ zicp*~RHQo5PD7f}icUm3FEfClj3(N7lbOtC8LL>w7Ls-jn!n*R7rDx>q&Tbe+(iyz z?-ZmM(N0;aa%(#+JU>epx^rtgV>~DA%rZwiD~P?bo==E&_Hl%hoG13qPuwKhxr5B) zBripHkZ7kO)u>H?QWjaiX2? ziM@(9sRb!yXqrnDm3=}Hg!@d_gt$7E(Ohb6?`iFP)#gD=?6 zF-{SC=SObvU(!0SJIO}0lb^rwFlC8$YVrh)Xii%?@dCXWNVM}h<9U-=%;!B~@2qDF zpYat3IZo`IOI#&s=XPfo_vC0N4}~a7Nh(lqCC)#O7w9|oTrx*Ph!brxFv@_2f z?YvKXV{KtOyZM@E=L{G4foSJLIl*^CJJ+~LD(9AgTiYq@`5{VEo@&&l0Zn*@4m?K>`Z1W1jAJs< z&RiC=g80VT%nrU_KgT$Q*Yf8*IkUfale81xSfwdP6{4N`JjFA#BiiXf+>@i75sW2i z=Uww+R`3y<_>|r3<0vP&KL z*FL|j-_%exYst)oO*jZ(`+Iyaowj9T4_p!b(D_xYlP~>mkwP(fjNl`xJ^(&IWapu5 zyI`b)E#&dQyXbxdk&O*y*EdS>`G*nNLpgR0vK~m*hEDEFXZNM!1E~f&$Xk~EJ4;@p z_5+{E*9&Hi>JWJ_rlRbJR7z`&w`NVc~Lp5rU;Gf7a9VknS z(yoK2_U+m0v;ph4L^`^yh#bni;+IQ*iI4aZ*Z`<0{7_oMM}iQUh4PoSHjd9;80QbC z=o`RPK!1BY8k7FUp!3wxrWslFkT#OvCVSxlP@qpFoN;-`i~*`Jy` znnE7f$;YE+eiZHqm*V8i6e5phKcL)9tRTK8&gHi-Ln%uss$x^y%102%?W*&}H&&5H zVu(u0P#qdoQXV$ZyWRM!nWn9_c_c;-)3Fu!=!gpRLovP~|7zOdo=S4B?Z3sP1$_pe zzV9#jMJD$`+p(OMrp*LKmH`({XrC0EqZPmR@I*4h^lT+Q78BHk{$%SHOp`a%~@$Hci{NYVMxwR$D#!2$U|w5Xa+g-208JMy4_HdH1fYVG_!T z4C;u9sHLH|ys&F;?d0j#KKb;kZ)DQZY(Nww-Hu0aJ}$;0Qo;>fk^Xk1v?ACwyo7g1 zqIx`sx7yXQG&P9NzX0aU0}g(iIdECjrkBVsi3P+bgh`Y^{B6$iT9=Qitm49vNFavz z{FZyhy_Ic6U1ZLk< zI}h46x(rvdkt|tE8j8`y(}b)Av#A`m7EB@~?39L5LMNe>tOY`+j|ibwPwJgb{KR*^ z+r-OGylULFV*L{rR$emhrrTDp$w&|JmsT9D#w)KLI9PXO!*!(A+nIFSRz$ACMp9)0 zsr6_^da$UID35*%T)!AN@pldyNIJiF?EDwmr@QU{Djgp zqs4rYzvD)s8kgZttoJy8*>>{lik4+-f-}b_n1 zCnzf@1E#X5ptWYRdhrb-^xwYz&oA>AF29*)fGRf*13FZJrId&TcYXRzr_Q~47mV7v zqnb$=Dl$yRvP6R>;k-J4$m!O84ALdG&T4#zToiEMzdyAHkUe8qJKEL&S8-qnAwgie z#jm58&jH!brIi2R+aexD65R4TryCV;qdtP4I2RcE^a0dz$g~@%7o#5f?D+R$!x(f+ z@9Jqe`y(YBwP!_9DwdtsBM|=mQpFyl+t|0F{<0F(U)TDtrt++95kIM{9Q4c(x+GQH z2z)bi({W1pBwP|s37>>xvIdlqLt7|iYw;t|D}+-@Q;92-;$zuz6dY@d#uhJ8Ey4_E zgHcuG_kqQ-nt!5m6L--(DB(uzr4QT_W>u)KE$uk&$*sugn%B4Ag66|V-B!}_kZnlX z!LtXCTX+aRq*_VGDR#|H<>vuIY0v&7R(prYqzYN&aIu8avg3ruFHpx2b;g@p>+I*k>b9sSX+LjAda{-cTx(tSij?Fx zvl?ocInDy!hOMsXi%aoyNiKHzYdGiu%aP{t-CQ2Tt<0k(%xpt-uzK#V%=?b z3@;sh{%AFCNv3u*%PK;W+~ze9dk8bA{C(5Rx05Ep$hH|Xfj4(%4qg_ui6-)KPJ@@A z4i93hXFP+M=lJS#nXxz&b;hu>&o!s=E``~_DdrP+?n zEUopE&%Cx{{Di4jJ-qJ4ZSA#VNn=c(60-?j0B2+tvzfGw&DhEP>D3JR?HT(f#g%fj zKI93taTf44xA^8#5@(H`Sz!2x(HA5g&3#1ZwV^7*uN=m>7YqYawYVn;HD>JPSDYkF z=&@9*88bqhZz@Y!+BrbU@%L+RMXM$I(^S7$RcP|q zg0||^qo&5SmERUqaKN;MN0nwQG|cE<40z~-ilMZr;gMTLF34JFn(2VsF@sU}@mno1 zhuBPPB_xC!iK&EyZ#{80v5!cX{6n}AN6aCJn!m^mP&}EXJT682BR$O+spOlZNf#e- zTk~f^oO_(;MiR78%H*_`aGfuX_j_~5@VaYdc}aQG-{*47;*#a*Gph+CzuJA+XsQjS zf)h8NJ9`f3=DQzzoU2Do4V()}d(Yu_u6}vDoI+E_N58pVH_T<4sw5;n7O@ZF_*l|B z;@rOze@Pp*bYKB}Z2+TkjZD;kVT!Xwz0KVW)se9>$A~N!? z6O|adE;8lm-1==r%ik`$rg`$M&mkJi&gq=nHLu61`2~fsK7IQY;jc3gW$D!s{?9y{ QtKVVn68U@3AwIn6|9EUxb^rhX literal 0 HcmV?d00001 diff --git a/crates/clawhdf5-format/tests/fixtures/blosc2/b2nd_3d.out b/crates/clawhdf5-format/tests/fixtures/blosc2/b2nd_3d.out new file mode 100644 index 0000000000000000000000000000000000000000..c29f5f0ad5461edad2d9de4271b870192dbecd8c GIT binary patch literal 2520 zcmZwJY0#Eq7{>84Ygx0Ugpj40C<$4+>V1psQKS;Bk|HY07p}2n3o-U3*%FgvEMqLW z(HN2tW5_aNYqHE}LYBe!UCsT>ydO04pW{5Q>wex3_w%3QDPsM(w#4@6h|cJO9_WL_G$ zR^BLYmbb|}kDZ+DY~RIw zC%L<`&gMPMd%Npm?=AAoH;} z9w+)+C&`oL5P7O+PQ&S*8;+3}<@_x7W6WbQ-u>BfqMRfryFU+8?H6DgW;nmd*`>&R zxl+5cNUlbE9`rA*agL%uIP#4PQm~jkD(ZW zF_?t&aS1L5ck1r2xJHghN5m)yy< zJMpEtxbv+$?gV#oapxDg4lUe?4ZN$3yf^NoP0_}CSl-!Ax zk~{IW{7!z4pRmTxoha`71-(=0o#Ia7&idZrPH?C6PI0Fyy;HkW)xA^Psq{{Cl(jpt zn|XI|CvzwEk+nOSJF%agJDEF)J8_`7g*%PjNwquCfTAi zJkOoXoj3#BiP1h=yOSok*E_j|J8`~yy%QJ8>F!JKa3`1EnaiETnb~*?f1u&cFisSA zO7Gl(xoGLm=l0e36&rfr4R_j`i#vOvJ9^S>HRg zJNb6|-OjlabMY`5?ksfvG)nK(?!>F+hC6yE-o*#bxD%h?Gjr`ua3|}X%$?FZ>)rX? zGsT^M!JWjN(mQDr+1mT#PTpK{CvzupC$_cMy;Ie_QYZHoPCC-govNQZ?xX?k21@RvW1N@XIYIi(D!r3VkwZPB zcZxe{xLxNY?&Q)tqa}ALy^|-{Cra*AdM7n{r*zK$xl^@p=Y8I6#DhK22SYF&GjSK@ zVG$}U$Cvm4>rmWjgKg0PolxA_52bgCJEeDuJA+Z&8HEX$f@vtd)6$(A>^J%DO7Gku jxfAzGy%XjCI*&o`WbOoa;yE*S;$>Ofc>|?)mNxwdF;+b& literal 0 HcmV?d00001 diff --git a/crates/clawhdf5-format/tests/fixtures/blosc2/b2nd_4d.b2f b/crates/clawhdf5-format/tests/fixtures/blosc2/b2nd_4d.b2f new file mode 100644 index 0000000000000000000000000000000000000000..6fa77def7053c5fa36e16e15ab53569008c522ee GIT binary patch literal 6804 zcmbVR30zjy693l7sA-fXTL=VNcU(%pqDA(WqdtB=^|wc(bEMA7oaocWkJKx+Zgwd_t! zEs~_A=X-QW-;-K8-J?T#+Gyz^9v#y2wz<*VZWfrQ%_6fJ!V!fD4o278)&e}HrKj5_ ztE;7Z{tX=>9B9n+#n%+CtS!F!aEzO{iYPJIl4Iidyh>b7{4$wAYV^A+a-p8UV4RvJOt*@^_*0N?X+_=AEPQlG<{SAm z7U{XiJcHM=q^JFZ81b|)GR_Bv0saU|$&VnK^+L+H_$16(AiYgu@pytTH-n^wW!^Zz z1?n?#CN=L|*?2=#>_39Cp@?aPjUS+}2tjw;tN{)c(VU((DdbiUtD#zrHL z30!FWO3wtT@l2bU>J7C@ty2fpQT4f+Xr`Jkm~WXm4D`bc^poabBs09`VIh`bD+7HW zwrMXEX&S5@VI|&X&7anEf%eNbO^YU zLoEY@@`GJyg|_JIaYhj?;3t$dDj3zY1waceo#@da!%A!ErXC&AQ%i^rHO-Jn$N}LV z9l{SYMj6i->Beiu3S)<{$2efLR&7-$HCQF9v1-0rq%v%Ch(z%)P3v5(=`l^?wA>_3 z-@GjyqWQe>y>Zznr>d&EJTmXAooX+O>_zqCZEFFpX&Y+WBtFypRa;N-U*hbwCwH3G z$@jM8j%U+NTu+qb2FFNpx93w5|5hL=kh>oj|AR_lczG#4+=H(!VQ~g_?L$B*JU0hx z-b4Ngot0KHTA653Uo!fp5q<8C7&c0J%2h_|w!+kJhTa2&K@jTb+AHHM9b|4+V@<_Z zGH&LJShoeo&!SgY(EpMvH94*?pWhJ0j?Qs?#!g7h z$jWJYkGp@;@LBWHo44tZFl1!fD@#g-RB6)k{wD?{JHJKBjS47-_Me2lx=EL;eR=t% z=vprN)*UC#ULdpigWadjUm`PWe)@*3vgOKLCPkfKXkeexKt{%KW#$0%Dg$T?@EW+m zVZ7cPYSLn0G{1%8EdRbphI(leQfAK!i`um-SX*^R#jxgW+9wVfdFp&XcvPbu5($XKOLi6aIlEGFXe&MAhn;3n-xGB>!-dyJ=X*KWk$@lcip9v_lUOwU^P(u&m zaqb7r><2DuhO5a=f_94L?1U@BAA!bLtLpgJxJ;fuEe9#M$6=%6Bg7S4AOs)QVHN>A z1Uv(j1!92&t=Eq@()Bz%wy$(kUr1z!8cz>7UwXac; zoVDmlfb5T&VJ6%rIrAunVyB3$te5eNWtY_ zj2e;k__Dd%@1?duii!17zI^)J1#fTK?lX=|p8~MmeI@#c|967Xmvv(z7=W&|s_n%h};3#uFtyzbD2K|G}}h zvlBa3vroobIB*!_Cn1w(mza=4sdqb$*>3SX-6uXs&D$MjB~y4Ebm@+9lR2vge*PWd zQRwohw`~QfB~0oACb^ZEz8=R?b^7vKgj7b$w)9bW3+3>46h(Njg)1fJnA3aFgOZW9 zkRr@=hg+jP2FMG0(#`c+WOJ*V>X#A!O_V$A3_jC{P06S8DWSJ)#p;g_=6`?X_u)@Z zT(o>;?k9)GOh{R}B75(rM;}joa>m^BcQ$Qzd=C!@+}3@YDpeE$xZ?pQUvR=Wpf87} zr6}i}$hH)@9OEYPyLE(1bW3CpCB_r~E{OfjTFE7jP%;G3Erf}vhps(^`R_HjY71le zMT#`RFHEIN;!miI+p-hqzCoEgF?#{t-G)b3eu)wELs;>oB%qS(}cI-LuRbVig>~))u7Mv!N^X}IC z6GddE%t(7{?PfAwePnH5>0*a31Z5moVv7FLK^tHQ|9%Fv1%`t+Q?Rz)r#l^$yV$cK z&l$4m03Yy%rI}pdqGnzOKGcl1N^^EXxAI5e;_LOlWt58d-Nam?tVSkLRw$demFN#L zZ@+<I^Heg~dp$pY{r+v4?~Ajj zez&~%Xg|KYEX>|K{BTv6#hKW1Kp5?hn^M)9kFTxy$k?N_xErEm=y$|s^yBM%x{2N> zoniTwJvW2Qf6XWrS-)Ag-j9EJtnlg|LE&{89{K$AZ$`<;22EG4S)Z3*7*e^0V`px` z?NV$u%c0-92NQJZo5!rw>Jxw$G&2x*Q8Nz%6Eq`}(H6@bt}}lGYVvb{j0@!n3>d`2 zEx<^R)gcwa-9fMKSyqr@Dzar2a!a}jc3bzAiPN56v04J-q3(UgO`c9Bu6O^5(^%bc zkGtc*VZ8KsDurYQ8@{~oOGLi%f_*KDtDvJe&`r!Gde=bKC!RQz_;*3vQK<&*?~LTp z`1%JqMBm#EmoB49O|)w;y{5P!1G{;1Xa?pnR= x;Q<$~J;8HXP>IrIfs=>>rhg5*-DgU zjBQ3`8;;N>HKL-L`~JMAbNUxP=X>r2?m{-?Mm`il2~Rm%*%o-#?pM4)%X}2vIX0*GrO}d z2Qh_X`4XpcHWzal-{b~vk^UgVejAAcg1L0Obb`BX&3RZfqnmg=jiTB(D& zs+am{h(>FIrs!{)r`NPnYqd!`v_}VZOrPkSF6*jp=$6vOJ&`?fN5Lo>WujtKkJ`~N znn$~MI=V;i7#PE2Y)p!&F)J3u(pVGgV_WQs594s0h%<2^h>XZ?o&0zRrL6NO>oiEy z>5e`afE0|u#6)Mlb=F`VwqPgr;V}MahA1C4eK;!OY3xD z59sb(3M2zwjwgeeUUc`MX@Lym7_*H7LBaa-a0+w zFEJ=mtn+eAi#gU=VVw=JHQuw%kvM6ci;>Pc_gbf5qEiXgP#X==9BuJ5dbpi|7=afs z$vU&Jz&fk3-a5PSp>C{VfI#{Qtb%t1H0;h0>bry4_+u6i- z`M%pZ%8z-Lmw44Ww;9&SrUz6&52>^&sH*Cyp`NhLpVdvh)nCIk#yV52voK9(i*{+B z4qNB6&g%%ACnWEC9yo-u+Dbt?2n_? zIcJ^p$ZVawD1>4ti^|rigT{Cg?a>+C(Kpc+HAANu0LMSNIk` zB|2G|llfVgC0Kz~Sd;bGl&$%cb$YQMhjO%aUg1p6=j;3j*K#v=ChZ)z&Ux#6lh#fq zWmg{kK}A(Y6;)le)j-YF#_ja5&OnXOSWU9dY%S7qtR0Pz zjhvA$3P*`37Y`@xG>KNxA-cMqzA-dL#ss%BJ?6%eSQTqyW9*1MaWIZq=WJYZJDHIK z52ooSLC-& zN$WfkwcJj#XdNBnndlY$V_1x`&XkxD^Q^Nn*2SjSX`KUcJU)%{aoO!;Np$|`cFG}D z=W#SaD|EC@Pq#A|BQXJ!F#~hm&MNC{ayuWmonzKHhfDa{I=8Ho(VLtr(J94B)~Vx7 zZpOB5=Z;Q_+j)_%auydRI_naheLTeD-sB6s!fU+AUzxheIrN})id(0OYFej>TB@D8 z=vh6l!5XP?dRfyo*X^vd&PHw5`_?(ClR9IauXNp;jL4Yi+-avmRP`p;OVjCQodMqD zF)`6Pv#j&F+j%=STW4P!ayzLyS0aOTa(R;rqhz8}3-!&(P_ zEWt`Y)<$f{9vpHzAA6H8;TwE!9Wj$PIgcN!s2{7cAFDPSusPeXlOL-O2XX{o;KW2{ z5tnie-{BTN);>R0YCEZqm8$bgqI19UdXr12taTn$JwH~<$aG*^FS1cBHrYRQ7!GU+D0e0)5njM5~Hp2O8h+*#NDV;hfJQPP8Ed&uhLG05>ILsE^S#yIkfl_k-V zM{%gpY+5P1UMIAY5|Y@GZM8jCPP=S-zo9v?+urm2U+25u|Nid%-Fv^BB6nl25I28M zxD5aWw{SvXn&&IS4zhjXqzscA1#?P?#EGEP6`&39A!<7nWQqi1LLcXoEik7Sh(fbzJ~4wp3^?Q-+_a|*u?al{E|y|x(D!uipF9CQwy67TXrr$2zr&v$F+nB z07eL#qyd8s*}?`RJFzo`Fd|x%n0ESX@kMD(Lvu&>BgQO_+9G|UFIKJHyzOf@Z~x@X z+;e4>S8JQv?mm1n{Q9lBj)Ad-wVmVkJs!Tn;YVVN%ca-qZ=yGYFGj}cipr*zwwow# z0>YxA#VMz9t~XKM;0skP9SW3}U+BT;*d&ZY$;GOghFc#&7%yxnw`+9XD#!0Y0*C$* zW1=)yO47I;h~}eG&ZE=Um0cNX)<8r0Lhda@Y~3rw1Yx%qQXp$8qh&( zYu4Ae*>P4bV~JdM1VBun01gs`hBBebt(U3V$W>bi;FC{Zy!jghQ896Vh!fJj%>z}< zC5BLNK~e=9c}{!bZg>APVCh{SAlR$NT<)1E2mgFzKcOqXWf&j`aerWr0bF$uHS_Hq z5*uNrm0a^Q0UR7MDp4O{{FHYU*3}disU3ir6AenYGtM zc>TKMDPfpqNddTY5P`@`Ak|%IA!K}5AM^jCKHRVt&SPEBaA2T-s+STR)3~+Y;=Iq( zKO|BpPR_{5zff_fXMkj}=OLfQme%%;J0pdnMC7x$0u!~Ry^BiJ`C3bi%&crN5nR0k zC%ILXdwv+1L>dU7Q`}mLTKuPYm3#&B0`_?Lg%17x<{gp|KZRU1`90|XJQ<->xPb%~ zFJXpcT7a#w)&Nlr1ce`s9iuDGjZ4WmGf!PxKew>F($Lh>=6?h1`u=(r|YOkO&cqi<@_#;809_YGzJhndIsXsuW3VE*d(^Ow8A;-x3mjRFrr!JFlqX za&_I!+g+ckFIr~IwO;SIZMTO{kl=7$VVU%Yx|{9yA3hx(`HNIEFtc2P&0{a$|3GB4 zI4QmAMq_KogZ`mcqoAO~S@7AqOgoKFils&?S5;QmHce-3soaJV0TC2iKpBt~A}~RCU6lT2WcM^Iu>51K^lW00000 literal 0 HcmV?d00001 diff --git a/crates/clawhdf5-format/tests/fixtures/blosc2/b2nd_zero_chunk.out b/crates/clawhdf5-format/tests/fixtures/blosc2/b2nd_zero_chunk.out new file mode 100644 index 0000000000000000000000000000000000000000..fe7c16ec92a0ff95c712aa2320a982eea4de5db8 GIT binary patch literal 2400 zcmeI!`Ipai9L4ebx-XeQB4ppPCJJLJdyMSG6wS!KCdU?82jSSJW1TW-vaf@1?1~UI z%3(N&lc9{UP9hSHQo7%d$vOSdf8hNmJkRUg#~TPo9uz_eltmTPKux@iCU^}o=zt#R zgLg3;<1h)+kcfF$id9&Tt=NSW9KW9d~dKk6A9uXGN`)J!4fZ(&||wYhlsW z&bnG}>u*DCl#RC!Y^Kez1-9JQ+9um>d+j?rWGC#jowuLuH@j$r_O`3(>9D1YEtUg9Wi@Vs8cOL_%= z&TDvGkMiao?J?fPdwM?~;v;>WPw^R^=nH(Aul7y8-FN#wKjh#0DL>~#4&+sxk|?J- zk*JL*G)HU1qBDA-AKt?VjK^fmz--LN64lv=ZP<-(Rp+?soW*6;xsAV7=W#1w#jK1~ zvTF9C>bzpFS{r-Ix&?KH*=U=fIg=~8s&mFJ+Eu$5)Cp%^7GiOh zXB9@UHXE=>P^S}nurCL3xav&dbk63dT&z0lRc9CX@c@tU6w_7bI&bkWX7ik$&kK7g zuc$haUdJ1GOVw%TUA?yt@S#51$9udd_$R8f+}HSK)%nWztIi2O?dLrkav~oJqcqAt ztkVL~x}DB=JE${Kb*5lu*6plTo$W!L!#JTj8MuNQxC>aA<+l8)Q&x4VT1~5;Ri}M$ zI|Fn(p*qtnA*hqA+u5u-Uk7zgTDo1ZU+k9sVa^A2N(6NxRVRwg7|j@VWKY!@%#j?& zc+OxV=Wz)?3+n7vom3uY8Z(&58@$8&%*yK^15NvW1 z7cg0yyqP<=hx>V0oBR_m@+xogt~NQ`^LQa`a(S<;O|In)y{WhMH@$;4xvvlQ;Xc+U zYm-0ndA`_J`Ucz$c?8^9A!`$5fZBbnxYljpktO;!!QODF7*+3s%P(T62jNW1X$1#2R985-VO} zCE9#jW~*(3#M%>xb=-cm49k>QcO+JJ=9XAR`81zpHNM1pY|NH?Lt=GTodFVSG$*P~ z0+YCqE4Y?hB-UQlITF;lq&g483e_q2P^{YCKw`D_SnudPB-S90Q=NF9=^y)CU+SxT zy?-vTQasg&+>e%HhFs1Ca=T>)%n-+GbDaTD*|na`)&2ochKKn8 literal 0 HcmV?d00001 diff --git a/crates/clawhdf5-format/tests/fixtures/blosc2/bitshuffle_odd_blocks.b2f b/crates/clawhdf5-format/tests/fixtures/blosc2/bitshuffle_odd_blocks.b2f new file mode 100644 index 0000000000000000000000000000000000000000..6e71e429af7876af0fbed4a4adcac4b60b17c597 GIT binary patch literal 2204 zcmZA33#^WH9KiACoE%3kxs-@RF2#~XB;_(89Tg>#NUo=m2%|(pIF*@LN64)Xa~m;J zQLK~7F;WMK+zm--sK#{v{MfsldG>yu^M9T_@Af{=^WAS}!Lq)MvWNG1VL-AgNs>O_ z+Sy9XTU;UOn!fwWQD$LkaV{<@7Q6a(nT0!(WM`7Ld0+AL%U5Pxx!(Tm9ZBZlBsrKQ z8ELbM6=wQ^v*-N1`pWo<(@8mQq#Ct}>l@OHRw{a(pX--=@(3z*`O*TUq!5Ahog&E9dK1*21 zIyR6`0fp@4ASd{l^ZZGghZc{x3>B$HZR*jGrnICTkJ5#n^kpE!c!_c3FqK^9@E%L~ zfREY8mwe4`_H%?&{K_BvL;5vIQj&61q6T%S&)qbqHCc4xNuH%YgBihSCh#WHnaKhc zv4XXH!sl#d2YWcgaXgx&BxR{gv{Q#WXw3bz<{>)Ljb8L)5W|Re#_>9DF_ZZ$Vg+l7 zcJld(Z`gyhlVkiuv~!VYr??+gifE@QHK<1e?jhCABi2vwG<|rEp^T(xJJTJ{Vj5DY%R1e=DoHsiQJt7Osdk!KKR^~8=|(RGFqq*)I}?bxGmUqc zM;^;r!+P>5Alli(0giK;b6n&8PXW=+UZS0ooZ&Yvk$6|pPFX5ajoL&zjc7`=^B^7RLQi7u3?k;vC|)Ip zXeXDs}fOM5L2Wv94RElN4t{I@PI*E|C;nBRg@UL_>7SOl+r4x)jZ? zotUX8x2|9AMm5o8`4NVsG^xh)`|hMZXlQ5 z3}7&YJjiH@DWR0<%wax@si2ZoRI!<@)KJSV_Huw@{KOglq*3D}X-Z4l(2-2KkwZ`N z7|1;gWh9R=j!8VtOv+e5IZJty)x5*|Y~wS&U=RB^#0h@k5B{O?1xb=dI&JAh7qYmC z-t;4%Aw0k%jAa6)Ok*}Lv4|zS!AjP#i4XXgI=-ZygB-P&+oM@*5o#;k3HCr_}K^(au4lom2eId7_<*xR_|?Dy|`m?%YaW z1~G(TG;HT7`!hs4FOX_ynfWc&uz_f&n(ah8yNPxV6YZSlEPr#Mw~KaK(~baWJRjg+-9}?}NoS&+9Bv_xf!syh zoe_+tVLLPIb6G&#oux!OtJ%PNRPzZth<3i?Fega0^Ow1?w@M?Owqy`@C)G|b^X=qQ zKoO&OoC!=O+L=w#Sfc8`(k)wM0Ah?B^&a`IU40M-va%oJ;9IXQG`O=|LU? z7)&9aE9CecoJdJ^s2 qK>Qi{tptcgoDU$Cx;u?gT~cyR%Nv1^3}zzxTV}j&Pjk{im_VFJ6vLKF1FL zqu+VsOZREAkz794f|J~nyg_M9_KuFyU>r!qP!u>7kB&9mvp7C>B?l=d)m`AeMU7VPgnFYRrCo}^(j?Nw!N!e zUC?_}U~+X)A5c*rQb`|C$zY#Z(u?m>d;X-4s_nlu*5@ z{&#i7m4fV1fytGuEKydLX+>6P#bn#(BqwXKPHRj#*^o`zkS)r~HswvWy?@@WbJO}s zB+-un>SszKg+WSTh=wsj!zSC_KR=EM8pk9}VVb5)wtX6DWH3V+rnLEPpUVZE%OzdO zHC>r(drv&6$r;s{JgLYrRpf-Ka!OT`ZSUXHq8w0B4yhzZR5IE2{`oLED2z^uAW9LF zZSTNA3|$msau7#1#nD3v^islP+xzG3I(Ge5ez)5fWI@Vrf@%TfF%`gq*_l~+MYHmn z=477cOtyVm(vp!G$}pv6TqbB-CTU8hY06~V`%jUS3{pylXjn#Q*ks%L=d)O%ES6~n ztF&UW?Q_Us4ePYVl*0x#X#-o7$2R3nw!MGeu4C8lkx4=6Xcm0O}gv@o_u1Gtv*gJ}8TNIA646BTBA`3bYVQ1$KQzg|d5Qxo5*2 literal 0 HcmV?d00001 diff --git a/crates/clawhdf5-format/tests/fixtures/blosc2/delta_int16_blosclz.out b/crates/clawhdf5-format/tests/fixtures/blosc2/delta_int16_blosclz.out new file mode 100644 index 0000000000000000000000000000000000000000..e8918075fcd369b427c22516ac6c889feda14e8f GIT binary patch literal 2000 zcmXBUgQ8eh00q$`cc0j{ZPvDJ+qP5Nwr$&1Qrmv9t#+of{$XmWn*U7D#0rx%xxy4p ztuRf~E6mW$3bQo3!W_-5Fi-OJnys#(xnErOP66|`2Hpsm^k?bRXZs7^s=bqTtvThLuSf}ZLX^j4psulfc3H6R$M zLBU`R35IG|FkB;oks1|@)|g;9Eg!6)6@!(uauB9zJ*^*X zpbdkKv~jSBHVrn@=D`-)GT2I62is`dU^{Ie?4TWkowReXi*^lm)9%3@+B4Wodk6by z-(WxO9~__qgM)N%aEJ~K4%6Ym5jrwBN=FCB=-A*m9Uq*a6N8gbBr^-4WcWyMnuQPjIj93+~qg!Gn4zcvz1FkLt1DaXk?{si%Uc^-S=ro(rDW3&D$e zDR^101h49~;B~zbys5W>xAji&uHFmY*9XCe`Y8BVTBv0~ zE440YqqYU@)V`pDIu>+N=YlTkTF_113wo$$K`-?#=%c;`{nWo;fCd%}(%^z28d@++ Q!wW`eWWgwnE*Mk(1#5BRUjP6A literal 0 HcmV?d00001 diff --git a/crates/clawhdf5-format/tests/fixtures/blosc2/delta_int32_lz4.b2f b/crates/clawhdf5-format/tests/fixtures/blosc2/delta_int32_lz4.b2f new file mode 100644 index 0000000000000000000000000000000000000000..819f61ea83e25425fa436a5a11a9d617c67313a3 GIT binary patch literal 1584 zcma)+&r20i9L3K(=lw_?h(1#x!$mGasU$KXvDySJS{P(O(hvLLLeVHtLGj1JKR|d{ zP-svtM5$IKB-O)xiG?L)whbi{LIp+boO@>m(I_8$=AF6c`<;7c?w#o$ymq+uPEEs2 zd;m~0&I5mul7TFo4Slp{=?k6+cdN+d!BKho!UQk@XKX5xW-1+9&tvi(qBFoW5Oylu zg>(+l><$B4D+@U=?E=~KuZMtk(!jAZY$@e|kl@QGu((wq#@Nwv~x z^N1tS?jqU2(@qhGn+Fsk(RKu&*+jFMMvE|JvW~J|&~BjJNPSBLz4f-YispG57liG- zNLek|dx`dC>MO$UO&OOXsx)a=C{{h#wZ^V>o^A-gE3s;4T%@@~V_Dc4XDR0ddw^X-ri{xGRrc}O{*F{piQvYivw?ghiztf) zFL!{KJ4k&<_?Jr*+bE?pkJ2a;Hp(%|a>3pT+Q+F+h@iLJ^tz{kb4Zlk(CTR^yOGv| zN@#AgEm9OdW?SgwiHSZ*C4#3QZKDrS4hv5Il+!<>elCLPhi&v1G+)vf5jOfO%2C1I z*R!cTjf< zKcx4K@U~pb_H}DywZ04kW_#`CQRsGZJ#tLU$FnRlFKLu>{qrqdWRN9RFAJMqG} zca8Qs^@a#?+OLC`b9sBtc~l3ffyuR@I%Ih1i@4SBS`pN4Tuiw1>@cPRu2n_-y z;+2?J2(M6HNq8mYm5f(%UMYB`#d;FXbACSI9&W#N^T zS2kYRdF9}hlUFWYxq0Q`6~QYnuYA1n^NQqEfLB3Yg?JU_RfJbjUd4D7=T(ANNnWLR zmF883S6N==c$Mc>fmcOdm3UR=RfShoUe$P2=T(DOOcy)!uRgr`^6JN{Kd%A22J#xjYcQ`NyoT}`#%nmQ5xhq78pUfguQ9yF z@*2l$Jg*77Ci0rZYcj7Xyr%M+#%nsSSY9)D#qpZSYZkBByyozl%WEF5cwX~)E#S3~ z*CJkvc`f0!l-Dv|%XzKfwUXB=UaNVn;kB06I$rB}ZQ!+$*Ct+@d2QjfmDe_2+j;Ha zwUgH_Ub}hi;kB37K3@BI9pH74*CAeqc^%<(l-Ds{$9bLLb&}U9UZ;7T;dPeRIbP>^ zUEp<**Ck$;d0pXkmDe?1*LmIGb(7aEUblJO;dPhSJzn>DJ>d0_*CSq!c|GCvl-D!< EAC<{Yy#N3J literal 0 HcmV?d00001 diff --git a/crates/clawhdf5-format/tests/fixtures/blosc2/delta_shuffle_v16.b2f b/crates/clawhdf5-format/tests/fixtures/blosc2/delta_shuffle_v16.b2f new file mode 100644 index 0000000000000000000000000000000000000000..5f0cc42b8317931a2ee462d80b093cea1a592701 GIT binary patch literal 1019 zcmb7DKS%;$6n~$mUW1THS_IKmOIy_3(9#%AjSU60)zH*<1Pu*MrGla^4WXz*oq?i8 zdw(qagQl*bSd&T~=k-2MBTnn+!M*ps@B7_*-~0Xee3#qHy9Zl)JMatuTaW0_81cda z8@d-|W#W4YW2Yc9Ivs_Z_%uKo$X(vZ5DGOYc=TuYz?K1CagUsuc@u8w0PGBwRv~q` zKEbnNj4KFZG3PNZA$C=kW%5ZKZVWYT1c(2{;)!G`5sM4a8zB%3g(IOr(BX7B+#avb z?{$0rY(WdeV~JEU5swKsQ6U%zMZ%$Az~OQ_JZ`Vw=k>TX3%udrI9u1x{K!Xp`4~1? z5EdnlTSI~wATW1}L<~2}H+UUj60roz`y<1#$}WeYElM3(YKx6#t5L1h-#_ZrS|*#z z6pE#CrBou_x2JUbk8aQC_MC1n==PFsujuxgZg1%JmTvFp_MUDZ==PCr zpXfFu@P9uxly1Z5Hk@t)bQ?jpk#rkHx6yPPL$|SX8%MYCbell8iFBJpx5;#yLbs`O zn?|?kbelo9ujn?DZnNn2HQi>@Z4TY$(rq5y=F@Ef-4@dA8@er`+qZOEOt&R;TS~WO zbX!ih6?9ukw^elej&7^z_C4KxpxYX{t)<&Kx~-?%k96BWw~ch$M7Pa!+d{Xkbo+^J z+vv8PZa>p)2i2`%~SLt?*Zokv*I^Ay2?IzuB a(d{OKTKC5dM12Z1?J{6BIL`t~n^IU~s{kzaUo+-a`Zh(SY~|GB-hu4>Wp{J$MjB zb4mn79Xu+@5@loX5&J0dk&8({NwliFw$X88LqDp&ny;$9?&({1jvYF2c6j7CUI7?x z(LgOaaC;k$lwNx)&ScRu%1(v`#qM!73(P`2pD%Cno?*N;4ZQ%oCn{AlSxkTeMtgvw z=TU;zO2BUFCGxpS_BuIzr*$9obJPuCW3+dL=2OIRn0)qqVkTQ5US%uW%VaRQjbW39 zg0}HNf|G`=50%jG_G`2PMyL<`&@@u}$+J~7HdBSQEX}pTdX^~svXRcyyq@cP&?Z9) z%h@18vD|~hv3ST|Lw=6MOk{G=dqHe{oUvx;~t?a$yIVADHQ!H?tk0FjE z(e_EqG598c3NJW`my*P^XPDt6S`y~`F&(*_sDU_xRzwL8+Y+A9LAh7cQGUWHb0`p@+@XbY#iVwo8BwENK_ erH!FFx$hSh_$ey3>x!PNvVXE>2;4hg^uTX=K@=1K literal 0 HcmV?d00001 diff --git a/crates/clawhdf5-format/tests/fixtures/blosc2/delta_uint64_blosclz.out b/crates/clawhdf5-format/tests/fixtures/blosc2/delta_uint64_blosclz.out new file mode 100644 index 0000000000000000000000000000000000000000..20735223546c90661fa2654826e0ddfeddfb4f6c GIT binary patch literal 3200 zcmZ9~RZ|rJ6h&d;(p`eINFz!}3X%fS9g@-p|NAN}-OS*6TIc1Pvz~V_Jmlqq@(;iL zOSk)UdqB5`bbCa%$8>u_x2JUbk8aQC_MC1n==PFsujuxgZg1%JmTvFp_MUDZ==PCr zpXfFu@P9uxly1Z5Hk@t)bQ?jpk#rkHx6yPPL$|SX8%MYCbell8iFBJpx5;#yLbs`O zn?|?kbelo9ujn?DZnNn2HQi>@Z4TY$(rq5y=F@Ef-4@dA8@er`+qZOEOt&R;TS~WO zbX!ih6?9ukw^elej&7^z_C4KxpxYX{t)<&Kx~-?%k96BWw~ch$M7Pa!+d{Xkbo+^J z+vv8PZa>p)2i2`%~SLt?*Zokv*I^Ay2?IzuB a(d{h+SQ!sku3Nu;-P*Mv_SUW2wr}6Ib?dI( zyLau~yKn#geS7zU*f4Q)^;fT6yMF!J)vLE|-@bMC?!Ei>@7=u%V#CB=y?*`b?b~0qApv0I+#LKQn+l{$(S~ zWOSFnTu7|TVIhDkWY9wr7Q)CO4`Rc_(beNJzk-2h0>cMMVx2o8WBcqdiRlJdA{~)Q nmz9y7g98|oAglqzU;r!y(ZLi(VCl*RDP7M2C6#ZWt6%^CT@d4> literal 0 HcmV?d00001 diff --git a/crates/clawhdf5-format/tests/fixtures/blosc2/delta_uint8_lz4.out b/crates/clawhdf5-format/tests/fixtures/blosc2/delta_uint8_lz4.out new file mode 100644 index 0000000000000000000000000000000000000000..b70a6d9599651fe994a3e7abcfb7640a4284fd88 GIT binary patch literal 2000 zcmeIygH|92006*j+qP|Owr$&XTWz*&+qP}nwq5rh_xgud#>URh&cVUK$;rvZ#l_9d&BMdP%gf8h$H&jlFCZZB=g%KOK|vuQ zAz@)*5fKqlQBg55F>!Hm2?+^FNl7UwDQRhG85tQ_Sy?$bIeB?`1qB5~MMWhgC1qu0 z6%`d#RaG@LHFb4$4Gj%VO-(H=Ep2UW9UUEAU0pprJ$-$B0|Ns?Lqj7YBV%J@6B83t zQ&TfDGjnru3kwTNOG_&&D{E_O8yg#2TU$FjJ9~S32L}g7M@J_oCue787Z(>-S64ST zH+Oe;4-XGdPfsr|FK=&eA0HoIUtd2zzrTO~`uqC_1Ox;I2LAi^FDNJ|I5;>YBqTI6 zG%PGEJUl!iA|f&}GAb%6IyyQgCMGsEHZCqMK0ZDnAt5m_F)1l2IXO8cB_%aAH7zYI zJv}`mBO^02Gb<}AJ3BikCnq;IH!m+QKR>^qprEj@u&Ai0xVX5aq@=X8w5+VGyu7@k zqN1|0vZ|`8y1Kfirlz*Gwyv(OzP`Spp`o#{v8k!4xw*NerKPpCwXLnKy}iAoqocF4 zv#YDCySuxmr>D2Kx390SzrTNAU|?`?aA;^~czAeZWMp)7bZl&Fe0+RjVq$V~a%yU7 zdU|?hW@dJFc5ZHNetv#oVPSD`acOC3d3kwdWo31Bb!}~JeSLjnV`Fo3b8BmBdwY9l rXJ>bJcW-ZRe}Dhr;NbA^@aX91`1ttbs`!bW+1_+10zE8|?|7NE2aBLl-(kZv@Uvw(aZAkGA0OCZ+R zs{c1ZhoP2{A!5mrJxkWiNy({s@USon-hQfQCcimKYZal!GN*7^ty zx{M43FMz%RI*(nR|G&I_JqJUBh}_|lh7Q9}l?D6ePTlJwxKM(Tm64r;1E>vzHGmil YfV$AZ6h>eGut5Ug98gmE2D%Cc0LuthMF0Q* literal 0 HcmV?d00001 diff --git a/crates/clawhdf5-format/tests/fixtures/blosc2/delta_v3.out b/crates/clawhdf5-format/tests/fixtures/blosc2/delta_v3.out new file mode 100644 index 0000000000000000000000000000000000000000..bc826e3e48d64d9208868bccdf969ec327295470 GIT binary patch literal 900 zcmd7Q17aWs006Mswr$(CZQHi5wr$(CZQHi(?id#>URh&cVUK$;rvZ#l_9d&BMdP%gf8h$H&jlFCZWwC@3f-BqS^>EFvNz zDk>@_CMGT}E+HWyDJdx>B_%B_Eh8f%D=RA}Cnqm2ub`lysHmuC#4Z(v|xXlQ6;WMph?Y+_D3@8ICz=;-L=gwv|=H~A1?&0C# z>FMd^<>l?|?c?L)>+9?1=lAd5KYxG!fPjF&z`&rOpy1%(kdTnj(9p23u<-Ekh=_>D z$jGRusOaeEn3$N@*x0zZxcK<^goK2|#Kfeeq~zq}l$4az)YP=JwDk1!jEs!T%*?E; ztnBRUoSdB8+}ympy!`z9f`Wp=!os4WqT=G>l9H0r($ccBvhwosii(QL%F3#$s_N?M znwpy0+SS?mrsn46mX?;**4DPRw)Xb+j*gDb&d#o`uI}#co}QlG z-rl~xzW)CHfq{X+!NH-Sq2b}-k&%(n(b2K7vGMWoiHV8H$;qjysp;wInVFf{+1a_d zx%v6|g@uL1#l@wirRC-2m6es%)z!7Nwe|J&jg5`X&CRW?t?ljYot>TC-QB&tz5V_D egM)*^!^5MaqvPY_larIv)6=uFv-AJK|Nj7+jm1U) literal 0 HcmV?d00001 diff --git a/crates/clawhdf5-format/tests/fixtures/blosc2/generate.py b/crates/clawhdf5-format/tests/fixtures/blosc2/generate.py new file mode 100644 index 0000000..f79c0f7 --- /dev/null +++ b/crates/clawhdf5-format/tests/fixtures/blosc2/generate.py @@ -0,0 +1,129 @@ +"""Generate the Blosc2 frames the `filters_blosc2` unit tests decode. + +Each case is `.b2f` (a Blosc2 contiguous frame, what the HDF5 Blosc2 +filter stores per chunk) and `.out` (what decoding it must give: the +first chunk of a plain frame, or the whole array in C order for a B2ND +frame), or `.err` (a frame clawhdf5 must refuse; the file holds a word +the error message must contain). + +These cover what files written by h5py + hdf5plugin never contain, but a +Blosc2 frame may: special chunks (repeated value, NaN, uninitialised), the +delta filter over many blocks and odd type sizes, bit shuffle of blocks that +are not a multiple of 8 elements, shuffle with a byte-group size, forced +stream splitting, multi-chunk B2ND arrays with padded edge chunks and a +chunk of zeros, and features clawhdf5 refuses (dictionaries, registered +filters). + +Written with python-blosc2 4.13.1 (c-blosc2 3.3.4) in a scratch venv +(`pip install blosc2`). Re-run only to regenerate: + + python generate.py +""" +import os +import sys + +import blosc2 +import numpy as np + +out = sys.argv[1] + + +def save(name, frame, expected): + with open(os.path.join(out, name + ".b2f"), "wb") as f: + f.write(frame) + with open(os.path.join(out, name + ".out"), "wb") as f: + f.write(expected) + + +def save_err(name, frame, word): + with open(os.path.join(out, name + ".b2f"), "wb") as f: + f.write(frame) + with open(os.path.join(out, name + ".err"), "w") as f: + f.write(word) + + +def plain(data, **cparams): + """A one-chunk super-chunk frame of `data`, as hdf5-blosc2 writes.""" + data = np.ascontiguousarray(data) + cp = blosc2.CParams(typesize=data.dtype.itemsize, **cparams) + sc = blosc2.SChunk(chunksize=data.nbytes, cparams=cp) + sc.append_data(data) + return sc.to_cframe(), data.tobytes() + + +def special(nitems, dtype, kind, value=None): + dt = np.dtype(dtype) + sc = blosc2.SChunk(chunksize=nitems * dt.itemsize, + cparams=blosc2.CParams(typesize=dt.itemsize)) + sc.fill_special(nitems, kind, value) + return sc.to_cframe() + + +# Special chunks. A repeated value stays in the frame as a 33+ byte chunk; +# NaN and uninitialised chunks become special offsets. +save("value_i4", special(300, "G!2ZVfzdQDng&MGz>rP@0A(|8D*ylh literal 0 HcmV?d00001 diff --git a/crates/clawhdf5-format/tests/fixtures/blosc2/nan_f8.b2f b/crates/clawhdf5-format/tests/fixtures/blosc2/nan_f8.b2f new file mode 100644 index 0000000000000000000000000000000000000000..939321b8a1a22d1ec8d36ce0eba5c4cf325cff1e GIT binary patch literal 172 zcmbQYBFQMNC^0vc;SvJ_L*jWL0Rn552r+~*U50Qt6QB$Rup$nSN)U@P;UWXWMFz%0 zH`u@m8CfCff#l?~4D9z98167Iurji9Z~#R>SOX@{2xGwMrYVd-J!}v?=YYb>H{fzG F1^`Ikm#Z273A)J9O*?6#JvY5xEQYxcn~{O z->)Y&>hA@1_(u>LCN}ENiH;z21l2sq_%jfN{S^;$W*5RVh9Dydf<kOxRvZkJWoHb3bZ7<=_@`P&9F2cL-4Hnm-Q zjV>#gGVVB|{ zuUn<$>W8hJy+bc&7FSlWth_>EvdWr!^v$dtUA;~QpN~#V#~0i#uc~Wme>yNWg@MFH zWmPnFe&BUI=65D6Dn1QgKp<7sHn#N)jJ}#%{JO!!&Mz#bpt@7n$l{QryLUjy`IyA? z>%}GI_ZuE{J{=l+J^yhP1Bq{y-KM^0zloKd^D)1m@W{B-?7Um0l{JlR-Tf~n-zMihF`s1{IZVa;1!lsRNbX#XlCQ&ej+jB+705p>iU*;`q0a11~?R# zQ`OpQXb!A<2b?{BIVlZabi15V-_!xDzk0v?70trQzXjB^OYeZCor|YW5U`${onK76 zSM#8)vlr4_{IrHq)zCFKh_gR>{N$PQQSp#KNf~7Lq`QA?`t36C$;r26-vJ9-CwHII zp%>$lGxCZ{@76T5{M$P;J_8W9NGNUxbOE6kW0Ny;;oX$FmQDaL^B&61$|WEwt*pLB z-_+6$3I_U#OUceBlvdV0eB9MHJT<%UWdqB`BP=Pes-YzU#ef{p^ zSA+?ypa%+eIhLH6S4<>VH?(vya`1~tD#E^zMd*dtl+5b-){efx@!9#6?-*7tAu$;x zbsd9ae!*dv6RzeIfGem?Z9V-XQ*)qIEIXfwgu?cndyOn?kAOHri<;YeK@;y+zM)t- zH;c+DYiL6`4!eT&!Xx8TbMgsgjFQG~eG@Bi+m)28yjx}Dn#R`d{^7~DA9#f%6}E#K&1@XO zWf}OQ5)!2jln7MKElDe>Y40<(v~~9MNd{lFb@f88m|t1NuyI3mt?gaYs%VYv5Ie=d z$_-yvR@XK#u_BaJ)i$+t4~$OEEqz9qICg018C%#md4OYbZ-h~5qICy=PtyhWL^WCQ0iMc z=z|lp?-3LlgGONxG-|he5t=2of{FVS4YwYdoqJ%vs7jV#FHWv)2~Q3QSsjx8eprx} zf2QkkxcSa@qLFX1Pf)Itui>1T8(E;ZVtSq+8Cc@3j}Bb(qboWk2t>`Lg?fBQ*ikS$ z=ggm1{+yRB{ocda%}I*WZwf}`T1o|63PS`}lD5Zaa|jq|8Oscw^)MxA-M<``PO`zG ln3$QNeg7EpFn(UBKa0QjEEN0>7=Qf^x?x*t@bB60)qf=)>URJD literal 0 HcmV?d00001 diff --git a/crates/clawhdf5-format/tests/fixtures/blosc2/never_split.out b/crates/clawhdf5-format/tests/fixtures/blosc2/never_split.out new file mode 100644 index 0000000000000000000000000000000000000000..8a1d2f344681e2d95ae3430e9ef5b3586e7bb84c GIT binary patch literal 3000 zcmYM$hkwua7RT}PKF1cdU5OPV)YjO0tG!aS_Z~H4hN@AG*ehskp|Oh_BPdZ@?b>^* zeTy2m===N0@4ElO^YMP1*HK9>pGii^F1e+U1WQ?|BDJNFd@Ze|v-FaIGF-lu?`5jY zmIbm*Vq~LilU=f3j>#FhBsb)NJeODUK@uSa(jhYfkrzc!0u@jL_0SkC&>r3J4F+Qb z#vlUIFc*ul0&B1d+p!mia02IW4R`PuFYpc@kxWx*2F;?mwV(!T8LgzXw4pZBR@zy6 zYJVN7qjiE#(pmbWF3}kMSz~pl{;EgylwQ*7dS9REU;17XS#nEjnJtIqvqDzV%2_q5 zV@<4uwY9GHjfGm6jkR!_ZgXsrEw{C{$+p`bJ7~x3oL#Y7W|B-&OD4%Cd8D8emvT~# zI*p~dw53iT86d-CjD*V+iIgabmesOJw#shm9GBnairjKKubqw`GQb}>@i}!$QKvfU zqA6NYrw96B2*NNP6EPk0u>?P19byrOUvU^Gah^JNsq>OLs>wBtX4LE&q=obgEvuEO z(}+5)snd%(!}UArOx4-CK$q((-Kbl1x9+FTX}zd7sPkN3>ED{zQdoNPr%qlgOr7%7 zsb`I?rM0te)`vPHZJb5e44Z2UZH29~&9=kh?XaDs&NaJDofMKzGIKk5rHGW23e>41 zO{9gilP=O*25~!{&UBe83#qeKHp_O|BM0P!oRh0^M;>uIZ{?#Tg^0J)JLMNt|R zQ4R}X+liBSIYgcFa*aBV<%PTxK@y~#@bBa|V}i6Zi29ukap;sFP0pHHYTYB3i@6>b5M`zIlZiR^dYzN#_6P@PJrdI0@NvEm8nzDLa5WxdVJD} zpw4_-Y%#XMw%AVF$L*Z5i+02AS|Uj$=_SDF6qQm^i8}SAskD;z(w#a(WTcFhNp3q! zWCe9%WhZrx$Vs2h6Y9JbMRKQ;mD?%gbSmRZG^9>jbVg4M#Bh8^oynMk1>8;yHef4$ z!2xdPG<9y_4?M?T)G_trcKkJvIz_2dQLAfRZKBP!y>`_;I!H(A7!B7MPG_aAaXNeT zklW5Ry-S@J`c^+`QcGPa9xE?OU5*lc^JBOKp{{x2?8| zI=|Uz>Rh+`+>WP{RdPxJDJG@4oto0X>2#DH)EVk-@?@DM^JFP+@&@YckbQF0r*mJP z$V+ZVd6Uy16S8wVg;5;kQ4MvRPCIl#Zw%me#!zR9(~0JGyqmlm2XPE%a2dDpfZKV) zo19ovaytQr3j0 zCABn`$+BA>-sE5_%k9*%hSrQXxr_C*{?r-mbY?l77~9D0csj@Ij9s#u)JZ~}jFLri zQO7G*Wv3Iu?RYxa3FWvW45(&+VL%3v!(|`6-L_UoofSj~oc%c8a^3 zT${ye&h2#6n-OwjrL|If1jdLY+t4&TD+2j-M-5PR*~yv^0xVQyb`4 zyvd$UUlwbWj@JmCsqvFE=zorIphWU-7r zPs5%ET9E5`1*=Y-Cf3{)%j@}Y8{>55uvpQyn#J;Ze!z~Kr*qqs#Y*FJg81!};C8&8 zH}dtox3A|va67Z9vs~8j+u6!uc|AWXm)Y}&@*jD_VkJgOq-W1_u~@~ZQ^|chAuLux z&qGMWtoTK3%2Il}Fn$5j^Vk<)!(=65veV|WW&yB@O>w2Egif}s>sZ*ERNzmy_oe|WD;I|XWVnw^2$5H1ni*??v zQYR60(z54)62xM8-%fR(PJ8LfZzq&JA1f323r4!0uW>z(XU~tjp5J26|KxTANs;PP z&%fZeQ<=qTfDnE=o!qxG6r+4SpT}R&>v=4TwU^({DYu>b?D-4!9GZ;V$;6%qX<^>v zvec>Ve!({G+Zm+8*z<6mLY;-WOjq+4+{R+<=S}{d#k$F!d&PS1dY+EO%E_J=vC`D3 zNu8#=$sJtJhp^}4Y?ABwVq3{yFxKMu?RY)Ez?*!R-%d(btepII3iI2kNS!+F7wq7Q zHCV!A9D6>UH+ix9b~f=Q$ID^fU_puFblWyzn<6RO%6e8emlL~O&;ZL Y@+|&>2|bU+PVD1$PT>N3{s2$#52yv@>Hq)$ literal 0 HcmV?d00001 diff --git a/crates/clawhdf5-format/tests/fixtures/blosc2/shuffle_meta2.b2f b/crates/clawhdf5-format/tests/fixtures/blosc2/shuffle_meta2.b2f new file mode 100644 index 0000000000000000000000000000000000000000..3426ff41e2b77760cb775a10f92be50a81ce2b16 GIT binary patch literal 1003 zcmbQYBFQMNC^0vc;SvJ_L*jWL0R+simk2S0FkOc5_!mGK%=REf96*A90hqYRz;Ka) z@z4!6MzBg21}2COhRJ6c*zYkg++ko~Wh~`bz|X({!uHGz4CpFVfI7s1;uUdcucn`w zpDfY#@&5nXvr(#Bo3;yGxtFxHEc@t+8*69qno7H7E!`%vRy1VGw`D7CRi78zxU}OY z*GAQhGj|H@PR=#|_;=^z*T1JewmJ8@?dJxLX-A48I9Dgy=5(#yU|V)jPNJ{5ki{*T z=h&l|p5sqvv`80PFME_|)A;ISEy z8Ob?)xf_kY@ho;Ou;@G5X|Xn`L~`AsuRD0ZKR9Sm#kc-Z?wKR%GkV`^xy42>bhica zvn8xGkeYFaYjNyG{%`9gWZQ!atd2cQ@;UP4i5ctnM}aoI;*)P%pImG3DxvNR@AP>) zHnUwj&li1c?!t$guF@?}ToF5h(c^~sluwU*0&wElN2Cc@yI zPVdu=TGtM`Nwg^55Zs+C!FR|*Mz)-5`sPA8y`xii@NI89{kLLI(R1G#f%1Jme!bRn zcU=CGebXRnZ&5mT&h$C6>Uvk)krXaJxAK>%<-0o*=g4(g>nBt_J-7UaR!^68{leYi zi~kiae|Oz1bpN+L>-N@*oG-6SPPJ~mxOInQ`NMrXf+hBbd|my-Z1tz@`DHIWk4x9H zAH6^6a^4MY_K<&l@1DMI{eSu6ro+ae$M3(s`f>iFbM~Rff2%IP^f7cp*&R(;`JTCt z&VASsTqGr`m;Cq~E70WAx}nFV%a=Xp+c$qs+H<~tP4^JcX-tux6ti7 z(@YF^POZD!%YJCZorB?*)~jug{J7meFaGk!TlX7pF8n7yOSbPMU+d|aFMp`sT==ov zOTRp;Ub=iA?{v$l*4)V-3nC3;t{qh?O!xTLerxVTkOS3PPA_@&&d2w-w4U*!b7!7} zSp91*+vj(B+3MEIcbhUFzuuCwE6U=o?hG>&!IXcjLOtIuvNE!BaDXx$5Nkj)7c>vS Ycwl-8BQVRcL9*OAposDf6lo9#0LW;=B>(^b literal 0 HcmV?d00001 diff --git a/crates/clawhdf5-format/tests/fixtures/blosc2/shuffle_meta2.out b/crates/clawhdf5-format/tests/fixtures/blosc2/shuffle_meta2.out new file mode 100644 index 0000000000000000000000000000000000000000..da0be8caf5f846bb2d8805b1156153769c6af9e3 GIT binary patch literal 4000 zcmZ|Rdvwor9LMpYl3OlE*dca~OcY8xUEFH5%65`Dk(iS%j#A`IoE45%$(dLeCzq48 zRYO2yf*uj$$4k;3VeraX!g;T*#OC3SZ}2T*r;v%1VCC zZ@Hg8@(7Rd6wk16lZMZ!1>3R{yE2`9nZbd~WHyKMZr;O*oXp2Ki-j!Wi(JYQu3$OW zb2C3<6?gMH9$+2gYtzjAYQqlf%IMA&_SGE78yVfnwMTIrCvY|?Q|x)Gr)N!hccI=IgZhtN9<{w$>>gzUCbqXm2YqjEBFyV zVHJ0=h6i|<^*qVHc$OEqchQ~pc4wxs7yI!VMt8F89FE{vj%U1grrOgP-I;4I;3CF* zr_^4}cliOgu##VK5BIT_(H%3Tx%<_Y(Va9qoqZVHiT6&H^K8a@XN;Zf&ZEw!b0(kW ze7?k`EaNJc^IYB8?(fl^c<&^;^PB&R?wq!h-MPp;yO^DMDdW9!m3=*?! literal 0 HcmV?d00001 diff --git a/crates/clawhdf5-format/tests/fixtures/blosc2/uninit_i8.b2f b/crates/clawhdf5-format/tests/fixtures/blosc2/uninit_i8.b2f new file mode 100644 index 0000000000000000000000000000000000000000..58a2a30ca38485a13acd7aa435f63bdfd2cce945 GIT binary patch literal 172 zcmbQYBFQMNC^0vc;SvJ_L*jWL0Rn552r+~*U50R&7@!OWup$nSN)U^Q;UWVNF&?_X z23E+(3Q-RvC!b|tzsJCEhk=2Wk)4ABC<4M7FnLB815US0VFc=7gXlR26jr_gmxD0? D_(2$h literal 0 HcmV?d00001 diff --git a/crates/clawhdf5-format/tests/fixtures/blosc2/uninit_i8.out b/crates/clawhdf5-format/tests/fixtures/blosc2/uninit_i8.out new file mode 100644 index 0000000000000000000000000000000000000000..a64a5a93fb4aef4d5f63d79cb2582731b9ac5063 GIT binary patch literal 512 NcmZQz7zHCa1ONg600961 literal 0 HcmV?d00001 diff --git a/crates/clawhdf5-format/tests/fixtures/blosc2/value_f8.b2f b/crates/clawhdf5-format/tests/fixtures/blosc2/value_f8.b2f new file mode 100644 index 0000000000000000000000000000000000000000..156a4b45a5828182186c608b81325f6d55af10ca GIT binary patch literal 212 zcmbQYBFQMNC^0vc;SvJ_L*jWL0RmT+2r+~*U50SjFF+X#8X!d+AOfVE{lY~ChKmf0 zhiEY9gL=fp_C5*r)R%u literal 0 HcmV?d00001 diff --git a/crates/clawhdf5-format/tests/fixtures/blosc2/value_i4.b2f b/crates/clawhdf5-format/tests/fixtures/blosc2/value_i4.b2f new file mode 100644 index 0000000000000000000000000000000000000000..e803bc270990804c4d5d85ddf3304dd43e066e22 GIT binary patch literal 208 zcmbQYBFQMNC^0vc;SvJ_L*jWL0Rk762r+~*U50R2Hb5B+Dj-EHAOfVEWy3`VhKmf0 zhiG#K_<G!2ZVfgz9v029|$6aWAK literal 0 HcmV?d00001 diff --git a/crates/clawhdf5-format/tests/fixtures/blosc2/zero_u2.b2f b/crates/clawhdf5-format/tests/fixtures/blosc2/zero_u2.b2f new file mode 100644 index 0000000000000000000000000000000000000000..5978d316b3e51edc7adb5d00b30a9f3fe5679ecc GIT binary patch literal 172 zcmbQYBFQMNC^0vc;SvJ_L*jWL0Rn552r+~*U50S@7eE;dU`0$Il^_=Xf{P3c7a152 z-CzSNWMqY?2a=P|GO*ucV7SA;z{<$Z!2uKjVGWo(Ba8v38>cV=^{_$ooC69g-+;@( F7yz4Q8Pos( literal 0 HcmV?d00001 diff --git a/crates/clawhdf5-format/tests/fixtures/blosc2/zero_u2.out b/crates/clawhdf5-format/tests/fixtures/blosc2/zero_u2.out new file mode 100644 index 0000000000000000000000000000000000000000..1dc89f8e47e591af4c36a35046077f0ba1d1ef9d GIT binary patch literal 4000 ocmeIuF#!Mo0K%a4Pi+hzh(KY$fB^#r3>YwAz<>b*1`NCh1`nVB0RR91 literal 0 HcmV?d00001 diff --git a/crates/clawhdf5-format/tests/fixtures/blosc2/zstd_dict.b2f b/crates/clawhdf5-format/tests/fixtures/blosc2/zstd_dict.b2f new file mode 100644 index 0000000000000000000000000000000000000000..89477d8da8c9adced12097e3aca12bf7f65a733e GIT binary patch literal 4978 zcma)=c{o*T8^+f*%d8?HGG)wAQBEl&LkWpWO439ni6Y9-q)>_mO-f1w4WdpW^H@ky z5+N1MMJl1b`&nD(JLme&AK$)&uD$m1zVCBCzx#gH$@5E%R``3aUCumU7{;>(-|!mh zpCZk;@YHetXBIyAi*zYn#D{M}bh%l0EyL6@oWceH_CgLn_j-oI#8)wb&lskOVfZ;a z_`=Qbh7iM)O8>s#_mvE1fQMnCBpGHMFT+@fGfbBt!x#)p?E6Omp@V-fk!8^m% z-ZrX|2JV8NWW9?#QpCdc8~0ne{p#zNBp-bVjV>1CVD)_5oiOFq_02EdbbsdvNysRwY8yUbMLNA~7o3`$Z zjX#!r;p&at;=8qtEu9~}4DgCb$|~=POH4YOo|#ive*bavOFogIa-%i$$4|Djb)3I= zg>OJeL{!|N6KB(}XBYgw_o8J!8-gOD_8mHwd@18*eo1xR)90^0ejC7NN2-j`pD=m) zOlLRGRlXayY#+QiBOo?RUPaT;*laph6y2S?$$ne_VRtt4;~?@ z5lZTM<0qM0JGf*QmQ_D)Zs!>yH9}c^?08cPJIDDRE7k@CNA8Y0bo^{uMs`72^`oZe zuRnhqyt(^QpY=gocf`aWJ9GZ(&D_$v51zJmeE2%RFCr~Bdd%1fe^}xci+Zb1bcN&H~R>>o$f)Mjtqwl$w^6Q&?Hs(ERdkS06`6Qbtit+i;S(&1~1fn`h^j zRzG~&&Lb=(GfGW&{2vy!4)Yd!t?}O+v1{L<<2c-#d1d$No7&!d`aU2aI#f~4weR4`GilcbZ;n-2&UBjRv2tx- zXk=8}k)*SiGH(@>)zo8EpL-cW@!^VUS_YG*&9HZNM_o6D?PP?pQdMn3Q}Y@2u8WrX z_yvXU+?#Oh^o1*b-YLHK;7M!er*A*`h7294q%nB&@lcCqmi~Yjsh8nQ;4qqFbjO)m zh=ejPR~Hz0R12%tKQ|f`>)85o;f!-ZPxeX)=eS!Z@Q?lH=lD5HcRo*2OJB{$fsSuK zyF(=0IZUM>ua9Q1#?YM)Zrc6O#Rs@Aw9CaSH}Z<_J$%}R_4MhyW+s{}V{?flq-`bx>sIZv5B({X{vN?(#&+~K5@v`msT zipV=eN>&MaoNR9E*+%sKh z>-7%_-@W(Hi8B|kq35d}f#n1^QI?)2Q;42RR{3G$(Fcy4hB$8*SJXakMH~F!0aujN z^o;&M8@RYHTjL)T9t8-T!E*9Tsvb7Bzaa>SN<$3>CQ!LE%RQ(8;)EJFu{EY@TS@KP{p966bYGUG#oH~E`Ms86hq2^up59+E>?7nOq zU4a@fb{7zO=3)lKb+3;0^&R_y(SjsqmUb{zc3;u^6QOl5tNeb$vsb*LQX@eKs_RzP%^LzL9eCD zXN|~^C@rqE8Ty+*3=9lKO3%tIx=Uv(ASNxZqGdpb?=-mEJ>1SlE7hQJ-*xwK1YykT zI))&$!`y|)C&6q!NxlHq6jwcjz`2GolJW^vae>EjUm#-7zQZR|E}`YhY92MUzwP?Y z2#O7tSJNW&*}K5?A&2ecJTSssoPR@Wryvk98pnoCnaheQbX&{;RPjnyZgD023x@H7 zBP@Z~0#Qw~c643DI%U*egg7D=2o2}?jANq!IEGpd`$7nTs9=Pr&ZWZ$E0GXiy!+A% zIk16aqPeyGT=!+({=wmtyHKPX*cXhjLqJ?cQAOJj{x!?R&C7coh!vY~?9}J5D6f>${IaB(I<&dp=hT8hYs8E0WE2%e7+@n9@MlClcw+T(D#4s)>` z|4mzW#bVDC-Ak&#*N#s;{ruDdh)-y3iW$UL#3uy#>$eIksvogk3)RRdpqCJzW;wep zMSKd|9)l{twgKu|fLc%-B-hqoxP%fp62?)~IWxBqf}w-$;}w#Sg?`7Gl1)bJ+DNK4MwG$A~MGgcr}unjFN_qk?Azs*`yF~c^8zQg1`&uZmXA;^HGtrbKl}NBk|!Jt-vimq@6xEe4jj1C1an3;pld(zS&K&CrrVub0y5R65KQS+>vqRQIGEvWeq;=O{ZE_jb< zxyZ}gACnAfo_vv76GUt4`1tiFKjtfCO+8F9m=6|HMrF;JX0yUF*2iGZI9EkA9fOI} ztY$mo;L&*?#Zg2Tge1&;ZLi?oBS&k26c7bE$``R^PdrAk%QtQpg7-+e>~dt4G<1!S zq8we2sr@nTQFXAK{4zSPHwa05ARv+>d~BBUd?ek#P}1_rb7@)NeJv14w#tqx+Q{A* zO=yS$DPrMcC^H!%MgihIB0fb+XLmGs5Jk(x)2UY|;8CJ3GZTE5bO7ItB;UoX zfQ*VT@!~Z*7(q}F`xGk>&~p`dza4^3A_5j+R%oRWlY`b$Qq#dWY2!#7Wiu-FMIot- zLJ~+JGfJH@ruA&-*4r;Q3=#i$^2IAR^Ds)$tnd}1lr(&dh;3_(_WrA{8IBc)Q8nlN z8xekmA(bR6*^ig9v<-}|wUrzBxK7k&>}s2Tq!Jq^?%)!Mnfhc(+Vxu)GY2CQRZB%n z-()JQ?p1!9pe~Aj=~OL>e*Ca+01|W5OiB~$0=Go&K5zt8fGALfmRIk-^q~siwXWer zjIu5Zm-+Z%aG_>KIjI}rLqrtO!GUZ#=43>_K;j>wD)?7WK~ELlVj!bIK#c}6_|QU6 zHYHJ8jSiV2xQf-hTu`8}wH;EMP>(zTmGrV-I3E z*KV-$e_IE-0qQcKs7lWr{DP3`Ffu@X6r<3b?|TM#he+bNgDz4X21wXq0C#XX%ci8N z`lc6e$reRPUHZmTt#Hhs426Bv;XF`ElPFCMQ?^wqfKejj3*3QR-r9vR;L8U;a`LKSxjq53uMV7Ng0v b8U~NK{^ymz!ILim?vt--ysO&qKbQOmmA}MC literal 0 HcmV?d00001 diff --git a/crates/clawhdf5-format/tests/fixtures/blosc2/zstd_dict.err b/crates/clawhdf5-format/tests/fixtures/blosc2/zstd_dict.err new file mode 100644 index 0000000..658c6ba --- /dev/null +++ b/crates/clawhdf5-format/tests/fixtures/blosc2/zstd_dict.err @@ -0,0 +1 @@ +dictionar \ No newline at end of file diff --git a/crates/clawhdf5/Cargo.toml b/crates/clawhdf5/Cargo.toml index a6401fe..1eca373 100644 --- a/crates/clawhdf5/Cargo.toml +++ b/crates/clawhdf5/Cargo.toml @@ -47,8 +47,9 @@ lzf = ["clawhdf5-format/lzf"] bitshuffle = ["clawhdf5-format/bitshuffle"] bzip2 = ["clawhdf5-format/bzip2"] blosc = ["clawhdf5-format/blosc"] +blosc2 = ["clawhdf5-format/blosc2"] # Every plugin filter. -plugin-filters = ["lzf", "bitshuffle", "bzip2", "blosc"] +plugin-filters = ["lzf", "bitshuffle", "bzip2", "blosc", "blosc2"] # Dataset::verify_provenance() — recompute a dataset's SHA-256 and compare # against its stored _provenance_sha256 attribute. On by default, matching # clawhdf5-format's own default-on `provenance` feature. diff --git a/crates/clawhdf5/tests/plugin_filters_interop.rs b/crates/clawhdf5/tests/plugin_filters_interop.rs index 81cc87c..00911c6 100644 --- a/crates/clawhdf5/tests/plugin_filters_interop.rs +++ b/crates/clawhdf5/tests/plugin_filters_interop.rs @@ -77,7 +77,7 @@ except ImportError: hdf5plugin = None path = sys.argv[1] FILTERS = eval('(' + sys.argv[2] + ')') -cases = [ +cases = eval('(' + sys.argv[3] + ')') if len(sys.argv) > 3 else [ (') { let dir = tempfile::tempdir().unwrap(); let path = dir.path().join(format!("{tag}.h5")); - let n: usize = run_python(GENERATE, &[path.to_str().unwrap(), filters]) - .parse() - .unwrap(); + let mut args = vec![path.to_str().unwrap(), filters]; + args.extend(cases); + let n: usize = run_python(GENERATE, &args).parse().unwrap(); assert!(n > 0); let file = File::open(&path).unwrap(); for i in 0..n { @@ -353,6 +363,110 @@ fn blosc_written_by_clawhdf5_reads_in_hdf5plugin() { } } +#[cfg(feature = "blosc2")] +#[test] +fn blosc2_written_by_hdf5plugin_reads_exactly() { + if !have_python("h5py, hdf5plugin") { + return; + } + // Every codec hdf5plugin's Blosc2 offers, each lossless filter, and + // levels from "store" to maximum. Truncating precision is lossy, so it + // is checked separately against h5py's own reading. + check_h5py_written( + "blosc2", + r#"[(f'{c} {f} {l}', hdf5plugin.Blosc2(cname=c, clevel=l, filters=f)) + for c in ['blosclz', 'lz4', 'lz4hc', 'zlib', 'zstd'] + for f, l in [(hdf5plugin.Blosc2.NOFILTER, 5), + (hdf5plugin.Blosc2.SHUFFLE, 9), + (hdf5plugin.Blosc2.BITSHUFFLE, 1), + (hdf5plugin.Blosc2.DELTA, 5)]] + + [('blosclz level 0', hdf5plugin.Blosc2(cname='blosclz', clevel=0)), + ('zstd level 0 bitshuffle', + hdf5plugin.Blosc2(cname='zstd', clevel=0, filters=hdf5plugin.Blosc2.BITSHUFFLE))]"#, + ); +} + +/// Blosc2 over more shapes and dtypes: every integer and float width, +/// 1-D to 5-D chunks (B2ND arrays from 2-D on) with partial edge chunks and +/// block shapes that pad the chunk, datasets of zeros, of one repeated value +/// and of NaN (Blosc2's "special" chunks), and Fletcher32 before Blosc2 +/// (which makes hdf5-blosc2 fall back from B2ND to a plain frame). +#[cfg(feature = "blosc2")] +#[test] +fn blosc2_shapes_and_special_chunks_read_exactly() { + if !have_python("h5py, hdf5plugin") { + return; + } + let cases = r#"[(dt, shape, chunks, kind) + for dt in ['u2', 'f8'] + for shape, chunks in [((777,), (100,)), + ((37, 53), (10, 16)), + ((9, 10, 11), (4, 5, 3)), + ((6, 7, 5, 9), (3, 2, 5, 4))] + for kind in ['ramp', 'noise']] + + [(' Date: Sat, 26 Sep 2026 10:22:52 -0500 Subject: [PATCH 16/43] docs: Blosc2 reads (read only); ZFP is the one plugin filter left Record the `blosc2` feature in the changelog, the README's feature table and the crate table, and mark the Blosc2 half of the known "Filters" issue fixed (dated, with the conformance run that shows h5ex_d_blosc2 reading). What stays open: ZFP, writing Blosc2, and the Blosc2 features hdf5plugin never writes (dictionaries, lazy chunks, variable-length blocks, user-defined codecs and registered filters), which are errors. Co-Authored-By: Claude Opus 5.5 (1M context) --- CHANGELOG.md | 21 +++++++++++++++++++++ CLAUDE.md | 2 +- README.md | 16 +++++++++------- docs/known-issues.md | 15 ++++++++++----- 4 files changed, 41 insertions(+), 13 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 29804fc..aebeb93 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,26 @@ ## Unreleased +### Blosc2 (2026-09-26) +- **Blosc2 (filter 32026) reads, in pure Rust.** Files written with + hdf5plugin's `Blosc2` failed with `UnsupportedFilter`. New feature + `blosc2` (`clawhdf5-format` and `clawhdf5`, included in `plugin-filters`) + decodes the Blosc2 contiguous frame hdf5-blosc2 stores per chunk, the + B2ND arrays it uses for chunks of 2 or more dimensions (blocks gathered + back into C order), Blosc2 chunks with their special values (zeros, NaN, + uninitialised, one repeated value), and the shuffle, bit-shuffle, delta + and truncate-precision filters, over the BloscLZ, LZ4/LZ4HC, Zlib and + Zstandard codecs shared with Blosc 1. Read only: there is no Blosc2 + encoder. Dictionaries, lazy chunks, variable-length blocks, user-defined + codecs and registered Blosc2 filters (e.g. bytedelta) are errors; + uninitialised chunks read as zeros. Tested against h5py 3.16 + + hdf5plugin 7.1 (every codec, filter and level 0-9; 1- to 5-D chunks with + partial edge chunks; every integer width and f4/f8; datasets of zeros, + one value and NaN; Fletcher32 before Blosc2) and against frames from + python-blosc2 4.13.1 for what hdf5plugin never writes + (`crates/clawhdf5-format/tests/fixtures/blosc2/`); the decoder is + fuzzed. Conformance: 576 of 697 files ok (was 575) — h5ex_d_blosc2. + ### Concurrent reads (2026-09-26) - **Full reads of chunked datasets scale with threads again when rayon's pool has one thread.** Each full read handed its chunks to rayon to @@ -372,6 +392,7 @@ missing feature ("unsupported filter: 32026 (Blosc2, not implemented by clawhdf5)"). - **Not implemented:** Blosc2 (32026) and ZFP (32013) remain a clear error. + (Blosc2 reads since the `blosc2` feature, see above.) - **Wrong data: a chunk that decodes short read as zeros** (pre-existing, every filter). HDF5 stores every chunk at the full chunk size, so a filter pipeline that decodes to fewer bytes means a corrupt chunk; every chunk diff --git a/CLAUDE.md b/CLAUDE.md index 1cb7aba..b029c5d 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -11,7 +11,7 @@ Cargo workspace with 18 crates under `crates/` (plus `libaec-sys`, an internal F |-------|------| | `clawhdf5-format` | HDF5 binary spec parser (superblock, B-tree, heap) — also holds shared type definitions and physical constants | | `clawhdf5-io` | Read/write implementation | -| `clawhdf5-filters` | Deflate backends (zlib-rs, zlib-ng, Apple Compression); the HDF5 filter pipeline, the filter registry (`clawhdf5_format::filter_registry`) and the other codecs (LZ4, Zstd, SZIP, N-Bit, scale-offset, pcodec, and the pure-Rust plugin filters LZF, bitshuffle, bzip2, Blosc 1) live in `clawhdf5-format`. No Blosc2 or ZFP. | +| `clawhdf5-filters` | Deflate backends (zlib-rs, zlib-ng, Apple Compression); the HDF5 filter pipeline, the filter registry (`clawhdf5_format::filter_registry`) and the other codecs (LZ4, Zstd, SZIP, N-Bit, scale-offset, pcodec, and the pure-Rust plugin filters LZF, bitshuffle, bzip2, Blosc 1, and Blosc2 read-only) live in `clawhdf5-format`. No ZFP. | | `clawhdf5-derive` | Proc-macro derive for HDF5-serializable structs | | `clawhdf5` | Main facade crate | | `clawhdf5-netcdf4` | NetCDF-4 compatibility layer | diff --git a/README.md b/README.md index 4ae4d1c..25f43a7 100644 --- a/README.md +++ b/README.md @@ -668,7 +668,7 @@ clawhdf5 workspace (17 crates, ~86K lines of Rust in src/, ~104K with tests ├── Core HDF5 │ ├── clawhdf5-format — Binary parser/writer (no_std-capable), shared type definitions │ ├── clawhdf5-io — I/O abstraction (file/memory readers; optional mmap, async, HSDS, MPI) -│ ├── clawhdf5-filters — Fast deflate path (zlib-ng); the filter registry and the lz4/zstd/pcodec/szip/LZF/bitshuffle/bzip2/Blosc filters live in clawhdf5-format +│ ├── clawhdf5-filters — Fast deflate path (zlib-ng); the filter registry and the lz4/zstd/pcodec/szip/LZF/bitshuffle/bzip2/Blosc/Blosc2 filters live in clawhdf5-format │ ├── clawhdf5-derive — Proc macros │ ├── clawhdf5 — High-level API │ ├── clawhdf5-netcdf4 — NetCDF-4 support @@ -779,13 +779,15 @@ stores keep their setting. Opt out with `float16 = false` or | `bitshuffle` | no | Bitshuffle filter (id 32008) with its LZ4 and Zstandard modes: read and write. Pure Rust (lz4_flex, ruzstd) | | `bzip2` | no | bzip2 filter (id 307): read and write. Pure Rust (the `bzip2` crate's libbz2-rs-sys backend compiles no C) | | `blosc` | no | Blosc 1 filter (id 32001): reads BloscLZ, LZ4/LZ4HC, Snappy, Zlib and Zstandard frames with byte or bit shuffle; writes LZ4, Snappy, Zlib or Zstandard (not BloscLZ). Pure Rust | -| `plugin-filters` | no | All four above | +| `blosc2` | no | Blosc2 filter (id 32026), read only: hdf5plugin's frames and B2ND (n-D) chunks, BloscLZ, LZ4/LZ4HC, Zlib and Zstandard, with shuffle, bit shuffle, delta or truncated precision. Pure Rust | +| `plugin-filters` | no | All five above | -Blosc2 (32026) and ZFP (32013) are not implemented: reading them fails with -`UnsupportedFilter`, whose message names the filter. Any other filter can be -supplied at run time with `filter_registry::register_filter` (a decoder -closure, or a `FilterCodec` that also encodes). The facade (`clawhdf5`) -forwards `lzf`, `bitshuffle`, `bzip2`, `blosc` and `plugin-filters`. Write +ZFP (32013) is not implemented: reading it fails with `UnsupportedFilter`, +whose message names the filter. clawhdf5 cannot write Blosc2. Any other +filter can be supplied at run time with `filter_registry::register_filter` (a +decoder closure, or a `FilterCodec` that also encodes). The facade +(`clawhdf5`) forwards `lzf`, `bitshuffle`, `bzip2`, `blosc`, `blosc2` and +`plugin-filters`. Write with `DatasetBuilder::with_lzf()`, `with_bitshuffle(..)`, `with_bzip2(..)` and `with_blosc(..)`; h5py + hdf5plugin read the result (tested both ways in `crates/clawhdf5/tests/plugin_filters_interop.rs`). The pure-Rust Zstandard diff --git a/docs/known-issues.md b/docs/known-issues.md index 2657cdf..2c42b2f 100644 --- a/docs/known-issues.md +++ b/docs/known-issues.md @@ -210,11 +210,16 @@ fill-value item that did is fixed). bitshuffle, bzip2 and Blosc 1 (`bitshuffle`, `bzip2`, `blosc`, or `plugin-filters` for all), read and write, pure Rust; h5ex_d_lzf, h5ex_d_bshuf, h5ex_d_bzip2 and h5ex_d_blosc now read (conformance 573 of - 697 ok). **Still open:** Blosc2 (32026 — hdf5plugin stores each chunk as a - Blosc2 super-chunk frame, and n-D chunks as B2ND arrays) and ZFP (32013); - both fail with an `UnsupportedFilter` error that names the filter, and - either can be plugged in with `filter_registry::register_filter` (32023, - Granular BitRound, too, since 2026-09-26 even with the `pcodec` feature). + 697 ok). **Fixed 2026-09-26** for Blosc2 (32026, `blosc2` feature, also + in `plugin-filters`), read only: hdf5plugin's frames and B2ND arrays, every + codec and filter it offers; h5ex_d_blosc2 now reads (conformance 576 of + 697 ok, tank, `conformance/run.sh --no-fetch`). Blosc2 frames using + dictionaries, lazy chunks, variable-length blocks, user-defined codecs or + registered filters (e.g. bytedelta) are refused with an error. + **Still open:** ZFP (32013) fails with an `UnsupportedFilter` error that + names the filter, and can be plugged in with + `filter_registry::register_filter` (32023, Granular BitRound, too, since + 2026-09-26 even with the `pcodec` feature). clawhdf5 cannot write Blosc2. - **Wrong data: a chunk whose filters decode to fewer bytes than the chunk read with zeros for the missing bytes** (any filter; found reviewing the plugin filters). **Fixed 2026-09-26:** it is an error naming the chunk. A corrupt From d3d73676c08b9dfabaa7b84093c0d37ec3ece38c Mon Sep 17 00:00:00 2001 From: osobh Date: Sat, 26 Sep 2026 10:28:10 -0500 Subject: [PATCH 17/43] style(format): iterate the dimensions when checking them against their maxima Co-Authored-By: Claude Opus 5.5 (1M context) --- crates/clawhdf5-format/src/dataspace.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/crates/clawhdf5-format/src/dataspace.rs b/crates/clawhdf5-format/src/dataspace.rs index 8eb37e1..eedba2f 100644 --- a/crates/clawhdf5-format/src/dataspace.rs +++ b/crates/clawhdf5-format/src/dataspace.rs @@ -121,9 +121,9 @@ impl Dataspace { // Read max dimensions if flags bit 0 is set let max_dimensions = if flags & 0x01 != 0 { let mut max_dims = Vec::with_capacity(rank as usize); - for i in 0..rank as usize { + for &dim in &dimensions { let val = read_length(data, pos, length_size)?; - if dimensions[i] > val { + if dim > val { return Err(FormatError::InvalidDataspace( "dataspace dimension size is greater than its maximum size", )); From b3058ca46eb36a4d66090de679dd705e6b61106b Mon Sep 17 00:00:00 2001 From: osobh Date: Sat, 26 Sep 2026 10:28:28 -0500 Subject: [PATCH 18/43] feat: decode the superblock extension at open; read metadata cache images libhdf5 decodes the messages of a v2/v3 superblock's extension when it opens a file (H5F__super_read) and refuses the file when one does not decode. We never looked at them, so we opened cve-2020-10810 (a File Space Info message too short for the free-space manager addresses it announces) and cve-2020-10812 (a metadata cache image past the end of the file), both of which libhdf5 refuses. A file written with a metadata cache image keeps its metadata cache entries in an image block the extension points at; libhdf5 loads them over the file's own bytes before it reads any metadata (H5C__load_cache_image, H5C__reconstruct_cache_contents). In h5clear_mdc_image.h5 the root group's header exists only in the image, so every reader failed with InvalidObjectHeaderVersion(0). The new clawhdf5_format::superblock_ext module: - read_superblock_extension decodes the v1 B-tree K, File Space Info and Metadata Cache Image messages with libhdf5's checks (versions, page size 512 B .. 1 GiB, the addresses a persisting message lists, the image inside the file), with the new FormatError::InvalidSuperblockExtension; - apply_cache_image checks an image block as libhdf5 does (signature, version, recorded length, entry types, rings, ages, addresses inside the file and not repeated, flush-dependency parents) and returns the file's bytes with every entry written at its address (FormatError::InvalidCacheImage); - metadata_view does both. File, MmapFile and LazyFile (and so h5rs) call metadata_view at open and read an image file through the patched copy; the conformance probe does the same. The image's trailing checksum is not verified, as libhdf5 does not verify it. tests/fixtures/h5clear_mdc_image.h5 is libhdf5's own test file. Co-Authored-By: Claude Opus 5.5 (1M context) --- conformance/probe/src/main.rs | 13 + crates/clawhdf5-format/src/error.rs | 14 + crates/clawhdf5-format/src/lib.rs | 1 + crates/clawhdf5-format/src/superblock_ext.rs | 630 ++++++++++++++++++ crates/clawhdf5/src/lazy.rs | 17 +- crates/clawhdf5/src/mmap_file.rs | 15 +- crates/clawhdf5/src/reader.rs | 25 +- .../tests/fixtures/h5clear_mdc_image.h5 | Bin 0 -> 23467 bytes .../tests/header_validation_interop.rs | 91 ++- crates/clawhdf5/tests/metadata_cache_image.rs | 48 ++ 10 files changed, 839 insertions(+), 15 deletions(-) create mode 100644 crates/clawhdf5-format/src/superblock_ext.rs create mode 100644 crates/clawhdf5/tests/fixtures/h5clear_mdc_image.h5 create mode 100644 crates/clawhdf5/tests/metadata_cache_image.rs diff --git a/conformance/probe/src/main.rs b/conformance/probe/src/main.rs index 844f343..c4cf524 100644 --- a/conformance/probe/src/main.rs +++ b/conformance/probe/src/main.rs @@ -712,6 +712,19 @@ fn main() { return; } }; + // libhdf5 decodes the superblock extension at open, and loads a + // metadata cache image over the file's own metadata. + let view = match guarded(|| { + clawhdf5_format::superblock_ext::metadata_view(hdf5, &sb).map_err(e) + }) { + Ok(v) => v, + Err(msg) => { + top.insert("open_error".into(), Value::String(msg)); + println!("{}", Value::Object(top)); + return; + } + }; + let hdf5: &[u8] = view.as_deref().unwrap_or(hdf5); top.insert("superblock_version".into(), json!(sb.version)); let ctx = Ctx { data: hdf5, diff --git a/crates/clawhdf5-format/src/error.rs b/crates/clawhdf5-format/src/error.rs index ddd9ba7..7f9e3d2 100644 --- a/crates/clawhdf5-format/src/error.rs +++ b/crates/clawhdf5-format/src/error.rs @@ -235,6 +235,14 @@ pub enum FormatError { /// element size that overflows, contiguous storage past the end of the /// file, compact data of the wrong size. InvalidDatasetStorage(&'static str), + /// A superblock extension message libhdf5 refuses to decode when it + /// opens the file (the reason is libhdf5's own error text): a File Space + /// Info message that runs off its end or has a bad page size, a metadata + /// cache image outside the file, … + InvalidSuperblockExtension(&'static str), + /// A metadata cache image block libhdf5 refuses to load (the reason is + /// libhdf5's own error text). + InvalidCacheImage(&'static str), } impl fmt::Display for FormatError { @@ -515,6 +523,12 @@ impl fmt::Display for FormatError { FormatError::InvalidDatasetStorage(why) => { write!(f, "invalid dataset storage: {why}") } + FormatError::InvalidSuperblockExtension(why) => { + write!(f, "invalid superblock extension: {why}") + } + FormatError::InvalidCacheImage(why) => { + write!(f, "invalid metadata cache image: {why}") + } } } } diff --git a/crates/clawhdf5-format/src/lib.rs b/crates/clawhdf5-format/src/lib.rs index 3e30f9d..e10711b 100644 --- a/crates/clawhdf5-format/src/lib.rs +++ b/crates/clawhdf5-format/src/lib.rs @@ -118,6 +118,7 @@ pub mod selection; pub mod shared_message; pub mod signature; pub mod superblock; +pub mod superblock_ext; pub mod symbol_table; #[cfg(all( test, diff --git a/crates/clawhdf5-format/src/superblock_ext.rs b/crates/clawhdf5-format/src/superblock_ext.rs new file mode 100644 index 0000000..d7ec3cc --- /dev/null +++ b/crates/clawhdf5-format/src/superblock_ext.rs @@ -0,0 +1,630 @@ +//! The superblock extension of a version 2 or 3 superblock, and the +//! metadata cache image it can point to. +//! +//! libhdf5 reads the extension when it opens a file (`H5F__super_read`) and +//! decodes the messages that configure the file: v1 B-tree "K" values, File +//! Space Info, and the Metadata Cache Image. A message that does not decode +//! makes the file fail to open, so [`read_superblock_extension`] decodes and +//! checks them the way libhdf5 does. +//! +//! A metadata cache image (written with `H5Pset_mdc_image_config`) is a +//! block holding serialized metadata cache entries — object headers, B-tree +//! nodes, heaps — each with its file address. libhdf5 loads it into its +//! cache before it reads any other metadata (`H5C__load_cache_image`, +//! `H5C__reconstruct_cache_contents`), and the entries take the place of +//! the file's bytes at their addresses: the file itself may hold stale or +//! no metadata there (in `h5clear_mdc_image.h5` the root group's header is +//! only in the image). [`apply_cache_image`] does the same with bytes: it +//! returns a copy of the file with every entry written at its address, so +//! every parser reads what libhdf5 reads. + +#[cfg(not(feature = "std"))] +use alloc::{collections::BTreeSet, vec::Vec}; +#[cfg(feature = "std")] +use std::collections::BTreeSet; + +use crate::error::FormatError; +use crate::message_type::MessageType; +use crate::object_header::ObjectHeader; +use crate::superblock::Superblock; + +/// Message type of the File Space Info message. +const MSG_FSINFO: u16 = 0x0017; +/// Message type of the Metadata Cache Image message. +const MSG_MDCI: u16 = 0x0018; +/// Header message flag: the library did not know the message when it wrote +/// it back (`H5O_MSG_FLAG_WAS_UNKNOWN`); libhdf5 then ignores its contents. +const MSG_FLAG_WAS_UNKNOWN: u8 = 0x20; + +/// `H5F_FILE_SPACE_PAGE_SIZE_MIN` / `_MAX`. +const PAGE_SIZE_MIN: u64 = 512; +const PAGE_SIZE_MAX: u64 = 1024 * 1024 * 1024; +/// libhdf5's default file space page size, used for a version 0 message. +const PAGE_SIZE_DEFAULT: u64 = 4096; +/// Free-space managers whose addresses a persisting version 1 File Space +/// Info message lists (`H5F_MEM_PAGE_SUPER` .. `H5F_MEM_PAGE_NTYPES`), and +/// a version 0 one (`H5FD_MEM_SUPER` .. `H5FD_MEM_NTYPES`). +const FSM_ADDRS_V1: usize = 12; +const FSM_ADDRS_V0: usize = 6; + +/// Metadata cache image block limits (`H5Cimage.c`, `H5ACprivate.h`). +const MDCI_SIGNATURE: &[u8; 4] = b"MDCI"; +const MDCI_HAVE_RESIZE_STATUS: u8 = 0x01; +const MDCI_ENTRY_IS_FD_PARENT: u8 = 0x04; +const MDCI_ENTRY_IS_FD_CHILD: u8 = 0x08; +/// `H5AC_NTYPES`: entry type ids are below this. +const MDCI_NTYPES: u8 = 30; +/// `H5C_RING_NTYPES`. +const MDCI_RING_NTYPES: u8 = 6; +/// `H5AC__CACHE_IMAGE__ENTRY_AGEOUT__MAX`. +const MDCI_AGE_MAX: u8 = 100; + +/// A decoded File Space Info message (0x0017), mapped to version 1 as +/// libhdf5 maps a version 0 one. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct FileSpaceInfo { + /// Message version as stored (0 or 1). + pub version: u8, + /// File space strategy (`H5F_fspace_strategy_t`). + pub strategy: u8, + /// Whether free space is persisted. + pub persist: bool, + /// Free-space section threshold. + pub threshold: u64, + /// File space page size. + pub page_size: u64, +} + +/// Where a metadata cache image block is (Metadata Cache Image message, +/// 0x0018). +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct CacheImageLocation { + /// Address of the image block. + pub address: u64, + /// Length of the image block in bytes. + pub length: u64, +} + +/// The messages of a superblock extension that libhdf5 decodes at open. +#[derive(Debug, Clone, Default, PartialEq, Eq)] +pub struct SuperblockExtension { + /// v1 B-tree "K" values (chunk index, symbol table node, symbol table + /// leaf), when the extension overrides the defaults. + pub btree_k: Option<(u16, u16, u16)>, + /// The File Space Info message. + pub file_space_info: Option, + /// The metadata cache image, when the file has one. + pub cache_image: Option, +} + +fn ext_err(why: &'static str) -> FormatError { + FormatError::InvalidSuperblockExtension(why) +} + +const RAN_OFF: &str = "ran off end of input buffer while decoding"; + +/// A little-endian cursor over one message or block, failing with +/// `overrun` when it runs off the end. +struct Cursor<'a> { + data: &'a [u8], + pos: usize, + overrun: FormatError, +} + +impl<'a> Cursor<'a> { + fn new(data: &'a [u8], overrun: FormatError) -> Self { + Cursor { + data, + pos: 0, + overrun, + } + } + + fn take(&mut self, n: usize) -> Result<&'a [u8], FormatError> { + let end = self + .pos + .checked_add(n) + .filter(|&e| e <= self.data.len()) + .ok_or_else(|| self.overrun.clone())?; + let s = &self.data[self.pos..end]; + self.pos = end; + Ok(s) + } + + fn u8(&mut self) -> Result { + Ok(self.take(1)?[0]) + } + + fn uint(&mut self, width: u8) -> Result { + let b = self.take(width as usize)?; + Ok(b.iter() + .rev() + .fold(0u64, |acc, &x| (acc << 8) | u64::from(x))) + } + + /// An address of `width` bytes; `None` when undefined (all ones). + fn addr(&mut self, width: u8) -> Result, FormatError> { + let v = self.uint(width)?; + let undef = if width >= 8 { + u64::MAX + } else { + (1u64 << (8 * u32::from(width))) - 1 + }; + Ok((v != undef).then_some(v)) + } +} + +/// Decode and check the superblock extension of `sb`, as libhdf5 does when +/// it opens the file. `data` is the file from the superblock on, up to the +/// end of file the superblock records (its end is libhdf5's "eoa"). +/// +/// Returns `Ok(None)` for a superblock without an extension (versions 0 +/// and 1 have none). A message libhdf5 fails to decode, or a cache image +/// that does not lie inside the file, is an error: libhdf5 refuses to open +/// such a file (`cve-2020-10810`: a File Space Info message too short for +/// the free-space manager addresses it announces; `cve-2020-10812`: a cache +/// image past the end of the file). +pub fn read_superblock_extension( + data: &[u8], + sb: &Superblock, +) -> Result, FormatError> { + let os = sb.offset_size; + let ls = sb.length_size; + let undef = if os >= 8 { + u64::MAX + } else { + (1u64 << (8 * u32::from(os))) - 1 + }; + let Some(addr) = sb.superblock_extension_address.filter(|&a| a != undef) else { + return Ok(None); + }; + let addr = usize::try_from(addr).map_err(|_| ext_err("address out of range"))?; + let header = ObjectHeader::parse(data, addr, os, ls)?; + let eoa = data.len() as u64; + + let mut ext = SuperblockExtension::default(); + for msg in &header.messages { + match msg.msg_type { + MessageType::BTreeKValues => { + let mut c = Cursor::new(&msg.data, ext_err(RAN_OFF)); + if c.u8()? != 0 { + return Err(ext_err("bad version number for v1 B-tree 'K' message")); + } + let chunk = c.uint(2)? as u16; + let snode = c.uint(2)? as u16; + let leaf = c.uint(2)? as u16; + ext.btree_k = Some((chunk, snode, leaf)); + } + MessageType::Unknown(MSG_FSINFO) if msg.flags & MSG_FLAG_WAS_UNKNOWN == 0 => { + ext.file_space_info = Some(decode_fsinfo(&msg.data, os, ls)?); + } + MessageType::Unknown(MSG_MDCI) => { + let mut c = Cursor::new(&msg.data, ext_err(RAN_OFF)); + if c.u8()? != 0 { + return Err(ext_err( + "bad version number for metadata cache image message", + )); + } + let address = c.addr(os)?; + let length = c.uint(ls)?; + let Some(address) = address else { + return Err(ext_err("metadata cache image address is undefined")); + }; + if address.checked_add(length).is_none_or(|end| end > eoa) { + return Err(ext_err( + "metadata cache image: address plus size exceeds file eoa", + )); + } + ext.cache_image = Some(CacheImageLocation { address, length }); + } + _ => {} + } + } + Ok(Some(ext)) +} + +/// `H5O__fsinfo_decode` plus the checks `H5F__super_read` makes on it. +fn decode_fsinfo(data: &[u8], os: u8, ls: u8) -> Result { + let mut c = Cursor::new(data, ext_err(RAN_OFF)); + let version = c.u8()?; + let info = if version == 0 { + let old_strategy = c.u8()?; + let threshold = c.uint(ls)?; + // H5F_file_space_type_t: 1 ALL_PERSIST, 2 ALL, 3 AGGR_VFD, 4 VFD. + let (strategy, persist) = match old_strategy { + 1 => { + for _ in 0..FSM_ADDRS_V0 { + c.addr(os)?; + } + (0, true) + } + 2 => (0, false), + 3 => (2, false), + 4 => (3, false), + _ => return Err(ext_err("invalid file space strategy")), + }; + FileSpaceInfo { + version, + strategy, + persist, + threshold, + page_size: PAGE_SIZE_DEFAULT, + } + } else { + if version > 1 { + return Err(ext_err("File space info message's version out of bounds")); + } + let strategy = c.u8()?; + let persist = c.u8()? != 0; + let threshold = c.uint(ls)?; + let page_size = c.uint(ls)?; + if page_size == 0 || page_size > PAGE_SIZE_MAX { + return Err(ext_err("invalid page size in file space info")); + } + c.uint(2)?; // page end metadata threshold + c.addr(os)?; // EOA before the free-space managers + if persist { + for _ in 0..FSM_ADDRS_V1 { + c.addr(os)?; + } + } + FileSpaceInfo { + version, + strategy, + persist, + threshold, + page_size, + } + }; + if info.page_size < PAGE_SIZE_MIN { + return Err(ext_err("file space page size too small")); + } + Ok(info) +} + +/// One entry of a metadata cache image: `len` bytes at `image_offset` in +/// the image block, belonging at file address `address`. +struct ImageEntry { + address: u64, + image_offset: usize, + len: usize, +} + +/// Decode the metadata cache image at `location` in `data` (the file from +/// the superblock on, up to its recorded end of file) and return a copy of +/// `data` with every cached entry written at its address — the metadata +/// libhdf5 reads for this file. The image is checked as libhdf5 checks it +/// (`H5C__decode_cache_image_header`, `H5C__reconstruct_cache_entry`): +/// signature and version, the image length it records, entry types, rings +/// and ages in range, entry addresses inside the file and not repeated, +/// flush-dependency parents that are earlier entries. +/// +/// libhdf5 does not verify the block's trailing checksum when it loads an +/// image, so neither does this. +pub fn apply_cache_image( + data: &[u8], + location: CacheImageLocation, + offset_size: u8, + length_size: u8, +) -> Result, FormatError> { + let bad = FormatError::InvalidCacheImage; + let start = usize::try_from(location.address).map_err(|_| bad("address out of range"))?; + let len = usize::try_from(location.length).map_err(|_| bad("length out of range"))?; + let block = start + .checked_add(len) + .and_then(|end| data.get(start..end)) + .ok_or(bad("image block extends past the end of the file"))?; + let eoa = data.len() as u64; + let mut c = Cursor::new(block, bad(RAN_OFF)); + + // Header: signature, version, flags, image data length, entry count. + if c.take(4)? != MDCI_SIGNATURE { + return Err(bad("bad metadata cache image header signature")); + } + if c.u8()? != 0 { + return Err(bad("bad metadata cache image version")); + } + if c.u8()? & MDCI_HAVE_RESIZE_STATUS != 0 { + return Err(bad("MDC resize status not yet supported")); + } + if c.uint(length_size)? != location.length { + return Err(bad("bad metadata cache image data length")); + } + let n_entries = c.uint(4)?; + if n_entries == 0 { + return Err(bad("bad metadata cache entry count")); + } + + let mut entries = Vec::new(); + let mut seen = BTreeSet::new(); + for _ in 0..n_entries { + let type_id = c.u8()?; + if type_id >= MDCI_NTYPES { + return Err(bad("type id is out of valid range")); + } + let flags = c.u8()?; + if c.u8()? >= MDCI_RING_NTYPES { + return Err(bad("ring is out of valid range")); + } + if c.u8()? > MDCI_AGE_MAX { + return Err(bad("entry age is out of policy range")); + } + let children = c.uint(2)?; + // libhdf5 checks the parent flag against the child count only in + // debug builds (release builds refuse any entry with children); the + // image format's own rule is checked here. + if (flags & MDCI_ENTRY_IS_FD_PARENT != 0) != (children > 0) { + return Err(bad("flush dependency parent flag and child count disagree")); + } + c.uint(2)?; // dirty dependency children: reset for a read-only open + let parents = c.uint(2)?; + if (flags & MDCI_ENTRY_IS_FD_CHILD != 0) != (parents > 0) { + return Err(bad("flush dependency child flag and parent count disagree")); + } + c.uint(4)?; // LRU rank + let address = c + .addr(offset_size)? + .filter(|&a| a < eoa) + .ok_or(bad("invalid entry address range"))?; + let size = c.uint(length_size)?; + if size == 0 { + return Err(bad("invalid entry size")); + } + for _ in 0..parents { + let parent = c + .addr(offset_size)? + .ok_or(bad("invalid flush dependency parent offset"))?; + if !seen.contains(&parent) { + return Err(bad("flush dependency parent not in the image")); + } + } + let len = usize::try_from(size).map_err(|_| bad(RAN_OFF))?; + let image_offset = c.pos; + c.take(len)?; + if !seen.insert(address) { + return Err(bad("duplicate addresses in cache")); + } + entries.push(ImageEntry { + address, + image_offset, + len, + }); + } + + let mut out = data.to_vec(); + for e in &entries { + // address < eoa <= usize::MAX, and the entry's bytes came from the + // block, so neither conversion nor the sum can fail. + let at = e.address as usize; + let end = at + e.len; + if end > out.len() { + out.resize(end, 0); + } + out[at..end].copy_from_slice(&block[e.image_offset..e.image_offset + e.len]); + } + Ok(out) +} + +/// What a reader must do before reading a file's metadata, in one call: +/// check the superblock extension ([`read_superblock_extension`]) and, +/// when the file has a metadata cache image, return the file's bytes with +/// the image applied ([`apply_cache_image`]). `Ok(None)` means read `data` +/// as it is. +pub fn metadata_view(data: &[u8], sb: &Superblock) -> Result>, FormatError> { + match read_superblock_extension(data, sb)? { + Some(SuperblockExtension { + cache_image: Some(location), + .. + }) => apply_cache_image(data, location, sb.offset_size, sb.length_size).map(Some), + _ => Ok(None), + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn sb_v2(ext: u64) -> Superblock { + Superblock { + version: 2, + offset_size: 8, + length_size: 8, + base_address: 0, + eof_address: 0, + root_group_address: 0, + group_leaf_node_k: None, + group_internal_node_k: None, + indexed_storage_internal_node_k: None, + free_space_address: None, + driver_info_address: None, + consistency_flags: 0, + superblock_extension_address: Some(ext), + checksum: None, + page_size: None, + } + } + + /// A file whose superblock extension (a version 1 object header at 48) + /// holds the given messages, padded to `len` bytes. + fn file_with_ext(messages: &[(u16, &[u8])], len: usize) -> Vec { + let mut body = Vec::new(); + for (t, d) in messages { + let padded = d.len().div_ceil(8) * 8; + body.extend_from_slice(&t.to_le_bytes()); + body.extend_from_slice(&(padded as u16).to_le_bytes()); + body.extend_from_slice(&[0x14, 0, 0, 0]); + body.extend_from_slice(d); + body.resize(body.len() + padded - d.len(), 0); + } + let mut f = vec![0u8; 48]; + f.push(1); + f.push(0); + f.extend_from_slice(&(messages.len() as u16).to_le_bytes()); + f.extend_from_slice(&1u32.to_le_bytes()); + f.extend_from_slice(&(body.len() as u32).to_le_bytes()); + f.extend_from_slice(&[0; 4]); + f.extend_from_slice(&body); + f.resize(len, 0); + f + } + + fn fsinfo_v1(page_size: u64, persist: bool, n_addrs: usize) -> Vec { + let mut m = vec![1, 1, u8::from(persist)]; + m.extend_from_slice(&1u64.to_le_bytes()); + m.extend_from_slice(&page_size.to_le_bytes()); + m.extend_from_slice(&0u16.to_le_bytes()); + m.extend_from_slice(&u64::MAX.to_le_bytes()); + for _ in 0..n_addrs { + m.extend_from_slice(&u64::MAX.to_le_bytes()); + } + m + } + + fn mdci(address: u64, length: u64) -> Vec { + let mut m = vec![0]; + m.extend_from_slice(&address.to_le_bytes()); + m.extend_from_slice(&length.to_le_bytes()); + m + } + + #[test] + fn no_extension() { + assert_eq!( + read_superblock_extension(&[0; 64], &sb_v2(u64::MAX)).unwrap(), + None + ); + } + + #[test] + fn file_space_info_as_libhdf5_decodes_it() { + // What FileWriter::with_page_size writes. + let f = file_with_ext(&[(MSG_FSINFO, &fsinfo_v1(4096, false, 0))], 256); + let ext = read_superblock_extension(&f, &sb_v2(48)).unwrap().unwrap(); + assert_eq!(ext.file_space_info.unwrap().page_size, 4096); + let f = file_with_ext(&[(MSG_FSINFO, &fsinfo_v1(4096, true, 12))], 512); + assert!(read_superblock_extension(&f, &sb_v2(48)).is_ok()); + + let refused = |m: Vec| { + let f = file_with_ext(&[(MSG_FSINFO, &m)], 512); + read_superblock_extension(&f, &sb_v2(48)).unwrap_err() + }; + // Persisting, but too short for the manager addresses. + let mut short = fsinfo_v1(4096, true, 12); + short.truncate(short.len() - 8); + assert_eq!(refused(short), ext_err(RAN_OFF)); + assert!(matches!( + refused(fsinfo_v1(256, false, 0)), + FormatError::InvalidSuperblockExtension(_) + )); + assert!(matches!( + refused(fsinfo_v1(0, false, 0)), + FormatError::InvalidSuperblockExtension(_) + )); + let mut v2 = fsinfo_v1(4096, false, 0); + v2[0] = 2; + assert!(matches!( + refused(v2), + FormatError::InvalidSuperblockExtension(_) + )); + // cve-2020-10810: version 0, strategy ALL_PERSIST, and a message of + // 32 bytes that cannot hold the six addresses that follow. + let mut v0 = vec![0u8, 1]; + v0.extend_from_slice(&[0, 1, 0, 0, 0, 0, 0, 0]); + v0.resize(32, 0xff); + assert_eq!(refused(v0), ext_err(RAN_OFF)); + // A version 0 message without persistence is fine. + let mut v0 = vec![0u8, 2]; + v0.extend_from_slice(&[0; 8]); + let f = file_with_ext(&[(MSG_FSINFO, &v0)], 256); + assert!(read_superblock_extension(&f, &sb_v2(48)).is_ok()); + } + + #[test] + fn cache_image_location_must_be_inside_the_file() { + let f = file_with_ext(&[(MSG_MDCI, &mdci(128, 64))], 192); + let ext = read_superblock_extension(&f, &sb_v2(48)).unwrap().unwrap(); + assert_eq!( + ext.cache_image, + Some(CacheImageLocation { + address: 128, + length: 64 + }) + ); + // cve-2020-10812: 256 MiB at 0x10100 in a 2565-byte file. + let f = file_with_ext(&[(MSG_MDCI, &mdci(0x10100, 0x1000_0000))], 2565); + assert!(matches!( + read_superblock_extension(&f, &sb_v2(48)), + Err(FormatError::InvalidSuperblockExtension(_)) + )); + let f = file_with_ext(&[(MSG_MDCI, &mdci(u64::MAX, 8))], 256); + assert!(read_superblock_extension(&f, &sb_v2(48)).is_err()); + } + + /// A cache image block with `entries` of (address, bytes). + fn image(entries: &[(u64, &[u8])]) -> Vec { + let mut b = Vec::new(); + b.extend_from_slice(MDCI_SIGNATURE); + b.push(0); + b.push(0); + b.extend_from_slice(&0u64.to_le_bytes()); // length, patched below + b.extend_from_slice(&(entries.len() as u32).to_le_bytes()); + for (addr, bytes) in entries { + b.extend_from_slice(&[5, 0x02, 1, 0]); // type, flags (in LRU), ring, age + b.extend_from_slice(&[0; 6]); // children, dirty children, parents + b.extend_from_slice(&0i32.to_le_bytes()); + b.extend_from_slice(&addr.to_le_bytes()); + b.extend_from_slice(&(bytes.len() as u64).to_le_bytes()); + b.extend_from_slice(bytes); + } + b.extend_from_slice(&[0; 4]); // checksum (not verified, as in libhdf5) + let n = b.len() as u64; + b[6..14].copy_from_slice(&n.to_le_bytes()); + b + } + + #[test] + fn cache_image_entries_replace_the_file_bytes() { + let img = image(&[(16, b"HEADER"), (40, b"NODE")]); + let mut f = vec![0u8; 64]; + let at = f.len() as u64; + f.extend_from_slice(&img); + let loc = CacheImageLocation { + address: at, + length: img.len() as u64, + }; + let out = apply_cache_image(&f, loc, 8, 8).unwrap(); + assert_eq!(out.len(), f.len()); + assert_eq!(&out[16..22], b"HEADER"); + assert_eq!(&out[40..44], b"NODE"); + assert_eq!(&out[..16], &f[..16]); + + let bad = |img: Vec| { + let mut f = vec![0u8; 64]; + f.extend_from_slice(&img); + let loc = CacheImageLocation { + address: 64, + length: img.len() as u64, + }; + apply_cache_image(&f, loc, 8, 8).unwrap_err() + }; + let mut sig = image(&[(16, b"x")]); + sig[0] = b'X'; + assert!(matches!(bad(sig), FormatError::InvalidCacheImage(_))); + assert!(matches!( + bad(image(&[(16, b"a"), (16, b"b")])), + FormatError::InvalidCacheImage("duplicate addresses in cache") + )); + assert!(matches!( + bad(image(&[(1 << 20, b"far")])), + FormatError::InvalidCacheImage("invalid entry address range") + )); + let mut len = image(&[(16, b"x")]); + len[6] ^= 1; + assert!(matches!(bad(len), FormatError::InvalidCacheImage(_))); + let mut cut = image(&[(16, b"abcdef")]); + let n = cut.len() as u64 - 8; + cut.truncate(cut.len() - 8); + cut[6..14].copy_from_slice(&n.to_le_bytes()); + assert!(matches!(bad(cut), FormatError::InvalidCacheImage(_))); + } +} diff --git a/crates/clawhdf5/src/lazy.rs b/crates/clawhdf5/src/lazy.rs index 3232ee6..8746f5f 100644 --- a/crates/clawhdf5/src/lazy.rs +++ b/crates/clawhdf5/src/lazy.rs @@ -46,6 +46,9 @@ pub struct LazyFile { /// End of the HDF5 data (`Superblock::data_end`, absolute). end: usize, superblock: Superblock, + /// The metadata as libhdf5 reads it when the file holds a metadata + /// cache image (see `superblock_ext::metadata_view`); `None` otherwise. + overlay: Option>, root_header: ObjectHeader, /// Cache of parsed object headers, keyed by address. header_cache: RefCell>, @@ -82,7 +85,13 @@ impl LazyFile { let superblock = Superblock::parse(data, 0)?; // Refuse a truncated file; read nothing past the recorded end of file. let end = base + superblock.data_end(base as u64, whole_len)? as usize; - let data = &reader.as_bytes()[base..end]; + // Decode the superblock extension as libhdf5 does at open, and load + // a metadata cache image over the file's metadata. + let overlay = clawhdf5_format::superblock_ext::metadata_view( + &reader.as_bytes()[base..end], + &superblock, + )?; + let data = overlay.as_deref().unwrap_or(&reader.as_bytes()[base..end]); let root_header = ObjectHeader::parse( data, superblock.root_group_address as usize, @@ -94,6 +103,7 @@ impl LazyFile { base, end, superblock, + overlay, root_header, header_cache: RefCell::new(HashMap::new()), }) @@ -111,7 +121,10 @@ impl LazyFile { } fn hdf5_bytes(&self) -> &[u8] { - &self.reader.as_bytes()[self.base..self.end] + match &self.overlay { + Some(v) => v, + None => &self.reader.as_bytes()[self.base..self.end], + } } /// Returns a reference to the parsed superblock. diff --git a/crates/clawhdf5/src/mmap_file.rs b/crates/clawhdf5/src/mmap_file.rs index 2a8400b..05d581a 100644 --- a/crates/clawhdf5/src/mmap_file.rs +++ b/crates/clawhdf5/src/mmap_file.rs @@ -38,6 +38,9 @@ pub struct MmapFile { /// End of the HDF5 data (`Superblock::data_end`, absolute). end: usize, superblock: Superblock, + /// The metadata as libhdf5 reads it when the file holds a metadata + /// cache image (see `superblock_ext::metadata_view`); `None` otherwise. + overlay: Option>, } impl MmapFile { @@ -50,18 +53,28 @@ impl MmapFile { let superblock = Superblock::parse(data, 0)?; // Refuse a truncated file; read nothing past the recorded end of file. let end = base + superblock.data_end(base as u64, whole_len)? as usize; + // Decode the superblock extension as libhdf5 does at open, and load + // a metadata cache image over the file's metadata. + let overlay = clawhdf5_format::superblock_ext::metadata_view( + &reader.as_bytes()[base..end], + &superblock, + )?; Ok(Self { reader, base, end, superblock, + overlay, }) } /// The file's bytes from the superblock on — the space HDF5 addresses /// index into. fn hdf5_bytes(&self) -> &[u8] { - &self.reader.as_bytes()[self.base..self.end] + match &self.overlay { + Some(v) => v, + None => &self.reader.as_bytes()[self.base..self.end], + } } /// Size of the user block before the superblock (0 for most files). diff --git a/crates/clawhdf5/src/reader.rs b/crates/clawhdf5/src/reader.rs index 40ac08b..c892eea 100644 --- a/crates/clawhdf5/src/reader.rs +++ b/crates/clawhdf5/src/reader.rs @@ -20,6 +20,7 @@ use clawhdf5_format::message_type::MessageType; use clawhdf5_format::object_header::ObjectHeader; use clawhdf5_format::signature; use clawhdf5_format::superblock::Superblock; +use clawhdf5_format::superblock_ext; use crate::error::Error; use crate::types::{AttrValue, DType, classify_datatype, read_attrs}; @@ -55,6 +56,11 @@ struct FileData { base: usize, /// End of the HDF5 data in the file (`Superblock::data_end`, absolute). end: usize, + /// The file's metadata as libhdf5 reads it when the file holds a + /// metadata cache image: the bytes from the superblock to the end of + /// file with the image's entries written in + /// ([`superblock_ext::metadata_view`]). `None` for every other file. + overlay: Option>, } impl FileData { @@ -68,11 +74,26 @@ impl FileData { let end = superblock.data_end(base as u64, whole.len() as u64)?; // data_end is at most the file length (less the user block). let end = base + end as usize; - Ok((Self { backing, base, end }, superblock)) + // libhdf5 decodes the superblock extension at open (a message it + // cannot decode fails the open) and loads a metadata cache image + // over the file's own metadata. + let overlay = superblock_ext::metadata_view(&whole[base..end], &superblock)?; + Ok(( + Self { + backing, + base, + end, + overlay, + }, + superblock, + )) } fn as_bytes(&self) -> &[u8] { - &self.backing.whole_file()[self.base..self.end] + match &self.overlay { + Some(v) => v, + None => &self.backing.whole_file()[self.base..self.end], + } } fn len(&self) -> usize { diff --git a/crates/clawhdf5/tests/fixtures/h5clear_mdc_image.h5 b/crates/clawhdf5/tests/fixtures/h5clear_mdc_image.h5 new file mode 100644 index 0000000000000000000000000000000000000000..6ed8b702cfbd9ae5f61ea2698fb65d404cddf7ef GIT binary patch literal 23467 zcmeI)f1H%_{{Qi5`(dqqtZZ3HrX;IhR;?^nrfgZMtRySRsFg*s5@(UlU}+_6>9DdG z35yOZ$x572o$MhjhpZHXBq57*Rwwm+Jeu+Oer})J_Yb$*_pi_Ax)`>??pPyJ~Cp%tmof1ht{A2aU9r)u8{BZ~V zcin+A3-ZUL1Q)dbx#&OZoOxiNJP;h0xEFUM2hYHmU5`%q-*tQb_}Bi=z5^}z=cS+} zTESO(8|()^J?sw`_p#qT+tbh%?U9a-aPc_qc|God@cakCb99F1=>pHy4MFsP=j;XBx+k}fV%;14;Af_R7=#m$ zg}$&o2f4^Y2>B?$Nc`T#>+?Q&jR(W)JOo~=_s#1)44&6(^4$I5`3Jyj@Y=kN6XCU- zf}t1&ujdSm##tDPb5WRlT*P_;!YD=wrlJ%x@Ou~U-{Av&v%FeYOPE``^2C8pyVlwl?!C`Se6q7n;Gg=+lXC5s)8hWEM; z`ohn>2 zyxynr44y+h)}aCKqY)bs$7XEBxA?tF9y^{4Uo*b0Ll^>IJEtNazRtbR*JCEU*Ad)| za@>Xrc+b73FJc8=K@6+07XN_P{tlY31I^ftKnun%q@pd-k%8a43}?sFP=M1h0=_Rt zVibJMpNYA+3ze9U1@PWiVIivV5Z=QF@cw>`Pw*MM$6sL^zC$M0*cCzaL>BrY8#%~D z9)9mq$c~O9V{kUc;v9^F<4O_ELlnM77UOZ$;3+JDua#OXh4=XOh6dL_`S<`c03;w;5ai87b1*{FbR|4>t#9W;A>_jUPTP6um-+% z+H>v$;cMbxbirZpz8`^JI2t9Gic-u#8D=7aa#Ub0Tv{Uy9gu-T&<#f*3yw=JZO|4S zkqP(diQdS@Ah_&@_UMGJ@Hp<{c$N#_UoNRghsO%Sb9o&1384Tk`@?e{3eV^DcrK6Q zKE6%{z;P!JLogh^?hE1T#=nF2-TtBQdA=8XUD+4L6imf*_9o2%`k{mmz`*RH6!=V=_GcHSqbfVZYg4tc)-ia@b$AaR_Y-V}$F}Vw#PJoj<7arjmYgpQ zzAu{CZbpE|zNZ~O9h-v4g5!~6k?$klKeqdR8Gw`Edw&$h;X-&1-h$`<5F7Csw%{9h zKCi>|PIx_zDW2PN_&)VL>Un%`dM$@1ANOOO1J64Yr^E5)T!c`N+%94rMhSdBmmz`* z*mf1J$1U(Z{TDohCs2!*u#4;3i&i||A70x*@Om6Sdf+JZh4&&CgW>%c0k6w@;utsy z6LB#dW3GnhzX@K`-MAMIq7qezq6S`19b%|Q0~)akZ{mGy!WMjwX0%A*?~8-r^&N{r z@Vo1ZeVK+C@Y=jzx54`}A6}F9Xfd9~Qh1-duGjH4KEOtNfjG9J30_lx zYe_{qG7&@;y5dOm$MFc^bd16ID8?1={!ND0c?G89PjHO(K63wW@8vr z;nEIWa1?Sd1ZJ%HSpsv1d8|n&g}GLYX(+?Zm#tI>c>FpK^Lug7zF9QRoRGa`=d z*o`z^FZHZ#--s<}!Y;I;uRSu+1NNJ1KjK%U(&qTw4dzo0PJz$&*!E{22*<=+ASc-)Tg zoW0=l9gp%c8nzt^&vzOen=XLIy&AJ{JBm;YuVW@EumGNK2^>?`U=HrXVmt?P;7vG& zeTg5j7c21wY~O+%NMU?(JnM$OI1!$A5~jj`X9S+Zb3cR{EKP1d%=#%fZpPqw*W)vM zi+`a38?hChGr&38!ZEZbvf&tN+ac%$$JG!trtA7{ezUkuMX7GX?98Ol)!&s~FM@LC>7Zoj~K z72d%{e2txG*^2iI>(9}IJ!nf?7i3`&@-YTM`plXf42IV+1`{v^rMN!1Hg9gjJa|n{ zU@2ZfJ^qOp)}axb;rVwXm2+iyKJ-O84nuDo53lDeOvL3V!>tHo8fIb+s_;0LVHFzi z3A|U{n<^~A5-f+;^9H;hpCoyoc3>~kcy1?nU47ua8G>ARO@)|%63jpZbK!L?#(X@C zC3p#I@lV9@9d=>A*1TSmk5hTv89gxoA&kZZTn3j8Fw6TP4`*OJtP{<(*>Fim5ZMT! z5Mh)ef=aj?h#nY-e4K+Ru&%9u^{7in^u!<(z;l}Q=Cj9I1eb%*3nySCth1-%R#f3> znB9-T%v}vL@iR2Rx}-g0nRQ5%$F+!|0dcrbfWCADkqtisnD1t{xoqZo9QWyt0XP+& zZ!OGsv)kXQH=9Hg<%@sfQ zhERwwN)bUNqNqg-4HygSD06Z)s^I5mbM8%;W8cBNydGxdL+}{pmN~cu=1&@9jX8Wb z%-I)Uj`}%vJFKHJa1?y5pL63d*US>@g|6rWGsJwdzX<^zrz41LgiweuO5yW6u(sd4 z845qQn<>}eHkcb8zdwAwS!BQCzIn3{W{sKR`A5Oyn^QjD&(HS%4e!CcF+VPW$G8>d zhINXW@fsTNDI%yu6t#%KzBrl?;Ju#?^X4U(T_3|-v0i9T-{Hu{5cu5BVaBv!+wt6S zGY_K?hI!%me*kMU?^rm_`?>ux+=xm%3bVrVm=lM>W0?z%zwYnYdkuo{8bT<9=P5-5 z9xIAkIA%}A4BU!);rLvKe_#W?f#alo=E({;Ry!^>A*EGfjPAwSvD4>m#P`WQ$4T?! zB#g#I@ObyYG50ydU>t9g~-%9vk48JsS)0SH$4>yam6YEo1KCFdL7C$2|?>a4~%TE%4Zn zAdFIYzDh(_Iw@dmsnHQ3Uf~Chme+F$}g} zj#-$8#dr~KU=zHSzu|3cg2(r~Y4ja}EF6yjJZBKu2%!*Rlp=yk*me=7;}$HyV_1&Y z(TJ}R#Zs)ndThlmv~9!R64@AvCfZuE?u?_6hcht=*PsFqzMCG4=x$Vf>~aOVwmIBlu?*jE}hUDr(i71d$ZU1+!{C!>(Uv0F%(6x zCN78dsad)fE(fDOhG7D(#vD9?I=lmyOk~5%55sG#gy)Q*5%t)JCbVMg&xF|+f|+Js znMLOOS1@<2M|;8iEQGmdrkPhB>qn&WxEpd{#)e@wntNv27WT0|&fu|i?NFG*r7%Cu zMsv^W@>oIS!t;4e5ty;Hs7EEt>^iJN96ON8SZr2hBOmVb608TmKr_tC9&o;DBrbx- zai7hwR!pa@7xG})46_k3B{<%(gJfPzf_A2x|}fCZH5%N)`MJAA_IW?Y{tK>}*uS zyjhEnVWzZZob&m&!G3F#dYDCi-uJV95Prt@_|L-UZ-V{iR1chhLREDfX9`}U%j6@h_NCm24&X{ZYI1gr*`QkCmr&ah6KG)1E!(A}fR-gf9 z*KTxVOq<2pJY0g+*nl5konfvG!YI_zW){WKjC9(}f)LD$ec*8%ug$DK!<={$K5sC_!oCRZ#S*N7`LP`y^9Ia^&x_$h*w>n|)p2_u zqHNcp0b3EEEfbFM1qj2jyNtfMaQ|hf$3`@v75zS^FU*mD!ejl4joBhDJ0YmG&TVQHT8bqf5ke9Cw{?Zb%|{WY!sjnQ4OU_j{shnW2$thbc-)_n z#<32E`@RRSV-GTD>kZF04pVV6I-?)*aXzNwc0}&@p}jKNe` z`$n+>@55TNJ)F1Bhx5(my!ElUYh7#2YCW3C+N>{x$Etu?8-um4*_Xk7Jy<*cJptF@ zURY;0!1~zQ*Sgm1HkZv@^VD3t3UgpxXeNGuRG8i7vYBh1y3b-*e>UPrbcOkDcALv) zuE%j7zgIFhJ21{?v-UGU3C!*&JeS9DpLF)=i+mpYIl<2W=6fBy9?#`*+~;&y5AMf0 z2nbz77eX+PXW%Z>U==oEC#)y^-Y5*`?X3CC%XRn)DU9FFznQDW zh``LOgL!M-wWU7`X4Mq7Z$T8VzGW_+%S)ppdMyX zfHuc}b7(9|;TUg*eT_CeHm5?k0A|sBSO)WEJ^OqQbEG>?#Mv-IZpA}*5uRfZ3UL|A zVP750i*L~;HIXwn^0*4m<86G6R`i)KC*mBKOKT9vZe-GygF=`wX2xP{r|$sPX5KJN z!khOw9i&$9?C;jx;KMjn}4ci?fnhK=|a zI?~rKnOT!r$C7=`tTWg)&qksI<%nV>(&6();1byP2wuge2)Gn1TvCg>~4DboOyx+Z?xk zw&pdT%~Ufyjy*8foMRot<8v?re$ToT&I>oe`CR8}M3*22y&r_<-WusP~{?pWHa>#N}B2D9Gl@>oMr4Cj2!Z);ZT(WS7CG#kx5GtIm* zi>!UEYpq$WN3Au@PqWe7GtPFkin1CCw z5HG>wxKDt`L4*)S1X08ghv)J*K^_mmBv@aXn}3J3qu)n)J)SEB_bJEYuts#Q(s@dA z&AfgaW@#GRSuo42*PN4!!WycP{(r%G@HmWx^LEyA&!GX|BY+@62qS_hVu&Na7$1b^ zIUSe5|I3v1qIqdO*n$3RSf3TLJsoD|ax}uZzOM8+#}|fkd*-0+=G-8dV`iW^E@^MmH#;k=?b`T@+zgK0MhogXyk z9)LN#7Upak+vccsnK?Nh=HPmmf4yK0Wvvu}`DQ(|0nM=Hv41jdhILd7pTb({F#7Xg zJ>>JFus@CfZDvmhVfzt<$KM2>Z$8?85v;Ku#Tr;gdHyb}J^n=ad~?hGEl6eT?hSKq z49u^)umom`HCG$@&5fbBp6&Z#P4*#n!o11B*)UV)Ax3)~)@wn85Jm)1#1MzaJ_^=y zmtqdgsde}cX3;S)8>gWXOYuH-B9p!oF#)p>rp@fC#tN8E?w3KI8B~aAsHM$Z+m3c@ zn_Xw%GMG;_cpd(j%ghTQj0mELAr61U6@}n2Ck3uQltQI2Ds{6QcMVKEbc>d0|9gUkq{h@A`MOcj;^mS&Pi}9F=hY-W(XvL4wdijGimLo76mtrog%gu6s zw`=|Be6&kX6ksYUVdh(JTSo`k$6D6qNQ}UhSb&%C30l&hg;B}-Db*^=%-?N&Lem`p7o4vJoAJ)%i zeF$c%_3IK$r`_LUt%Sd$YD1gf`xfFF_+9MZFq6JTsE2iZ25o+B2*Z5#Snv6mKJ(sl z{u|cTet%qw`(VB7_qEo<&Nn-k>^!mgZCz{4YCUSLX&q_KTQ{4#c?iP{t${VG^{BPx zX7+L3x<8z6E`#;4wXbz;TlTRYZDv2`w)5efbS12zHzI|8>so78KUX;aZO%Jy?fI;; zt)Z=t^I=_UcALv#n5Sl?b*#DAfm|5?=ReJNvpWJa*F1Hfboz2o1ZzM)1DNkon9F9a z$8n$YFbj{u&k248h{5bOmpzxqai7Jo&h)c{pA+IR-_35X$8&ic_xSv()F!fc2hP7Dp@kt=C4vdh$Y8FFpzD!JlBgcQPhnHmo5dqrFNONo| z%qeRpbM#%9lZUb$!o@J>o`pH~3(VQ}tj*E!h`_qc9Q+aH++cW)GhohIpT+PM%*o?m z4&H+Kw7mfH@89S~+Ynfr&4$_X7VO`RAdmA=0<)(UW>_=2nggs)h5dJ8DL%w*bfm(>15j= zWb^o3%)(<>ho8`ez9A^V-55>VRak%*u>rp#NZ+ZLg4)4J?9eACy zWT6n&=26tcy4)-e!3=ac8e?z+7UM1K#NqT8;Hu<)y=fbZS*U@Z4b1#r^cTXpYnMJ4 zhnuhjA0mamekj5%aOsO8l%p0KkxE|bT=WDHs_a}J9b{!IbY{dK8Gq21dYhUZyAlufX)|%$0*=X*WY37v~Z|xg`HLLZg z8Ebx;jpm-2md3XAaTwOM)~x2R8Ebx;jpm-$G!_w9`!>R?Hiykv^V4h$p#H9_H5_EY@G|gi;E+T?C@M3$9+yfG493+ ze2H}SISF2m=khr2Qwj5TD>CR0!FkjMwwn>;aRJP`Dwv06mGhFhNa1lGjKz&`-t!$a zqdWa0a5creA`Da=eWH5=w#IiA53+UH>>=!#1HS+ve^J zL}6w&qbJ!BW?cd6J+m%ATNbR>%3wWajb**~cUTV|%ytNu!g}syY(_gW!DFQG*m}`9 z&zIiec#HsTePO*=4(mZ{F>^4JwK-P; zbF2>Llo@Ca6~Uam5a!@>Fy~s*ZjPM;bLt70qmRR!+y!&cj56n}@ywu|FlWC*CXY{r zIe9aZE+WFE}H1&GjQ#xXb61%2-yD+>hUAG(l;EXSm>Iz2jKI6NcNvb`*b{rdi(^>Z@zl`eE9ro*uMio zGH@i)d7O(dDo~3?_+#-b6rv25(RL43;!Cur?RbQ7JC@>Otf9}0&E)Y=OvSy3VH+~i z$sTm$@#&a>hw%n}Mv%TUa4jBz=U$8k>_!%CV-dj;G@_M1L}%#-=h|n(n)@@fCwr{R z&2qmtcIl4^n1ecOK?Z$8QHqC>`yE63MBIs$_y%3*I~`?sBDr5S?O{|Rh9(5*GxH;; zP40Ir?UUf|?$*G013w#_iwgL=Fz42t=eE|i?lpV;p4c4scfHP4`#aus?6V75w2ebK zoY#)S`QtoTpRQ-0*0kkdGUnqoG{N5?kH(FNv!59qLMfu~oB{f>5k@7PFF2HTKNsAH zXJ8$kPTvrixsSpc+dABNV}Ji-hF=WpNxx?`*Zh9etT&&{)E=}IViuNQ15(*%5SGyI z@3GQo^LJDw@O$62@cY`H?Bn;WG4^RfkT#DMftehKwQ>lhaDJdQZT@D<-(&fCz`A=E z{N8s0?u2vF&NoNlJh5}V*0t8G)}z*%){(`u&qoZ-C3mIG+P4hWtk$E}n$B%IZyks8 z&CVrTA6LV=)|%CN)blkUz_!^Oh8Y?|6N2nh2tNxr{~e^wd28pSt+UNp>tkzQ>sqth zTsCvfQ?t_V3p^7IpT=Ud@^5Jh>qVTsZeg-h#&2Dqq%=I|#^9UNS z7ZLhu5l1@P=CbGVIPMc5hs;htOZYj#&j9AT+3odsE|23r7s0t)zlXGbJPXc|I?rmJ z`MsWVqZeW~`#2w5gxg>Z`#I9-v;Lflh3HS)1(=JMu?>gNUVuNr@Ap1{`CEw?nh>PT z%riG@5l1@tV!gT=J8&dzW~+5-9lk~v_BjLArZ2&HPwPVS(7JIUUWYYbcXGo_-wEeM z&Ep83fmxbHe=eq=725+b3G)!c_vl97C^)bAG(1l!qA)W9w3)f)T_s{@g1P!vIPcnl z?GUW{%+!CvdC}uw*4_ZK@B>)S9Ruq%>p<(VFJQg63Ld)~+tz#6qXz3?y*2>Wll!x6 zy?6<%2mcQ1y(8$ip1TR=Wg#M{MI2_C^`Kd2z1INi(feSY{sQaC^I^UC3e2j*$rYb_ zGR(RAV2=F&bLtG3Lyy6nybI>wHkfn6VU8_=IkgA1^nD6*@?@BU55S!J1?JefFlV!2 zj#k2)Y=Svh2y?F1$F$u9bM_0Eqr+fMJ^^!Ze{%3W`u3tP+vnjnJdZfq(SIT?f&JrA zf#ukYj`@=rZIQOR(O0E2R%a|@zp-?EqG)8t0iWZUo|1^J9}X~oHH=Z;SYgPl;{x48bTX8;M{GJHt>Q2tiG?ZUX>QlDJ=o_8tYJ?IkjJ;Sm!xvu5)>QcsQ z%f;89ODcZ@mxkoJihJl1=HKg*#lO*|DY>pYD6t*MNURI|(LPIh=ft`(xsLkddzO;S z#5&s_4YD*R*Rd{%?FxVV$5NQQL7Dz|k0qX5*K|v4m-^!amfYaJ6@yM+AuEb#rnZ8<^Ox$VsdVk4vmG2PM|=e7^4*9}W-M}{ZX1*av}=>>^(V{#qkUExwPBC*asBe8Byu45w;+ZCe{>%!5A zb>^9gbv(JQDNJmao|RbVj!CTd{YSERSbuh6yV40QmZEbK>o-#`tZMxq@23Rqj|8x1kJ39T}JLlk&d@1j%2mB}ezyC9@WhwmA8PxyF plE!V}(_sgH`O_-@>|tYn`#S`ZuOy%J|FsPM_1UfiH`RUhe*p5$_ErD@ literal 0 HcmV?d00001 diff --git a/crates/clawhdf5/tests/header_validation_interop.rs b/crates/clawhdf5/tests/header_validation_interop.rs index 0bfd30e..74dc419 100644 --- a/crates/clawhdf5/tests/header_validation_interop.rs +++ b/crates/clawhdf5/tests/header_validation_interop.rs @@ -42,8 +42,9 @@ macro_rules! skip_if_no_python { /// Runs `body` (Python, with `h5py`, `numpy as np`, `struct` imported and /// `d` the output directory) and then, for every `NAME.h5` it wrote, -/// prints `NAME ok` when h5py opens and reads dataset `d` and `NAME ERROR` -/// otherwise. Returns those lines, sorted. +/// prints `NAME ok` when h5py opens and reads dataset `d` (or the dataset +/// the body names in `DSET`) and `NAME ERROR` otherwise. Returns those +/// lines, sorted. fn h5py_verdicts(dir: &Path, body: &str) -> Vec { let script = format!( r#" @@ -54,7 +55,7 @@ for path in sorted(glob.glob(os.path.join(d, "*.h5"))): name = os.path.basename(path)[:-3] try: with h5py.File(path, "r") as f: - f["d"][()] + f[globals().get("DSET", "d")][()] print(name, "ok") except Exception: print(name, "ERROR") @@ -78,14 +79,14 @@ for path in sorted(glob.glob(os.path.join(d, "*.h5"))): lines } -/// Whether clawhdf5 opens and reads dataset `d` of `path` (as raw bytes of -/// whatever type it has). -fn clawhdf5_reads(path: &Path) -> Result<(), String> { +/// Whether clawhdf5 opens and reads dataset `dset` of `path` (as raw bytes +/// of whatever type it has). +fn clawhdf5_reads(path: &Path, dset: &str) -> Result<(), String> { let file = File::open(path).map_err(|e| format!("open: {e}"))?; - let ds = file.dataset("d").map_err(|e| format!("dataset: {e}"))?; + let ds = file.dataset(dset).map_err(|e| format!("dataset: {e}"))?; ds.dtype().map_err(|e| format!("dtype: {e}"))?; ds.shape().map_err(|e| format!("shape: {e}"))?; - file.read_multi(&["d"]) + file.read_multi(&[dset]) .map(|_| ()) .map_err(|e| format!("read: {e}")) } @@ -93,10 +94,15 @@ fn clawhdf5_reads(path: &Path) -> Result<(), String> { /// h5py's verdict for each file must be `expected`, and clawhdf5 must read /// exactly the files h5py reads. fn assert_agrees_with_h5py(dir: &Path, verdicts: &[String], expected: &[&str]) { + assert_agrees_with_h5py_on(dir, verdicts, expected, "d"); +} + +/// [`assert_agrees_with_h5py`] reading dataset `dset`. +fn assert_agrees_with_h5py_on(dir: &Path, verdicts: &[String], expected: &[&str], dset: &str) { assert_eq!(verdicts, expected, "h5py's view changed"); for line in verdicts { let (name, verdict) = line.split_once(' ').unwrap(); - let ours = clawhdf5_reads(&dir.join(format!("{name}.h5"))); + let ours = clawhdf5_reads(&dir.join(format!("{name}.h5")), dset); match verdict { "ok" => assert!(ours.is_ok(), "{name}: h5py reads it, we fail: {ours:?}"), _ => assert!(ours.is_err(), "{name}: h5py refuses it, we read it"), @@ -244,7 +250,7 @@ for libver in ("earliest", "latest"): ], ); for name in ["earliest_size2", "latest_size8"] { - let err = clawhdf5_reads(&dir.path().join(format!("{name}.h5"))).unwrap_err(); + let err = clawhdf5_reads(&dir.path().join(format!("{name}.h5")), "d").unwrap_err(); assert!( err.contains("stored datatype size in chunk layout"), "{name}: {err}" @@ -557,3 +563,68 @@ for name, n in (("overflow", (1 << 62) + 2), ("pasteof", 1000)): assert!(mm.dataset("d").is_err(), "{name}: MmapFile"); } } + +/// libhdf5 decodes the superblock extension's messages when it opens a file +/// and refuses the file when one does not decode: cve-2020-10810 (a File +/// Space Info message too short for what it announces), cve-2020-10812 (a +/// metadata cache image past the end of the file). We did not look at those +/// messages and opened such files. +#[test] +fn superblock_extension_messages_libhdf5_refuses_are_refused() { + skip_if_no_python!(); + let dir = tempfile::tempdir().unwrap(); + let mdc = Path::new(env!("CARGO_MANIFEST_DIR")).join("tests/fixtures/h5clear_mdc_image.h5"); + let body = format!( + r#" +{FIX_OHDR_PY} +def ext_addr(buf): + assert buf[8] in (2, 3) + return int.from_bytes(buf[20:28], "little") +def save(name, buf): + open(os.path.join(d, name + ".h5"), "wb").write(buf) + +# A paged file: its superblock extension holds a File Space Info message +# (version 1, strategy page, not persisting, threshold 1, page size 4096). +DSET = "DSET" +good = os.path.join(d, "paged_good.h5") +with h5py.File(good, "w", libver="latest", fs_strategy="page", fs_page_size=4096) as f: + f.create_dataset("DSET", data=np.arange(10, dtype=" ext +# Page size 256 (under libhdf5's minimum of 512). +bad = bytearray(data); bad[at + 8:at + 16] = struct.pack(" ext +length = at + 5 + 8 +bad = bytearray(data); bad[length:length + 8] = struct.pack(" PathBuf { + PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("tests/fixtures/h5clear_mdc_image.h5") +} + +fn expected() -> Vec { + (0..50).flat_map(|i| (0..100).map(move |j| i * j)).collect() +} + +#[test] +fn file_reads_through_the_cache_image() { + let file = File::open(fixture()).unwrap(); + assert_eq!(file.root().datasets().unwrap(), ["DSET"]); + let ds = file.dataset("DSET").unwrap(); + assert_eq!(ds.shape().unwrap(), [50, 100]); + assert_eq!(ds.read_i32().unwrap(), expected()); + + let file = File::from_bytes(std::fs::read(fixture()).unwrap()).unwrap(); + assert_eq!( + file.dataset("DSET").unwrap().read_i32().unwrap(), + expected() + ); +} + +#[test] +fn mmap_and_lazy_files_read_through_the_cache_image() { + let mm = MmapFile::open(fixture()).unwrap(); + assert_eq!(mm.dataset("DSET").unwrap().read_i32().unwrap(), expected()); + let lazy = LazyFile::from_bytes(std::fs::read(fixture()).unwrap()).unwrap(); + assert_eq!( + lazy.dataset("DSET").unwrap().read_i32().unwrap(), + expected() + ); +} From d110b1d9451945162356267e6f49b0b607ea8e63 Mon Sep 17 00:00:00 2001 From: osobh Date: Sat, 26 Sep 2026 10:31:52 -0500 Subject: [PATCH 19/43] fix(format): scale-offset and shuffle decode chunks as libhdf5 does Scale-offset (filter 6) now follows H5Z__filter_scaleoffset: - the packed codes start at byte 21 whatever size the chunk records for minval (libhdf5 reads min(8, size) bytes of minval and always starts the codes at buf_offset 21). We started them after minval plus 8 bytes, so a chunk recording a size of 0 (cve-2025-44905 /Scale_offset_short_data_be) decoded differently from h5py; - with a fill value defined, a code equal to the all-ones code of minbits bits is the fill value, including minbits 0 (code 0): a chunk of nothing but fill values read as minval; - minbits of the full width stores the elements as they are (no minval added), and an integer scale factor of the full width means the chunk was left untouched; minbits or a scale factor wider than the type is an error; - the class parameter (integer or float) decides the decode, a scale type that does not match it is refused, and E-scale is refused, as in libhdf5 (no library writes it; it was decoded here unchecked); - minval is the stored bytes zero-extended, as libhdf5 reads it. Codes past the end of the chunk stay an error, as in libhdf5 releases after 2.0 ("Buffer too short"; 2.0 reads past the buffer, cve-2025-2308). Shuffle (filter 2) uses its own parameter as the element size, as libhdf5 does, instead of the dataset's element size; a parameter larger than the chunk leaves the chunk as it is (cve-2025-44905 /Shuffle_float_data_be), and a parameter of 0 is an error. Co-Authored-By: Claude Opus 5.5 (1M context) --- crates/clawhdf5-format/src/filters.rs | 376 ++++++++++++++++---------- 1 file changed, 236 insertions(+), 140 deletions(-) diff --git a/crates/clawhdf5-format/src/filters.rs b/crates/clawhdf5-format/src/filters.rs index 36f2e96..45b54b2 100644 --- a/crates/clawhdf5-format/src/filters.rs +++ b/crates/clawhdf5-format/src/filters.rs @@ -183,7 +183,7 @@ pub(crate) static BUILTIN_FILTERS: &[BuiltinFilter] = &[ BuiltinFilter { id: FILTER_SHUFFLE, name: "shuffle", - decode: |d, c| shuffle_decompress(d, c.element_size), + decode: |d, c| shuffle_decompress(d, shuffle_type_size(c.client_data(), c.element_size)?), encode: Some(|d, c| shuffle_compress(d, c.element_size)), }, BuiltinFilter { @@ -280,23 +280,6 @@ pub(crate) static BUILTIN_FILTERS: &[BuiltinFilter] = &[ }, ]; -/// Decode the HDF5 scale-offset filter (id 6). -/// -/// Supports all three scale-offset variants: -/// - `H5Z_SO_FLOAT_DSCALE` (0): `value = minval + code / 10^D` -/// - `H5Z_SO_FLOAT_ESCALE` (1): `value = minval + code * 2^E` -/// - `H5Z_SO_INT` (2): `value = minval + code` -/// -/// Compressed buffer layout: `minbits` (u32 LE) · `minval_width` (1 byte) -/// · `minval` (`minval_width` bytes) · 8 reserved bytes · MSB-first packed -/// codes (`nelmts * minbits` bits). The all-ones code is reserved for the -/// defined fill value. -/// -/// `cd` is the `H5Zscaleoffset.c` parameter block: `[0]`=scale type, -/// `[1]`=scale factor (decimal digits D for D-scale, binary exponent E for -/// E-scale, interpreted as i32 for negative exponents), `[2]`=element count, -/// `[4]`=element size, `[5]`=signed flag, `[6]`=byte order (1 = big-endian), -/// `[7]`=fill defined, `[8..]`=fill value bits. /// `f64::powi` equivalent that works under `no_std` (no libm/std available). /// Exponentiation by squaring, matching `powi`'s semantics for negative /// exponents via reciprocal. @@ -318,6 +301,32 @@ fn powi_f64(base: f64, mut exp: i32) -> f64 { if neg { 1.0 / result } else { result } } +/// Decode the HDF5 scale-offset filter (id 6) as libhdf5 does +/// (`H5Z__filter_scaleoffset`, reverse direction). +/// +/// - Integers (`H5Z_SO_INT`): `value = minval + code`. +/// - Floats, D-scale (`H5Z_SO_FLOAT_DSCALE`): `value = code / 10^D + min`. +/// - Floats, E-scale: refused, as libhdf5 refuses it ("E-scaling method not +/// supported"); no library writes it. +/// +/// Compressed buffer layout: `minbits` (u32 LE) · the size of `minval` in +/// bytes (1 byte; libhdf5 uses at most 8 of them) · `minval` · packed codes +/// at byte 21, whatever the stored size of `minval` (`buf_offset` is fixed) +/// · MSB-first, `minbits` bits per element. With a fill value defined, the +/// all-ones code of `minbits` bits is the fill value — for `minbits == 0` +/// that is every element. `minbits` equal to the element's full width means +/// the elements are stored as they are (in little-endian order), and an +/// integer scale factor of the full width means the filter left the chunk +/// untouched. +/// +/// `cd` is the `H5Zscaleoffset.c` parameter block: `[0]` scale type, `[1]` +/// scale factor, `[2]` element count, `[3]` class (0 integer, 1 float), +/// `[4]` element size, `[5]` signed, `[6]` byte order (1 = big-endian), `[7]` +/// fill defined, `[8..]` fill value bits. +/// +/// Packed data too short for its codes is an error (libhdf5 2.0 read past +/// the end of the chunk buffer — `cve-2025-2308` — and later releases +/// refuse it: "Buffer too short"). fn scaleoffset_decompress( data: &[u8], cd: &[u32], @@ -326,74 +335,101 @@ fn scaleoffset_decompress( const H5Z_SO_FLOAT_DSCALE: u32 = 0; const H5Z_SO_FLOAT_ESCALE: u32 = 1; const H5Z_SO_INT: u32 = 2; + /// Where the packed codes start (`buf_offset` in `H5Zscaleoffset.c`). + const BUF_OFFSET: usize = 21; + let err = |why: &str| FormatError::ChunkedReadError(format!("scale-offset: {why}")); if cd.len() < 8 { - return Err(FormatError::ChunkedReadError( - "scale-offset: missing filter client data".into(), - )); + return Err(err("missing filter client data")); } let scale_type = cd[0]; - let is_float = scale_type == H5Z_SO_FLOAT_DSCALE || scale_type == H5Z_SO_FLOAT_ESCALE; - if scale_type != H5Z_SO_INT && !is_float { - return Err(FormatError::UnsupportedFilter(FILTER_SCALEOFFSET)); + let is_float = match cd[3] { + 0 => false, + 1 => true, + _ => return Err(err("cannot use C integer datatype for cast")), + }; + if is_float && scale_type != H5Z_SO_FLOAT_DSCALE && scale_type != H5Z_SO_FLOAT_ESCALE + || !is_float && scale_type != H5Z_SO_INT + { + return Err(err("invalid scale type")); + } + if scale_type == H5Z_SO_FLOAT_ESCALE { + return Err(err("E-scaling method not supported")); } let nelmts = cd[2] as usize; let elem_size = cd[4] as usize; - if elem_size == 0 || elem_size > 8 || (is_float && elem_size != 4 && elem_size != 8) { - return Err(FormatError::ChunkedReadError( - "scale-offset: unsupported element size".into(), - )); + let size_ok = if is_float { + matches!(elem_size, 4 | 8) + } else { + matches!(elem_size, 1 | 2 | 4 | 8) + }; + if !size_ok { + return Err(err("cannot use C integer datatype for cast")); + } + let full_bits = elem_size * 8; + // An integer's scale factor is the number of bits kept; all of them + // means the filter stored the chunk as it was. + if !is_float && (cd[1] as i32).max(0) as usize > full_bits { + return Err(err("minimum number of bits exceeds maximum")); + } + if !is_float && cd[1] as i32 == full_bits as i32 { + return Ok(data.to_vec()); } // The decoded output must match the chunk's uncompressed size; reject an // element count that would over-allocate (e.g. minbits == 0 with a huge // nelmts and no packed payload to bound it). let out_bytes = nelmts .checked_mul(elem_size) - .ok_or_else(|| FormatError::ChunkedReadError("scale-offset: size overflow".into()))?; + .ok_or_else(|| err("size overflow"))?; if expected_bytes != 0 && out_bytes > expected_bytes { - return Err(FormatError::ChunkedReadError( - "scale-offset: element count exceeds chunk size".into(), - )); + return Err(err("element count exceeds chunk size")); } let signed = cd[5] == 1; let big_endian = cd[6] == 1; let fill_defined = cd[7] == 1; - // --- header: minbits, then minval, then 8 reserved bytes --- + // --- header: minbits, then the size of minval and minval --- if data.len() < 5 { - return Err(FormatError::ChunkedReadError( - "scale-offset: truncated header".into(), - )); + return Err(err("buffer too short")); } let minbits = u32::from_le_bytes([data[0], data[1], data[2], data[3]]) as usize; - let minval_width = data[4] as usize; - let minval_end = 5 + minval_width; - if data.len() < minval_end { - return Err(FormatError::ChunkedReadError( - "scale-offset: truncated minval".into(), - )); + if minbits > full_bits { + return Err(err("minimum number of bits exceeds size of type")); } - let minval_bytes = &data[5..minval_end]; + let minval_size = usize::from(data[4]).min(8); + let minval_bytes = data + .get(5..5 + minval_size) + .ok_or_else(|| err("buffer too short"))?; + let minval = minval_bytes + .iter() + .rev() + .fold(0u64, |acc, &b| (acc << 8) | u64::from(b)); - // --- unpack the per-element codes (MSB-first), shared by both variants --- - if minbits > 64 { - return Err(FormatError::ChunkedReadError( - "scale-offset: implausible minbits".into(), - )); + // Full precision: the elements follow as they were, little-endian. + if minbits == full_bits { + let raw = data + .get(BUF_OFFSET..) + .and_then(|d| d.get(..out_bytes)) + .ok_or_else(|| err("buffer too short"))?; + let mut out = raw.to_vec(); + if big_endian { + for e in out.chunks_exact_mut(elem_size) { + e.reverse(); + } + } + return Ok(out); } + + // --- unpack the per-element codes (MSB-first) --- let codes: Vec = if minbits == 0 { - // No packed payload: every element equals minval. + // No packed payload: every code is 0. vec![0u64; nelmts] } else { - let packed = data.get(minval_end + 8..).ok_or_else(|| { - FormatError::ChunkedReadError("scale-offset: truncated packed data".into()) - })?; + let packed = data.get(BUF_OFFSET..).unwrap_or(&[]); let need_bits = nelmts .checked_mul(minbits) - .ok_or_else(|| FormatError::ChunkedReadError("scale-offset: size overflow".into()))?; + .ok_or_else(|| err("size overflow"))?; if packed.len() * 8 < need_bits { - return Err(FormatError::ChunkedReadError( - "scale-offset: packed data too short".into(), - )); + return Err(err("packed data too short")); } let mut out = Vec::with_capacity(nelmts); let mut bitpos = 0usize; @@ -408,35 +444,28 @@ fn scaleoffset_decompress( } out }; - // The fill code (all ones) only exists when there are bits to pack. - let has_fill_code = fill_defined && minbits > 0 && minbits < 64; - // Computed for all 1..=64 widths; `1 << 64` would overflow, so saturate. - let fill_code: u64 = if minbits == 0 { - 0 - } else if minbits >= 64 { - u64::MAX - } else { - (1u64 << minbits) - 1 + // With a fill value defined, the all-ones code of `minbits` bits (0 + // when minbits is 0) stands for it. minbits < 64 here. + let fill_code: u64 = (1u64 << minbits) - 1; + let fill_bits = || { + let lo = u64::from(*cd.get(8).unwrap_or(&0)); + let hi = u64::from(*cd.get(9).unwrap_or(&0)); + lo | (hi << 32) }; if is_float { - let is_escale = scale_type == H5Z_SO_FLOAT_ESCALE; let scale_factor = cd[1] as i32; - let minval = read_le_float(minval_bytes, elem_size); + let minval = bits_to_float(minval, elem_size); let fill_value = if fill_defined { - let lo = *cd.get(8).unwrap_or(&0) as u64; - let hi = *cd.get(9).unwrap_or(&0) as u64; - bits_to_float(lo | (hi << 32), elem_size) + bits_to_float(fill_bits(), elem_size) } else { 0.0 }; let values: Vec = codes .iter() .map(|&code| { - if has_fill_code && code == fill_code { + if fill_defined && code == fill_code { fill_value - } else if is_escale { - minval + code as f64 * powi_f64(2.0, scale_factor) } else if elem_size == 4 { // H5Z_scaleoffset_modify_3/4 for `float`: the code is // read as an `int` and everything is single precision, @@ -456,21 +485,19 @@ fn scaleoffset_decompress( .collect(); Ok(write_floats(&values, elem_size, big_endian)) } else { - let minval = read_le_int(minval_bytes, signed); let fill_value: i64 = if fill_defined { - let lo = *cd.get(8).unwrap_or(&0) as u64; - let hi = *cd.get(9).unwrap_or(&0) as u64; - sign_extend(lo | (hi << 32), elem_size, signed) + sign_extend(fill_bits(), elem_size, signed) } else { 0 }; let values: Vec = codes .iter() .map(|&code| { - if has_fill_code && code == fill_code { + if fill_defined && code == fill_code { fill_value } else { - minval.wrapping_add(code as i64) + // `(type)(buf[i] + minval)`: wraps at the element width. + (code.wrapping_add(minval)) as i64 } }) .collect(); @@ -478,21 +505,6 @@ fn scaleoffset_decompress( } } -/// Read a little-endian float of `size` bytes (4 = f32, otherwise f64) as f64. -fn read_le_float(bytes: &[u8], size: usize) -> f64 { - if size == 4 { - let mut b = [0u8; 4]; - let n = bytes.len().min(4); - b[..n].copy_from_slice(&bytes[..n]); - f32::from_le_bytes(b) as f64 - } else { - let mut b = [0u8; 8]; - let n = bytes.len().min(8); - b[..n].copy_from_slice(&bytes[..n]); - f64::from_le_bytes(b) - } -} - /// Interpret the low bits of `raw` as an IEEE float of `size` bytes. fn bits_to_float(raw: u64, size: usize) -> f64 { if size == 4 { @@ -525,16 +537,6 @@ fn write_floats(values: &[f64], elem_size: usize, big_endian: bool) -> Vec { out } -/// Read a little-endian integer of `bytes.len()` bytes, sign-extending when -/// `signed`. Used for the scale-offset `minval` field. -fn read_le_int(bytes: &[u8], signed: bool) -> i64 { - let mut raw: u64 = 0; - for (i, &b) in bytes.iter().enumerate().take(8) { - raw |= (b as u64) << (i * 8); - } - sign_extend(raw, bytes.len().min(8), signed) -} - /// Interpret the low `size` bytes of `raw` as a (possibly signed) integer. fn sign_extend(raw: u64, size: usize, signed: bool) -> i64 { if size == 0 || size >= 8 { @@ -1216,6 +1218,23 @@ fn zstd_compress(data: &[u8], level: u32) -> Result, FormatError> { /// Unshuffle (decompress direction): reconstruct interleaved element bytes. /// On disk: all byte-0s of each element together, then all byte-1s, etc. /// Output: elements in natural order. +/// The element size the shuffle filter works with: its parameter, as +/// libhdf5 uses it (`H5Z__filter_shuffle`), not the dataset's element size. +/// They are the same in every file a library wrote; a corrupt parameter +/// larger than the chunk makes libhdf5 leave the chunk as it is, and so +/// does [`shuffle_decompress`] (`cve-2025-44905`'s `Shuffle_float_data_be`). +/// A zero parameter is an error ("invalid shuffle parameters"); a pipeline +/// without the parameter (never written by libhdf5) uses the element size. +fn shuffle_type_size(cd: &[u32], element_size: usize) -> Result { + match cd { + [] => Ok(element_size), + [0] | [_, _, ..] => Err(FormatError::FilterError( + "invalid shuffle parameters".into(), + )), + [size] => Ok(*size as usize), + } +} + fn shuffle_decompress(data: &[u8], element_size: usize) -> Result, FormatError> { if element_size <= 1 { return Ok(data.to_vec()); @@ -2434,48 +2453,125 @@ mod tests { } } - fn as_f64(bytes: &[u8]) -> Vec { - bytes - .as_chunks::<8>() - .0 - .iter() - .map(|c| f64::from_le_bytes(*c)) - .collect() + /// E-scale: libhdf5 refuses it on read and write ("E-scaling method not + /// supported"); it was decoded here, never checked against anything. + #[test] + fn scaleoffset_float_escale_is_refused_as_in_libhdf5() { + let cd = [1u32, 1, 4, 1, 8, 0, 0, 0]; + let mut raw = vec![2, 0, 0, 0, 8]; + raw.extend_from_slice(&[0; 16]); + raw.push(0x1B); + assert!(scaleoffset_decompress(&raw, &cd, 0).is_err()); } + /// A scale type that does not match the class is refused, as libhdf5 + /// refuses it ("invalid scale type"). #[test] - fn scaleoffset_float_escale_e1() { - // f64 [0.0, 2.0, 4.0, 6.0], E=1 (×2^1=2), fill_defined=0. - // cd: scale_type=1, E=1, nelmts=4, elem_size=8. - let cd = [1u32, 1, 4, 0, 8, 0, 0, 0]; - let raw: &[u8] = &[ - 2, 0, 0, 0, // minbits=2 - 8, // minval_width=8 - 0, 0, 0, 0, 0, 0, 0, 0, // minval=0.0f64 - 0, 0, 0, 0, 0, 0, 0, 0, // 8 reserved bytes - 0x1B, // packed codes: 00 01 10 11 MSB-first - ]; - let got = as_f64(&scaleoffset_decompress(raw, &cd, 0).unwrap()); - assert_eq!(got, vec![0.0, 2.0, 4.0, 6.0]); + fn scaleoffset_scale_type_must_match_the_class() { + let mut raw = vec![2, 0, 0, 0, 8]; + raw.extend_from_slice(&[0; 16]); + raw.push(0x1B); + assert!(scaleoffset_decompress(&raw, &[0, 0, 4, 0, 4, 1, 0, 0], 0).is_err()); + assert!(scaleoffset_decompress(&raw, &[2, 0, 4, 1, 4, 1, 0, 0], 0).is_err()); + assert!(scaleoffset_decompress(&raw, &[2, 0, 4, 7, 4, 1, 0, 0], 0).is_err()); } + /// `cve-2025-44905` `/Scale_offset_short_data_be`, chunk (4, 0): the + /// stored size of `minval` is 0. libhdf5 reads a `minval` of 0 and the + /// packed codes from byte 21 regardless; we read them from byte 13 + /// (5 + size + 8), so the values differed from h5py's. #[test] - fn scaleoffset_float_escale_neg_exp() { - // f64 [0.0, 0.5, 1.0, 1.5], E=-1 (×2^-1=0.5), fill_defined=0. - // cd[1] = 0xFFFF_FFFF which casts to i32 = -1. - let cd = [1u32, 0xFFFF_FFFF, 4, 0, 8, 0, 0, 0]; - let raw: &[u8] = &[ - 2, 0, 0, 0, // minbits=2 - 8, // minval_width=8 - 0, 0, 0, 0, 0, 0, 0, 0, // minval=0.0f64 - 0, 0, 0, 0, 0, 0, 0, 0, // 8 reserved bytes - 0x1B, // packed codes: 00 01 10 11 MSB-first - ]; - let got = as_f64(&scaleoffset_decompress(raw, &cd, 0).unwrap()); - let exp = [0.0f64, 0.5, 1.0, 1.5]; - for (g, e) in got.iter().zip(exp.iter()) { - assert!((g - e).abs() < 1e-9, "got {g} expected {e}"); - } + fn scaleoffset_codes_start_at_byte_21_whatever_the_minval_size() { + // big-endian i16, 12 elements, fill -2 (cd 65534), minbits 3. + let mut cd = vec![2u32, 0, 12, 0, 2, 1, 1, 1, 65534]; + cd.resize(20, 0); + let raw = unhex("0300000000d20e00000000000034000000000000000400000000"); + let got = scaleoffset_decompress(&raw, &cd, 24).unwrap(); + // Codes of 3 bits from byte 21 (04 00 00 00 00): 0, 1, 0, ...; h5py + // reads the chunk's first row as 0, 1, 0. + let want: Vec = vec![0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]; + let want: Vec = want.iter().flat_map(|v| v.to_be_bytes()).collect(); + assert_eq!(got, want); + } + + /// With a fill value defined, libhdf5 compares each code with the + /// all-ones code of `minbits` bits — which for `minbits == 0` is 0, so a + /// chunk with no packed codes reads as all fill values (the compressor + /// writes that for a chunk of nothing but fill values). It read as + /// `minval` here. + #[test] + fn scaleoffset_minbits_zero_with_a_fill_value_is_all_fill() { + let mut cd = vec![2u32, 0, 3, 0, 4, 1, 0, 1, (-7i32) as u32]; + cd.resize(20, 0); + let mut raw = vec![0, 0, 0, 0, 8]; + raw.extend_from_slice(&5i64.to_le_bytes()); + raw.extend_from_slice(&[0; 8]); + assert_eq!( + scaleoffset_decompress(&raw, &cd, 12).unwrap(), + i32_le(&[-7, -7, -7]) + ); + // Without a fill value every element is minval. + cd[7] = 0; + assert_eq!( + scaleoffset_decompress(&raw, &cd, 12).unwrap(), + i32_le(&[5, 5, 5]) + ); + } + + /// `minbits` of the full width stores the elements as they are + /// (little-endian), without `minval`; a full-width integer scale factor + /// means the filter left the chunk untouched. + #[test] + fn scaleoffset_full_width_is_stored_as_is() { + let mut cd = vec![2u32, 0, 2, 0, 2, 1, 1, 0]; + cd.resize(20, 0); + let mut raw = vec![16, 0, 0, 0, 8]; + raw.extend_from_slice(&100i64.to_le_bytes()); + raw.extend_from_slice(&[0; 8]); + raw.extend_from_slice(&[0x34, 0x12, 0xfe, 0xff]); + assert_eq!( + scaleoffset_decompress(&raw, &cd, 4).unwrap(), + [0x12, 0x34, 0xff, 0xfe] + ); + cd[1] = 16; + assert_eq!( + scaleoffset_decompress(&[1, 2, 3, 4], &cd, 4).unwrap(), + [1, 2, 3, 4] + ); + cd[1] = 17; + assert!(scaleoffset_decompress(&[1, 2, 3, 4], &cd, 4).is_err()); + // minbits wider than the type. + cd[1] = 0; + raw[0] = 17; + assert!(scaleoffset_decompress(&raw, &cd, 4).is_err()); + } + + /// The shuffle filter uses its own parameter as the element size, as + /// libhdf5 does; a parameter larger than the chunk leaves the chunk as + /// it is (`cve-2025-44905` `/Shuffle_float_data_be`, whose parameter is + /// 4261347332: h5py and h5dump read the stored bytes unshuffled). + #[test] + fn shuffle_uses_its_parameter() { + let data: Vec = (0..16).collect(); + let shuffled = shuffle_compress(&data, 4).unwrap(); + let pipeline = |cd: Vec| FilterPipeline { + version: 2, + filters: vec![one_filter(FILTER_SHUFFLE, cd)], + }; + // The dataset's element size says 2; the parameter says 4. + assert_eq!( + decompress_chunk(&shuffled, &pipeline(vec![4]), 16, 2).unwrap(), + data + ); + assert_eq!( + decompress_chunk(&shuffled, &pipeline(vec![4_261_347_332]), 16, 4).unwrap(), + shuffled + ); + assert!(decompress_chunk(&shuffled, &pipeline(vec![0]), 16, 4).is_err()); + assert_eq!( + decompress_chunk(&shuffled, &pipeline(vec![]), 16, 4).unwrap(), + data + ); } // --- N-Bit (filter id 5) -------------------------------------------------- From 6b3d003950edd92f838083cd998c16cb3f5d81f1 Mon Sep 17 00:00:00 2001 From: osobh Date: Sat, 26 Sep 2026 10:36:01 -0500 Subject: [PATCH 20/43] fix(format): refuse chunk index entries libhdf5 mis-reads - A dataset without filters stores every chunk at the chunk's full size. A chunk its index records at another size was read at that size, with the rest of the chunk left as zeros (cve-2025-44904's Scale_offset_float_data_le: 38- and 37-byte chunks for 48-byte chunks, where HDF5 2.0 fills the rest with whatever its buffer held). It is now refused, as later libhdf5 releases refuse it ("incorrect chunk size returned from index for unfiltered chunk"): chunked_read::list_chunks_for_read, used by every read path. - A v1 B-tree chunk key carries 0 in the element-size dimension. libhdf5 compares that coordinate when it looks a chunk up, so whether it finds a chunk keyed otherwise depends on where the key falls (in cve-2025-44905 /Shuffle_float_data_le, offset 4096, it does not, and h5py reads fill values); we read the chunk. Such a key is now refused. Co-Authored-By: Claude Opus 5.5 (1M context) --- crates/clawhdf5-format/src/chunked_read.rs | 64 +++++++++++++++++-- crates/clawhdf5-format/src/partial_read.rs | 5 +- .../tests/header_validation_interop.rs | 44 +++++++++++++ 3 files changed, 107 insertions(+), 6 deletions(-) diff --git a/crates/clawhdf5-format/src/chunked_read.rs b/crates/clawhdf5-format/src/chunked_read.rs index 70176e2..f9d6567 100644 --- a/crates/clawhdf5-format/src/chunked_read.rs +++ b/crates/clawhdf5-format/src/chunked_read.rs @@ -461,6 +461,17 @@ fn collect_chunk_info_inner( file_data[pos + 7], ]); let offsets = read_key_offsets(file_data, pos, ndims, chunk_dimensions)?; + // A chunk's key carries 0 in the element-size dimension. libhdf5 + // compares that coordinate too when it looks a chunk up + // (`H5D__btree_found`), so whether it finds a chunk keyed + // otherwise depends on where the key falls; in `cve-2025-44905` + // `/Shuffle_float_data_le` (offset 4096) it does not, and h5py + // reads fill values there. Such a key is refused here. + if chunk_dimensions.is_some() && offsets.last().is_some_and(|&o| o != 0) { + return Err(FormatError::ChunkedReadError(format!( + "chunk key {offsets:?} has a non-zero element offset" + ))); + } pos += key_size; // Parse child address @@ -820,6 +831,47 @@ pub fn list_chunks( Ok((chunks, chunk_dims)) } +/// [`list_chunks`] for reading the chunks through `pipeline`: a dataset +/// without filters stores every chunk at the chunk's full size, and a chunk +/// the index records at another size is refused, as libhdf5 refuses it +/// ("incorrect chunk size returned from index for unfiltered chunk"). Such +/// a chunk was read at its recorded size, with the rest of the chunk left +/// as zeros or fill values: `cve-2025-44904`'s `Scale_offset_float_data_le` +/// has chunks of 38 and 37 bytes for 48-byte chunks, where HDF5 2.0 reads +/// whatever its buffer held for the missing bytes. +pub fn list_chunks_for_read( + file_data: &[u8], + layout: &DataLayout, + dataspace: &Dataspace, + elem_size: usize, + pipeline: Option<&FilterPipeline>, + offset_size: u8, + length_size: u8, +) -> Result<(Vec, Vec), FormatError> { + let (chunks, chunk_dims) = list_chunks( + file_data, + layout, + dataspace, + elem_size, + offset_size, + length_size, + )?; + if pipeline.is_none_or(|p| p.filters.is_empty()) { + let chunk_bytes = checked_chunk_byte_len(&chunk_dims, elem_size)?; + if let Some(c) = chunks + .iter() + .find(|c| c.address != u64::MAX && c.chunk_size as usize != chunk_bytes) + { + return Err(FormatError::ChunkedReadError(format!( + "incorrect chunk size returned from index for unfiltered chunk at {:?}: \ + {} bytes, expected {chunk_bytes}", + c.offsets, c.chunk_size + ))); + } + } + Ok((chunks, chunk_dims)) +} + pub fn read_chunked_data( file_data: &[u8], layout: &DataLayout, @@ -831,11 +883,12 @@ pub fn read_chunked_data( ) -> Result, FormatError> { check_chunk_element_size(layout, datatype, offset_size)?; let elem_size = datatype.type_size() as usize; - let (chunks, chunk_dims) = list_chunks( + let (chunks, chunk_dims) = list_chunks_for_read( file_data, layout, dataspace, elem_size, + pipeline, offset_size, length_size, )?; @@ -979,11 +1032,12 @@ pub fn read_chunked_data_cached( // lookup is keyed by this dataset's chunk-index address, so another // dataset's index or chunks are never used for this read. let chunks = cache.chunks_for(addr, rank, || { - list_chunks( + list_chunks_for_read( file_data, layout, dataspace, elem_size, + pipeline, offset_size, length_size, ) @@ -1290,11 +1344,12 @@ pub fn read_chunked_data_sweep( // lookup is keyed by this dataset's chunk-index address, so another // dataset's index or chunks are never used for this read. let chunks = cache.chunks_for(addr, rank, || { - list_chunks( + list_chunks_for_read( file_data, layout, dataspace, elem_size, + pipeline, offset_size, length_size, ) @@ -1431,11 +1486,12 @@ pub fn read_chunked_data_indexed( addr, rank, || { - list_chunks( + list_chunks_for_read( file_data, layout, dataspace, elem_size, + pipeline, offset_size, length_size, ) diff --git a/crates/clawhdf5-format/src/partial_read.rs b/crates/clawhdf5-format/src/partial_read.rs index 9865c46..b75887c 100644 --- a/crates/clawhdf5-format/src/partial_read.rs +++ b/crates/clawhdf5-format/src/partial_read.rs @@ -18,7 +18,7 @@ use alloc::{format, vec, vec::Vec}; #[cfg(feature = "std")] use std::string as alloc_or_std; -use crate::chunked_read::{alloc_output, checked_byte_len, list_chunks}; +use crate::chunked_read::{alloc_output, checked_byte_len, list_chunks_for_read}; use crate::data_layout::DataLayout; use crate::data_read::extract_selection_from_buffer; use crate::dataspace::Dataspace; @@ -294,11 +294,12 @@ pub fn read_selection( btree_address: Some(_), .. } => { - let (chunks, chunk_dims) = list_chunks( + let (chunks, chunk_dims) = list_chunks_for_read( file_data, layout, dataspace, elem_size, + pipeline, offset_size, length_size, )?; diff --git a/crates/clawhdf5/tests/header_validation_interop.rs b/crates/clawhdf5/tests/header_validation_interop.rs index 74dc419..25614c5 100644 --- a/crates/clawhdf5/tests/header_validation_interop.rs +++ b/crates/clawhdf5/tests/header_validation_interop.rs @@ -628,3 +628,47 @@ save("mdc_past_eof", bad) "DSET", ); } + +/// Chunk index entries HDF5 2.0 mis-reads, refused here. An unfiltered +/// chunk the index records at less than the chunk's size (`cve-2025-44904`): +/// HDF5 2.0 fills the rest of the chunk with whatever its buffer held, and +/// later libhdf5 releases refuse it ("incorrect chunk size returned from +/// index for unfiltered chunk"); we read the rest as zeros. A chunk keyed +/// with a non-zero element offset (`cve-2025-44905`): libhdf5's lookup +/// compares that coordinate too, so whether it finds the chunk depends on +/// where the key falls (in `cve-2025-44905` it does not, and h5py reads +/// fill values; in this file it does); we read the chunk. +#[test] +fn chunk_index_entries_libhdf5_misreads_are_refused() { + skip_if_no_python!(); + let dir = tempfile::tempdir().unwrap(); + run_python( + dir.path(), + r#" +good = os.path.join(d, "good.h5") +with h5py.File(good, "w", libver="earliest") as f: + f.create_dataset("d", data=np.arange(100, dtype=" 0 and data[tree + 5] == 0 +key = tree + 8 + 16 # the first chunk's key: size, filter mask, offsets +second = key + 24 + 8 # a key (4 + 4 + 2 x 8 bytes), then a child address +assert struct.unpack_from(" Date: Sat, 26 Sep 2026 10:36:44 -0500 Subject: [PATCH 21/43] docs: chunked full reads decode in place; small pools no longer block CHANGELOG entry for the chunked read changes, and the known-issues entry on concurrent chunked reads updated: both causes it names (per-read page faults, readers waiting on a small pool) are fixed; the 16-thread comparison with h5py stays open until re-measured on an idle machine. Co-Authored-By: Claude Opus 5.5 (1M context) --- CHANGELOG.md | 46 ++++++++++++++++++++++++++++++++++++++++++++ docs/known-issues.md | 21 +++++++++++++++----- 2 files changed, 62 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 29804fc..a877c30 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,52 @@ ## Unreleased +### Chunked full reads (2026-09-26) +- **Chunks are decoded straight into the output, into reused buffers.** A + full read of a chunked dataset faulted in about three times its size in + fresh pages: every chunk was decoded into a new buffer per filter stage + (the cached reader behind `read_*` decoded 128 chunks at a time before + placing any; the uncached one behind `MmapFile`, `LazyFile` and + `verify_provenance` decoded the whole dataset first), then assembled into + a byte buffer, which the typed readers copied once more. Now each chunk + is decoded into buffers the thread keeps between chunks and reads + (`clawhdf5_format::filters::DecodeScratch`, + `decompress_chunk_exact_with`: deflate inflates into a kept buffer with a + reset inflater, shuffle into the other one, Fletcher32 is checked in + place; other filters go through the registry as before) and copied + directly to its place in the output. Chunks still go into the file's + chunk cache when the whole dataset fits. Selection reads decode the + chunks they touch the same way. +- **Typed full reads of chunked data skip the byte buffer.** `read_f32`, + `read_f64`, `read_i32`, `read_i64` and `read_u64` (on `File`, `MmapFile` + and `LazyFile`) of a chunked dataset stored as that type in native byte + order decode every chunk into the returned `Vec` (huge-page backed when + large, like the byte readers' output); other types and byte orders + convert as before. New public + `clawhdf5_format::data_read::read_chunked_native`. +- **Reading threads no longer wait for a busy rayon pool.** A full read + handed its chunks to rayon and the calling thread slept until the pool had + decoded them, so with a small pool (2-4 threads) readers outside it queued + behind its workers. The calling thread now decodes too, and pool workers + join in only when free; a helper the pool starts after the read has + finished returns at once. A single read still spreads over the default + pool. This replaces the one-thread-pool special case below for full + reads. Chunks are placed from several threads only when the chunk index + puts them on the chunk grid at distinct places; a corrupt index is read + one chunk at a time, and the error reported is still the first failing + chunk's. New test `crates/clawhdf5/tests/busy_decode_pool.rs`. +- **Fixed:** in a filtered dataset, a chunk stored with every filter skipped + (filter mask) and shorter than a chunk read with zeros in place of its + missing part through `File`'s `read_*`; it is now an error naming the + chunk, as `MmapFile`/`LazyFile` already made it. +- New h5py comparison `crates/clawhdf5/tests/chunked_read_paths_interop.rs`: + every chunked read path (cached and uncached full reads, `MmapFile`, + `LazyFile`, small, strided and point selections, with and without the + `parallel` feature) for 1-8-byte integers and 2-8-byte floats in both + byte orders, through deflate, shuffle, Fletcher32, LZF, SZIP and Blosc, + with partial edge chunks, sparse datasets and fill values, and datasets + larger than the chunk cache. + ### Concurrent reads (2026-09-26) - **Full reads of chunked datasets scale with threads again when rayon's pool has one thread.** Each full read handed its chunks to rayon to diff --git a/docs/known-issues.md b/docs/known-issues.md index 2657cdf..0d2556b 100644 --- a/docs/known-issues.md +++ b/docs/known-issues.md @@ -26,8 +26,9 @@ performance" below. ## Concurrent and contiguous read performance (measured 2026-09-26) -**Status:** open for chunked full reads (one cause fixed 2026-09-26); the -contiguous item is fixed (2026-09-26). Measured on +**Status:** open for chunked full reads at 16 threads until re-measured +(the causes identified below are fixed as of 2026-09-26); the contiguous +item is fixed (2026-09-26). Measured on tank with `concurrent_read` against h5py 3.16 / HDF5 2.0 (`BENCHMARKS.md`, "Concurrent reads"): - **Partly fixed 2026-09-26.** Full reads of chunked datasets from several threads @@ -48,9 +49,19 @@ tank with `concurrent_read` against h5py 3.16 / HDF5 2.0 (`BENCHMARKS.md`, readers outside it still wait on its workers. Datasets larger than the cache's budget were already read without inserting into it, and skipping its lookups entirely gained only a few percent at 16 threads. Remaining - per-read overhead, not yet addressed: each full `read_f32` of a chunked - dataset faults in about three times its size in fresh pages (the output, - the `f32` copy of it, and a new buffer per decoded chunk). + per-read overhead: each full `read_f32` of a chunked dataset faults in + about three times its size in fresh pages (the output, the `f32` copy of + it, and a new buffer per decoded chunk). + **Fixed 2026-09-26** (both causes; see `CHANGELOG.md`, "Chunked full + reads"): chunks are decoded into buffers each thread reuses and copied + straight into the output, and the typed readers decode into the `Vec` + they return, so a full read no longer faults in a buffer per chunk or a + second copy of its output; and the reading + thread decodes its own chunks with pool workers helping when free, so no + reader waits on a small or busy pool + (`crates/clawhdf5/tests/busy_decode_pool.rs`). The 16-thread comparison + with h5py processes has not been re-measured yet (tank was busy with + other work); this item stays open until it is. - Contiguous datasets read 4x slower than h5py on one thread (2.5 vs 9.8 GB/s full, 0.12x for 256 x 256 hyperslabs). **Fixed 2026-09-26** (re-measured on tank at `408f69e`: 13665 MB/s From 8295d016141ea491bfe892b1f7462cab8f0fb4db Mon Sep 17 00:00:00 2001 From: osobh Date: Sat, 26 Sep 2026 10:37:40 -0500 Subject: [PATCH 22/43] format: a chunk offset past usize writes nothing Placing a chunk cast its u64 offsets to usize; on a 32-bit target an offset past the address space wrapped into the output (and could then overlap another chunk's region when chunks are placed concurrently). Such an offset is past the dataset, so it now saturates and the chunk writes nothing, as the concurrent-placement check already assumed. No change on 64-bit targets, where the cast cannot wrap (so no test here). Co-Authored-By: Claude Opus 5.5 (1M context) --- crates/clawhdf5-format/src/chunked_read.rs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/crates/clawhdf5-format/src/chunked_read.rs b/crates/clawhdf5-format/src/chunked_read.rs index 712aab8..ddadda7 100644 --- a/crates/clawhdf5-format/src/chunked_read.rs +++ b/crates/clawhdf5-format/src/chunked_read.rs @@ -155,7 +155,9 @@ impl ChunkPlacer { &mut long }; for (o, &off) in chunk_offsets.iter_mut().zip(offsets) { - *o = off as usize; + // An offset past `usize` is past the dataset: the chunk writes + // nothing (a plain cast would wrap it into the output). + *o = usize::try_from(off).unwrap_or(usize::MAX); } // SAFETY: the caller's contract. unsafe { From f512bf3d09c502c818322a4b8e77e3f450473bad Mon Sep 17 00:00:00 2001 From: osobh Date: Sat, 26 Sep 2026 10:38:50 -0500 Subject: [PATCH 23/43] conformance: report a cache image libhdf5 cannot load where it does libhdf5 loads a metadata cache image when it first reads metadata (the root group), not at open, so for cve-2025-6269-1..4 and cve-2025-6516 (all corrupt images) h5py opens the file and fails on "/". The probe reported the image's error as an open error, which made those files our-errors; it now records it on the root object, where h5py reports it. File::open still refuses such a file outright: nothing in it can be read. Co-Authored-By: Claude Opus 5.5 (1M context) --- conformance/probe/src/main.rs | 32 ++++++++++++++++++++++++++------ 1 file changed, 26 insertions(+), 6 deletions(-) diff --git a/conformance/probe/src/main.rs b/conformance/probe/src/main.rs index c4cf524..9fa9929 100644 --- a/conformance/probe/src/main.rs +++ b/conformance/probe/src/main.rs @@ -712,18 +712,35 @@ fn main() { return; } }; - // libhdf5 decodes the superblock extension at open, and loads a - // metadata cache image over the file's own metadata. - let view = match guarded(|| { - clawhdf5_format::superblock_ext::metadata_view(hdf5, &sb).map_err(e) - }) { - Ok(v) => v, + // libhdf5 decodes the superblock extension at open (File::open does + // the same), and loads a metadata cache image over the file's own + // metadata. It loads the image only when it first reads metadata — the + // root group — so a file whose image it cannot load still opens and + // every object fails; File::open refuses such a file outright. The + // probe records the image's error where libhdf5 reports it. + use clawhdf5_format::superblock_ext; + let ext = match guarded(|| superblock_ext::read_superblock_extension(hdf5, &sb).map_err(e)) { + Ok(x) => x, Err(msg) => { top.insert("open_error".into(), Value::String(msg)); println!("{}", Value::Object(top)); return; } }; + let mut image_error = None; + let view = match ext.and_then(|x| x.cache_image) { + None => None, + Some(loc) => match guarded(|| { + superblock_ext::apply_cache_image(hdf5, loc, sb.offset_size, sb.length_size) + .map_err(e) + }) { + Ok(v) => Some(v), + Err(msg) => { + image_error = Some(msg); + None + } + }, + }; let hdf5: &[u8] = view.as_deref().unwrap_or(hdf5); top.insert("superblock_version".into(), json!(sb.version)); let ctx = Ctx { @@ -752,6 +769,9 @@ fn main() { let mut rec = Map::new(); rec.insert("path".into(), Value::String(p.clone())); let r = guarded(|| { + if let Some(msg) = &image_error { + return Err(msg.clone()); + } let h = ctx.header(addr)?; Ok(h) }); From 67958b08d9f76c2e2d17f4b5c34fc869229ca51f Mon Sep 17 00:00:00 2001 From: osobh Date: Sat, 26 Sep 2026 10:39:25 -0500 Subject: [PATCH 24/43] conformance: list the corrupt objects HDF5 2.0 reads through a bug Four of the remaining our-errors are objects the reference (h5py 3.16 / HDF5 2.0) reads only through a libhdf5 bug, and clawhdf5 refuses: cve-2025-2308 (scale-offset codes past the end of the chunk), cve-2025-44904 (short unfiltered chunks), bad_nbit_parms_walk.h5 (an N-Bit parameter list one value short; libhdf5's own test_filter_bad_params now requires the read to fail) and cve-2025-44905 /Shuffle_float_data_le (a chunk key libhdf5's lookup misses, reading fill values). report.py lists them under Known not-our-bug and counts them in the summary; they stay our-errors in the class counts. Co-Authored-By: Claude Opus 5.5 (1M context) --- conformance/report.py | 48 +++++++++++++++++++++++++++++++++++++------ 1 file changed, 42 insertions(+), 6 deletions(-) diff --git a/conformance/report.py b/conformance/report.py index dc82413..3331898 100644 --- a/conformance/report.py +++ b/conformance/report.py @@ -98,13 +98,40 @@ def is_h5py_be_vlen(i): and ">" in (i.get("ours_dtype") or "")) +# Objects the reference (h5py 3.16 / HDF5 2.0) reads only because of an +# HDF5 2.0 bug, and that clawhdf5 refuses: each one reads past a buffer or +# returns bytes the file does not hold, and libhdf5's develop branch refuses the +# first three. (file, object) -> why. Checked 2026-09-26 against HDF5 2.0.0 +# and HDFGroup/hdf5 develop sources; see docs/known-issues.md. +LIBHDF5_BUGS = { + ("cve_hdf5/cvefiles/cve-2025-2308.h5", "/Scale_offset_long_long_data_le"): + "scale-offset codes run past the end of the chunk: HDF5 2.0 reads past its buffer; " + "libhdf5's develop branch refuses the chunk (\"Buffer too short\")", + ("cve_hdf5/cvefiles/cve-2025-44904.h5", "/Scale_offset_float_data_le"): + "unfiltered chunks of 38 and 37 bytes for 48-byte chunks: HDF5 2.0 fills the rest with " + "whatever its buffer held; libhdf5's develop branch refuses them (\"incorrect chunk size returned " + "from index for unfiltered chunk\")", + ("hdf5/test/testfiles/bad_nbit_parms_walk.h5", "/Nbit_int_data_le"): + "an N-Bit parameter list one value short: HDF5 2.0 reads past the list; libhdf5's own " + "test (`test_filter_bad_params`, test/dsets.c) now requires the read to fail", + ("cve_hdf5/cvefiles/cve-2025-44905.h5", "/Shuffle_float_data_le"): + "a chunk B-tree key with element offset 4096: libhdf5's lookup misses the chunk and " + "returns fill values for data the file holds; clawhdf5 refuses the key", +} + + +def is_libhdf5_bug(rel, i): + return i["kind"] == "our-error" and any( + f == rel and i["detail"].startswith(obj + ":") for (f, obj) in LIBHDF5_BUGS) + + known = collections.defaultdict(list) for r in rows: - if r["class"] != "mismatch": - continue iss = issues.get(r["file"], []) - if iss and all(is_h5py_be_vlen(i) for i in iss): + if r["class"] == "mismatch" and iss and all(is_h5py_be_vlen(i) for i in iss): known["h5py-be-vlen"].append(r["file"]) + if r["class"] == "our-error" and iss and all(is_libhdf5_bug(r["file"], i) for i in iss): + known["libhdf5-2.0"].append(r["file"]) # --- the CVE corpus: clawhdf5 vs h5dump vs h5py ------------------------------ @@ -230,9 +257,13 @@ for c in sorted(by_corpus): w(f"| {c} | {sum(cnt.values())} | " + " | ".join(str(cnt.get(k, 0)) for k in CLASSES) + " |") w(f"| **all** | **{len(rows)}** | " + " | ".join(f"**{total.get(k, 0)}**" for k in CLASSES) + " |") w("") -n_known = sum(len(v) for v in known.values()) -if n_known: - w(f"{n_known} of the {total.get('mismatch', 0)} mismatches are a known h5py bug, not ours (see *Known not-our-bug*).") +if known["h5py-be-vlen"]: + w(f"{len(known['h5py-be-vlen'])} of the {total.get('mismatch', 0)} mismatches are a known h5py bug, " + "not ours (see *Known not-our-bug*).") + w("") +if known["libhdf5-2.0"]: + w(f"{len(known['libhdf5-2.0'])} of the {total.get('our-error', 0)} our-errors are corrupt data that " + "HDF5 2.0 reads only through a bug and clawhdf5 refuses (see *Known not-our-bug*).") w("") w("Corpora (fetched by `conformance/fetch-corpus.sh` into the gitignored `conformance/.cache/`):") w("") @@ -311,6 +342,11 @@ if res["incomparable"]: w(" (FP8 -> float16, bfloat16 -> float32, x87 long double -> float128) the values are not") w(" compared (shape and presence still are): " + ", ".join(f"{k} ({n}x)" for k, n in res["incomparable"]) + ".") +w("- **Corrupt data HDF5 2.0 reads through a bug.** clawhdf5 refuses these objects; h5py 3.16 /") +w(" HDF5 2.0 returns values for them that the file does not hold:") +for (f, obj), why in sorted(LIBHDF5_BUGS.items()): + here = "" if f in known["libhdf5-2.0"] else " (not an our-error in this run)" + w(f" - `{f}` `{obj}`: {why}{here}.") w("- **References** are compared by presence only (`R`), not by target.") w("") if res.get("ref_only_errors"): From 378afa1584164109fc5c5651bb28cc07dcb9ed1b Mon Sep 17 00:00:00 2001 From: osobh Date: Sat, 26 Sep 2026 10:40:18 -0500 Subject: [PATCH 25/43] docs: changelog and known issues for the remaining conformance errors Conformance on tank, conformance/run.sh --no-fetch (2026-09-26): 597 of 697 ok, 6 our-errors (4 corrupt objects HDF5 2.0 reads through a bug, the Blosc2 and ZFP filters), 2 mismatches (the known h5py big-endian VL bug). Closes the known-issues entries for metadata cache images, cve-2024-32624, cve-2020-10810/10812, and unfiltered chunks of the wrong size; the N-Bit / 64-bit scale-offset entry is recorded as not our bug. Co-Authored-By: Claude Opus 5.5 (1M context) --- CHANGELOG.md | 54 ++++++++++++++++++++++++++++++++++++++++++++ docs/known-issues.md | 37 +++++++++++++++++++++++++----- 2 files changed, 85 insertions(+), 6 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 29804fc..c13a8da 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,60 @@ ## Unreleased +### Remaining conformance errors (2026-09-26) +Conformance on tank, `conformance/run.sh --no-fetch`: 597 of 697 files +ok (575 before). Of the 6 our-errors left, 4 are corrupt data HDF5 2.0 +reads only through a bug (listed in `CONFORMANCE.md`), 2 are the Blosc2 and +ZFP filters; the 2 mismatches are the known h5py big-endian VL bug. +- **Metadata cache images are read.** A file written with a metadata cache + image keeps its metadata cache entries in an image block the superblock + extension points at, and libhdf5 reads them in place of the file's own + bytes; in `h5clear_mdc_image.h5` the root group exists only there, and + every reader failed with `InvalidObjectHeaderVersion(0)`. `File`, + `MmapFile` and `LazyFile` (and `h5rs`) now apply the image at open + (`clawhdf5_format::superblock_ext`), with libhdf5's checks. A file whose + image libhdf5 cannot load opens in libhdf5 but nothing in it can be read; + `File::open` refuses it. +- **The superblock extension is decoded at open, as libhdf5 does:** a File + Space Info or Metadata Cache Image message libhdf5 cannot decode makes the + open fail (`cve-2020-10810`, `cve-2020-10812` were opened). + `FormatError::InvalidSuperblockExtension`, `InvalidCacheImage`. +- **Dataset storage libhdf5 refuses at open is refused at open** + (`FormatError::InvalidDatasetStorage`, `data_read::check_dataset_storage`): + an element count times element size that overflows (`cve-2024-32624` + `/Dset_OBJREF` opened and reported its shape), contiguous storage past the + end of the file, compact data of the wrong size. An empty contiguous + dataset at a defined address, which clawhdf5 up to v2.7.0 wrote, still + opens. +- **Wrong or missing data fixed:** + - a simple dataspace of rank 0 holds one element (it held 0; + `cve-2020-18494`), and contiguous storage larger than the dataset reads + (`cve-2024-32623`, `cve-2025-2309`; libhdf5 ignores the excess); + - scale-offset: the packed codes start at byte 21 whatever size the chunk + records for `minval`, a chunk with `minbits` 0 and a fill value is all + fill values, full-width `minbits` stores the elements as they are + (these decoded differently from libhdf5); E-scale is refused, as in + libhdf5; codes past the end of the chunk stay an error + (`cve-2025-2308`, where HDF5 2.0 reads past its buffer); + - shuffle uses its own parameter as the element size, as libhdf5 does + (`cve-2025-44905`); + - an unfiltered chunk the index records at other than the chunk's size is + refused (it read with zeros for the missing bytes; `cve-2025-44904`), + as is a chunk B-tree key with a non-zero element offset. +- **Refused as libhdf5 refuses them:** a v1 group with an empty link name + fails its listing (`FormatError::InvalidLinkName`; lookups still work, + `cve-2021-46244`); dataspaces with more than 32 dimensions, a rank on a + scalar or null dataspace, or a dimension over its maximum + (`FormatError::InvalidDataspace`). +- `ObjectHeader::object_class` classifies a header as libhdf5 does (a + dataset needs a datatype *and* a dataspace). +- Conformance harness: user-defined links were listed as objects by the + reference, unopenable objects were not deduplicated, nested array types + were hashed wrong (`tarray3.h5`), and the attributes of objects h5py + cannot open were compared; all fixed. `CONFORMANCE.md` lists the corrupt + objects HDF5 2.0 reads through a bug (`bad_nbit_parms_walk.h5` among + them: libhdf5's own test now requires that read to fail). + ### Concurrent reads (2026-09-26) - **Full reads of chunked datasets scale with threads again when rayon's pool has one thread.** Each full read handed its chunks to rayon to diff --git a/docs/known-issues.md b/docs/known-issues.md index 2657cdf..b7c471a 100644 --- a/docs/known-issues.md +++ b/docs/known-issues.md @@ -202,9 +202,25 @@ fill-value item that did is fixed). its datatype message stores (12 with 4-byte offsets), and the global heap is read with libhdf5's header padding (`crates/clawhdf5/tests/vl_offset4_interop.rs`). - - Metadata cache images are not supported. + - Metadata cache images are not supported. **Fixed 2026-09-26:** the + image is applied at open, as libhdf5 loads it over the file's metadata + (`clawhdf5_format::superblock_ext`); `h5clear_mdc_image.h5` reads + (`crates/clawhdf5/tests/metadata_cache_image.rs`). A file whose image + libhdf5 cannot load (`cve-2025-6269-*`, `cve-2025-6516`) opens in + libhdf5 with nothing readable in it; `File::open` refuses it. - x87 long double and binary128 are refused. - N-Bit on 64-bit scale-offset data and some N-Bit parameter layouts fail. + **Not our bug (checked 2026-09-26):** both are corrupt files HDF5 2.0 + reads only by reading past a buffer. `cve-2025-2308`'s + `/Scale_offset_long_long_data_le` has scale-offset codes that run past + the end of the chunk (libhdf5's develop branch refuses it, "Buffer too + short"); `bad_nbit_parms_walk.h5` has an N-Bit parameter list one value + short (libhdf5's own `test_filter_bad_params` in `test/dsets.c` now + requires that read to fail). We refuse both; `CONFORMANCE.md` lists them + under *Known not-our-bug*. Scale-offset did decode three cases + differently from libhdf5 (codes after a `minval` of recorded size other + than 8, `minbits` 0 with a fill value, full-width `minbits`): fixed + 2026-09-26. - **Filters:** blosc, blosc2, bitshuffle, bzip2, LZF and zfp are not implemented. **Fixed 2026-09-26** for LZF (default-on `lzf` feature), bitshuffle, bzip2 and Blosc 1 (`bitshuffle`, `bzip2`, `blosc`, or @@ -219,7 +235,11 @@ fill-value item that did is fixed). read with zeros for the missing bytes** (any filter; found reviewing the plugin filters). **Fixed 2026-09-26:** it is an error naming the chunk. A corrupt chunk must never read as zeros. Unfiltered chunks are read at their stored - size and are not checked this way. + size and are not checked this way. **Fixed 2026-09-26** for unfiltered + chunks too: in a dataset without filters a chunk the index records at + other than the chunk's size is refused, as libhdf5's develop branch + refuses it (`cve-2025-44904`, where HDF5 2.0 fills the rest from its + buffer). - **Crash:** a hostile Blosc chunk (frame size below its header) panicked in builds with overflow checks. **Fixed 2026-09-26**; the new decoders are fuzzed in the unit tests. @@ -232,13 +252,18 @@ fill-value item that did is fixed). refused. 17 of the 18 now fail as in libhdf5 (conformance on tank, `conformance/run.sh --no-fetch`, 2026-09-26: 571 of 697 ok). Still read where libhdf5 refuses: - - `cve-2024-32624.h5` `/Dset_OBJREF`: a dataspace whose storage size + - ~~`cve-2024-32624.h5` `/Dset_OBJREF`: a dataspace whose storage size overflows 64 bits. `File::dataset` and `shape()` succeed (libhdf5 - refuses at open); reading the values fails. - - `cve-2020-10810.h5`, `cve-2020-10812.h5` (whole files libhdf5 cannot + refuses at open); reading the values fails.~~ **Fixed 2026-09-26:** + `File::dataset` (and `MmapFile`, `LazyFile`) refuse it at open + (`FormatError::InvalidDatasetStorage`), as they do contiguous storage + past the end of the file. + - ~~`cve-2020-10810.h5`, `cve-2020-10812.h5` (whole files libhdf5 cannot open, not among the 18): libhdf5 decodes the superblock extension's File Space Info and metadata-cache-image messages at open and refuses these - files; we do not decode those messages at open. + files; we do not decode those messages at open.~~ **Fixed 2026-09-26:** + the superblock extension is decoded at open with libhdf5's checks, and + both files are refused. - Deliberately not refused, because clawhdf5 up to v2.7.0 wrote them: a float sign bit position outside the type, and a size-0 string type. - Not refused because current libhdf5 reads it though HDF5 2.0.0 From bf4aefcd00abd5629701a207fc42f0fb9d494973 Mon Sep 17 00:00:00 2001 From: osobh Date: Sat, 26 Sep 2026 11:22:07 -0500 Subject: [PATCH 26/43] format: a one-thread pool decodes on the caller only; keep 1 MiB scratch Review follow-ups. With run_with_helpers a one-thread rayon pool gave each read a second core (the caller plus the worker), so --decode-threads 1 no longer matched h5py's one core per call; such a pool now adds no helper. Per-thread decode scratch is kept up to 1 MiB per buffer (was 4 MiB), bounding what never-exiting pool workers hold. Co-Authored-By: Claude Opus 5.5 (1M context) --- crates/clawhdf5-format/src/filters.rs | 7 +++++-- crates/clawhdf5-format/src/parallel_read.rs | 6 ++++++ 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/crates/clawhdf5-format/src/filters.rs b/crates/clawhdf5-format/src/filters.rs index 872b8d8..266e530 100644 --- a/crates/clawhdf5-format/src/filters.rs +++ b/crates/clawhdf5-format/src/filters.rs @@ -183,8 +183,11 @@ enum Stage { } impl DecodeScratch { - /// Largest buffer [`trim`](Self::trim) keeps (4 MiB). - pub const RETAIN_BYTES: usize = 4 << 20; + /// Largest buffer [`trim`](Self::trim) keeps (1 MiB): enough for common + /// chunk sizes (a 256 x 256 `f32` chunk is 256 KiB) while bounding what + /// every long-lived thread (rayon's workers never exit) holds on to, at + /// two buffers each. + pub const RETAIN_BYTES: usize = 1 << 20; /// An empty scratch; buffers are allocated on first use. pub fn new() -> Self { diff --git a/crates/clawhdf5-format/src/parallel_read.rs b/crates/clawhdf5-format/src/parallel_read.rs index 61d06cd..d9927c5 100644 --- a/crates/clawhdf5-format/src/parallel_read.rs +++ b/crates/clawhdf5-format/src/parallel_read.rs @@ -47,6 +47,12 @@ pub fn pool_can_parallelise() -> bool { /// item beyond the caller's first. pub(crate) fn helper_count(items: usize) -> usize { let pool = rayon::current_num_threads(); + // A one-thread pool means "decode on the calling thread" (the setting + // benchmarks use to compare with h5py, where each call decodes on its + // caller): no helper, so one read never uses two cores. + if pool <= 1 { + return 0; + } let others = if rayon::current_thread_index().is_some() { pool.saturating_sub(1) } else { From 989335b67b5da5f9ace23783320c87e891b4a432 Mon Sep 17 00:00:00 2001 From: osobh Date: Sat, 26 Sep 2026 11:26:28 -0500 Subject: [PATCH 27/43] fix(format): bound a Blosc2 frame's offsets chunk by the HDF5 chunk size parse_frame sized the offsets chunk from the frame header's own nbytes and chunksize, so a 173-byte frame declaring 32 Mi chunks, with a 40-byte repeated-value offsets chunk, built 256 MiB (up to 2 GiB) of offsets for a 1 MiB HDF5 chunk and then returned 4 bytes. The offsets chunk is now capped at the output limit (at least 128 bytes); a frame whose nbytes/chunksize imply more chunks than that is refused before anything is allocated. tests/blosc2_alloc_bounds.rs measures peak allocation with a counting global allocator; the reviewer's frame failed it (decoded Ok(4)) before. Co-Authored-By: Claude Opus 5.5 (1M context) --- crates/clawhdf5-format/src/filters_blosc2.rs | 38 ++-- .../tests/blosc2_alloc_bounds.rs | 191 ++++++++++++++++++ 2 files changed, 213 insertions(+), 16 deletions(-) create mode 100644 crates/clawhdf5-format/tests/blosc2_alloc_bounds.rs diff --git a/crates/clawhdf5-format/src/filters_blosc2.rs b/crates/clawhdf5-format/src/filters_blosc2.rs index 5f332f8..d2b1ef0 100644 --- a/crates/clawhdf5-format/src/filters_blosc2.rs +++ b/crates/clawhdf5-format/src/filters_blosc2.rs @@ -451,7 +451,9 @@ struct Frame<'a> { offsets: Vec, } -fn parse_frame(buf: &[u8]) -> Result, FormatError> { +/// Parse a frame's header and decode its offsets chunk, holding no more +/// than an HDF5 chunk of `limit` bytes needs. +fn parse_frame(buf: &[u8], limit: usize) -> Result, FormatError> { if buf.len() < FRAME_HEADER_MINLEN { return Err(err("truncated frame header")); } @@ -497,29 +499,33 @@ fn parse_frame(buf: &[u8]) -> Result, FormatError> { .ok_or_else(|| err("chunks run past the frame"))?; let nbytes = usize::try_from(nbytes).map_err(|_| err("bad decoded size"))?; let chunksize = chunksize as usize; - // The offsets chunk follows the data chunks. + // The offsets chunk follows the data chunks: one `i64` per chunk. The + // frame header's sizes are the file's word, so they must not size it: + // it may be no larger than the HDF5 chunk (`limit`, at least 128 + // bytes), i.e. one Blosc2 chunk per 8 bytes of output. hdf5-blosc2 + // writes one chunk per frame; only python-blosc2 arrays of tiny + // chunks come near the cap. let off_src = &buf[data_end..]; - let expected = if nbytes == 0 { + let max_offsets = limit.max(128); + let off_len = if nbytes == 0 { 0 } else if chunksize > 0 { - nbytes.div_ceil(chunksize) + nbytes + .div_ceil(chunksize) + .checked_mul(8) + .filter(|&n| n <= max_offsets) + .ok_or_else(|| err("frame has more chunks than the HDF5 chunk can hold"))? } else { // Variable-size chunks: the offsets chunk tells how many. - usize::MAX - }; - let off_limit = if expected == usize::MAX { - 1 << 24 - } else { - expected - .checked_mul(8) - .ok_or_else(|| err("too many chunks"))? + max_offsets }; let offsets = if nbytes == 0 { Vec::new() } else { - blosc2_decompress_chunk(off_src, off_limit)? + blosc2_decompress_chunk(off_src, off_len)? }; - if !offsets.len().is_multiple_of(8) || (expected != usize::MAX && offsets.len() != off_limit) { + let expected = if chunksize > 0 { off_len } else { offsets.len() }; + if !offsets.len().is_multiple_of(8) || offsets.len() != expected { return Err(err("offsets chunk does not match the number of chunks")); } Ok(Frame { @@ -711,7 +717,7 @@ fn decode_frame( limit: usize, cd_shape: Option<&[usize]>, ) -> Result, FormatError> { - let frame = parse_frame(input)?; + let frame = parse_frame(input, limit)?; let meta = match frame.metalayer(b"b2nd")? { Some(m) => Some(m), None => frame.metalayer(b"caterva")?, @@ -1065,7 +1071,7 @@ mod tests { if w.is_err() { continue; } - let frame = parse_frame(&f).unwrap(); + let frame = parse_frame(&f, 1 << 20).unwrap(); let raw: [u8; 8] = frame.offsets[..8].try_into().unwrap(); let off = i64::from_le_bytes(raw); if off >= 0 { diff --git a/crates/clawhdf5-format/tests/blosc2_alloc_bounds.rs b/crates/clawhdf5-format/tests/blosc2_alloc_bounds.rs new file mode 100644 index 0000000..11e7812 --- /dev/null +++ b/crates/clawhdf5-format/tests/blosc2_alloc_bounds.rs @@ -0,0 +1,191 @@ +//! Crafted Blosc2 frames and chunks cannot make the decoder allocate out of +//! proportion to the HDF5 chunk it decodes. +//! +//! A frame's header, its offsets chunk and its chunk headers all declare +//! sizes, and the decoder used to allocate what they declared: a 173-byte +//! frame whose offsets chunk claimed 2 GiB was decoded in full before any +//! check failed. Every allocation is now bounded by the output limit (the +//! HDF5 chunk's size) and the input's length. +//! +//! Peak heap use is measured with a counting global allocator; the tests +//! share it, so each holds `SERIAL` for its whole run. +#![cfg(feature = "blosc2")] + +use std::alloc::{GlobalAlloc, Layout, System}; +use std::sync::Mutex; +use std::sync::atomic::{AtomicUsize, Ordering}; + +use clawhdf5_format::filters_blosc2::{blosc2_decompress, blosc2_decompress_chunk}; + +struct Counting; + +static CURRENT: AtomicUsize = AtomicUsize::new(0); +static PEAK: AtomicUsize = AtomicUsize::new(0); +static SERIAL: Mutex<()> = Mutex::new(()); + +unsafe impl GlobalAlloc for Counting { + unsafe fn alloc(&self, layout: Layout) -> *mut u8 { + let p = unsafe { System.alloc(layout) }; + if !p.is_null() { + let now = CURRENT.fetch_add(layout.size(), Ordering::Relaxed) + layout.size(); + PEAK.fetch_max(now, Ordering::Relaxed); + } + p + } + + unsafe fn alloc_zeroed(&self, layout: Layout) -> *mut u8 { + let p = unsafe { System.alloc_zeroed(layout) }; + if !p.is_null() { + let now = CURRENT.fetch_add(layout.size(), Ordering::Relaxed) + layout.size(); + PEAK.fetch_max(now, Ordering::Relaxed); + } + p + } + + unsafe fn dealloc(&self, ptr: *mut u8, layout: Layout) { + unsafe { System.dealloc(ptr, layout) }; + CURRENT.fetch_sub(layout.size(), Ordering::Relaxed); + } +} + +#[global_allocator] +static ALLOC: Counting = Counting; + +/// Bytes allocated at the peak of `f`, above what was live when it started. +fn peak_during(f: impl FnOnce() -> T) -> (T, usize) { + let base = CURRENT.load(Ordering::Relaxed); + PEAK.store(base, Ordering::Relaxed); + let out = f(); + (out, PEAK.load(Ordering::Relaxed).saturating_sub(base)) +} + +/// What decoding one HDF5 chunk of `limit` bytes from `input` may hold at +/// once: the output, a few blocks of scratch (each no larger than the +/// output), the offsets table, and the Zstandard decoder's state. +fn bound(limit: usize, input: &[u8]) -> usize { + 6 * limit + 2 * input.len() + (1 << 20) +} + +fn lock() -> std::sync::MutexGuard<'static, ()> { + SERIAL.lock().unwrap_or_else(|e| e.into_inner()) +} + +/// A 32-byte (extended) Blosc2 chunk header. +fn chunk_header(ts: u8, nbytes: i32, blocksize: i32, cbytes: i32, special: u8) -> Vec { + let mut c = vec![5u8, 1, 0x05, ts]; + for v in [nbytes, blocksize, cbytes] { + c.extend_from_slice(&v.to_le_bytes()); + } + c.resize(32, 0); + c[31] = special << 4; + c +} + +/// A chunk of `nbytes` bytes that repeats one value (special type 3). +fn repeated(value: &[u8], nbytes: i32, blocksize: i32) -> Vec { + let mut c = chunk_header(value.len() as u8, nbytes, blocksize, 32 + value.len() as i32, 3); + c.extend_from_slice(value); + c +} + +/// A frame offset recording a special chunk of `kind` (1 zeros, 2 NaN). +fn special_offset(kind: u8) -> [u8; 8] { + (((0x80 | kind) as i64) << 56).to_le_bytes() +} + +/// A B2ND metalayer. +fn nd_meta(shape: &[i64], chunks: &[i32], blocks: &[i32]) -> Vec { + let n = shape.len() as u8; + let mut m = vec![0x95, 0, n, 0x90 | n]; + for s in shape { + m.push(0xd3); + m.extend_from_slice(&s.to_be_bytes()); + } + for dims in [chunks, blocks] { + m.push(0x90 | n); + for d in dims { + m.push(0xd2); + m.extend_from_slice(&d.to_be_bytes()); + } + } + m +} + +/// A contiguous frame: header (with a `b2nd` metalayer if given), the data +/// chunks, then the offsets chunk. +fn frame( + meta: Option<&[u8]>, + nbytes: i64, + typesize: i32, + chunksize: i32, + data: &[u8], + offsets: &[u8], +) -> Vec { + let mut h = vec![0u8; 91]; + h[0] = 0x9e; + h[1] = 0xa8; + h[2..10].copy_from_slice(b"b2frame\0"); + h[25] = 2; + match meta { + Some(m) => { + h.extend_from_slice(&[0xde, 0, 1, 0xa4]); + h.extend_from_slice(b"b2nd"); + let at = h.len() as i32 + 5; + h.push(0xd2); + h.extend_from_slice(&at.to_be_bytes()); + h.push(0xc6); + h.extend_from_slice(&(m.len() as u32).to_be_bytes()); + h.extend_from_slice(m); + } + None => h.extend_from_slice(&[0xde, 0, 0]), + } + let header_len = h.len() as i32; + h[11..15].copy_from_slice(&header_len.to_be_bytes()); + h[30..38].copy_from_slice(&nbytes.to_be_bytes()); + h[39..47].copy_from_slice(&(data.len() as i64).to_be_bytes()); + h[48..52].copy_from_slice(&typesize.to_be_bytes()); + h[58..62].copy_from_slice(&chunksize.to_be_bytes()); + h.extend_from_slice(data); + h.extend_from_slice(offsets); + let len = h.len() as u64; + h[16..24].copy_from_slice(&len.to_be_bytes()); + h +} + +/// The frame header's own sizes must not size the offsets chunk: a frame +/// declaring 32 Mi chunks of 4 bytes, whose offsets chunk (40 bytes) says +/// "one repeated offset, 256 MiB of them", made the decoder build all +/// 256 MiB of offsets for a 1 MiB HDF5 chunk and then return 4 bytes. +#[test] +fn offsets_chunk_is_bounded_by_the_output_limit() { + let _g = lock(); + let limit = 1 << 20; + let offsets_len: i32 = 256 << 20; + let nchunks = offsets_len as i64 / 8; + let offsets = repeated(&special_offset(1), offsets_len, 64 << 20); + let f = frame(None, nchunks * 4, 4, 4, &[], &offsets); + let (r, peak) = peak_during(|| blosc2_decompress(&f, limit)); + assert!(r.is_err(), "decoded {:?} bytes", r.map(|v| v.len())); + assert!( + peak <= bound(limit, &f), + "peak {peak} bytes for a {}-byte frame", + f.len() + ); + // The same frame with a variable chunk size (0): the offsets chunk + // alone says how many chunks there are. + let f = frame(None, nchunks * 4, 4, 0, &[], &offsets); + let (r, peak) = peak_during(|| blosc2_decompress(&f, limit)); + assert!(r.is_err()); + assert!(peak <= bound(limit, &f), "chunksize 0: peak {peak} bytes"); +} + +/// A legitimate frame of this shape (one chunk, its offset special) still +/// decodes. +#[test] +fn small_frames_still_decode() { + let _g = lock(); + let offsets = repeated(&special_offset(1), 8, 8); + let f = frame(None, 64, 4, 64, &[], &offsets); + assert_eq!(blosc2_decompress(&f, 64).unwrap(), vec![0; 64]); + let _ = blosc2_decompress_chunk; +} From e05530a80519153f51d2c6845934a4bf4410e0c2 Mon Sep 17 00:00:00 2001 From: osobh Date: Sat, 26 Sep 2026 11:27:06 -0500 Subject: [PATCH 28/43] fix(format): an empty Blosc2 chunk no longer allocates its block size A chunk header with nbytes 0 and no special type kept its declared block size (up to 512 MiB): the block size was clamped to nbytes only when nbytes was positive, and the scratch blocks were allocated before the (empty) block loop, so a 20-byte chunk allocated about 1 GiB. The block size is now clamped to nbytes always, and an empty chunk returns before any scratch is allocated. A frame chunk must also decode to the size the frame header gives it (chunksize, or the remainder for the last chunk), and is decoded with that as its limit, so an empty chunk in a frame for a non-empty HDF5 chunk is an error rather than an empty result. Co-Authored-By: Claude Opus 5.5 (1M context) --- crates/clawhdf5-format/src/filters_blosc2.rs | 39 +++++++++++++------ .../tests/blosc2_alloc_bounds.rs | 33 +++++++++++++++- 2 files changed, 59 insertions(+), 13 deletions(-) diff --git a/crates/clawhdf5-format/src/filters_blosc2.rs b/crates/clawhdf5-format/src/filters_blosc2.rs index d2b1ef0..812ff24 100644 --- a/crates/clawhdf5-format/src/filters_blosc2.rs +++ b/crates/clawhdf5-format/src/filters_blosc2.rs @@ -209,7 +209,10 @@ fn read_header(src: &[u8]) -> Result { if flags2 & VL_BLOCKS != 0 { return Err(err("variable-length blocks are not supported")); } - if nbytes > 0 && blocksize > nbytes { + // c-blosc2 clamps only a non-empty chunk's block size; clamping an + // empty one too keeps its scratch blocks from being sized by the + // header (up to 512 MiB each). + if blocksize > nbytes { blocksize = nbytes; } h.blocksize = blocksize; @@ -234,9 +237,6 @@ pub fn blosc2_decompress_chunk(src: &[u8], limit: usize) -> Result, Form if memcpyed && h.cbytes != h.nbytes + h.overhead { return Err(err("stored chunk has the wrong size")); } - if h.nbytes == 0 && h.cbytes == h.overhead && h.special == 0 { - return Ok(Vec::new()); - } let nbytes = h.nbytes; let mut out = vec![0u8; nbytes]; if h.special != 0 { @@ -288,6 +288,9 @@ pub fn blosc2_decompress_chunk(src: &[u8], limit: usize) -> Result, Form return Err(err(&format!("filter {f} is not supported"))); } } + if nbytes == 0 { + return Ok(out); + } let blocksize = h.blocksize; let nblocks = nbytes.div_ceil(blocksize); let leftover = nbytes % blocksize; @@ -524,7 +527,11 @@ fn parse_frame(buf: &[u8], limit: usize) -> Result, FormatError> { } else { blosc2_decompress_chunk(off_src, off_len)? }; - let expected = if chunksize > 0 { off_len } else { offsets.len() }; + let expected = if chunksize > 0 { + off_len + } else { + offsets.len() + }; if !offsets.len().is_multiple_of(8) || offsets.len() != expected { return Err(err("offsets chunk does not match the number of chunks")); } @@ -548,15 +555,18 @@ impl Frame<'_> { } let raw: [u8; 8] = self.offsets[8 * n..8 * n + 8].try_into().unwrap(); let offset = i64::from_le_bytes(raw); + // Every chunk but the last holds `chunksize` bytes. + let size = if self.chunksize == 0 { + None + } else if n == self.nchunks - 1 && !self.nbytes.is_multiple_of(self.chunksize) { + Some(self.nbytes % self.chunksize) + } else { + Some(self.chunksize) + }; if offset < 0 { // A special chunk, recorded in the offset's top byte. - if self.chunksize == 0 { + let Some(size) = size else { return Err(err("special chunk in a frame without a chunk size")); - } - let size = if n == self.nchunks - 1 && !self.nbytes.is_multiple_of(self.chunksize) { - self.nbytes % self.chunksize - } else { - self.chunksize }; if size > limit { return Err(err("decoded size exceeds the limit")); @@ -586,7 +596,12 @@ impl Frame<'_> { .and_then(|o| self.header_len.checked_add(o)) .filter(|&s| s < end && end - s >= MIN_HEADER) .ok_or_else(|| err("chunk offset out of range"))?; - blosc2_decompress_chunk(&self.buf[start..end], limit) + let data = + blosc2_decompress_chunk(&self.buf[start..end], limit.min(size.unwrap_or(limit)))?; + if size.is_some_and(|s| s != data.len()) { + return Err(err("chunk size does not match the frame's chunk size")); + } + Ok(data) } /// The content of metalayer `name`, if the frame has it. diff --git a/crates/clawhdf5-format/tests/blosc2_alloc_bounds.rs b/crates/clawhdf5-format/tests/blosc2_alloc_bounds.rs index 11e7812..833cd53 100644 --- a/crates/clawhdf5-format/tests/blosc2_alloc_bounds.rs +++ b/crates/clawhdf5-format/tests/blosc2_alloc_bounds.rs @@ -83,7 +83,13 @@ fn chunk_header(ts: u8, nbytes: i32, blocksize: i32, cbytes: i32, special: u8) - /// A chunk of `nbytes` bytes that repeats one value (special type 3). fn repeated(value: &[u8], nbytes: i32, blocksize: i32) -> Vec { - let mut c = chunk_header(value.len() as u8, nbytes, blocksize, 32 + value.len() as i32, 3); + let mut c = chunk_header( + value.len() as u8, + nbytes, + blocksize, + 32 + value.len() as i32, + 3, + ); c.extend_from_slice(value); c } @@ -189,3 +195,28 @@ fn small_frames_still_decode() { assert_eq!(blosc2_decompress(&f, 64).unwrap(), vec![0; 64]); let _ = blosc2_decompress_chunk; } + +/// A chunk that decodes to nothing kept its declared block size (up to +/// 512 MiB) and allocated two scratch blocks of it: about 1 GiB for a +/// 20-byte chunk. +#[test] +fn empty_chunk_does_not_allocate_its_block_size() { + let _g = lock(); + let mut c = vec![5u8, 1, 0x01, 1]; + for v in [0i32, 0x1FFF_F000, 20] { + c.extend_from_slice(&v.to_le_bytes()); + } + c.resize(20, 0); + let (r, peak) = peak_during(|| blosc2_decompress_chunk(&c, 1 << 20)); + assert_eq!(r.map(|v| v.len()).unwrap_or(0), 0); + assert!( + peak <= bound(0, &c), + "peak {peak} bytes for a 20-byte chunk" + ); + // Inside a frame for a non-empty HDF5 chunk it is an error, not data. + let offsets = repeated(&0i64.to_le_bytes(), 8, 8); + let f = frame(None, 64, 4, 64, &c, &offsets); + let (r, peak) = peak_during(|| blosc2_decompress(&f, 64)); + assert!(r.is_err(), "decoded {:?}", r.map(|v| v.len())); + assert!(peak <= bound(64, &f), "in a frame: peak {peak} bytes"); +} From 22dc87b07c810665a7825a6b2a14488b89e3d54d Mon Sep 17 00:00:00 2001 From: osobh Date: Sat, 26 Sep 2026 11:30:39 -0500 Subject: [PATCH 29/43] fix(format): never hold a B2ND chunk's padding B2ND chunks were decoded whole, padding included, with up to 16x the HDF5 chunk size as their limit, so a crafted frame made each chunk allocate and fill up to 16x the output (4 GiB for a 256 MiB HDF5 chunk). The padding is the real bound (prod(ceil(c/b)*b) per chunk), but that can be 2^ndim times the array, so it is no longer held at all: - A Blosc2 chunk is now decoded block by block (decode_blocks), each block handed to a sink as it is ready, with at most three blocks of scratch. blosc2_decompress_chunk and plain frames still collect every block. - reassemble places each B2ND block straight into the output and skips blocks that are all padding (they are not decoded unless the delta filter needs the first block). A frame's NaN chunks are handed over one B2ND block at a time and its zero chunks cost nothing. - A B2ND chunk must decode to exactly its padded size, its Blosc2 blocks must be whole B2ND blocks no larger than the output, and a chunk may not be larger than the array (hdf5-blosc2's chunk is the array), so a block is never larger than the output. Peak allocation for a 10-D array padded to 13x (NaN, repeated-value and stored-block chunks) and for a 16x chunk was 4.5 MB and 17.8 MB for 315 KB and 1 MiB outputs before, and is now within the tests' bound. Blosc2 files written by hdf5plugin in 9-D and 12-D, an 8 MiB single chunk, 1x1x1 and edge-chunk shapes still read exactly. Co-Authored-By: Claude Opus 5.5 (1M context) --- crates/clawhdf5-format/src/filters_blosc2.rs | 377 +++++++++++++----- .../tests/blosc2_alloc_bounds.rs | 128 ++++++ 2 files changed, 408 insertions(+), 97 deletions(-) diff --git a/crates/clawhdf5-format/src/filters_blosc2.rs b/crates/clawhdf5-format/src/filters_blosc2.rs index 812ff24..71ad4b6 100644 --- a/crates/clawhdf5-format/src/filters_blosc2.rs +++ b/crates/clawhdf5-format/src/filters_blosc2.rs @@ -225,20 +225,59 @@ fn read_header(src: &[u8]) -> Result { /// Decompress one Blosc2 chunk (header included), refusing to produce more /// than `limit` bytes. pub fn blosc2_decompress_chunk(src: &[u8], limit: usize) -> Result, FormatError> { + let (h, src) = open_chunk(src, limit)?; + let mut out = vec![0u8; h.nbytes]; + decode_blocks(&h, src, &mut |_, _| true, &mut |at, block| { + out[at..at + block.len()].copy_from_slice(block); + })?; + Ok(out) +} + +/// Read and check a chunk's header: its decoded size must be within +/// `limit`. Returns the header and the chunk's bytes. +fn open_chunk(src: &[u8], limit: usize) -> Result<(ChunkHeader, &[u8]), FormatError> { let h = read_header(src)?; if h.nbytes > limit { return Err(err("decoded size exceeds the limit")); } - let src = &src[..h.cbytes]; if h.special > SPECIAL_UNINIT { return Err(err(&format!("unknown special chunk type {}", h.special))); } - let memcpyed = h.flags & FLAG_MEMCPYED != 0; - if memcpyed && h.cbytes != h.nbytes + h.overhead { + if h.flags & FLAG_MEMCPYED != 0 && h.cbytes != h.nbytes + h.overhead { return Err(err("stored chunk has the wrong size")); } + let src = &src[..h.cbytes]; + Ok((h, src)) +} + +/// Decode a chunk block by block into a destination the caller has +/// zeroed: `sink(offset, block)` receives each decoded block (the last may +/// be short), except blocks for which `want(offset, len)` is false and +/// the blocks of a special all-zeros chunk. Holds at most three blocks of scratch, so a +/// caller that keeps only part of each block (B2ND padding) never holds +/// the whole chunk. +fn decode_blocks( + h: &ChunkHeader, + src: &[u8], + want: &mut dyn FnMut(usize, usize) -> bool, + sink: &mut dyn FnMut(usize, &[u8]), +) -> Result<(), FormatError> { let nbytes = h.nbytes; - let mut out = vec![0u8; nbytes]; + // `read_header` clamps the block size to the decoded size. + let blocksize = h.blocksize; + let nblocks = if nbytes == 0 { + 0 + } else { + nbytes.div_ceil(blocksize) + }; + let leftover = nbytes % blocksize.max(1); + let block_len = |j: usize| { + if j == nblocks - 1 && leftover > 0 { + leftover + } else { + blocksize + } + }; if h.special != 0 { // Filled per block, as c-blosc2 does: each block must hold whole // values. @@ -253,19 +292,30 @@ pub fn blosc2_decompress_chunk(src: &[u8], limit: usize) -> Result, Form }; if let Some(value) = fill { let ts = value.len(); - let blocksize = h.blocksize.max(1); - if !blocksize.is_multiple_of(ts) || !(nbytes % blocksize).is_multiple_of(ts) { + if !blocksize.max(1).is_multiple_of(ts) || !leftover.is_multiple_of(ts) { return Err(err("special chunk blocks are not whole values")); } - for v in out.chunks_exact_mut(ts) { - v.copy_from_slice(value); + if nblocks > 0 { + let block: Vec = value.iter().copied().cycle().take(blocksize).collect(); + for j in 0..nblocks { + let len = block_len(j); + if want(j * blocksize, len) { + sink(j * blocksize, &block[..len]); + } + } } } - return Ok(out); + return Ok(()); } - if memcpyed { - out.copy_from_slice(&src[h.overhead..]); - return Ok(out); + if h.flags & FLAG_MEMCPYED != 0 { + let data = &src[h.overhead..]; + for j in 0..nblocks { + let (at, len) = (j * blocksize, block_len(j)); + if want(at, len) { + sink(at, &data[at..at + len]); + } + } + return Ok(()); } if h.blosc2_flags & B2_USEDICT != 0 { return Err(err("dictionary-compressed chunks are not supported")); @@ -289,11 +339,8 @@ pub fn blosc2_decompress_chunk(src: &[u8], limit: usize) -> Result, Form } } if nbytes == 0 { - return Ok(out); + return Ok(()); } - let blocksize = h.blocksize; - let nblocks = nbytes.div_ceil(blocksize); - let leftover = nbytes % blocksize; let bstarts_end = nblocks .checked_mul(4) .and_then(|n| n.checked_add(h.overhead)) @@ -302,12 +349,19 @@ pub fn blosc2_decompress_chunk(src: &[u8], limit: usize) -> Result, Form return Err(err("block table is truncated")); } let dont_split = h.flags & FLAG_DONT_SPLIT != 0; + // The delta filter's reference is the first block, decoded. + let delta = h.filters.contains(&FILTER_DELTA); + let mut first = Vec::new(); let mut tmp = vec![0u8; blocksize]; let mut tmp2 = vec![0u8; blocksize]; let mut zstd = None; for j in 0..nblocks { let is_leftover = j == nblocks - 1 && leftover > 0; - let bsize = if is_leftover { leftover } else { blocksize }; + let bsize = block_len(j); + let wanted = want(j * blocksize, bsize); + if !wanted && !(delta && j == 0) { + continue; + } let start = le_i32(src, h.overhead + 4 * j)?; if start <= 0 || start as usize >= src.len() { return Err(err("block start out of range")); @@ -355,23 +409,21 @@ pub fn blosc2_decompress_chunk(src: &[u8], limit: usize) -> Result, Form } } } - let (done, rest) = out.split_at_mut(j * blocksize); - let dest = &mut rest[..bsize]; - run_filters_backward(&h, cur, &mut tmp2[..bsize], done, dest); + run_filters_backward(h, cur, &mut tmp2[..bsize], &first); + if delta && j == 0 { + first = cur.to_vec(); + } + if wanted { + sink(j * blocksize, cur); + } } - Ok(out) + Ok(()) } -/// Undo the chunk's filters on one decoded block `cur`, leaving the result -/// in `dest`. `done` is the chunk decoded so far (the delta filter's -/// reference is the first block). -fn run_filters_backward( - h: &ChunkHeader, - cur: &mut [u8], - scratch: &mut [u8], - done: &[u8], - dest: &mut [u8], -) { +/// Undo the chunk's filters on one decoded block `cur`, in place. `first` +/// is the decoded first block (empty while decoding it), the delta +/// filter's reference. +fn run_filters_backward(h: &ChunkHeader, cur: &mut [u8], scratch: &mut [u8], first: &[u8]) { let bsize = cur.len(); let ts = h.typesize; for i in (0..6).rev() { @@ -395,13 +447,12 @@ fn run_filters_backward( bitunshuffle_block(&cur[..body], &mut scratch[..body], n, ts); cur[..body].copy_from_slice(&scratch[..body]); } - FILTER_DELTA => delta_decode(done, cur, ts), + FILTER_DELTA => delta_decode(first, cur, ts), // Truncating precision is lossy and has nothing to undo. FILTER_NONE | FILTER_TRUNC_PREC => {} _ => unreachable!("filters are checked before decoding"), } } - dest.copy_from_slice(cur); } /// Byte unshuffle (`unshuffle_generic`): trailing bytes that do not make a @@ -416,18 +467,19 @@ fn unshuffle(bytes: usize, src: &[u8], dest: &mut [u8]) { dest[n * bytes..].copy_from_slice(&src[n * bytes..]); } -/// `delta_decoder`: the first block is XOR-accumulated over its own -/// elements; every other block is XORed with the (decoded) first block. +/// `delta_decoder`: the first block (`first` is empty) is XOR-accumulated +/// over its own elements; every other block is XORed with the (decoded) +/// first block. /// Elements are 1, 2, 4 or 8 bytes wide (other sizes: 8 if a multiple of /// 8, else 1); a trailing partial element is left alone. -fn delta_decode(done: &[u8], cur: &mut [u8], typesize: usize) { +fn delta_decode(first: &[u8], cur: &mut [u8], typesize: usize) { let w = match typesize { 1 | 2 | 4 | 8 => typesize, t if t.is_multiple_of(8) => 8, _ => 1, }; let n = cur.len() / w; - if done.is_empty() { + if first.is_empty() { for i in 1..n { for b in 0..w { cur[i * w + b] ^= cur[(i - 1) * w + b]; @@ -435,7 +487,7 @@ fn delta_decode(done: &[u8], cur: &mut [u8], typesize: usize) { } } else { // The first block is at least as long as any other. - for (c, r) in cur[..n * w].iter_mut().zip(done) { + for (c, r) in cur[..n * w].iter_mut().zip(first) { *c ^= *r; } } @@ -548,14 +600,15 @@ fn parse_frame(buf: &[u8], limit: usize) -> Result, FormatError> { } impl Frame<'_> { - /// Decode chunk `n`, refusing more than `limit` bytes. - fn chunk(&self, n: usize, limit: usize) -> Result, FormatError> { + /// Where chunk `n` is: its size from the frame header (every chunk but + /// the last holds `chunksize` bytes; unknown if that is 0), and either + /// the special value its offset records or its bytes. + fn locate(&self, n: usize) -> Result<(Option, Located<'_>), FormatError> { if n >= self.nchunks { return Err(err("frame has no such chunk")); } let raw: [u8; 8] = self.offsets[8 * n..8 * n + 8].try_into().unwrap(); let offset = i64::from_le_bytes(raw); - // Every chunk but the last holds `chunksize` bytes. let size = if self.chunksize == 0 { None } else if n == self.nchunks - 1 && !self.nbytes.is_multiple_of(self.chunksize) { @@ -565,26 +618,20 @@ impl Frame<'_> { }; if offset < 0 { // A special chunk, recorded in the offset's top byte. - let Some(size) = size else { + if size.is_none() { return Err(err("special chunk in a frame without a chunk size")); - }; - if size > limit { - return Err(err("decoded size exceeds the limit")); } let kind = raw[7] & 7; return match kind { - SPECIAL_ZERO | SPECIAL_UNINIT => Ok(vec![0u8; size]), - SPECIAL_NAN => { - let nan: &[u8] = match self.typesize { + SPECIAL_ZERO | SPECIAL_UNINIT => Ok((size, Located::Zeros)), + SPECIAL_NAN => Ok(( + size, + Located::Fill(match self.typesize { 4 => &[0x00, 0x00, 0xc0, 0x7f], 8 => &[0, 0, 0, 0, 0, 0, 0xf8, 0x7f], _ => return Err(err("NaN chunk of a type that is not f32 or f64")), - }; - if !size.is_multiple_of(nan.len()) { - return Err(err("NaN chunk is not whole values")); - } - Ok(nan.iter().copied().cycle().take(size).collect()) - } + }), + )), _ => Err(err(&format!("unknown special chunk offset {kind}"))), }; } @@ -596,12 +643,90 @@ impl Frame<'_> { .and_then(|o| self.header_len.checked_add(o)) .filter(|&s| s < end && end - s >= MIN_HEADER) .ok_or_else(|| err("chunk offset out of range"))?; - let data = - blosc2_decompress_chunk(&self.buf[start..end], limit.min(size.unwrap_or(limit)))?; - if size.is_some_and(|s| s != data.len()) { + Ok((size, Located::Data(&self.buf[start..end]))) + } + + /// Open chunk `n` (a regular one), which must decode to the frame's + /// size for it, and at most `limit` bytes. + fn open<'s>( + &self, + size: Option, + src: &'s [u8], + limit: usize, + ) -> Result<(ChunkHeader, &'s [u8]), FormatError> { + let (h, src) = open_chunk(src, limit.min(size.unwrap_or(limit)))?; + if size.is_some_and(|s| s != h.nbytes) { return Err(err("chunk size does not match the frame's chunk size")); } - Ok(data) + Ok((h, src)) + } + + /// Decode chunk `n`, refusing more than `limit` bytes. + fn chunk(&self, n: usize, limit: usize) -> Result, FormatError> { + let (size, at) = self.locate(n)?; + let src = match at { + Located::Data(src) => src, + Located::Zeros | Located::Fill(_) => { + let size = size.unwrap_or(0); + if size > limit { + return Err(err("decoded size exceeds the limit")); + } + return match at { + Located::Fill(value) if !size.is_multiple_of(value.len()) => { + Err(err("NaN chunk is not whole values")) + } + Located::Fill(value) => Ok(value.iter().copied().cycle().take(size).collect()), + _ => Ok(vec![0u8; size]), + }; + } + }; + let (h, src) = self.open(size, src, limit)?; + let mut out = vec![0u8; h.nbytes]; + decode_blocks(&h, src, &mut |_, _| true, &mut |at, block| { + out[at..at + block.len()].copy_from_slice(block); + })?; + Ok(out) + } + + /// Decode chunk `n` (exactly `size` bytes) block by block, as + /// [`decode_blocks`], into a destination the caller has zeroed. Its + /// blocks must be whole multiples of `unit` bytes and at most + /// `max_block`; special chunks are handed over `unit` bytes at a time. + fn chunk_blocks( + &self, + n: usize, + size: usize, + unit: usize, + max_block: usize, + want: &mut dyn FnMut(usize, usize) -> bool, + sink: &mut dyn FnMut(usize, &[u8]), + ) -> Result<(), FormatError> { + let (frame_size, at) = self.locate(n)?; + if frame_size.is_some_and(|s| s != size) { + return Err(err("chunk size does not match the b2nd shape")); + } + match at { + Located::Zeros => Ok(()), + Located::Fill(value) => { + if !unit.is_multiple_of(value.len()) || !size.is_multiple_of(unit) { + return Err(err("NaN chunk is not whole values")); + } + let block: Vec = value.iter().copied().cycle().take(unit).collect(); + for k in 0..size / unit { + if want(k * unit, unit) { + sink(k * unit, &block); + } + } + Ok(()) + } + Located::Data(src) => { + let (h, src) = self.open(Some(size), src, size)?; + if h.nbytes > 0 && (!h.blocksize.is_multiple_of(unit) || h.blocksize > max_block) { + return Err(err("chunk blocks do not match the b2nd blocks")); + } + decode_blocks(&h, src, want, sink) + } + } } /// The content of metalayer `name`, if the frame has it. @@ -653,6 +778,16 @@ impl Frame<'_> { } } +/// Where a frame's chunk is. +enum Located<'a> { + /// All zeros (or uninitialised, read as zeros). + Zeros, + /// One value repeated (NaN). + Fill(&'static [u8]), + /// A Blosc2 chunk, running to the end of the data section. + Data(&'a [u8]), +} + /// The B2ND (or Caterva) metalayer: shape, chunk shape, block shape. #[derive(Debug, PartialEq, Eq)] struct NdMeta { @@ -753,6 +888,12 @@ fn decode_frame( } /// Gather a B2ND array's blocks into one C-order buffer. +/// +/// Each Blosc2 chunk holds one B2ND chunk padded to whole blocks, which can +/// be many times the array (a crafted frame's shapes are the file's word). +/// So the chunks are never decoded whole: each block is placed in the +/// output as it is decoded, blocks that are all padding are skipped, and +/// no more than a few blocks (each within the array) are held at once. fn reassemble(frame: &Frame<'_>, nd: &NdMeta, limit: usize) -> Result, FormatError> { let ts = frame.typesize; let ndim = nd.shape.len(); @@ -778,10 +919,15 @@ fn reassemble(frame: &Frame<'_>, nd: &NdMeta, limit: usize) -> Result, F if total == 0 { return Ok(out); } - // Padding to whole blocks grows a chunk by less than 2x per dimension; - // hdf5-blosc2's block shapes keep it far below 16x (a crafted frame - // could otherwise make each chunk enormous). - if ext_bytes > limit.saturating_mul(16) { + // hdf5-blosc2's chunk is the array. A chunk larger than the array + // would be padding decoded for nothing; with chunks and blocks within + // the array, a block is never larger than the output, and a chunk + // pads each dimension by less than one block. + if (0..ndim).any(|i| nd.chunkshape[i] > nd.shape[i]) { + return Err(err("b2nd chunk is larger than the array")); + } + // A Blosc2 chunk of `ext_bytes`, exactly the padded chunk. + if ext_bytes > i32::MAX as usize { return Err(too_big()); } let nchunks: usize = grid.iter().product(); @@ -789,7 +935,6 @@ fn reassemble(frame: &Frame<'_>, nd: &NdMeta, limit: usize) -> Result, F return Err(err("chunk count does not match the b2nd shape")); } let blocks_in_chunk: Vec = (0..ndim).map(|i| ext[i] / nd.blockshape[i]).collect(); - let nblocks: usize = blocks_in_chunk.iter().product(); let block_items: usize = nd.blockshape.iter().product(); let block_bytes = block_items * ts; // Strides (in elements) of the output array and of a block. @@ -799,50 +944,88 @@ fn reassemble(frame: &Frame<'_>, nd: &NdMeta, limit: usize) -> Result, F out_stride[i] = out_stride[i + 1] * nd.shape[i + 1]; blk_stride[i] = blk_stride[i + 1] * nd.blockshape[i + 1]; } + let geometry = BlockGeometry { + nd, + blocks_in_chunk: &blocks_in_chunk, + }; let mut cidx = vec![0usize; ndim]; - let mut bidx = vec![0usize; ndim]; - let mut gstart = vec![0usize; ndim]; - let mut valid = vec![0usize; ndim]; + let mut w = BlockPlace::new(ndim); + let mut p = BlockPlace::new(ndim); let mut pos = vec![0usize; ndim]; for n in 0..nchunks { unravel(n, &grid, &mut cidx); - let data = frame.chunk(n, ext_bytes)?; - if data.len() != ext_bytes { - return Err(err("chunk size does not match the b2nd shape")); - } - for b in 0..nblocks { - unravel(b, &blocks_in_chunk, &mut bidx); - let mut empty = false; - for i in 0..ndim { - let in_chunk = bidx[i] * nd.blockshape[i]; - gstart[i] = cidx[i] * nd.chunkshape[i] + in_chunk; - let lim_chunk = nd.chunkshape[i].saturating_sub(in_chunk); - let lim_shape = nd.shape[i].saturating_sub(gstart[i]); - valid[i] = nd.blockshape[i].min(lim_chunk).min(lim_shape); - empty |= valid[i] == 0; - } - if empty { - continue; - } - let block = &data[b * block_bytes..(b + 1) * block_bytes]; - // Copy row by row along the last dimension. - let row = valid[ndim - 1] * ts; - let rows: usize = valid[..ndim - 1].iter().product(); - for r in 0..rows { - unravel(r, &valid[..ndim - 1], &mut pos[..ndim - 1]); - let mut src = 0; - let mut dst = gstart[ndim - 1]; - for i in 0..ndim - 1 { - src += pos[i] * blk_stride[i]; - dst += (gstart[i] + pos[i]) * out_stride[i]; + let blocks = |at: usize, len: usize| at / block_bytes..(at + len).div_ceil(block_bytes); + let mut want = + |at: usize, len: usize| blocks(at, len).any(|b| geometry.place(b, &cidx, &mut w)); + let mut sink = |at: usize, data: &[u8]| { + for b in blocks(at, data.len()) { + if !geometry.place(b, &cidx, &mut p) { + continue; + } + let block = &data[b * block_bytes - at..][..block_bytes]; + // Copy row by row along the last dimension. + let row = p.valid[ndim - 1] * ts; + let rows: usize = p.valid[..ndim - 1].iter().product(); + for r in 0..rows { + unravel(r, &p.valid[..ndim - 1], &mut pos[..ndim - 1]); + let mut src = 0; + let mut dst = p.gstart[ndim - 1]; + for i in 0..ndim - 1 { + src += pos[i] * blk_stride[i]; + dst += (p.gstart[i] + pos[i]) * out_stride[i]; + } + out[dst * ts..dst * ts + row].copy_from_slice(&block[src * ts..src * ts + row]); } - out[dst * ts..dst * ts + row].copy_from_slice(&block[src * ts..src * ts + row]); } - } + }; + frame.chunk_blocks(n, ext_bytes, block_bytes, limit, &mut want, &mut sink)?; } Ok(out) } +/// The block layout of a B2ND array's chunks. +struct BlockGeometry<'a> { + nd: &'a NdMeta, + blocks_in_chunk: &'a [usize], +} + +/// Where one block of a chunk lands: its first element in the array and +/// how much of it (per dimension) lies within the chunk and the array. +struct BlockPlace { + bidx: Vec, + gstart: Vec, + valid: Vec, +} + +impl BlockPlace { + fn new(ndim: usize) -> BlockPlace { + BlockPlace { + bidx: vec![0; ndim], + gstart: vec![0; ndim], + valid: vec![0; ndim], + } + } +} + +impl BlockGeometry<'_> { + /// Place block `b` of the chunk at grid index `cidx`; false if it is + /// all padding. + fn place(&self, b: usize, cidx: &[usize], p: &mut BlockPlace) -> bool { + let nd = self.nd; + unravel(b, self.blocks_in_chunk, &mut p.bidx); + let mut empty = false; + for i in 0..nd.shape.len() { + let in_chunk = p.bidx[i] * nd.blockshape[i]; + p.gstart[i] = cidx[i] * nd.chunkshape[i] + in_chunk; + let lim_chunk = nd.chunkshape[i].saturating_sub(in_chunk); + let lim_shape = nd.shape[i].saturating_sub(p.gstart[i]); + p.valid[i] = nd.blockshape[i].min(lim_chunk).min(lim_shape); + empty |= p.valid[i] == 0; + } + !empty + } +} + /// C-order multi-index of `n` in a grid of `dims`. fn unravel(mut n: usize, dims: &[usize], idx: &mut [usize]) { for i in (0..dims.len()).rev() { diff --git a/crates/clawhdf5-format/tests/blosc2_alloc_bounds.rs b/crates/clawhdf5-format/tests/blosc2_alloc_bounds.rs index 833cd53..a6ba114 100644 --- a/crates/clawhdf5-format/tests/blosc2_alloc_bounds.rs +++ b/crates/clawhdf5-format/tests/blosc2_alloc_bounds.rs @@ -220,3 +220,131 @@ fn empty_chunk_does_not_allocate_its_block_size() { assert!(r.is_err(), "decoded {:?}", r.map(|v| v.len())); assert!(peak <= bound(64, &f), "in a frame: peak {peak} bytes"); } + +/// B2ND chunks were decoded whole, padding included, with up to 16x the +/// HDF5 chunk size as their limit. Blocks are now placed as they are +/// decoded, so the padding is never held. +/// +/// Ten dimensions: nine of 3 split into blocks of 2 (padded to 4) and one +/// of 4, so each chunk is 13x the array. One chunk, stored three ways: as a +/// NaN chunk in the frame's offsets, as a repeated-value chunk, and as a +/// chunk of stored (uncompressed) blocks. +#[test] +fn b2nd_padding_is_never_held() { + let _g = lock(); + let ts = 4usize; + let mut shape = vec![3i64; 9]; + shape.push(4); + let chunks: Vec = shape.iter().map(|&s| s as i32).collect(); + let mut blocks = vec![2i32; 9]; + blocks.push(4); + let meta = nd_meta(&shape, &chunks, &blocks); + let items: usize = shape.iter().product::() as usize; + let limit = items * ts; + let block_bytes = ts * blocks.iter().product::() as usize; + let ext_bytes = ts * 4usize.pow(9) * 4; + assert!(ext_bytes > 13 * limit); + let offsets = |off: [u8; 8]| repeated(&off, 8, 8); + + let value = 1.5f32.to_le_bytes(); + let stored = { + // Every block stored raw: block k holds the value k. + let mut c = chunk_header(4, ext_bytes as i32, block_bytes as i32, 0, 0); + c[2] = 0x02 | 0x10; // memcpyed, not split + c.truncate(16); + for k in 0..ext_bytes / block_bytes { + c.extend((k as f32).to_le_bytes().repeat(block_bytes / 4)); + } + let n = c.len() as i32; + c[12..16].copy_from_slice(&n.to_le_bytes()); + c + }; + let cases: Vec<(&str, Vec)> = vec![ + ( + "NaN offset", + frame( + Some(&meta), + ext_bytes as i64, + 4, + ext_bytes as i32, + &[], + &offsets(special_offset(2)), + ), + ), + ( + "repeated value", + frame( + Some(&meta), + ext_bytes as i64, + 4, + ext_bytes as i32, + &repeated(&value, ext_bytes as i32, block_bytes as i32), + &offsets(0i64.to_le_bytes()), + ), + ), + ( + "stored blocks", + frame( + Some(&meta), + ext_bytes as i64, + 4, + ext_bytes as i32, + &stored, + &offsets(0i64.to_le_bytes()), + ), + ), + ]; + for (name, f) in cases { + let (r, peak) = peak_during(|| blosc2_decompress(&f, limit)); + let out = r.unwrap_or_else(|e| panic!("{name}: {e}")); + assert_eq!(out.len(), limit, "{name}"); + match name { + "NaN offset" => assert!( + out.chunks(4) + .all(|v| f32::from_le_bytes(v.try_into().unwrap()).is_nan()) + ), + "repeated value" => assert!(out.chunks(4).all(|v| v == value)), + _ => { + // Element (i0..i9) lies in block (i0/2, .., i8/2), numbered + // in C order over a 2x..x2x1 grid of blocks. + let mut idx = [0usize; 10]; + for (e, v) in out.chunks(4).enumerate() { + let mut n = e; + for d in (0..10).rev() { + idx[d] = n % shape[d] as usize; + n /= shape[d] as usize; + } + let k = idx[..9].iter().fold(0, |k, &i| k * 2 + i / 2); + assert_eq!( + f32::from_le_bytes(v.try_into().unwrap()), + k as f32, + "{name} {e}" + ); + } + } + } + assert!( + peak <= bound(limit, &f), + "{name}: peak {peak} bytes for a {limit}-byte chunk ({}-byte frame)", + f.len() + ); + } +} + +/// A B2ND chunk larger than the array (here 16x, the old cap) is refused, +/// or at least never allocated. +#[test] +fn b2nd_chunk_larger_than_the_array_is_not_allocated() { + let _g = lock(); + let limit = 1 << 20; + let c = 16 * limit as i32; + let meta = nd_meta(&[limit as i64], &[c], &[c]); + let offsets = repeated(&special_offset(1), 8, 8); + let f = frame(Some(&meta), c as i64, 1, c, &[], &offsets); + let (r, peak) = peak_during(|| blosc2_decompress(&f, limit)); + assert!( + peak <= bound(limit, &f), + "peak {peak} bytes ({:?})", + r.map(|v| v.len()) + ); +} From d9e4dfb6e6de63a66c0238630b58c13405c767d5 Mon Sep 17 00:00:00 2001 From: osobh Date: Sat, 26 Sep 2026 11:34:28 -0500 Subject: [PATCH 30/43] fix(format): cap the Zstandard window at what the output can need ruzstd reserves a frame's declared window (up to its 100 MiB default) when a decoder is reset for a new frame, before decoding anything. The Blosc, Blosc2 and bitshuffle decoders reuse one decoder per chunk, so a Blosc2 chunk of two 16-byte streams, each declaring a 96 MiB window, allocated 128 MiB. zstd_decode_into now sets the decoder's maximum window to twice the stream's output (at least 128 KiB): c-blosc, c-blosc2 and bitshuffle compress each block in one call with its size known, so libzstd's window never exceeds the block. Found by tracking peak allocation in the Blosc2 fuzz test. Co-Authored-By: Claude Opus 5.5 (1M context) --- .../clawhdf5-format/src/filters_bitshuffle.rs | 8 +++ .../tests/blosc2_alloc_bounds.rs | 51 ++++++++++++++++++- 2 files changed, 57 insertions(+), 2 deletions(-) diff --git a/crates/clawhdf5-format/src/filters_bitshuffle.rs b/crates/clawhdf5-format/src/filters_bitshuffle.rs index d20835e..6b23291 100644 --- a/crates/clawhdf5-format/src/filters_bitshuffle.rs +++ b/crates/clawhdf5-format/src/filters_bitshuffle.rs @@ -218,12 +218,20 @@ pub(crate) fn bitshuffle_decode( } /// Decode Zstandard frames into exactly `dst`, failing if they hold more. +/// +/// ruzstd reserves a frame's declared window (by default up to 100 MiB) +/// before decoding it, so the window is capped at what the output could +/// need: twice `dst` (window sizes are rounded up), and at least 128 KiB. +/// The encoders behind these filters (c-blosc, c-blosc2, bitshuffle) +/// compress each block in one call with its size known, so libzstd's +/// window never exceeds the block. #[cfg(any(feature = "bitshuffle", feature = "blosc"))] pub(crate) fn zstd_decode_into( decoder: &mut ruzstd::decoding::FrameDecoder, frames: &[u8], dst: &mut [u8], ) -> Result { + decoder.set_max_window_size((2 * dst.len()).max(1 << 17) as u64); decoder .decode_all(frames, dst) .map_err(|e| FormatError::DecompressionError(format!("zstd: {e}"))) diff --git a/crates/clawhdf5-format/tests/blosc2_alloc_bounds.rs b/crates/clawhdf5-format/tests/blosc2_alloc_bounds.rs index a6ba114..5dc71b0 100644 --- a/crates/clawhdf5-format/tests/blosc2_alloc_bounds.rs +++ b/crates/clawhdf5-format/tests/blosc2_alloc_bounds.rs @@ -61,9 +61,11 @@ fn peak_during(f: impl FnOnce() -> T) -> (T, usize) { /// What decoding one HDF5 chunk of `limit` bytes from `input` may hold at /// once: the output, a few blocks of scratch (each no larger than the -/// output), the offsets table, and the Zstandard decoder's state. +/// output), the offsets table, and the Zstandard decoder's state, which has +/// a fixed ceiling: a window of at most 128 KiB (or twice the stream) and a +/// block's table of sequences (up to 98,303 of 12 bytes, 1.2 MB). fn bound(limit: usize, input: &[u8]) -> usize { - 6 * limit + 2 * input.len() + (1 << 20) + 6 * limit + 2 * input.len() + (2 << 20) } fn lock() -> std::sync::MutexGuard<'static, ()> { @@ -348,3 +350,48 @@ fn b2nd_chunk_larger_than_the_array_is_not_allocated() { r.map(|v| v.len()) ); } + +/// ruzstd reserves a frame's declared window (up to 100 MiB) before it +/// decodes a frame with a decoder it has used before: a Blosc2 chunk of +/// two 16-byte Zstandard streams, each declaring a 96 MiB window, +/// allocated 96 MiB. c-blosc2 compresses each block with its size known, +/// so its windows never exceed the block. +#[test] +fn zstd_window_is_bounded_by_the_output() { + let _g = lock(); + let mut z = 0xfd2f_b528u32.to_le_bytes().to_vec(); + // No single segment, no checksum; window 2^26 + 4/8 of it = 96 MiB. + z.extend_from_slice(&[0x00, (16 << 3) | 4]); + // One raw block, last, of 16 bytes. + let h = 1 | (16 << 3); + z.extend_from_slice(&[h as u8, (h >> 8) as u8, 0]); + z.extend_from_slice(&[7; 16]); + // Two blocks of 16 bytes, one stream each (not split), Zstandard + // (codec 4). + let chunk = |z: &[u8]| { + let mut c = vec![5u8, 1, 0x10 | (4 << 5), 1]; + for v in [32i32, 16, 0] { + c.extend_from_slice(&v.to_le_bytes()); + } + let first = 24 + 4 + z.len(); + c.extend_from_slice(&24i32.to_le_bytes()); + c.extend_from_slice(&(first as i32).to_le_bytes()); + for _ in 0..2 { + c.extend_from_slice(&(z.len() as i32).to_le_bytes()); + c.extend_from_slice(z); + } + let n = c.len() as i32; + c[12..16].copy_from_slice(&n.to_le_bytes()); + c + }; + let c = chunk(&z); + let (r, peak) = peak_during(|| blosc2_decompress_chunk(&c, 32)); + assert!(peak <= bound(32, &c), "peak {peak} bytes ({r:?})"); + assert!(r.is_err(), "{r:?}"); + // The same streams with a window they can use read. + z[5] = 0; + assert_eq!( + blosc2_decompress_chunk(&chunk(&z), 32).unwrap(), + vec![7; 32] + ); +} From 2ba4bc97d8ae01cfb88eac7b019fb6947e9e270a Mon Sep 17 00:00:00 2001 From: osobh Date: Sat, 26 Sep 2026 11:35:08 -0500 Subject: [PATCH 31/43] test(format): fuzz Blosc2 decoding for peak allocation The Blosc2 fuzz tests checked only for panics and the size of the output, so the offsets-chunk amplification passed all 40,000 iterations. tests/blosc2_alloc_bounds.rs now measures peak allocation (a counting global allocator) and asserts it stays within 6x the HDF5 chunk size plus twice the input plus 2 MiB (ruzstd's fixed state) for: - 20,000 mutated fixture frames, decoded with their real chunk size as the limit, with edits aimed at the frame's and chunks' size fields; - 20,000 mutated first chunks on their own; - 5,000 frames built from random header sizes, offsets chunks and B2ND shapes (padding chunk and block shapes, chunks larger than the array, NaN, zero and repeated-value chunks). Against the code before this series every test in the file fails (the fuzz tests at frame iteration 3913, a zstd window, and random frame 289, the offsets chunk); now the worst frame peaks at 0.48 of the bound. Co-Authored-By: Claude Opus 5.5 (1M context) --- crates/clawhdf5-format/src/filters_blosc2.rs | 3 +- .../tests/blosc2_alloc_bounds.rs | 244 ++++++++++++++++++ 2 files changed, 246 insertions(+), 1 deletion(-) diff --git a/crates/clawhdf5-format/src/filters_blosc2.rs b/crates/clawhdf5-format/src/filters_blosc2.rs index 71ad4b6..a70a906 100644 --- a/crates/clawhdf5-format/src/filters_blosc2.rs +++ b/crates/clawhdf5-format/src/filters_blosc2.rs @@ -1246,7 +1246,8 @@ mod tests { } /// Random and mutated frames: errors are fine, panics are not, and no - /// output past the limit. + /// output past the limit. Peak allocation is fuzzed separately, with a + /// counting allocator, in `tests/blosc2_alloc_bounds.rs`. #[test] fn fuzzed_frames_never_panic() { let limit = 20_000; diff --git a/crates/clawhdf5-format/tests/blosc2_alloc_bounds.rs b/crates/clawhdf5-format/tests/blosc2_alloc_bounds.rs index 5dc71b0..1667186 100644 --- a/crates/clawhdf5-format/tests/blosc2_alloc_bounds.rs +++ b/crates/clawhdf5-format/tests/blosc2_alloc_bounds.rs @@ -395,3 +395,247 @@ fn zstd_window_is_bounded_by_the_output() { vec![7; 32] ); } + +/// xorshift64*: deterministic, so a failure reproduces. +struct Rng(u64); + +impl Rng { + fn next(&mut self) -> u64 { + let mut x = self.0; + x ^= x >> 12; + x ^= x << 25; + x ^= x >> 27; + self.0 = x; + x.wrapping_mul(0x2545_F491_4F6C_DD1D) + } + + fn below(&mut self, n: usize) -> usize { + (self.next() % n.max(1) as u64) as usize + } + + /// A size that tends to the edges: small, a power of two, huge. + fn size(&mut self) -> i64 { + match self.below(6) { + 0 => self.below(64) as i64, + 1 => 1 << self.below(31), + 2 => i32::MAX as i64 - self.below(4096) as i64, + 3 => (1i64 << self.below(62)) + self.below(8) as i64, + 4 => MAX_BLOCK - self.below(3) as i64, + _ => self.next() as i32 as i64, + } + } +} + +const MAX_BLOCK: i64 = 536_866_816; + +/// One to four edits: bytes, or a size field written little-endian (chunk +/// headers) or big-endian (frame headers), most often at a header's size +/// fields. +fn mutate(rng: &mut Rng, seed: &[u8], data_at: usize) -> Vec { + let mut v = seed.to_vec(); + for _ in 0..1 + rng.below(4) { + let len = v.len(); + if len < 16 { + v.push(rng.next() as u8); + continue; + } + match rng.below(8) { + 0 => { + let i = rng.below(len); + v[i] ^= 1 << rng.below(8); + } + 1 => { + let i = rng.below(len); + v[i] = rng.next() as u8; + } + 2 => { + // Frame header: nbytes, cbytes (i64), typesize, chunksize. + let x = rng.size(); + match rng.below(4) { + 0 if len >= 38 => v[30..38].copy_from_slice(&x.to_be_bytes()), + 1 if len >= 47 => v[39..47].copy_from_slice(&x.to_be_bytes()), + 2 if len >= 52 => v[48..52].copy_from_slice(&(x as i32).to_be_bytes()), + _ if len >= 62 => v[58..62].copy_from_slice(&(x as i32).to_be_bytes()), + _ => {} + } + } + 3 | 4 => { + // A chunk header's nbytes, blocksize or cbytes: in the first + // data chunk, or anywhere (the offsets chunk comes last). + let at = if rng.below(2) == 0 && data_at + 16 <= len { + data_at + 4 * (1 + rng.below(3)) + } else { + rng.below(len - 3) + }; + let x = rng.size() as i32; + v[at..at + 4].copy_from_slice(&x.to_le_bytes()); + } + 5 => v.truncate(rng.below(len)), + 6 => { + let at = rng.below(len); + v[at] = [0x10, 0x20, 0x30, 0x40, 0x05, 0x07, 0x02][rng.below(7)]; + } + _ => { + let i = rng.below(len - 3); + let x = rng.size() as i32; + v[i..i + 4].copy_from_slice(&x.to_be_bytes()); + } + } + } + v +} + +/// Every fixture frame that decodes, with its decoded size. +fn seeds() -> Vec<(Vec, usize)> { + let dir = std::path::Path::new(env!("CARGO_MANIFEST_DIR")).join("tests/fixtures/blosc2"); + let mut v = Vec::new(); + for e in std::fs::read_dir(dir).unwrap() { + let p = e.unwrap().path(); + if p.extension().is_some_and(|x| x == "b2f") + && let Ok(out) = std::fs::read(p.with_extension("out")) + { + v.push((std::fs::read(&p).unwrap(), out.len())); + } + } + v.sort(); + assert!(v.len() >= 20, "fixtures missing"); + v +} + +fn header_len(frame: &[u8]) -> usize { + i32::from_be_bytes(frame[11..15].try_into().unwrap()) as usize +} + +/// Mutated fixture frames, decoded with their HDF5 chunk size as the +/// limit, and their first chunks on their own: whatever they declare, no +/// decode holds more than a small multiple of the output and the input. +#[test] +fn fuzzed_frames_and_chunks_stay_within_the_allocation_bound() { + let _g = lock(); + let seeds = seeds(); + let mut rng = Rng(0xb2a1); + let mut worst = (0.0f64, String::new()); + for i in 0..20_000 { + let (seed, limit) = &seeds[rng.below(seeds.len())]; + let f = mutate(&mut rng, seed, header_len(seed)); + let (r, peak) = peak_during(|| blosc2_decompress(&f, *limit)); + if let Ok(out) = &r { + assert!(out.len() <= *limit, "iteration {i}: output past the limit"); + } + assert!( + peak <= bound(*limit, &f), + "iteration {i}: peak {peak} bytes for a {limit}-byte chunk from {} bytes ({:?})", + f.len(), + r.map(|v| v.len()) + ); + let ratio = peak as f64 / bound(*limit, &f) as f64; + if ratio > worst.0 { + worst = ( + ratio, + format!( + "frame iteration {i}: peak {peak}, limit {limit}, input {}", + f.len() + ), + ); + } + } + for i in 0..20_000 { + let (seed, _) = &seeds[rng.below(seeds.len())]; + let at = header_len(seed); + let chunk = &seed[at..]; + let c = mutate(&mut rng, chunk, 0); + let limit = 1 << 16; + let (r, peak) = peak_during(|| blosc2_decompress_chunk(&c, limit)); + assert!( + peak <= bound(limit, &c), + "chunk iteration {i}: peak {peak} bytes from {} bytes ({:?})", + c.len(), + r.map(|v| v.len()) + ); + } + eprintln!("worst peak / bound: {:.2} ({})", worst.0, worst.1); +} + +/// Frames built from random header sizes, offsets chunks and B2ND shapes +/// (chunk and block shapes that pad, special and repeated-value chunks). +#[test] +fn random_frames_stay_within_the_allocation_bound() { + let _g = lock(); + let mut rng = Rng(0xb2a2); + for i in 0..5_000 { + let ts = [1usize, 2, 4, 8][rng.below(4)]; + let ndim = 1 + rng.below(8); + let mut shape = Vec::new(); + let mut chunks = Vec::new(); + let mut blocks = Vec::new(); + for _ in 0..ndim { + let s = 1 + rng.below(if ndim > 3 { 4 } else { 40 }); + let c = if rng.below(8) == 0 { + s * (1 + rng.below(4)) + } else { + 1 + rng.below(s) + }; + let b = 1 + rng.below(c); + shape.push(s as i64); + chunks.push(c as i32); + blocks.push(b as i32); + } + let items: usize = shape.iter().product::() as usize; + let limit = items * ts; + let meta = nd_meta(&shape, &chunks, &blocks); + let ext: usize = ts + * chunks + .iter() + .zip(&blocks) + .map(|(&c, &b)| (c as usize).div_ceil(b as usize) * b as usize) + .product::(); + let nchunks: usize = shape + .iter() + .zip(&chunks) + .map(|(&s, &c)| (s as usize).div_ceil(c as usize)) + .product(); + let block_bytes = ts * blocks.iter().product::() as usize; + let chunksize = if rng.below(4) == 0 { + rng.size() + } else { + ext as i64 + }; + let nbytes = if rng.below(4) == 0 { + rng.size() + } else { + (nchunks * ext) as i64 + }; + let off_n = if rng.below(4) == 0 { + rng.size() as i32 + } else { + 8 * nchunks as i32 + }; + let (data, off) = match rng.below(3) { + 0 => (Vec::new(), special_offset(1 + rng.below(2) as u8)), + _ => { + let bs = if rng.below(4) == 0 { + rng.size() as i32 + } else { + block_bytes as i32 + }; + let value: Vec = (0..ts).map(|_| rng.next() as u8).collect(); + let n = if rng.below(4) == 0 { + rng.size() as i32 + } else { + ext as i32 + }; + (repeated(&value, n, bs), 0i64.to_le_bytes()) + } + }; + let offsets = repeated(&off, off_n, off_n.clamp(1, 8)); + let meta = (rng.below(4) != 0).then_some(meta.as_slice()); + let f = frame(meta, nbytes, ts as i32, chunksize as i32, &data, &offsets); + let (r, peak) = peak_during(|| blosc2_decompress(&f, limit)); + assert!( + peak <= bound(limit, &f), + "iteration {i}: peak {peak} bytes for a {limit}-byte chunk ({:?}, shape {shape:?} \ + chunks {chunks:?} blocks {blocks:?})", + r.map(|v| v.len()) + ); + } +} From 742ed4dfb81a488f6269c7c81ddac8ba5e43efed Mon Sep 17 00:00:00 2001 From: osobh Date: Sat, 26 Sep 2026 11:35:13 -0500 Subject: [PATCH 32/43] fix(format): read a v1 chunk B-tree where libhdf5's lookup finds chunks libhdf5 does not walk the chunk B-tree to read a dataset: it looks each chunk up (H5B_find with H5D__btree_cmp3 and H5D__btree_found), asking for the element-size coordinate as 0. collect_chunk_info_checked now parses the tree with its keys and returns each stored chunk only when that lookup, replayed over the scaled keys, finds it. A key with a non-zero element-size coordinate is therefore found in a 1-D dataset (cmp3 compares only the first coordinate there, and found compares with <=) and missed in a dataset of rank 2 or more, which reads fill values. The previous commit refused every such key, which refused 1-D files libhdf5 reads correctly; before that, the rank-2 case read the chunk's data where h5py reads fill values (cve-2025-44905 /Shuffle_float_data_le, now identical to h5py, so it leaves the conformance report's list of libhdf5 bugs). Test: chunk_keys_with_an_element_offset_read_as_libhdf5_reads_them compares 1-D and 2-D files against h5py's values. It fails on the previous commit (the 1-D file is refused) and with the refusal removed (the 2-D file reads 0..23 where h5py reads fill values). Co-Authored-By: Claude Opus 5.5 (1M context) --- CHANGELOG.md | 11 +- conformance/report.py | 7 +- crates/clawhdf5-format/src/chunked_read.rs | 355 +++++++++++++----- .../tests/header_validation_interop.rs | 74 +++- 4 files changed, 322 insertions(+), 125 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c13a8da..8cc4248 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -40,8 +40,15 @@ ZFP filters; the 2 mismatches are the known h5py big-endian VL bug. - shuffle uses its own parameter as the element size, as libhdf5 does (`cve-2025-44905`); - an unfiltered chunk the index records at other than the chunk's size is - refused (it read with zeros for the missing bytes; `cve-2025-44904`), - as is a chunk B-tree key with a non-zero element offset. + refused (it read with zeros for the missing bytes; `cve-2025-44904`); + - a v1 B-tree chunk index is read as libhdf5 reads it: each chunk is + looked up the way `H5B_find` / `H5D__btree_cmp3` / `H5D__btree_found` + look it up, and a chunk that lookup does not find reads as fill values. + A key with a non-zero element-size coordinate is found in a 1-D dataset + and not in one of rank 2 or more (`cve-2025-44905` + `/Shuffle_float_data_le`, which read the chunk's data where h5py reads + fill values); an interim fix refused every such key, including 1-D + files libhdf5 reads correctly. - **Refused as libhdf5 refuses them:** a v1 group with an empty link name fails its listing (`FormatError::InvalidLinkName`; lookups still work, `cve-2021-46244`); dataspaces with more than 32 dimensions, a rank on a diff --git a/conformance/report.py b/conformance/report.py index 3331898..85ad0aa 100644 --- a/conformance/report.py +++ b/conformance/report.py @@ -100,8 +100,8 @@ def is_h5py_be_vlen(i): # Objects the reference (h5py 3.16 / HDF5 2.0) reads only because of an # HDF5 2.0 bug, and that clawhdf5 refuses: each one reads past a buffer or -# returns bytes the file does not hold, and libhdf5's develop branch refuses the -# first three. (file, object) -> why. Checked 2026-09-26 against HDF5 2.0.0 +# returns bytes the file does not hold, and libhdf5's develop branch refuses all +# three. (file, object) -> why. Checked 2026-09-26 against HDF5 2.0.0 # and HDFGroup/hdf5 develop sources; see docs/known-issues.md. LIBHDF5_BUGS = { ("cve_hdf5/cvefiles/cve-2025-2308.h5", "/Scale_offset_long_long_data_le"): @@ -114,9 +114,6 @@ LIBHDF5_BUGS = { ("hdf5/test/testfiles/bad_nbit_parms_walk.h5", "/Nbit_int_data_le"): "an N-Bit parameter list one value short: HDF5 2.0 reads past the list; libhdf5's own " "test (`test_filter_bad_params`, test/dsets.c) now requires the read to fail", - ("cve_hdf5/cvefiles/cve-2025-44905.h5", "/Shuffle_float_data_le"): - "a chunk B-tree key with element offset 4096: libhdf5's lookup misses the chunk and " - "returns fill values for data the file holds; clawhdf5 refuses the key", } diff --git a/crates/clawhdf5-format/src/chunked_read.rs b/crates/clawhdf5-format/src/chunked_read.rs index f9d6567..e943a03 100644 --- a/crates/clawhdf5-format/src/chunked_read.rs +++ b/crates/clawhdf5-format/src/chunked_read.rs @@ -316,26 +316,41 @@ pub fn collect_chunk_info( offset_size: u8, length_size: u8, ) -> Result, FormatError> { - collect_chunk_info_inner( + let _ = length_size; + let mut chunks = Vec::new(); + parse_chunk_node( file_data, btree_address, ndims, None, offset_size, - length_size, 0, - ) + &mut chunks, + )?; + Ok(chunks) } -/// [`collect_chunk_info`] for a layout with these `chunk_dimensions` (the -/// layout message's list, element size last), checking every key of the -/// B-tree as libhdf5 does (`H5D__btree_decode_key`): each coordinate offset -/// must be a multiple of its chunk dimension. That includes the keys that -/// only bound a node (internal-node keys and each node's final key), which -/// is where a corrupt chunk dimension shows when the chunks themselves all -/// start at offset 0 in that dimension (`cve-2018-11205`). A key that fails -/// ("bad coordinate offset") means a corrupt index or chunk dimension; the -/// chunks were read at the wrong place, or the dataset read as fill values. +/// The chunks of a v1 B-tree chunk index as libhdf5 reads them, for a +/// layout with these `chunk_dimensions` (the layout message's list, element +/// size last). +/// +/// Every key of the B-tree is checked as libhdf5 checks it +/// (`H5D__btree_decode_key`): each coordinate offset must be a multiple of +/// its chunk dimension. That includes the keys that only bound a node +/// (internal-node keys and each node's final key), which is where a +/// corrupt chunk dimension shows when the chunks themselves all start at +/// offset 0 in that dimension (`cve-2018-11205`). A key that fails ("bad +/// coordinate offset") means a corrupt index or chunk dimension. +/// +/// libhdf5 does not read a chunk by walking the tree: it looks each chunk +/// up (`H5B_find` with `H5D__btree_cmp3` and `H5D__btree_found`), comparing +/// the element-size coordinate too, which it asks for as 0. So a chunk is +/// returned only where that lookup finds it: a key whose element-size +/// coordinate is not 0 is found in a 1-D dataset (the comparison looks at +/// that coordinate only against the next key) but not in a dataset of rank +/// 2 or more (`cve-2025-44905` `/Shuffle_float_data_le`), which then reads +/// as fill values, and a tree whose keys are out of order loses the chunks +/// libhdf5's binary search misses. pub fn collect_chunk_info_checked( file_data: &[u8], btree_address: u64, @@ -343,15 +358,141 @@ pub fn collect_chunk_info_checked( offset_size: u8, length_size: u8, ) -> Result, FormatError> { - collect_chunk_info_inner( + let _ = length_size; + let ndims = chunk_dimensions.len(); + if ndims == 0 { + return Err(FormatError::ChunkedReadError( + "chunk layout has no dimensions".into(), + )); + } + let mut stored = Vec::new(); + let mut root = parse_chunk_node( file_data, btree_address, - chunk_dimensions.len(), + ndims, Some(chunk_dimensions), offset_size, - length_size, 0, - ) + &mut stored, + )?; + // Keys were checked to be multiples of non-zero dimensions. + root.scale_keys(chunk_dimensions); + // Look every stored chunk's position up. A lookup finds a chunk only at + // that chunk's own position, so each is returned at most once. + let mut returned = vec![false; stored.len()]; + let mut wanted = vec![0u64; ndims]; + for chunk in &stored { + for (w, (&o, &d)) in wanted + .iter_mut() + .zip(chunk.offsets.iter().zip(chunk_dimensions)) + { + *w = o / u64::from(d); + } + wanted[ndims - 1] = 0; + if let Some(i) = root.find(&wanted) { + returned[i] = true; + } + } + Ok(stored + .into_iter() + .zip(returned) + .filter_map(|(c, r)| r.then_some(c)) + .collect()) +} + +/// A node of a v1 B-tree chunk index: its `n + 1` keys (`ndims` +/// coordinates each, flattened; byte offsets as stored, or scaled by the +/// chunk dimensions after [`ChunkNode::scale_keys`]) and its `n` children, +/// a leaf's as indices into the list of stored chunks. +struct ChunkNode { + ndims: usize, + keys: Vec, + children: ChunkChildren, +} + +enum ChunkChildren { + Nodes(Vec), + Chunks(Vec), +} + +impl ChunkNode { + fn key(&self, i: usize) -> &[u64] { + &self.keys[i * self.ndims..(i + 1) * self.ndims] + } + + fn len(&self) -> usize { + match &self.children { + ChunkChildren::Nodes(n) => n.len(), + ChunkChildren::Chunks(c) => c.len(), + } + } + + fn scale_keys(&mut self, dims: &[u32]) { + for (k, &d) in self.keys.iter_mut().zip(dims.iter().cycle()) { + *k /= u64::from(d); + } + if let ChunkChildren::Nodes(nodes) = &mut self.children { + for n in nodes { + n.scale_keys(dims); + } + } + } + + /// `H5B_find_helper` over scaled keys: binary search for the child + /// whose keys bracket `scaled`, then `H5D__btree_found` at the leaf. + /// Returns the index of the chunk found. + fn find(&self, scaled: &[u64]) -> Option { + let (mut lt, mut rt) = (0, self.len()); + let mut idx = 0; + let mut cmp = core::cmp::Ordering::Greater; + while lt < rt && cmp != core::cmp::Ordering::Equal { + idx = (lt + rt) / 2; + cmp = btree_cmp3(self.key(idx), scaled, self.key(idx + 1)); + if cmp == core::cmp::Ordering::Less { + rt = idx; + } else { + lt = idx + 1; + } + } + if cmp != core::cmp::Ordering::Equal { + return None; + } + match &self.children { + ChunkChildren::Nodes(nodes) => nodes[idx].find(scaled), + ChunkChildren::Chunks(chunks) => { + // "Is this *really* the requested chunk?" + let lt_key = self.key(idx); + let found = scaled + .iter() + .zip(lt_key) + .all(|(&s, &k)| s < k.wrapping_add(1)); + found.then_some(chunks[idx]) + } + } + } +} + +/// `H5D__btree_cmp3`: where `scaled` falls against a child's left and +/// right keys. `Less` is left of the child, `Greater` right of it. With a +/// rank-1 dataset (two coordinates, element size last) libhdf5 compares +/// only the first coordinate, and the second against the right key. +fn btree_cmp3(lt: &[u64], scaled: &[u64], rt: &[u64]) -> core::cmp::Ordering { + use core::cmp::Ordering; + if scaled.len() == 2 { + if scaled[0] > rt[0] || (scaled[0] == rt[0] && scaled[1] >= rt[1]) { + Ordering::Greater + } else if scaled[0] < lt[0] { + Ordering::Less + } else { + Ordering::Equal + } + } else if scaled >= rt { + Ordering::Greater + } else if scaled < lt { + Ordering::Less + } else { + Ordering::Equal + } } /// Check one v1 B-tree chunk key's offsets (see @@ -368,24 +509,25 @@ fn check_key_offsets(offsets: &[u64], chunk_dimensions: &[u32]) -> Result<(), Fo } /// Read the `ndims` 8-byte offsets of the chunk key at `pos` (after its -/// chunk size and filter mask) and check them when `chunk_dimensions` is -/// given. +/// chunk size and filter mask) into `out`, checking them when +/// `chunk_dimensions` is given. fn read_key_offsets( file_data: &[u8], pos: usize, ndims: usize, chunk_dimensions: Option<&[u32]>, -) -> Result, FormatError> { - let mut offsets = Vec::with_capacity(ndims); + out: &mut Vec, +) -> Result<(), FormatError> { + let start = out.len(); let mut kp = pos + 8; for _ in 0..ndims { - offsets.push(read_offset(file_data, kp, CHUNK_KEY_OFFSET_SIZE)?); + out.push(read_offset(file_data, kp, CHUNK_KEY_OFFSET_SIZE)?); kp += CHUNK_KEY_OFFSET_SIZE as usize; } if let Some(dims) = chunk_dimensions { - check_key_offsets(&offsets, dims)?; + check_key_offsets(&out[start..], dims)?; } - Ok(offsets) + Ok(()) } /// Width of each chunk offset in a v1 chunk B-tree key, independent of the @@ -396,15 +538,17 @@ const CHUNK_KEY_OFFSET_SIZE: u8 = 8; /// protection), matching `btree_v1.rs`'s `MAX_BTREE_DEPTH`. const MAX_CHUNK_BTREE_DEPTH: usize = 64; -fn collect_chunk_info_inner( +/// Parse the v1 B-tree chunk index node at `btree_address` and its +/// subtree, appending its chunks to `stored` in tree order. +fn parse_chunk_node( file_data: &[u8], btree_address: u64, ndims: usize, chunk_dimensions: Option<&[u32]>, offset_size: u8, - _length_size: u8, depth: usize, -) -> Result, FormatError> { + stored: &mut Vec, +) -> Result { if depth > MAX_CHUNK_BTREE_DEPTH { return Err(FormatError::NestingDepthExceeded); } @@ -439,85 +583,68 @@ fn collect_chunk_info_inner( .and_then(|n| n.checked_add(8)) .ok_or_else(|| FormatError::ChunkedReadError("chunk key too large".into()))?; - if node_level == 0 { - // Leaf node: keys and children interleaved - // key[0], child[0], key[1], child[1], ..., key[N-1], child[N-1], key[N] - let needed = entries_used * (key_size + os) + key_size; - ensure_len(file_data, pos, needed)?; + // key[0], child[0], key[1], child[1], ..., key[N-1], child[N-1], key[N] + let needed = entries_used * (key_size + os) + key_size; + ensure_len(file_data, pos, needed)?; - let mut chunks = Vec::with_capacity(entries_used); - for _ in 0..entries_used { - // Parse key - let chunk_size = u32::from_le_bytes([ - file_data[pos], - file_data[pos + 1], - file_data[pos + 2], - file_data[pos + 3], - ]); - let filter_mask = u32::from_le_bytes([ - file_data[pos + 4], - file_data[pos + 5], - file_data[pos + 6], - file_data[pos + 7], - ]); - let offsets = read_key_offsets(file_data, pos, ndims, chunk_dimensions)?; - // A chunk's key carries 0 in the element-size dimension. libhdf5 - // compares that coordinate too when it looks a chunk up - // (`H5D__btree_found`), so whether it finds a chunk keyed - // otherwise depends on where the key falls; in `cve-2025-44905` - // `/Shuffle_float_data_le` (offset 4096) it does not, and h5py - // reads fill values there. Such a key is refused here. - if chunk_dimensions.is_some() && offsets.last().is_some_and(|&o| o != 0) { - return Err(FormatError::ChunkedReadError(format!( - "chunk key {offsets:?} has a non-zero element offset" - ))); - } - pos += key_size; - - // Parse child address - let address = read_offset(file_data, pos, offset_size)?; - pos += os; - - chunks.push(ChunkInfo { + let mut keys = Vec::with_capacity((entries_used + 1) * ndims); + let mut chunks = Vec::new(); + let mut child_addrs = Vec::new(); + for _ in 0..entries_used { + let chunk_size = u32::from_le_bytes([ + file_data[pos], + file_data[pos + 1], + file_data[pos + 2], + file_data[pos + 3], + ]); + let filter_mask = u32::from_le_bytes([ + file_data[pos + 4], + file_data[pos + 5], + file_data[pos + 6], + file_data[pos + 7], + ]); + let k = keys.len(); + read_key_offsets(file_data, pos, ndims, chunk_dimensions, &mut keys)?; + pos += key_size; + let address = read_offset(file_data, pos, offset_size)?; + pos += os; + if node_level == 0 { + chunks.push(stored.len()); + stored.push(ChunkInfo { chunk_size, filter_mask, - offsets, + offsets: keys[k..].to_vec(), address, }); + } else { + child_addrs.push(address); } - // The final key only bounds the node; libhdf5 still checks it. - read_key_offsets(file_data, pos, ndims, chunk_dimensions)?; - Ok(chunks) + } + // The final key only bounds the node; libhdf5 still checks it. + read_key_offsets(file_data, pos, ndims, chunk_dimensions, &mut keys)?; + + let children = if node_level == 0 { + ChunkChildren::Chunks(chunks) } else { - // Internal node: recurse into children - let needed = entries_used * (key_size + os) + key_size; - ensure_len(file_data, pos, needed)?; - - let mut child_addrs = Vec::with_capacity(entries_used); - for _ in 0..entries_used { - read_key_offsets(file_data, pos, ndims, chunk_dimensions)?; - pos += key_size; - let child_addr = read_offset(file_data, pos, offset_size)?; - child_addrs.push(child_addr); - pos += os; - } - read_key_offsets(file_data, pos, ndims, chunk_dimensions)?; - - let mut all_chunks = Vec::new(); + let mut nodes = Vec::with_capacity(child_addrs.len()); for child_addr in child_addrs { - let child_chunks = collect_chunk_info_inner( + nodes.push(parse_chunk_node( file_data, child_addr, ndims, chunk_dimensions, offset_size, - _length_size, depth + 1, - )?; - all_chunks.extend(child_chunks); + stored, + )?); } - Ok(all_chunks) - } + ChunkChildren::Nodes(nodes) + }; + Ok(ChunkNode { + ndims, + keys, + children, + }) } /// Generate ChunkInfo entries for an implicit index (v4 index type 2). @@ -1754,6 +1881,25 @@ mod tests { /// Build a B-tree v1 type 1 leaf node with given chunk infos. fn build_chunk_btree_leaf(chunks: &[ChunkInfo], ndims: usize, offset_size: u8) -> Vec { + build_chunk_btree_leaf_to(chunks, &vec![0; ndims], offset_size) + } + + /// A leaf whose final key is the one libhdf5 writes past the last chunk: + /// each coordinate of the last chunk plus its chunk dimension (the + /// element size last). + fn build_chunk_btree_leaf_dims(chunks: &[ChunkInfo], dims: &[u32], offset_size: u8) -> Vec { + let last = &chunks.last().expect("a chunk").offsets; + let end: Vec = dims + .iter() + .enumerate() + .map(|(d, &c)| last.get(d).copied().unwrap_or(0) + u64::from(c)) + .collect(); + build_chunk_btree_leaf_to(chunks, &end, offset_size) + } + + /// A leaf holding `chunks`, with final key `end`. + fn build_chunk_btree_leaf_to(chunks: &[ChunkInfo], end: &[u64], offset_size: u8) -> Vec { + let ndims = end.len(); let _os = offset_size as usize; let entries_used = chunks.len() as u16; let mut buf = Vec::new(); @@ -1795,8 +1941,8 @@ mod tests { // checks; 0 always is) buf.extend_from_slice(&0u32.to_le_bytes()); // chunk_size buf.extend_from_slice(&0u32.to_le_bytes()); // filter_mask - for _ in 0..ndims { - write_offset(&mut buf, 0, 8); + for &e in end { + write_offset(&mut buf, e, 8); } buf @@ -1812,8 +1958,11 @@ mod tests { offsets, address, }; - let good = - build_chunk_btree_leaf(&[chunk(vec![0, 0], 0x100), chunk(vec![10, 0], 0x200)], 2, 8); + let good = build_chunk_btree_leaf_dims( + &[chunk(vec![0, 0], 0x100), chunk(vec![10, 0], 0x200)], + &[10, 8], + 8, + ); assert_eq!( collect_chunk_info_checked(&good, 0, &[10, 8], 8, 8) .unwrap() @@ -2044,7 +2193,6 @@ mod tests { ) -> (Vec, DataLayout, Dataspace) { let os: u8 = 8; let elem_size = 8usize; - let ndims = 2; // rank(1) + 1 let total = values.len(); // Place chunk data starting at offset 0x2000 @@ -2075,12 +2223,13 @@ mod tests { } // Build B-tree at offset 0x100 - let btree = build_chunk_btree_leaf(&chunk_infos, ndims, os); + let dims = [chunk_size_elems as u32, elem_size as u32]; + let btree = build_chunk_btree_leaf_dims(&chunk_infos, &dims, os); let btree_addr = 0x100usize; file_data[btree_addr..btree_addr + btree.len()].copy_from_slice(&btree); let layout = DataLayout::Chunked { - chunk_dimensions: vec![chunk_size_elems as u32, elem_size as u32], + chunk_dimensions: dims.to_vec(), btree_address: Some(btree_addr as u64), version: 3, chunk_index_type: None, @@ -2218,7 +2367,6 @@ mod tests { let os: u8 = 8; let elem_size = 8usize; - let ndims = 2; let chunk_elems = 10usize; let total = 20usize; @@ -2257,12 +2405,13 @@ mod tests { data_offset += compressed.len() + 16; // some padding } - let btree = build_chunk_btree_leaf(&chunk_infos, ndims, os); + let dims = [chunk_elems as u32, elem_size as u32]; + let btree = build_chunk_btree_leaf_dims(&chunk_infos, &dims, os); let btree_addr = 0x100usize; file_data[btree_addr..btree_addr + btree.len()].copy_from_slice(&btree); let layout = DataLayout::Chunked { - chunk_dimensions: vec![chunk_elems as u32, elem_size as u32], + chunk_dimensions: dims.to_vec(), btree_address: Some(btree_addr as u64), version: 3, chunk_index_type: None, @@ -2300,7 +2449,6 @@ mod tests { // 4x6 dataset with chunk size 2x3 => 4 chunks let os: u8 = 8; let elem_size = 4usize; // f32 - let ndims = 3; // rank(2) + 1 let ds_dims = [4usize, 6]; let chunk_dims = [2usize, 3]; @@ -2340,12 +2488,13 @@ mod tests { } } - let btree = build_chunk_btree_leaf(&chunk_infos, ndims, os); + let dims = [chunk_dims[0] as u32, chunk_dims[1] as u32, elem_size as u32]; + let btree = build_chunk_btree_leaf_dims(&chunk_infos, &dims, os); let btree_addr = 0x100usize; file_data[btree_addr..btree_addr + btree.len()].copy_from_slice(&btree); let layout = DataLayout::Chunked { - chunk_dimensions: vec![chunk_dims[0] as u32, chunk_dims[1] as u32, elem_size as u32], + chunk_dimensions: dims.to_vec(), btree_address: Some(btree_addr as u64), version: 3, chunk_index_type: None, diff --git a/crates/clawhdf5/tests/header_validation_interop.rs b/crates/clawhdf5/tests/header_validation_interop.rs index 25614c5..d0a1273 100644 --- a/crates/clawhdf5/tests/header_validation_interop.rs +++ b/crates/clawhdf5/tests/header_validation_interop.rs @@ -629,17 +629,13 @@ save("mdc_past_eof", bad) ); } -/// Chunk index entries HDF5 2.0 mis-reads, refused here. An unfiltered -/// chunk the index records at less than the chunk's size (`cve-2025-44904`): -/// HDF5 2.0 fills the rest of the chunk with whatever its buffer held, and -/// later libhdf5 releases refuse it ("incorrect chunk size returned from -/// index for unfiltered chunk"); we read the rest as zeros. A chunk keyed -/// with a non-zero element offset (`cve-2025-44905`): libhdf5's lookup -/// compares that coordinate too, so whether it finds the chunk depends on -/// where the key falls (in `cve-2025-44905` it does not, and h5py reads -/// fill values; in this file it does); we read the chunk. +/// An unfiltered chunk the index records at less than the chunk's size +/// (`cve-2025-44904`): HDF5 2.0 fills the rest of the chunk with whatever +/// its buffer held, and later libhdf5 releases refuse it ("incorrect chunk +/// size returned from index for unfiltered chunk"); we used to read the +/// rest as zeros, and now refuse it. #[test] -fn chunk_index_entries_libhdf5_misreads_are_refused() { +fn unfiltered_chunk_of_the_wrong_size_is_refused() { skip_if_no_python!(); let dir = tempfile::tempdir().unwrap(); run_python( @@ -658,8 +654,6 @@ second = key + 24 + 8 # a key (4 + 4 + 2 x 8 bytes), then a child address assert struct.unpack_from(" 0 and data[tree + 5] == 0 + ndims = len(shape) + 1 + key_size = 8 + 8 * ndims + key = tree + 8 + 16 + which * (key_size + 8) + last = key + 8 + 8 * (ndims - 1) + assert struct.unpack_from(" = std::fs::read_to_string(dir.path().join(format!("{name}.h5.txt"))) + .unwrap() + .split_whitespace() + .map(|v| v.parse().unwrap()) + .collect(); + let file = File::open(&path).unwrap(); + let got = file.dataset("d").unwrap().read_i32(); + assert_eq!(got.as_deref().ok(), Some(&expected[..]), "{name}: {got:?}"); } } From 7515e5dcbdd13c3399cf79f3f8371e6a92adb1e2 Mon Sep 17 00:00:00 2001 From: osobh Date: Sat, 26 Sep 2026 11:35:19 -0500 Subject: [PATCH 33/43] docs: CHANGELOG for the Blosc2 allocation bounds Co-Authored-By: Claude Opus 5.5 (1M context) --- CHANGELOG.md | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index aebeb93..51e3040 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -21,6 +21,21 @@ python-blosc2 4.13.1 for what hdf5plugin never writes (`crates/clawhdf5-format/tests/fixtures/blosc2/`); the decoder is fuzzed. Conformance: 576 of 697 files ok (was 575) — h5ex_d_blosc2. +- **A crafted Blosc2 chunk cannot allocate more than a few times its HDF5 + chunk size.** Found in review before release: the sizes a frame declares + sized the decoder's buffers. A 173-byte frame whose offsets chunk claimed + 2 GiB was decoded in full for a 1 MiB chunk; an empty chunk allocated its + declared block size twice (about 1 GiB); a B2ND chunk was decoded whole + with its padding (up to 16x the chunk); and a Zstandard stream's declared + window (up to 100 MiB) was reserved as the decoder was reused, which also + affected Blosc 1 and bitshuffle with Zstandard. Now the offsets chunk is + capped at the chunk size, block sizes are clamped to the chunk, B2ND + blocks are placed as they are decoded (padding is never held), and the + Zstandard window is capped at twice the stream's output (at least + 128 KiB). A B2ND chunk may no longer be larger than its array, which + hdf5-blosc2 never writes. `tests/blosc2_alloc_bounds.rs` measures peak + allocation for these frames and for 45,000 fuzzed ones: at most 6x the + chunk size, twice the input and 2 MiB of Zstandard state. ### Concurrent reads (2026-09-26) - **Full reads of chunked datasets scale with threads again when rayon's From 3da118d2ee953ed12b17db4e84c4a45693ad8eff Mon Sep 17 00:00:00 2001 From: osobh Date: Sat, 26 Sep 2026 11:36:00 -0500 Subject: [PATCH 34/43] style(format): iterate the chunk index in BlockGeometry::place clippy's needless_range_loop, missed before the B2ND streaming commit. Co-Authored-By: Claude Opus 5.5 (1M context) --- crates/clawhdf5-format/src/filters_blosc2.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/crates/clawhdf5-format/src/filters_blosc2.rs b/crates/clawhdf5-format/src/filters_blosc2.rs index a70a906..f4d1690 100644 --- a/crates/clawhdf5-format/src/filters_blosc2.rs +++ b/crates/clawhdf5-format/src/filters_blosc2.rs @@ -1014,9 +1014,9 @@ impl BlockGeometry<'_> { let nd = self.nd; unravel(b, self.blocks_in_chunk, &mut p.bidx); let mut empty = false; - for i in 0..nd.shape.len() { + for (i, &c) in cidx.iter().enumerate() { let in_chunk = p.bidx[i] * nd.blockshape[i]; - p.gstart[i] = cidx[i] * nd.chunkshape[i] + in_chunk; + p.gstart[i] = c * nd.chunkshape[i] + in_chunk; let lim_chunk = nd.chunkshape[i].saturating_sub(in_chunk); let lim_shape = nd.shape[i].saturating_sub(p.gstart[i]); p.valid[i] = nd.blockshape[i].min(lim_chunk).min(lim_shape); From a6ed3a5c7d287e444358d307bc8dc545c6aca448 Mon Sep 17 00:00:00 2001 From: osobh Date: Sat, 26 Sep 2026 11:36:10 -0500 Subject: [PATCH 35/43] fix(format): resolve cache-image flush-dependency parents as libhdf5 does The review suggested libhdf5 loads every image entry and resolves flush-dependency parents afterwards. It does not: H5C__reconstruct_cache_contents (HDF5 1.14.6 and 2.0.0, and develop) inserts each entry and then searches the cache index for its parents in the same loop, failing with "fd parent not in cache?!?" when one is missing. So a parent must be an earlier image entry, as before, or metadata cached before the image loads: the superblock (address 0) and the superblock extension's object header, which libhdf5 reads to find the image. Those two were refused as parents; they are now accepted. A parent listed after its child is still refused, as libhdf5 refuses it, and so is an entry that is its own parent ("Child entry flush dependency parent can't be itself"). apply_cache_image takes the superblock to know the extension address. Test: superblock_ext::tests::flush_dependency_parents_must_already_be_cached (parent-first loads, child-first refused, extension header accepted, self-parent refused); the extension-header case fails without the fix. Co-Authored-By: Claude Opus 5.5 (1M context) --- conformance/probe/src/main.rs | 2 +- crates/clawhdf5-format/src/superblock_ext.rs | 91 +++++++++++++++++--- 2 files changed, 81 insertions(+), 12 deletions(-) diff --git a/conformance/probe/src/main.rs b/conformance/probe/src/main.rs index 9fa9929..fe619a0 100644 --- a/conformance/probe/src/main.rs +++ b/conformance/probe/src/main.rs @@ -731,7 +731,7 @@ fn main() { let view = match ext.and_then(|x| x.cache_image) { None => None, Some(loc) => match guarded(|| { - superblock_ext::apply_cache_image(hdf5, loc, sb.offset_size, sb.length_size) + superblock_ext::apply_cache_image(hdf5, loc, &sb) .map_err(e) }) { Ok(v) => Some(v), diff --git a/crates/clawhdf5-format/src/superblock_ext.rs b/crates/clawhdf5-format/src/superblock_ext.rs index d7ec3cc..2db3d8b 100644 --- a/crates/clawhdf5-format/src/superblock_ext.rs +++ b/crates/clawhdf5-format/src/superblock_ext.rs @@ -297,16 +297,16 @@ struct ImageEntry { /// (`H5C__decode_cache_image_header`, `H5C__reconstruct_cache_entry`): /// signature and version, the image length it records, entry types, rings /// and ages in range, entry addresses inside the file and not repeated, -/// flush-dependency parents that are earlier entries. +/// flush-dependency parents already in the cache. /// /// libhdf5 does not verify the block's trailing checksum when it loads an /// image, so neither does this. pub fn apply_cache_image( data: &[u8], location: CacheImageLocation, - offset_size: u8, - length_size: u8, + sb: &Superblock, ) -> Result, FormatError> { + let (offset_size, length_size) = (sb.offset_size, sb.length_size); let bad = FormatError::InvalidCacheImage; let start = usize::try_from(location.address).map_err(|_| bad("address out of range"))?; let len = usize::try_from(location.length).map_err(|_| bad("length out of range"))?; @@ -336,6 +336,17 @@ pub fn apply_cache_image( } let mut entries = Vec::new(); + // What is in libhdf5's cache when it loads the image: the superblock + // and the superblock extension's object header (read to find the image). + // Each entry's flush-dependency parents are looked up in the cache as + // the entry is inserted (`H5C__reconstruct_cache_contents` searches the + // index inside the loop that inserts the entries, in HDF5 1.14.6 and + // 2.0.0 alike), so a parent must be one of those or an earlier entry. + let mut cached = BTreeSet::new(); + cached.insert(0); + if let Some(ext) = sb.superblock_extension_address { + cached.insert(ext); + } let mut seen = BTreeSet::new(); for _ in 0..n_entries { let type_id = c.u8()?; @@ -374,8 +385,8 @@ pub fn apply_cache_image( let parent = c .addr(offset_size)? .ok_or(bad("invalid flush dependency parent offset"))?; - if !seen.contains(&parent) { - return Err(bad("flush dependency parent not in the image")); + if !seen.contains(&parent) && !cached.contains(&parent) { + return Err(bad("fd parent not in cache")); } } let len = usize::try_from(size).map_err(|_| bad(RAN_OFF))?; @@ -415,7 +426,7 @@ pub fn metadata_view(data: &[u8], sb: &Superblock) -> Result>, Fo Some(SuperblockExtension { cache_image: Some(location), .. - }) => apply_cache_image(data, location, sb.offset_size, sb.length_size).map(Some), + }) => apply_cache_image(data, location, sb).map(Some), _ => Ok(None), } } @@ -562,18 +573,37 @@ mod tests { /// A cache image block with `entries` of (address, bytes). fn image(entries: &[(u64, &[u8])]) -> Vec { + let with_deps: Vec<_> = entries.iter().map(|&(a, b)| (a, b, 0, None)).collect(); + image_with_deps(&with_deps) + } + + /// A cache image block with `entries` of (address, bytes, flush + /// dependency children, flush dependency parent). + fn image_with_deps(entries: &[(u64, &[u8], u16, Option)]) -> Vec { let mut b = Vec::new(); b.extend_from_slice(MDCI_SIGNATURE); b.push(0); b.push(0); b.extend_from_slice(&0u64.to_le_bytes()); // length, patched below b.extend_from_slice(&(entries.len() as u32).to_le_bytes()); - for (addr, bytes) in entries { - b.extend_from_slice(&[5, 0x02, 1, 0]); // type, flags (in LRU), ring, age - b.extend_from_slice(&[0; 6]); // children, dirty children, parents + for &(addr, bytes, children, parent) in entries { + let mut flags = 0x02; // in LRU + if children > 0 { + flags |= MDCI_ENTRY_IS_FD_PARENT; + } + if parent.is_some() { + flags |= MDCI_ENTRY_IS_FD_CHILD; + } + b.extend_from_slice(&[5, flags, 1, 0]); // type, flags, ring, age + b.extend_from_slice(&children.to_le_bytes()); + b.extend_from_slice(&0u16.to_le_bytes()); // dirty children + b.extend_from_slice(&u16::from(parent.is_some()).to_le_bytes()); b.extend_from_slice(&0i32.to_le_bytes()); b.extend_from_slice(&addr.to_le_bytes()); b.extend_from_slice(&(bytes.len() as u64).to_le_bytes()); + if let Some(p) = parent { + b.extend_from_slice(&p.to_le_bytes()); + } b.extend_from_slice(bytes); } b.extend_from_slice(&[0; 4]); // checksum (not verified, as in libhdf5) @@ -592,7 +622,7 @@ mod tests { address: at, length: img.len() as u64, }; - let out = apply_cache_image(&f, loc, 8, 8).unwrap(); + let out = apply_cache_image(&f, loc, &sb_v2(u64::MAX)).unwrap(); assert_eq!(out.len(), f.len()); assert_eq!(&out[16..22], b"HEADER"); assert_eq!(&out[40..44], b"NODE"); @@ -605,7 +635,7 @@ mod tests { address: 64, length: img.len() as u64, }; - apply_cache_image(&f, loc, 8, 8).unwrap_err() + apply_cache_image(&f, loc, &sb_v2(u64::MAX)).unwrap_err() }; let mut sig = image(&[(16, b"x")]); sig[0] = b'X'; @@ -627,4 +657,43 @@ mod tests { cut[6..14].copy_from_slice(&n.to_le_bytes()); assert!(matches!(bad(cut), FormatError::InvalidCacheImage(_))); } + + /// libhdf5 resolves an entry's flush-dependency parents as it inserts + /// the entry (`H5C__reconstruct_cache_contents`): a parent must be an + /// earlier entry, or the superblock or its extension's object header, + /// which are cached before the image loads. A parent listed after its + /// child fails ("fd parent not in cache?!?"). + #[test] + fn flush_dependency_parents_must_already_be_cached() { + let load = |img: Vec| { + let mut f = vec![0u8; 64]; + f.extend_from_slice(&img); + let loc = CacheImageLocation { + address: 64, + length: img.len() as u64, + }; + apply_cache_image(&f, loc, &sb_v2(48)) + }; + // Parent first, as libhdf5 writes images. + assert!( + load(image_with_deps(&[ + (16, b"P", 1, None), + (40, b"C", 0, Some(16)) + ])) + .is_ok() + ); + // Child first: libhdf5 does not find the parent. + assert_eq!( + load(image_with_deps(&[ + (40, b"C", 0, Some(16)), + (16, b"P", 1, None) + ])) + .unwrap_err(), + FormatError::InvalidCacheImage("fd parent not in cache") + ); + // The superblock extension's header (at 48 here) is in the cache. + assert!(load(image_with_deps(&[(40, b"C", 0, Some(48))])).is_ok()); + // An entry cannot be its own parent. + assert!(load(image_with_deps(&[(40, b"C", 1, Some(40))])).is_err()); + } } From 60502593b74b9ff62ccbc8376bc1aed7eaad24d3 Mon Sep 17 00:00:00 2001 From: osobh Date: Sat, 26 Sep 2026 11:42:43 -0500 Subject: [PATCH 36/43] fix: apply a metadata cache image without copying the file apply_cache_image returned a copy of the whole file with the image's entries written in, and File (mmap by default), MmapFile and LazyFile used that copy for every read: opening a 1 GiB sparse file with an image needed 2 GB of memory, and an 8 GiB one aborted the process, where de2a53f (which ignored the image) opened them in a few MB. The metadata parsers read one contiguous slice, so the image still has to be laid over the file's bytes; it is now laid over a private copy that costs only the pages it touches: - clawhdf5_format::superblock_ext::CacheImage decodes the image into an entry list (address, offset in the block, length) and applies it to any destination; cache_image_state tells an opener whether the file has no image, a loadable one, or one libhdf5 cannot load; apply_cache_image_in_place is for readers that own their buffer. apply_cache_image and metadata_view (which copied) are gone. - clawhdf5_io::HDF5Read::private_copy returns a writable private copy of a reader's bytes: MmapReader gives a MAP_PRIVATE copy-on-write mapping (memmap2 map_copy), so only the pages the entries land on are copied; the default copies the bytes (in-memory readers). - File, MmapFile and LazyFile write the image into that mapping (crate::cache_image). File::from_bytes / open_buffered patch their own buffer in place, copying only the image block, as libhdf5 does. A file without an image is read straight from the mapping, unchanged. An image entry that runs past the end of file is now refused: libhdf5 checks only that it starts inside the file, and the images libhdf5 writes never do this, but those bytes have nowhere to go in a view of the file. Tests: tests/cache_image_memory.rs has libhdf5 (through ctypes) add an image to a 1 GiB sparse file and bounds resident-memory growth for all three openers at 256 MiB; it fails on the previous commit (File::open grew 2,148,720,640 bytes). reader.rs zero_copy_tests check that a file without an image is read from the mapping itself and that an image goes into a copy-on-write mapping, not a heap copy; clawhdf5-io checks that private_copy writes never reach the reader or the file. Co-Authored-By: Claude Opus 5.5 (1M context) --- CHANGELOG.md | 16 +- conformance/probe/src/main.rs | 30 +- crates/clawhdf5-format/src/superblock_ext.rs | 369 ++++++++++++------- crates/clawhdf5-io/src/lib.rs | 59 +++ crates/clawhdf5-io/src/mmap.rs | 29 ++ crates/clawhdf5-io/src/prefetch.rs | 4 + crates/clawhdf5/src/cache_image.rs | 85 +++++ crates/clawhdf5/src/lazy.rs | 34 +- crates/clawhdf5/src/lib.rs | 1 + crates/clawhdf5/src/mmap_file.rs | 27 +- crates/clawhdf5/src/reader.rs | 78 +++- crates/clawhdf5/tests/cache_image_memory.rs | 139 +++++++ 12 files changed, 695 insertions(+), 176 deletions(-) create mode 100644 crates/clawhdf5/src/cache_image.rs create mode 100644 crates/clawhdf5/tests/cache_image_memory.rs diff --git a/CHANGELOG.md b/CHANGELOG.md index 8cc4248..3b0efd6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,9 +13,19 @@ ZFP filters; the 2 mismatches are the known h5py big-endian VL bug. bytes; in `h5clear_mdc_image.h5` the root group exists only there, and every reader failed with `InvalidObjectHeaderVersion(0)`. `File`, `MmapFile` and `LazyFile` (and `h5rs`) now apply the image at open - (`clawhdf5_format::superblock_ext`), with libhdf5's checks. A file whose - image libhdf5 cannot load opens in libhdf5 but nothing in it can be read; - `File::open` refuses it. + (`clawhdf5_format::superblock_ext::CacheImage`), with libhdf5's checks. + The file is not copied to do it: a mapped file gets the image's entries + written into a private copy-on-write mapping + (`clawhdf5_io::HDF5Read::private_copy`, `MAP_PRIVATE`), so only the pages + they land on are copied, and a buffer the opener owns (`File::from_bytes`, + `open_buffered`) is patched in place; files without an image are read + from the mapping exactly as before. (An interim version copied the whole + file onto the heap: 2 GB of memory to open a 1 GiB sparse file with an + image, and an abort for an 8 GiB one; `tests/cache_image_memory.rs` + guards it.) An image entry that runs past the end of file is refused + (libhdf5 checks only its start; the images it writes never do this). A + file whose image libhdf5 cannot load opens in libhdf5 but nothing in it + can be read; `File::open` refuses it. - **The superblock extension is decoded at open, as libhdf5 does:** a File Space Info or Metadata Cache Image message libhdf5 cannot decode makes the open fail (`cve-2020-10810`, `cve-2020-10812` were opened). diff --git a/conformance/probe/src/main.rs b/conformance/probe/src/main.rs index fe619a0..8c25a88 100644 --- a/conformance/probe/src/main.rs +++ b/conformance/probe/src/main.rs @@ -718,8 +718,8 @@ fn main() { // root group — so a file whose image it cannot load still opens and // every object fails; File::open refuses such a file outright. The // probe records the image's error where libhdf5 reports it. - use clawhdf5_format::superblock_ext; - let ext = match guarded(|| superblock_ext::read_superblock_extension(hdf5, &sb).map_err(e)) { + use clawhdf5_format::superblock_ext::{self, CacheImageState}; + let state = match guarded(|| superblock_ext::cache_image_state(hdf5, &sb).map_err(e)) { Ok(x) => x, Err(msg) => { top.insert("open_error".into(), Value::String(msg)); @@ -728,18 +728,22 @@ fn main() { } }; let mut image_error = None; - let view = match ext.and_then(|x| x.cache_image) { - None => None, - Some(loc) => match guarded(|| { - superblock_ext::apply_cache_image(hdf5, loc, &sb) - .map_err(e) - }) { - Ok(v) => Some(v), - Err(msg) => { - image_error = Some(msg); - None + let view = match state { + CacheImageState::Absent => None, + CacheImageState::Unloadable(err) => { + image_error = Some(e(err)); + None + } + CacheImageState::Loaded(image) => { + let mut v = hdf5.to_vec(); + match image.block(hdf5).and_then(|b| image.apply(b, &mut v)) { + Ok(()) => Some(v), + Err(err) => { + image_error = Some(e(err)); + None + } } - }, + } }; let hdf5: &[u8] = view.as_deref().unwrap_or(hdf5); top.insert("superblock_version".into(), json!(sb.version)); diff --git a/crates/clawhdf5-format/src/superblock_ext.rs b/crates/clawhdf5-format/src/superblock_ext.rs index 2db3d8b..e1d1cf3 100644 --- a/crates/clawhdf5-format/src/superblock_ext.rs +++ b/crates/clawhdf5-format/src/superblock_ext.rs @@ -14,9 +14,11 @@ //! `H5C__reconstruct_cache_contents`), and the entries take the place of //! the file's bytes at their addresses: the file itself may hold stale or //! no metadata there (in `h5clear_mdc_image.h5` the root group's header is -//! only in the image). [`apply_cache_image`] does the same with bytes: it -//! returns a copy of the file with every entry written at its address, so -//! every parser reads what libhdf5 reads. +//! only in the image). [`CacheImage::apply`] does the same with bytes: it +//! writes every entry at its address, so every parser reads what libhdf5 +//! reads. It writes into whatever the opener gives it — a private +//! copy-on-write mapping of the file, or a buffer the opener owns — so the +//! file is never copied whole. #[cfg(not(feature = "std"))] use alloc::{collections::BTreeSet, vec::Vec}; @@ -284,150 +286,243 @@ fn decode_fsinfo(data: &[u8], os: u8, ls: u8) -> Result Result, FormatError> { - let (offset_size, length_size) = (sb.offset_size, sb.length_size); + entries: Vec, +} + +/// What an opener must do about a file's metadata cache image. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum CacheImageState { + /// The file has no image: its bytes are its metadata. + Absent, + /// The file has an image that loads: apply it with [`CacheImage::apply`]. + Loaded(CacheImage), + /// The file has an image libhdf5 fails to load. libhdf5 still opens the + /// file (the image loads at the first metadata read), and that read + /// fails with this error. + Unloadable(FormatError), +} + +impl CacheImage { + /// Decode the metadata cache image at `location` in `data` (the file + /// from the superblock on, up to its recorded end of file). The image is + /// checked as libhdf5 checks it (`H5C__decode_cache_image_header`, + /// `H5C__reconstruct_cache_entry`): signature and version, the image + /// length it records, entry types, rings and ages in range, entry + /// addresses inside the file and not repeated, flush-dependency parents + /// already in the cache. + /// + /// One check is stricter than libhdf5's: an entry must end inside the + /// file. libhdf5 checks only that it starts there, and serves the rest + /// from the image; the images libhdf5 writes never do this (every entry + /// lies below the image block, which is written last), and the bytes an + /// entry would put past the end of file have nowhere to go in a view of + /// the file. + /// + /// libhdf5 does not verify the block's trailing checksum when it loads + /// an image, so neither does this. + pub fn decode( + data: &[u8], + location: CacheImageLocation, + sb: &Superblock, + ) -> Result { + let (offset_size, length_size) = (sb.offset_size, sb.length_size); + let bad = FormatError::InvalidCacheImage; + let block = image_block(data, location)?; + let eoa = data.len() as u64; + let mut c = Cursor::new(block, bad(RAN_OFF)); + + // Header: signature, version, flags, image data length, entry count. + if c.take(4)? != MDCI_SIGNATURE { + return Err(bad("bad metadata cache image header signature")); + } + if c.u8()? != 0 { + return Err(bad("bad metadata cache image version")); + } + if c.u8()? & MDCI_HAVE_RESIZE_STATUS != 0 { + return Err(bad("MDC resize status not yet supported")); + } + if c.uint(length_size)? != location.length { + return Err(bad("bad metadata cache image data length")); + } + let n_entries = c.uint(4)?; + if n_entries == 0 { + return Err(bad("bad metadata cache entry count")); + } + + let mut entries = Vec::new(); + // What is in libhdf5's cache when it loads the image: the superblock + // and the superblock extension's object header (read to find the + // image). Each entry's flush-dependency parents are looked up in the + // cache as the entry is inserted (`H5C__reconstruct_cache_contents` + // searches the index inside the loop that inserts the entries, in + // HDF5 1.14.6 and 2.0.0 alike), so a parent must be one of those or + // an earlier entry. + let mut cached = BTreeSet::new(); + cached.insert(0); + if let Some(ext) = sb.superblock_extension_address { + cached.insert(ext); + } + let mut seen = BTreeSet::new(); + for _ in 0..n_entries { + let type_id = c.u8()?; + if type_id >= MDCI_NTYPES { + return Err(bad("type id is out of valid range")); + } + let flags = c.u8()?; + if c.u8()? >= MDCI_RING_NTYPES { + return Err(bad("ring is out of valid range")); + } + if c.u8()? > MDCI_AGE_MAX { + return Err(bad("entry age is out of policy range")); + } + let children = c.uint(2)?; + // libhdf5 checks the parent flag against the child count only in + // debug builds (release builds refuse any entry with children); + // the image format's own rule is checked here. + if (flags & MDCI_ENTRY_IS_FD_PARENT != 0) != (children > 0) { + return Err(bad("flush dependency parent flag and child count disagree")); + } + c.uint(2)?; // dirty dependency children: reset for a read-only open + let parents = c.uint(2)?; + if (flags & MDCI_ENTRY_IS_FD_CHILD != 0) != (parents > 0) { + return Err(bad("flush dependency child flag and parent count disagree")); + } + c.uint(4)?; // LRU rank + let address = c + .addr(offset_size)? + .filter(|&a| a < eoa) + .ok_or(bad("invalid entry address range"))?; + let size = c.uint(length_size)?; + if size == 0 { + return Err(bad("invalid entry size")); + } + for _ in 0..parents { + let parent = c + .addr(offset_size)? + .ok_or(bad("invalid flush dependency parent offset"))?; + if !seen.contains(&parent) && !cached.contains(&parent) { + return Err(bad("fd parent not in cache")); + } + } + let len = usize::try_from(size).map_err(|_| bad(RAN_OFF))?; + let image_offset = c.pos; + c.take(len)?; + if address.checked_add(size).is_none_or(|end| end > eoa) { + return Err(bad("entry extends past the end of file")); + } + if !seen.insert(address) { + return Err(bad("duplicate addresses in cache")); + } + entries.push(ImageEntry { + address, + image_offset, + len, + }); + } + Ok(CacheImage { location, entries }) + } + + /// Where the image block is. + pub fn location(&self) -> CacheImageLocation { + self.location + } + + /// The number of entries in the image. + pub fn len(&self) -> usize { + self.entries.len() + } + + /// Whether the image has no entries (a decoded image always has some). + pub fn is_empty(&self) -> bool { + self.entries.is_empty() + } + + /// The file ranges (address, length) the image's entries replace. + pub fn entry_ranges(&self) -> impl Iterator + '_ { + self.entries.iter().map(|e| (e.address, e.len)) + } + + /// The image block in `data`, the bytes [`Self::decode`] read it from. + pub fn block<'a>(&self, data: &'a [u8]) -> Result<&'a [u8], FormatError> { + image_block(data, self.location) + } + + /// Write every entry over `dst`, the file's bytes from the superblock + /// on (as long as the `data` the image was decoded from), taking the + /// entries from `block` (the image block, see [`Self::block`]). `block` + /// must not alias `dst`: an entry may land on the block itself. + pub fn apply(&self, block: &[u8], dst: &mut [u8]) -> Result<(), FormatError> { + let short = || FormatError::InvalidCacheImage("image applied to the wrong file"); + for e in &self.entries { + let src = block + .get(e.image_offset..e.image_offset + e.len) + .ok_or_else(short)?; + let at = usize::try_from(e.address).map_err(|_| short())?; + dst.get_mut(at..at + e.len) + .ok_or_else(short)? + .copy_from_slice(src); + } + Ok(()) + } +} + +fn image_block(data: &[u8], location: CacheImageLocation) -> Result<&[u8], FormatError> { let bad = FormatError::InvalidCacheImage; let start = usize::try_from(location.address).map_err(|_| bad("address out of range"))?; let len = usize::try_from(location.length).map_err(|_| bad("length out of range"))?; - let block = start + start .checked_add(len) .and_then(|end| data.get(start..end)) - .ok_or(bad("image block extends past the end of the file"))?; - let eoa = data.len() as u64; - let mut c = Cursor::new(block, bad(RAN_OFF)); - - // Header: signature, version, flags, image data length, entry count. - if c.take(4)? != MDCI_SIGNATURE { - return Err(bad("bad metadata cache image header signature")); - } - if c.u8()? != 0 { - return Err(bad("bad metadata cache image version")); - } - if c.u8()? & MDCI_HAVE_RESIZE_STATUS != 0 { - return Err(bad("MDC resize status not yet supported")); - } - if c.uint(length_size)? != location.length { - return Err(bad("bad metadata cache image data length")); - } - let n_entries = c.uint(4)?; - if n_entries == 0 { - return Err(bad("bad metadata cache entry count")); - } - - let mut entries = Vec::new(); - // What is in libhdf5's cache when it loads the image: the superblock - // and the superblock extension's object header (read to find the image). - // Each entry's flush-dependency parents are looked up in the cache as - // the entry is inserted (`H5C__reconstruct_cache_contents` searches the - // index inside the loop that inserts the entries, in HDF5 1.14.6 and - // 2.0.0 alike), so a parent must be one of those or an earlier entry. - let mut cached = BTreeSet::new(); - cached.insert(0); - if let Some(ext) = sb.superblock_extension_address { - cached.insert(ext); - } - let mut seen = BTreeSet::new(); - for _ in 0..n_entries { - let type_id = c.u8()?; - if type_id >= MDCI_NTYPES { - return Err(bad("type id is out of valid range")); - } - let flags = c.u8()?; - if c.u8()? >= MDCI_RING_NTYPES { - return Err(bad("ring is out of valid range")); - } - if c.u8()? > MDCI_AGE_MAX { - return Err(bad("entry age is out of policy range")); - } - let children = c.uint(2)?; - // libhdf5 checks the parent flag against the child count only in - // debug builds (release builds refuse any entry with children); the - // image format's own rule is checked here. - if (flags & MDCI_ENTRY_IS_FD_PARENT != 0) != (children > 0) { - return Err(bad("flush dependency parent flag and child count disagree")); - } - c.uint(2)?; // dirty dependency children: reset for a read-only open - let parents = c.uint(2)?; - if (flags & MDCI_ENTRY_IS_FD_CHILD != 0) != (parents > 0) { - return Err(bad("flush dependency child flag and parent count disagree")); - } - c.uint(4)?; // LRU rank - let address = c - .addr(offset_size)? - .filter(|&a| a < eoa) - .ok_or(bad("invalid entry address range"))?; - let size = c.uint(length_size)?; - if size == 0 { - return Err(bad("invalid entry size")); - } - for _ in 0..parents { - let parent = c - .addr(offset_size)? - .ok_or(bad("invalid flush dependency parent offset"))?; - if !seen.contains(&parent) && !cached.contains(&parent) { - return Err(bad("fd parent not in cache")); - } - } - let len = usize::try_from(size).map_err(|_| bad(RAN_OFF))?; - let image_offset = c.pos; - c.take(len)?; - if !seen.insert(address) { - return Err(bad("duplicate addresses in cache")); - } - entries.push(ImageEntry { - address, - image_offset, - len, - }); - } - - let mut out = data.to_vec(); - for e in &entries { - // address < eoa <= usize::MAX, and the entry's bytes came from the - // block, so neither conversion nor the sum can fail. - let at = e.address as usize; - let end = at + e.len; - if end > out.len() { - out.resize(end, 0); - } - out[at..end].copy_from_slice(&block[e.image_offset..e.image_offset + e.len]); - } - Ok(out) + .ok_or(bad("image block extends past the end of the file")) } -/// What a reader must do before reading a file's metadata, in one call: -/// check the superblock extension ([`read_superblock_extension`]) and, -/// when the file has a metadata cache image, return the file's bytes with -/// the image applied ([`apply_cache_image`]). `Ok(None)` means read `data` -/// as it is. -pub fn metadata_view(data: &[u8], sb: &Superblock) -> Result>, FormatError> { +/// What an opener must do before reading a file's metadata: check the +/// superblock extension ([`read_superblock_extension`]; an error means +/// libhdf5 refuses to open the file) and decode any metadata cache image +/// ([`CacheImage::decode`]). `data` is the file from the superblock on, up +/// to its recorded end of file. +pub fn cache_image_state(data: &[u8], sb: &Superblock) -> Result { match read_superblock_extension(data, sb)? { Some(SuperblockExtension { cache_image: Some(location), .. - }) => apply_cache_image(data, location, sb).map(Some), - _ => Ok(None), + }) => Ok(match CacheImage::decode(data, location, sb) { + Ok(image) => CacheImageState::Loaded(image), + Err(e) => CacheImageState::Unloadable(e), + }), + _ => Ok(CacheImageState::Absent), + } +} + +/// [`cache_image_state`] for a reader that holds the file's bytes in a +/// buffer of its own: check the superblock extension and write any cache +/// image over `data` in place (only the image block is copied). An image +/// libhdf5 cannot load is an error here: such a reader has no way to open +/// the file and fail each object instead. +pub fn apply_cache_image_in_place(data: &mut [u8], sb: &Superblock) -> Result<(), FormatError> { + match cache_image_state(data, sb)? { + CacheImageState::Absent => Ok(()), + CacheImageState::Unloadable(e) => Err(e), + CacheImageState::Loaded(image) => { + let block = image.block(data)?.to_vec(); + image.apply(&block, data) + } } } @@ -435,6 +530,18 @@ pub fn metadata_view(data: &[u8], sb: &Superblock) -> Result>, Fo mod tests { use super::*; + /// The file's bytes with the image at `loc` applied. + fn apply_cache_image( + data: &[u8], + loc: CacheImageLocation, + sb: &Superblock, + ) -> Result, FormatError> { + let image = CacheImage::decode(data, loc, sb)?; + let mut out = data.to_vec(); + image.apply(image.block(data)?, &mut out)?; + Ok(out) + } + fn sb_v2(ext: u64) -> Superblock { Superblock { version: 2, @@ -651,6 +758,12 @@ mod tests { let mut len = image(&[(16, b"x")]); len[6] ^= 1; assert!(matches!(bad(len), FormatError::InvalidCacheImage(_))); + // An entry that starts inside the file (64 bytes, then a 60-byte + // image) but runs past its end. + assert!(matches!( + bad(image(&[(123, b"8 bytes!")])), + FormatError::InvalidCacheImage("entry extends past the end of file") + )); let mut cut = image(&[(16, b"abcdef")]); let n = cut.len() as u64 - 8; cut.truncate(cut.len() - 8); diff --git a/crates/clawhdf5-io/src/lib.rs b/crates/clawhdf5-io/src/lib.rs index 1f42019..5de13b7 100644 --- a/crates/clawhdf5-io/src/lib.rs +++ b/crates/clawhdf5-io/src/lib.rs @@ -41,6 +41,65 @@ pub trait HDF5Read { fn is_empty(&self) -> bool { self.as_bytes().is_empty() } + + /// A private, writable copy of [`Self::as_bytes`]: writes to it stay in + /// this process and never reach the underlying storage. + /// + /// Readers use it to lay a file's metadata cache image over the file's + /// own metadata. The default copies the bytes; a memory-mapped reader + /// returns a copy-on-write mapping instead, so only the pages written to + /// are copied and the rest stay shared with the page cache. + fn private_copy(&self) -> io::Result { + Ok(PrivateCopy::Owned(self.as_bytes().to_vec())) + } +} + +/// A private, writable copy of a file's bytes (see +/// [`HDF5Read::private_copy`]). +pub enum PrivateCopy { + /// The bytes copied onto the heap. + Owned(Vec), + /// A copy-on-write mapping of the file: pages are copied only when + /// written to. + #[cfg(feature = "mmap")] + Mapped(memmap2::MmapMut), +} + +impl PrivateCopy { + /// Whether this is a copy-on-write mapping rather than a heap copy. + pub fn is_mapped(&self) -> bool { + !matches!(self, PrivateCopy::Owned(_)) + } +} + +impl std::fmt::Debug for PrivateCopy { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("PrivateCopy") + .field("len", &self.len()) + .field("mapped", &self.is_mapped()) + .finish() + } +} + +impl std::ops::Deref for PrivateCopy { + type Target = [u8]; + fn deref(&self) -> &[u8] { + match self { + PrivateCopy::Owned(v) => v, + #[cfg(feature = "mmap")] + PrivateCopy::Mapped(m) => m, + } + } +} + +impl std::ops::DerefMut for PrivateCopy { + fn deref_mut(&mut self) -> &mut [u8] { + match self { + PrivateCopy::Owned(v) => v, + #[cfg(feature = "mmap")] + PrivateCopy::Mapped(m) => m, + } + } } /// Read-write access to HDF5 data. diff --git a/crates/clawhdf5-io/src/mmap.rs b/crates/clawhdf5-io/src/mmap.rs index 70d99c9..756d5e3 100644 --- a/crates/clawhdf5-io/src/mmap.rs +++ b/crates/clawhdf5-io/src/mmap.rs @@ -87,6 +87,19 @@ impl HDF5Read for MmapReader { fn as_bytes(&self) -> &[u8] { &self.mmap } + + /// A private copy-on-write mapping of the file (`MAP_PRIVATE`): only the + /// pages written to are copied. + fn private_copy(&self) -> io::Result { + if self.mmap.is_empty() { + return Ok(crate::PrivateCopy::Owned(Vec::new())); + } + // SAFETY: as for `open`: the caller keeps the file from being + // modified while the mapping is alive. Writes to a private mapping + // never reach the file. + let map = unsafe { memmap2::MmapOptions::new().map_copy(&self._file)? }; + Ok(crate::PrivateCopy::Mapped(map)) + } } /// Writable memory-mapped file for read-write HDF5 access. @@ -218,6 +231,22 @@ mod tests { fs::remove_file(&path).ok(); } + #[test] + fn private_copy_is_a_copy_on_write_mapping() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("cow.bin"); + fs::write(&path, [1u8, 2, 3, 4]).unwrap(); + let reader = MmapReader::open(&path).unwrap(); + let mut copy = reader.private_copy().unwrap(); + assert!(copy.is_mapped()); + copy[1] = 99; + assert_eq!(©[..], &[1, 99, 3, 4]); + // Neither the reader's mapping nor the file sees the write. + assert_eq!(reader.as_bytes(), &[1, 2, 3, 4]); + drop(copy); + assert_eq!(fs::read(&path).unwrap(), [1, 2, 3, 4]); + } + #[test] fn mmap_reader_read_at() { let dir = std::env::temp_dir(); diff --git a/crates/clawhdf5-io/src/prefetch.rs b/crates/clawhdf5-io/src/prefetch.rs index a4dc451..bf3ad9b 100644 --- a/crates/clawhdf5-io/src/prefetch.rs +++ b/crates/clawhdf5-io/src/prefetch.rs @@ -195,6 +195,10 @@ impl HDF5Read for PrefetchReader { fn as_bytes(&self) -> &[u8] { self.inner.as_bytes() } + + fn private_copy(&self) -> std::io::Result { + self.inner.private_copy() + } } // --------------------------------------------------------------------------- diff --git a/crates/clawhdf5/src/cache_image.rs b/crates/clawhdf5/src/cache_image.rs new file mode 100644 index 0000000..1ceb741 --- /dev/null +++ b/crates/clawhdf5/src/cache_image.rs @@ -0,0 +1,85 @@ +//! Metadata cache images, as the file openers apply them. +//! +//! A file written with a metadata cache image keeps metadata cache entries +//! (object headers, B-tree nodes, heaps) in an image block, and libhdf5 +//! reads those entries in place of the file's own bytes at their addresses +//! (see `clawhdf5_format::superblock_ext`). The metadata parsers read one +//! contiguous byte slice, so the image has to be laid over the file's bytes +//! — without copying the file: +//! +//! - an opener that holds the file in a buffer it owns writes the entries +//! into that buffer (only the image block is copied, as libhdf5 copies +//! it); +//! - an opener that maps the file ([`File::open`](crate::File::open), +//! [`MmapFile`](crate::MmapFile), [`LazyFile::open_mmap`] +//! (crate::LazyFile::open_mmap)) writes them into a private copy-on-write +//! mapping of the file ([`clawhdf5_io::HDF5Read::private_copy`]): only the +//! pages the entries land on are copied, and the rest of the file stays +//! shared with the page cache; +//! - a file without an image is read from the original bytes, as before. + +use clawhdf5_format::error::FormatError; +use clawhdf5_format::superblock::Superblock; +use clawhdf5_format::superblock_ext::{self, CacheImageState}; +use clawhdf5_io::PrivateCopy; + +use crate::error::Error; + +/// A file's metadata, as an opener must read it. +pub(crate) enum ImageView { + /// Read the opener's bytes: the file has no cache image, or it was + /// written into a buffer the opener owns. + Plain, + /// The file with its cache image written in: a private copy of the + /// whole file (the HDF5 data at the same offsets as in the file). + Patched(PrivateCopy), + /// The file has a cache image libhdf5 cannot load. + Unloadable(FormatError), +} + +/// Check the superblock extension of the file whose bytes are `whole` (the +/// HDF5 data in `base..end`) and lay any cache image over a private copy +/// of the file made by `copy`. An error means libhdf5 refuses the file. +pub(crate) fn private_view( + whole: &[u8], + base: usize, + end: usize, + sb: &Superblock, + copy: impl FnOnce() -> std::io::Result, +) -> Result { + let data = &whole[base..end]; + Ok(match superblock_ext::cache_image_state(data, sb)? { + CacheImageState::Absent => ImageView::Plain, + CacheImageState::Unloadable(e) => ImageView::Unloadable(e), + CacheImageState::Loaded(image) => { + let mut view = copy().map_err(Error::Io)?; + let dst = + view.get_mut(base..end) + .ok_or(Error::Format(FormatError::InvalidCacheImage( + "the file changed while it was opened", + )))?; + image.apply(image.block(data)?, dst)?; + ImageView::Patched(view) + } + }) +} + +/// [`private_view`] for a file held in `whole`, a buffer the opener owns: +/// the image is written into it in place. +pub(crate) fn in_place( + whole: &mut [u8], + base: usize, + end: usize, + sb: &Superblock, +) -> Result { + let data = &mut whole[base..end]; + Ok(match superblock_ext::cache_image_state(data, sb)? { + CacheImageState::Absent => ImageView::Plain, + CacheImageState::Unloadable(e) => ImageView::Unloadable(e), + CacheImageState::Loaded(image) => { + let block = image.block(data)?.to_vec(); + image.apply(&block, data)?; + ImageView::Plain + } + }) +} diff --git a/crates/clawhdf5/src/lazy.rs b/crates/clawhdf5/src/lazy.rs index 8746f5f..cb688e9 100644 --- a/crates/clawhdf5/src/lazy.rs +++ b/crates/clawhdf5/src/lazy.rs @@ -46,9 +46,13 @@ pub struct LazyFile { /// End of the HDF5 data (`Superblock::data_end`, absolute). end: usize, superblock: Superblock, - /// The metadata as libhdf5 reads it when the file holds a metadata - /// cache image (see `superblock_ext::metadata_view`); `None` otherwise. - overlay: Option>, + /// A file that holds a metadata cache image, with the image written in: + /// the reader's [`HDF5Read::private_copy`] of the whole file (a + /// copy-on-write mapping for [`clawhdf5_io::MmapReader`], so only the + /// pages the image's entries land on are copied; see + /// `crate::cache_image`). `None` for a file without an image, read + /// straight from the reader. + patched: Option, root_header: ObjectHeader, /// Cache of parsed object headers, keyed by address. header_cache: RefCell>, @@ -87,11 +91,19 @@ impl LazyFile { let end = base + superblock.data_end(base as u64, whole_len)? as usize; // Decode the superblock extension as libhdf5 does at open, and load // a metadata cache image over the file's metadata. - let overlay = clawhdf5_format::superblock_ext::metadata_view( - &reader.as_bytes()[base..end], - &superblock, - )?; - let data = overlay.as_deref().unwrap_or(&reader.as_bytes()[base..end]); + let view = + crate::cache_image::private_view(reader.as_bytes(), base, end, &superblock, || { + reader.private_copy() + })?; + let patched = match view { + crate::cache_image::ImageView::Plain => None, + crate::cache_image::ImageView::Patched(p) => Some(p), + crate::cache_image::ImageView::Unloadable(e) => return Err(e.into()), + }; + let data = match &patched { + Some(p) => &p[base..end], + None => &reader.as_bytes()[base..end], + }; let root_header = ObjectHeader::parse( data, superblock.root_group_address as usize, @@ -103,7 +115,7 @@ impl LazyFile { base, end, superblock, - overlay, + patched, root_header, header_cache: RefCell::new(HashMap::new()), }) @@ -121,8 +133,8 @@ impl LazyFile { } fn hdf5_bytes(&self) -> &[u8] { - match &self.overlay { - Some(v) => v, + match &self.patched { + Some(p) => &p[self.base..self.end], None => &self.reader.as_bytes()[self.base..self.end], } } diff --git a/crates/clawhdf5/src/lib.rs b/crates/clawhdf5/src/lib.rs index f8098fa..222e432 100644 --- a/crates/clawhdf5/src/lib.rs +++ b/crates/clawhdf5/src/lib.rs @@ -24,6 +24,7 @@ //! builder.write("output.h5").unwrap(); //! ``` +mod cache_image; pub mod error; pub mod lazy; #[cfg(feature = "mmap")] diff --git a/crates/clawhdf5/src/mmap_file.rs b/crates/clawhdf5/src/mmap_file.rs index 05d581a..0de5b02 100644 --- a/crates/clawhdf5/src/mmap_file.rs +++ b/crates/clawhdf5/src/mmap_file.rs @@ -38,9 +38,11 @@ pub struct MmapFile { /// End of the HDF5 data (`Superblock::data_end`, absolute). end: usize, superblock: Superblock, - /// The metadata as libhdf5 reads it when the file holds a metadata - /// cache image (see `superblock_ext::metadata_view`); `None` otherwise. - overlay: Option>, + /// A file that holds a metadata cache image, with the image written in: + /// a private copy-on-write mapping of the whole file, so only the pages + /// the image's entries land on are copied (see `crate::cache_image`). + /// `None` for a file without an image, read straight from the mapping. + patched: Option, } impl MmapFile { @@ -55,24 +57,29 @@ impl MmapFile { let end = base + superblock.data_end(base as u64, whole_len)? as usize; // Decode the superblock extension as libhdf5 does at open, and load // a metadata cache image over the file's metadata. - let overlay = clawhdf5_format::superblock_ext::metadata_view( - &reader.as_bytes()[base..end], - &superblock, - )?; + let view = + crate::cache_image::private_view(reader.as_bytes(), base, end, &superblock, || { + clawhdf5_io::HDF5Read::private_copy(&reader) + })?; + let patched = match view { + crate::cache_image::ImageView::Plain => None, + crate::cache_image::ImageView::Patched(p) => Some(p), + crate::cache_image::ImageView::Unloadable(e) => return Err(e.into()), + }; Ok(Self { reader, base, end, superblock, - overlay, + patched, }) } /// The file's bytes from the superblock on — the space HDF5 addresses /// index into. fn hdf5_bytes(&self) -> &[u8] { - match &self.overlay { - Some(v) => v, + match &self.patched { + Some(p) => &p[self.base..self.end], None => &self.reader.as_bytes()[self.base..self.end], } } diff --git a/crates/clawhdf5/src/reader.rs b/crates/clawhdf5/src/reader.rs index c892eea..9f6da0f 100644 --- a/crates/clawhdf5/src/reader.rs +++ b/crates/clawhdf5/src/reader.rs @@ -20,8 +20,8 @@ use clawhdf5_format::message_type::MessageType; use clawhdf5_format::object_header::ObjectHeader; use clawhdf5_format::signature; use clawhdf5_format::superblock::Superblock; -use clawhdf5_format::superblock_ext; +use crate::cache_image::{self, ImageView}; use crate::error::Error; use crate::types::{AttrValue, DType, classify_datatype, read_attrs}; @@ -56,17 +56,19 @@ struct FileData { base: usize, /// End of the HDF5 data in the file (`Superblock::data_end`, absolute). end: usize, - /// The file's metadata as libhdf5 reads it when the file holds a - /// metadata cache image: the bytes from the superblock to the end of - /// file with the image's entries written in - /// ([`superblock_ext::metadata_view`]). `None` for every other file. - overlay: Option>, + /// A mapped file that holds a metadata cache image, with the image + /// written in: a private copy-on-write mapping of the whole file, so + /// only the pages the image's entries land on are copied (see + /// `crate::cache_image`). `None` for every other file: a file without + /// an image is read straight from the mapping, and an owned buffer has + /// the image written into it in place. + patched: Option, } impl FileData { /// Locate the superblock and parse it. A truncated file is refused, and /// bytes past the recorded end of file are not read, as in libhdf5. - fn new(backing: Backing) -> Result<(Self, Superblock), Error> { + fn new(mut backing: Backing) -> Result<(Self, Superblock), Error> { let whole = backing.whole_file(); let (user_block, hdf5) = signature::split_user_block(whole)?; let base = user_block.len(); @@ -77,21 +79,34 @@ impl FileData { // libhdf5 decodes the superblock extension at open (a message it // cannot decode fails the open) and loads a metadata cache image // over the file's own metadata. - let overlay = superblock_ext::metadata_view(&whole[base..end], &superblock)?; + let view = match &mut backing { + Backing::Owned(v) => cache_image::in_place(v, base, end, &superblock)?, + #[cfg(feature = "mmap")] + Backing::Mmap(r) => { + cache_image::private_view(r.as_bytes(), base, end, &superblock, || { + clawhdf5_io::HDF5Read::private_copy(r) + })? + } + }; + let patched = match view { + ImageView::Plain => None, + ImageView::Patched(p) => Some(p), + ImageView::Unloadable(e) => return Err(e.into()), + }; Ok(( Self { backing, base, end, - overlay, + patched, }, superblock, )) } fn as_bytes(&self) -> &[u8] { - match &self.overlay { - Some(v) => v, + match &self.patched { + Some(p) => &p[self.base..self.end], None => &self.backing.whole_file()[self.base..self.end], } } @@ -1282,3 +1297,44 @@ mod sibling_file_name_tests { } } } + +#[cfg(all(test, feature = "mmap"))] +mod zero_copy_tests { + use super::*; + + /// Where `File::open` reads metadata from: `Some(true)` for the file's + /// own mapping, `Some(false)` for a private copy-on-write mapping. + fn reads_from_the_mapping(f: &File) -> Option { + let Backing::Mmap(r) = &f.data.backing else { + return None; + }; + let mapped = r.as_bytes()[f.data.base..].as_ptr(); + match &f.data.patched { + None => Some(std::ptr::eq(f.as_bytes().as_ptr(), mapped)), + Some(p) => { + assert!(p.is_mapped(), "the image went into a heap copy of the file"); + Some(false) + } + } + } + + #[test] + fn a_file_without_a_cache_image_is_read_from_the_mapping() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("plain.h5"); + let mut b = crate::FileBuilder::new(); + b.create_dataset("d").with_f64_data(&[1.0, 2.0]); + b.write(&path).unwrap(); + let f = File::open(&path).unwrap(); + assert_eq!(reads_from_the_mapping(&f), Some(true)); + assert_eq!(f.dataset("d").unwrap().read_f64().unwrap(), [1.0, 2.0]); + } + + #[test] + fn a_cache_image_goes_into_a_copy_on_write_mapping() { + let path = std::path::Path::new(env!("CARGO_MANIFEST_DIR")) + .join("tests/fixtures/h5clear_mdc_image.h5"); + let f = File::open(path).unwrap(); + assert_eq!(reads_from_the_mapping(&f), Some(false)); + } +} diff --git a/crates/clawhdf5/tests/cache_image_memory.rs b/crates/clawhdf5/tests/cache_image_memory.rs new file mode 100644 index 0000000..16186ba --- /dev/null +++ b/crates/clawhdf5/tests/cache_image_memory.rs @@ -0,0 +1,139 @@ +//! Opening a file with a metadata cache image must not copy the file. +//! +//! The image's entries are laid over the file's bytes in a private +//! copy-on-write mapping (`File::open`, `MmapFile::open`, +//! `LazyFile::open_mmap`), so only the pages they land on are copied. The +//! first implementation copied the whole file onto the heap at open: a +//! 1 GiB file that takes a few KB on disk needed 2 GB of memory, and an +//! 8 GiB one aborted the process. Here libhdf5 itself (the library h5py +//! bundles, through ctypes: `H5Pset_mdc_image_config`) adds an image to a +//! 1 GiB sparse file, and the process's resident memory must stay far below +//! the file's size while each opener lists the file and reads its small +//! dataset. +//! +//! One test in its own binary, so no other test's allocations land in the +//! measurement. Linux only (it reads `VmRSS` from `/proc/self/status`). +//! Skipped when python3 with h5py is unavailable, unless +//! `CLAWHDF5_REQUIRE_INTEROP=1`. + +#![cfg(target_os = "linux")] + +use std::path::Path; +use std::process::Command; + +use clawhdf5::{File, LazyFile, MmapFile}; + +fn python() -> String { + std::env::var("CLAWHDF5_PYTHON").unwrap_or_else(|_| "python3".to_string()) +} + +fn python_available() -> bool { + Command::new(python()) + .args(["-c", "import h5py, numpy"]) + .output() + .map(|o| o.status.success()) + .unwrap_or(false) +} + +fn rss_bytes() -> u64 { + let status = std::fs::read_to_string("/proc/self/status").unwrap(); + let line = status.lines().find(|l| l.starts_with("VmRSS:")).unwrap(); + let kb: u64 = line.split_whitespace().nth(1).unwrap().parse().unwrap(); + kb * 1024 +} + +/// A 1 GiB sparse file: `/big`, 2^27 `f8` with only its last element +/// written, `/small` = 0..10, with a metadata cache image added by libhdf5. +fn make_file(path: &Path) { + let script = format!( + r#" +import ctypes, glob, os, h5py, numpy as np +path = "{path}" +libs = glob.glob(os.path.join(os.path.dirname(h5py.__file__), os.pardir, "h5py.libs", "libhdf5-*.so*")) +assert libs, "no libhdf5 bundled with h5py" +lib = ctypes.CDLL(libs[0]) +class Cfg(ctypes.Structure): + _fields_ = [("version", ctypes.c_int), ("generate_image", ctypes.c_bool), + ("save_resize_status", ctypes.c_bool), ("entry_ageout", ctypes.c_int)] +with h5py.File(path, "w", libver="latest") as f: + d = f.create_dataset("big", shape=(2**27,), dtype="f8") + d[-1] = 7.5 + f.create_dataset("small", data=np.arange(10, dtype="= 0 +f = h5py.File(h5py.h5f.open(path.encode(), h5py.h5f.ACC_RDWR, fapl=fapl)) +f["small"][()]; f["big"].shape +f.close() +assert os.path.getsize(path) >= 2**30 +with open(path, "rb") as fh: + fh.seek(-(1 << 20), 2) + assert b"MDCI" in fh.read(), "libhdf5 wrote no cache image" +with h5py.File(path, "r") as f: + assert list(f["small"][()]) == list(range(10)) +"#, + path = path.display() + ); + let out = Command::new(python()) + .args(["-c", &script]) + .output() + .expect("failed to run python"); + assert!( + out.status.success(), + "python failed:\n{}", + String::from_utf8_lossy(&out.stderr) + ); +} + +#[test] +fn a_cache_image_does_not_copy_the_file() { + if !python_available() { + assert!( + !std::env::var("CLAWHDF5_REQUIRE_INTEROP").is_ok_and(|v| v == "1"), + "CLAWHDF5_REQUIRE_INTEROP=1 but python3 with h5py is not available" + ); + eprintln!("SKIP: python3 with h5py not available"); + return; + } + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("sparse_image.h5"); + make_file(&path); + + const LIMIT: u64 = 256 << 20; + let small: Vec = (0..10).collect(); + let before = rss_bytes(); + { + let f = File::open(&path).unwrap(); + let mut names = f.root().datasets().unwrap(); + names.sort(); + assert_eq!(names, ["big", "small"]); + assert_eq!(f.dataset("small").unwrap().read_i32().unwrap(), small); + assert_eq!(f.dataset("big").unwrap().shape().unwrap(), [1 << 27]); + let grew = rss_bytes().saturating_sub(before); + assert!( + grew < LIMIT, + "File::open: resident memory grew {grew} bytes" + ); + } + let before = rss_bytes(); + { + let f = MmapFile::open(&path).unwrap(); + assert_eq!(f.dataset("small").unwrap().read_i32().unwrap(), small); + let grew = rss_bytes().saturating_sub(before); + assert!( + grew < LIMIT, + "MmapFile::open: resident memory grew {grew} bytes" + ); + } + let before = rss_bytes(); + { + let f = LazyFile::open_mmap(&path).unwrap(); + assert_eq!(f.dataset("small").unwrap().read_i32().unwrap(), small); + let grew = rss_bytes().saturating_sub(before); + assert!( + grew < LIMIT, + "LazyFile::open_mmap: resident memory grew {grew} bytes" + ); + } +} From 6559a91495c12b005a434bddadcae091d787fc66 Mon Sep 17 00:00:00 2001 From: osobh Date: Sat, 26 Sep 2026 11:46:28 -0500 Subject: [PATCH 37/43] fix: open a file whose cache image cannot load, and fail its objects For a metadata cache image libhdf5 cannot load, libhdf5 opens the file and fails the first metadata read (the image loads on the first H5C_protect after open); h5py reports the error on the root group. The conformance probe reported it that way, but File::open refused the file, so the gate counted cve-2025-6269-1..4 and cve-2025-6516 as agreeing with h5py for behaviour the library did not have. The library now behaves as the probe reports: File (mmap, buffered and from_bytes) and MmapFile open the file and every object lookup (dataset, dataset_at, group, group listings and attributes, VL decoding) fails with the image's error; LazyFile reads the root group's header at open, so its open is that first read and fails. Probe and library take the three-way decision (refuse at open / image loads / image cannot load) from the same clawhdf5_format::superblock_ext::cache_image_state. One deliberate difference from libhdf5 remains, documented: after the failed first read libhdf5 reads the file's own metadata, which the image was meant to replace and may be stale; here every lookup keeps failing. File::cache_image_error / MmapFile::cache_image_error expose the error to code that parses as_bytes() itself; h5rs checks it before reading any object header (h5rs ls on cve-2025-6269-1 said "invalid object header version: 0" from the stale bytes). Test: metadata_cache_image.rs an_image_libhdf5_cannot_load_fails_every_object (the fixture with its image signature broken; h5py opens that file and fails the first read with "Bad metadata cache image header signature"). It fails on the previous commit, where File::open refuses the file. Co-Authored-By: Claude Opus 5.5 (1M context) --- CHANGELOG.md | 12 ++++- conformance/probe/src/main.rs | 10 ++-- crates/clawhdf5-tools/src/h5.rs | 5 ++ crates/clawhdf5/src/lazy.rs | 3 ++ crates/clawhdf5/src/mmap_file.rs | 35 ++++++++++--- crates/clawhdf5/src/reader.rs | 49 ++++++++++++++----- crates/clawhdf5/tests/metadata_cache_image.rs | 46 +++++++++++++++++ docs/known-issues.md | 16 ++++-- 8 files changed, 148 insertions(+), 28 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3b0efd6..d256a15 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -24,8 +24,16 @@ ZFP filters; the 2 mismatches are the known h5py big-endian VL bug. image, and an abort for an 8 GiB one; `tests/cache_image_memory.rs` guards it.) An image entry that runs past the end of file is refused (libhdf5 checks only its start; the images it writes never do this). A - file whose image libhdf5 cannot load opens in libhdf5 but nothing in it - can be read; `File::open` refuses it. + file whose image libhdf5 cannot load (`cve-2025-6269-*`, `cve-2025-6516`) + opens, as in libhdf5, and every object lookup fails with the image's + error (`File`, `MmapFile`; `LazyFile` reads the root group at open, so + its open fails). libhdf5 fails only its first metadata read and then + reads the file's own, possibly stale, bytes; those are never read here. + An interim version refused such a file at `File::open` while the + conformance probe reported it as libhdf5 does, so the gate counted five + files as agreeing with h5py that the library did not open; probe and + library now take the decision from the same + `superblock_ext::cache_image_state`. - **The superblock extension is decoded at open, as libhdf5 does:** a File Space Info or Metadata Cache Image message libhdf5 cannot decode makes the open fail (`cve-2020-10810`, `cve-2020-10812` were opened). diff --git a/conformance/probe/src/main.rs b/conformance/probe/src/main.rs index 8c25a88..71c3139 100644 --- a/conformance/probe/src/main.rs +++ b/conformance/probe/src/main.rs @@ -712,12 +712,14 @@ fn main() { return; } }; - // libhdf5 decodes the superblock extension at open (File::open does - // the same), and loads a metadata cache image over the file's own + // libhdf5 decodes the superblock extension at open (an error refuses + // the file), and loads a metadata cache image over the file's own // metadata. It loads the image only when it first reads metadata — the // root group — so a file whose image it cannot load still opens and - // every object fails; File::open refuses such a file outright. The - // probe records the image's error where libhdf5 reports it. + // that read fails. The library decides all three cases with the same + // `cache_image_state`: `File` and `MmapFile` open such a file and fail + // every object lookup with the image's error, which is what the probe + // records here (on the root group, where libhdf5 reports it). use clawhdf5_format::superblock_ext::{self, CacheImageState}; let state = match guarded(|| superblock_ext::cache_image_state(hdf5, &sb).map_err(e)) { Ok(x) => x, diff --git a/crates/clawhdf5-tools/src/h5.rs b/crates/clawhdf5-tools/src/h5.rs index 1f4354a..5f71a14 100644 --- a/crates/clawhdf5-tools/src/h5.rs +++ b/crates/clawhdf5-tools/src/h5.rs @@ -234,6 +234,11 @@ impl H5 { } pub fn header(&self, addr: u64) -> Result { + // A metadata cache image libhdf5 cannot load: the file opens, and + // every object fails (its bytes may hold stale metadata). + if let Some(e) = self.file.cache_image_error() { + return Err(Error::at(addr, format!("metadata cache image: {e}"))); + } let off = usize::try_from(addr).map_err(|_| Error::at(addr, "address out of range"))?; ObjectHeader::parse(self.data(), off, self.os(), self.ls()) .map_err(|e| Error::at(addr, format!("object header: {e}"))) diff --git a/crates/clawhdf5/src/lazy.rs b/crates/clawhdf5/src/lazy.rs index cb688e9..34f46cc 100644 --- a/crates/clawhdf5/src/lazy.rs +++ b/crates/clawhdf5/src/lazy.rs @@ -98,6 +98,9 @@ impl LazyFile { let patched = match view { crate::cache_image::ImageView::Plain => None, crate::cache_image::ImageView::Patched(p) => Some(p), + // libhdf5 opens such a file and fails its first metadata read; + // a LazyFile reads the root group's header at open, so the open + // is that read. crate::cache_image::ImageView::Unloadable(e) => return Err(e.into()), }; let data = match &patched { diff --git a/crates/clawhdf5/src/mmap_file.rs b/crates/clawhdf5/src/mmap_file.rs index 0de5b02..040c02f 100644 --- a/crates/clawhdf5/src/mmap_file.rs +++ b/crates/clawhdf5/src/mmap_file.rs @@ -43,6 +43,9 @@ pub struct MmapFile { /// the image's entries land on are copied (see `crate::cache_image`). /// `None` for a file without an image, read straight from the mapping. patched: Option, + /// The file has a metadata cache image libhdf5 cannot load: every + /// object lookup fails with this error (see `File`). + image_error: Option, } impl MmapFile { @@ -61,10 +64,10 @@ impl MmapFile { crate::cache_image::private_view(reader.as_bytes(), base, end, &superblock, || { clawhdf5_io::HDF5Read::private_copy(&reader) })?; - let patched = match view { - crate::cache_image::ImageView::Plain => None, - crate::cache_image::ImageView::Patched(p) => Some(p), - crate::cache_image::ImageView::Unloadable(e) => return Err(e.into()), + let (patched, image_error) = match view { + crate::cache_image::ImageView::Plain => (None, None), + crate::cache_image::ImageView::Patched(p) => (Some(p), None), + crate::cache_image::ImageView::Unloadable(e) => (None, Some(e)), }; Ok(Self { reader, @@ -72,6 +75,7 @@ impl MmapFile { end, superblock, patched, + image_error, }) } @@ -99,7 +103,7 @@ impl MmapFile { /// Resolve a path and return a `MmapDataset` handle. pub fn dataset(&self, path: &str) -> Result, Error> { - let data = self.hdf5_bytes(); + let data = self.meta()?; let addr = group_v2::resolve_path_any(data, &self.superblock, path)?; let hdr = self.parse_header(addr)?; if !has_message(&hdr, MessageType::DataLayout) { @@ -114,7 +118,7 @@ impl MmapFile { /// Resolve a path and return a `MmapGroup` handle. pub fn group(&self, path: &str) -> Result, Error> { - let data = self.hdf5_bytes(); + let data = self.meta()?; let addr = group_v2::resolve_path_any(data, &self.superblock, path)?; Ok(MmapGroup { file: self, @@ -129,14 +133,29 @@ impl MmapFile { self.hdf5_bytes() } + /// The error of a metadata cache image libhdf5 cannot load, when the + /// file has one (see [`crate::File::cache_image_error`]). + pub fn cache_image_error(&self) -> Option<&FormatError> { + self.image_error.as_ref() + } + /// Returns a reference to the parsed superblock. pub fn superblock(&self) -> &Superblock { &self.superblock } + /// The bytes to read metadata from; fails for a file whose cache image + /// cannot be loaded. + fn meta(&self) -> Result<&[u8], FormatError> { + match &self.image_error { + Some(e) => Err(e.clone()), + None => Ok(self.hdf5_bytes()), + } + } + fn parse_header(&self, address: u64) -> Result { ObjectHeader::parse( - self.hdf5_bytes(), + self.meta()?, address as usize, self.superblock.offset_size, self.superblock.length_size, @@ -257,7 +276,7 @@ impl<'f> MmapGroup<'f> { /// [`group_v2::resolve_group_children`]); dangling, external and /// user-defined links are left out. fn children(&self) -> Result, Error> { - let data = self.file.hdf5_bytes(); + let data = self.file.meta()?; group_v2::resolve_group_children(data, &self.file.superblock, self.address) .map_err(Error::Format) } diff --git a/crates/clawhdf5/src/reader.rs b/crates/clawhdf5/src/reader.rs index 9f6da0f..023e417 100644 --- a/crates/clawhdf5/src/reader.rs +++ b/crates/clawhdf5/src/reader.rs @@ -63,6 +63,11 @@ struct FileData { /// an image is read straight from the mapping, and an owned buffer has /// the image written into it in place. patched: Option, + /// The file has a metadata cache image libhdf5 cannot load. libhdf5 + /// opens such a file and fails its first metadata read (the image loads + /// then); every object lookup here fails with this error, and no + /// metadata is read from the file's own, possibly stale, bytes. + image_error: Option, } impl FileData { @@ -88,10 +93,10 @@ impl FileData { })? } }; - let patched = match view { - ImageView::Plain => None, - ImageView::Patched(p) => Some(p), - ImageView::Unloadable(e) => return Err(e.into()), + let (patched, image_error) = match view { + ImageView::Plain => (None, None), + ImageView::Patched(p) => (Some(p), None), + ImageView::Unloadable(e) => (None, Some(e)), }; Ok(( Self { @@ -99,6 +104,7 @@ impl FileData { base, end, patched, + image_error, }, superblock, )) @@ -114,6 +120,15 @@ impl FileData { fn len(&self) -> usize { self.as_bytes().len() } + + /// The bytes to read metadata from; fails for a file whose cache image + /// cannot be loaded (see [`Self::image_error`]). + fn meta(&self) -> Result<&[u8], FormatError> { + match &self.image_error { + Some(e) => Err(e.clone()), + None => Ok(self.as_bytes()), + } + } } // --------------------------------------------------------------------------- @@ -200,7 +215,7 @@ impl File { /// /// The path uses `/` separators (e.g., `"group1/values"`). pub fn dataset(&self, path: &str) -> Result, Error> { - let data = self.data.as_bytes(); + let data = self.data.meta()?; let addr = group_v2::resolve_path_any(data, &self.superblock, path)?; let hdr = self.parse_header(addr)?; if !has_message(&hdr, MessageType::DataLayout) { @@ -235,7 +250,7 @@ impl File { /// The path uses `/` separators (e.g., `"sensors"`). /// Use `"/"` or `""` for the root group. pub fn group(&self, path: &str) -> Result, Error> { - let data = self.data.as_bytes(); + let data = self.data.meta()?; let addr = group_v2::resolve_path_any(data, &self.superblock, path)?; Ok(Group { file: self, @@ -291,11 +306,23 @@ impl File { /// Returns the file's bytes from the superblock on (after any user /// block). Every HDF5 address in the file indexes this slice, so it is - /// what the `clawhdf5_format` parsers expect as `file_data`. + /// what the `clawhdf5_format` parsers expect as `file_data`. For a file + /// with a metadata cache image these are the bytes with the image + /// applied; when the image cannot be loaded they are the file's own + /// bytes, whose metadata may be stale (every object lookup fails then). pub fn as_bytes(&self) -> &[u8] { self.data.as_bytes() } + /// The error of a metadata cache image libhdf5 cannot load, when the + /// file has one. Such a file opens, as in libhdf5, and every object + /// lookup fails with this error; code that parses [`Self::as_bytes`] + /// itself should check it first, since those bytes then hold the + /// file's own, possibly stale, metadata. + pub fn cache_image_error(&self) -> Option<&FormatError> { + self.data.image_error.as_ref() + } + /// Size of the user block before the superblock (0 for most files). /// Matches h5py's `File.userblock_size`. pub fn user_block_size(&self) -> u64 { @@ -340,7 +367,7 @@ impl File { raw: &[u8], ) -> Result>, Error> { crate::vlen::decode_string_bytes( - self.as_bytes(), + self.data.meta()?, datatype, raw, self.offset_size(), @@ -358,7 +385,7 @@ impl File { raw: &[u8], ) -> Result>, Error> { crate::vlen::decode_vlen( - self.as_bytes(), + self.data.meta()?, datatype, raw, self.offset_size(), @@ -368,7 +395,7 @@ impl File { fn parse_header(&self, address: u64) -> Result { ObjectHeader::parse( - self.data.as_bytes(), + self.data.meta()?, address as usize, self.superblock.offset_size, self.superblock.length_size, @@ -490,7 +517,7 @@ impl<'f> Group<'f> { /// [`group_v2::resolve_group_children`]); dangling, external and /// user-defined links are left out. fn children(&self) -> Result, Error> { - let data = self.file.data.as_bytes(); + let data = self.file.data.meta()?; group_v2::resolve_group_children(data, &self.file.superblock, self.address) .map_err(Error::Format) } diff --git a/crates/clawhdf5/tests/metadata_cache_image.rs b/crates/clawhdf5/tests/metadata_cache_image.rs index 3cbc496..92463e8 100644 --- a/crates/clawhdf5/tests/metadata_cache_image.rs +++ b/crates/clawhdf5/tests/metadata_cache_image.rs @@ -46,3 +46,49 @@ fn mmap_and_lazy_files_read_through_the_cache_image() { expected() ); } + +/// A file whose cache image libhdf5 cannot load: libhdf5 opens it, and the +/// first metadata read fails with the image's error (h5py: +/// `Unable to get group info (Bad metadata cache image header signature)`); +/// later reads get the file's own bytes, which here are stale (the root +/// group's header is zeros: "bad object header version number"). The +/// openers agree on the open and the failure: `File` and `MmapFile` open +/// and fail every object lookup with the image's error (never reading the +/// stale bytes), `LazyFile` reads the root group's header at open and so +/// fails there. `File::open` used to refuse the file, while the +/// conformance probe reported it as h5py does. +#[test] +fn an_image_libhdf5_cannot_load_fails_every_object() { + let mut bytes = std::fs::read(fixture()).unwrap(); + let at = bytes.windows(4).position(|w| w == b"MDCI").unwrap(); + bytes[at] = b'X'; + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("bad_image.h5"); + std::fs::write(&path, &bytes).unwrap(); + + let is_image_error = |e: clawhdf5::Error| { + matches!( + e, + clawhdf5::Error::Format(clawhdf5_format::error::FormatError::InvalidCacheImage( + "bad metadata cache image header signature" + )) + ) + }; + for file in [ + File::open(&path).unwrap(), + File::from_bytes(bytes.clone()).unwrap(), + ] { + assert!(is_image_error(file.root().datasets().unwrap_err())); + assert!(is_image_error(file.root().attrs().unwrap_err())); + assert!(is_image_error(file.dataset("DSET").unwrap_err())); + assert!(is_image_error(file.group("/").map(|_| ()).unwrap_err())); + assert!(is_image_error(file.dataset_at(96).unwrap_err())); + } + let mm = MmapFile::open(&path).unwrap(); + assert!(is_image_error(mm.root().datasets().unwrap_err())); + assert!(is_image_error(mm.dataset("DSET").unwrap_err())); + assert!(is_image_error(mm.group("/").map(|_| ()).unwrap_err())); + assert!(is_image_error( + LazyFile::from_bytes(bytes).map(|_| ()).unwrap_err() + )); +} diff --git a/docs/known-issues.md b/docs/known-issues.md index b7c471a..8328f14 100644 --- a/docs/known-issues.md +++ b/docs/known-issues.md @@ -205,9 +205,19 @@ fill-value item that did is fixed). - Metadata cache images are not supported. **Fixed 2026-09-26:** the image is applied at open, as libhdf5 loads it over the file's metadata (`clawhdf5_format::superblock_ext`); `h5clear_mdc_image.h5` reads - (`crates/clawhdf5/tests/metadata_cache_image.rs`). A file whose image - libhdf5 cannot load (`cve-2025-6269-*`, `cve-2025-6516`) opens in - libhdf5 with nothing readable in it; `File::open` refuses it. + (`crates/clawhdf5/tests/metadata_cache_image.rs`), without copying the + file (a private copy-on-write mapping takes the image's entries; + `tests/cache_image_memory.rs`). A file whose image libhdf5 cannot load + (`cve-2025-6269-*`, `cve-2025-6516`) opens, as in libhdf5, and every + object lookup fails with the image's error. Differences from libhdf5 + that remain: libhdf5 fails only the first metadata read and then reads + the file's own (possibly stale) metadata, where we keep failing; an + image entry that runs past the end of file is refused (libhdf5 checks + only its start); a flush-dependency parent flag is checked against the + child count as libhdf5's debug build checks it (HDF5 2.0 release + builds refuse every entry that has children, even in images they + wrote); the superblock extension's driver-info and shared-message table + messages are not decoded at open. - x87 long double and binary128 are refused. - N-Bit on 64-bit scale-offset data and some N-Bit parameter layouts fail. **Not our bug (checked 2026-09-26):** both are corrupt files HDF5 2.0 From d493d4792e1d711cdd4746ca9fa101e79246ab81 Mon Sep 17 00:00:00 2001 From: osobh Date: Sat, 26 Sep 2026 11:49:13 -0500 Subject: [PATCH 38/43] fix: apply the superblock extension and cache image in every opener File, MmapFile and LazyFile decoded the superblock extension and laid a metadata cache image over the file's metadata; the other readers did not, so the same file read differently by entry point: NativeVol, AsyncHDF5File and MpiVol (clawhdf5-io) and the external source files of a virtual dataset (clawhdf5-format vds.rs) read a file with an image from its own bytes, which libhdf5 does not (they may be stale, or zeros: h5clear_mdc_image.h5 failed with InvalidObjectHeaderVersion(0)), and skipped the extension checks File::open makes (cve-2020-10810/10812). Each of them owns its buffer, so each now calls the shared superblock_ext::apply_cache_image_in_place, which checks the extension and writes the image's entries in place (only the image block is copied). These readers read whole datasets and cannot open a file and fail each object, so an image libhdf5 cannot load is refused with the image's error, never read around. clawhdf5-io's vol::load_hdf5 wraps it for NativeVol (at open; for from_bytes the error is reported on read, as a truncated file already was) and MpiVol. The MpiVol edit is minimal and was not compiled: the mpi-io feature needs an MPI installation this machine does not have (mpi-sys's build script panics). Tests: NativeVol (open_path and from_bytes), AsyncHDF5File and a VDS whose source file is h5clear_mdc_image.h5 (vds_interop.rs, against h5py) read the fixture's values; the corrupted-image variants are refused. Each fails without its fix. Co-Authored-By: Claude Opus 5.5 (1M context) --- CHANGELOG.md | 12 +++++ crates/clawhdf5-format/src/vds.rs | 23 +++++++- crates/clawhdf5-io/src/async_read.rs | 41 +++++++++++++++ crates/clawhdf5-io/src/mpi_vol.rs | 5 +- crates/clawhdf5-io/src/vol.rs | 78 ++++++++++++++++++++++++++-- crates/clawhdf5/tests/vds_interop.rs | 23 ++++++++ 6 files changed, 176 insertions(+), 6 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d256a15..8912fed 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -34,6 +34,18 @@ ZFP filters; the 2 mismatches are the known h5py big-endian VL bug. files as agreeing with h5py that the library did not open; probe and library now take the decision from the same `superblock_ext::cache_image_state`. +- **Every other opener applies the superblock extension and the cache + image too** (`superblock_ext::apply_cache_image_in_place`, writing into + the buffer each already owns): `clawhdf5_io`'s `NativeVol` (at `open`, + and on read for `from_bytes`), `AsyncHDF5File`, `MpiVol` (a minimal edit + through the same `vol::load_hdf5`; the `mpi-io` feature cannot be built + without an MPI installation, so it was not compiled), and the external + source files of a virtual dataset. They read a file with an image from + its own bytes — stale metadata, or none (`h5clear_mdc_image.h5` failed + with `InvalidObjectHeaderVersion(0)`) — and skipped the extension checks + `File::open` makes. These readers cannot open a file and fail each + object, so an image libhdf5 cannot load is refused with the image's + error. - **The superblock extension is decoded at open, as libhdf5 does:** a File Space Info or Metadata Cache Image message libhdf5 cannot decode makes the open fail (`cve-2020-10810`, `cve-2020-10812` were opened). diff --git a/crates/clawhdf5-format/src/vds.rs b/crates/clawhdf5-format/src/vds.rs index f5d5237..d67b832 100644 --- a/crates/clawhdf5-format/src/vds.rs +++ b/crates/clawhdf5-format/src/vds.rs @@ -803,7 +803,11 @@ impl<'a, 'r> Sources<'a, 'r> { let resolver = self.resolver.ok_or_else(|| { vds_err("external-file virtual dataset sources require a file resolver") })?; - self.cached_file = Some((String::from(name), resolver(name)?)); + let mut bytes = resolver(name)?; + if let Some(b) = bytes.as_mut() { + load_source_file(b)?; + } + self.cached_file = Some((String::from(name), bytes)); } // An external file is handed over whole; its addresses are relative // to its superblock, so skip any user block. @@ -851,6 +855,23 @@ impl<'a, 'r> Sources<'a, 'r> { } } +/// Check an external source file's superblock extension as libhdf5 does +/// when it opens the file, and write any metadata cache image over its +/// metadata in place: libhdf5 reads the image's entries instead of the +/// file's own, possibly stale, bytes (`crate::superblock_ext`). A source +/// file whose image cannot be loaded is an error, as other corrupt source +/// files are here. +fn load_source_file(whole: &mut [u8]) -> Result<(), FormatError> { + let base = crate::signature::find_signature(whole)?; + let sb = crate::superblock::Superblock::parse(&whole[base..], 0)?; + // The end of file the superblock records; a truncated source file is + // read as before, up to its length. + let end = sb + .data_end(base as u64, whole.len() as u64) + .map_or(whole.len(), |e| base + e as usize); + crate::superblock_ext::apply_cache_image_in_place(&mut whole[base..end], &sb) +} + /// Whether elements of `dt` contain addresses into their own file: /// variable-length data (global-heap IDs) or references. fn holds_file_addresses(dt: &Datatype) -> bool { diff --git a/crates/clawhdf5-io/src/async_read.rs b/crates/clawhdf5-io/src/async_read.rs index 30969fa..f5277c0 100644 --- a/crates/clawhdf5-io/src/async_read.rs +++ b/crates/clawhdf5-io/src/async_read.rs @@ -288,6 +288,12 @@ impl AsyncHDF5File { // superblock records, as libhdf5 does. let end = superblock.data_end(user_block as u64, whole_len)?; data.truncate(end as usize); + // Check the superblock extension as libhdf5 does at open, and write + // any metadata cache image over the file's metadata (libhdf5 reads + // the image's entries instead of the file's own, possibly stale, + // bytes). An image libhdf5 cannot load is refused: this reader has + // no way to open the file and fail each object instead. + clawhdf5_format::superblock_ext::apply_cache_image_in_place(&mut data, &superblock)?; Ok(Self { data, superblock }) } @@ -430,6 +436,41 @@ mod tests { fw.finish().unwrap() } + /// libhdf5's `h5clear_mdc_image.h5` (see the clawhdf5 crate's + /// `tests/metadata_cache_image.rs`): the root group's header exists + /// only in the file's metadata cache image. + fn cache_image_fixture() -> Vec { + std::fs::read(concat!( + env!("CARGO_MANIFEST_DIR"), + "/../clawhdf5/tests/fixtures/h5clear_mdc_image.h5" + )) + .unwrap() + } + + #[tokio::test] + async fn reads_through_a_metadata_cache_image() { + let bytes = cache_image_fixture(); + let file = AsyncHDF5File::from_bytes(bytes.clone()).unwrap(); + let info = file.read_dataset_raw("DSET").await.unwrap(); + assert_eq!(info.shape, [50, 100]); + let values: Vec = info + .raw + .chunks_exact(4) + .map(|b| i32::from_le_bytes(b.try_into().unwrap())) + .collect(); + let expected: Vec = (0..50).flat_map(|i| (0..100).map(move |j| i * j)).collect(); + assert_eq!(values, expected); + + // An image libhdf5 cannot load is refused at open. + let mut bad = bytes; + let at = bad.windows(4).position(|w| w == b"MDCI").unwrap(); + bad[at] = b'X'; + assert!(matches!( + AsyncHDF5File::from_bytes(bad), + Err(AsyncHDF5Error::Format(FormatError::InvalidCacheImage(_))) + )); + } + // --- AsyncMemoryReader tests --- #[tokio::test] diff --git a/crates/clawhdf5-io/src/mpi_vol.rs b/crates/clawhdf5-io/src/mpi_vol.rs index 87a82ea..fdf4d56 100644 --- a/crates/clawhdf5-io/src/mpi_vol.rs +++ b/crates/clawhdf5-io/src/mpi_vol.rs @@ -191,7 +191,10 @@ fn mpi_collective_read(vol: &MpiVol, location: &str, path: &str) -> Result>, location: Option, + /// Why the bytes given to [`NativeVol::from_bytes`] cannot be read + /// (what `open` would have refused them for). + load_error: Option, } /// The HDF5 bytes of a whole file and its superblock: from the superblock @@ -228,12 +231,35 @@ pub(crate) fn hdf5_view( Ok((&data[..end as usize], sb)) } +/// Check a whole file as libhdf5 does when it opens it ([`hdf5_view`], and +/// the superblock extension), and write any metadata cache image over the +/// file's metadata in place: libhdf5 reads the image's entries instead of +/// the file's own bytes at their addresses, which may be stale +/// (`clawhdf5_format::superblock_ext`). A file whose image libhdf5 cannot +/// load is refused: a connector that reads whole datasets has no way to +/// open the file and fail each object instead. +pub(crate) fn load_hdf5(whole: &mut [u8]) -> Result<(), VolError> { + use clawhdf5_format::superblock_ext::apply_cache_image_in_place; + let err = |e: clawhdf5_format::error::FormatError| VolError::DataError(e.to_string()); + let (len, sb) = { + let (data, sb) = hdf5_view(whole)?; + (data.len(), sb) + }; + // hdf5_view's bytes start at the superblock, after any user block. + let base = clawhdf5_format::signature::split_user_block(whole) + .map_err(err)? + .0 + .len(); + apply_cache_image_in_place(&mut whole[base..base + len], &sb).map_err(err) +} + impl NativeVol { /// Create a new native VOL connector. pub fn new() -> Self { Self { data: None, location: None, + load_error: None, } } @@ -245,10 +271,12 @@ impl NativeVol { } /// Create a native VOL connector from bytes already in memory. - pub fn from_bytes(data: Vec) -> Self { + pub fn from_bytes(mut data: Vec) -> Self { + let load_error = load_hdf5(&mut data).err().map(|e| e.to_string()); Self { data: Some(data), location: Some("".into()), + load_error, } } @@ -281,10 +309,12 @@ impl VirtualObjectLayer for NativeVol { } fn open(&mut self, location: &str) -> Result<(), VolError> { - let data = std::fs::read(location)?; - // Refuse a truncated file at open, as libhdf5 does. - hdf5_view(&data)?; + let mut data = std::fs::read(location)?; + // Refuse a truncated file at open, as libhdf5 does, and load any + // metadata cache image. + load_hdf5(&mut data)?; self.data = Some(data); + self.load_error = None; self.location = Some(location.to_string()); Ok(()) } @@ -299,6 +329,9 @@ impl VirtualObjectLayer for NativeVol { let data = self.data.as_ref().ok_or_else(|| { VolError::Io(io::Error::new(io::ErrorKind::NotConnected, "file not open")) })?; + if let Some(e) = &self.load_error { + return Err(VolError::DataError(e.clone())); + } use clawhdf5_format::{ data_layout::DataLayout, data_read::read_raw_data_full, dataspace::Dataspace, @@ -434,6 +467,43 @@ mod tests { assert_eq!(raw.len(), 24); } + /// libhdf5's `h5clear_mdc_image.h5` (see the clawhdf5 crate's + /// `tests/metadata_cache_image.rs`): the root group's header exists + /// only in the file's metadata cache image, so reading the file's own + /// bytes finds zeros there. + #[test] + fn native_vol_reads_through_a_metadata_cache_image() { + let path = concat!( + env!("CARGO_MANIFEST_DIR"), + "/../clawhdf5/tests/fixtures/h5clear_mdc_image.h5" + ); + let expected: Vec = (0..50) + .flat_map(|i| (0..100).map(move |j| i * j)) + .flat_map(i32::to_le_bytes) + .collect(); + let vol = NativeVol::open_path(path).unwrap(); + assert_eq!(vol.read_dataset("DSET").unwrap(), expected); + let bytes = std::fs::read(path).unwrap(); + let vol = NativeVol::from_bytes(bytes.clone()); + assert_eq!(vol.read_dataset("DSET").unwrap(), expected); + + // An image libhdf5 cannot load is refused, not read around. + let mut bad = bytes; + let at = bad.windows(4).position(|w| w == b"MDCI").unwrap(); + bad[at] = b'X'; + let err = NativeVol::from_bytes(bad.clone()) + .read_dataset("DSET") + .unwrap_err(); + assert!(err.to_string().contains("cache image"), "{err}"); + let dir = tempfile::tempdir().unwrap(); + let bad_path = dir.path().join("bad_image.h5"); + std::fs::write(&bad_path, &bad).unwrap(); + let err = NativeVol::open_path(bad_path.to_str().unwrap()) + .err() + .unwrap(); + assert!(err.to_string().contains("cache image"), "{err}"); + } + #[test] fn vol_error_display() { let err = VolError::Unsupported("read_dataset".into()); diff --git a/crates/clawhdf5/tests/vds_interop.rs b/crates/clawhdf5/tests/vds_interop.rs index 17069b0..71e512e 100644 --- a/crates/clawhdf5/tests/vds_interop.rs +++ b/crates/clawhdf5/tests/vds_interop.rs @@ -486,3 +486,26 @@ fn vds_libhdf5_test_files() { vec![5, 10, 10] ); } + +/// A source file with a metadata cache image (libhdf5's +/// `h5clear_mdc_image.h5`, whose root group's header exists only in the +/// image) is read through the image, as libhdf5 opens it. Source files were +/// read from their own bytes, and this one failed on the root group. +#[test] +fn vds_source_file_with_a_metadata_cache_image() { + skip_if_no_python!(); + let dir = tempfile::tempdir().unwrap(); + let fixture = Path::new(env!("CARGO_MANIFEST_DIR")).join("tests/fixtures/h5clear_mdc_image.h5"); + std::fs::copy(&fixture, dir.path().join("src.h5")).unwrap(); + generate( + dir.path(), + r#" +with h5py.File("vds.h5", "w", libver="latest") as f: + lay = h5py.VirtualLayout(shape=(3, 100), dtype="i4") + lay[0:3, :] = h5py.VirtualSource("src.h5", "DSET", shape=(50, 100))[5:8, :] + f.create_virtual_dataset("v", lay) +expect("vds.h5", "v", "image_source") +"#, + ); + assert_matches_libhdf5(dir.path(), "vds.h5", "v", "image_source"); +} From 4c01267b76c05fc36a8395d9df6f1155f74efc69 Mon Sep 17 00:00:00 2001 From: osobh Date: Sat, 26 Sep 2026 11:51:20 -0500 Subject: [PATCH 39/43] test: every scale-offset dataset h5py writes reads as h5py reads it The scale-offset fix (d110b1d) was covered only by unit vectors from CVE chunks, and documented as three corner cases. The review found it is much bigger: of 1480 scale-offset datasets h5py writes (every integer type i1..u8, f4 and f8, both byte orders, with and without a fill value, scaleoffset 0..full width), v2.7.0's decoder read 332 differently from h5py: 151 returned wrong values with no error (82 integer datasets with scaleoffset=0 and a wide range, 51 full-width i4/u4/i8/u8, 18 f4 D-scale datasets with a large range) and 181 failed to read. The cause in every case is a chunk libhdf5 stores at full width, whose elements were decoded as offsets from minval. tests/scaleoffset_interop.rs generates that matrix with h5py at test time, stores h5py's decoded values uncompressed next to it, and compares every dataset's bytes. It passes on this branch; with the filters.rs before d110b1d it reports "332 of 1480 scale-offset datasets differ from h5py". CHANGELOG: a Correctness entry stating this was silent wrong data in every release that decoded scale-offset (v2.2.0 to v2.7.0), replacing the corner-case wording. docs/known-issues.md: a fixed entry with the affected cases. Co-Authored-By: Claude Opus 5.5 (1M context) --- CHANGELOG.md | 32 +++- crates/clawhdf5/tests/scaleoffset_interop.rs | 151 +++++++++++++++++++ docs/known-issues.md | 40 ++++- 3 files changed, 216 insertions(+), 7 deletions(-) create mode 100644 crates/clawhdf5/tests/scaleoffset_interop.rs diff --git a/CHANGELOG.md b/CHANGELOG.md index 8912fed..253e9a0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -61,12 +61,10 @@ ZFP filters; the 2 mismatches are the known h5py big-endian VL bug. - a simple dataspace of rank 0 holds one element (it held 0; `cve-2020-18494`), and contiguous storage larger than the dataset reads (`cve-2024-32623`, `cve-2025-2309`; libhdf5 ignores the excess); - - scale-offset: the packed codes start at byte 21 whatever size the chunk - records for `minval`, a chunk with `minbits` 0 and a fill value is all - fill values, full-width `minbits` stores the elements as they are - (these decoded differently from libhdf5); E-scale is refused, as in - libhdf5; codes past the end of the chunk stay an error - (`cve-2025-2308`, where HDF5 2.0 reads past its buffer); + - scale-offset returned wrong values for ordinary h5py files — see + *Correctness* below; E-scale is refused, as in libhdf5; codes past the + end of the chunk stay an error (`cve-2025-2308`, where HDF5 2.0 reads + past its buffer); - shuffle uses its own parameter as the element size, as libhdf5 does (`cve-2025-44905`); - an unfiltered chunk the index records at other than the chunk's size is @@ -878,6 +876,28 @@ ZFP filters; the 2 mismatches are the known h5py big-endian VL bug. - CI keeps zlib-ng building and tested; the arm64 job no longer needs cmake. ### Correctness +- **Scale-offset data read wrong values in every release that decoded it + (v2.2.0 to v2.7.0), silently, on ordinary h5py files** (fixed + 2026-09-26). Of 1480 scale-offset datasets h5py writes across every + integer type (`i1` .. `u8`), `f4` and `f8`, both byte orders, with and + without a fill value, and `scaleoffset` from 0 to the full width, 332 did + not read as h5py reads them: **151 returned wrong values with no error** + and 181 failed to read. The common cause was a chunk libhdf5 stores at + full width (`minbits` equal to the type's width), which it does for any + full-width `scaleoffset` and on its own whenever a chunk's values span + most of the type's range: `scaleoffset=0` integer data with a wide range + (82 datasets, all wrong values), full-width `u4`/`i4`/`u8`/`i8` (51 wrong + values; the narrower types and the rest failed with "truncated minval" or + "implausible minbits"), and `f4` D-scale data with a large range (18, + wrong values). Such a chunk holds the elements as they are; they were + decoded as offsets from `minval`. Also fixed, found on crafted files: the + packed codes start at byte 21 whatever size the chunk records for + `minval` (`cve-2025-44905` `/Scale_offset_short_data_be`), and a chunk + with `minbits` 0 and a fill value is all fill values (it read as + `minval`). The whole matrix is now an interop test + (`crates/clawhdf5/tests/scaleoffset_interop.rs`, generated by h5py at + test time, every dataset compared); on v2.7.0's decoder it reports the + 332. See `docs/known-issues.md`. - **Corrupt files libhdf5 refuses are now refused instead of read.** On the HDF Group's CVE reproducers, 18 objects that libhdf5 (HDF5 2.0, through h5py) refuses to open were read by clawhdf5, some as wrong data (a chunk diff --git a/crates/clawhdf5/tests/scaleoffset_interop.rs b/crates/clawhdf5/tests/scaleoffset_interop.rs new file mode 100644 index 0000000..d31c0db --- /dev/null +++ b/crates/clawhdf5/tests/scaleoffset_interop.rs @@ -0,0 +1,151 @@ +//! Scale-offset datasets written by h5py (libhdf5) read exactly as h5py +//! reads them. +//! +//! h5py writes the whole matrix the filter has: every integer type (`i1` .. +//! `u8`) and `f4`/`f8`, both byte orders, with and without a fill value +//! (the type's minimum, maximum, or another value), with random, constant, +//! all-fill and extreme data, and every interesting `scaleoffset` setting +//! (0 = let libhdf5 choose the bits, a few bits, full width less one, full +//! width; decimal scale factors 0..7 for floats) — about 1480 datasets. For +//! each one h5py's decoded values are stored uncompressed next to it, and +//! the raw bytes clawhdf5 decodes must equal them. +//! +//! Until 2026-09-26 clawhdf5 silently returned wrong values for 332 of +//! these, in every release that decoded scale-offset: ordinary `u8`, `u4`, +//! `i8` and `f4` data with a wide range (where libhdf5 stores the elements +//! as they are, at full width), chunks whose `minval` field was recorded at +//! another size than 8 bytes, and chunks with `minbits` 0 and a fill value. +//! +//! Skipped when python3 with h5py is unavailable, unless +//! `CLAWHDF5_REQUIRE_INTEROP=1`. + +use std::process::Command; + +use clawhdf5::File; + +fn python() -> String { + std::env::var("CLAWHDF5_PYTHON").unwrap_or_else(|_| "python3".to_string()) +} + +fn python_available() -> bool { + Command::new(python()) + .args(["-c", "import h5py, numpy"]) + .output() + .map(|o| o.status.success()) + .unwrap_or(false) +} + +const GENERATE: &str = r#" +import sys, h5py, numpy as np +rng = np.random.default_rng(7) +out, expect = sys.argv[1], sys.argv[2] +made = [] +with h5py.File(out, "w") as f: + def make(name, a, so, fv, desc): + try: + d = f.create_dataset(name, data=a, chunks=(16,), scaleoffset=so, fillvalue=fv) + except Exception: + return # a combination libhdf5 refuses to write + d.attrs["desc"] = desc + made.append(name) + i = 0 + for dt in ["i1", "u1", "i2", "u2", "i4", "u4", "i8", "u8"]: + for bo in "<>": + t = np.dtype(bo + dt) + info = np.iinfo(t) + for so in [0, 1, 3, 8 * t.itemsize - 1, 8 * t.itemsize]: + for fill in [None, "min", "max", "mid"]: + for pat in ["rand", "const", "allfill", "extreme"]: + n = 64 + fv = {None: None, "min": info.min, "max": info.max, "mid": t.type(7)}[fill] + if pat == "rand": + lo = max(info.min, -50) if so else info.min + hi = min(info.max, 50) if so else info.max + a = rng.integers(lo, hi, size=n, endpoint=True, + dtype=np.int64 if t.kind == "i" else np.uint64).astype(t) + elif pat == "const": + a = np.full(n, 3, t) + elif pat == "extreme": + a = np.array([info.min, info.max] * (n // 2), t) + else: + if fv is None: + continue + a = np.full(n, fv, t) + make(f"d{i}", a, so, fv, f"{bo}{dt} so={so} fill={fill} pat={pat}") + i += 1 + for dt in ["f4", "f8"]: + for bo in "<>": + t = np.dtype(bo + dt) + for so in [0, 1, 2, 4, 7]: + for fill in [None, -1.5, 0.0]: + for pat in ["rand", "const", "allfill", "neg", "big"]: + n = 50 + if pat == "rand": + a = rng.normal(size=n).astype(t) * 10 + elif pat == "const": + a = np.full(n, 2.25, t) + elif pat == "neg": + a = -np.abs(rng.normal(size=n)).astype(t) * 1000 + elif pat == "big": + a = rng.normal(size=n).astype(t) * 1e6 + else: + if fill is None: + continue + a = np.full(n, fill, t) + make(f"d{i}", a, so, fill, f"{bo}{dt} so={so} fill={fill} pat={pat}") + i += 1 +# What libhdf5 decodes, stored uncompressed in the same datatype. +with h5py.File(out, "r") as f, h5py.File(expect, "w") as e: + for name in made: + e.create_dataset(name, data=f[name][()]) +print(len(made)) +"#; + +#[test] +fn every_scale_offset_dataset_reads_as_h5py_reads_it() { + if !python_available() { + assert!( + !std::env::var("CLAWHDF5_REQUIRE_INTEROP").is_ok_and(|v| v == "1"), + "CLAWHDF5_REQUIRE_INTEROP=1 but python3 with h5py is not available" + ); + eprintln!("SKIP: python3 with h5py not available"); + return; + } + let dir = tempfile::tempdir().unwrap(); + let written = dir.path().join("scaleoffset.h5"); + let expected = dir.path().join("expected.h5"); + let out = Command::new(python()) + .args(["-c", GENERATE]) + .arg(&written) + .arg(&expected) + .output() + .expect("failed to run python"); + assert!( + out.status.success(), + "python failed:\n{}", + String::from_utf8_lossy(&out.stderr) + ); + let count: usize = String::from_utf8_lossy(&out.stdout).trim().parse().unwrap(); + assert!(count > 1400, "only {count} datasets written"); + + let file = File::open(&written).unwrap(); + let reference = File::open(&expected).unwrap(); + let mut names = reference.root().datasets().unwrap(); + names.sort(); + assert_eq!(names.len(), count); + let mut wrong = Vec::new(); + for name in &names { + let want = reference.read_multi(&[name]).unwrap().remove(0); + let got = file.read_multi(&[name]).map(|mut v| v.remove(0)); + if got.as_ref().ok() != Some(&want) { + let desc = file.dataset(name).unwrap().attrs().unwrap().remove("desc"); + wrong.push(format!("{name} {desc:?}: {:?}", got.map(|g| g.len()))); + } + } + assert!( + wrong.is_empty(), + "{} of {count} scale-offset datasets differ from h5py:\n{}", + wrong.len(), + wrong.join("\n") + ); +} diff --git a/docs/known-issues.md b/docs/known-issues.md index 8328f14..5d1aac7 100644 --- a/docs/known-issues.md +++ b/docs/known-issues.md @@ -63,6 +63,43 @@ tank with `concurrent_read` against h5py 3.16 / HDF5 2.0 (`BENCHMARKS.md`, `CHANGELOG.md`). The chunked-read scaling item above is still open. Values are correct; this is speed only. +## Scale-offset data read back wrong values + +**Status:** fixed 2026-09-26, after v2.7.0. **Every release that decoded +the scale-offset filter (v2.2.0 to v2.7.0) is affected**, on ordinary +files h5py writes, with no error. + +Found by the review of the 2026-09-26 conformance work: h5py wrote 1480 +scale-offset datasets (every integer type `i1` .. `u8`, `f4` and `f8`, +little- and big-endian, with no fill value and with the type's minimum, +maximum or another value as fill, random, constant, all-fill and extreme +data, `scaleoffset` 0, 1, 3, full width less one and full width for +integers, decimal scale factors 0, 1, 2, 4 and 7 for floats). v2.7.0's +decoder read 332 of them differently from h5py: **151 returned wrong +values with no error**, 181 failed to read. + +| Case | Datasets | v2.7.0 | +|---|---|---| +| integer, `scaleoffset=0` (libhdf5 picks the bits), data spanning most of the type's range | 82 | wrong values | +| integer, `scaleoffset` = full width: `i4`/`u4` (10), `i8`/`u8` (41) | 51 | wrong values | +| integer, `scaleoffset` = full width, the other datasets (every `i1`..`u2` one, most `i4`/`u4`, some `i8`/`u8`) | 181 | "truncated minval" / "implausible minbits" | +| `f4` D-scale, factor 4 or 7, values up to about 10^6 | 18 | wrong values | + +In every case libhdf5 stored a chunk at full width (`minbits` equal to the +type's width): the chunk then holds the elements as they are, and they +were decoded as offsets from `minval`. Two more differences were found on +crafted files and fixed with them: the packed codes start at byte 21 +whatever size the chunk records for `minval` (`cve-2025-44905` +`/Scale_offset_short_data_be`), and a chunk with `minbits` 0 and a fill +value is all fill values (it read as `minval`). + +**Fix:** `clawhdf5_format::filters` decodes a scale-offset chunk as +`H5Z__filter_scaleoffset` does. **Test:** the whole matrix is +`crates/clawhdf5/tests/scaleoffset_interop.rs`, generated by h5py at test +time and compared dataset by dataset; on v2.7.0's decoder it reports the +332. **Existing data:** the files were always right; only reads were +wrong, so re-reading with a fixed build gives the correct values. + ## Silent wrong data found by the 2026-09-25 HDF5 audit **Status:** fixed after v2.7.0 (2026-09-25). **Every release up @@ -230,7 +267,8 @@ fill-value item that did is fixed). under *Known not-our-bug*. Scale-offset did decode three cases differently from libhdf5 (codes after a `minval` of recorded size other than 8, `minbits` 0 with a fill value, full-width `minbits`): fixed - 2026-09-26. + 2026-09-26, and the full-width case was silent wrong data on ordinary + h5py files (see *Scale-offset data read back wrong values* above). - **Filters:** blosc, blosc2, bitshuffle, bzip2, LZF and zfp are not implemented. **Fixed 2026-09-26** for LZF (default-on `lzf` feature), bitshuffle, bzip2 and Blosc 1 (`bitshuffle`, `bzip2`, `blosc`, or From 00f94d57eddddeed8736259e40a86fbac52efdcd Mon Sep 17 00:00:00 2001 From: osobh Date: Sat, 26 Sep 2026 11:52:01 -0500 Subject: [PATCH 40/43] test(io): read the cache-image fixture's values with as_chunks clippy (with the async feature) flags chunks_exact with a constant size in the AsyncHDF5File cache-image test added in d493d47. Co-Authored-By: Claude Opus 5.5 (1M context) --- crates/clawhdf5-io/src/async_read.rs | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/crates/clawhdf5-io/src/async_read.rs b/crates/clawhdf5-io/src/async_read.rs index f5277c0..fa87355 100644 --- a/crates/clawhdf5-io/src/async_read.rs +++ b/crates/clawhdf5-io/src/async_read.rs @@ -455,8 +455,10 @@ mod tests { assert_eq!(info.shape, [50, 100]); let values: Vec = info .raw - .chunks_exact(4) - .map(|b| i32::from_le_bytes(b.try_into().unwrap())) + .as_chunks::<4>() + .0 + .iter() + .map(|&b| i32::from_le_bytes(b)) .collect(); let expected: Vec = (0..50).flat_map(|i| (0..100).map(move |j| i * j)).collect(); assert_eq!(values, expected); From 55e0e7e9cf2ede678122fb5a08a783c66ba5ae9b Mon Sep 17 00:00:00 2001 From: osobh Date: Sat, 26 Sep 2026 11:53:46 -0500 Subject: [PATCH 41/43] docs: conformance numbers after the review fixes (598 of 697 ok) cve-2025-44905 now reads as h5py reads it (the v1 chunk B-tree lookup), leaving 5 our-errors: cve-2025-2308, cve-2025-44904 and bad_nbit_parms_walk (corrupt data HDF5 2.0 reads through a bug), and the Blosc2 and ZFP filters. The five unloadable-cache-image files stay ok, now with the library behaving as the probe reports. Co-Authored-By: Claude Opus 5.5 (1M context) --- CHANGELOG.md | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 253e9a0..da05b6a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,10 +3,13 @@ ## Unreleased ### Remaining conformance errors (2026-09-26) -Conformance on tank, `conformance/run.sh --no-fetch`: 597 of 697 files -ok (575 before). Of the 6 our-errors left, 4 are corrupt data HDF5 2.0 +Conformance on tank, `conformance/run.sh --no-fetch`: 598 of 697 files +ok (575 before). Of the 5 our-errors left, 3 are corrupt data HDF5 2.0 reads only through a bug (listed in `CONFORMANCE.md`), 2 are the Blosc2 and -ZFP filters; the 2 mismatches are the known h5py big-endian VL bug. +ZFP filters; the 2 mismatches are the known h5py big-endian VL bug. The +five files whose cache image libhdf5 cannot load (`cve-2025-6269-*`, +`cve-2025-6516`) count as ok because the library, like libhdf5, opens them +and fails their objects (see below). - **Metadata cache images are read.** A file written with a metadata cache image keeps its metadata cache entries in an image block the superblock extension points at, and libhdf5 reads them in place of the file's own From c5334b1c97e59558b8537707dd06e6cbba6a91e5 Mon Sep 17 00:00:00 2001 From: osobh Date: Sat, 26 Sep 2026 12:10:57 -0500 Subject: [PATCH 42/43] docs: conformance report after this batch (599 of 697 ok) Regenerated on tank: ok 575 -> 599, our-error 10 -> 4, mismatch 20 -> 2, no panics, hangs, crashes or OOM. Newly ok: metadata-cache-image files, the CVE open-time checks, v1 chunk-key lookup as libhdf5 does it, the scale-offset fixes, Blosc2, and harness corrections. Baseline raised. h5rs check --data flags none of the 435 fully-read ok files. Co-Authored-By: Claude Opus 5.5 (1M context) --- CONFORMANCE.md | 108 ++++++++++++++++++-------------------- conformance/baseline.json | 47 ++++++++++++----- 2 files changed, 86 insertions(+), 69 deletions(-) diff --git a/CONFORMANCE.md b/CONFORMANCE.md index 07ce59b..1a8e2b7 100644 --- a/CONFORMANCE.md +++ b/CONFORMANCE.md @@ -13,15 +13,15 @@ fatal. This file is generated by `conformance/run.sh`; do not edit it by hand. | | | |---|---| -| date | 2026-09-26 14:18 UTC | -| clawhdf5 commit | `73a01f1256fb9bf1b1e7601f755af9e8273cec4e` | +| date | 2026-09-26 17:10 UTC | +| clawhdf5 commit | `d0e3beb3aa8290aae523ce280b4380e75484b6bc` | | machine | `tank`: AMD Ryzen 7 7800X3D 8-Core Processor, 16 CPUs, 61 GiB, Linux 7.0.0-34-generic x86_64 | | command | `conformance/run.sh --no-fetch --update-baseline` | | rustc | rustc 1.98.1 (48a229cea 2026-09-01) | | reference | h5py 3.16.0, HDF5 2.0.0, numpy 2.5.3, hdf5plugin 7.1.0, Python 3.14.4 | | h5dump | Version 1.14.6 (CVE corpus only) | | limits | 20 s timeout (SIGKILL), 4096 MiB address space, per process; 16 files in parallel | -| runtime | 23 s probing + comparing (0 s fetch/build before it) | +| runtime | 24 s probing + comparing (0 s fetch/build before it) | ## Results @@ -36,16 +36,18 @@ A file's class is the first that applies: | corpus | files | ok | our-error | mismatch | h5py-cannot-read | panic | hang | crash | oom | |---|---|---|---|---|---|---|---|---|---| | NCAS-CMS_pyfive | 33 | 32 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | -| cve_hdf5 | 147 | 100 | 6 | 9 | 32 | 0 | 0 | 0 | 0 | +| cve_hdf5 | 147 | 113 | 2 | 0 | 32 | 0 | 0 | 0 | 0 | | h5py_data | 4 | 4 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| hdf5 | 466 | 392 | 4 | 10 | 60 | 0 | 0 | 0 | 0 | +| hdf5 | 466 | 403 | 2 | 1 | 60 | 0 | 0 | 0 | 0 | | netcdf-c | 20 | 20 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | | netcdf4-python | 18 | 18 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | | usnistgov_h5wasm | 5 | 5 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | | xarray-data | 4 | 4 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| **all** | **697** | **575** | **10** | **20** | **92** | **0** | **0** | **0** | **0** | +| **all** | **697** | **599** | **4** | **2** | **92** | **0** | **0** | **0** | **0** | -2 of the 20 mismatches are a known h5py bug, not ours (see *Known not-our-bug*). +2 of the 2 mismatches are a known h5py bug, not ours (see *Known not-our-bug*). + +3 of the 4 our-errors are corrupt data that HDF5 2.0 reads only through a bug and clawhdf5 refuses (see *Known not-our-bug*). Corpora (fetched by `conformance/fetch-corpus.sh` into the gitignored `conformance/.cache/`): @@ -70,26 +72,14 @@ Grouped by normalised error message. *files* counts files whose class this cause | files | objects | error | examples | |---:|---:|---|---| -| 3 | 3 | `DataSizeMismatch { expected: N, actual: N }` | `cve_hdf5/cvefiles/cve-2020-18494.h5`, `cve_hdf5/cvefiles/cve-2024-32623.h5`, `cve_hdf5/cvefiles/cve-2025-2309.h5` | -| 2 | 2 | `ChunkedReadError("…")` | `cve_hdf5/cvefiles/cve-2025-2308.h5`, `hdf5/test/testfiles/bad_nbit_parms_walk.h5` | -| 2 | 2 | `UnsupportedFilter(N)` | `hdf5/HDF5Examples/C/H5FLT/tfiles/h5ex_d_blosc2.h5`, `hdf5/HDF5Examples/C/H5FLT/tfiles/h5ex_d_zfp.h5` | -| 1 | 1 | `UnexpectedEof { expected: N, available: N }` | `cve_hdf5/cvefiles/cve-2019-9151.h5` | -| 1 | 1 | `MissingMessage(Dataspace)` | `cve_hdf5/cvefiles/cve-2024-33874.h5` | -| 1 | 1 | `InvalidObjectHeaderVersion(N)` | `hdf5/tools/test/testfiles/h5clear_mdc_image.h5` | +| 3 | 3 | `ChunkedReadError("…")` | `cve_hdf5/cvefiles/cve-2025-2308.h5`, `cve_hdf5/cvefiles/cve-2025-44904.h5`, `hdf5/test/testfiles/bad_nbit_parms_walk.h5` | +| 1 | 1 | `UnsupportedFilter(N)` | `hdf5/HDF5Examples/C/H5FLT/tfiles/h5ex_d_zfp.h5` | ## Mismatch root causes | files | objects | cause | examples | |---:|---:|---|---| -| 13 | 14 | `missing-object` | `cve_hdf5/cvefiles/cve-2019-8397.h5`, `cve_hdf5/cvefiles/cve-2019-8398.h5`, `cve_hdf5/cvefiles/cve-2021-46243.h5` (+10 more) | -| 2 | 6 | `extra-attr` | `cve_hdf5/cvefiles/cve-2018-17438`, `cve_hdf5/cvefiles/cve-2018-17439` | | 1 | 1 | `attr-values: ours=vlen(>u8) h5py=object layout=- filters=-` | `NCAS-CMS_pyfive/tests/data/attr_datatypes.hdf5` | -| 1 | 4 | `extra-object` | `cve_hdf5/cvefiles/cve-2021-46244.h5` | -| 1 | 1 | `values: ours=i2 h5py=>i2 layout=chunked filters=[6]` | `cve_hdf5/cvefiles/cve-2025-44905.h5` | -| 1 | 1 | `values: ours=>f4 h5py=>f4 layout=chunked filters=[2]` | `cve_hdf5/cvefiles/cve-2025-44905.h5` | -| 1 | 1 | `values: ours=f4,i:>f4}8) h5py=object layout=contiguous filters=-` | `hdf5/tools/test/testfiles/tcomplex_be.h5` | ## CVE corpus: clawhdf5 vs h5dump vs h5py @@ -102,7 +92,7 @@ columns are. | tool | read | error | panic | crash | hang | oom | |---|---:|---:|---:|---:|---:|---:| -| clawhdf5 | 140 | 7 | 0 | 0 | 0 | 0 | +| clawhdf5 | 121 | 26 | 0 | 0 | 0 | 0 | | h5dump 1.14.6 | 16 | 129 | 0 | 2 | 0 | 0 | | h5py 3.16.0 / HDF5 2.0.0 | 115 | 31 | 0 | 1 | 0 | 0 | @@ -156,27 +146,27 @@ columns are. | cvefiles/cve-2018-17435.h5 | error exit | read 3 obj, 1 errors | read 3 obj, 1 errors | ok | | cvefiles/cve-2018-17436 | error exit | open error | open error | h5py-cannot-read | | cvefiles/cve-2018-17437.h5 | error exit | read 3 obj, 1 errors | read 3 obj, 1 errors | ok | -| cvefiles/cve-2018-17438 | error exit | read 3 obj, 1 errors | read 3 obj, 1 errors | mismatch | -| cvefiles/cve-2018-17439 | error exit | read 3 obj, 1 errors | read 3 obj, 1 errors | mismatch | +| cvefiles/cve-2018-17438 | error exit | read 3 obj, 1 errors | read 3 obj, 1 errors | ok | +| cvefiles/cve-2018-17439 | error exit | read 3 obj, 1 errors | read 3 obj, 1 errors | ok | | cvefiles/cve-2019-8396.h5 | error exit | read 3 obj, 2 errors | read 3 obj, 2 errors | ok | -| cvefiles/cve-2019-8397.h5 | error exit | read 3 obj, 2 errors | read 2 obj, 1 errors | mismatch | -| cvefiles/cve-2019-8398.h5 | error exit | read 3 obj, 2 errors | read 2 obj, 1 errors | mismatch | -| cvefiles/cve-2019-9151.h5 | error exit | read 3 obj, 1 errors | read 3 obj, 2 errors | our-error | +| cvefiles/cve-2019-8397.h5 | error exit | read 2 obj, 1 errors | read 2 obj, 1 errors | ok | +| cvefiles/cve-2019-8398.h5 | error exit | read 2 obj, 1 errors | read 2 obj, 1 errors | ok | +| cvefiles/cve-2019-9151.h5 | error exit | read 3 obj, 1 errors | read 3 obj, 2 errors | ok | | cvefiles/cve-2019-9152.h5 | error exit | read 3 obj, 1 errors | read 3 obj, 1 errors | ok | | cvefiles/cve-2020-10809 | error exit | open error | open error | h5py-cannot-read | -| cvefiles/cve-2020-10810.h5 | error exit | open error | read 2 obj | h5py-cannot-read | +| cvefiles/cve-2020-10810.h5 | error exit | open error | open error | h5py-cannot-read | | cvefiles/cve-2020-10811.h5 | error exit | read 25 obj, 1 errors | read 25 obj, 1 errors | ok | -| cvefiles/cve-2020-10812.h5 | error exit | open error | read 2 obj | h5py-cannot-read | +| cvefiles/cve-2020-10812.h5 | error exit | open error | open error | h5py-cannot-read | | cvefiles/cve-2020-18232.h5 | error exit | read 3 obj, 2 errors | read 3 obj, 2 errors | ok | -| cvefiles/cve-2020-18494.h5 | ok | read 2 obj | read 2 obj, 1 errors | our-error | +| cvefiles/cve-2020-18494.h5 | ok | read 2 obj | read 2 obj | ok | | cvefiles/cve-2021-36977.h5 | error exit | read 1 obj, 1 errors | read 1 obj, 1 errors | ok | | cvefiles/cve-2021-37501.h5 | error exit | read 18 obj, 1 errors | read 18 obj, 1 errors | ok | | cvefiles/cve-2021-45829.h5 | error exit | read 1 obj, 2 errors | read 1 obj, 1 errors | ok | -| cvefiles/cve-2021-45830.h5 | error exit | open error | read 1 obj, 1 errors | h5py-cannot-read | +| cvefiles/cve-2021-45830.h5 | error exit | open error | open error | h5py-cannot-read | | cvefiles/cve-2021-45833.h5 | error exit | read 2 obj, 1 errors | read 2 obj, 1 errors | ok | | cvefiles/cve-2021-46242.h5 | error exit | open error | read 1 obj, 1 errors | h5py-cannot-read | -| cvefiles/cve-2021-46243.h5 | error exit | read 3 obj, 2 errors | read 2 obj, 1 errors | mismatch | -| cvefiles/cve-2021-46244.h5 | error exit | read 2 obj, 1 errors | read 6 obj, 4 errors | mismatch | +| cvefiles/cve-2021-46243.h5 | error exit | read 2 obj, 1 errors | read 2 obj, 1 errors | ok | +| cvefiles/cve-2021-46244.h5 | error exit | read 2 obj, 1 errors | read 2 obj, 1 errors | ok | | cvefiles/cve-2024-29157.h5 | error exit | read 4 obj, 7 errors | read 4 obj, 7 errors | ok | | cvefiles/cve-2024-29158.h5 | ok | read 3 obj, 1 errors | read 3 obj, 1 errors | ok | | cvefiles/cve-2024-29159.h5 | error exit | read 2 obj, 1 errors | read 2 obj, 1 errors | ok | @@ -201,50 +191,50 @@ columns are. | cvefiles/cve-2024-32615.h5 | error exit | read 4 obj, 1 errors | read 4 obj, 1 errors | ok | | cvefiles/cve-2024-32616.h5 | error exit | read 10 obj, 7 errors | read 10 obj, 6 errors | ok | | cvefiles/cve-2024-32617.h5 | error exit | read 1 obj, 1 errors | read 1 obj, 1 errors | ok | -| cvefiles/cve-2024-32618.h5 | error exit | read 4 obj, 2 errors | read 3 obj, 1 errors | mismatch | +| cvefiles/cve-2024-32618.h5 | error exit | read 3 obj, 1 errors | read 3 obj, 1 errors | ok | | cvefiles/cve-2024-32619.h5 | error exit | read 3 obj, 2 errors | read 3 obj, 2 errors | ok | | cvefiles/cve-2024-32620.h5 | error exit | read 1 obj, 1 errors | read 1 obj, 1 errors | ok | | cvefiles/cve-2024-32621.h5 | ok | read 3 obj, 1 errors | read 3 obj, 1 errors | ok | | cvefiles/cve-2024-32622.h5 | ok | read 3 obj, 1 errors | read 3 obj, 1 errors | ok | -| cvefiles/cve-2024-32623.h5 | ok | read 6 obj | read 6 obj, 1 errors | our-error | -| cvefiles/cve-2024-32624.h5 | error exit | read 6 obj, 1 errors | read 6 obj | ok | +| cvefiles/cve-2024-32623.h5 | ok | read 6 obj | read 6 obj | ok | +| cvefiles/cve-2024-32624.h5 | error exit | read 6 obj, 1 errors | read 6 obj, 1 errors | ok | | cvefiles/cve-2024-33873.h5 | error exit | read 4 obj, 1 errors | read 4 obj, 1 errors | ok | -| cvefiles/cve-2024-33874.h5 | ok | read 6 obj, 1 errors | read 6 obj, 2 errors | our-error | +| cvefiles/cve-2024-33874.h5 | ok | read 6 obj, 1 errors | read 6 obj, 1 errors | ok | | cvefiles/cve-2024-33875.h5 | ok | read 2 obj | read 2 obj | ok | | cvefiles/cve-2024-33876.h5 | ok | read 3 obj, 1 errors | read 3 obj, 1 errors | ok | | cvefiles/cve-2024-33877.h5 | error exit | read 8 obj, 1 errors | read 8 obj, 1 errors | ok | | cvefiles/cve-2025-2153.h5 | error exit | open error | read 1 obj, 1 errors | h5py-cannot-read | | cvefiles/cve-2025-2308.h5 | error exit | read 25 obj, 1 errors | read 25 obj, 2 errors | our-error | -| cvefiles/cve-2025-2309.h5 | ok | read 6 obj, 1 errors | read 6 obj, 1 errors | our-error | +| cvefiles/cve-2025-2309.h5 | ok | read 6 obj, 1 errors | read 6 obj | ok | | cvefiles/cve-2025-2310.h5 | error exit | read 24 obj, 8 errors | read 24 obj, 8 errors | ok | -| cvefiles/cve-2025-2912.h5 | error exit | open error | read 1 obj, 1 errors | h5py-cannot-read | -| cvefiles/cve-2025-2913.h5 | error exit | open error | read 1 obj, 1 errors | h5py-cannot-read | -| cvefiles/cve-2025-2914.h5 | error exit | open error | read 1 obj, 1 errors | h5py-cannot-read | -| cvefiles/cve-2025-2915.h5 | error exit | open error | read 1 obj, 1 errors | h5py-cannot-read | -| cvefiles/cve-2025-2923.h5 | error exit | open error | read 1 obj, 1 errors | h5py-cannot-read | +| cvefiles/cve-2025-2912.h5 | error exit | open error | open error | h5py-cannot-read | +| cvefiles/cve-2025-2913.h5 | error exit | open error | open error | h5py-cannot-read | +| cvefiles/cve-2025-2914.h5 | error exit | open error | open error | h5py-cannot-read | +| cvefiles/cve-2025-2915.h5 | error exit | open error | open error | h5py-cannot-read | +| cvefiles/cve-2025-2923.h5 | error exit | open error | open error | h5py-cannot-read | | cvefiles/cve-2025-2924.h5 | error exit | read 1 obj, 1 errors | read 1 obj, 1 errors | ok | | cvefiles/cve-2025-2925.h5 | error exit | read 1 obj, 1 errors | read 1 obj, 1 errors | ok | -| cvefiles/cve-2025-2926.h5 | error exit | open error | read 1 obj, 1 errors | h5py-cannot-read | -| cvefiles/cve-2025-44904.h5 | error exit | read 25 obj, 1 errors | read 25 obj, 1 errors | mismatch | -| cvefiles/cve-2025-44905.h5 | error exit | read 25 obj, 3 errors | read 25 obj, 3 errors | mismatch | +| cvefiles/cve-2025-2926.h5 | error exit | open error | open error | h5py-cannot-read | +| cvefiles/cve-2025-44904.h5 | error exit | read 25 obj, 1 errors | read 25 obj, 2 errors | our-error | +| cvefiles/cve-2025-44905.h5 | error exit | read 25 obj, 3 errors | read 25 obj, 3 errors | ok | | cvefiles/cve-2025-6269-1.h5 | error exit | read 1 obj, 1 errors | read 1 obj, 1 errors | ok | | cvefiles/cve-2025-6269-2.h5 | error exit | read 1 obj, 1 errors | read 1 obj, 1 errors | ok | | cvefiles/cve-2025-6269-3.h5 | error exit | read 1 obj, 1 errors | read 1 obj, 1 errors | ok | | cvefiles/cve-2025-6269-4.h5 | error exit | read 1 obj, 1 errors | read 1 obj, 1 errors | ok | -| cvefiles/cve-2025-6270-1.h5 | error exit | open error | read 1 obj, 1 errors | h5py-cannot-read | -| cvefiles/cve-2025-6270-2.h5 | error exit | open error | read 1 obj, 1 errors | h5py-cannot-read | -| cvefiles/cve-2025-6270-3.h5 | error exit | open error | read 1 obj, 1 errors | h5py-cannot-read | +| cvefiles/cve-2025-6270-1.h5 | error exit | open error | open error | h5py-cannot-read | +| cvefiles/cve-2025-6270-2.h5 | error exit | open error | open error | h5py-cannot-read | +| cvefiles/cve-2025-6270-3.h5 | error exit | open error | open error | h5py-cannot-read | | cvefiles/cve-2025-6516.h5 | error exit | read 1 obj, 1 errors | read 1 obj, 1 errors | ok | | cvefiles/cve-2025-6750.h5 | error exit | open error | read 1 obj, 1 errors | h5py-cannot-read | -| cvefiles/cve-2025-6816.h5 | error exit | open error | read 1 obj, 1 errors | h5py-cannot-read | -| cvefiles/cve-2025-6817.h5 | error exit | open error | read 1 obj | h5py-cannot-read | -| cvefiles/cve-2025-6818.h5 | error exit | open error | read 1 obj | h5py-cannot-read | -| cvefiles/cve-2025-6856.h5 | error exit | open error | read 1 obj, 1 errors | h5py-cannot-read | +| cvefiles/cve-2025-6816.h5 | error exit | open error | open error | h5py-cannot-read | +| cvefiles/cve-2025-6817.h5 | error exit | open error | open error | h5py-cannot-read | +| cvefiles/cve-2025-6818.h5 | error exit | open error | open error | h5py-cannot-read | +| cvefiles/cve-2025-6856.h5 | error exit | open error | open error | h5py-cannot-read | | cvefiles/cve-2025-6857.h5 | error exit | read 1 obj, 1 errors | read 1 obj, 1 errors | ok | -| cvefiles/cve-2025-6858.h5 | SIGSEGV | open error | read 1 obj, 1 errors | h5py-cannot-read | +| cvefiles/cve-2025-6858.h5 | SIGSEGV | open error | open error | h5py-cannot-read | | cvefiles/cve-2025-7067.h5 | error exit | read 1 obj, 1 errors | read 1 obj, 1 errors | ok | -| cvefiles/cve-2025-7068.h5 | error exit | open error | read 1 obj, 1 errors | h5py-cannot-read | -| cvefiles/cve-2025-7069.h5 | error exit | open error | read 1 obj, 1 errors | h5py-cannot-read | +| cvefiles/cve-2025-7068.h5 | error exit | open error | open error | h5py-cannot-read | +| cvefiles/cve-2025-7069.h5 | error exit | open error | open error | h5py-cannot-read | | cvefiles/cve-2026-26200.h5 | error exit | read 1 obj, 1 errors | read 1 obj, 1 errors | ok | | cvefiles/cve-2026-34734.h5 | error exit | read 2 obj, 1 errors | read 2 obj, 1 errors | ok | | cvefiles/cve-2026-92627.h5 | error exit | read 2 obj, 1 errors | read 2 obj, 1 errors | ok | @@ -275,6 +265,11 @@ columns are. - **Types h5py widens.** Where h5py reads a type into a numpy type of a different size (FP8 -> float16, bfloat16 -> float32, x87 long double -> float128) the values are not compared (shape and presence still are): dataset file type size 1 -> numpy float16 (2) (15x), attr file type size 1 -> numpy float16 (2) (15x), dataset file type size 2 -> numpy float32 (4) (2x), dataset file type size 8 -> numpy float128 (16) (1x), dataset file type size 12 -> numpy float128 (16) (1x), attr file type size 2 -> numpy float32 (4) (1x), dataset file type size 2 -> numpy >f4 (4) (1x), attr file type size 2 -> numpy >f4 (4) (1x). +- **Corrupt data HDF5 2.0 reads through a bug.** clawhdf5 refuses these objects; h5py 3.16 / + HDF5 2.0 returns values for them that the file does not hold: + - `cve_hdf5/cvefiles/cve-2025-2308.h5` `/Scale_offset_long_long_data_le`: scale-offset codes run past the end of the chunk: HDF5 2.0 reads past its buffer; libhdf5's develop branch refuses the chunk ("Buffer too short"). + - `cve_hdf5/cvefiles/cve-2025-44904.h5` `/Scale_offset_float_data_le`: unfiltered chunks of 38 and 37 bytes for 48-byte chunks: HDF5 2.0 fills the rest with whatever its buffer held; libhdf5's develop branch refuses them ("incorrect chunk size returned from index for unfiltered chunk"). + - `hdf5/test/testfiles/bad_nbit_parms_walk.h5` `/Nbit_int_data_le`: an N-Bit parameter list one value short: HDF5 2.0 reads past the list; libhdf5's own test (`test_filter_bad_params`, test/dsets.c) now requires the read to fail. - **References** are compared by presence only (`R`), not by target. ## Objects h5py fails on but clawhdf5 reads @@ -282,7 +277,6 @@ columns are. - 19 x `OSError: Can't synchronously read data (no appropriate function for conversion path)` - 1 x `TypeError: unhandled dtype kind M (dtype('…'))` - 1 x `TypeError: No NumPy equivalent for TypeTimeID exists` -- 1 x `KeyError: "…"` - 1 x `ValueError: Insufficient precision in available types to represent (N, N, N, N, N)` ## Reproduce diff --git a/conformance/baseline.json b/conformance/baseline.json index df3cb4a..4c496f9 100644 --- a/conformance/baseline.json +++ b/conformance/baseline.json @@ -1,15 +1,15 @@ { "comment": "conformance/run.sh fails if the ok count drops below `ok` or a file in `ok_files` stops being ok. Regenerate with `conformance/run.sh --update-baseline` after an intended change.", - "commit": "73a01f1256fb9bf1b1e7601f755af9e8273cec4e", - "date": "2026-09-26 14:18 UTC", + "commit": "d0e3beb3aa8290aae523ce280b4380e75484b6bc", + "date": "2026-09-26 17:10 UTC", "reference": "h5py 3.16.0 / HDF5 2.0.0", "files": 697, - "ok": 575, + "ok": 599, "counts": { "h5py-cannot-read": 92, - "mismatch": 20, - "ok": 575, - "our-error": 10 + "mismatch": 2, + "ok": 599, + "our-error": 4 }, "per_corpus": { "NCAS-CMS_pyfive": { @@ -18,18 +18,17 @@ }, "cve_hdf5": { "h5py-cannot-read": 32, - "mismatch": 9, - "ok": 100, - "our-error": 6 + "ok": 113, + "our-error": 2 }, "h5py_data": { "ok": 4 }, "hdf5": { "h5py-cannot-read": 60, - "mismatch": 10, - "ok": 392, - "our-error": 4 + "mismatch": 1, + "ok": 403, + "our-error": 2 }, "netcdf-c": { "ok": 20 @@ -117,14 +116,22 @@ "cve_hdf5/cvefiles/cve-2018-17434.h5", "cve_hdf5/cvefiles/cve-2018-17435.h5", "cve_hdf5/cvefiles/cve-2018-17437.h5", + "cve_hdf5/cvefiles/cve-2018-17438", + "cve_hdf5/cvefiles/cve-2018-17439", "cve_hdf5/cvefiles/cve-2019-8396.h5", + "cve_hdf5/cvefiles/cve-2019-8397.h5", + "cve_hdf5/cvefiles/cve-2019-8398.h5", + "cve_hdf5/cvefiles/cve-2019-9151.h5", "cve_hdf5/cvefiles/cve-2019-9152.h5", "cve_hdf5/cvefiles/cve-2020-10811.h5", "cve_hdf5/cvefiles/cve-2020-18232.h5", + "cve_hdf5/cvefiles/cve-2020-18494.h5", "cve_hdf5/cvefiles/cve-2021-36977.h5", "cve_hdf5/cvefiles/cve-2021-37501.h5", "cve_hdf5/cvefiles/cve-2021-45829.h5", "cve_hdf5/cvefiles/cve-2021-45833.h5", + "cve_hdf5/cvefiles/cve-2021-46243.h5", + "cve_hdf5/cvefiles/cve-2021-46244.h5", "cve_hdf5/cvefiles/cve-2024-29157.h5", "cve_hdf5/cvefiles/cve-2024-29158.h5", "cve_hdf5/cvefiles/cve-2024-29159.h5", @@ -148,18 +155,23 @@ "cve_hdf5/cvefiles/cve-2024-32615.h5", "cve_hdf5/cvefiles/cve-2024-32616.h5", "cve_hdf5/cvefiles/cve-2024-32617.h5", + "cve_hdf5/cvefiles/cve-2024-32618.h5", "cve_hdf5/cvefiles/cve-2024-32619.h5", "cve_hdf5/cvefiles/cve-2024-32620.h5", "cve_hdf5/cvefiles/cve-2024-32621.h5", "cve_hdf5/cvefiles/cve-2024-32622.h5", + "cve_hdf5/cvefiles/cve-2024-32623.h5", "cve_hdf5/cvefiles/cve-2024-32624.h5", "cve_hdf5/cvefiles/cve-2024-33873.h5", + "cve_hdf5/cvefiles/cve-2024-33874.h5", "cve_hdf5/cvefiles/cve-2024-33875.h5", "cve_hdf5/cvefiles/cve-2024-33876.h5", "cve_hdf5/cvefiles/cve-2024-33877.h5", + "cve_hdf5/cvefiles/cve-2025-2309.h5", "cve_hdf5/cvefiles/cve-2025-2310.h5", "cve_hdf5/cvefiles/cve-2025-2924.h5", "cve_hdf5/cvefiles/cve-2025-2925.h5", + "cve_hdf5/cvefiles/cve-2025-44905.h5", "cve_hdf5/cvefiles/cve-2025-6269-1.h5", "cve_hdf5/cvefiles/cve-2025-6269-2.h5", "cve_hdf5/cvefiles/cve-2025-6269-3.h5", @@ -183,6 +195,7 @@ "h5py_data/vlen_string_s390x.h5", "hdf5/HDF5Examples/C/H5FLT/tfiles/h5ex_d_bitgroom.h5", "hdf5/HDF5Examples/C/H5FLT/tfiles/h5ex_d_blosc.h5", + "hdf5/HDF5Examples/C/H5FLT/tfiles/h5ex_d_blosc2.h5", "hdf5/HDF5Examples/C/H5FLT/tfiles/h5ex_d_bshuf.h5", "hdf5/HDF5Examples/C/H5FLT/tfiles/h5ex_d_bzip2.h5", "hdf5/HDF5Examples/C/H5FLT/tfiles/h5ex_d_granularbr.h5", @@ -264,6 +277,7 @@ "hdf5/test/testfiles/tmtimeo.h5", "hdf5/test/testfiles/tnullspace.h5", "hdf5/test/testfiles/tsizeslheap.h5", + "hdf5/tools/test/testfiles/bigendian/tall.h5", "hdf5/tools/test/testfiles/bigendian/tdset2.h5", "hdf5/tools/test/testfiles/binfp64.h5", "hdf5/tools/test/testfiles/binin16.h5", @@ -284,6 +298,7 @@ "hdf5/tools/test/testfiles/h5clear_fsm_persist_noclose.h5", "hdf5/tools/test/testfiles/h5clear_fsm_persist_user_equal.h5", "hdf5/tools/test/testfiles/h5clear_fsm_persist_user_less.h5", + "hdf5/tools/test/testfiles/h5clear_mdc_image.h5", "hdf5/tools/test/testfiles/h5clear_sec2_v0.h5", "hdf5/tools/test/testfiles/h5clear_sec2_v2.h5", "hdf5/tools/test/testfiles/h5copy_extlinks_src.h5", @@ -337,6 +352,7 @@ "hdf5/tools/test/testfiles/h5diff_softlinks.h5", "hdf5/tools/test/testfiles/h5diff_strings1.h5", "hdf5/tools/test/testfiles/h5diff_strings2.h5", + "hdf5/tools/test/testfiles/h5diff_types.h5", "hdf5/tools/test/testfiles/h5fc_edge_v3.h5", "hdf5/tools/test/testfiles/h5fc_err_level.h5", "hdf5/tools/test/testfiles/h5fc_ext1_f.h5", @@ -414,9 +430,11 @@ "hdf5/tools/test/testfiles/tCVE_2018_11206_fill_new.h5", "hdf5/tools/test/testfiles/tCVE_2018_11206_fill_old.h5", "hdf5/tools/test/testfiles/taindices.h5", + "hdf5/tools/test/testfiles/tall.h5", "hdf5/tools/test/testfiles/tarray1.h5", "hdf5/tools/test/testfiles/tarray1_big.h5", "hdf5/tools/test/testfiles/tarray2.h5", + "hdf5/tools/test/testfiles/tarray3.h5", "hdf5/tools/test/testfiles/tarray4.h5", "hdf5/tools/test/testfiles/tarray5.h5", "hdf5/tools/test/testfiles/tarray8.h5", @@ -449,6 +467,7 @@ "hdf5/tools/test/testfiles/textlinksrc.h5", "hdf5/tools/test/testfiles/textlinktar.h5", "hdf5/tools/test/testfiles/textpfe.h5", + "hdf5/tools/test/testfiles/tfcontents1.h5", "hdf5/tools/test/testfiles/tfcontents2.h5", "hdf5/tools/test/testfiles/tfilters.h5", "hdf5/tools/test/testfiles/tfloat16.h5", @@ -505,6 +524,7 @@ "hdf5/tools/test/testfiles/tstr3.h5", "hdf5/tools/test/testfiles/tudfilter.h5", "hdf5/tools/test/testfiles/tudfilter2.h5", + "hdf5/tools/test/testfiles/tudlink.h5", "hdf5/tools/test/testfiles/tvldtypes1.h5", "hdf5/tools/test/testfiles/tvldtypes2.h5", "hdf5/tools/test/testfiles/tvldtypes3.h5", @@ -513,6 +533,8 @@ "hdf5/tools/test/testfiles/tvlenstr_array.h5", "hdf5/tools/test/testfiles/tvlstr.h5", "hdf5/tools/test/testfiles/tvms.h5", + "hdf5/tools/test/testfiles/twithub.h5", + "hdf5/tools/test/testfiles/twithub513.h5", "hdf5/tools/test/testfiles/txtfp32.h5", "hdf5/tools/test/testfiles/txtfp64.h5", "hdf5/tools/test/testfiles/txtin16.h5", @@ -558,6 +580,7 @@ "hdf5/tools/test/testfiles/xml/tenum.h5", "hdf5/tools/test/testfiles/xml/test35.nc", "hdf5/tools/test/testfiles/xml/tloop2.h5", + "hdf5/tools/test/testfiles/xml/tmany.h5", "hdf5/tools/test/testfiles/xml/tname-amp.h5", "hdf5/tools/test/testfiles/xml/tname-apos.h5", "hdf5/tools/test/testfiles/xml/tname-gt.h5", From f2ff2c424fb8cee1c48b5c05f1a246bd3ca5eadd Mon Sep 17 00:00:00 2001 From: osobh Date: Sat, 26 Sep 2026 12:20:47 -0500 Subject: [PATCH 43/43] bench: chunked full reads now beat an h5py process pool Idle-start run on tank at c5334b1 (noisier than the last: compare ratios within the run). Full reads of deflate data at 16 threads: 4944 MB/s vs 3135 for 16 h5py processes (1.58x; 0.69x-0.76x before). One thread with the default pool: 6143 MB/s, 15x one h5py call. The concurrent-read known issue is closed. Co-Authored-By: Claude Opus 5.5 (1M context) --- BENCHMARKS.md | 39 +++++++++++++++++++++++++++++++++++++++ docs/known-issues.md | 8 +++++--- 2 files changed, 44 insertions(+), 3 deletions(-) diff --git a/BENCHMARKS.md b/BENCHMARKS.md index 0bc5fe7..62beeb6 100644 --- a/BENCHMARKS.md +++ b/BENCHMARKS.md @@ -484,6 +484,45 @@ explain the slower windows. ## Concurrent reads +### Results after in-place chunk decoding (2026-09-26, tank, `c5334b1`) + +Same machine, files and commands, re-run after chunked reads started +decoding into reusable per-thread buffers straight into the (typed) output, +with the calling thread decoding alongside the pool. Load average 1.78 at +the start; it rose to 6-9 during the runs (the clawhdf5 runs' own threads, +and it stayed around 5-6 through the h5py runs, so something else was +active). **This run was noisier than the previous one: h5py's own contiguous +figures are about 40% lower than in the run below, and ours dropped +similarly, so compare ratios within a run rather than MB/s across runs.** +h5py was re-run in the same session. + +Each read decoding on its calling thread (`--decode-threads 1`, like h5py): + +| layout | mode | threads | clawhdf5 MB/s (eff) | h5py threads MB/s (eff) | h5py processes MB/s (eff) | vs h5py processes | +|---|---|---:|---:|---:|---:|---:| +| deflate | distinct | 1 | 670 (1.00) | 410 (1.00) | 397 (1.00) | 1.69x | +| deflate | distinct | 4 | 2434 (0.91) | 406 (0.25) | 1470 (0.93) | 1.66x | +| deflate | distinct | 8 | 3749 (0.70) | 406 (0.12) | 2398 (0.76) | 1.56x | +| deflate | distinct | 16 | 4944 (0.46) | 390 (0.06) | 3135 (0.49) | 1.58x | +| deflate | same | 1 | 211 (1.00) | 125 (1.00) | 124 (1.00) | 1.70x | +| deflate | same | 16 | 1835 (0.54) | 122 (0.06) | 961 (0.48) | 1.91x | +| contiguous | distinct | 1 | 6718 (1.00) | 5545 (1.00) | 5200 (1.00) | 1.29x | +| contiguous | distinct | 16 | 11035 (0.10) | 4950 (0.06) | 10558 (0.13) | 1.05x | +| contiguous | same | 1 | 14483 (1.00) | 2593 (1.00) | 2737 (1.00) | 5.29x | +| contiguous | same | 16 | 132175 (0.57) | 2224 (0.05) | 14809 (0.34) | 8.93x | + +With the default rayon pool, deflate `distinct` reads 6143 MB/s from a single +thread (15x h5py's 410 on one call) and 4556 MB/s at 16 threads (1.45x h5py +processes); the other rows are within the noise of the table above. + +What changed: full reads of chunked datasets were 0.69x-0.76x of h5py +processes at 16 threads in the run below, and are 1.58x here; with one +thread they were 1.44x and are 1.69x. Minor page faults for the 16-thread +run fell from about 4.6M to 0.2M (`/usr/bin/time -v`, provisional, loaded +machine). clawhdf5 now reads faster than 16 h5py processes in every row of +this benchmark except contiguous full reads at 16 threads, where both +saturate memory bandwidth (1.05x). + ### Results after the read fixes (2026-09-26, tank, `408f69e`) Same machine, files and commands as the first run below, re-run on an idle diff --git a/docs/known-issues.md b/docs/known-issues.md index a58f42e..287e501 100644 --- a/docs/known-issues.md +++ b/docs/known-issues.md @@ -26,9 +26,11 @@ performance" below. ## Concurrent and contiguous read performance (measured 2026-09-26) -**Status:** open for chunked full reads at 16 threads until re-measured -(the causes identified below are fixed as of 2026-09-26); the contiguous -item is fixed (2026-09-26). Measured on +**Status:** fixed (2026-09-26). Re-measured on tank at `c5334b1`: full +reads of chunked deflate data at 16 threads run at 4944 MB/s against 3135 +for 16 h5py processes (1.58x; 0.69x-0.76x before), and contiguous reads +are 1.29x h5py on one thread (`BENCHMARKS.md`, "Results after in-place +chunk decoding"). The history below is kept for reference. Measured on tank with `concurrent_read` against h5py 3.16 / HDF5 2.0 (`BENCHMARKS.md`, "Concurrent reads"): - **Partly fixed 2026-09-26.** Full reads of chunked datasets from several threads