From 1c85986079651d82f0d16a5ae3f9af59fc8311e8 Mon Sep 17 00:00:00 2001 From: osobh Date: Fri, 25 Sep 2026 21:55:51 -0500 Subject: [PATCH] fix(format): read huge, tiny and filtered fractal heap objects MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A heap ID's type is in bits 4-5 of its first byte (H5HF_ID_TYPE_MASK 0x30); bits 6-7 are the ID version. The reader took the type from bits 6-7, so every huge object ID (0x10) was decoded as a managed one and failed — and since dense attributes are read all at once, one attribute over the heap's 4 KiB managed limit made every attribute on its object unreadable (netcdf4-python's issue671.nc / issue672.nc). - Huge objects (type 1): located directly from the ID when address and length fit in it, otherwise through the huge-object v2 B-tree (record types 1 and 2); filtered huge objects are decoded with the heap's pipeline and their filter mask. - Tiny objects (type 2): read from the ID itself. - Filtered heaps: the header's pipeline is parsed (it was skipped short, so the header checksum was read from the wrong place), indirect-block entries for direct blocks carry their filtered size and mask, and direct blocks are decoded before objects are read from them. - An unknown ID version is an error. FractalHeapHeader gains huge_btree_address, filter_pipeline, root_direct_block_filtered_size, root_direct_block_filter_mask, offset_size and length_size; read_managed_object now accepts any ID type. Regression tests (h5py-written, compared with h5py): dense_attribute_stored_as_a_huge_heap_object, dense_group_with_a_huge_link, dense_group_with_a_filtered_link_heap; unit tests tiny_object_is_read_from_the_id, huge_object_with_a_direct_id, unknown_heap_id_version_is_refused. Co-Authored-By: Claude Opus 5.5 (1M context) --- CHANGELOG.md | 12 + crates/clawhdf5-format/src/file_writer.rs | 6 + crates/clawhdf5-format/src/fractal_heap.rs | 459 +++++++++++++++--- .../clawhdf5/tests/dense_storage_interop.rs | 112 ++++- docs/known-issues.md | 3 +- 5 files changed, 518 insertions(+), 74 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9d94b66..462672c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -310,6 +310,18 @@ estimate instead of libhdf5's per-depth record capacities, and the listing failed. The same B-tree code indexes dense attributes, shared messages and chunks. + - Fractal-heap "huge" objects (larger than the heap's managed-object + limit, 4 KiB by default — e.g. an 8 KiB dense attribute or a link with a + very long name) and "tiny" objects are now read; the ID type was taken + from the wrong bits (6-7, the version, instead of 4-5), so a huge object + failed and took every attribute on its object down with it (NetCDF-4 + files such as netcdf4-python's `issue671.nc`). Huge objects are found + directly from the ID or through the huge-object v2 B-tree, filtered or + not. + - Heaps with an I/O filter pipeline (a group created with a filter on its + creation property list compresses its link heap) are now read: the + header's pipeline was skipped with the wrong size, so its checksum was + looked for in the wrong place, and filtered direct blocks were read raw. - `clawhdf5-format` writer — **files libhdf5 rejects or reads wrong:** - Extensible Array (one unlimited dimension): chunks from index 244 on were written but never indexed and read as 0, by libhdf5 and by us. diff --git a/crates/clawhdf5-format/src/file_writer.rs b/crates/clawhdf5-format/src/file_writer.rs index 95d5834..dbf59f2 100644 --- a/crates/clawhdf5-format/src/file_writer.rs +++ b/crates/clawhdf5-format/src/file_writer.rs @@ -1947,6 +1947,12 @@ mod tests { root_block_address: 0, current_rows_in_root_indirect_block: 0, managed_objects_count: 0, + huge_btree_address: u64::MAX, + filter_pipeline: None, + root_direct_block_filtered_size: 0, + root_direct_block_filter_mask: 0, + offset_size: 8, + length_size: 8, }; let (off, len) = fh.decode_managed_id(&id).unwrap(); assert_eq!(off, 100); diff --git a/crates/clawhdf5-format/src/fractal_heap.rs b/crates/clawhdf5-format/src/fractal_heap.rs index c71a41f..9bc4ba4 100644 --- a/crates/clawhdf5-format/src/fractal_heap.rs +++ b/crates/clawhdf5-format/src/fractal_heap.rs @@ -1,12 +1,14 @@ //! HDF5 Fractal Heap parsing for v2 group link storage. #[cfg(not(feature = "std"))] -use alloc::vec::Vec; +use alloc::{format, vec::Vec}; #[cfg(feature = "checksum")] use byteorder::{ByteOrder, LittleEndian}; +use crate::btree_v2::{BTreeV2Header, collect_btree_v2_records}; use crate::error::FormatError; +use crate::filter_pipeline::FilterPipeline; /// Parsed fractal heap header (signature "FRHP"). #[derive(Debug, Clone)] @@ -33,6 +35,23 @@ pub struct FractalHeapHeader { pub current_rows_in_root_indirect_block: u16, /// Total number of managed objects. pub managed_objects_count: u64, + /// Address of the v2 B-tree indexing "huge" objects (undefined address + /// when the heap has none). Huge objects are those larger than + /// `max_managed_object_size`; they live outside the heap's blocks. + pub huge_btree_address: u64, + /// The heap's I/O filter pipeline, if it has one. It applies to managed + /// direct blocks and to huge objects. + pub filter_pipeline: Option, + /// Stored (filtered) size of the root direct block; meaningful only when + /// the heap is filtered and its root is a direct block. + pub root_direct_block_filtered_size: u64, + /// Filter mask of the root direct block (bit *i* set = filter *i* + /// skipped); meaningful only when the heap is filtered. + pub root_direct_block_filter_mask: u32, + /// Size of addresses in the file ("Size of Offsets"). + pub offset_size: u8, + /// Size of lengths in the file ("Size of Lengths"). + pub length_size: u8, } fn read_offset(data: &[u8], pos: usize, size: u8) -> Result { @@ -79,6 +98,38 @@ fn is_undefined(val: u64, offset_size: u8) -> bool { } } +/// Little-endian unsigned integer of up to 8 bytes. +fn le_uint(bytes: &[u8]) -> u64 { + bytes + .iter() + .take(8) + .enumerate() + .fold(0u64, |acc, (i, &b)| acc | (u64::from(b) << (i * 8))) +} + +fn heap_error(msg: &str) -> FormatError { + FormatError::ChunkedReadError(format!("fractal heap: {msg}")) +} + +/// Heap ID type, from bits 4-5 of an ID's first byte (libhdf5's +/// `H5HF_ID_TYPE_MASK`, 0x30); bits 6-7 are the ID version, which must be 0. +const HEAP_ID_MANAGED: u8 = 0; +const HEAP_ID_HUGE: u8 = 1; +const HEAP_ID_TINY: u8 = 2; + +/// The type (0 managed, 1 huge, 2 tiny) of a heap ID from its first byte, +/// refusing an ID version other than 0. +fn heap_id_type(first: u8) -> Result { + if first >> 6 != 0 { + return Err(heap_error("unsupported heap ID version")); + } + Ok((first >> 4) & 0x03) +} + +/// v2 B-tree record types indexing a heap's huge objects. +const BTREE_HUGE_INDIRECT: u8 = 1; +const BTREE_HUGE_INDIRECT_FILTERED: u8 = 2; + impl FractalHeapHeader { /// Parse a fractal heap header at the given offset. pub fn parse( @@ -122,11 +173,17 @@ impl FractalHeapHeader { ]); pos += 4; - // Skip several fixed fields: next_huge_object_id(ls), btree_huge_objects_address(os), - // free_space_managed_blocks(ls), managed_block_free_space_manager_address(os), + // next_huge_object_id (length_size) + ensure_len(file_data, pos, ls)?; + pos += ls; + // btree_huge_objects_address (offset_size) + let huge_btree_address = read_offset(file_data, pos, offset_size)?; + pos += os; + + // Skip: free_space_managed_blocks(ls), managed_block_free_space_manager_address(os), // managed_space_in_heap(ls), allocated_managed_space_in_heap(ls), // direct_block_allocation_iterator_offset(ls) - let skip_size = 5 * ls + 2 * os; + let skip_size = 4 * ls + os; ensure_len(file_data, pos, skip_size)?; pos += skip_size; @@ -134,14 +191,9 @@ impl FractalHeapHeader { let managed_objects_count = read_offset(file_data, pos, length_size)?; pos += ls; - // huge_objects_size (length_size) - pos += ls; - // huge_objects_count (length_size) - pos += ls; - // tiny_objects_size (length_size) - pos += ls; - // tiny_objects_count (length_size) - pos += ls; + // huge_objects_size, huge_objects_count, tiny_objects_size, + // tiny_objects_count (length_size each) + pos += 4 * ls; // table_width (2) ensure_len(file_data, pos, 2)?; @@ -175,16 +227,28 @@ impl FractalHeapHeader { ensure_len(file_data, pos, 2)?; let current_rows_in_root_indirect_block = u16::from_le_bytes([file_data[pos], file_data[pos + 1]]); - #[allow(unused_variables, unused_mut, unused_assignments)] - let mut pos = pos + 2; + pos += 2; - // Skip IO filter encoded info if present + // With I/O filters: root direct block's filtered size (length_size), + // its filter mask (4), then the encoded filter pipeline message. + let mut filter_pipeline = None; + let mut root_direct_block_filtered_size = 0; + let mut root_direct_block_filter_mask = 0; if io_filter_encoded_length > 0 { - // root_block_filter_info_size (length_size) + filter_mask (4) - #[allow(unused_assignments)] - { - pos += ls + 4; - } + root_direct_block_filtered_size = read_offset(file_data, pos, length_size)?; + pos += ls; + ensure_len(file_data, pos, 4)?; + root_direct_block_filter_mask = u32::from_le_bytes([ + file_data[pos], + file_data[pos + 1], + file_data[pos + 2], + file_data[pos + 3], + ]); + pos += 4; + let n = io_filter_encoded_length as usize; + ensure_len(file_data, pos, n)?; + filter_pipeline = Some(FilterPipeline::parse(&file_data[pos..pos + n])?); + pos += n; } // Validate header checksum @@ -200,6 +264,8 @@ impl FractalHeapHeader { }); } } + #[cfg(not(feature = "checksum"))] + let _ = pos; Ok(FractalHeapHeader { heap_id_length, @@ -213,13 +279,19 @@ impl FractalHeapHeader { root_block_address, current_rows_in_root_indirect_block, managed_objects_count, + huge_btree_address, + filter_pipeline, + root_direct_block_filtered_size, + root_direct_block_filter_mask, + offset_size, + length_size, }) } /// Decode a managed heap ID into (offset_in_heap, object_length). /// /// The heap ID layout for managed objects (type 0): - /// - Byte 0: bits 6-7 = type (0), bits 4-5 = version (0), bits 0-3 = reserved + /// - Byte 0: bits 6-7 = version (0), bits 4-5 = type (0), bits 0-3 = reserved /// - Bytes 1+: offset (max_heap_size bits, LE) then length (remaining bits, LE) pub fn decode_managed_id(&self, id_bytes: &[u8]) -> Result<(u64, u64), FormatError> { if id_bytes.is_empty() { @@ -229,8 +301,8 @@ impl FractalHeapHeader { }); } - let id_type = (id_bytes[0] >> 6) & 0x03; - if id_type != 0 { + let id_type = heap_id_type(id_bytes[0])?; + if id_type != HEAP_ID_MANAGED { return Err(FormatError::InvalidHeapIdType(id_type)); } @@ -269,12 +341,183 @@ impl FractalHeapHeader { Ok((heap_offset, length_val)) } - /// Read a managed object from the heap given its raw heap ID bytes. + /// Read any object from the heap given its raw heap ID bytes: managed + /// (stored in the heap's blocks), huge (stored outside them, found + /// directly from the ID or through the huge-object v2 B-tree, optionally + /// filtered) or tiny (stored in the ID itself). + /// + /// Despite its name this accepts every ID type; `offset_size` must match + /// the one the header was parsed with. pub fn read_managed_object( &self, file_data: &[u8], id_bytes: &[u8], offset_size: u8, + ) -> Result, FormatError> { + let Some(&first) = id_bytes.first() else { + return Err(FormatError::UnexpectedEof { + expected: 1, + available: 0, + }); + }; + match heap_id_type(first)? { + HEAP_ID_MANAGED => self.read_heap_managed(file_data, id_bytes, offset_size), + HEAP_ID_HUGE => self.read_huge_object(file_data, id_bytes), + HEAP_ID_TINY => self.read_tiny_object(id_bytes), + other => Err(FormatError::InvalidHeapIdType(other)), + } + } + + /// Whether a huge object's ID holds its address and length directly + /// (libhdf5 does this when they fit in the ID), rather than a key into + /// the huge-object B-tree. + fn huge_ids_direct(&self) -> bool { + let room = usize::from(self.heap_id_length).saturating_sub(1); + let os = usize::from(self.offset_size); + let ls = usize::from(self.length_size); + if self.filter_pipeline.is_some() { + room >= os + ls + 4 + ls + } else { + room >= os + ls + } + } + + /// Read a huge object (heap ID type 1). + fn read_huge_object(&self, file_data: &[u8], id: &[u8]) -> Result, FormatError> { + let os = usize::from(self.offset_size); + let ls = usize::from(self.length_size); + // (address, stored length, filter mask, decoded length); the last two + // only matter for a filtered heap. + let (addr, stored_len, mask, mem_len) = if self.huge_ids_direct() { + let body = &id[1..]; + let need = if self.filter_pipeline.is_some() { + os + ls + 4 + ls + } else { + os + ls + }; + ensure_len(body, 0, need)?; + let addr = le_uint(&body[..os]); + let len = le_uint(&body[os..os + ls]); + if self.filter_pipeline.is_some() { + let mask = u32::from_le_bytes([ + body[os + ls], + body[os + ls + 1], + body[os + ls + 2], + body[os + ls + 3], + ]); + let mem = le_uint(&body[os + ls + 4..os + ls + 4 + ls]); + (addr, len, mask, mem) + } else { + (addr, len, 0, len) + } + } else { + let key_len = (usize::from(self.heap_id_length).saturating_sub(1)).min(8); + ensure_len(id, 1, key_len)?; + let key = le_uint(&id[1..1 + key_len]); + self.find_huge_record(file_data, key)? + }; + + let start = usize::try_from(addr).map_err(|_| heap_error("huge object address"))?; + let len = usize::try_from(stored_len).map_err(|_| heap_error("huge object length"))?; + ensure_len(file_data, start, len)?; + let stored = &file_data[start..start + len]; + match &self.filter_pipeline { + None => Ok(stored.to_vec()), + Some(pipeline) => { + let mem = usize::try_from(mem_len).map_err(|_| heap_error("huge object size"))?; + let out = crate::filters::decompress_chunk_masked(stored, pipeline, mem, 1, mask)?; + if out.len() != mem { + return Err(heap_error("filtered huge object decoded to the wrong size")); + } + Ok(out) + } + } + } + + /// Look up huge object `key` in the huge-object v2 B-tree, returning + /// (address, stored length, filter mask, decoded length). + fn find_huge_record( + &self, + file_data: &[u8], + key: u64, + ) -> Result<(u64, u64, u32, u64), FormatError> { + if is_undefined(self.huge_btree_address, self.offset_size) { + return Err(heap_error( + "huge object ID but the heap has no huge-object index", + )); + } + let hdr = BTreeV2Header::parse( + file_data, + self.huge_btree_address as usize, + self.offset_size, + self.length_size, + )?; + let os = usize::from(self.offset_size); + let ls = usize::from(self.length_size); + let filtered = self.filter_pipeline.is_some(); + let (expected_type, rec_len) = if filtered { + (BTREE_HUGE_INDIRECT_FILTERED, os + ls + 4 + ls + ls) + } else { + (BTREE_HUGE_INDIRECT, os + ls + ls) + }; + if hdr.tree_type != expected_type || usize::from(hdr.record_size) < rec_len { + return Err(heap_error("unexpected huge-object B-tree record type")); + } + let records = + collect_btree_v2_records(file_data, &hdr, self.offset_size, self.length_size)?; + for rec in &records { + let d = &rec.data; + if d.len() < rec_len { + continue; + } + let addr = le_uint(&d[..os]); + let len = le_uint(&d[os..os + ls]); + if filtered { + let mask = u32::from_le_bytes([ + d[os + ls], + d[os + ls + 1], + d[os + ls + 2], + d[os + ls + 3], + ]); + let mem = le_uint(&d[os + ls + 4..os + 2 * ls + 4]); + let id = le_uint(&d[os + 2 * ls + 4..os + 3 * ls + 4]); + if id == key { + return Ok((addr, len, mask, mem)); + } + } else { + let id = le_uint(&d[os + ls..os + 2 * ls]); + if id == key { + return Ok((addr, len, 0, len)); + } + } + } + Err(heap_error("huge object not found in its B-tree")) + } + + /// Read a tiny object (heap ID type 2), stored in the ID itself. + fn read_tiny_object(&self, id: &[u8]) -> Result, FormatError> { + // libhdf5 uses a one-byte length (low 4 bits of byte 0) unless the ID + // is long enough to need 12 bits, which then borrow byte 1. + let extended = usize::from(self.heap_id_length).saturating_sub(1) > 17; + let (len, start) = if extended { + ensure_len(id, 0, 2)?; + ( + ((usize::from(id[0] & 0x0F)) << 8 | usize::from(id[1])) + 1, + 2, + ) + } else { + (usize::from(id[0] & 0x0F) + 1, 1) + }; + ensure_len(id, start, len)?; + Ok(id[start..start + len].to_vec()) + } + + /// Read a managed object (heap ID type 0). + fn read_heap_managed( + &self, + file_data: &[u8], + id_bytes: &[u8], + offset_size: u8, ) -> Result, FormatError> { let (heap_offset, obj_len) = self.decode_managed_id(id_bytes)?; @@ -289,12 +532,15 @@ impl FractalHeapHeader { // Root is a direct block self.read_from_direct_block( file_data, - self.root_block_address as usize, - self.starting_block_size, - 0, // block offset in heap = 0 for root + DirectBlock { + addr: self.root_block_address as usize, + size: self.starting_block_size, + heap_offset: 0, + filtered_size: self.root_direct_block_filtered_size, + filter_mask: self.root_direct_block_filter_mask, + }, heap_offset, obj_len as usize, - offset_size, ) } else { // Root is an indirect block — limit recursion to 64 levels @@ -313,27 +559,41 @@ impl FractalHeapHeader { /// Read an object from a direct block. /// - /// The heap offset is relative to the start of the block (including its header), - /// so we just add it to the block address minus the block's heap offset. - #[allow(clippy::too_many_arguments)] + /// The heap offset is relative to the start of the block (including its + /// header), so we just add it to the block address minus the block's heap + /// offset. A filtered heap stores each direct block (header included) + /// through its filter pipeline, so the block is decoded first. fn read_from_direct_block( &self, file_data: &[u8], - block_addr: usize, - _block_size: u64, - block_heap_offset: u64, + block: DirectBlock, target_offset: u64, length: usize, - _offset_size: u8, ) -> Result, FormatError> { - if target_offset < block_heap_offset { + if target_offset < block.heap_offset { return Err(FormatError::UnexpectedEof { - expected: block_heap_offset as usize, + expected: block.heap_offset as usize, available: target_offset as usize, }); } - let local_offset = (target_offset - block_heap_offset) as usize; - let pos = block_addr + let local_offset = (target_offset - block.heap_offset) as usize; + if let Some(pipeline) = &self.filter_pipeline { + let stored_len = usize::try_from(block.filtered_size) + .map_err(|_| heap_error("direct block size"))?; + let size = usize::try_from(block.size).map_err(|_| heap_error("direct block size"))?; + ensure_len(file_data, block.addr, stored_len)?; + let decoded = crate::filters::decompress_chunk_masked( + &file_data[block.addr..block.addr + stored_len], + pipeline, + size, + 1, + block.filter_mask, + )?; + ensure_len(&decoded, local_offset, length)?; + return Ok(decoded[local_offset..local_offset + length].to_vec()); + } + let pos = block + .addr .checked_add(local_offset) .ok_or(FormatError::UnexpectedEof { expected: usize::MAX, @@ -371,19 +631,13 @@ impl FractalHeapHeader { let iblock_header = 5 + offset_size as usize + block_offset_bytes; let mut pos = iblock_addr + iblock_header; - // Compute block sizes for each row using the doubling table let tw = self.table_width as u64; - let nrows_usize = nrows as usize; - - // Build table of (block_size, heap_offset) for each child entry let mut current_heap_offset = iblock_heap_offset; // Rows below max_direct_rows hold direct blocks; rows at/above hold // child indirect blocks. (NOT the FRHP "starting rows" field.) let start_indirect = self.max_direct_rows(); - - // Read child addresses for direct block rows let max_direct_rows = nrows_usize.min(start_indirect); for row in 0..max_direct_rows { @@ -393,35 +647,49 @@ impl FractalHeapHeader { let child_addr = read_offset(file_data, pos, offset_size)?; pos += offset_size as usize; - if self.io_filter_encoded_length > 0 { - // filtered_size(length_size) + filter_mask(4) - // Skip for now - we don't handle filtered direct blocks in fractal heaps - pos += 4; // filter_mask - simplified - } + // A filtered heap stores each direct block's filtered size + // (length_size) and filter mask (4) after its address. + let (filtered_size, filter_mask) = if self.filter_pipeline.is_some() { + let size = read_offset(file_data, pos, self.length_size)?; + pos += usize::from(self.length_size); + ensure_len(file_data, pos, 4)?; + let mask = u32::from_le_bytes([ + file_data[pos], + file_data[pos + 1], + file_data[pos + 2], + file_data[pos + 3], + ]); + pos += 4; + (size, mask) + } else { + (0, 0) + }; - if !is_undefined(child_addr, offset_size) { - let block_end = current_heap_offset + block_size; - if target_offset >= current_heap_offset && target_offset < block_end { - return self.read_from_direct_block( - file_data, - child_addr as usize, - block_size, - current_heap_offset, - target_offset, - length, - offset_size, - ); - } + let block_end = current_heap_offset.saturating_add(block_size); + if !is_undefined(child_addr, offset_size) + && target_offset >= current_heap_offset + && target_offset < block_end + { + return self.read_from_direct_block( + file_data, + DirectBlock { + addr: child_addr as usize, + size: block_size, + heap_offset: current_heap_offset, + filtered_size, + filter_mask, + }, + target_offset, + length, + ); } - current_heap_offset += block_size; + current_heap_offset = block_end; } } - // If we have indirect block rows - // A child indirect block in row r spans exactly that row's block size - // of heap space, so it has as many rows as a table of that total size - // needs (not `row - start_indirect + 1`, which undercounts and makes - // every object past the root's direct rows unreachable). + // Rows at and above `start_indirect` hold child indirect blocks. A + // child in row r spans exactly that row's block size of heap space, + // so it has as many rows as a table of that total size needs. for row in start_indirect..nrows_usize { let child_space = self.block_size_for_row(row); let child_nrows = self.rows_for_size(child_space); @@ -495,6 +763,16 @@ impl FractalHeapHeader { } } +/// A managed direct block's location, extent and (for a filtered heap) its +/// stored size and filter mask. +struct DirectBlock { + addr: usize, + size: u64, + heap_offset: u64, + filtered_size: u64, + filter_mask: u32, +} + #[cfg(test)] mod tests { use super::*; @@ -640,7 +918,7 @@ mod tests { let hdr = FractalHeapHeader::parse(&file_data, 0, 8, 8).unwrap(); // Build a managed heap ID: - // byte 0: type=0 (bits 6-7 = 00), version=0 (bits 4-5), reserved (bits 0-3) + // byte 0: version=0 (bits 6-7), type=0 (bits 4-5), reserved (bits 0-3) // bytes 1-6: offset (max_heap_size=16 bits) then length (remaining bits) // For offset=0, length=13: // payload = offset | (length << 16) = 0 | (13 << 16) = 0x000D0000 @@ -704,9 +982,46 @@ mod tests { fn invalid_heap_id_type() { let (file_data, _) = build_simple_heap(8, 8); let hdr = FractalHeapHeader::parse(&file_data, 0, 8, 8).unwrap(); - // Type = 1 (tiny) in bits 6-7 - let id = vec![0x40u8, 0, 0, 0, 0, 0, 0]; // bit 6 set = type 1 + // Type = 1 (huge) in bits 4-5 is not a managed ID + let id = vec![0x10u8, 0, 0, 0, 0, 0, 0]; let err = hdr.decode_managed_id(&id).unwrap_err(); assert_eq!(err, FormatError::InvalidHeapIdType(1)); } + + #[test] + fn tiny_object_is_read_from_the_id() { + let (file_data, _) = build_simple_heap(8, 8); + let hdr = FractalHeapHeader::parse(&file_data, 0, 8, 8).unwrap(); + // Type 2 (0x20), length - 1 in the low 4 bits, data after. + let id = [0x20 | 2, b'a', b'b', b'c', 0, 0, 0]; + assert_eq!(hdr.read_managed_object(&file_data, &id, 8).unwrap(), b"abc"); + // A length running past the ID is an error, not a short read. + let id = [0x20 | 9, b'a', b'b', b'c', 0, 0, 0]; + assert!(hdr.read_managed_object(&file_data, &id, 8).is_err()); + } + + #[test] + fn huge_object_with_a_direct_id() { + // With IDs long enough for an address and a length, libhdf5 stores + // huge objects' location in the ID instead of the huge-object B-tree. + let (mut file_data, _) = build_simple_heap(8, 8); + let mut hdr = FractalHeapHeader::parse(&file_data, 0, 8, 8).unwrap(); + hdr.heap_id_length = 17; + file_data[900..905].copy_from_slice(b"huge!"); + let mut id = vec![0x10u8]; + id.extend_from_slice(&900u64.to_le_bytes()); + id.extend_from_slice(&5u64.to_le_bytes()); + assert_eq!( + hdr.read_managed_object(&file_data, &id, 8).unwrap(), + b"huge!" + ); + } + + #[test] + fn unknown_heap_id_version_is_refused() { + let (file_data, _) = build_simple_heap(8, 8); + let hdr = FractalHeapHeader::parse(&file_data, 0, 8, 8).unwrap(); + let id = [0x40u8, 0, 0, 0, 0, 0, 0]; + assert!(hdr.read_managed_object(&file_data, &id, 8).is_err()); + } } diff --git a/crates/clawhdf5/tests/dense_storage_interop.rs b/crates/clawhdf5/tests/dense_storage_interop.rs index ecfff7a..7f8206a 100644 --- a/crates/clawhdf5/tests/dense_storage_interop.rs +++ b/crates/clawhdf5/tests/dense_storage_interop.rs @@ -8,7 +8,7 @@ use std::process::Command; -use clawhdf5::File; +use clawhdf5::{AttrValue, File}; fn python() -> String { std::env::var("CLAWHDF5_PYTHON").unwrap_or_else(|_| "python3".to_string()) @@ -163,3 +163,113 @@ fn dense_group_with_a_three_level_name_index() { vec![1.0] ); } + +/// The attribute names h5py reports for `obj`, sorted. +fn h5py_attr_names(path: &str, obj: &str) -> Vec { + let out = run_python(&format!( + "import h5py\n\ + with h5py.File(r'{path}', 'r') as f:\n\ + \x20 print('\\x1f'.join(sorted(f[{obj:?}].attrs.keys())))\n" + )); + out.split('\x1f') + .filter(|s| !s.is_empty()) + .map(str::to_string) + .collect() +} + +#[test] +fn dense_attribute_stored_as_a_huge_heap_object() { + skip_if_no_python!(); + // More than 8 attributes puts them in dense storage; one larger than the + // heap's 4 KiB managed-object limit is stored as a "huge" object, outside + // the heap blocks and found through the huge-object v2 B-tree. Its heap ID + // (type bits 4-5 = 1) was misread as a managed ID, and the error made + // every attribute on the object unreadable. NetCDF-4 files hit this + // (netcdf-c's issue671.nc / issue672.nc). + let (_dir, path) = h5py_file( + "d = f.create_dataset('d', data=[1.0])\n\ + for i in range(10):\n\ + \x20 d.attrs['a%d' % i] = i\n\ + d.attrs['big'] = np.arange(1024, dtype='f8')\n\ + d.attrs['bigger'] = np.arange(20000, dtype='i8') * 3\n", + ); + let f = File::open(&path).unwrap(); + let attrs = f.dataset("d").unwrap().attrs().unwrap(); + let mut names: Vec = attrs.keys().cloned().collect(); + names.sort(); + assert_eq!(names, h5py_attr_names(&path, "d")); + for i in 0..10 { + assert!( + matches!(attrs[&format!("a{i}")], AttrValue::I64(v) if v == i), + "a{i}: {:?}", + attrs[&format!("a{i}")] + ); + } + let big: Vec = (0..1024).map(f64::from).collect(); + assert!(matches!(&attrs["big"], AttrValue::F64Array(v) if *v == big)); + let bigger: Vec = (0..20000).map(|v| v * 3).collect(); + assert!(matches!(&attrs["bigger"], AttrValue::I64Array(v) if *v == bigger)); +} + +/// A group whose link heap has a deflate I/O filter (set on the group +/// creation property list), with 3 000 links and one link whose message is +/// larger than the heap's managed-object limit, so it is a huge object. +fn huge_link_group(filtered: bool) -> (tempfile::TempDir, String) { + let filter = if filtered { + "import ctypes, glob, os\n\ + lib = ctypes.CDLL(glob.glob(os.path.join(os.path.dirname(h5py.__file__), '..', 'h5py.libs', 'libhdf5-*.so*'))[0])\n\ + lib.H5Pset_deflate.argtypes = [ctypes.c_int64, ctypes.c_uint]\n\ + assert lib.H5Pset_deflate(gcpl.id, 6) >= 0\n" + } else { + "" + }; + h5py_file(&format!( + "t = f.create_dataset('t', data=[1.0])\n\ + gcpl = h5py.h5p.create(h5py.h5p.GROUP_CREATE)\n\ + {filter}\ + h5py.h5g.create(f.id, b'g', gcpl=gcpl)\n\ + g = f['g']\n\ + for i in range(3000):\n\ + \x20 g['l%05d' % i] = t\n\ + g['L' * 5000] = t\n" + )) +} + +fn check_huge_link_group(filtered: bool) { + let (_dir, path) = huge_link_group(filtered); + assert_same_listing(&path, "g"); + let f = File::open(&path).unwrap(); + let huge = format!("g/{}", "L".repeat(5000)); + assert_eq!(f.dataset(&huge).unwrap().read_f64().unwrap(), vec![1.0]); + assert_eq!( + f.dataset("g/l02999").unwrap().read_f64().unwrap(), + vec![1.0] + ); +} + +#[test] +fn dense_group_with_a_huge_link() { + skip_if_no_python!(); + check_huge_link_group(false); +} + +#[test] +fn dense_group_with_a_filtered_link_heap() { + skip_if_no_python!(); + // libhdf5 applies a group's filter pipeline to its link heap: direct + // blocks and huge objects are stored deflated, and the heap header + // carries the pipeline. The header's checksum was looked for in the + // wrong place, and filtered blocks were read raw. + if run_python( + "import h5py, glob, os\nprint(len(glob.glob(os.path.join(os.path.dirname(h5py.__file__), '..', 'h5py.libs', 'libhdf5-*.so*'))))", + ) == "0" + { + assert!( + !interop_required(), + "CLAWHDF5_REQUIRE_INTEROP=1 but h5py's bundled libhdf5 was not found" + ); + eprintln!("SKIP: h5py's bundled libhdf5 not found (needed to set the filter)"); + return; + } + check_huge_link_group(true); +} diff --git a/docs/known-issues.md b/docs/known-issues.md index 39cdcf2..774c70e 100644 --- a/docs/known-issues.md +++ b/docs/known-issues.md @@ -83,7 +83,8 @@ the VDS item, which is marked. - Soft links are left out of `datasets()`. - **Dense attributes:** a large attribute stored as a fractal-heap "huge" object makes every attribute on the object fail. This affects real NetCDF - files (`issue671.nc`). + files (`issue671.nc`). **Fixed 2026-09-25:** huge and tiny heap objects, + and filtered heaps, are read. - **Other readers:** - VL-string datasets are not readable through `File`. - Metadata cache images are not supported.