Merge branch 'fix/p1-attrs-links' into fix/p1-read-gaps

# Conflicts:
#	crates/clawhdf5-format/src/attribute.rs
#	crates/clawhdf5-format/src/group_v1.rs
#	crates/clawhdf5/src/lazy.rs
#	crates/clawhdf5/src/mmap_file.rs
#	docs/known-issues.md

crates/clawhdf5-format/src/attribute.rs

@@ -394,43 +394,80 @@ pub fn find_attribute<'a>(
 ///
 /// Use this instead of `extract_attributes` when reading files that may use dense storage
 /// (e.g., objects with many attributes, typically >8).
+///
+/// Fails if any attribute cannot be read; see [`extract_attributes_tolerant`]
+/// to read the others.
 pub fn extract_attributes_full(
     file_data: &[u8],
     header: &ObjectHeader,
     offset_size: u8,
     length_size: u8,
+) -> Result<Vec<AttributeMessage>, FormatError> {
+    extract_attributes_with(file_data, header, offset_size, length_size, &mut Err)
+}
+
+/// Like [`extract_attributes_full`], but an attribute that cannot be read
+/// (a corrupt or unsupported attribute message, or a heap object that cannot
+/// be located) is left out and its error returned alongside the attributes
+/// that could be read, instead of failing them all.
+///
+/// Errors in the structures that index the attributes (the Attribute Info
+/// message, the dense-storage heap header or B-tree) still fail the call:
+/// then it is unknown which attributes exist at all.
+pub fn extract_attributes_tolerant(
+    file_data: &[u8],
+    header: &ObjectHeader,
+    offset_size: u8,
+    length_size: u8,
+) -> Result<(Vec<AttributeMessage>, Vec<FormatError>), FormatError> {
+    let mut errors = Vec::new();
+    let attrs = extract_attributes_with(file_data, header, offset_size, length_size, &mut |e| {
+        errors.push(e);
+        Ok(())
+    })?;
+    Ok((attrs, errors))
+}
+
+/// Read every attribute; each one that fails goes to `on_error`, which
+/// either stops the read (returns the error) or skips that attribute.
+fn extract_attributes_with(
+    file_data: &[u8],
+    header: &ObjectHeader,
+    offset_size: u8,
+    length_size: u8,
+    on_error: &mut dyn FnMut(FormatError) -> Result<(), FormatError>,
 ) -> Result<Vec<AttributeMessage>, FormatError> {
     let mut attrs = Vec::new();
 
     // Collect compact attributes (inline in OH)
     for msg in &header.messages {
         if msg.msg_type == MessageType::Attribute {
-            if shared_message::is_shared(msg.flags) {
+            let attr = if shared_message::is_shared(msg.flags) {
                 // Shared attribute: resolve the reference to get actual attribute data
-                let shared_ref =
-                    shared_message::parse_shared_ref_sized(&msg.data, offset_size, length_size)?;
-                let resolved_data = shared_message::resolve_shared_message(
-                    file_data,
-                    &shared_ref,
-                    MessageType::Attribute,
-                    offset_size,
-                    length_size,
-                )?;
-                let attr = AttributeMessage::parse_in_file(
-                    &resolved_data,
-                    file_data,
-                    offset_size,
-                    length_size,
-                )?;
-                attrs.push(attr);
+                shared_message::parse_shared_ref_sized(&msg.data, offset_size, length_size)
+                    .and_then(|shared_ref| {
+                        shared_message::resolve_shared_message(
+                            file_data,
+                            &shared_ref,
+                            MessageType::Attribute,
+                            offset_size,
+                            length_size,
+                        )
+                    })
+                    .and_then(|resolved| {
+                        AttributeMessage::parse_in_file(
+                            &resolved,
+                            file_data,
+                            offset_size,
+                            length_size,
+                        )
+                    })
             } else {
-                let attr = AttributeMessage::parse_in_file(
-                    &msg.data,
-                    file_data,
-                    offset_size,
-                    length_size,
-                )?;
-                attrs.push(attr);
+                AttributeMessage::parse_in_file(&msg.data, file_data, offset_size, length_size)
+            };
+            match attr {
+                Ok(attr) => attrs.push(attr),
+                Err(e) => on_error(e)?,
             }
         }
     }
@@ -440,9 +477,15 @@ pub fn extract_attributes_full(
     if let Some(info) = attr_info
         && let Some(fh_addr) = info.fractal_heap_address
     {
-        let dense_attrs =
-            extract_dense_attributes(file_data, &info, fh_addr, offset_size, length_size)?;
-        attrs.extend(dense_attrs);
+        extract_dense_attributes(
+            file_data,
+            &info,
+            fh_addr,
+            offset_size,
+            length_size,
+            &mut attrs,
+            on_error,
+        )?;
     }
 
     Ok(attrs)
@@ -469,7 +512,9 @@ fn extract_dense_attributes(
     fh_addr: u64,
     offset_size: u8,
     length_size: u8,
-) -> Result<Vec<AttributeMessage>, FormatError> {
+    attrs: &mut Vec<AttributeMessage>,
+    on_error: &mut dyn FnMut(FormatError) -> Result<(), FormatError>,
+) -> Result<(), FormatError> {
     // Parse fractal heap
     let fh = FractalHeapHeader::parse(file_data, fh_addr as usize, offset_size, length_size)?;
 
@@ -483,28 +528,32 @@ fn extract_dense_attributes(
     let btree_hdr = BTreeV2Header::parse(file_data, btree_addr as usize, offset_size, length_size)?;
     let records = collect_btree_v2_records(file_data, &btree_hdr, offset_size, length_size)?;
 
-    let mut attrs = Vec::new();
     for record in &records {
         // Per HDF5 spec, both type 8 and type 9 records start with heap_id:
         //   Type 8: heap_id(8) + msg_flags(1) + creation_order(4) + hash(4)
         //   Type 9: heap_id(8) + msg_flags(1) + creation_order(4)
-        let id_offset = 0;
-
-        if record.data.len() < id_offset + fh.heap_id_length as usize {
+        let id_len = fh.heap_id_length as usize;
+        let Some(id_bytes) = record.data.get(..id_len) else {
+            on_error(FormatError::UnexpectedEof {
+                expected: id_len,
+                available: record.data.len(),
+            })?;
             continue;
-        }
-        let id_bytes = &record.data[id_offset..id_offset + fh.heap_id_length as usize];
-
-        // Read attribute message from fractal heap
-        let attr_data = fh.read_managed_object(file_data, id_bytes, offset_size)?;
+        };
 
         // The data in the heap is a complete attribute message
-        let attr =
-            AttributeMessage::parse_in_file(&attr_data, file_data, offset_size, length_size)?;
-        attrs.push(attr);
+        let attr = fh
+            .read_managed_object(file_data, id_bytes, offset_size)
+            .and_then(|attr_data| {
+                AttributeMessage::parse_in_file(&attr_data, file_data, offset_size, length_size)
+            });
+        match attr {
+            Ok(attr) => attrs.push(attr),
+            Err(e) => on_error(e)?,
+        }
     }
 
-    Ok(attrs)
+    Ok(())
 }
 
 #[cfg(test)]

crates/clawhdf5-format/src/btree_v2.rs

@@ -323,39 +323,21 @@ fn collect_internal_records(
     let records_start = pos;
     pos += records_total;
 
-    // Compute sizes for child pointers
-    // max_records at child depth - for variable-width nrec encoding
+    // Child pointer layout, as libhdf5 computes it (H5B2__hdr_init): the
+    // child's record count is always encoded in the width needed for a
+    // *leaf's* maximum, and — below the first internal level — the child
+    // subtree's total record count in the width needed for the most records
+    // a subtree of that depth can hold.
     let child_depth = depth - 1;
-    let max_nrec_child = if child_depth == 0 {
-        max_leaf_nrec
-    } else {
-        // For internal nodes at child_depth, the true max_nrec depends on the
-        // node size, record size, and the recursive width of child pointer
-        // entries (which themselves depend on max_nrec at deeper levels).
-        // Computing the exact value requires iterating from the leaf level
-        // upward, as described in the HDF5 spec (III.A.2 "Computing the Size
-        // of B-tree Nodes").
-        //
-        // We use `max_leaf_nrec * 2` as a conservative upper bound. This
-        // over-estimates the nrec encoding width, which means we may read
-        // slightly more bytes per child pointer than strictly necessary, but
-        // never fewer. The over-read bytes are harmless because we only
-        // decode `num_records` entries (the actual count from the node header).
-        //
-        // Known limitation: for very deep trees (depth > 3) with small record
-        // sizes, the true max could exceed this estimate, causing us to
-        // under-allocate the nrec encoding width and misparse child pointers.
-        // In practice, HDF5 B-tree v2 depths rarely exceed 2-3.
-        max_leaf_nrec * 2
-    };
-    let nrec_width = bytes_for_max_records(max_nrec_child);
-
-    // Total records in subtree width (only if depth > 1)
+    let nrec_width = bytes_for_max_records(max_leaf_nrec);
     let total_nrec_width = if depth > 1 {
-        // Width to hold total records in a subtree
-        // We compute max possible total records at this subtree depth
-        let max_total = header_max_total_records(max_leaf_nrec, depth - 1);
-        bytes_for_max_records(max_total)
+        bytes_for_max_records(cum_max_records(
+            node_size,
+            record_size,
+            offset_size,
+            max_leaf_nrec,
+            child_depth,
+        ))
     } else {
         0
     };
@@ -435,14 +417,36 @@ fn collect_internal_records(
     Ok(())
 }
 
-/// Estimate maximum total records at a given depth (for variable-width encoding).
-fn header_max_total_records(max_leaf_nrec: u64, depth: u16) -> u64 {
-    // Conservative: branching factor * max_leaf at each level
-    let mut total = max_leaf_nrec;
-    for _ in 0..depth {
-        total = total.saturating_mul(max_leaf_nrec.max(2));
+/// Most records a subtree whose root is at `depth` can hold (libhdf5's
+/// `cum_max_nrec`): a leaf holds `max_leaf_nrec`; an internal node at depth
+/// `d` holds `max_nrec(d)` records and `max_nrec(d) + 1` subtrees of depth
+/// `d - 1`, where `max_nrec(d)` is what fits in a node once each record is
+/// paired with a child pointer of the width depth `d` needs.
+fn cum_max_records(
+    node_size: u32,
+    record_size: u16,
+    offset_size: u8,
+    max_leaf_nrec: u64,
+    depth: u16,
+) -> u64 {
+    // Internal node overhead: signature(4) + version(1) + type(1) + checksum(4).
+    const PREFIX: u64 = 10;
+    let nrec_width = bytes_for_max_records(max_leaf_nrec) as u64;
+    let mut cum = max_leaf_nrec;
+    let mut cum_width = 0u64;
+    for d in 1..=depth {
+        let ptr = u64::from(offset_size) + nrec_width + if d > 1 { cum_width } else { 0 };
+        let max_nrec = u64::from(node_size)
+            .saturating_sub(PREFIX)
+            .saturating_sub(ptr)
+            / (u64::from(record_size) + ptr).max(1);
+        cum = max_nrec
+            .saturating_add(1)
+            .saturating_mul(cum)
+            .saturating_add(max_nrec);
+        cum_width = bytes_for_max_records(cum) as u64;
     }
-    total
+    cum
 }
 
 #[cfg(test)]
@@ -512,9 +516,15 @@ mod tests {
         child_nrec: u64,
     ) -> Vec<u8> {
         let max_leaf = max_records_leaf(node_size, record_size);
-        let nrec_width = bytes_for_max_records(if depth == 1 { max_leaf } else { max_leaf * 2 });
+        let nrec_width = bytes_for_max_records(max_leaf);
         let total_width = if depth > 1 {
-            bytes_for_max_records(header_max_total_records(max_leaf, depth - 1))
+            bytes_for_max_records(cum_max_records(
+                node_size,
+                record_size,
+                8,
+                max_leaf,
+                depth - 1,
+            ))
         } else {
             0
         };
@@ -673,4 +683,18 @@ mod tests {
         let records = collect_btree_v2_records(&header, &hdr, 8, 8).unwrap();
         assert!(records.is_empty());
     }
+
+    #[test]
+    fn subtree_capacity_matches_libhdf5() {
+        // A link-name index (11-byte records, 512-byte nodes, 8-byte
+        // addresses): libhdf5's H5B2__hdr_init gives 45 records per leaf,
+        // then cum_max_nrec 1 149 at depth 1 and 26 449 at depth 2 — two
+        // bytes of subtree count in a depth-3 root's child pointers, where
+        // leaf_max^3 = 91 125 would need three.
+        let leaf = max_records_leaf(512, 11);
+        assert_eq!(leaf, 45);
+        assert_eq!(cum_max_records(512, 11, 8, leaf, 0), 45);
+        assert_eq!(cum_max_records(512, 11, 8, leaf, 1), 1_149);
+        assert_eq!(cum_max_records(512, 11, 8, leaf, 2), 26_449);
+    }
 }

crates/clawhdf5-format/src/file_writer.rs

@@ -1947,6 +1947,12 @@ mod tests {
             root_block_address: 0,
             current_rows_in_root_indirect_block: 0,
             managed_objects_count: 0,
+            huge_btree_address: u64::MAX,
+            filter_pipeline: None,
+            root_direct_block_filtered_size: 0,
+            root_direct_block_filter_mask: 0,
+            offset_size: 8,
+            length_size: 8,
         };
         let (off, len) = fh.decode_managed_id(&id).unwrap();
         assert_eq!(off, 100);

crates/clawhdf5-format/src/fractal_heap.rs

@@ -1,12 +1,14 @@
 //! HDF5 Fractal Heap parsing for v2 group link storage.
 
 #[cfg(not(feature = "std"))]
-use alloc::vec::Vec;
+use alloc::{format, vec::Vec};
 
 #[cfg(feature = "checksum")]
 use byteorder::{ByteOrder, LittleEndian};
 
+use crate::btree_v2::{BTreeV2Header, collect_btree_v2_records};
 use crate::error::FormatError;
+use crate::filter_pipeline::FilterPipeline;
 
 /// Parsed fractal heap header (signature "FRHP").
 #[derive(Debug, Clone)]
@@ -33,6 +35,23 @@ pub struct FractalHeapHeader {
     pub current_rows_in_root_indirect_block: u16,
     /// Total number of managed objects.
     pub managed_objects_count: u64,
+    /// Address of the v2 B-tree indexing "huge" objects (undefined address
+    /// when the heap has none). Huge objects are those larger than
+    /// `max_managed_object_size`; they live outside the heap's blocks.
+    pub huge_btree_address: u64,
+    /// The heap's I/O filter pipeline, if it has one. It applies to managed
+    /// direct blocks and to huge objects.
+    pub filter_pipeline: Option<FilterPipeline>,
+    /// Stored (filtered) size of the root direct block; meaningful only when
+    /// the heap is filtered and its root is a direct block.
+    pub root_direct_block_filtered_size: u64,
+    /// Filter mask of the root direct block (bit *i* set = filter *i*
+    /// skipped); meaningful only when the heap is filtered.
+    pub root_direct_block_filter_mask: u32,
+    /// Size of addresses in the file ("Size of Offsets").
+    pub offset_size: u8,
+    /// Size of lengths in the file ("Size of Lengths").
+    pub length_size: u8,
 }
 
 fn read_offset(data: &[u8], pos: usize, size: u8) -> Result<u64, FormatError> {
@@ -79,6 +98,38 @@ fn is_undefined(val: u64, offset_size: u8) -> bool {
     }
 }
 
+/// Little-endian unsigned integer of up to 8 bytes.
+fn le_uint(bytes: &[u8]) -> u64 {
+    bytes
+        .iter()
+        .take(8)
+        .enumerate()
+        .fold(0u64, |acc, (i, &b)| acc | (u64::from(b) << (i * 8)))
+}
+
+fn heap_error(msg: &str) -> FormatError {
+    FormatError::ChunkedReadError(format!("fractal heap: {msg}"))
+}
+
+/// Heap ID type, from bits 4-5 of an ID's first byte (libhdf5's
+/// `H5HF_ID_TYPE_MASK`, 0x30); bits 6-7 are the ID version, which must be 0.
+const HEAP_ID_MANAGED: u8 = 0;
+const HEAP_ID_HUGE: u8 = 1;
+const HEAP_ID_TINY: u8 = 2;
+
+/// The type (0 managed, 1 huge, 2 tiny) of a heap ID from its first byte,
+/// refusing an ID version other than 0.
+fn heap_id_type(first: u8) -> Result<u8, FormatError> {
+    if first >> 6 != 0 {
+        return Err(heap_error("unsupported heap ID version"));
+    }
+    Ok((first >> 4) & 0x03)
+}
+
+/// v2 B-tree record types indexing a heap's huge objects.
+const BTREE_HUGE_INDIRECT: u8 = 1;
+const BTREE_HUGE_INDIRECT_FILTERED: u8 = 2;
+
 impl FractalHeapHeader {
     /// Parse a fractal heap header at the given offset.
     pub fn parse(
@@ -122,11 +173,17 @@ impl FractalHeapHeader {
         ]);
         pos += 4;
 
-        // Skip several fixed fields: next_huge_object_id(ls), btree_huge_objects_address(os),
-        // free_space_managed_blocks(ls), managed_block_free_space_manager_address(os),
+        // next_huge_object_id (length_size)
+        ensure_len(file_data, pos, ls)?;
+        pos += ls;
+        // btree_huge_objects_address (offset_size)
+        let huge_btree_address = read_offset(file_data, pos, offset_size)?;
+        pos += os;
+
+        // Skip: free_space_managed_blocks(ls), managed_block_free_space_manager_address(os),
         // managed_space_in_heap(ls), allocated_managed_space_in_heap(ls),
         // direct_block_allocation_iterator_offset(ls)
-        let skip_size = 5 * ls + 2 * os;
+        let skip_size = 4 * ls + os;
         ensure_len(file_data, pos, skip_size)?;
         pos += skip_size;
 
@@ -134,14 +191,9 @@ impl FractalHeapHeader {
         let managed_objects_count = read_offset(file_data, pos, length_size)?;
         pos += ls;
 
-        // huge_objects_size (length_size)
-        pos += ls;
-        // huge_objects_count (length_size)
-        pos += ls;
-        // tiny_objects_size (length_size)
-        pos += ls;
-        // tiny_objects_count (length_size)
-        pos += ls;
+        // huge_objects_size, huge_objects_count, tiny_objects_size,
+        // tiny_objects_count (length_size each)
+        pos += 4 * ls;
 
         // table_width (2)
         ensure_len(file_data, pos, 2)?;
@@ -175,16 +227,28 @@ impl FractalHeapHeader {
         ensure_len(file_data, pos, 2)?;
         let current_rows_in_root_indirect_block =
             u16::from_le_bytes([file_data[pos], file_data[pos + 1]]);
-        #[allow(unused_variables, unused_mut, unused_assignments)]
-        let mut pos = pos + 2;
+        pos += 2;
 
-        // Skip IO filter encoded info if present
+        // With I/O filters: root direct block's filtered size (length_size),
+        // its filter mask (4), then the encoded filter pipeline message.
+        let mut filter_pipeline = None;
+        let mut root_direct_block_filtered_size = 0;
+        let mut root_direct_block_filter_mask = 0;
         if io_filter_encoded_length > 0 {
-            // root_block_filter_info_size (length_size) + filter_mask (4)
-            #[allow(unused_assignments)]
-            {
-                pos += ls + 4;
-            }
+            root_direct_block_filtered_size = read_offset(file_data, pos, length_size)?;
+            pos += ls;
+            ensure_len(file_data, pos, 4)?;
+            root_direct_block_filter_mask = u32::from_le_bytes([
+                file_data[pos],
+                file_data[pos + 1],
+                file_data[pos + 2],
+                file_data[pos + 3],
+            ]);
+            pos += 4;
+            let n = io_filter_encoded_length as usize;
+            ensure_len(file_data, pos, n)?;
+            filter_pipeline = Some(FilterPipeline::parse(&file_data[pos..pos + n])?);
+            pos += n;
         }
 
         // Validate header checksum
@@ -200,6 +264,8 @@ impl FractalHeapHeader {
                 });
             }
         }
+        #[cfg(not(feature = "checksum"))]
+        let _ = pos;
 
         Ok(FractalHeapHeader {
             heap_id_length,
@@ -213,13 +279,19 @@ impl FractalHeapHeader {
             root_block_address,
             current_rows_in_root_indirect_block,
             managed_objects_count,
+            huge_btree_address,
+            filter_pipeline,
+            root_direct_block_filtered_size,
+            root_direct_block_filter_mask,
+            offset_size,
+            length_size,
         })
     }
 
     /// Decode a managed heap ID into (offset_in_heap, object_length).
     ///
     /// The heap ID layout for managed objects (type 0):
-    /// - Byte 0: bits 6-7 = type (0), bits 4-5 = version (0), bits 0-3 = reserved
+    /// - Byte 0: bits 6-7 = version (0), bits 4-5 = type (0), bits 0-3 = reserved
     /// - Bytes 1+: offset (max_heap_size bits, LE) then length (remaining bits, LE)
     pub fn decode_managed_id(&self, id_bytes: &[u8]) -> Result<(u64, u64), FormatError> {
         if id_bytes.is_empty() {
@@ -229,8 +301,8 @@ impl FractalHeapHeader {
             });
         }
 
-        let id_type = (id_bytes[0] >> 6) & 0x03;
-        if id_type != 0 {
+        let id_type = heap_id_type(id_bytes[0])?;
+        if id_type != HEAP_ID_MANAGED {
             return Err(FormatError::InvalidHeapIdType(id_type));
         }
 
@@ -269,12 +341,183 @@ impl FractalHeapHeader {
         Ok((heap_offset, length_val))
     }
 
-    /// Read a managed object from the heap given its raw heap ID bytes.
+    /// Read any object from the heap given its raw heap ID bytes: managed
+    /// (stored in the heap's blocks), huge (stored outside them, found
+    /// directly from the ID or through the huge-object v2 B-tree, optionally
+    /// filtered) or tiny (stored in the ID itself).
+    ///
+    /// Despite its name this accepts every ID type; `offset_size` must match
+    /// the one the header was parsed with.
     pub fn read_managed_object(
         &self,
         file_data: &[u8],
         id_bytes: &[u8],
         offset_size: u8,
+    ) -> Result<Vec<u8>, FormatError> {
+        let Some(&first) = id_bytes.first() else {
+            return Err(FormatError::UnexpectedEof {
+                expected: 1,
+                available: 0,
+            });
+        };
+        match heap_id_type(first)? {
+            HEAP_ID_MANAGED => self.read_heap_managed(file_data, id_bytes, offset_size),
+            HEAP_ID_HUGE => self.read_huge_object(file_data, id_bytes),
+            HEAP_ID_TINY => self.read_tiny_object(id_bytes),
+            other => Err(FormatError::InvalidHeapIdType(other)),
+        }
+    }
+
+    /// Whether a huge object's ID holds its address and length directly
+    /// (libhdf5 does this when they fit in the ID), rather than a key into
+    /// the huge-object B-tree.
+    fn huge_ids_direct(&self) -> bool {
+        let room = usize::from(self.heap_id_length).saturating_sub(1);
+        let os = usize::from(self.offset_size);
+        let ls = usize::from(self.length_size);
+        if self.filter_pipeline.is_some() {
+            room >= os + ls + 4 + ls
+        } else {
+            room >= os + ls
+        }
+    }
+
+    /// Read a huge object (heap ID type 1).
+    fn read_huge_object(&self, file_data: &[u8], id: &[u8]) -> Result<Vec<u8>, FormatError> {
+        let os = usize::from(self.offset_size);
+        let ls = usize::from(self.length_size);
+        // (address, stored length, filter mask, decoded length); the last two
+        // only matter for a filtered heap.
+        let (addr, stored_len, mask, mem_len) = if self.huge_ids_direct() {
+            let body = &id[1..];
+            let need = if self.filter_pipeline.is_some() {
+                os + ls + 4 + ls
+            } else {
+                os + ls
+            };
+            ensure_len(body, 0, need)?;
+            let addr = le_uint(&body[..os]);
+            let len = le_uint(&body[os..os + ls]);
+            if self.filter_pipeline.is_some() {
+                let mask = u32::from_le_bytes([
+                    body[os + ls],
+                    body[os + ls + 1],
+                    body[os + ls + 2],
+                    body[os + ls + 3],
+                ]);
+                let mem = le_uint(&body[os + ls + 4..os + ls + 4 + ls]);
+                (addr, len, mask, mem)
+            } else {
+                (addr, len, 0, len)
+            }
+        } else {
+            let key_len = (usize::from(self.heap_id_length).saturating_sub(1)).min(8);
+            ensure_len(id, 1, key_len)?;
+            let key = le_uint(&id[1..1 + key_len]);
+            self.find_huge_record(file_data, key)?
+        };
+
+        let start = usize::try_from(addr).map_err(|_| heap_error("huge object address"))?;
+        let len = usize::try_from(stored_len).map_err(|_| heap_error("huge object length"))?;
+        ensure_len(file_data, start, len)?;
+        let stored = &file_data[start..start + len];
+        match &self.filter_pipeline {
+            None => Ok(stored.to_vec()),
+            Some(pipeline) => {
+                let mem = usize::try_from(mem_len).map_err(|_| heap_error("huge object size"))?;
+                let out = crate::filters::decompress_chunk_masked(stored, pipeline, mem, 1, mask)?;
+                if out.len() != mem {
+                    return Err(heap_error("filtered huge object decoded to the wrong size"));
+                }
+                Ok(out)
+            }
+        }
+    }
+
+    /// Look up huge object `key` in the huge-object v2 B-tree, returning
+    /// (address, stored length, filter mask, decoded length).
+    fn find_huge_record(
+        &self,
+        file_data: &[u8],
+        key: u64,
+    ) -> Result<(u64, u64, u32, u64), FormatError> {
+        if is_undefined(self.huge_btree_address, self.offset_size) {
+            return Err(heap_error(
+                "huge object ID but the heap has no huge-object index",
+            ));
+        }
+        let hdr = BTreeV2Header::parse(
+            file_data,
+            self.huge_btree_address as usize,
+            self.offset_size,
+            self.length_size,
+        )?;
+        let os = usize::from(self.offset_size);
+        let ls = usize::from(self.length_size);
+        let filtered = self.filter_pipeline.is_some();
+        let (expected_type, rec_len) = if filtered {
+            (BTREE_HUGE_INDIRECT_FILTERED, os + ls + 4 + ls + ls)
+        } else {
+            (BTREE_HUGE_INDIRECT, os + ls + ls)
+        };
+        if hdr.tree_type != expected_type || usize::from(hdr.record_size) < rec_len {
+            return Err(heap_error("unexpected huge-object B-tree record type"));
+        }
+        let records =
+            collect_btree_v2_records(file_data, &hdr, self.offset_size, self.length_size)?;
+        for rec in &records {
+            let d = &rec.data;
+            if d.len() < rec_len {
+                continue;
+            }
+            let addr = le_uint(&d[..os]);
+            let len = le_uint(&d[os..os + ls]);
+            if filtered {
+                let mask = u32::from_le_bytes([
+                    d[os + ls],
+                    d[os + ls + 1],
+                    d[os + ls + 2],
+                    d[os + ls + 3],
+                ]);
+                let mem = le_uint(&d[os + ls + 4..os + 2 * ls + 4]);
+                let id = le_uint(&d[os + 2 * ls + 4..os + 3 * ls + 4]);
+                if id == key {
+                    return Ok((addr, len, mask, mem));
+                }
+            } else {
+                let id = le_uint(&d[os + ls..os + 2 * ls]);
+                if id == key {
+                    return Ok((addr, len, 0, len));
+                }
+            }
+        }
+        Err(heap_error("huge object not found in its B-tree"))
+    }
+
+    /// Read a tiny object (heap ID type 2), stored in the ID itself.
+    fn read_tiny_object(&self, id: &[u8]) -> Result<Vec<u8>, FormatError> {
+        // libhdf5 uses a one-byte length (low 4 bits of byte 0) unless the ID
+        // is long enough to need 12 bits, which then borrow byte 1.
+        let extended = usize::from(self.heap_id_length).saturating_sub(1) > 17;
+        let (len, start) = if extended {
+            ensure_len(id, 0, 2)?;
+            (
+                ((usize::from(id[0] & 0x0F)) << 8 | usize::from(id[1])) + 1,
+                2,
+            )
+        } else {
+            (usize::from(id[0] & 0x0F) + 1, 1)
+        };
+        ensure_len(id, start, len)?;
+        Ok(id[start..start + len].to_vec())
+    }
+
+    /// Read a managed object (heap ID type 0).
+    fn read_heap_managed(
+        &self,
+        file_data: &[u8],
+        id_bytes: &[u8],
+        offset_size: u8,
     ) -> Result<Vec<u8>, FormatError> {
         let (heap_offset, obj_len) = self.decode_managed_id(id_bytes)?;
 
@@ -289,12 +532,15 @@ impl FractalHeapHeader {
             // Root is a direct block
             self.read_from_direct_block(
                 file_data,
-                self.root_block_address as usize,
-                self.starting_block_size,
-                0, // block offset in heap = 0 for root
+                DirectBlock {
+                    addr: self.root_block_address as usize,
+                    size: self.starting_block_size,
+                    heap_offset: 0,
+                    filtered_size: self.root_direct_block_filtered_size,
+                    filter_mask: self.root_direct_block_filter_mask,
+                },
                 heap_offset,
                 obj_len as usize,
-                offset_size,
             )
         } else {
             // Root is an indirect block — limit recursion to 64 levels
@@ -313,27 +559,41 @@ impl FractalHeapHeader {
 
     /// Read an object from a direct block.
     ///
-    /// The heap offset is relative to the start of the block (including its header),
-    /// so we just add it to the block address minus the block's heap offset.
-    #[allow(clippy::too_many_arguments)]
+    /// The heap offset is relative to the start of the block (including its
+    /// header), so we just add it to the block address minus the block's heap
+    /// offset. A filtered heap stores each direct block (header included)
+    /// through its filter pipeline, so the block is decoded first.
     fn read_from_direct_block(
         &self,
         file_data: &[u8],
-        block_addr: usize,
-        _block_size: u64,
-        block_heap_offset: u64,
+        block: DirectBlock,
         target_offset: u64,
         length: usize,
-        _offset_size: u8,
     ) -> Result<Vec<u8>, FormatError> {
-        if target_offset < block_heap_offset {
+        if target_offset < block.heap_offset {
             return Err(FormatError::UnexpectedEof {
-                expected: block_heap_offset as usize,
+                expected: block.heap_offset as usize,
                 available: target_offset as usize,
             });
         }
-        let local_offset = (target_offset - block_heap_offset) as usize;
-        let pos = block_addr
+        let local_offset = (target_offset - block.heap_offset) as usize;
+        if let Some(pipeline) = &self.filter_pipeline {
+            let stored_len = usize::try_from(block.filtered_size)
+                .map_err(|_| heap_error("direct block size"))?;
+            let size = usize::try_from(block.size).map_err(|_| heap_error("direct block size"))?;
+            ensure_len(file_data, block.addr, stored_len)?;
+            let decoded = crate::filters::decompress_chunk_masked(
+                &file_data[block.addr..block.addr + stored_len],
+                pipeline,
+                size,
+                1,
+                block.filter_mask,
+            )?;
+            ensure_len(&decoded, local_offset, length)?;
+            return Ok(decoded[local_offset..local_offset + length].to_vec());
+        }
+        let pos = block
+            .addr
             .checked_add(local_offset)
             .ok_or(FormatError::UnexpectedEof {
                 expected: usize::MAX,
@@ -371,19 +631,13 @@ impl FractalHeapHeader {
         let iblock_header = 5 + offset_size as usize + block_offset_bytes;
         let mut pos = iblock_addr + iblock_header;
 
-        // Compute block sizes for each row using the doubling table
         let tw = self.table_width as u64;
-
         let nrows_usize = nrows as usize;
-
-        // Build table of (block_size, heap_offset) for each child entry
         let mut current_heap_offset = iblock_heap_offset;
 
         // Rows below max_direct_rows hold direct blocks; rows at/above hold
         // child indirect blocks. (NOT the FRHP "starting rows" field.)
         let start_indirect = self.max_direct_rows();
-
-        // Read child addresses for direct block rows
         let max_direct_rows = nrows_usize.min(start_indirect);
 
         for row in 0..max_direct_rows {
@@ -393,60 +647,74 @@ impl FractalHeapHeader {
                 let child_addr = read_offset(file_data, pos, offset_size)?;
                 pos += offset_size as usize;
 
-                if self.io_filter_encoded_length > 0 {
-                    // filtered_size(length_size) + filter_mask(4)
-                    // Skip for now - we don't handle filtered direct blocks in fractal heaps
-                    pos += 4; // filter_mask - simplified
-                }
+                // A filtered heap stores each direct block's filtered size
+                // (length_size) and filter mask (4) after its address.
+                let (filtered_size, filter_mask) = if self.filter_pipeline.is_some() {
+                    let size = read_offset(file_data, pos, self.length_size)?;
+                    pos += usize::from(self.length_size);
+                    ensure_len(file_data, pos, 4)?;
+                    let mask = u32::from_le_bytes([
+                        file_data[pos],
+                        file_data[pos + 1],
+                        file_data[pos + 2],
+                        file_data[pos + 3],
+                    ]);
+                    pos += 4;
+                    (size, mask)
+                } else {
+                    (0, 0)
+                };
 
-                if !is_undefined(child_addr, offset_size) {
-                    let block_end = current_heap_offset + block_size;
-                    if target_offset >= current_heap_offset && target_offset < block_end {
-                        return self.read_from_direct_block(
-                            file_data,
-                            child_addr as usize,
-                            block_size,
-                            current_heap_offset,
-                            target_offset,
-                            length,
-                            offset_size,
-                        );
-                    }
+                let block_end = current_heap_offset.saturating_add(block_size);
+                if !is_undefined(child_addr, offset_size)
+                    && target_offset >= current_heap_offset
+                    && target_offset < block_end
+                {
+                    return self.read_from_direct_block(
+                        file_data,
+                        DirectBlock {
+                            addr: child_addr as usize,
+                            size: block_size,
+                            heap_offset: current_heap_offset,
+                            filtered_size,
+                            filter_mask,
+                        },
+                        target_offset,
+                        length,
+                    );
                 }
-                current_heap_offset += block_size;
+                current_heap_offset = block_end;
             }
         }
 
-        // If we have indirect block rows
+        // Rows at and above `start_indirect` hold child indirect blocks. A
+        // child in row r spans exactly that row's block size of heap space,
+        // so it has as many rows as a table of that total size needs.
         for row in start_indirect..nrows_usize {
-            let _block_size = self.block_size_for_row(row);
-            let child_nrows = row - start_indirect + 1;
+            let child_space = self.block_size_for_row(row);
+            let child_nrows = self.rows_for_size(child_space);
 
             for _col in 0..tw {
                 let child_addr = read_offset(file_data, pos, offset_size)?;
                 pos += offset_size as usize;
 
-                if !is_undefined(child_addr, offset_size) {
-                    // Calculate total heap space covered by this indirect block child
-                    let total_child_space = self.indirect_block_heap_size(child_nrows);
-                    let block_end = current_heap_offset + total_child_space;
-                    if target_offset >= current_heap_offset && target_offset < block_end {
-                        return self.read_from_indirect_block(
-                            file_data,
-                            child_addr as usize,
-                            child_nrows as u16,
-                            current_heap_offset,
-                            target_offset,
-                            length,
-                            offset_size,
-                            depth_remaining - 1,
-                        );
-                    }
-                    current_heap_offset += total_child_space;
-                } else {
-                    let total_child_space = self.indirect_block_heap_size(child_nrows);
-                    current_heap_offset += total_child_space;
+                let block_end = current_heap_offset.saturating_add(child_space);
+                if !is_undefined(child_addr, offset_size)
+                    && target_offset >= current_heap_offset
+                    && target_offset < block_end
+                {
+                    return self.read_from_indirect_block(
+                        file_data,
+                        child_addr as usize,
+                        child_nrows,
+                        current_heap_offset,
+                        target_offset,
+                        length,
+                        offset_size,
+                        depth_remaining - 1,
+                    );
                 }
+                current_heap_offset = block_end;
             }
         }
 
@@ -475,25 +743,34 @@ impl FractalHeapHeader {
         log2 + 2
     }
 
+    /// Rows an indirect block needs to span `size` bytes of heap space:
+    /// `log2(size) - log2(starting_block_size * table_width) + 1`, as
+    /// libhdf5's `H5HF__dtable_size_to_rows`.
+    fn rows_for_size(&self, size: u64) -> u16 {
+        let log2 = |v: u64| 63u32.saturating_sub(v.max(1).leading_zeros());
+        let first_row_bits = log2(self.starting_block_size) + log2(u64::from(self.table_width));
+        (log2(size).saturating_sub(first_row_bits) + 1) as u16
+    }
+
     /// Get block size for a given row in the doubling table.
     fn block_size_for_row(&self, row: usize) -> u64 {
         let sbs = self.starting_block_size;
         if row <= 1 {
             sbs
         } else {
-            sbs * (1u64 << (row - 1))
+            sbs.saturating_mul(1u64.checked_shl((row - 1) as u32).unwrap_or(u64::MAX))
         }
     }
+}
 
-    /// Total heap space covered by an indirect block with the given number of rows.
-    fn indirect_block_heap_size(&self, nrows: usize) -> u64 {
-        let tw = self.table_width as u64;
-        let mut total = 0u64;
-        for row in 0..nrows {
-            total += self.block_size_for_row(row) * tw;
-        }
-        total
-    }
+/// A managed direct block's location, extent and (for a filtered heap) its
+/// stored size and filter mask.
+struct DirectBlock {
+    addr: usize,
+    size: u64,
+    heap_offset: u64,
+    filtered_size: u64,
+    filter_mask: u32,
 }
 
 #[cfg(test)]
@@ -641,7 +918,7 @@ mod tests {
         let hdr = FractalHeapHeader::parse(&file_data, 0, 8, 8).unwrap();
 
         // Build a managed heap ID:
-        // byte 0: type=0 (bits 6-7 = 00), version=0 (bits 4-5), reserved (bits 0-3)
+        // byte 0: version=0 (bits 6-7), type=0 (bits 4-5), reserved (bits 0-3)
         // bytes 1-6: offset (max_heap_size=16 bits) then length (remaining bits)
         // For offset=0, length=13:
         // payload = offset | (length << 16) = 0 | (13 << 16) = 0x000D0000
@@ -705,9 +982,46 @@ mod tests {
     fn invalid_heap_id_type() {
         let (file_data, _) = build_simple_heap(8, 8);
         let hdr = FractalHeapHeader::parse(&file_data, 0, 8, 8).unwrap();
-        // Type = 1 (tiny) in bits 6-7
-        let id = vec![0x40u8, 0, 0, 0, 0, 0, 0]; // bit 6 set = type 1
+        // Type = 1 (huge) in bits 4-5 is not a managed ID
+        let id = vec![0x10u8, 0, 0, 0, 0, 0, 0];
         let err = hdr.decode_managed_id(&id).unwrap_err();
         assert_eq!(err, FormatError::InvalidHeapIdType(1));
     }
+
+    #[test]
+    fn tiny_object_is_read_from_the_id() {
+        let (file_data, _) = build_simple_heap(8, 8);
+        let hdr = FractalHeapHeader::parse(&file_data, 0, 8, 8).unwrap();
+        // Type 2 (0x20), length - 1 in the low 4 bits, data after.
+        let id = [0x20 | 2, b'a', b'b', b'c', 0, 0, 0];
+        assert_eq!(hdr.read_managed_object(&file_data, &id, 8).unwrap(), b"abc");
+        // A length running past the ID is an error, not a short read.
+        let id = [0x20 | 9, b'a', b'b', b'c', 0, 0, 0];
+        assert!(hdr.read_managed_object(&file_data, &id, 8).is_err());
+    }
+
+    #[test]
+    fn huge_object_with_a_direct_id() {
+        // With IDs long enough for an address and a length, libhdf5 stores
+        // huge objects' location in the ID instead of the huge-object B-tree.
+        let (mut file_data, _) = build_simple_heap(8, 8);
+        let mut hdr = FractalHeapHeader::parse(&file_data, 0, 8, 8).unwrap();
+        hdr.heap_id_length = 17;
+        file_data[900..905].copy_from_slice(b"huge!");
+        let mut id = vec![0x10u8];
+        id.extend_from_slice(&900u64.to_le_bytes());
+        id.extend_from_slice(&5u64.to_le_bytes());
+        assert_eq!(
+            hdr.read_managed_object(&file_data, &id, 8).unwrap(),
+            b"huge!"
+        );
+    }
+
+    #[test]
+    fn unknown_heap_id_version_is_refused() {
+        let (file_data, _) = build_simple_heap(8, 8);
+        let hdr = FractalHeapHeader::parse(&file_data, 0, 8, 8).unwrap();
+        let id = [0x40u8, 0, 0, 0, 0, 0, 0];
+        assert!(hdr.read_managed_object(&file_data, &id, 8).is_err());
+    }
 }

crates/clawhdf5-format/src/group_v1.rs

@@ -80,6 +80,53 @@ pub fn find_v1_soft_link(
     offset_size: u8,
     length_size: u8,
 ) -> Result<Option<String>, FormatError> {
+    let mut found = None;
+    for_each_v1_soft_link(
+        file_data,
+        sym_table_msg,
+        offset_size,
+        length_size,
+        |link_name| link_name == name,
+        |_, target| {
+            found = Some(target);
+            false
+        },
+    )?;
+    Ok(found)
+}
+
+/// Every soft link in a v1 group, as `(name, target path)`.
+pub fn v1_soft_links(
+    file_data: &[u8],
+    sym_table_msg: &SymbolTableMessage,
+    offset_size: u8,
+    length_size: u8,
+) -> Result<Vec<(String, String)>, FormatError> {
+    let mut links = Vec::new();
+    for_each_v1_soft_link(
+        file_data,
+        sym_table_msg,
+        offset_size,
+        length_size,
+        |_| true,
+        |name, target| {
+            links.push((String::from(name), target));
+            true
+        },
+    )?;
+    Ok(links)
+}
+
+/// Visit the soft links of a v1 group whose name passes `wanted`, with their
+/// target paths, until `visit` returns false.
+fn for_each_v1_soft_link(
+    file_data: &[u8],
+    sym_table_msg: &SymbolTableMessage,
+    offset_size: u8,
+    length_size: u8,
+    wanted: impl Fn(&str) -> bool,
+    mut visit: impl FnMut(&str, String) -> bool,
+) -> Result<(), FormatError> {
     let heap = LocalHeap::parse(
         file_data,
         sym_table_msg.local_heap_address as usize,
@@ -103,7 +150,8 @@ pub fn find_v1_soft_link(
                 heap.validate_free_list(file_data, length_size)?;
                 heap_checked = true;
             }
-            if heap.read_string(file_data, entry.link_name_offset)? != name {
+            let name = heap.read_string(file_data, entry.link_name_offset)?;
+            if !wanted(&name) {
                 continue;
             }
             let value_offset = u32::from_le_bytes([
@@ -112,12 +160,19 @@ pub fn find_v1_soft_link(
                 entry.scratch_pad[2],
                 entry.scratch_pad[3],
             ]);
-            return heap
-                .read_string(file_data, u64::from(value_offset))
-                .map(Some);
+            let target = heap.read_string(file_data, u64::from(value_offset))?;
+            if !visit(&name, target) {
+                return Ok(());
+            }
         }
     }
-    Ok(None)
+    Ok(())
+}
+
+/// Whether a v1 symbol-table entry is a soft link (no object header of its
+/// own; its target path is in the local heap).
+pub fn is_v1_soft_link(entry: &GroupEntry) -> bool {
+    entry.cache_type == CACHE_TYPE_SOFT_LINK
 }
 
 /// Extract the SymbolTableMessage from an object header's messages.

crates/clawhdf5-format/src/group_v2.rs

@@ -38,6 +38,24 @@ pub fn resolve_v2_group_entries(
     }
 }
 
+/// First user-defined link type (HDF5 reserves 2-63; 64 is external).
+const FIRST_USER_DEFINED_LINK_TYPE: u8 = 65;
+
+/// Parse a Link message, or `None` for a user-defined link (type 65-255).
+///
+/// A user-defined link's target is only meaningful to the application that
+/// registered its class, so, like libhdf5 without that class, we cannot
+/// follow it. Leaving it out lets the rest of the group be listed and
+/// resolved instead of one such link failing the whole group; reserved
+/// types (2-63) are still an error.
+fn parse_link(data: &[u8], offset_size: u8) -> Result<Option<LinkMessage>, FormatError> {
+    match LinkMessage::parse(data, offset_size) {
+        Ok(link) => Ok(Some(link)),
+        Err(FormatError::InvalidLinkType(t)) if t >= FIRST_USER_DEFINED_LINK_TYPE => Ok(None),
+        Err(e) => Err(e),
+    }
+}
+
 /// Extract link entries from Link messages directly in the object header (compact storage).
 fn resolve_compact_entries(
     object_header: &ObjectHeader,
@@ -46,7 +64,9 @@ fn resolve_compact_entries(
     let mut entries = Vec::new();
     for msg in &object_header.messages {
         if msg.msg_type == MessageType::Link {
-            let link = LinkMessage::parse(&msg.data, offset_size)?;
+            let Some(link) = parse_link(&msg.data, offset_size)? else {
+                continue;
+            };
             if let LinkTarget::Hard {
                 object_header_address,
             } = link.link_target
@@ -98,7 +118,9 @@ fn for_each_dense_link(
 
         // Read managed object from fractal heap
         let link_data = fh.read_managed_object(file_data, id_bytes, offset_size)?;
-        visit(LinkMessage::parse(&link_data, offset_size)?);
+        if let Some(link) = parse_link(&link_data, offset_size)? {
+            visit(link);
+        }
     }
     Ok(())
 }
@@ -178,7 +200,9 @@ fn find_symbolic_link(
     } else {
         for msg in &object_header.messages {
             if msg.msg_type == MessageType::Link {
-                let link = LinkMessage::parse(&msg.data, offset_size)?;
+                let Some(link) = parse_link(&msg.data, offset_size)? else {
+                    continue;
+                };
                 if link.name == name && is_symbolic(&link.link_target) {
                     found = Some(link.link_target);
                 }
@@ -231,32 +255,135 @@ pub fn resolve_path_any(
     superblock: &Superblock,
     path: &str,
 ) -> Result<u64, FormatError> {
-    resolve_path_following_links(file_data, superblock, path, 0)
+    resolve_path_following_links(
+        file_data,
+        superblock,
+        superblock.root_group_address,
+        path,
+        0,
+    )
+}
+
+/// Resolve `path` relative to the group at `group_address` (an absolute path
+/// starts at the root group instead), following soft links. This is how a
+/// relative soft link's target is resolved: from the group holding the link.
+pub fn resolve_path_from(
+    file_data: &[u8],
+    superblock: &Superblock,
+    group_address: u64,
+    path: &str,
+) -> Result<u64, FormatError> {
+    let start = if path.starts_with('/') {
+        superblock.root_group_address
+    } else {
+        group_address
+    };
+    resolve_path_following_links(file_data, superblock, start, path, 0)
+}
+
+/// The children of the group at `group_address` that can be opened, as h5py
+/// lists them: hard links, and soft links resolved to the object they point
+/// at (under the soft link's own name). Links that cannot be followed are
+/// left out rather than failing the listing — a dangling or cyclic soft link
+/// (h5py lists its name but cannot open it), an external link (another
+/// file), and a user-defined link. An object header that is not a group has
+/// no children.
+///
+/// Any other error, such as a corrupt structure met while resolving a soft
+/// link, is returned.
+pub fn resolve_group_children(
+    file_data: &[u8],
+    superblock: &Superblock,
+    group_address: u64,
+) -> Result<Vec<GroupEntry>, FormatError> {
+    let os = superblock.offset_size;
+    let ls = superblock.length_size;
+    let header = ObjectHeader::parse(file_data, group_address as usize, os, ls)?;
+
+    let mut entries = Vec::new();
+    let mut soft = Vec::new();
+    if is_v1_group(&header) {
+        let sym_msg = header
+            .messages
+            .iter()
+            .find(|m| m.msg_type == MessageType::SymbolTable)
+            .ok_or_else(|| FormatError::PathNotFound(String::from("no symbol table message")))?;
+        let stm = SymbolTableMessage::parse(&sym_msg.data, os)?;
+        let all = group_v1::resolve_v1_group_entries(file_data, &stm, os, ls)?;
+        if all.iter().any(group_v1::is_v1_soft_link) {
+            soft = group_v1::v1_soft_links(file_data, &stm, os, ls)?;
+        }
+        entries.extend(all.into_iter().filter(|e| !group_v1::is_v1_soft_link(e)));
+    } else if is_v2_group(&header) {
+        let mut visit = |link: LinkMessage| match link.link_target {
+            LinkTarget::Hard {
+                object_header_address,
+            } => entries.push(GroupEntry {
+                name: link.name,
+                object_header_address,
+                cache_type: 0,
+            }),
+            LinkTarget::Soft { target_path } => soft.push((link.name, target_path)),
+            LinkTarget::External { .. } => {}
+        };
+        let link_info = find_link_info(&header, os)?;
+        if let Some(fh_addr) = link_info.fractal_heap_address {
+            for_each_dense_link(file_data, &link_info, fh_addr, os, ls, visit)?;
+        } else {
+            for msg in &header.messages {
+                if msg.msg_type == MessageType::Link
+                    && let Some(link) = parse_link(&msg.data, os)?
+                {
+                    visit(link);
+                }
+            }
+        }
+    }
+
+    for (name, target) in soft {
+        match resolve_path_from(file_data, superblock, group_address, &target) {
+            Ok(object_header_address) => entries.push(GroupEntry {
+                name,
+                object_header_address,
+                cache_type: 0,
+            }),
+            // Dangling, cyclic, or ending in another file: not openable here.
+            Err(
+                FormatError::PathNotFound(_)
+                | FormatError::NestingDepthExceeded
+                | FormatError::ExternalLinkUnsupported { .. },
+            ) => {}
+            Err(e) => return Err(e),
+        }
+    }
+    Ok(entries)
 }
 
 /// Soft links followed while resolving one path. Guards against link cycles
 /// (`a -> b -> a`), which are legal to create.
 const MAX_SOFT_LINK_DEPTH: u8 = 16;
 
+/// Walk `path` from the group at `start`, following soft links.
 fn resolve_path_following_links(
     file_data: &[u8],
     superblock: &Superblock,
+    start: u64,
     path: &str,
     depth: u8,
 ) -> Result<u64, FormatError> {
-    let components: Vec<&str> = path.split('/').filter(|s| !s.is_empty()).collect();
+    let components: Vec<&str> = path
+        .split('/')
+        .filter(|s| !s.is_empty() && *s != ".")
+        .collect();
     if components.is_empty() {
-        return Ok(superblock.root_group_address);
+        return Ok(start);
     }
 
     let os = superblock.offset_size;
     let ls = superblock.length_size;
 
-    let root_header =
-        ObjectHeader::parse(file_data, superblock.root_group_address as usize, os, ls)?;
-
-    let mut current_addr = superblock.root_group_address;
-    let mut current_header = root_header;
+    let mut current_addr = start;
+    let mut current_header = ObjectHeader::parse(file_data, start as usize, os, ls)?;
 
     for (i, component) in components.iter().enumerate() {
         let entries = resolve_group_entries(file_data, &current_header, os, ls)?;
@@ -280,20 +407,17 @@ fn resolve_path_following_links(
                         }
                         // A relative target is relative to the group holding
                         // the link; then the rest of the original path.
-                        let mut full = String::new();
-                        if !target_path.starts_with('/') {
-                            for parent in &components[..i] {
-                                full.push('/');
-                                full.push_str(parent);
-                            }
-                        }
-                        full.push('/');
-                        full.push_str(&target_path);
+                        let from = if target_path.starts_with('/') {
+                            superblock.root_group_address
+                        } else {
+                            current_addr
+                        };
+                        let mut full = target_path;
                         for rest in &components[i + 1..] {
                             full.push('/');
                             full.push_str(rest);
                         }
-                        resolve_path_following_links(file_data, superblock, &full, depth + 1)
+                        resolve_path_following_links(file_data, superblock, from, &full, depth + 1)
                     }
                     Some(LinkTarget::External {
                         filename,

crates/clawhdf5-format/src/shared_message.rs

@@ -545,7 +545,8 @@ pub fn message_data<'a>(
 ///
 /// For type 1/3 (shared in another object header), reads the target object header
 /// and finds the message of the specified type.
-/// For type 2 (SOHM), uses the fractal heap from the SOHM table.
+/// For type 2 (SOHM), uses the fractal heap from the file's SOHM table,
+/// loaded from the superblock extension on demand.
 pub fn resolve_shared_message(
     file_data: &[u8],
     shared_ref: &SharedMessageRef,
@@ -553,13 +554,18 @@ pub fn resolve_shared_message(
     offset_size: u8,
     length_size: u8,
 ) -> Result<Vec<u8>, FormatError> {
+    let table = if shared_ref.heap_id.is_some() {
+        load_sohm_table(file_data, offset_size, length_size)?
+    } else {
+        None
+    };
     resolve_shared_message_with_sohm(
         file_data,
         shared_ref,
         target_msg_type,
         offset_size,
         length_size,
-        None,
+        table.as_ref(),
     )
 }
 

crates/clawhdf5/src/lazy.rs

@@ -12,25 +12,23 @@
 use std::cell::RefCell;
 use std::collections::HashMap;
 
-use clawhdf5_format::attribute::extract_attributes_full;
 use clawhdf5_format::data_layout::DataLayout;
 use clawhdf5_format::data_read;
 use clawhdf5_format::dataspace::Dataspace;
 use clawhdf5_format::datatype::Datatype;
 use clawhdf5_format::error::FormatError;
 use clawhdf5_format::filter_pipeline::FilterPipeline;
-use clawhdf5_format::group_v1::{self, GroupEntry};
+use clawhdf5_format::group_v1::GroupEntry;
 use clawhdf5_format::group_v2;
 use clawhdf5_format::message_type::MessageType;
 use clawhdf5_format::object_header::ObjectHeader;
 use clawhdf5_format::signature;
 use clawhdf5_format::superblock::Superblock;
-use clawhdf5_format::symbol_table::SymbolTableMessage;
 
 use clawhdf5_io::HDF5Read;
 
 use crate::error::Error;
-use crate::types::{AttrValue, DType, attrs_to_map, classify_datatype};
+use crate::types::{AttrValue, DType, classify_datatype, read_attrs};
 
 /// A lazy HDF5 file handle that parses metadata on demand.
 ///
@@ -246,17 +244,26 @@ impl<'f, R: HDF5Read> LazyGroup<'f, R> {
     }
 
     /// Read all attributes of this group.
+    ///
+    /// An attribute that cannot be read — a corrupt or unsupported attribute
+    /// message, or a dense-storage heap object that cannot be located — is
+    /// left out of the map instead of failing every attribute on the object;
+    /// [`attrs_with_errors`](Self::attrs_with_errors) reports which failed.
+    /// Values that are returned are complete (never partially decoded). An
+    /// error in the index of the attributes itself (the attribute info
+    /// message, the dense heap header or B-tree) still fails the call.
     pub fn attrs(&self) -> Result<HashMap<String, AttrValue>, Error> {
+        self.attrs_with_errors().map(|(attrs, _)| attrs)
+    }
+
+    /// Like [`attrs`](Self::attrs), also returning one error for each
+    /// attribute that could not be read and was left out.
+    pub fn attrs_with_errors(
+        &self,
+    ) -> Result<(HashMap<String, AttrValue>, Vec<FormatError>), Error> {
         let hdr = self.file.get_or_parse_header(self.address)?;
         let data = self.file.hdf5_bytes();
-        let attr_msgs =
-            extract_attributes_full(data, &hdr, self.file.offset_size(), self.file.length_size())?;
-        Ok(attrs_to_map(
-            &attr_msgs,
-            data,
-            self.file.offset_size(),
-            self.file.length_size(),
-        ))
+        read_attrs(data, &hdr, self.file.offset_size(), self.file.length_size())
     }
 
     /// Get a dataset within this group by name.
@@ -289,12 +296,14 @@ impl<'f, R: HDF5Read> LazyGroup<'f, R> {
         })
     }
 
+    /// This group's links that can be opened: hard links, and soft links
+    /// resolved to their targets (see
+    /// [`group_v2::resolve_group_children`]); dangling, external and
+    /// user-defined links are left out.
     fn children(&self) -> Result<Vec<GroupEntry>, Error> {
-        let hdr = self.file.get_or_parse_header(self.address)?;
         let data = self.file.hdf5_bytes();
-        let os = self.file.offset_size();
-        let ls = self.file.length_size();
-        resolve_group_entries(data, &hdr, os, ls).map_err(Error::Format)
+        group_v2::resolve_group_children(data, &self.file.superblock, self.address)
+            .map_err(Error::Format)
     }
 }
 
@@ -414,20 +423,30 @@ impl<'f, R: HDF5Read> LazyDataset<'f, R> {
     }
 
     /// Read all attributes of this dataset.
+    ///
+    /// An attribute that cannot be read — a corrupt or unsupported attribute
+    /// message, or a dense-storage heap object that cannot be located — is
+    /// left out of the map instead of failing every attribute on the object;
+    /// [`attrs_with_errors`](Self::attrs_with_errors) reports which failed.
+    /// Values that are returned are complete (never partially decoded). An
+    /// error in the index of the attributes itself (the attribute info
+    /// message, the dense heap header or B-tree) still fails the call.
     pub fn attrs(&self) -> Result<HashMap<String, AttrValue>, Error> {
+        self.attrs_with_errors().map(|(attrs, _)| attrs)
+    }
+
+    /// Like [`attrs`](Self::attrs), also returning one error for each
+    /// attribute that could not be read and was left out.
+    pub fn attrs_with_errors(
+        &self,
+    ) -> Result<(HashMap<String, AttrValue>, Vec<FormatError>), Error> {
         let data = self.file.hdf5_bytes();
-        let attr_msgs = extract_attributes_full(
+        read_attrs(
             data,
             &self.header,
             self.file.offset_size(),
             self.file.length_size(),
-        )?;
-        Ok(attrs_to_map(
-            &attr_msgs,
-            data,
-            self.file.offset_size(),
-            self.file.length_size(),
-        ))
+        )
     }
 
     /// A header message's payload, resolved through the shared-message
@@ -544,33 +563,3 @@ fn is_group(header: &ObjectHeader) -> bool {
             || m.msg_type == MessageType::SymbolTable
     })
 }
-
-fn resolve_group_entries(
-    file_data: &[u8],
-    object_header: &ObjectHeader,
-    offset_size: u8,
-    length_size: u8,
-) -> Result<Vec<GroupEntry>, FormatError> {
-    let is_v1 = object_header
-        .messages
-        .iter()
-        .any(|m| m.msg_type == MessageType::SymbolTable);
-    let is_v2 = object_header
-        .messages
-        .iter()
-        .any(|m| m.msg_type == MessageType::LinkInfo || m.msg_type == MessageType::Link);
-
-    if is_v1 {
-        let sym_msg = object_header
-            .messages
-            .iter()
-            .find(|m| m.msg_type == MessageType::SymbolTable)
-            .ok_or_else(|| FormatError::PathNotFound("no symbol table message".into()))?;
-        let stm = SymbolTableMessage::parse(&sym_msg.data, offset_size)?;
-        group_v1::resolve_v1_group_entries(file_data, &stm, offset_size, length_size)
-    } else if is_v2 {
-        group_v2::resolve_v2_group_entries(file_data, object_header, offset_size, length_size)
-    } else {
-        Ok(Vec::new())
-    }
-}

crates/clawhdf5/src/mmap_file.rs

@@ -7,25 +7,23 @@
 
 use std::collections::HashMap;
 
-use clawhdf5_format::attribute::extract_attributes_full;
 use clawhdf5_format::data_layout::DataLayout;
 use clawhdf5_format::data_read;
 use clawhdf5_format::dataspace::Dataspace;
 use clawhdf5_format::datatype::Datatype;
 use clawhdf5_format::error::FormatError;
 use clawhdf5_format::filter_pipeline::FilterPipeline;
-use clawhdf5_format::group_v1::{self, GroupEntry};
+use clawhdf5_format::group_v1::GroupEntry;
 use clawhdf5_format::group_v2;
 use clawhdf5_format::message_type::MessageType;
 use clawhdf5_format::object_header::ObjectHeader;
 use clawhdf5_format::signature;
 use clawhdf5_format::superblock::Superblock;
-use clawhdf5_format::symbol_table::SymbolTableMessage;
 
 use clawhdf5_io::MmapReader;
 
 use crate::error::Error;
-use crate::types::{AttrValue, DType, attrs_to_map, classify_datatype};
+use crate::types::{AttrValue, DType, classify_datatype, read_attrs};
 
 /// An HDF5 file opened via memory mapping.
 ///
@@ -174,17 +172,26 @@ impl<'f> MmapGroup<'f> {
     }
 
     /// Read all attributes of this group.
+    ///
+    /// An attribute that cannot be read — a corrupt or unsupported attribute
+    /// message, or a dense-storage heap object that cannot be located — is
+    /// left out of the map instead of failing every attribute on the object;
+    /// [`attrs_with_errors`](Self::attrs_with_errors) reports which failed.
+    /// Values that are returned are complete (never partially decoded). An
+    /// error in the index of the attributes itself (the attribute info
+    /// message, the dense heap header or B-tree) still fails the call.
     pub fn attrs(&self) -> Result<HashMap<String, AttrValue>, Error> {
-        let data = self.file.hdf5_bytes();
+        self.attrs_with_errors().map(|(attrs, _)| attrs)
+    }
+
+    /// Like [`attrs`](Self::attrs), also returning one error for each
+    /// attribute that could not be read and was left out.
+    pub fn attrs_with_errors(
+        &self,
+    ) -> Result<(HashMap<String, AttrValue>, Vec<FormatError>), Error> {
         let hdr = self.file.parse_header(self.address)?;
-        let attr_msgs =
-            extract_attributes_full(data, &hdr, self.file.offset_size(), self.file.length_size())?;
-        Ok(attrs_to_map(
-            &attr_msgs,
-            data,
-            self.file.offset_size(),
-            self.file.length_size(),
-        ))
+        let data = self.file.hdf5_bytes();
+        read_attrs(data, &hdr, self.file.offset_size(), self.file.length_size())
     }
 
     /// Get a dataset within this group by name.
@@ -217,12 +224,14 @@ impl<'f> MmapGroup<'f> {
         })
     }
 
+    /// This group's links that can be opened: hard links, and soft links
+    /// resolved to their targets (see
+    /// [`group_v2::resolve_group_children`]); dangling, external and
+    /// user-defined links are left out.
     fn children(&self) -> Result<Vec<GroupEntry>, Error> {
         let data = self.file.hdf5_bytes();
-        let hdr = self.file.parse_header(self.address)?;
-        let os = self.file.offset_size();
-        let ls = self.file.length_size();
-        resolve_group_entries(data, &hdr, os, ls).map_err(Error::Format)
+        group_v2::resolve_group_children(data, &self.file.superblock, self.address)
+            .map_err(Error::Format)
     }
 }
 
@@ -361,20 +370,30 @@ impl<'f> MmapDataset<'f> {
     }
 
     /// Read all attributes of this dataset.
+    ///
+    /// An attribute that cannot be read — a corrupt or unsupported attribute
+    /// message, or a dense-storage heap object that cannot be located — is
+    /// left out of the map instead of failing every attribute on the object;
+    /// [`attrs_with_errors`](Self::attrs_with_errors) reports which failed.
+    /// Values that are returned are complete (never partially decoded). An
+    /// error in the index of the attributes itself (the attribute info
+    /// message, the dense heap header or B-tree) still fails the call.
     pub fn attrs(&self) -> Result<HashMap<String, AttrValue>, Error> {
+        self.attrs_with_errors().map(|(attrs, _)| attrs)
+    }
+
+    /// Like [`attrs`](Self::attrs), also returning one error for each
+    /// attribute that could not be read and was left out.
+    pub fn attrs_with_errors(
+        &self,
+    ) -> Result<(HashMap<String, AttrValue>, Vec<FormatError>), Error> {
         let data = self.file.hdf5_bytes();
-        let attr_msgs = extract_attributes_full(
+        read_attrs(
             data,
             &self.header,
             self.file.offset_size(),
             self.file.length_size(),
-        )?;
-        Ok(attrs_to_map(
-            &attr_msgs,
-            data,
-            self.file.offset_size(),
-            self.file.length_size(),
-        ))
+        )
     }
 
     /// A header message's payload, resolved through the shared-message
@@ -490,33 +509,3 @@ fn is_group(header: &ObjectHeader) -> bool {
             || m.msg_type == MessageType::SymbolTable
     })
 }
-
-fn resolve_group_entries(
-    file_data: &[u8],
-    object_header: &ObjectHeader,
-    offset_size: u8,
-    length_size: u8,
-) -> Result<Vec<GroupEntry>, FormatError> {
-    let is_v1 = object_header
-        .messages
-        .iter()
-        .any(|m| m.msg_type == MessageType::SymbolTable);
-    let is_v2 = object_header
-        .messages
-        .iter()
-        .any(|m| m.msg_type == MessageType::LinkInfo || m.msg_type == MessageType::Link);
-
-    if is_v1 {
-        let sym_msg = object_header
-            .messages
-            .iter()
-            .find(|m| m.msg_type == MessageType::SymbolTable)
-            .ok_or_else(|| FormatError::PathNotFound("no symbol table message".into()))?;
-        let stm = SymbolTableMessage::parse(&sym_msg.data, offset_size)?;
-        group_v1::resolve_v1_group_entries(file_data, &stm, offset_size, length_size)
-    } else if is_v2 {
-        group_v2::resolve_v2_group_entries(file_data, object_header, offset_size, length_size)
-    } else {
-        Ok(Vec::new())
-    }
-}

crates/clawhdf5/src/reader.rs

@@ -7,7 +7,6 @@
 
 use std::collections::HashMap;
 
-use clawhdf5_format::attribute::extract_attributes_full;
 use clawhdf5_format::chunk_cache::ChunkCache;
 use clawhdf5_format::data_layout::DataLayout;
 use clawhdf5_format::data_read;
@@ -15,16 +14,15 @@ use clawhdf5_format::dataspace::Dataspace;
 use clawhdf5_format::datatype::{Datatype, DatatypeByteOrder};
 use clawhdf5_format::error::FormatError;
 use clawhdf5_format::filter_pipeline::FilterPipeline;
-use clawhdf5_format::group_v1::{self, GroupEntry};
+use clawhdf5_format::group_v1::GroupEntry;
 use clawhdf5_format::group_v2;
 use clawhdf5_format::message_type::MessageType;
 use clawhdf5_format::object_header::ObjectHeader;
 use clawhdf5_format::signature;
 use clawhdf5_format::superblock::Superblock;
-use clawhdf5_format::symbol_table::SymbolTableMessage;
 
 use crate::error::Error;
-use crate::types::{AttrValue, DType, attrs_to_map, classify_datatype};
+use crate::types::{AttrValue, DType, classify_datatype, read_attrs};
 
 // ---------------------------------------------------------------------------
 // FileData — internal storage for either owned bytes or an mmap
@@ -322,17 +320,26 @@ impl<'f> Group<'f> {
     }
 
     /// Read all attributes of this group.
+    ///
+    /// An attribute that cannot be read — a corrupt or unsupported attribute
+    /// message, or a dense-storage heap object that cannot be located — is
+    /// left out of the map instead of failing every attribute on the object;
+    /// [`attrs_with_errors`](Self::attrs_with_errors) reports which failed.
+    /// Values that are returned are complete (never partially decoded). An
+    /// error in the index of the attributes itself (the attribute info
+    /// message, the dense heap header or B-tree) still fails the call.
     pub fn attrs(&self) -> Result<HashMap<String, AttrValue>, Error> {
-        let data = self.file.data.as_bytes();
+        self.attrs_with_errors().map(|(attrs, _)| attrs)
+    }
+
+    /// Like [`attrs`](Self::attrs), also returning one error for each
+    /// attribute that could not be read and was left out.
+    pub fn attrs_with_errors(
+        &self,
+    ) -> Result<(HashMap<String, AttrValue>, Vec<FormatError>), Error> {
         let hdr = self.file.parse_header(self.address)?;
-        let attr_msgs =
-            extract_attributes_full(data, &hdr, self.file.offset_size(), self.file.length_size())?;
-        Ok(attrs_to_map(
-            &attr_msgs,
-            data,
-            self.file.offset_size(),
-            self.file.length_size(),
-        ))
+        let data = self.file.data.as_bytes();
+        read_attrs(data, &hdr, self.file.offset_size(), self.file.length_size())
     }
 
     /// Get a dataset within this group by name.
@@ -365,12 +372,14 @@ impl<'f> Group<'f> {
         })
     }
 
+    /// This group's links that can be opened: hard links, and soft links
+    /// resolved to their targets (see
+    /// [`group_v2::resolve_group_children`]); dangling, external and
+    /// user-defined links are left out.
     fn children(&self) -> Result<Vec<GroupEntry>, Error> {
         let data = self.file.data.as_bytes();
-        let hdr = self.file.parse_header(self.address)?;
-        let os = self.file.offset_size();
-        let ls = self.file.length_size();
-        resolve_group_entries(data, &hdr, os, ls).map_err(Error::Format)
+        group_v2::resolve_group_children(data, &self.file.superblock, self.address)
+            .map_err(Error::Format)
     }
 }
 
@@ -759,20 +768,30 @@ impl<'f> Dataset<'f> {
     }
 
     /// Read all attributes of this dataset.
+    ///
+    /// An attribute that cannot be read — a corrupt or unsupported attribute
+    /// message, or a dense-storage heap object that cannot be located — is
+    /// left out of the map instead of failing every attribute on the object;
+    /// [`attrs_with_errors`](Self::attrs_with_errors) reports which failed.
+    /// Values that are returned are complete (never partially decoded). An
+    /// error in the index of the attributes itself (the attribute info
+    /// message, the dense heap header or B-tree) still fails the call.
     pub fn attrs(&self) -> Result<HashMap<String, AttrValue>, Error> {
+        self.attrs_with_errors().map(|(attrs, _)| attrs)
+    }
+
+    /// Like [`attrs`](Self::attrs), also returning one error for each
+    /// attribute that could not be read and was left out.
+    pub fn attrs_with_errors(
+        &self,
+    ) -> Result<(HashMap<String, AttrValue>, Vec<FormatError>), Error> {
         let data = self.file.data.as_bytes();
-        let attr_msgs = extract_attributes_full(
+        read_attrs(
             data,
             &self.header,
             self.file.offset_size(),
             self.file.length_size(),
-        )?;
-        Ok(attrs_to_map(
-            &attr_msgs,
-            data,
-            self.file.offset_size(),
-            self.file.length_size(),
-        ))
+        )
     }
 
     /// Verify this dataset's content against its stored provenance hash
@@ -987,37 +1006,6 @@ fn is_group(header: &ObjectHeader) -> bool {
     })
 }
 
-fn resolve_group_entries(
-    file_data: &[u8],
-    object_header: &ObjectHeader,
-    offset_size: u8,
-    length_size: u8,
-) -> Result<Vec<GroupEntry>, FormatError> {
-    let is_v1 = object_header
-        .messages
-        .iter()
-        .any(|m| m.msg_type == MessageType::SymbolTable);
-    let is_v2 = object_header
-        .messages
-        .iter()
-        .any(|m| m.msg_type == MessageType::LinkInfo || m.msg_type == MessageType::Link);
-
-    if is_v1 {
-        let sym_msg = object_header
-            .messages
-            .iter()
-            .find(|m| m.msg_type == MessageType::SymbolTable)
-            .ok_or_else(|| FormatError::PathNotFound("no symbol table message".into()))?;
-        let stm = SymbolTableMessage::parse(&sym_msg.data, offset_size)?;
-        group_v1::resolve_v1_group_entries(file_data, &stm, offset_size, length_size)
-    } else if is_v2 {
-        group_v2::resolve_v2_group_entries(file_data, object_header, offset_size, length_size)
-    } else {
-        // Empty group or unrecognized — return empty
-        Ok(Vec::new())
-    }
-}
-
 #[cfg(test)]
 mod sibling_file_name_tests {
     use super::sibling_file_name;

crates/clawhdf5/src/types.rs

@@ -155,6 +155,33 @@ pub(crate) fn classify_datatype(dt: &clawhdf5_format::datatype::Datatype) -> DTy
 /// Read attribute messages into a `HashMap<String, AttrValue>`.
 ///
 /// Best-effort: attributes that can't be decoded are silently skipped.
+/// The attributes of the object with header `header` that could be read,
+/// and one error for each that could not (see
+/// [`extract_attributes_tolerant`](clawhdf5_format::attribute::extract_attributes_tolerant)).
+pub(crate) fn read_attrs(
+    file_data: &[u8],
+    header: &clawhdf5_format::object_header::ObjectHeader,
+    offset_size: u8,
+    length_size: u8,
+) -> Result<
+    (
+        HashMap<String, AttrValue>,
+        Vec<clawhdf5_format::error::FormatError>,
+    ),
+    crate::Error,
+> {
+    let (msgs, errors) = clawhdf5_format::attribute::extract_attributes_tolerant(
+        file_data,
+        header,
+        offset_size,
+        length_size,
+    )?;
+    Ok((
+        attrs_to_map(&msgs, file_data, offset_size, length_size),
+        errors,
+    ))
+}
+
 pub(crate) fn attrs_to_map(
     attrs: &[clawhdf5_format::attribute::AttributeMessage],
     file_data: &[u8],

crates/clawhdf5/tests/dense_storage_interop.rs

@@ -0,0 +1,418 @@
+//! Dense ("new-style") link and attribute storage written by libhdf5 (via
+//! h5py): groups whose links live in a fractal heap indexed by a v2 B-tree,
+//! and objects whose attributes do. Every listing and value is compared with
+//! what h5py itself reports for the same file.
+//!
+//! Skipped when python3 with h5py is unavailable, unless
+//! `CLAWHDF5_REQUIRE_INTEROP=1`.
+
+use std::process::Command;
+
+use clawhdf5::{AttrValue, File};
+
+fn python() -> String {
+    std::env::var("CLAWHDF5_PYTHON").unwrap_or_else(|_| "python3".to_string())
+}
+
+fn interop_required() -> bool {
+    std::env::var("CLAWHDF5_REQUIRE_INTEROP").is_ok_and(|v| v == "1")
+}
+
+fn python_available() -> bool {
+    Command::new(python())
+        .args(["-c", "import h5py"])
+        .output()
+        .map(|o| o.status.success())
+        .unwrap_or(false)
+}
+
+macro_rules! skip_if_no_python {
+    () => {
+        if !python_available() {
+            assert!(
+                !interop_required(),
+                "CLAWHDF5_REQUIRE_INTEROP=1 but python3 with h5py is not available"
+            );
+            eprintln!("SKIP: python3 with h5py not available");
+            return;
+        }
+    };
+}
+
+fn run_python(script: &str) -> String {
+    let output = Command::new(python())
+        .args(["-c", script])
+        .output()
+        .expect("failed to run python");
+    if !output.status.success() {
+        panic!(
+            "Python script failed:\nSTDOUT: {}\nSTDERR: {}",
+            String::from_utf8_lossy(&output.stdout),
+            String::from_utf8_lossy(&output.stderr)
+        );
+    }
+    String::from_utf8_lossy(&output.stdout).trim().to_string()
+}
+
+/// Have h5py write a file with `body` (which sees `f`, `h5py` and `np`),
+/// returning the temp dir holding it and its path.
+fn h5py_file(body: &str) -> (tempfile::TempDir, String) {
+    let dir = tempfile::tempdir().unwrap();
+    let path = dir.path().join("dense.h5").display().to_string();
+    let script = format!(
+        "import h5py, numpy as np\n\
+         with h5py.File(r'{path}', 'w', libver='latest') as f:\n{}",
+        indent(body)
+    );
+    run_python(&script);
+    (dir, path)
+}
+
+fn indent(body: &str) -> String {
+    body.lines()
+        .map(|l| format!("    {l}\n"))
+        .collect::<String>()
+}
+
+/// The datasets and groups h5py lists in `group`, sorted: links h5py can
+/// resolve (hard and soft), without dangling soft links or external links.
+fn h5py_listing(path: &str, group: &str) -> (Vec<String>, Vec<String>) {
+    let out = run_python(&format!(
+        "import h5py\n\
+         ds, gs = [], []\n\
+         with h5py.File(r'{path}', 'r') as f:\n\
+         \x20   g = f[{group:?}]\n\
+         \x20   for k in g.keys():\n\
+         \x20       if isinstance(g.get(k, getlink=True), h5py.ExternalLink):\n\
+         \x20           continue\n\
+         \x20       try:\n\
+         \x20           o = g[k]\n\
+         \x20       except Exception:\n\
+         \x20           continue\n\
+         \x20       (ds if isinstance(o, h5py.Dataset) else gs).append(k)\n\
+         print('\\x1f'.join(sorted(ds)))\n\
+         print('\\x1f'.join(sorted(gs)))\n"
+    ));
+    let mut lines = out.lines();
+    let split = |l: Option<&str>| -> Vec<String> {
+        l.unwrap_or("")
+            .split('\x1f')
+            .filter(|s| !s.is_empty())
+            .map(str::to_string)
+            .collect()
+    };
+    let ds = split(lines.next());
+    let gs = split(lines.next());
+    (ds, gs)
+}
+
+fn our_listing(path: &str, group: &str) -> (Vec<String>, Vec<String>) {
+    let f = File::open(path).unwrap();
+    let g = f.group(group).unwrap();
+    let mut ds = g.datasets().unwrap();
+    let mut gs = g.groups().unwrap();
+    ds.sort();
+    gs.sort();
+    (ds, gs)
+}
+
+fn assert_same_listing(path: &str, group: &str) {
+    let ours = our_listing(path, group);
+    let theirs = h5py_listing(path, group);
+    assert_eq!(ours.0.len(), theirs.0.len(), "dataset count in {group}");
+    assert_eq!(ours, theirs, "listing of {group}");
+}
+
+#[test]
+fn dense_group_whose_heap_outgrows_the_root_direct_rows() {
+    skip_if_no_python!();
+    // Long link names make the link heap larger than the root indirect
+    // block's direct rows can hold (512 KiB with h5py's defaults), so links
+    // live in child indirect blocks. Those were sized from the wrong row
+    // count, and every link past the direct rows was unreachable.
+    let (_dir, path) = h5py_file(
+        "t = f.create_dataset('t', data=[1.0])\n\
+         g = f.create_group('g')\n\
+         for i in range(2500):\n\
+         \x20   g['n%05d_' % i + 'x' * 240] = t\n",
+    );
+    assert_same_listing(&path, "g");
+    let f = File::open(&path).unwrap();
+    let last = format!("g/n02499_{}", "x".repeat(240));
+    assert_eq!(f.dataset(&last).unwrap().read_f64().unwrap(), vec![1.0]);
+}
+
+#[test]
+fn dense_group_with_a_three_level_name_index() {
+    skip_if_no_python!();
+    // 24 000 links give the link-name v2 B-tree a depth of 3. Internal-node
+    // child pointers carry the subtree's total record count in a width that
+    // depends on the most records a subtree can hold; the reader estimated
+    // that as leaf_max^depth, read the root's pointers 3 bytes wide instead
+    // of 2, and decoded garbage heap IDs.
+    let (_dir, path) = h5py_file(
+        "t = f.create_dataset('t', data=[1.0])\n\
+         g = f.create_group('g')\n\
+         for i in range(24000):\n\
+         \x20   g['l%06d' % i] = t\n",
+    );
+    assert_same_listing(&path, "g");
+    let f = File::open(&path).unwrap();
+    assert_eq!(
+        f.dataset("g/l023999").unwrap().read_f64().unwrap(),
+        vec![1.0]
+    );
+}
+
+/// The attribute names h5py reports for `obj`, sorted.
+fn h5py_attr_names(path: &str, obj: &str) -> Vec<String> {
+    let out = run_python(&format!(
+        "import h5py\n\
+         with h5py.File(r'{path}', 'r') as f:\n\
+         \x20   print('\\x1f'.join(sorted(f[{obj:?}].attrs.keys())))\n"
+    ));
+    out.split('\x1f')
+        .filter(|s| !s.is_empty())
+        .map(str::to_string)
+        .collect()
+}
+
+#[test]
+fn dense_attribute_stored_as_a_huge_heap_object() {
+    skip_if_no_python!();
+    // More than 8 attributes puts them in dense storage; one larger than the
+    // heap's 4 KiB managed-object limit is stored as a "huge" object, outside
+    // the heap blocks and found through the huge-object v2 B-tree. Its heap ID
+    // (type bits 4-5 = 1) was misread as a managed ID, and the error made
+    // every attribute on the object unreadable. NetCDF-4 files hit this
+    // (netcdf-c's issue671.nc / issue672.nc).
+    let (_dir, path) = h5py_file(
+        "d = f.create_dataset('d', data=[1.0])\n\
+         for i in range(10):\n\
+         \x20   d.attrs['a%d' % i] = i\n\
+         d.attrs['big'] = np.arange(1024, dtype='f8')\n\
+         d.attrs['bigger'] = np.arange(20000, dtype='i8') * 3\n",
+    );
+    let f = File::open(&path).unwrap();
+    let attrs = f.dataset("d").unwrap().attrs().unwrap();
+    let mut names: Vec<String> = attrs.keys().cloned().collect();
+    names.sort();
+    assert_eq!(names, h5py_attr_names(&path, "d"));
+    for i in 0..10 {
+        assert!(
+            matches!(attrs[&format!("a{i}")], AttrValue::I64(v) if v == i),
+            "a{i}: {:?}",
+            attrs[&format!("a{i}")]
+        );
+    }
+    let big: Vec<f64> = (0..1024).map(f64::from).collect();
+    assert!(matches!(&attrs["big"], AttrValue::F64Array(v) if *v == big));
+    let bigger: Vec<i64> = (0..20000).map(|v| v * 3).collect();
+    assert!(matches!(&attrs["bigger"], AttrValue::I64Array(v) if *v == bigger));
+}
+
+/// A group whose link heap has a deflate I/O filter (set on the group
+/// creation property list), with 3 000 links and one link whose message is
+/// larger than the heap's managed-object limit, so it is a huge object.
+fn huge_link_group(filtered: bool) -> (tempfile::TempDir, String) {
+    let filter = if filtered {
+        "import ctypes, glob, os\n\
+         lib = ctypes.CDLL(glob.glob(os.path.join(os.path.dirname(h5py.__file__), '..', 'h5py.libs', 'libhdf5-*.so*'))[0])\n\
+         lib.H5Pset_deflate.argtypes = [ctypes.c_int64, ctypes.c_uint]\n\
+         assert lib.H5Pset_deflate(gcpl.id, 6) >= 0\n"
+    } else {
+        ""
+    };
+    h5py_file(&format!(
+        "t = f.create_dataset('t', data=[1.0])\n\
+         gcpl = h5py.h5p.create(h5py.h5p.GROUP_CREATE)\n\
+         {filter}\
+         h5py.h5g.create(f.id, b'g', gcpl=gcpl)\n\
+         g = f['g']\n\
+         for i in range(3000):\n\
+         \x20   g['l%05d' % i] = t\n\
+         g['L' * 5000] = t\n"
+    ))
+}
+
+fn check_huge_link_group(filtered: bool) {
+    let (_dir, path) = huge_link_group(filtered);
+    assert_same_listing(&path, "g");
+    let f = File::open(&path).unwrap();
+    let huge = format!("g/{}", "L".repeat(5000));
+    assert_eq!(f.dataset(&huge).unwrap().read_f64().unwrap(), vec![1.0]);
+    assert_eq!(
+        f.dataset("g/l02999").unwrap().read_f64().unwrap(),
+        vec![1.0]
+    );
+}
+
+#[test]
+fn dense_group_with_a_huge_link() {
+    skip_if_no_python!();
+    check_huge_link_group(false);
+}
+
+#[test]
+fn dense_group_with_a_filtered_link_heap() {
+    skip_if_no_python!();
+    // libhdf5 applies a group's filter pipeline to its link heap: direct
+    // blocks and huge objects are stored deflated, and the heap header
+    // carries the pipeline. The header's checksum was looked for in the
+    // wrong place, and filtered blocks were read raw.
+    if run_python(
+        "import h5py, glob, os\nprint(len(glob.glob(os.path.join(os.path.dirname(h5py.__file__), '..', 'h5py.libs', 'libhdf5-*.so*'))))",
+    ) == "0"
+    {
+        assert!(
+            !interop_required(),
+            "CLAWHDF5_REQUIRE_INTEROP=1 but h5py's bundled libhdf5 was not found"
+        );
+        eprintln!("SKIP: h5py's bundled libhdf5 not found (needed to set the filter)");
+        return;
+    }
+    check_huge_link_group(true);
+}
+
+fn fixture(name: &str) -> String {
+    format!("{}/tests/fixtures/{name}", env!("CARGO_MANIFEST_DIR"))
+}
+
+/// `tall.h5` and `tudlink.h5` are libhdf5's own tool test files
+/// (`tools/test/testfiles`, BSD-style HDF5 licence). Each has user-defined
+/// links of class 187, which h5py lists by name but cannot open, and h5dump
+/// prints as `USERDEFINED_LINK`. One such link made the whole group
+/// unlistable (`InvalidLinkType(187)`); it is now left out of the listing
+/// like any other link that cannot be followed.
+#[test]
+fn user_defined_links_do_not_break_the_listing() {
+    let f = File::open(fixture("tall.h5")).unwrap();
+    let g2 = f.group("g2").unwrap();
+    let mut ds = g2.datasets().unwrap();
+    ds.sort();
+    assert_eq!(ds, ["dset2.1", "dset2.2"]);
+    assert!(g2.groups().unwrap().is_empty());
+    assert!(f.dataset("g2/udlink").is_err());
+
+    let f = File::open(fixture("tudlink.h5")).unwrap();
+    assert!(f.root().datasets().unwrap().is_empty());
+    assert!(f.root().groups().unwrap().is_empty());
+}
+
+/// Soft links (absolute, relative, to a dataset and to a group), a dangling
+/// one, a cycle and an external link, in an old-style (symbol table) or
+/// new-style (link message) group.
+fn soft_link_file(libver: &str) -> (tempfile::TempDir, String) {
+    let dir = tempfile::tempdir().unwrap();
+    let path = dir.path().join("soft.h5").display().to_string();
+    run_python(&format!(
+        "import h5py, numpy as np\n\
+         with h5py.File(r'{path}', 'w', libver='{libver}') as f:\n\
+         \x20   f.create_dataset('a/b/deep', data=np.arange(3.0))\n\
+         \x20   f.create_dataset('plain', data=[1.0])\n\
+         \x20   f['soft_ds'] = h5py.SoftLink('/a/b/deep')\n\
+         \x20   f['soft_grp'] = h5py.SoftLink('/a')\n\
+         \x20   f['a/rel'] = h5py.SoftLink('b/deep')\n\
+         \x20   f['a/rel_grp'] = h5py.SoftLink('b')\n\
+         \x20   f['dangling'] = h5py.SoftLink('/nope')\n\
+         \x20   f['loop1'] = h5py.SoftLink('/loop2')\n\
+         \x20   f['loop2'] = h5py.SoftLink('/loop1')\n\
+         \x20   f['ext'] = h5py.ExternalLink('elsewhere.h5', '/x')\n"
+    ));
+    (dir, path)
+}
+
+fn check_soft_links(libver: &str) {
+    let (_dir, path) = soft_link_file(libver);
+    assert_same_listing(&path, "/");
+    assert_same_listing(&path, "a");
+
+    let f = File::open(&path).unwrap();
+    let root = f.root();
+    let deep = vec![0.0, 1.0, 2.0];
+    assert_eq!(root.dataset("soft_ds").unwrap().read_f64().unwrap(), deep);
+    let a = root.group("soft_grp").unwrap();
+    assert_eq!(a.dataset("rel").unwrap().read_f64().unwrap(), deep);
+    assert_eq!(a.group("rel_grp").unwrap().datasets().unwrap(), ["deep"]);
+    // A dangling link is not listed and cannot be opened.
+    assert!(root.dataset("dangling").is_err());
+    assert!(root.dataset("loop1").is_err());
+
+    // The memory-mapped and lazy handles list the same way.
+    let (ds, gs) = h5py_listing(&path, "/");
+    let m = clawhdf5::MmapFile::open(&path).unwrap();
+    let mut mds = m.root().datasets().unwrap();
+    let mut mgs = m.root().groups().unwrap();
+    mds.sort();
+    mgs.sort();
+    assert_eq!((mds, mgs), (ds.clone(), gs.clone()));
+    let l = clawhdf5::LazyFile::from_bytes(std::fs::read(&path).unwrap()).unwrap();
+    let mut lds = l.root().datasets().unwrap();
+    let mut lgs = l.root().groups().unwrap();
+    lds.sort();
+    lgs.sort();
+    assert_eq!((lds, lgs), (ds, gs));
+}
+
+/// Soft links were left out of `datasets()`/`groups()` (and could not be
+/// opened by name from a group handle); in old-style groups, where a soft
+/// link has no object header address, they made the listing fail. h5py
+/// lists a soft link under its own name as whatever it points at.
+#[test]
+fn soft_links_are_listed_as_their_targets() {
+    skip_if_no_python!();
+    check_soft_links("latest");
+    check_soft_links("earliest");
+}
+
+/// One unreadable attribute used to fail `attrs()` for every attribute on
+/// the object. Now it is left out (and reported by `attrs_with_errors`),
+/// and the others are returned with their full values.
+#[test]
+fn one_unreadable_attribute_does_not_hide_the_others() {
+    skip_if_no_python!();
+    let (dir, path) = h5py_file(
+        "d = f.create_dataset('d', data=[1.0])\n\
+         for i in range(10):\n\
+         \x20   d.attrs['a%d' % i] = float(i)\n\
+         d.attrs['zz_broken_attribute'] = 42.0\n",
+    );
+    // Dense attributes live in a fractal heap whose blocks carry no checksum
+    // by default: give the one named `zz_broken_attribute` a nonexistent
+    // attribute message version (the byte 9 before its name in a v3 message).
+    let mut bytes = std::fs::read(&path).unwrap();
+    let needle = b"zz_broken_attribute";
+    let at = bytes
+        .windows(needle.len())
+        .position(|w| w == needle)
+        .expect("attribute name in the file");
+    assert_eq!(bytes[at - 9], 3, "expected a version-3 attribute message");
+    bytes[at - 9] = 0x7f;
+    let broken = dir.path().join("broken.h5");
+    std::fs::write(&broken, &bytes).unwrap();
+
+    for file in [
+        File::open(&broken).unwrap(),
+        File::from_bytes(bytes.clone()).unwrap(),
+    ] {
+        let ds = file.dataset("d").unwrap();
+        let (attrs, errors) = ds.attrs_with_errors().unwrap();
+        assert_eq!(errors.len(), 1, "{errors:?}");
+        assert_eq!(attrs.len(), 10);
+        for i in 0..10 {
+            assert!(
+                matches!(attrs[&format!("a{i}")], AttrValue::F64(v) if v == f64::from(i)),
+                "a{i}"
+            );
+        }
+        assert!(!attrs.contains_key("zz_broken_attribute"));
+        assert_eq!(ds.attrs().unwrap().len(), 10);
+    }
+    let m = clawhdf5::MmapFile::open(&broken).unwrap();
+    assert_eq!(
+        m.dataset("d").unwrap().attrs_with_errors().unwrap().1.len(),
+        1
+    );
+    let l = clawhdf5::LazyFile::from_bytes(bytes).unwrap();
+    assert_eq!(l.dataset("d").unwrap().attrs().unwrap().len(), 10);
+}

crates/clawhdf5/tests/sohm_interop.rs

@@ -0,0 +1,141 @@
+//! Files with shared object header messages (SOHM: datatypes, dataspaces,
+//! filter pipelines and attributes stored once in a file-wide heap and
+//! referenced by heap ID), written by libhdf5 through h5py.
+//!
+//! Skipped when python3 with h5py is unavailable, unless
+//! `CLAWHDF5_REQUIRE_INTEROP=1`.
+
+use std::process::Command;
+
+use clawhdf5::{AttrValue, File};
+
+fn python() -> String {
+    std::env::var("CLAWHDF5_PYTHON").unwrap_or_else(|_| "python3".to_string())
+}
+
+fn interop_required() -> bool {
+    std::env::var("CLAWHDF5_REQUIRE_INTEROP").is_ok_and(|v| v == "1")
+}
+
+fn python_available() -> bool {
+    Command::new(python())
+        .args(["-c", "import h5py"])
+        .output()
+        .map(|o| o.status.success())
+        .unwrap_or(false)
+}
+
+fn run_python(script: &str) -> String {
+    let output = Command::new(python())
+        .args(["-c", script])
+        .output()
+        .expect("failed to run python");
+    if !output.status.success() {
+        panic!(
+            "Python script failed:\nSTDOUT: {}\nSTDERR: {}",
+            String::from_utf8_lossy(&output.stdout),
+            String::from_utf8_lossy(&output.stderr)
+        );
+    }
+    String::from_utf8_lossy(&output.stdout).trim().to_string()
+}
+
+/// h5py has no binding for the SOHM property-list calls, so they go through
+/// the libhdf5 that h5py bundles. `None` when that library is not found.
+fn sohm_file(path: &str, libver: &str, mesg_types: u32) -> Option<()> {
+    let out = run_python(&format!(
+        "import ctypes, glob, os, h5py, numpy as np\n\
+         libs = glob.glob(os.path.join(os.path.dirname(h5py.__file__), '..', 'h5py.libs', 'libhdf5-*.so*'))\n\
+         if not libs:\n\
+         \x20   print('nolib'); raise SystemExit\n\
+         lib = ctypes.CDLL(libs[0])\n\
+         lib.H5Pset_shared_mesg_nindexes.argtypes = [ctypes.c_int64, ctypes.c_uint]\n\
+         lib.H5Pset_shared_mesg_index.argtypes = [ctypes.c_int64, ctypes.c_uint, ctypes.c_uint, ctypes.c_uint]\n\
+         fcpl = h5py.h5p.create(h5py.h5p.FILE_CREATE)\n\
+         assert lib.H5Pset_shared_mesg_nindexes(fcpl.id, 1) >= 0\n\
+         assert lib.H5Pset_shared_mesg_index(fcpl.id, 0, {mesg_types}, 1) >= 0\n\
+         fapl = h5py.h5p.create(h5py.h5p.FILE_ACCESS)\n\
+         low = h5py.h5f.LIBVER_EARLIEST if '{libver}' == 'earliest' else h5py.h5f.LIBVER_LATEST\n\
+         fapl.set_libver_bounds(low, h5py.h5f.LIBVER_LATEST)\n\
+         fid = h5py.h5f.create(r'{path}'.encode(), h5py.h5f.ACC_TRUNC, fcpl=fcpl, fapl=fapl)\n\
+         with h5py.File(fid) as f:\n\
+         \x20   for i in range(4):\n\
+         \x20       ds = f.create_dataset('d%d' % i, shape=(50,), dtype='<f8', chunks=(10,), compression='gzip', fillvalue=-9.0)\n\
+         \x20       ds[0:20] = np.arange(20.0) + i\n\
+         \x20       ds.attrs['shared_attr'] = np.arange(10.0)\n\
+         \x20       ds.attrs['units'] = 'm/s'\n\
+         \x20   f.create_dataset('contig', data=np.arange(7, dtype='<i4') * 2)\n\
+         print('ok')\n"
+    ));
+    (out == "ok").then_some(())
+}
+
+/// Every message type libhdf5 can share (`H5O_SHMESG_ALL_FLAG`), and each on
+/// its own.
+const MESG_TYPES: [(u32, &str); 6] = [
+    (0x182A, "all"),
+    (0x02, "dataspace"),
+    (0x08, "datatype"),
+    (0x20, "fill value"),
+    (0x800, "filter pipeline"),
+    (0x1000, "attribute"),
+];
+
+/// A message shared through the SOHM heap was only resolved on the one path
+/// that loaded the SOHM table itself (shared fill values); datatypes,
+/// dataspaces, filter pipelines and attributes stored there failed with
+/// "invalid shared message version: 2", so such files' datasets could not be
+/// read at all.
+#[test]
+fn sohm_shared_messages_resolve() {
+    if !python_available() {
+        assert!(
+            !interop_required(),
+            "CLAWHDF5_REQUIRE_INTEROP=1 but python3 with h5py is not available"
+        );
+        eprintln!("SKIP: python3 with h5py not available");
+        return;
+    }
+    let dir = tempfile::tempdir().unwrap();
+    for libver in ["earliest", "latest"] {
+        for (flags, what) in MESG_TYPES {
+            let path = dir.path().join("sohm.h5").display().to_string();
+            if sohm_file(&path, libver, flags).is_none() {
+                assert!(
+                    !interop_required(),
+                    "CLAWHDF5_REQUIRE_INTEROP=1 but h5py's bundled libhdf5 was not found"
+                );
+                eprintln!("SKIP: h5py's bundled libhdf5 not found");
+                return;
+            }
+            let case = format!("{what}, libver {libver}");
+            let f = File::open(&path).unwrap();
+            let d2 = f.dataset("d2").unwrap_or_else(|e| panic!("{case}: {e}"));
+            let mut expected: Vec<f64> = (0..20).map(|v| f64::from(v) + 2.0).collect();
+            expected.resize(50, -9.0);
+            assert_eq!(
+                d2.read_f64().unwrap_or_else(|e| panic!("{case}: {e}")),
+                expected,
+                "{case}"
+            );
+            let (attrs, errors) = d2.attrs_with_errors().unwrap();
+            assert!(errors.is_empty(), "{case}: {errors:?}");
+            let shared: Vec<f64> = (0..10).map(f64::from).collect();
+            assert!(
+                matches!(&attrs["shared_attr"], AttrValue::F64Array(v) if *v == shared),
+                "{case}: {:?}",
+                attrs.get("shared_attr")
+            );
+            assert!(
+                matches!(&attrs["units"], AttrValue::String(s) if s == "m/s"),
+                "{case}: {:?}",
+                attrs.get("units")
+            );
+            assert_eq!(
+                f.dataset("contig").unwrap().read_i32().unwrap(),
+                [0, 2, 4, 6, 8, 10, 12],
+                "{case}"
+            );
+        }
+    }
+}