Read HDF5 1.6-era files, user blocks, VDS, dense attributes and large groups #13
@@ -322,6 +322,11 @@
|
||||
creation property list compresses its link heap) are now read: the
|
||||
header's pipeline was skipped with the wrong size, so its checksum was
|
||||
looked for in the wrong place, and filtered direct blocks were read raw.
|
||||
- A user-defined link (link class 65-255, e.g. 187 in libhdf5's
|
||||
`tall.h5`/`tudlink.h5`) made its whole group unlistable. Such links
|
||||
cannot be followed without the application that registered the class, so
|
||||
they are now left out of `datasets()`/`groups()` and path lookup, as h5py
|
||||
leaves out links it cannot open; reserved link types are still an error.
|
||||
- `clawhdf5-format` writer — **files libhdf5 rejects or reads wrong:**
|
||||
- Extensible Array (one unlimited dimension): chunks from index 244 on were
|
||||
written but never indexed and read as 0, by libhdf5 and by us.
|
||||
|
||||
@@ -38,6 +38,24 @@ pub fn resolve_v2_group_entries(
|
||||
}
|
||||
}
|
||||
|
||||
/// First user-defined link type (HDF5 reserves 2-63; 64 is external).
|
||||
const FIRST_USER_DEFINED_LINK_TYPE: u8 = 65;
|
||||
|
||||
/// Parse a Link message, or `None` for a user-defined link (type 65-255).
|
||||
///
|
||||
/// A user-defined link's target is only meaningful to the application that
|
||||
/// registered its class, so, like libhdf5 without that class, we cannot
|
||||
/// follow it. Leaving it out lets the rest of the group be listed and
|
||||
/// resolved instead of one such link failing the whole group; reserved
|
||||
/// types (2-63) are still an error.
|
||||
fn parse_link(data: &[u8], offset_size: u8) -> Result<Option<LinkMessage>, FormatError> {
|
||||
match LinkMessage::parse(data, offset_size) {
|
||||
Ok(link) => Ok(Some(link)),
|
||||
Err(FormatError::InvalidLinkType(t)) if t >= FIRST_USER_DEFINED_LINK_TYPE => Ok(None),
|
||||
Err(e) => Err(e),
|
||||
}
|
||||
}
|
||||
|
||||
/// Extract link entries from Link messages directly in the object header (compact storage).
|
||||
fn resolve_compact_entries(
|
||||
object_header: &ObjectHeader,
|
||||
@@ -46,7 +64,9 @@ fn resolve_compact_entries(
|
||||
let mut entries = Vec::new();
|
||||
for msg in &object_header.messages {
|
||||
if msg.msg_type == MessageType::Link {
|
||||
let link = LinkMessage::parse(&msg.data, offset_size)?;
|
||||
let Some(link) = parse_link(&msg.data, offset_size)? else {
|
||||
continue;
|
||||
};
|
||||
if let LinkTarget::Hard {
|
||||
object_header_address,
|
||||
} = link.link_target
|
||||
@@ -98,7 +118,9 @@ fn for_each_dense_link(
|
||||
|
||||
// Read managed object from fractal heap
|
||||
let link_data = fh.read_managed_object(file_data, id_bytes, offset_size)?;
|
||||
visit(LinkMessage::parse(&link_data, offset_size)?);
|
||||
if let Some(link) = parse_link(&link_data, offset_size)? {
|
||||
visit(link);
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
@@ -178,7 +200,9 @@ fn find_symbolic_link(
|
||||
} else {
|
||||
for msg in &object_header.messages {
|
||||
if msg.msg_type == MessageType::Link {
|
||||
let link = LinkMessage::parse(&msg.data, offset_size)?;
|
||||
let Some(link) = parse_link(&msg.data, offset_size)? else {
|
||||
continue;
|
||||
};
|
||||
if link.name == name && is_symbolic(&link.link_target) {
|
||||
found = Some(link.link_target);
|
||||
}
|
||||
|
||||
@@ -273,3 +273,28 @@ fn dense_group_with_a_filtered_link_heap() {
|
||||
}
|
||||
check_huge_link_group(true);
|
||||
}
|
||||
|
||||
fn fixture(name: &str) -> String {
|
||||
format!("{}/tests/fixtures/{name}", env!("CARGO_MANIFEST_DIR"))
|
||||
}
|
||||
|
||||
/// `tall.h5` and `tudlink.h5` are libhdf5's own tool test files
|
||||
/// (`tools/test/testfiles`, BSD-style HDF5 licence). Each has user-defined
|
||||
/// links of class 187, which h5py lists by name but cannot open, and h5dump
|
||||
/// prints as `USERDEFINED_LINK`. One such link made the whole group
|
||||
/// unlistable (`InvalidLinkType(187)`); it is now left out of the listing
|
||||
/// like any other link that cannot be followed.
|
||||
#[test]
|
||||
fn user_defined_links_do_not_break_the_listing() {
|
||||
let f = File::open(fixture("tall.h5")).unwrap();
|
||||
let g2 = f.group("g2").unwrap();
|
||||
let mut ds = g2.datasets().unwrap();
|
||||
ds.sort();
|
||||
assert_eq!(ds, ["dset2.1", "dset2.2"]);
|
||||
assert!(g2.groups().unwrap().is_empty());
|
||||
assert!(f.dataset("g2/udlink").is_err());
|
||||
|
||||
let f = File::open(fixture("tudlink.h5")).unwrap();
|
||||
assert!(f.root().datasets().unwrap().is_empty());
|
||||
assert!(f.root().groups().unwrap().is_empty());
|
||||
}
|
||||
|
||||
BIN
Binary file not shown.
BIN
Binary file not shown.
@@ -76,6 +76,8 @@ the VDS item, which is marked.
|
||||
- **Old-style shared messages (version 1)** read the wrong address.
|
||||
- **Groups and links:**
|
||||
- Groups with a user-defined link type (e.g. 187) cannot be listed.
|
||||
**Fixed 2026-09-25:** user-defined links are skipped; the rest of the
|
||||
group lists.
|
||||
- Dense groups with more than about 22 000 links cannot be listed.
|
||||
**Fixed 2026-09-25:** two bugs — fractal-heap child indirect blocks had
|
||||
the wrong row count, and v2 B-tree internal nodes at depth 3+ were read
|
||||
|
||||
Reference in New Issue
Block a user