Read HDF5 1.6-era files, user blocks, VDS, dense attributes and large groups #13
@@ -322,6 +322,11 @@
|
|||||||
creation property list compresses its link heap) are now read: the
|
creation property list compresses its link heap) are now read: the
|
||||||
header's pipeline was skipped with the wrong size, so its checksum was
|
header's pipeline was skipped with the wrong size, so its checksum was
|
||||||
looked for in the wrong place, and filtered direct blocks were read raw.
|
looked for in the wrong place, and filtered direct blocks were read raw.
|
||||||
|
- A user-defined link (link class 65-255, e.g. 187 in libhdf5's
|
||||||
|
`tall.h5`/`tudlink.h5`) made its whole group unlistable. Such links
|
||||||
|
cannot be followed without the application that registered the class, so
|
||||||
|
they are now left out of `datasets()`/`groups()` and path lookup, as h5py
|
||||||
|
leaves out links it cannot open; reserved link types are still an error.
|
||||||
- `clawhdf5-format` writer — **files libhdf5 rejects or reads wrong:**
|
- `clawhdf5-format` writer — **files libhdf5 rejects or reads wrong:**
|
||||||
- Extensible Array (one unlimited dimension): chunks from index 244 on were
|
- Extensible Array (one unlimited dimension): chunks from index 244 on were
|
||||||
written but never indexed and read as 0, by libhdf5 and by us.
|
written but never indexed and read as 0, by libhdf5 and by us.
|
||||||
|
|||||||
@@ -38,6 +38,24 @@ pub fn resolve_v2_group_entries(
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// First user-defined link type (HDF5 reserves 2-63; 64 is external).
|
||||||
|
const FIRST_USER_DEFINED_LINK_TYPE: u8 = 65;
|
||||||
|
|
||||||
|
/// Parse a Link message, or `None` for a user-defined link (type 65-255).
|
||||||
|
///
|
||||||
|
/// A user-defined link's target is only meaningful to the application that
|
||||||
|
/// registered its class, so, like libhdf5 without that class, we cannot
|
||||||
|
/// follow it. Leaving it out lets the rest of the group be listed and
|
||||||
|
/// resolved instead of one such link failing the whole group; reserved
|
||||||
|
/// types (2-63) are still an error.
|
||||||
|
fn parse_link(data: &[u8], offset_size: u8) -> Result<Option<LinkMessage>, FormatError> {
|
||||||
|
match LinkMessage::parse(data, offset_size) {
|
||||||
|
Ok(link) => Ok(Some(link)),
|
||||||
|
Err(FormatError::InvalidLinkType(t)) if t >= FIRST_USER_DEFINED_LINK_TYPE => Ok(None),
|
||||||
|
Err(e) => Err(e),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
/// Extract link entries from Link messages directly in the object header (compact storage).
|
/// Extract link entries from Link messages directly in the object header (compact storage).
|
||||||
fn resolve_compact_entries(
|
fn resolve_compact_entries(
|
||||||
object_header: &ObjectHeader,
|
object_header: &ObjectHeader,
|
||||||
@@ -46,7 +64,9 @@ fn resolve_compact_entries(
|
|||||||
let mut entries = Vec::new();
|
let mut entries = Vec::new();
|
||||||
for msg in &object_header.messages {
|
for msg in &object_header.messages {
|
||||||
if msg.msg_type == MessageType::Link {
|
if msg.msg_type == MessageType::Link {
|
||||||
let link = LinkMessage::parse(&msg.data, offset_size)?;
|
let Some(link) = parse_link(&msg.data, offset_size)? else {
|
||||||
|
continue;
|
||||||
|
};
|
||||||
if let LinkTarget::Hard {
|
if let LinkTarget::Hard {
|
||||||
object_header_address,
|
object_header_address,
|
||||||
} = link.link_target
|
} = link.link_target
|
||||||
@@ -98,7 +118,9 @@ fn for_each_dense_link(
|
|||||||
|
|
||||||
// Read managed object from fractal heap
|
// Read managed object from fractal heap
|
||||||
let link_data = fh.read_managed_object(file_data, id_bytes, offset_size)?;
|
let link_data = fh.read_managed_object(file_data, id_bytes, offset_size)?;
|
||||||
visit(LinkMessage::parse(&link_data, offset_size)?);
|
if let Some(link) = parse_link(&link_data, offset_size)? {
|
||||||
|
visit(link);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
@@ -178,7 +200,9 @@ fn find_symbolic_link(
|
|||||||
} else {
|
} else {
|
||||||
for msg in &object_header.messages {
|
for msg in &object_header.messages {
|
||||||
if msg.msg_type == MessageType::Link {
|
if msg.msg_type == MessageType::Link {
|
||||||
let link = LinkMessage::parse(&msg.data, offset_size)?;
|
let Some(link) = parse_link(&msg.data, offset_size)? else {
|
||||||
|
continue;
|
||||||
|
};
|
||||||
if link.name == name && is_symbolic(&link.link_target) {
|
if link.name == name && is_symbolic(&link.link_target) {
|
||||||
found = Some(link.link_target);
|
found = Some(link.link_target);
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -273,3 +273,28 @@ fn dense_group_with_a_filtered_link_heap() {
|
|||||||
}
|
}
|
||||||
check_huge_link_group(true);
|
check_huge_link_group(true);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn fixture(name: &str) -> String {
|
||||||
|
format!("{}/tests/fixtures/{name}", env!("CARGO_MANIFEST_DIR"))
|
||||||
|
}
|
||||||
|
|
||||||
|
/// `tall.h5` and `tudlink.h5` are libhdf5's own tool test files
|
||||||
|
/// (`tools/test/testfiles`, BSD-style HDF5 licence). Each has user-defined
|
||||||
|
/// links of class 187, which h5py lists by name but cannot open, and h5dump
|
||||||
|
/// prints as `USERDEFINED_LINK`. One such link made the whole group
|
||||||
|
/// unlistable (`InvalidLinkType(187)`); it is now left out of the listing
|
||||||
|
/// like any other link that cannot be followed.
|
||||||
|
#[test]
|
||||||
|
fn user_defined_links_do_not_break_the_listing() {
|
||||||
|
let f = File::open(fixture("tall.h5")).unwrap();
|
||||||
|
let g2 = f.group("g2").unwrap();
|
||||||
|
let mut ds = g2.datasets().unwrap();
|
||||||
|
ds.sort();
|
||||||
|
assert_eq!(ds, ["dset2.1", "dset2.2"]);
|
||||||
|
assert!(g2.groups().unwrap().is_empty());
|
||||||
|
assert!(f.dataset("g2/udlink").is_err());
|
||||||
|
|
||||||
|
let f = File::open(fixture("tudlink.h5")).unwrap();
|
||||||
|
assert!(f.root().datasets().unwrap().is_empty());
|
||||||
|
assert!(f.root().groups().unwrap().is_empty());
|
||||||
|
}
|
||||||
|
|||||||
BIN
Binary file not shown.
BIN
Binary file not shown.
@@ -76,6 +76,8 @@ the VDS item, which is marked.
|
|||||||
- **Old-style shared messages (version 1)** read the wrong address.
|
- **Old-style shared messages (version 1)** read the wrong address.
|
||||||
- **Groups and links:**
|
- **Groups and links:**
|
||||||
- Groups with a user-defined link type (e.g. 187) cannot be listed.
|
- Groups with a user-defined link type (e.g. 187) cannot be listed.
|
||||||
|
**Fixed 2026-09-25:** user-defined links are skipped; the rest of the
|
||||||
|
group lists.
|
||||||
- Dense groups with more than about 22 000 links cannot be listed.
|
- Dense groups with more than about 22 000 links cannot be listed.
|
||||||
**Fixed 2026-09-25:** two bugs — fractal-heap child indirect blocks had
|
**Fixed 2026-09-25:** two bugs — fractal-heap child indirect blocks had
|
||||||
the wrong row count, and v2 B-tree internal nodes at depth 3+ were read
|
the wrong row count, and v2 B-tree internal nodes at depth 3+ were read
|
||||||
|
|||||||
Reference in New Issue
Block a user