From 38d0d4de02f663b55a1191c31af10f3133df3abe Mon Sep 17 00:00:00 2001 From: osobh Date: Fri, 25 Sep 2026 21:57:18 -0500 Subject: [PATCH] fix(format): skip user-defined links instead of failing the group Link types 65-255 are user-defined: their target is only meaningful to the application that registered the link class. LinkMessage::parse rejects them with InvalidLinkType, and group traversal propagated that, so one such link made the whole group unlistable and every path through it unresolvable (libhdf5's tall.h5 and tudlink.h5, class 187). Group traversal (compact and dense) now leaves user-defined links out, the way h5py leaves out links it cannot open; reserved types (2-63) are still an error. Regression test: user_defined_links_do_not_break_the_listing, on libhdf5's own tools/test/testfiles tall.h5 and tudlink.h5 (BSD-style HDF5 licence, 10 KB and 1 KB), committed as fixtures. Co-Authored-By: Claude Opus 5.5 (1M context) --- CHANGELOG.md | 5 +++ crates/clawhdf5-format/src/group_v2.rs | 30 ++++++++++++++++-- .../clawhdf5/tests/dense_storage_interop.rs | 25 +++++++++++++++ crates/clawhdf5/tests/fixtures/tall.h5 | Bin 0 -> 9968 bytes crates/clawhdf5/tests/fixtures/tudlink.h5 | Bin 0 -> 904 bytes docs/known-issues.md | 2 ++ 6 files changed, 59 insertions(+), 3 deletions(-) create mode 100644 crates/clawhdf5/tests/fixtures/tall.h5 create mode 100644 crates/clawhdf5/tests/fixtures/tudlink.h5 diff --git a/CHANGELOG.md b/CHANGELOG.md index 462672c..95da497 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -322,6 +322,11 @@ creation property list compresses its link heap) are now read: the header's pipeline was skipped with the wrong size, so its checksum was looked for in the wrong place, and filtered direct blocks were read raw. + - A user-defined link (link class 65-255, e.g. 187 in libhdf5's + `tall.h5`/`tudlink.h5`) made its whole group unlistable. Such links + cannot be followed without the application that registered the class, so + they are now left out of `datasets()`/`groups()` and path lookup, as h5py + leaves out links it cannot open; reserved link types are still an error. - `clawhdf5-format` writer — **files libhdf5 rejects or reads wrong:** - Extensible Array (one unlimited dimension): chunks from index 244 on were written but never indexed and read as 0, by libhdf5 and by us. diff --git a/crates/clawhdf5-format/src/group_v2.rs b/crates/clawhdf5-format/src/group_v2.rs index 903c7d6..bcfa1be 100644 --- a/crates/clawhdf5-format/src/group_v2.rs +++ b/crates/clawhdf5-format/src/group_v2.rs @@ -38,6 +38,24 @@ pub fn resolve_v2_group_entries( } } +/// First user-defined link type (HDF5 reserves 2-63; 64 is external). +const FIRST_USER_DEFINED_LINK_TYPE: u8 = 65; + +/// Parse a Link message, or `None` for a user-defined link (type 65-255). +/// +/// A user-defined link's target is only meaningful to the application that +/// registered its class, so, like libhdf5 without that class, we cannot +/// follow it. Leaving it out lets the rest of the group be listed and +/// resolved instead of one such link failing the whole group; reserved +/// types (2-63) are still an error. +fn parse_link(data: &[u8], offset_size: u8) -> Result, FormatError> { + match LinkMessage::parse(data, offset_size) { + Ok(link) => Ok(Some(link)), + Err(FormatError::InvalidLinkType(t)) if t >= FIRST_USER_DEFINED_LINK_TYPE => Ok(None), + Err(e) => Err(e), + } +} + /// Extract link entries from Link messages directly in the object header (compact storage). fn resolve_compact_entries( object_header: &ObjectHeader, @@ -46,7 +64,9 @@ fn resolve_compact_entries( let mut entries = Vec::new(); for msg in &object_header.messages { if msg.msg_type == MessageType::Link { - let link = LinkMessage::parse(&msg.data, offset_size)?; + let Some(link) = parse_link(&msg.data, offset_size)? else { + continue; + }; if let LinkTarget::Hard { object_header_address, } = link.link_target @@ -98,7 +118,9 @@ fn for_each_dense_link( // Read managed object from fractal heap let link_data = fh.read_managed_object(file_data, id_bytes, offset_size)?; - visit(LinkMessage::parse(&link_data, offset_size)?); + if let Some(link) = parse_link(&link_data, offset_size)? { + visit(link); + } } Ok(()) } @@ -178,7 +200,9 @@ fn find_symbolic_link( } else { for msg in &object_header.messages { if msg.msg_type == MessageType::Link { - let link = LinkMessage::parse(&msg.data, offset_size)?; + let Some(link) = parse_link(&msg.data, offset_size)? else { + continue; + }; if link.name == name && is_symbolic(&link.link_target) { found = Some(link.link_target); } diff --git a/crates/clawhdf5/tests/dense_storage_interop.rs b/crates/clawhdf5/tests/dense_storage_interop.rs index 7f8206a..92af6ee 100644 --- a/crates/clawhdf5/tests/dense_storage_interop.rs +++ b/crates/clawhdf5/tests/dense_storage_interop.rs @@ -273,3 +273,28 @@ fn dense_group_with_a_filtered_link_heap() { } check_huge_link_group(true); } + +fn fixture(name: &str) -> String { + format!("{}/tests/fixtures/{name}", env!("CARGO_MANIFEST_DIR")) +} + +/// `tall.h5` and `tudlink.h5` are libhdf5's own tool test files +/// (`tools/test/testfiles`, BSD-style HDF5 licence). Each has user-defined +/// links of class 187, which h5py lists by name but cannot open, and h5dump +/// prints as `USERDEFINED_LINK`. One such link made the whole group +/// unlistable (`InvalidLinkType(187)`); it is now left out of the listing +/// like any other link that cannot be followed. +#[test] +fn user_defined_links_do_not_break_the_listing() { + let f = File::open(fixture("tall.h5")).unwrap(); + let g2 = f.group("g2").unwrap(); + let mut ds = g2.datasets().unwrap(); + ds.sort(); + assert_eq!(ds, ["dset2.1", "dset2.2"]); + assert!(g2.groups().unwrap().is_empty()); + assert!(f.dataset("g2/udlink").is_err()); + + let f = File::open(fixture("tudlink.h5")).unwrap(); + assert!(f.root().datasets().unwrap().is_empty()); + assert!(f.root().groups().unwrap().is_empty()); +} diff --git a/crates/clawhdf5/tests/fixtures/tall.h5 b/crates/clawhdf5/tests/fixtures/tall.h5 new file mode 100644 index 0000000000000000000000000000000000000000..918aeeeda416eafefdf960b0b6df7dbee30d8890 GIT binary patch literal 9968 zcmeHN%}-N75Z_m@6s%f6%y7C`rp+@eIMu&La!cU0)x5v3*L)!1(9UnI&sroe+w)t|1 zOWxTjH_Q*Z2h!s=GO#f#hAN&uM7)+y4RL)c$vLoxG-{u_e>jB6?f7jM{HB{&S5a^o zgx~cz%ZK>g{W~}aJ!;{1Mew6GP7PhQ2&UX3c(3@qckA|;V#P|F0P1sg4TgVKMSg#8{4amXML@8V71lji`+uKl%%DA^Q6TdBu2er zb=PXeu>|k@%%H@lAp9zbOiFf%>-m?{f% zPx8+uXBP9qP(15;sq#TN$1$gT9^3GJFG2l7K6^m#VOGQ;>v9himmMo1TX^ySPJE`jJKD()NOXKUX}lom*WN z<4ne_<~4H5pWdh9l!4&IyS~rlY_6{N`Jj)ztM4;XzRA}cdd>gpEntfF`=A`36>h_# zM7Vvz#C4Ys8SueFJK&EK$5A2xOrpNQ?G|Lv5By_9-~nBLkeUx&&;uFt$FC9KpC)pN z`iYP*AB+V(km(^hLv)U4fanqt62^osj0HWAfsTtrgG5(|kYE?)jQLPuy`g(>tv=z-7!p$GPt2lRf)MjV=Y{rg;m846U0RFvfJ0Tc*zfBOZo zmr{l^dHVOa#;kbw|GUyg7Kll9Dp4EK5s;&NHR3w}Ms2t!er&WQzf{8MlTM^KeHNx? z^0qKPSt=U;IZ}6F*eD^fH@O@>YkMFVuZ+L;zO&_XUsud@Jr|XZcv^9bO%zfEw}ZXh zf-PJnRpWNK!u*dV|GCWWPk@ literal 0 HcmV?d00001 diff --git a/crates/clawhdf5/tests/fixtures/tudlink.h5 b/crates/clawhdf5/tests/fixtures/tudlink.h5 new file mode 100644 index 0000000000000000000000000000000000000000..5dc0c7071d8c1497f21b4bd468047b6564423590 GIT binary patch literal 904 zcmeD5aB<`1lHy_j0S*oZ76t(@6Gr@pf(~Yg2#gPtPk=FS(2NYM42(cI38=gP)O-b~ zc^OdgkRVrA5EEuTjD|`xM2J9G958`VdPIeQhpS@%$iop(U&GSD2`1EF0Hrk$04s(t zDI%HRDHoO^VKl^8E(Q*8`i5D;22J0L9J|>|Q*tu%vJJsfY*2F$;zleCY5DmueE?eY BKrH|O literal 0 HcmV?d00001 diff --git a/docs/known-issues.md b/docs/known-issues.md index 774c70e..a437db4 100644 --- a/docs/known-issues.md +++ b/docs/known-issues.md @@ -76,6 +76,8 @@ the VDS item, which is marked. - **Old-style shared messages (version 1)** read the wrong address. - **Groups and links:** - Groups with a user-defined link type (e.g. 187) cannot be listed. + **Fixed 2026-09-25:** user-defined links are skipped; the rest of the + group lists. - Dense groups with more than about 22 000 links cannot be listed. **Fixed 2026-09-25:** two bugs — fractal-heap child indirect blocks had the wrong row count, and v2 B-tree internal nodes at depth 3+ were read