From 52cfcf20b2113a1f733e12076c827312f544e0aa Mon Sep 17 00:00:00 2001 From: osobh Date: Sat, 19 Sep 2026 14:24:04 -0700 Subject: [PATCH] feat(format): parse H5T_STD_REF references and decode object references HDF5 1.12 revised the reference datatype (class 7) in datatype message version 4: reference types 2-4 are the new H5T_STD_REF object / dataset-region / attribute references. Datatype::parse rejected them with InvalidReferenceType, so any dataset of that type was unreadable. h5py cannot write this type, which is why it had never been tested. A real file was produced by calling the libhdf5 bundled in the h5py wheel through ctypes (H5T_STD_REF_g, H5Rcreate_object, H5Dwrite); the 2 KB result is committed as tests/fixtures/std_ref_hdf5_2_0.h5 with its generator, gen_std_ref.py. - ReferenceType gains Object2, DatasetRegion2 and Attribute, accepted only from datatype version 4. - read_object_references decodes Object2 elements: type(1) flags(1) token_size(1) token, zero-padded to the element size; the token is the target's object header address. A null reference decodes to the undefined address; an external reference, a wrong type byte or a token that doesn't fit is an error. The fixture test follows both references and checks they resolve to the objects they were created from. Co-Authored-By: Claude Fable 5.1 --- CHANGELOG.md | 5 + crates/clawhdf5-format/src/data_read.rs | 60 +++++++++++ crates/clawhdf5-format/src/datatype.rs | 44 +++++++- .../tests/fixtures/gen_std_ref.py | 44 ++++++++ .../tests/fixtures/std_ref_hdf5_2_0.h5 | Bin 0 -> 2104 bytes .../clawhdf5-format/tests/reference_tests.rs | 94 ++++++++++++++++++ docs/known-issues.md | 20 ++-- 7 files changed, 258 insertions(+), 9 deletions(-) create mode 100644 crates/clawhdf5-format/tests/fixtures/gen_std_ref.py create mode 100644 crates/clawhdf5-format/tests/fixtures/std_ref_hdf5_2_0.h5 diff --git a/CHANGELOG.md b/CHANGELOG.md index 8e8ee83..705e7ac 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -22,6 +22,11 @@ type 5 — what `libver='latest'` uses for two or more unlimited dimensions; previously "unsupported chunked layout"). The four copies of the chunk-index dispatch are now one shared function, so every read path gets it. +- **`H5T_STD_REF` references** (HDF5 1.12+, datatype message version 4) parse: + `ReferenceType` gains `Object2`, `DatasetRegion2` and `Attribute`, and + `read_object_references` decodes the new object references. Previously any + dataset of this type failed with `InvalidReferenceType(2)`. Tested against a + file written by HDF5 2.0 itself (fixture + generator script committed). - **Automatic chunk sizes.** Asking for compression (or any filter) without `with_chunks` used to store the whole dataset as one chunk, so any read had to decompress everything and nothing could be decoded in parallel. Datasets up diff --git a/crates/clawhdf5-format/src/data_read.rs b/crates/clawhdf5-format/src/data_read.rs index 4aa83a4..0e6773f 100644 --- a/crates/clawhdf5-format/src/data_read.rs +++ b/crates/clawhdf5-format/src/data_read.rs @@ -1427,6 +1427,26 @@ pub fn read_object_references( } Ok(result) } + Datatype::Reference { + ref_type: crate::datatype::ReferenceType::Object2, + size, + } => { + let elem_size = *size as usize; + if elem_size == 0 { + return Ok(Vec::new()); + } + if !raw.len().is_multiple_of(elem_size) { + return Err(FormatError::DataSizeMismatch { + expected: 0, + actual: raw.len(), + }); + } + raw.chunks_exact(elem_size) + .map(|element| { + decode_std_object_ref(element).map(|address| ObjectReference { address }) + }) + .collect() + } _ => Err(FormatError::TypeMismatch { expected: "Reference(Object)", actual: datatype_name(datatype), @@ -1434,6 +1454,46 @@ pub fn read_object_references( } } +/// Decode one `H5T_STD_REF` object reference as stored in a dataset: +/// `type(1) flags(1) token_size(1) token(token_size)`, zero-padded to the +/// element size. For a reference within the same file the token is the target +/// object's header address. An all-zero element is a null reference and +/// decodes to the undefined address (`u64::MAX`). +fn decode_std_object_ref(element: &[u8]) -> Result { + const STD_REF_OBJECT: u8 = 2; + const FLAG_EXTERNAL: u8 = 0x01; + if element.iter().all(|&b| b == 0) { + return Ok(u64::MAX); + } + let [ref_type, flags, token_size, token @ ..] = element else { + return Err(FormatError::UnexpectedEof { + expected: 3, + available: element.len(), + }); + }; + if *ref_type != STD_REF_OBJECT { + return Err(FormatError::InvalidReferenceType(*ref_type)); + } + if flags & FLAG_EXTERNAL != 0 { + // Carries a file name as well; nothing here follows those. + return Err(FormatError::TypeMismatch { + expected: "object reference within this file", + actual: "external object reference", + }); + } + let n = *token_size as usize; + if n == 0 || n > 8 || n > token.len() { + return Err(FormatError::UnexpectedEof { + expected: 3 + n, + available: element.len(), + }); + } + Ok(token[..n] + .iter() + .rev() + .fold(0u64, |addr, &byte| (addr << 8) | u64::from(byte))) +} + /// Read region references from raw bytes. /// /// Region references encode a dataset selection (hyperslab, point list, etc.) diff --git a/crates/clawhdf5-format/src/datatype.rs b/crates/clawhdf5-format/src/datatype.rs index 19242c0..2a83e0b 100644 --- a/crates/clawhdf5-format/src/datatype.rs +++ b/crates/clawhdf5-format/src/datatype.rs @@ -36,8 +36,18 @@ pub enum CharacterSet { /// Reference type. #[derive(Debug, Clone, PartialEq)] pub enum ReferenceType { + /// Legacy object reference: the target's object header address. Object, + /// Legacy dataset region reference. DatasetRegion, + /// `H5T_STD_REF` object reference (HDF5 1.12+, datatype message version + /// 4): a small header followed by an object token. Decoded by + /// `data_read::read_object_references`. + Object2, + /// `H5T_STD_REF` dataset region reference. + DatasetRegion2, + /// `H5T_STD_REF` attribute reference. + Attribute, } /// A member of a compound datatype. @@ -424,9 +434,15 @@ impl Datatype { 7 => { // Reference let ref_type_val = bf0 & 0x0F; - let ref_type = match ref_type_val { - 0 => ReferenceType::Object, - 1 => ReferenceType::DatasetRegion, + // Datatype message version 4 (HDF5 1.12) revised this class: + // types 2-4 are the new `H5T_STD_REF` references, and the high + // nibble of the first flag byte carries their encoding version. + let ref_type = match (ref_type_val, version) { + (0, _) => ReferenceType::Object, + (1, _) => ReferenceType::DatasetRegion, + (2, 4..) => ReferenceType::Object2, + (3, 4..) => ReferenceType::DatasetRegion2, + (4, 4..) => ReferenceType::Attribute, _ => return Err(FormatError::InvalidReferenceType(ref_type_val)), }; Ok((Datatype::Reference { size, ref_type }, pos)) @@ -1563,6 +1579,28 @@ mod tests { assert_eq!(err, FormatError::InvalidCharacterSet(2)); } + #[test] + fn test_reference_v4_std_ref_from_hdf5_2_0() { + // Datatype message of an H5T_STD_REF dataset written by HDF5 2.0: + // class 7, version 4, type 2 (object), encoding version 1, 18 bytes. + let bytes = [0x47, 0x12, 0x00, 0x00, 0x12, 0x00, 0x00, 0x00]; + let (dt, consumed) = Datatype::parse(&bytes).unwrap(); + assert_eq!(consumed, 8); + assert_eq!( + dt, + Datatype::Reference { + size: 18, + ref_type: ReferenceType::Object2 + } + ); + // The new types are only valid from datatype version 4. + let old_version = [0x37, 0x12, 0x00, 0x00, 0x12, 0x00, 0x00, 0x00]; + assert_eq!( + Datatype::parse(&old_version).unwrap_err(), + FormatError::InvalidReferenceType(2) + ); + } + #[test] fn test_error_invalid_reference_type() { let buf = build_dt_header(7, 1, [5, 0, 0], 8); diff --git a/crates/clawhdf5-format/tests/fixtures/gen_std_ref.py b/crates/clawhdf5-format/tests/fixtures/gen_std_ref.py new file mode 100644 index 0000000..f73a8c5 --- /dev/null +++ b/crates/clawhdf5-format/tests/fixtures/gen_std_ref.py @@ -0,0 +1,44 @@ +"""Generate std_ref_hdf5_2_0.h5: a dataset of H5T_STD_REF (the reference +datatype introduced in HDF5 1.12, datatype message version 4) holding two +object references — to /target (a dataset) and /grp (a group). + +h5py has no API for this type, so the file is written by calling the libhdf5 +bundled in the h5py wheel directly through ctypes. Written with h5py 3.16.0 / +HDF5 2.0.0. Re-run only if the fixture ever needs regenerating: + + python gen_std_ref.py std_ref_hdf5_2_0.h5 +""" +import ctypes +import glob +import os +import sys + +import h5py +import numpy as np + +libdir = os.path.join(os.path.dirname(os.path.dirname(h5py.__file__)), "h5py.libs") +libs = [p for p in glob.glob(os.path.join(libdir, "libhdf5*.so*")) if "_hl" not in os.path.basename(p)] +lib = ctypes.CDLL(libs[0]) +lib.H5open() +hid = ctypes.c_int64 +std_ref = hid.in_dll(lib, "H5T_STD_REF_g").value + +lib.H5Screate_simple.restype = hid +lib.H5Screate_simple.argtypes = [ctypes.c_int, ctypes.POINTER(ctypes.c_uint64), ctypes.POINTER(ctypes.c_uint64)] +lib.H5Dcreate2.restype = hid +lib.H5Dcreate2.argtypes = [hid, ctypes.c_char_p, hid, hid, hid, hid, hid] +lib.H5Rcreate_object.argtypes = [hid, ctypes.c_char_p, hid, ctypes.c_void_p] +lib.H5Dwrite.argtypes = [hid, hid, hid, hid, hid, ctypes.c_void_p] +lib.H5Dclose.argtypes = [hid] + +with h5py.File(sys.argv[1], "w", libver="latest") as f: + f.create_dataset("target", data=np.arange(5, dtype="uwDa20ohR0SouGLA&$=vRjEEjs+`?#P4hBYdU}S(&5OUOrK%+(l=G;9n#?Yjj wglAtGnl2x8^=Jr;hQP=P0Z@euDxH~u7*uI8F>t`DZ6FT>fFiJJ8zR5}01qQlYXATM literal 0 HcmV?d00001 diff --git a/crates/clawhdf5-format/tests/reference_tests.rs b/crates/clawhdf5-format/tests/reference_tests.rs index af44b15..b1dcf40 100644 --- a/crates/clawhdf5-format/tests/reference_tests.rs +++ b/crates/clawhdf5-format/tests/reference_tests.rs @@ -316,3 +316,97 @@ print('ok') // Clean up let _ = std::fs::remove_file(&path); } + +// --------------------------------------------------------------------------- +// H5T_STD_REF (HDF5 1.12+ references, datatype message version 4) +// --------------------------------------------------------------------------- + +/// `fixtures/std_ref_hdf5_2_0.h5` (see `gen_std_ref.py`) holds a dataset of +/// `H5T_STD_REF` with two object references, written by HDF5 2.0 itself. The +/// datatype used to be rejected with `InvalidReferenceType(2)`. +#[test] +fn std_ref_object_references_from_hdf5_2_0() { + use clawhdf5_format::data_layout::DataLayout; + use clawhdf5_format::dataspace::Dataspace; + use clawhdf5_format::group_v2::resolve_path_any; + use clawhdf5_format::message_type::MessageType; + use clawhdf5_format::object_header::ObjectHeader; + use clawhdf5_format::signature::find_signature; + use clawhdf5_format::superblock::Superblock; + + let bytes: &[u8] = include_bytes!("fixtures/std_ref_hdf5_2_0.h5"); + let sb = Superblock::parse(bytes, find_signature(bytes).unwrap()).unwrap(); + let (os, ls) = (sb.offset_size, sb.length_size); + + let refs_addr = resolve_path_any(bytes, &sb, "refs").unwrap(); + let header = ObjectHeader::parse(bytes, refs_addr as usize, os, ls).unwrap(); + let message = |t: MessageType| { + &header + .messages + .iter() + .find(|m| m.msg_type == t) + .unwrap() + .data + }; + + let (datatype, _) = Datatype::parse(message(MessageType::Datatype)).unwrap(); + assert_eq!( + datatype, + Datatype::Reference { + size: 18, + ref_type: ReferenceType::Object2 + } + ); + let dataspace = Dataspace::parse(message(MessageType::Dataspace), ls).unwrap(); + let layout = DataLayout::parse(message(MessageType::DataLayout), os, ls).unwrap(); + let raw = + clawhdf5_format::data_read::read_raw_data(bytes, &layout, &dataspace, &datatype).unwrap(); + assert_eq!(raw.len(), 2 * 18); + + // The references point at the objects they were created from. + let refs = read_object_references(&raw, &datatype, os).unwrap(); + let addresses: Vec = refs.iter().map(|r| r.address).collect(); + assert_eq!( + addresses, + [ + resolve_path_any(bytes, &sb, "target").unwrap(), + resolve_path_any(bytes, &sb, "grp").unwrap(), + ] + ); + // And what they point at is a real object header. + for address in addresses { + ObjectHeader::parse(bytes, address as usize, os, ls).unwrap(); + } +} + +#[test] +fn std_ref_decoding_rejects_malformed_elements() { + let dt = Datatype::Reference { + size: 18, + ref_type: ReferenceType::Object2, + }; + let mut good = vec![0u8; 18]; + good[..4].copy_from_slice(&[2, 0, 8, 0xb3]); + assert_eq!( + read_object_references(&good, &dt, 8).unwrap()[0].address, + 0xb3 + ); + + // Null reference. + assert_eq!( + read_object_references(&[0u8; 18], &dt, 8).unwrap()[0].address, + u64::MAX + ); + for (what, patch) in [ + ("wrong reference type", (0usize, 3u8)), + ("external flag", (1, 1)), + ("token longer than the element", (2, 200)), + ("zero-length token", (2, 0)), + ] { + let mut bad = good.clone(); + bad[patch.0] = patch.1; + assert!(read_object_references(&bad, &dt, 8).is_err(), "{what}"); + } + // Not a whole number of elements. + assert!(read_object_references(&good[..17], &dt, 8).is_err()); +} diff --git a/docs/known-issues.md b/docs/known-issues.md index d3c8993..c762b1a 100644 --- a/docs/known-issues.md +++ b/docs/known-issues.md @@ -62,13 +62,21 @@ only files using the native type through the C API / h5py low-level API hit this ## Revised reference datatype (class 7, version 4) is not parsed -**Status:** open, unconfirmed against a real file. +**Status:** fixed 2026-09-19 for object references; region and attribute +references are recognised but not decoded. -**Summary:** HDF5 1.12+ `H5T_STD_REF` references use datatype version 4 with -reference types 2–4 (object2 / region2 / attribute), which `Datatype::parse` -rejects with `InvalidReferenceType`. h5py still writes the legacy v1 -object/region references, which read correctly, so no reproducing file has been -generated yet; one written with the C API (`H5T_STD_REF`) is needed. +**Summary:** HDF5 1.12+ `H5T_STD_REF` references use datatype message version 4 +with reference types 2-4 (object / region / attribute), which `Datatype::parse` +rejected with `InvalidReferenceType`. h5py still writes the legacy references, +so no file had been available to test against. + +**Fix:** a real file was produced by driving the libhdf5 bundled in the h5py +wheel through ctypes (`tests/fixtures/gen_std_ref.py` -> +`std_ref_hdf5_2_0.h5`). The three new types parse as +`ReferenceType::{Object2, DatasetRegion2, Attribute}`, and +`read_object_references` decodes `Object2` elements (type, flags, token size, +token = target object header address). External references (flag bit 0) and +the region/attribute payloads are errors rather than misreads. ## `clawhdf5-gpu` `gpu_tests` can hang under the default parallel test runner