feat(format): parse H5T_STD_REF references and decode object references
HDF5 1.12 revised the reference datatype (class 7) in datatype message version 4: reference types 2-4 are the new H5T_STD_REF object / dataset-region / attribute references. Datatype::parse rejected them with InvalidReferenceType, so any dataset of that type was unreadable. h5py cannot write this type, which is why it had never been tested. A real file was produced by calling the libhdf5 bundled in the h5py wheel through ctypes (H5T_STD_REF_g, H5Rcreate_object, H5Dwrite); the 2 KB result is committed as tests/fixtures/std_ref_hdf5_2_0.h5 with its generator, gen_std_ref.py. - ReferenceType gains Object2, DatasetRegion2 and Attribute, accepted only from datatype version 4. - read_object_references decodes Object2 elements: type(1) flags(1) token_size(1) token, zero-padded to the element size; the token is the target's object header address. A null reference decodes to the undefined address; an external reference, a wrong type byte or a token that doesn't fit is an error. The fixture test follows both references and checks they resolve to the objects they were created from. Co-Authored-By: Claude Fable 5.1 <[email protected]>
This commit is contained in:
osobh
co-authored by
Claude Fable 5.1
Claude Fable 5.1
parent
05c665a898
commit
52cfcf20b2
@@ -1427,6 +1427,26 @@ pub fn read_object_references(
|
||||
}
|
||||
Ok(result)
|
||||
}
|
||||
Datatype::Reference {
|
||||
ref_type: crate::datatype::ReferenceType::Object2,
|
||||
size,
|
||||
} => {
|
||||
let elem_size = *size as usize;
|
||||
if elem_size == 0 {
|
||||
return Ok(Vec::new());
|
||||
}
|
||||
if !raw.len().is_multiple_of(elem_size) {
|
||||
return Err(FormatError::DataSizeMismatch {
|
||||
expected: 0,
|
||||
actual: raw.len(),
|
||||
});
|
||||
}
|
||||
raw.chunks_exact(elem_size)
|
||||
.map(|element| {
|
||||
decode_std_object_ref(element).map(|address| ObjectReference { address })
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
_ => Err(FormatError::TypeMismatch {
|
||||
expected: "Reference(Object)",
|
||||
actual: datatype_name(datatype),
|
||||
@@ -1434,6 +1454,46 @@ pub fn read_object_references(
|
||||
}
|
||||
}
|
||||
|
||||
/// Decode one `H5T_STD_REF` object reference as stored in a dataset:
|
||||
/// `type(1) flags(1) token_size(1) token(token_size)`, zero-padded to the
|
||||
/// element size. For a reference within the same file the token is the target
|
||||
/// object's header address. An all-zero element is a null reference and
|
||||
/// decodes to the undefined address (`u64::MAX`).
|
||||
fn decode_std_object_ref(element: &[u8]) -> Result<u64, FormatError> {
|
||||
const STD_REF_OBJECT: u8 = 2;
|
||||
const FLAG_EXTERNAL: u8 = 0x01;
|
||||
if element.iter().all(|&b| b == 0) {
|
||||
return Ok(u64::MAX);
|
||||
}
|
||||
let [ref_type, flags, token_size, token @ ..] = element else {
|
||||
return Err(FormatError::UnexpectedEof {
|
||||
expected: 3,
|
||||
available: element.len(),
|
||||
});
|
||||
};
|
||||
if *ref_type != STD_REF_OBJECT {
|
||||
return Err(FormatError::InvalidReferenceType(*ref_type));
|
||||
}
|
||||
if flags & FLAG_EXTERNAL != 0 {
|
||||
// Carries a file name as well; nothing here follows those.
|
||||
return Err(FormatError::TypeMismatch {
|
||||
expected: "object reference within this file",
|
||||
actual: "external object reference",
|
||||
});
|
||||
}
|
||||
let n = *token_size as usize;
|
||||
if n == 0 || n > 8 || n > token.len() {
|
||||
return Err(FormatError::UnexpectedEof {
|
||||
expected: 3 + n,
|
||||
available: element.len(),
|
||||
});
|
||||
}
|
||||
Ok(token[..n]
|
||||
.iter()
|
||||
.rev()
|
||||
.fold(0u64, |addr, &byte| (addr << 8) | u64::from(byte)))
|
||||
}
|
||||
|
||||
/// Read region references from raw bytes.
|
||||
///
|
||||
/// Region references encode a dataset selection (hyperslab, point list, etc.)
|
||||
|
||||
@@ -36,8 +36,18 @@ pub enum CharacterSet {
|
||||
/// Reference type.
|
||||
#[derive(Debug, Clone, PartialEq)]
|
||||
pub enum ReferenceType {
|
||||
/// Legacy object reference: the target's object header address.
|
||||
Object,
|
||||
/// Legacy dataset region reference.
|
||||
DatasetRegion,
|
||||
/// `H5T_STD_REF` object reference (HDF5 1.12+, datatype message version
|
||||
/// 4): a small header followed by an object token. Decoded by
|
||||
/// `data_read::read_object_references`.
|
||||
Object2,
|
||||
/// `H5T_STD_REF` dataset region reference.
|
||||
DatasetRegion2,
|
||||
/// `H5T_STD_REF` attribute reference.
|
||||
Attribute,
|
||||
}
|
||||
|
||||
/// A member of a compound datatype.
|
||||
@@ -424,9 +434,15 @@ impl Datatype {
|
||||
7 => {
|
||||
// Reference
|
||||
let ref_type_val = bf0 & 0x0F;
|
||||
let ref_type = match ref_type_val {
|
||||
0 => ReferenceType::Object,
|
||||
1 => ReferenceType::DatasetRegion,
|
||||
// Datatype message version 4 (HDF5 1.12) revised this class:
|
||||
// types 2-4 are the new `H5T_STD_REF` references, and the high
|
||||
// nibble of the first flag byte carries their encoding version.
|
||||
let ref_type = match (ref_type_val, version) {
|
||||
(0, _) => ReferenceType::Object,
|
||||
(1, _) => ReferenceType::DatasetRegion,
|
||||
(2, 4..) => ReferenceType::Object2,
|
||||
(3, 4..) => ReferenceType::DatasetRegion2,
|
||||
(4, 4..) => ReferenceType::Attribute,
|
||||
_ => return Err(FormatError::InvalidReferenceType(ref_type_val)),
|
||||
};
|
||||
Ok((Datatype::Reference { size, ref_type }, pos))
|
||||
@@ -1563,6 +1579,28 @@ mod tests {
|
||||
assert_eq!(err, FormatError::InvalidCharacterSet(2));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_reference_v4_std_ref_from_hdf5_2_0() {
|
||||
// Datatype message of an H5T_STD_REF dataset written by HDF5 2.0:
|
||||
// class 7, version 4, type 2 (object), encoding version 1, 18 bytes.
|
||||
let bytes = [0x47, 0x12, 0x00, 0x00, 0x12, 0x00, 0x00, 0x00];
|
||||
let (dt, consumed) = Datatype::parse(&bytes).unwrap();
|
||||
assert_eq!(consumed, 8);
|
||||
assert_eq!(
|
||||
dt,
|
||||
Datatype::Reference {
|
||||
size: 18,
|
||||
ref_type: ReferenceType::Object2
|
||||
}
|
||||
);
|
||||
// The new types are only valid from datatype version 4.
|
||||
let old_version = [0x37, 0x12, 0x00, 0x00, 0x12, 0x00, 0x00, 0x00];
|
||||
assert_eq!(
|
||||
Datatype::parse(&old_version).unwrap_err(),
|
||||
FormatError::InvalidReferenceType(2)
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_error_invalid_reference_type() {
|
||||
let buf = build_dt_header(7, 1, [5, 0, 0], 8);
|
||||
|
||||
@@ -0,0 +1,44 @@
|
||||
"""Generate std_ref_hdf5_2_0.h5: a dataset of H5T_STD_REF (the reference
|
||||
datatype introduced in HDF5 1.12, datatype message version 4) holding two
|
||||
object references — to /target (a dataset) and /grp (a group).
|
||||
|
||||
h5py has no API for this type, so the file is written by calling the libhdf5
|
||||
bundled in the h5py wheel directly through ctypes. Written with h5py 3.16.0 /
|
||||
HDF5 2.0.0. Re-run only if the fixture ever needs regenerating:
|
||||
|
||||
python gen_std_ref.py std_ref_hdf5_2_0.h5
|
||||
"""
|
||||
import ctypes
|
||||
import glob
|
||||
import os
|
||||
import sys
|
||||
|
||||
import h5py
|
||||
import numpy as np
|
||||
|
||||
libdir = os.path.join(os.path.dirname(os.path.dirname(h5py.__file__)), "h5py.libs")
|
||||
libs = [p for p in glob.glob(os.path.join(libdir, "libhdf5*.so*")) if "_hl" not in os.path.basename(p)]
|
||||
lib = ctypes.CDLL(libs[0])
|
||||
lib.H5open()
|
||||
hid = ctypes.c_int64
|
||||
std_ref = hid.in_dll(lib, "H5T_STD_REF_g").value
|
||||
|
||||
lib.H5Screate_simple.restype = hid
|
||||
lib.H5Screate_simple.argtypes = [ctypes.c_int, ctypes.POINTER(ctypes.c_uint64), ctypes.POINTER(ctypes.c_uint64)]
|
||||
lib.H5Dcreate2.restype = hid
|
||||
lib.H5Dcreate2.argtypes = [hid, ctypes.c_char_p, hid, hid, hid, hid, hid]
|
||||
lib.H5Rcreate_object.argtypes = [hid, ctypes.c_char_p, hid, ctypes.c_void_p]
|
||||
lib.H5Dwrite.argtypes = [hid, hid, hid, hid, hid, ctypes.c_void_p]
|
||||
lib.H5Dclose.argtypes = [hid]
|
||||
|
||||
with h5py.File(sys.argv[1], "w", libver="latest") as f:
|
||||
f.create_dataset("target", data=np.arange(5, dtype="<i4"))
|
||||
f.create_group("grp")
|
||||
fid = f.id.id
|
||||
sid = lib.H5Screate_simple(1, (ctypes.c_uint64 * 1)(2), None)
|
||||
did = lib.H5Dcreate2(fid, b"refs", std_ref, sid, 0, 0, 0)
|
||||
refs = ((ctypes.c_ubyte * 64) * 2)() # H5R_ref_t is a 64-byte buffer
|
||||
assert lib.H5Rcreate_object(fid, b"/target", 0, ctypes.byref(refs[0])) == 0
|
||||
assert lib.H5Rcreate_object(fid, b"/grp", 0, ctypes.byref(refs[1])) == 0
|
||||
assert lib.H5Dwrite(did, std_ref, 0, 0, 0, ctypes.byref(refs)) == 0
|
||||
lib.H5Dclose(did)
|
||||
Binary file not shown.
@@ -316,3 +316,97 @@ print('ok')
|
||||
// Clean up
|
||||
let _ = std::fs::remove_file(&path);
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// H5T_STD_REF (HDF5 1.12+ references, datatype message version 4)
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/// `fixtures/std_ref_hdf5_2_0.h5` (see `gen_std_ref.py`) holds a dataset of
|
||||
/// `H5T_STD_REF` with two object references, written by HDF5 2.0 itself. The
|
||||
/// datatype used to be rejected with `InvalidReferenceType(2)`.
|
||||
#[test]
|
||||
fn std_ref_object_references_from_hdf5_2_0() {
|
||||
use clawhdf5_format::data_layout::DataLayout;
|
||||
use clawhdf5_format::dataspace::Dataspace;
|
||||
use clawhdf5_format::group_v2::resolve_path_any;
|
||||
use clawhdf5_format::message_type::MessageType;
|
||||
use clawhdf5_format::object_header::ObjectHeader;
|
||||
use clawhdf5_format::signature::find_signature;
|
||||
use clawhdf5_format::superblock::Superblock;
|
||||
|
||||
let bytes: &[u8] = include_bytes!("fixtures/std_ref_hdf5_2_0.h5");
|
||||
let sb = Superblock::parse(bytes, find_signature(bytes).unwrap()).unwrap();
|
||||
let (os, ls) = (sb.offset_size, sb.length_size);
|
||||
|
||||
let refs_addr = resolve_path_any(bytes, &sb, "refs").unwrap();
|
||||
let header = ObjectHeader::parse(bytes, refs_addr as usize, os, ls).unwrap();
|
||||
let message = |t: MessageType| {
|
||||
&header
|
||||
.messages
|
||||
.iter()
|
||||
.find(|m| m.msg_type == t)
|
||||
.unwrap()
|
||||
.data
|
||||
};
|
||||
|
||||
let (datatype, _) = Datatype::parse(message(MessageType::Datatype)).unwrap();
|
||||
assert_eq!(
|
||||
datatype,
|
||||
Datatype::Reference {
|
||||
size: 18,
|
||||
ref_type: ReferenceType::Object2
|
||||
}
|
||||
);
|
||||
let dataspace = Dataspace::parse(message(MessageType::Dataspace), ls).unwrap();
|
||||
let layout = DataLayout::parse(message(MessageType::DataLayout), os, ls).unwrap();
|
||||
let raw =
|
||||
clawhdf5_format::data_read::read_raw_data(bytes, &layout, &dataspace, &datatype).unwrap();
|
||||
assert_eq!(raw.len(), 2 * 18);
|
||||
|
||||
// The references point at the objects they were created from.
|
||||
let refs = read_object_references(&raw, &datatype, os).unwrap();
|
||||
let addresses: Vec<u64> = refs.iter().map(|r| r.address).collect();
|
||||
assert_eq!(
|
||||
addresses,
|
||||
[
|
||||
resolve_path_any(bytes, &sb, "target").unwrap(),
|
||||
resolve_path_any(bytes, &sb, "grp").unwrap(),
|
||||
]
|
||||
);
|
||||
// And what they point at is a real object header.
|
||||
for address in addresses {
|
||||
ObjectHeader::parse(bytes, address as usize, os, ls).unwrap();
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn std_ref_decoding_rejects_malformed_elements() {
|
||||
let dt = Datatype::Reference {
|
||||
size: 18,
|
||||
ref_type: ReferenceType::Object2,
|
||||
};
|
||||
let mut good = vec![0u8; 18];
|
||||
good[..4].copy_from_slice(&[2, 0, 8, 0xb3]);
|
||||
assert_eq!(
|
||||
read_object_references(&good, &dt, 8).unwrap()[0].address,
|
||||
0xb3
|
||||
);
|
||||
|
||||
// Null reference.
|
||||
assert_eq!(
|
||||
read_object_references(&[0u8; 18], &dt, 8).unwrap()[0].address,
|
||||
u64::MAX
|
||||
);
|
||||
for (what, patch) in [
|
||||
("wrong reference type", (0usize, 3u8)),
|
||||
("external flag", (1, 1)),
|
||||
("token longer than the element", (2, 200)),
|
||||
("zero-length token", (2, 0)),
|
||||
] {
|
||||
let mut bad = good.clone();
|
||||
bad[patch.0] = patch.1;
|
||||
assert!(read_object_references(&bad, &dt, 8).is_err(), "{what}");
|
||||
}
|
||||
// Not a whole number of elements.
|
||||
assert!(read_object_references(&good[..17], &dt, 8).is_err());
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user