feat(format): parse H5T_STD_REF references and decode object references
HDF5 1.12 revised the reference datatype (class 7) in datatype message version 4: reference types 2-4 are the new H5T_STD_REF object / dataset-region / attribute references. Datatype::parse rejected them with InvalidReferenceType, so any dataset of that type was unreadable. h5py cannot write this type, which is why it had never been tested. A real file was produced by calling the libhdf5 bundled in the h5py wheel through ctypes (H5T_STD_REF_g, H5Rcreate_object, H5Dwrite); the 2 KB result is committed as tests/fixtures/std_ref_hdf5_2_0.h5 with its generator, gen_std_ref.py. - ReferenceType gains Object2, DatasetRegion2 and Attribute, accepted only from datatype version 4. - read_object_references decodes Object2 elements: type(1) flags(1) token_size(1) token, zero-padded to the element size; the token is the target's object header address. A null reference decodes to the undefined address; an external reference, a wrong type byte or a token that doesn't fit is an error. The fixture test follows both references and checks they resolve to the objects they were created from. Co-Authored-By: Claude Fable 5.1 <[email protected]>
This commit is contained in:
osobh
co-authored by
Claude Fable 5.1
Claude Fable 5.1
parent
05c665a898
commit
52cfcf20b2
@@ -0,0 +1,44 @@
|
||||
"""Generate std_ref_hdf5_2_0.h5: a dataset of H5T_STD_REF (the reference
|
||||
datatype introduced in HDF5 1.12, datatype message version 4) holding two
|
||||
object references — to /target (a dataset) and /grp (a group).
|
||||
|
||||
h5py has no API for this type, so the file is written by calling the libhdf5
|
||||
bundled in the h5py wheel directly through ctypes. Written with h5py 3.16.0 /
|
||||
HDF5 2.0.0. Re-run only if the fixture ever needs regenerating:
|
||||
|
||||
python gen_std_ref.py std_ref_hdf5_2_0.h5
|
||||
"""
|
||||
import ctypes
|
||||
import glob
|
||||
import os
|
||||
import sys
|
||||
|
||||
import h5py
|
||||
import numpy as np
|
||||
|
||||
libdir = os.path.join(os.path.dirname(os.path.dirname(h5py.__file__)), "h5py.libs")
|
||||
libs = [p for p in glob.glob(os.path.join(libdir, "libhdf5*.so*")) if "_hl" not in os.path.basename(p)]
|
||||
lib = ctypes.CDLL(libs[0])
|
||||
lib.H5open()
|
||||
hid = ctypes.c_int64
|
||||
std_ref = hid.in_dll(lib, "H5T_STD_REF_g").value
|
||||
|
||||
lib.H5Screate_simple.restype = hid
|
||||
lib.H5Screate_simple.argtypes = [ctypes.c_int, ctypes.POINTER(ctypes.c_uint64), ctypes.POINTER(ctypes.c_uint64)]
|
||||
lib.H5Dcreate2.restype = hid
|
||||
lib.H5Dcreate2.argtypes = [hid, ctypes.c_char_p, hid, hid, hid, hid, hid]
|
||||
lib.H5Rcreate_object.argtypes = [hid, ctypes.c_char_p, hid, ctypes.c_void_p]
|
||||
lib.H5Dwrite.argtypes = [hid, hid, hid, hid, hid, ctypes.c_void_p]
|
||||
lib.H5Dclose.argtypes = [hid]
|
||||
|
||||
with h5py.File(sys.argv[1], "w", libver="latest") as f:
|
||||
f.create_dataset("target", data=np.arange(5, dtype="<i4"))
|
||||
f.create_group("grp")
|
||||
fid = f.id.id
|
||||
sid = lib.H5Screate_simple(1, (ctypes.c_uint64 * 1)(2), None)
|
||||
did = lib.H5Dcreate2(fid, b"refs", std_ref, sid, 0, 0, 0)
|
||||
refs = ((ctypes.c_ubyte * 64) * 2)() # H5R_ref_t is a 64-byte buffer
|
||||
assert lib.H5Rcreate_object(fid, b"/target", 0, ctypes.byref(refs[0])) == 0
|
||||
assert lib.H5Rcreate_object(fid, b"/grp", 0, ctypes.byref(refs[1])) == 0
|
||||
assert lib.H5Dwrite(did, std_ref, 0, 0, 0, ctypes.byref(refs)) == 0
|
||||
lib.H5Dclose(did)
|
||||
Binary file not shown.
@@ -316,3 +316,97 @@ print('ok')
|
||||
// Clean up
|
||||
let _ = std::fs::remove_file(&path);
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// H5T_STD_REF (HDF5 1.12+ references, datatype message version 4)
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/// `fixtures/std_ref_hdf5_2_0.h5` (see `gen_std_ref.py`) holds a dataset of
|
||||
/// `H5T_STD_REF` with two object references, written by HDF5 2.0 itself. The
|
||||
/// datatype used to be rejected with `InvalidReferenceType(2)`.
|
||||
#[test]
|
||||
fn std_ref_object_references_from_hdf5_2_0() {
|
||||
use clawhdf5_format::data_layout::DataLayout;
|
||||
use clawhdf5_format::dataspace::Dataspace;
|
||||
use clawhdf5_format::group_v2::resolve_path_any;
|
||||
use clawhdf5_format::message_type::MessageType;
|
||||
use clawhdf5_format::object_header::ObjectHeader;
|
||||
use clawhdf5_format::signature::find_signature;
|
||||
use clawhdf5_format::superblock::Superblock;
|
||||
|
||||
let bytes: &[u8] = include_bytes!("fixtures/std_ref_hdf5_2_0.h5");
|
||||
let sb = Superblock::parse(bytes, find_signature(bytes).unwrap()).unwrap();
|
||||
let (os, ls) = (sb.offset_size, sb.length_size);
|
||||
|
||||
let refs_addr = resolve_path_any(bytes, &sb, "refs").unwrap();
|
||||
let header = ObjectHeader::parse(bytes, refs_addr as usize, os, ls).unwrap();
|
||||
let message = |t: MessageType| {
|
||||
&header
|
||||
.messages
|
||||
.iter()
|
||||
.find(|m| m.msg_type == t)
|
||||
.unwrap()
|
||||
.data
|
||||
};
|
||||
|
||||
let (datatype, _) = Datatype::parse(message(MessageType::Datatype)).unwrap();
|
||||
assert_eq!(
|
||||
datatype,
|
||||
Datatype::Reference {
|
||||
size: 18,
|
||||
ref_type: ReferenceType::Object2
|
||||
}
|
||||
);
|
||||
let dataspace = Dataspace::parse(message(MessageType::Dataspace), ls).unwrap();
|
||||
let layout = DataLayout::parse(message(MessageType::DataLayout), os, ls).unwrap();
|
||||
let raw =
|
||||
clawhdf5_format::data_read::read_raw_data(bytes, &layout, &dataspace, &datatype).unwrap();
|
||||
assert_eq!(raw.len(), 2 * 18);
|
||||
|
||||
// The references point at the objects they were created from.
|
||||
let refs = read_object_references(&raw, &datatype, os).unwrap();
|
||||
let addresses: Vec<u64> = refs.iter().map(|r| r.address).collect();
|
||||
assert_eq!(
|
||||
addresses,
|
||||
[
|
||||
resolve_path_any(bytes, &sb, "target").unwrap(),
|
||||
resolve_path_any(bytes, &sb, "grp").unwrap(),
|
||||
]
|
||||
);
|
||||
// And what they point at is a real object header.
|
||||
for address in addresses {
|
||||
ObjectHeader::parse(bytes, address as usize, os, ls).unwrap();
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn std_ref_decoding_rejects_malformed_elements() {
|
||||
let dt = Datatype::Reference {
|
||||
size: 18,
|
||||
ref_type: ReferenceType::Object2,
|
||||
};
|
||||
let mut good = vec![0u8; 18];
|
||||
good[..4].copy_from_slice(&[2, 0, 8, 0xb3]);
|
||||
assert_eq!(
|
||||
read_object_references(&good, &dt, 8).unwrap()[0].address,
|
||||
0xb3
|
||||
);
|
||||
|
||||
// Null reference.
|
||||
assert_eq!(
|
||||
read_object_references(&[0u8; 18], &dt, 8).unwrap()[0].address,
|
||||
u64::MAX
|
||||
);
|
||||
for (what, patch) in [
|
||||
("wrong reference type", (0usize, 3u8)),
|
||||
("external flag", (1, 1)),
|
||||
("token longer than the element", (2, 200)),
|
||||
("zero-length token", (2, 0)),
|
||||
] {
|
||||
let mut bad = good.clone();
|
||||
bad[patch.0] = patch.1;
|
||||
assert!(read_object_references(&bad, &dt, 8).is_err(), "{what}");
|
||||
}
|
||||
// Not a whole number of elements.
|
||||
assert!(read_object_references(&good[..17], &dt, 8).is_err());
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user