quantumclaw/clawhdf5
3
1
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 1 Wiki Activity

fix(format): parse HDF5 2.0 native complex datatypes (class 11)

Browse Source 
Class 11 (datatype version 5) properties are a single base floating-point
datatype message, not a compound-style member list. The old parser read the
base type's bytes as member names, yielding a garbage datatype, and failed
with UnexpectedEof when a complex type was nested in a compound.

Parse the base type and surface the type as the equivalent {r, i} compound
(the shape h5py writes for numpy complex dtypes), with a size check against
the base type. Covered by byte-level tests taken from HDF5 2.0 output and an
h5py end-to-end test (writer_h5py_tests is now 27/27 against HDF5 2.0.0).

Found while validating a user report of InvalidDatatypeVersion
{ class: 6, version: 5 } against v2.1.0 (already fixed on main in a13ff51,
never released). Add docs/known-issues.md recording that report, this bug,
the open reference-v4 gap and a gpu_tests parallel-run hang; credit the
reporter in the changelog.

Co-Authored-By: Claude Fable 5.1 <[email protected]>
This commit is contained in:
osobh
2026-09-18 20:58:29 -07:00
co-authored by Claude Fable 5.1
parent 48c745a960
commit b55b7dbac5
4 changed files with 260 additions and 19 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/clawhdf5-format/src/datatype.rs
+100 -19
View File
@@ -546,27 +546,39 @@ impl Datatype {
}
}
11 => {
// Complex number — store as compound of two floats internally
// Parse like compound with version 3 and 2 members
// But actually class 11 has no special properties beyond class 6 compound.
// It's just recognized as a separate class. For now parse the 2 members
// as compound.
let num_members = (bf0 as u16) | ((bf1 as u16) << 8);
let mut members = Vec::with_capacity(num_members as usize);
let ob = offset_bytes_for_size(size);
for _ in 0..num_members {
let (name, name_len) = read_null_terminated_string(data, pos)?;
pos += name_len;
let byte_offset = read_uint(data, pos, ob)?;
pos += ob;
let (member_dt, consumed) = Self::parse_with_depth(&data[pos..], depth + 1)?;
pos += consumed;
members.push(CompoundMember {
name,
byte_offset,
datatype: member_dt,
// Complex number (HDF5 2.0, datatype version 5). The properties
// are a single base floating-point datatype message; an element
// is two consecutive base-type values (real, imaginary). There
// is no member list. Surface it as the equivalent two-member
// compound `{r, i}` — the same shape h5py writes for numpy
// complex dtypes — so downstream compound readers work as-is.
if version != 5 {
return Err(FormatError::InvalidDatatypeVersion {
class: class_id,
version,
});
}
let (base_type, consumed) = Self::parse_with_depth(&data[pos..], depth + 1)?;
pos += consumed;
let base_size = base_type.type_size();
if base_size.checked_mul(2) != Some(size) {
return Err(FormatError::DataSizeMismatch {
expected: (base_size as usize).saturating_mul(2),
actual: size as usize,
});
}
let members = vec![
CompoundMember {
name: String::from("r"),
byte_offset: 0,
datatype: base_type.clone(),
},
CompoundMember {
name: String::from("i"),
byte_offset: base_size as u64,
datatype: base_type,
},
];
Ok((Datatype::Compound { size, members }, pos))
}
_ => Err(FormatError::InvalidDatatypeClass(class_id)),
@@ -1126,6 +1138,75 @@ mod tests {
}
}
/// Real datatype message bytes emitted by HDF5 2.0 for the native complex
/// type `H5T_COMPLEX_IEEE_F64LE`: class 11, version 5, size 16, followed by
/// the base IEEE f64 datatype message.
const COMPLEX_F64_HDF5_2_0: [u8; 28] = [
0x5b, 0x01, 0x00, 0x00, 0x10, 0x00, 0x00, 0x00, 0x11, 0x20, 0x3f, 0x00, 0x08, 0x00, 0x00,
0x00, 0x00, 0x00, 0x40, 0x00, 0x34, 0x0b, 0x00, 0x34, 0xff, 0x03, 0x00, 0x00,
];
#[test]
fn test_complex_v5_from_hdf5_2_0() {
let (dt, consumed) = Datatype::parse(&COMPLEX_F64_HDF5_2_0).unwrap();
assert_eq!(consumed, COMPLEX_F64_HDF5_2_0.len());
match dt {
Datatype::Compound { size, members } => {
assert_eq!(size, 16);
assert_eq!(members.len(), 2);
assert_eq!((members[0].name.as_str(), members[0].byte_offset), ("r", 0));
assert_eq!((members[1].name.as_str(), members[1].byte_offset), ("i", 8));
for m in &members {
assert!(matches!(
m.datatype,
Datatype::FloatingPoint { size: 8, .. }
));
}
}
other => panic!("expected Compound, got {other:?}"),
}
}
#[test]
fn test_compound_with_complex_member_from_hdf5_2_0() {
// Compound { z: complex f64 @0, k: i64 @16 } as written by HDF5 2.0.
// Regression guard: the complex member must consume exactly its own
// bytes so the following member parses.
let mut bytes = vec![0x56, 0x02, 0x00, 0x00, 0x18, 0x00, 0x00, 0x00, b'z', 0x00, 0x00];
bytes.extend_from_slice(&COMPLEX_F64_HDF5_2_0);
bytes.extend_from_slice(&[b'k', 0x00, 0x10]);
bytes.extend_from_slice(&[
0x10, 0x08, 0x00, 0x00, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x40, 0x00,
]);
let (dt, consumed) = Datatype::parse(&bytes).unwrap();
assert_eq!(consumed, bytes.len());
match dt {
Datatype::Compound { size, members } => {
assert_eq!(size, 24);
assert_eq!(members.len(), 2);
assert!(matches!(
&members[0].datatype,
Datatype::Compound { size: 16, members } if members.len() == 2
));
assert_eq!((members[1].name.as_str(), members[1].byte_offset), ("k", 16));
}
other => panic!("expected Compound, got {other:?}"),
}
}
#[test]
fn test_complex_size_mismatch_rejected() {
let mut bytes = COMPLEX_F64_HDF5_2_0;
bytes[4] = 0x0c; // claims 12 bytes, base type is 8
assert!(matches!(
Datatype::parse(&bytes),
Err(FormatError::DataSizeMismatch {
expected: 16,
actual: 12
})
));
}
#[test]
fn test_reference_object() {
let buf = build_dt_header(7, 1, [0, 0, 0], 8);