fix(format): parse compound datatype versions 1 and 2 correctly
Compound datasets written with default libver bounds (datatype message version 1, i.e. plain h5py.File(path, 'w')) could not be read: the v1 member layout has 28 bytes of legacy array fields after the byte offset (dimensionality 1, reserved 3, permutation 4, reserved 4, four sizes 16) and the parser skipped 24, so every following member was read 4 bytes off. v2 was also wrong: it keeps the 8-byte name padding and has no array fields. Found by adding a default-libver axis to the h5py-generated-file tests (HDF5 2.0 raised the default low bound to 1.8, so "default" files are a distinct format path from libver='latest'). Adds byte-level v1/v2 regression tests, a truncation test, and fuzz corpus seeds for v1 compound and native complex. Co-Authored-By: Claude Fable 5.1 <[email protected]>
This commit is contained in:
osobh
co-authored by
Claude Fable 5.1
Claude Fable 5.1
parent
a8ab9ca054
commit
926dc457e0
Binary file not shown.
Binary file not shown.
@@ -372,7 +372,8 @@ impl Datatype {
|
||||
pos += name_len;
|
||||
let byte_offset = read_uint(data, pos, ob)?;
|
||||
pos += ob;
|
||||
let (member_dt, consumed) = Self::parse_with_depth(&data[pos..], depth + 1)?;
|
||||
let (member_dt, consumed) =
|
||||
Self::parse_with_depth(&data[pos..], depth + 1)?;
|
||||
pos += consumed;
|
||||
members.push(CompoundMember {
|
||||
name,
|
||||
@@ -381,24 +382,29 @@ impl Datatype {
|
||||
});
|
||||
}
|
||||
} else if version == 1 || version == 2 {
|
||||
// v1/v2: name, offset(4), dimensionality(1), reserved(3), dim_perm(4),
|
||||
// reserved_dims(up to 4*4=16), member datatype
|
||||
// v1/v2: name (null-terminated, padded to a multiple of 8
|
||||
// bytes), offset(4), member datatype. v1 additionally
|
||||
// carries the legacy per-member array fields between the
|
||||
// offset and the member datatype: dimensionality(1),
|
||||
// reserved(3), dim_perm(4), reserved(4), 4 dim sizes(16).
|
||||
// v1 is what default (non-`latest`) libver bounds emit.
|
||||
for _ in 0..num_members {
|
||||
let (name, name_len) = read_null_terminated_string(data, pos)?;
|
||||
pos += name_len;
|
||||
// v1: names padded to 8-byte boundary
|
||||
if version == 1 {
|
||||
let total_name_bytes = name_len;
|
||||
let padded = (total_name_bytes + 7) & !7;
|
||||
pos = pos - name_len + padded;
|
||||
}
|
||||
let padded = name_len.checked_add(7).ok_or(FormatError::UnexpectedEof {
|
||||
expected: usize::MAX,
|
||||
available: data.len(),
|
||||
})? & !7;
|
||||
ensure_len(data, pos, padded)?;
|
||||
pos += padded;
|
||||
ensure_len(data, pos, 4)?;
|
||||
let byte_offset = LittleEndian::read_u32(&data[pos..pos + 4]) as u64;
|
||||
pos += 4;
|
||||
// dimensionality(1) + reserved(3) + dim_perm(4) + 4 dim slots(16) = 24
|
||||
ensure_len(data, pos, 24)?;
|
||||
pos += 24;
|
||||
let (member_dt, consumed) = Self::parse_with_depth(&data[pos..], depth + 1)?;
|
||||
if version == 1 {
|
||||
ensure_len(data, pos, 28)?;
|
||||
pos += 28;
|
||||
}
|
||||
let (member_dt, consumed) =
|
||||
Self::parse_with_depth(&data[pos..], depth + 1)?;
|
||||
pos += consumed;
|
||||
members.push(CompoundMember {
|
||||
name,
|
||||
@@ -1138,6 +1144,82 @@ mod tests {
|
||||
}
|
||||
}
|
||||
|
||||
/// Real datatype message bytes emitted by h5py 3.16 / HDF5 2.0 with
|
||||
/// *default* libver bounds for [('x','f8'),('y','f8'),('id','i4')]:
|
||||
/// compound datatype version 1 (padded names + 28 bytes of legacy
|
||||
/// per-member array fields).
|
||||
fn compound_v1_bytes() -> Vec<u8> {
|
||||
let f64le: [u8; 20] = [
|
||||
0x11, 0x20, 0x3f, 0x00, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x40, 0x00, 0x34, 0x0b,
|
||||
0x00, 0x34, 0xff, 0x03, 0x00, 0x00,
|
||||
];
|
||||
let i32le: [u8; 12] = [
|
||||
0x10, 0x08, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, 0x00, 0x00, 0x20, 0x00,
|
||||
];
|
||||
let mut b = vec![0x16, 0x03, 0x00, 0x00, 0x14, 0x00, 0x00, 0x00];
|
||||
for (name, offset, dt) in [
|
||||
(&b"x"[..], 0u32, &f64le[..]),
|
||||
(&b"y"[..], 8, &f64le[..]),
|
||||
(&b"id"[..], 16, &i32le[..]),
|
||||
] {
|
||||
let mut padded = name.to_vec();
|
||||
padded.resize((name.len() + 1 + 7) & !7, 0);
|
||||
b.extend_from_slice(&padded);
|
||||
b.extend_from_slice(&offset.to_le_bytes());
|
||||
b.extend_from_slice(&[0u8; 28]);
|
||||
b.extend_from_slice(dt);
|
||||
}
|
||||
b
|
||||
}
|
||||
|
||||
fn assert_xyid_compound(dt: Datatype) {
|
||||
match dt {
|
||||
Datatype::Compound { size, members } => {
|
||||
assert_eq!(size, 20);
|
||||
let got: Vec<(&str, u64, u32)> = members
|
||||
.iter()
|
||||
.map(|m| (m.name.as_str(), m.byte_offset, m.datatype.type_size()))
|
||||
.collect();
|
||||
assert_eq!(got, vec![("x", 0, 8), ("y", 8, 8), ("id", 16, 4)]);
|
||||
}
|
||||
other => panic!("expected Compound, got {other:?}"),
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_compound_v1_default_libver() {
|
||||
let bytes = compound_v1_bytes();
|
||||
let (dt, consumed) = Datatype::parse(&bytes).unwrap();
|
||||
assert_eq!(consumed, bytes.len());
|
||||
assert_xyid_compound(dt);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_compound_v2_padded_names_no_array_fields() {
|
||||
// v2 = v1 without the 28 bytes of per-member array fields; names are
|
||||
// still padded to a multiple of 8 (matches libhdf5's H5O decoder).
|
||||
let v1 = compound_v1_bytes();
|
||||
let mut v2 = vec![0x26, 0x03, 0x00, 0x00, 0x14, 0x00, 0x00, 0x00];
|
||||
let mut pos = 8;
|
||||
for dt_len in [20usize, 20, 12] {
|
||||
v2.extend_from_slice(&v1[pos..pos + 8 + 4]); // padded name + offset
|
||||
pos += 8 + 4 + 28;
|
||||
v2.extend_from_slice(&v1[pos..pos + dt_len]);
|
||||
pos += dt_len;
|
||||
}
|
||||
let (dt, consumed) = Datatype::parse(&v2).unwrap();
|
||||
assert_eq!(consumed, v2.len());
|
||||
assert_xyid_compound(dt);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_compound_v1_truncated_is_error_not_panic() {
|
||||
let bytes = compound_v1_bytes();
|
||||
for cut in 8..bytes.len() {
|
||||
assert!(Datatype::parse(&bytes[..cut]).is_err(), "cut at {cut}");
|
||||
}
|
||||
}
|
||||
|
||||
/// Real datatype message bytes emitted by HDF5 2.0 for the native complex
|
||||
/// type `H5T_COMPLEX_IEEE_F64LE`: class 11, version 5, size 16, followed by
|
||||
/// the base IEEE f64 datatype message.
|
||||
@@ -1172,7 +1254,9 @@ mod tests {
|
||||
// Compound { z: complex f64 @0, k: i64 @16 } as written by HDF5 2.0.
|
||||
// Regression guard: the complex member must consume exactly its own
|
||||
// bytes so the following member parses.
|
||||
let mut bytes = vec![0x56, 0x02, 0x00, 0x00, 0x18, 0x00, 0x00, 0x00, b'z', 0x00, 0x00];
|
||||
let mut bytes = vec![
|
||||
0x56, 0x02, 0x00, 0x00, 0x18, 0x00, 0x00, 0x00, b'z', 0x00, 0x00,
|
||||
];
|
||||
bytes.extend_from_slice(&COMPLEX_F64_HDF5_2_0);
|
||||
bytes.extend_from_slice(&[b'k', 0x00, 0x10]);
|
||||
bytes.extend_from_slice(&[
|
||||
@@ -1188,7 +1272,10 @@ mod tests {
|
||||
&members[0].datatype,
|
||||
Datatype::Compound { size: 16, members } if members.len() == 2
|
||||
));
|
||||
assert_eq!((members[1].name.as_str(), members[1].byte_offset), ("k", 16));
|
||||
assert_eq!(
|
||||
(members[1].name.as_str(), members[1].byte_offset),
|
||||
("k", 16)
|
||||
);
|
||||
}
|
||||
other => panic!("expected Compound, got {other:?}"),
|
||||
}
|
||||
|
||||
@@ -235,17 +235,31 @@ fn h5py_reads_our_array_dataset() {
|
||||
#[test]
|
||||
#[ignore = "requires Python h5py module"]
|
||||
fn read_h5py_generated_compound() {
|
||||
let path = std::env::temp_dir().join("clawhdf5_h5py_compound.h5");
|
||||
check_h5py_generated_compound("latest", ", libver='latest'");
|
||||
}
|
||||
|
||||
/// Same file written with h5py's default format bounds. HDF5 2.0 raised the
|
||||
/// default low bound to 1.8, so "default" files exercise different on-disk
|
||||
/// structures than both `libver='latest'` and pre-2.0 defaults.
|
||||
#[test]
|
||||
#[ignore = "requires Python h5py module"]
|
||||
fn read_h5py_generated_compound_default_libver() {
|
||||
check_h5py_generated_compound("default", "");
|
||||
}
|
||||
|
||||
fn check_h5py_generated_compound(tag: &str, libver_kw: &str) {
|
||||
let path = std::env::temp_dir().join(format!("clawhdf5_h5py_compound_{tag}.h5"));
|
||||
let gen_script = format!(
|
||||
r#"
|
||||
import h5py, numpy as np
|
||||
dt = np.dtype([('x', 'f8'), ('y', 'f8'), ('id', 'i4')])
|
||||
data = np.array([(1.0, 2.0, 10), (3.0, 4.0, 20)], dtype=dt)
|
||||
f = h5py.File('{}', 'w', libver='latest')
|
||||
f = h5py.File('{}', 'w'{})
|
||||
f.create_dataset('particles', data=data)
|
||||
f.close()
|
||||
"#,
|
||||
path.display()
|
||||
path.display(),
|
||||
libver_kw
|
||||
);
|
||||
h5py_read(&path, &gen_script);
|
||||
|
||||
@@ -363,17 +377,31 @@ else:
|
||||
#[test]
|
||||
#[ignore = "requires Python h5py module"]
|
||||
fn read_h5py_generated_enum() {
|
||||
let path = std::env::temp_dir().join("clawhdf5_h5py_enum.h5");
|
||||
check_h5py_generated_enum("latest", ", libver='latest'");
|
||||
}
|
||||
|
||||
/// Same file written with h5py's default format bounds. HDF5 2.0 raised the
|
||||
/// default low bound to 1.8, so "default" files exercise different on-disk
|
||||
/// structures than both `libver='latest'` and pre-2.0 defaults.
|
||||
#[test]
|
||||
#[ignore = "requires Python h5py module"]
|
||||
fn read_h5py_generated_enum_default_libver() {
|
||||
check_h5py_generated_enum("default", "");
|
||||
}
|
||||
|
||||
fn check_h5py_generated_enum(tag: &str, libver_kw: &str) {
|
||||
let path = std::env::temp_dir().join(format!("clawhdf5_h5py_enum_{tag}.h5"));
|
||||
let gen_script = format!(
|
||||
r#"
|
||||
import h5py, numpy as np
|
||||
dt = h5py.enum_dtype({{"RED": 0, "GREEN": 1, "BLUE": 2}}, basetype=np.int32)
|
||||
data = np.array([1, 0, 2, 1], dtype=np.int32)
|
||||
f = h5py.File('{}', 'w', libver='latest')
|
||||
f = h5py.File('{}', 'w'{})
|
||||
f.create_dataset('colors', data=data, dtype=dt)
|
||||
f.close()
|
||||
"#,
|
||||
path.display()
|
||||
path.display(),
|
||||
libver_kw
|
||||
);
|
||||
h5py_read(&path, &gen_script);
|
||||
|
||||
|
||||
Reference in New Issue
Block a user