From 926dc457e09ee375317e5e38445a4f4c661e798c Mon Sep 17 00:00:00 2001 From: osobh Date: Sat, 19 Sep 2026 05:36:22 -0700 Subject: [PATCH] fix(format): parse compound datatype versions 1 and 2 correctly Compound datasets written with default libver bounds (datatype message version 1, i.e. plain h5py.File(path, 'w')) could not be read: the v1 member layout has 28 bytes of legacy array fields after the byte offset (dimensionality 1, reserved 3, permutation 4, reserved 4, four sizes 16) and the parser skipped 24, so every following member was read 4 bytes off. v2 was also wrong: it keeps the 8-byte name padding and has no array fields. Found by adding a default-libver axis to the h5py-generated-file tests (HDF5 2.0 raised the default low bound to 1.8, so "default" files are a distinct format path from libver='latest'). Adds byte-level v1/v2 regression tests, a truncation test, and fuzz corpus seeds for v1 compound and native complex. Co-Authored-By: Claude Fable 5.1 --- .../fuzz_datatype/seed_complex_f64_hdf5_2_0 | Bin 0 -> 28 bytes .../seed_compound_v1_default_libver | Bin 0 -> 180 bytes crates/clawhdf5-format/src/datatype.rs | 119 +++++++++++++++--- .../tests/writer_h5py_tests.rs | 40 +++++- 4 files changed, 137 insertions(+), 22 deletions(-) create mode 100644 crates/clawhdf5-format/fuzz/corpus/fuzz_datatype/seed_complex_f64_hdf5_2_0 create mode 100644 crates/clawhdf5-format/fuzz/corpus/fuzz_datatype/seed_compound_v1_default_libver diff --git a/crates/clawhdf5-format/fuzz/corpus/fuzz_datatype/seed_complex_f64_hdf5_2_0 b/crates/clawhdf5-format/fuzz/corpus/fuzz_datatype/seed_complex_f64_hdf5_2_0 new file mode 100644 index 0000000000000000000000000000000000000000..5a2bc929ce2a1edae29ad7c4039a3486037b3b44 GIT binary patch literal 28 hcma!MWMB|rU| Vec { + let f64le: [u8; 20] = [ + 0x11, 0x20, 0x3f, 0x00, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x40, 0x00, 0x34, 0x0b, + 0x00, 0x34, 0xff, 0x03, 0x00, 0x00, + ]; + let i32le: [u8; 12] = [ + 0x10, 0x08, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, 0x00, 0x00, 0x20, 0x00, + ]; + let mut b = vec![0x16, 0x03, 0x00, 0x00, 0x14, 0x00, 0x00, 0x00]; + for (name, offset, dt) in [ + (&b"x"[..], 0u32, &f64le[..]), + (&b"y"[..], 8, &f64le[..]), + (&b"id"[..], 16, &i32le[..]), + ] { + let mut padded = name.to_vec(); + padded.resize((name.len() + 1 + 7) & !7, 0); + b.extend_from_slice(&padded); + b.extend_from_slice(&offset.to_le_bytes()); + b.extend_from_slice(&[0u8; 28]); + b.extend_from_slice(dt); + } + b + } + + fn assert_xyid_compound(dt: Datatype) { + match dt { + Datatype::Compound { size, members } => { + assert_eq!(size, 20); + let got: Vec<(&str, u64, u32)> = members + .iter() + .map(|m| (m.name.as_str(), m.byte_offset, m.datatype.type_size())) + .collect(); + assert_eq!(got, vec![("x", 0, 8), ("y", 8, 8), ("id", 16, 4)]); + } + other => panic!("expected Compound, got {other:?}"), + } + } + + #[test] + fn test_compound_v1_default_libver() { + let bytes = compound_v1_bytes(); + let (dt, consumed) = Datatype::parse(&bytes).unwrap(); + assert_eq!(consumed, bytes.len()); + assert_xyid_compound(dt); + } + + #[test] + fn test_compound_v2_padded_names_no_array_fields() { + // v2 = v1 without the 28 bytes of per-member array fields; names are + // still padded to a multiple of 8 (matches libhdf5's H5O decoder). + let v1 = compound_v1_bytes(); + let mut v2 = vec![0x26, 0x03, 0x00, 0x00, 0x14, 0x00, 0x00, 0x00]; + let mut pos = 8; + for dt_len in [20usize, 20, 12] { + v2.extend_from_slice(&v1[pos..pos + 8 + 4]); // padded name + offset + pos += 8 + 4 + 28; + v2.extend_from_slice(&v1[pos..pos + dt_len]); + pos += dt_len; + } + let (dt, consumed) = Datatype::parse(&v2).unwrap(); + assert_eq!(consumed, v2.len()); + assert_xyid_compound(dt); + } + + #[test] + fn test_compound_v1_truncated_is_error_not_panic() { + let bytes = compound_v1_bytes(); + for cut in 8..bytes.len() { + assert!(Datatype::parse(&bytes[..cut]).is_err(), "cut at {cut}"); + } + } + /// Real datatype message bytes emitted by HDF5 2.0 for the native complex /// type `H5T_COMPLEX_IEEE_F64LE`: class 11, version 5, size 16, followed by /// the base IEEE f64 datatype message. @@ -1172,7 +1254,9 @@ mod tests { // Compound { z: complex f64 @0, k: i64 @16 } as written by HDF5 2.0. // Regression guard: the complex member must consume exactly its own // bytes so the following member parses. - let mut bytes = vec![0x56, 0x02, 0x00, 0x00, 0x18, 0x00, 0x00, 0x00, b'z', 0x00, 0x00]; + let mut bytes = vec![ + 0x56, 0x02, 0x00, 0x00, 0x18, 0x00, 0x00, 0x00, b'z', 0x00, 0x00, + ]; bytes.extend_from_slice(&COMPLEX_F64_HDF5_2_0); bytes.extend_from_slice(&[b'k', 0x00, 0x10]); bytes.extend_from_slice(&[ @@ -1188,7 +1272,10 @@ mod tests { &members[0].datatype, Datatype::Compound { size: 16, members } if members.len() == 2 )); - assert_eq!((members[1].name.as_str(), members[1].byte_offset), ("k", 16)); + assert_eq!( + (members[1].name.as_str(), members[1].byte_offset), + ("k", 16) + ); } other => panic!("expected Compound, got {other:?}"), } diff --git a/crates/clawhdf5-format/tests/writer_h5py_tests.rs b/crates/clawhdf5-format/tests/writer_h5py_tests.rs index 7506717..58d1660 100644 --- a/crates/clawhdf5-format/tests/writer_h5py_tests.rs +++ b/crates/clawhdf5-format/tests/writer_h5py_tests.rs @@ -235,17 +235,31 @@ fn h5py_reads_our_array_dataset() { #[test] #[ignore = "requires Python h5py module"] fn read_h5py_generated_compound() { - let path = std::env::temp_dir().join("clawhdf5_h5py_compound.h5"); + check_h5py_generated_compound("latest", ", libver='latest'"); +} + +/// Same file written with h5py's default format bounds. HDF5 2.0 raised the +/// default low bound to 1.8, so "default" files exercise different on-disk +/// structures than both `libver='latest'` and pre-2.0 defaults. +#[test] +#[ignore = "requires Python h5py module"] +fn read_h5py_generated_compound_default_libver() { + check_h5py_generated_compound("default", ""); +} + +fn check_h5py_generated_compound(tag: &str, libver_kw: &str) { + let path = std::env::temp_dir().join(format!("clawhdf5_h5py_compound_{tag}.h5")); let gen_script = format!( r#" import h5py, numpy as np dt = np.dtype([('x', 'f8'), ('y', 'f8'), ('id', 'i4')]) data = np.array([(1.0, 2.0, 10), (3.0, 4.0, 20)], dtype=dt) -f = h5py.File('{}', 'w', libver='latest') +f = h5py.File('{}', 'w'{}) f.create_dataset('particles', data=data) f.close() "#, - path.display() + path.display(), + libver_kw ); h5py_read(&path, &gen_script); @@ -363,17 +377,31 @@ else: #[test] #[ignore = "requires Python h5py module"] fn read_h5py_generated_enum() { - let path = std::env::temp_dir().join("clawhdf5_h5py_enum.h5"); + check_h5py_generated_enum("latest", ", libver='latest'"); +} + +/// Same file written with h5py's default format bounds. HDF5 2.0 raised the +/// default low bound to 1.8, so "default" files exercise different on-disk +/// structures than both `libver='latest'` and pre-2.0 defaults. +#[test] +#[ignore = "requires Python h5py module"] +fn read_h5py_generated_enum_default_libver() { + check_h5py_generated_enum("default", ""); +} + +fn check_h5py_generated_enum(tag: &str, libver_kw: &str) { + let path = std::env::temp_dir().join(format!("clawhdf5_h5py_enum_{tag}.h5")); let gen_script = format!( r#" import h5py, numpy as np dt = h5py.enum_dtype({{"RED": 0, "GREEN": 1, "BLUE": 2}}, basetype=np.int32) data = np.array([1, 0, 2, 1], dtype=np.int32) -f = h5py.File('{}', 'w', libver='latest') +f = h5py.File('{}', 'w'{}) f.create_dataset('colors', data=data, dtype=dt) f.close() "#, - path.display() + path.display(), + libver_kw ); h5py_read(&path, &gen_script);