diff --git a/crates/clawhdf5-format/fuzz/corpus/fuzz_datatype/seed_complex_f64_hdf5_2_0 b/crates/clawhdf5-format/fuzz/corpus/fuzz_datatype/seed_complex_f64_hdf5_2_0 new file mode 100644 index 0000000..5a2bc92 Binary files /dev/null and b/crates/clawhdf5-format/fuzz/corpus/fuzz_datatype/seed_complex_f64_hdf5_2_0 differ diff --git a/crates/clawhdf5-format/fuzz/corpus/fuzz_datatype/seed_compound_v1_default_libver b/crates/clawhdf5-format/fuzz/corpus/fuzz_datatype/seed_compound_v1_default_libver new file mode 100644 index 0000000..6595d3a Binary files /dev/null and b/crates/clawhdf5-format/fuzz/corpus/fuzz_datatype/seed_compound_v1_default_libver differ diff --git a/crates/clawhdf5-format/src/datatype.rs b/crates/clawhdf5-format/src/datatype.rs index 40b186e..19242c0 100644 --- a/crates/clawhdf5-format/src/datatype.rs +++ b/crates/clawhdf5-format/src/datatype.rs @@ -372,7 +372,8 @@ impl Datatype { pos += name_len; let byte_offset = read_uint(data, pos, ob)?; pos += ob; - let (member_dt, consumed) = Self::parse_with_depth(&data[pos..], depth + 1)?; + let (member_dt, consumed) = + Self::parse_with_depth(&data[pos..], depth + 1)?; pos += consumed; members.push(CompoundMember { name, @@ -381,24 +382,29 @@ impl Datatype { }); } } else if version == 1 || version == 2 { - // v1/v2: name, offset(4), dimensionality(1), reserved(3), dim_perm(4), - // reserved_dims(up to 4*4=16), member datatype + // v1/v2: name (null-terminated, padded to a multiple of 8 + // bytes), offset(4), member datatype. v1 additionally + // carries the legacy per-member array fields between the + // offset and the member datatype: dimensionality(1), + // reserved(3), dim_perm(4), reserved(4), 4 dim sizes(16). + // v1 is what default (non-`latest`) libver bounds emit. for _ in 0..num_members { let (name, name_len) = read_null_terminated_string(data, pos)?; - pos += name_len; - // v1: names padded to 8-byte boundary - if version == 1 { - let total_name_bytes = name_len; - let padded = (total_name_bytes + 7) & !7; - pos = pos - name_len + padded; - } + let padded = name_len.checked_add(7).ok_or(FormatError::UnexpectedEof { + expected: usize::MAX, + available: data.len(), + })? & !7; + ensure_len(data, pos, padded)?; + pos += padded; ensure_len(data, pos, 4)?; let byte_offset = LittleEndian::read_u32(&data[pos..pos + 4]) as u64; pos += 4; - // dimensionality(1) + reserved(3) + dim_perm(4) + 4 dim slots(16) = 24 - ensure_len(data, pos, 24)?; - pos += 24; - let (member_dt, consumed) = Self::parse_with_depth(&data[pos..], depth + 1)?; + if version == 1 { + ensure_len(data, pos, 28)?; + pos += 28; + } + let (member_dt, consumed) = + Self::parse_with_depth(&data[pos..], depth + 1)?; pos += consumed; members.push(CompoundMember { name, @@ -1138,6 +1144,82 @@ mod tests { } } + /// Real datatype message bytes emitted by h5py 3.16 / HDF5 2.0 with + /// *default* libver bounds for [('x','f8'),('y','f8'),('id','i4')]: + /// compound datatype version 1 (padded names + 28 bytes of legacy + /// per-member array fields). + fn compound_v1_bytes() -> Vec { + let f64le: [u8; 20] = [ + 0x11, 0x20, 0x3f, 0x00, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x40, 0x00, 0x34, 0x0b, + 0x00, 0x34, 0xff, 0x03, 0x00, 0x00, + ]; + let i32le: [u8; 12] = [ + 0x10, 0x08, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, 0x00, 0x00, 0x20, 0x00, + ]; + let mut b = vec![0x16, 0x03, 0x00, 0x00, 0x14, 0x00, 0x00, 0x00]; + for (name, offset, dt) in [ + (&b"x"[..], 0u32, &f64le[..]), + (&b"y"[..], 8, &f64le[..]), + (&b"id"[..], 16, &i32le[..]), + ] { + let mut padded = name.to_vec(); + padded.resize((name.len() + 1 + 7) & !7, 0); + b.extend_from_slice(&padded); + b.extend_from_slice(&offset.to_le_bytes()); + b.extend_from_slice(&[0u8; 28]); + b.extend_from_slice(dt); + } + b + } + + fn assert_xyid_compound(dt: Datatype) { + match dt { + Datatype::Compound { size, members } => { + assert_eq!(size, 20); + let got: Vec<(&str, u64, u32)> = members + .iter() + .map(|m| (m.name.as_str(), m.byte_offset, m.datatype.type_size())) + .collect(); + assert_eq!(got, vec![("x", 0, 8), ("y", 8, 8), ("id", 16, 4)]); + } + other => panic!("expected Compound, got {other:?}"), + } + } + + #[test] + fn test_compound_v1_default_libver() { + let bytes = compound_v1_bytes(); + let (dt, consumed) = Datatype::parse(&bytes).unwrap(); + assert_eq!(consumed, bytes.len()); + assert_xyid_compound(dt); + } + + #[test] + fn test_compound_v2_padded_names_no_array_fields() { + // v2 = v1 without the 28 bytes of per-member array fields; names are + // still padded to a multiple of 8 (matches libhdf5's H5O decoder). + let v1 = compound_v1_bytes(); + let mut v2 = vec![0x26, 0x03, 0x00, 0x00, 0x14, 0x00, 0x00, 0x00]; + let mut pos = 8; + for dt_len in [20usize, 20, 12] { + v2.extend_from_slice(&v1[pos..pos + 8 + 4]); // padded name + offset + pos += 8 + 4 + 28; + v2.extend_from_slice(&v1[pos..pos + dt_len]); + pos += dt_len; + } + let (dt, consumed) = Datatype::parse(&v2).unwrap(); + assert_eq!(consumed, v2.len()); + assert_xyid_compound(dt); + } + + #[test] + fn test_compound_v1_truncated_is_error_not_panic() { + let bytes = compound_v1_bytes(); + for cut in 8..bytes.len() { + assert!(Datatype::parse(&bytes[..cut]).is_err(), "cut at {cut}"); + } + } + /// Real datatype message bytes emitted by HDF5 2.0 for the native complex /// type `H5T_COMPLEX_IEEE_F64LE`: class 11, version 5, size 16, followed by /// the base IEEE f64 datatype message. @@ -1172,7 +1254,9 @@ mod tests { // Compound { z: complex f64 @0, k: i64 @16 } as written by HDF5 2.0. // Regression guard: the complex member must consume exactly its own // bytes so the following member parses. - let mut bytes = vec![0x56, 0x02, 0x00, 0x00, 0x18, 0x00, 0x00, 0x00, b'z', 0x00, 0x00]; + let mut bytes = vec![ + 0x56, 0x02, 0x00, 0x00, 0x18, 0x00, 0x00, 0x00, b'z', 0x00, 0x00, + ]; bytes.extend_from_slice(&COMPLEX_F64_HDF5_2_0); bytes.extend_from_slice(&[b'k', 0x00, 0x10]); bytes.extend_from_slice(&[ @@ -1188,7 +1272,10 @@ mod tests { &members[0].datatype, Datatype::Compound { size: 16, members } if members.len() == 2 )); - assert_eq!((members[1].name.as_str(), members[1].byte_offset), ("k", 16)); + assert_eq!( + (members[1].name.as_str(), members[1].byte_offset), + ("k", 16) + ); } other => panic!("expected Compound, got {other:?}"), } diff --git a/crates/clawhdf5-format/tests/writer_h5py_tests.rs b/crates/clawhdf5-format/tests/writer_h5py_tests.rs index 7506717..58d1660 100644 --- a/crates/clawhdf5-format/tests/writer_h5py_tests.rs +++ b/crates/clawhdf5-format/tests/writer_h5py_tests.rs @@ -235,17 +235,31 @@ fn h5py_reads_our_array_dataset() { #[test] #[ignore = "requires Python h5py module"] fn read_h5py_generated_compound() { - let path = std::env::temp_dir().join("clawhdf5_h5py_compound.h5"); + check_h5py_generated_compound("latest", ", libver='latest'"); +} + +/// Same file written with h5py's default format bounds. HDF5 2.0 raised the +/// default low bound to 1.8, so "default" files exercise different on-disk +/// structures than both `libver='latest'` and pre-2.0 defaults. +#[test] +#[ignore = "requires Python h5py module"] +fn read_h5py_generated_compound_default_libver() { + check_h5py_generated_compound("default", ""); +} + +fn check_h5py_generated_compound(tag: &str, libver_kw: &str) { + let path = std::env::temp_dir().join(format!("clawhdf5_h5py_compound_{tag}.h5")); let gen_script = format!( r#" import h5py, numpy as np dt = np.dtype([('x', 'f8'), ('y', 'f8'), ('id', 'i4')]) data = np.array([(1.0, 2.0, 10), (3.0, 4.0, 20)], dtype=dt) -f = h5py.File('{}', 'w', libver='latest') +f = h5py.File('{}', 'w'{}) f.create_dataset('particles', data=data) f.close() "#, - path.display() + path.display(), + libver_kw ); h5py_read(&path, &gen_script); @@ -363,17 +377,31 @@ else: #[test] #[ignore = "requires Python h5py module"] fn read_h5py_generated_enum() { - let path = std::env::temp_dir().join("clawhdf5_h5py_enum.h5"); + check_h5py_generated_enum("latest", ", libver='latest'"); +} + +/// Same file written with h5py's default format bounds. HDF5 2.0 raised the +/// default low bound to 1.8, so "default" files exercise different on-disk +/// structures than both `libver='latest'` and pre-2.0 defaults. +#[test] +#[ignore = "requires Python h5py module"] +fn read_h5py_generated_enum_default_libver() { + check_h5py_generated_enum("default", ""); +} + +fn check_h5py_generated_enum(tag: &str, libver_kw: &str) { + let path = std::env::temp_dir().join(format!("clawhdf5_h5py_enum_{tag}.h5")); let gen_script = format!( r#" import h5py, numpy as np dt = h5py.enum_dtype({{"RED": 0, "GREEN": 1, "BLUE": 2}}, basetype=np.int32) data = np.array([1, 0, 2, 1], dtype=np.int32) -f = h5py.File('{}', 'w', libver='latest') +f = h5py.File('{}', 'w'{}) f.create_dataset('colors', data=data, dtype=dt) f.close() "#, - path.display() + path.display(), + libver_kw ); h5py_read(&path, &gen_script);