fix(format): measure compound members by their stored size

The compound overlap check measured each earlier member with
Datatype::type_size, which is a fixed 16 for a variable-length type. On
disk a VL member takes 4 + offset size + 4 bytes, 12 in a file with
4-byte offsets, so a member right after one was refused as "member
overlaps with previous member" (and with the type, every attribute of
the object). libhdf5 measures members by their decoded, stored size
(times a v1 member's array dimensions); so does this now.

Reading VL values in such files is a separate, older gap, now recorded
in known-issues.

Co-Authored-By: Claude Opus 5.5 (1M context) <[email protected]>
This commit is contained in:
osobh
2026-09-26 01:24:17 -05:00
co-authored by Claude Opus 5.5
parent 17f09375ad
commit 9238605661
4 changed files with 78 additions and 10 deletions
@@ -313,3 +313,54 @@ with h5py.File(os.path.join(d, "big.h5"), "r") as f:
let values = file.dataset("d").unwrap().read_f64().unwrap();
assert_eq!(values, (0..10).map(f64::from).collect::<Vec<_>>());
}
/// A variable-length compound member takes 4 + offset size + 4 bytes, which
/// is 12 in a file with 4-byte offsets, and libhdf5 checks for overlapping
/// members with that stored size. A member right after one was refused as
/// "member overlaps with previous member" (the check took the 16 bytes of
/// an 8-byte-offset file), and with it every attribute of the object.
#[test]
fn variable_length_compound_members_in_files_with_4_byte_offsets() {
skip_if_no_python!();
let dir = tempfile::tempdir().unwrap();
run_python(
dir.path(),
r#"
from h5py import h5f, h5p
dt = np.dtype([("s", h5py.string_dtype()), ("i", "<i4")])
a = np.array([("x", 1), ("yy", 2)], dtype=dt)
for libver in ("earliest", "latest"):
fcpl = h5p.create(h5p.FILE_CREATE)
fcpl.set_sizes(4, 4)
fapl = h5p.create(h5p.FILE_ACCESS)
low = h5f.LIBVER_EARLIEST if libver == "earliest" else h5f.LIBVER_LATEST
fapl.set_libver_bounds(low, h5f.LIBVER_LATEST)
path = os.path.join(d, f"{libver}.h5")
with h5py.File(h5f.create(path.encode(), h5f.ACC_TRUNC, fcpl=fcpl, fapl=fapl)) as f:
f.create_dataset("c", data=a)
f.attrs["c"] = a
f.attrs["note"] = "kept"
with h5py.File(path, "r") as f:
assert f["c"][1]["i"] == 2
assert f.attrs["note"] == "kept"
"#,
);
for libver in ["earliest", "latest"] {
let file = File::open(dir.path().join(format!("{libver}.h5"))).unwrap();
let dtype = file.dataset("c").unwrap().dtype();
assert!(
matches!(&dtype, Ok(clawhdf5::DType::Compound(fields))
if fields.iter().map(|f| f.0.as_str()).eq(["s", "i"])),
"{libver}: {dtype:?}"
);
// (Variable-length values in files with 4-byte offsets are not
// decoded yet; see docs/known-issues.md. The attributes must at
// least be listed without errors.)
let (attrs, errors) = file.root().attrs_with_errors().unwrap();
assert!(errors.is_empty(), "{libver}: {errors:?}");
assert!(
attrs.contains_key("c") && attrs.contains_key("note"),
"{libver}: {attrs:?}"
);
}
}