fix(format): measure compound members by their stored size
The compound overlap check measured each earlier member with Datatype::type_size, which is a fixed 16 for a variable-length type. On disk a VL member takes 4 + offset size + 4 bytes, 12 in a file with 4-byte offsets, so a member right after one was refused as "member overlaps with previous member" (and with the type, every attribute of the object). libhdf5 measures members by their decoded, stored size (times a v1 member's array dimensions); so does this now. Reading VL values in such files is a separate, older gap, now recorded in known-issues. Co-Authored-By: Claude Opus 5.5 (1M context) <[email protected]>
This commit is contained in:
@@ -313,3 +313,54 @@ with h5py.File(os.path.join(d, "big.h5"), "r") as f:
|
||||
let values = file.dataset("d").unwrap().read_f64().unwrap();
|
||||
assert_eq!(values, (0..10).map(f64::from).collect::<Vec<_>>());
|
||||
}
|
||||
|
||||
/// A variable-length compound member takes 4 + offset size + 4 bytes, which
|
||||
/// is 12 in a file with 4-byte offsets, and libhdf5 checks for overlapping
|
||||
/// members with that stored size. A member right after one was refused as
|
||||
/// "member overlaps with previous member" (the check took the 16 bytes of
|
||||
/// an 8-byte-offset file), and with it every attribute of the object.
|
||||
#[test]
|
||||
fn variable_length_compound_members_in_files_with_4_byte_offsets() {
|
||||
skip_if_no_python!();
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
run_python(
|
||||
dir.path(),
|
||||
r#"
|
||||
from h5py import h5f, h5p
|
||||
dt = np.dtype([("s", h5py.string_dtype()), ("i", "<i4")])
|
||||
a = np.array([("x", 1), ("yy", 2)], dtype=dt)
|
||||
for libver in ("earliest", "latest"):
|
||||
fcpl = h5p.create(h5p.FILE_CREATE)
|
||||
fcpl.set_sizes(4, 4)
|
||||
fapl = h5p.create(h5p.FILE_ACCESS)
|
||||
low = h5f.LIBVER_EARLIEST if libver == "earliest" else h5f.LIBVER_LATEST
|
||||
fapl.set_libver_bounds(low, h5f.LIBVER_LATEST)
|
||||
path = os.path.join(d, f"{libver}.h5")
|
||||
with h5py.File(h5f.create(path.encode(), h5f.ACC_TRUNC, fcpl=fcpl, fapl=fapl)) as f:
|
||||
f.create_dataset("c", data=a)
|
||||
f.attrs["c"] = a
|
||||
f.attrs["note"] = "kept"
|
||||
with h5py.File(path, "r") as f:
|
||||
assert f["c"][1]["i"] == 2
|
||||
assert f.attrs["note"] == "kept"
|
||||
"#,
|
||||
);
|
||||
for libver in ["earliest", "latest"] {
|
||||
let file = File::open(dir.path().join(format!("{libver}.h5"))).unwrap();
|
||||
let dtype = file.dataset("c").unwrap().dtype();
|
||||
assert!(
|
||||
matches!(&dtype, Ok(clawhdf5::DType::Compound(fields))
|
||||
if fields.iter().map(|f| f.0.as_str()).eq(["s", "i"])),
|
||||
"{libver}: {dtype:?}"
|
||||
);
|
||||
// (Variable-length values in files with 4-byte offsets are not
|
||||
// decoded yet; see docs/known-issues.md. The attributes must at
|
||||
// least be listed without errors.)
|
||||
let (attrs, errors) = file.root().attrs_with_errors().unwrap();
|
||||
assert!(errors.is_empty(), "{libver}: {errors:?}");
|
||||
assert!(
|
||||
attrs.contains_key("c") && attrs.contains_key("note"),
|
||||
"{libver}: {attrs:?}"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user