fix(py): index lists of padded compounds no longer return uninitialised padding

np.concatenate copies structured dtypes field by field into np.empty, so
the padding of ds[[0, 3, 6]] held process memory. The runs' bytes are
joined in Rust, whole elements at a time, before anything becomes numpy:
the padding is the file's bytes (h5py's) and the result is still a view
of the Rust buffer. The h5py comparisons now compare every byte of
structured values; the new test failed on the padding before.

Co-Authored-By: Claude Opus 5.5 (1M context) <[email protected]>
This commit is contained in:
osobh
2026-09-26 08:54:04 -05:00
co-authored by Claude Opus 5.5
parent 24412a0e59
commit 8c51b05b9c
4 changed files with 116 additions and 52 deletions
+22 -2
View File
@@ -212,14 +212,19 @@ def assert_same(ours, theirs, what=""):
if theirs.dtype == object:
for a, b in zip(ours.ravel(), theirs.ravel()):
assert_same(a, b, what)
elif theirs.dtype.names is None and theirs.dtype.kind == "V":
assert ours.tobytes() == theirs.tobytes(), what
elif theirs.dtype.kind == "V":
# Structured and opaque: every byte, padding included (h5py's
# padding is zero; uninitialised memory there would leak).
if theirs.dtype.names is not None:
np.testing.assert_array_equal(ours, theirs, err_msg=what)
assert ours.tobytes() == theirs.tobytes(), f"{what}: bytes differ"
else:
np.testing.assert_array_equal(ours, theirs, err_msg=what)
elif isinstance(theirs, np.generic):
assert ours.dtype == theirs.dtype, what
if theirs.dtype.names is not None:
np.testing.assert_array_equal(np.asarray(ours), np.asarray(theirs), err_msg=what)
assert ours.tobytes() == theirs.tobytes(), f"{what}: bytes differ"
else:
assert ours == theirs or (ours != ours and theirs != theirs), what
else:
@@ -511,3 +516,18 @@ def test_a_library_panic_is_an_ordinary_exception():
clawhdf5._panic_for_test()
except Exception: # noqa: BLE001 - the point of the test
pass
def test_compound_padding_bytes_match_h5py(pair):
"""Every byte of a padded compound, padding included, is h5py's, for
index lists with many runs as well as slices. Joining the runs with
np.concatenate left the padding uninitialised: process memory ended up
in tobytes()."""
ours, theirs, _ = pair
keys = [[0, 3, 6], [1, 2, 5, 9], [0, 2, 4, 6, 8], slice(None), slice(1, 9, 3), 4, [9]]
for name in ["cmp/padded", "cmp/padded_chunked"]:
for _ in range(20): # garbage varies between runs; zeros do not
for key in keys:
assert ours[name][key].tobytes() == theirs[name][key].tobytes(), f"{name}[{key!r}]"
for key in [(slice(None), [0, 2]), ([0, 2, 3], slice(None)), ([1, 3], 1)]:
assert ours["cmp/nested_2d"][key].tobytes() == theirs["cmp/nested_2d"][key].tobytes(), key