fix(py): index lists of padded compounds no longer return uninitialised padding
np.concatenate copies structured dtypes field by field into np.empty, so the padding of ds[[0, 3, 6]] held process memory. The runs' bytes are joined in Rust, whole elements at a time, before anything becomes numpy: the padding is the file's bytes (h5py's) and the result is still a view of the Rust buffer. The h5py comparisons now compare every byte of structured values; the new test failed on the padding before. Co-Authored-By: Claude Opus 5.5 (1M context) <[email protected]>
This commit is contained in:
@@ -212,14 +212,19 @@ def assert_same(ours, theirs, what=""):
|
||||
if theirs.dtype == object:
|
||||
for a, b in zip(ours.ravel(), theirs.ravel()):
|
||||
assert_same(a, b, what)
|
||||
elif theirs.dtype.names is None and theirs.dtype.kind == "V":
|
||||
assert ours.tobytes() == theirs.tobytes(), what
|
||||
elif theirs.dtype.kind == "V":
|
||||
# Structured and opaque: every byte, padding included (h5py's
|
||||
# padding is zero; uninitialised memory there would leak).
|
||||
if theirs.dtype.names is not None:
|
||||
np.testing.assert_array_equal(ours, theirs, err_msg=what)
|
||||
assert ours.tobytes() == theirs.tobytes(), f"{what}: bytes differ"
|
||||
else:
|
||||
np.testing.assert_array_equal(ours, theirs, err_msg=what)
|
||||
elif isinstance(theirs, np.generic):
|
||||
assert ours.dtype == theirs.dtype, what
|
||||
if theirs.dtype.names is not None:
|
||||
np.testing.assert_array_equal(np.asarray(ours), np.asarray(theirs), err_msg=what)
|
||||
assert ours.tobytes() == theirs.tobytes(), f"{what}: bytes differ"
|
||||
else:
|
||||
assert ours == theirs or (ours != ours and theirs != theirs), what
|
||||
else:
|
||||
@@ -511,3 +516,18 @@ def test_a_library_panic_is_an_ordinary_exception():
|
||||
clawhdf5._panic_for_test()
|
||||
except Exception: # noqa: BLE001 - the point of the test
|
||||
pass
|
||||
|
||||
|
||||
def test_compound_padding_bytes_match_h5py(pair):
|
||||
"""Every byte of a padded compound, padding included, is h5py's, for
|
||||
index lists with many runs as well as slices. Joining the runs with
|
||||
np.concatenate left the padding uninitialised: process memory ended up
|
||||
in tobytes()."""
|
||||
ours, theirs, _ = pair
|
||||
keys = [[0, 3, 6], [1, 2, 5, 9], [0, 2, 4, 6, 8], slice(None), slice(1, 9, 3), 4, [9]]
|
||||
for name in ["cmp/padded", "cmp/padded_chunked"]:
|
||||
for _ in range(20): # garbage varies between runs; zeros do not
|
||||
for key in keys:
|
||||
assert ours[name][key].tobytes() == theirs[name][key].tobytes(), f"{name}[{key!r}]"
|
||||
for key in [(slice(None), [0, 2]), ([0, 2, 3], slice(None)), ([1, 3], 1)]:
|
||||
assert ours["cmp/nested_2d"][key].tobytes() == theirs["cmp/nested_2d"][key].tobytes(), key
|
||||
|
||||
Reference in New Issue
Block a user