osobhandClaude Opus 5.5
17edfe2cf0
test_threads_read_the_same_file passed with the GIL held. The new test_reads_release_the_gil measures the longest stall of a spinning Python thread while another reads: with py.detach removed from the read it stalled 0.062 s of a 0.064 s read and failed; with it, about 3 ms. test_errors_match_h5py now compares the result whenever h5py reads the key, instead of only checking that we raise when h5py raises, over a longer key list. Co-Authored-By: Claude Opus 5.5 (1M context) <[email protected]>
654 lines
27 KiB
Python
654 lines
27 KiB
Python
"""Every read through clawhdf5 compared against h5py (libhdf5) on files h5py
|
|
writes: dtypes, shapes, values and the type of what comes back (array,
|
|
numpy scalar, bytes, str, Empty), for every datatype the bindings map and a
|
|
spread of index expressions; plus the errors h5py gives for the same keys."""
|
|
|
|
import threading
|
|
from concurrent.futures import ThreadPoolExecutor
|
|
|
|
import numpy as np
|
|
import pytest
|
|
|
|
import clawhdf5
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# The generated file
|
|
# ---------------------------------------------------------------------------
|
|
|
|
NUMERIC = [
|
|
"<i1", "<i2", "<i4", "<i8", "<u1", "<u2", "<u4", "<u8",
|
|
">i2", ">i4", ">i8", ">u2", ">u4", ">u8",
|
|
"<f2", "<f4", "<f8", ">f2", ">f4", ">f8",
|
|
]
|
|
|
|
|
|
def _values(dtype, shape, seed):
|
|
rng = np.random.default_rng(seed)
|
|
dt = np.dtype(dtype)
|
|
n = int(np.prod(shape))
|
|
if dt.kind == "f":
|
|
return rng.standard_normal(n).astype(dt).reshape(shape)
|
|
info = np.iinfo(dt)
|
|
return rng.integers(info.min, info.max, size=n, dtype=dt.newbyteorder("=")).astype(dt).reshape(shape)
|
|
|
|
|
|
def _compound_dtype():
|
|
return np.dtype(
|
|
{
|
|
"names": ["id", "pos", "label", "flag", "vec"],
|
|
"formats": ["<i4", ">f8", "S6", "u1", ("<f4", (3,))],
|
|
"offsets": [0, 8, 16, 22, 24],
|
|
"itemsize": 40,
|
|
}
|
|
)
|
|
|
|
|
|
def _nested_dtype():
|
|
inner = np.dtype([("a", "<i2"), ("b", "<f4")])
|
|
return np.dtype([("x", "<u8"), ("inner", inner), ("c", "<c8")])
|
|
|
|
|
|
def _write_fixture(h5py, path):
|
|
str_dt = h5py.string_dtype()
|
|
ascii_dt = h5py.string_dtype("ascii")
|
|
with h5py.File(path, "w") as f:
|
|
# Numeric types, both byte orders, 1-D contiguous and 2-D chunked+gzip.
|
|
for i, dt in enumerate(NUMERIC):
|
|
name = dt.replace("<", "le_").replace(">", "be_")
|
|
f.create_dataset(f"num/{name}_1d", data=_values(dt, (37,), i))
|
|
f.create_dataset(
|
|
f"num/{name}_2d_gzip",
|
|
data=_values(dt, (13, 11), 100 + i),
|
|
chunks=(4, 5),
|
|
compression="gzip",
|
|
)
|
|
f.create_dataset("num/f8_3d", data=_values("<f8", (6, 7, 5), 7), chunks=(2, 3, 5))
|
|
f.create_dataset(
|
|
"num/i4_3d_shuffle",
|
|
data=_values("<i4", (5, 9, 4), 8),
|
|
chunks=(3, 4, 2),
|
|
shuffle=True,
|
|
fletcher32=True,
|
|
compression="gzip",
|
|
)
|
|
f.create_dataset("num/scalar_f8", data=np.float64(3.25))
|
|
f.create_dataset("num/scalar_i2_be", data=np.array(-7, dtype=">i2"))
|
|
f.create_dataset("num/zero_size", shape=(0, 3), dtype="<f4")
|
|
f.create_dataset("num/empty", data=h5py.Empty("<f8"))
|
|
sparse = f.create_dataset("num/sparse_fill", shape=(40,), chunks=(8,), dtype="<i4", fillvalue=-3)
|
|
sparse[10:14] = [1, 2, 3, 4]
|
|
f.create_dataset("num/resizable", data=np.arange(12.0), maxshape=(None,), chunks=(5,))
|
|
# Compact layout (low level: h5py's create_dataset cannot ask for it).
|
|
dcpl = h5py.h5p.create(h5py.h5p.DATASET_CREATE)
|
|
dcpl.set_layout(h5py.h5d.COMPACT)
|
|
space = h5py.h5s.create_simple((9,))
|
|
dsid = h5py.h5d.create(f.id, b"num/compact_u2", h5py.h5t.py_create(np.dtype("<u2")), space, dcpl=dcpl)
|
|
dsid.write(h5py.h5s.ALL, h5py.h5s.ALL, np.arange(9, dtype="<u2") * 7)
|
|
|
|
# bool, enum, complex.
|
|
f.create_dataset("misc/bool", data=np.array([True, False, True, True, False]))
|
|
enum_dt = h5py.enum_dtype({"RED": 0, "GREEN": 1, "BLUE": 42}, basetype="i2")
|
|
f.create_dataset("misc/enum", data=np.array([0, 42, 1, 1, 0], dtype="<i2"), dtype=enum_dt)
|
|
uenum_dt = h5py.enum_dtype({"LO": 0, "HI": 2**40}, basetype="u8")
|
|
f.create_dataset("misc/enum_u8", data=np.array([0, 2**40], dtype="<u8"), dtype=uenum_dt)
|
|
f.create_dataset("misc/c8", data=(np.arange(6) + 1j * np.arange(6)).astype("<c8"))
|
|
f.create_dataset("misc/c16_2d", data=(np.arange(12) - 2j).reshape(3, 4).astype("<c16"))
|
|
f.create_dataset("misc/opaque", data=np.array([b"\x00\x01\x02", b"\xff\xfe\xfd"], dtype="V3"))
|
|
|
|
# Strings.
|
|
f.create_dataset("str/fixed", data=np.array([b"alpha", b"be", b"", b"gamma!"], dtype="S6"))
|
|
utf8_4 = h5py.string_dtype("utf-8", 4)
|
|
f.create_dataset(
|
|
"str/fixed_2d_utf8",
|
|
data=np.array([["é".encode(), b"b"], [b"c", b"dd"]], dtype=utf8_4),
|
|
)
|
|
f.create_dataset("str/vlen", data=["", "one", "twø", "a" * 300, "ünïcödé"], dtype=str_dt)
|
|
f.create_dataset("str/vlen_ascii", data=[b"x", b"yy", b"zzz"], dtype=ascii_dt)
|
|
f.create_dataset(
|
|
"str/vlen_2d_gzip",
|
|
data=np.array([[f"r{r}c{c}" * (r + c) for c in range(5)] for r in range(6)], dtype=object),
|
|
dtype=str_dt,
|
|
chunks=(2, 2),
|
|
compression="gzip",
|
|
)
|
|
f.create_dataset("str/vlen_scalar", data="just one", dtype=str_dt)
|
|
|
|
# Variable-length sequences.
|
|
vl_i = h5py.vlen_dtype(np.dtype("<i4"))
|
|
seqs = np.empty(4, dtype=object)
|
|
seqs[:] = [np.arange(3, dtype="<i4"), np.array([], dtype="<i4"), np.arange(10, dtype="<i4") * -1, np.array([7], dtype="<i4")]
|
|
f.create_dataset("vlen/i4", data=seqs, dtype=vl_i)
|
|
vl_f = h5py.vlen_dtype(np.dtype(">f8"))
|
|
fseqs = np.empty(3, dtype=object)
|
|
fseqs[:] = [np.linspace(0, 1, 5).astype(">f8"), np.array([2.5], dtype=">f8"), np.array([], dtype=">f8")]
|
|
f.create_dataset("vlen/f8_be", data=fseqs, dtype=vl_f)
|
|
|
|
# Compounds.
|
|
cdt = _compound_dtype()
|
|
rec = np.zeros(10, dtype=cdt)
|
|
rec["id"] = np.arange(10) * 3
|
|
rec["pos"] = np.linspace(-1, 1, 10)
|
|
rec["label"] = [f"n{i}".encode() for i in range(10)]
|
|
rec["flag"] = np.arange(10) % 2
|
|
rec["vec"] = np.arange(30, dtype="<f4").reshape(10, 3)
|
|
f.create_dataset("cmp/padded", data=rec)
|
|
f.create_dataset("cmp/padded_chunked", data=rec, chunks=(3,), compression="gzip")
|
|
ndt = _nested_dtype()
|
|
nrec = np.zeros((4, 3), dtype=ndt)
|
|
nrec["x"] = np.arange(12).reshape(4, 3)
|
|
nrec["inner"]["a"] = -np.arange(12).reshape(4, 3)
|
|
nrec["inner"]["b"] = np.arange(12).reshape(4, 3) / 4
|
|
nrec["c"] = np.arange(12).reshape(4, 3) * (1 + 1j)
|
|
f.create_dataset("cmp/nested_2d", data=nrec)
|
|
vdt = np.dtype([("n", "<i4"), ("s", h5py.string_dtype())])
|
|
vrec = np.array([(1, "a"), (2, "bb")], dtype=vdt)
|
|
f.create_dataset("cmp/with_vlen", data=vrec)
|
|
|
|
# A true HDF5 array datatype (h5py's high level would widen the shape).
|
|
tid = h5py.h5t.array_create(h5py.h5t.py_create(np.dtype("<i4")), (2, 3))
|
|
space = h5py.h5s.create_simple((4,))
|
|
dsid = h5py.h5d.create(f.id, b"cmp/array_type", tid, space)
|
|
dsid.write(h5py.h5s.ALL, h5py.h5s.ALL, np.arange(24, dtype="<i4").reshape(4, 2, 3), mtype=tid)
|
|
|
|
# Unsupported: references.
|
|
f.create_dataset("unsupported/refs", data=[f["num"].ref, f["misc"].ref], dtype=h5py.ref_dtype)
|
|
|
|
# Groups and attributes.
|
|
g = f.create_group("deep/er/est")
|
|
g.create_dataset("leaf", data=np.arange(5))
|
|
f.create_group("empty_group")
|
|
f.attrs["i4"] = np.int32(-5)
|
|
f.attrs["u8"] = np.uint64(2**63 + 1)
|
|
f.attrs["f2"] = np.float16(1.5)
|
|
f.attrs["f8_arr"] = np.array([1.0, 2.5, -3.0])
|
|
f.attrs["f8_one"] = np.array([4.0])
|
|
f.attrs["i2_2d_be"] = np.arange(6, dtype=">i2").reshape(2, 3)
|
|
f.attrs["bool"] = True
|
|
f.attrs["bool_arr"] = np.array([True, False])
|
|
f.attrs["vstr"] = "héllo"
|
|
f.attrs["vstr_arr"] = ["a", "bcd", ""]
|
|
f.attrs["fstr"] = np.bytes_(b"fixed")
|
|
f.attrs["fstr_arr"] = np.array([b"x", b"yz"], dtype="S2")
|
|
f.attrs["empty"] = h5py.Empty("<i4")
|
|
f.attrs["complex"] = np.complex128(1 - 2j)
|
|
f.attrs["compound"] = np.array((7, 2.5), dtype=[("a", "<i2"), ("b", "<f8")])
|
|
f.attrs["enum"] = np.array(42, dtype=enum_dt)
|
|
f["num/le_f8_1d"].attrs["units"] = "m/s"
|
|
f["num/le_f8_1d"].attrs["scale"] = np.float32(0.5)
|
|
g.attrs["depth"] = np.int8(3)
|
|
|
|
|
|
@pytest.fixture(scope="module")
|
|
def pair(h5py, tmp_path_factory):
|
|
path = str(tmp_path_factory.mktemp("h5") / "fixture.h5")
|
|
_write_fixture(h5py, path)
|
|
theirs = h5py.File(path, "r")
|
|
ours = clawhdf5.File(path, "r")
|
|
yield ours, theirs, path
|
|
theirs.close()
|
|
ours.close()
|
|
|
|
|
|
def _all_datasets(h5py, f):
|
|
names = []
|
|
f.visititems(lambda n, o: names.append(n) if isinstance(o, h5py.Dataset) else None)
|
|
return sorted(names)
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Comparison helpers
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
def assert_same(ours, theirs, what=""):
|
|
if type(theirs).__name__ == "Empty":
|
|
assert isinstance(ours, clawhdf5.Empty), what
|
|
assert ours.dtype == theirs.dtype, what
|
|
return
|
|
assert type(ours) is type(theirs), f"{what}: {type(ours)} vs {type(theirs)}"
|
|
if isinstance(theirs, np.ndarray):
|
|
assert ours.shape == theirs.shape, what
|
|
assert ours.dtype == theirs.dtype, f"{what}: {ours.dtype} vs {theirs.dtype}"
|
|
if theirs.dtype == object:
|
|
for a, b in zip(ours.ravel(), theirs.ravel()):
|
|
assert_same(a, b, what)
|
|
elif theirs.dtype.kind == "V":
|
|
# Structured and opaque: every byte, padding included (h5py's
|
|
# padding is zero; uninitialised memory there would leak).
|
|
if theirs.dtype.names is not None:
|
|
np.testing.assert_array_equal(ours, theirs, err_msg=what)
|
|
assert ours.tobytes() == theirs.tobytes(), f"{what}: bytes differ"
|
|
else:
|
|
np.testing.assert_array_equal(ours, theirs, err_msg=what)
|
|
elif isinstance(theirs, np.generic):
|
|
assert ours.dtype == theirs.dtype, what
|
|
if theirs.dtype.names is not None:
|
|
np.testing.assert_array_equal(np.asarray(ours), np.asarray(theirs), err_msg=what)
|
|
assert ours.tobytes() == theirs.tobytes(), f"{what}: bytes differ"
|
|
else:
|
|
assert ours == theirs or (ours != ours and theirs != theirs), what
|
|
else:
|
|
assert ours == theirs, what
|
|
|
|
|
|
def keys_for(shape):
|
|
if shape == ():
|
|
return [(), Ellipsis]
|
|
keys = [(), Ellipsis, 0, -1, slice(None), slice(None, None, 2), slice(1, None, 3), slice(0, 0), np.int64(0),
|
|
np.array(0), np.array(-1, dtype="i1")]
|
|
n0 = shape[0]
|
|
if n0 == 0:
|
|
return [(), Ellipsis, slice(None), slice(None, None, 2), slice(0, 0)]
|
|
keys += [slice(n0 // 2, None), slice(-3, None), [0, n0 - 1] if n0 > 1 else [0], (Ellipsis,)]
|
|
if n0 > 5:
|
|
keys += [[1, 2, 3, 5], [0, 4, 5]]
|
|
if len(shape) >= 2:
|
|
n1 = shape[1]
|
|
keys += [
|
|
(0, 0),
|
|
(-1, -1),
|
|
(slice(1, 3), slice(None, None, 2)),
|
|
(Ellipsis, 1),
|
|
(1, Ellipsis),
|
|
(slice(None), [0, n1 - 1] if n1 > 1 else [0]),
|
|
(slice(None, None, 2), 1),
|
|
(slice(0, 2), slice(3, 1)),
|
|
(np.array(1) if n0 > 1 else np.array(0), slice(None)),
|
|
(slice(None), np.array(n1 - 1, dtype="u2")),
|
|
]
|
|
if len(shape) >= 3:
|
|
keys += [(0, slice(None), -1), (slice(1, None, 2), 2, slice(None, None, 3)), (Ellipsis, 0, 0), (0, Ellipsis, 1)]
|
|
return keys
|
|
|
|
|
|
# h5py 3.16 (HDF5 2.0) returns the elements of a variable-length sequence of
|
|
# big-endian floats unswapped (0.25 comes back as 2.6e-319), so these are
|
|
# checked against the values written instead (test_vlen_big_endian).
|
|
H5PY_MISREADS = {"vlen/f8_be"}
|
|
|
|
ERROR_KEYS_1D = [
|
|
slice(None, None, -1), 10**6, -(10**6), None, (0, 0, 0, 0, 0), [3, 1], (Ellipsis, Ellipsis), 1.5, "nope",
|
|
np.array(1.0), np.array(True), np.array(10**6), [0, 0], [], (), Ellipsis, (0,), [-1], np.array([1, 2]),
|
|
]
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Tests
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
def test_every_dataset_matches_h5py(h5py, pair):
|
|
ours, theirs, _ = pair
|
|
checked = 0
|
|
for name in _all_datasets(h5py, theirs):
|
|
if name.startswith("unsupported/") or name in H5PY_MISREADS or name == "cmp/with_vlen":
|
|
continue
|
|
t = theirs[name]
|
|
o = ours[name]
|
|
assert o.shape == t.shape, name
|
|
assert o.dtype == t.dtype, f"{name}: {o.dtype} vs {t.dtype}"
|
|
assert dict(o.dtype.metadata or {}) == dict(t.dtype.metadata or {}), name
|
|
assert o.ndim == t.ndim and o.size == t.size, name
|
|
assert o.maxshape == t.maxshape, name
|
|
assert o.name == t.name, name
|
|
if t.shape is None:
|
|
assert_same(o[()], t[()], name)
|
|
continue
|
|
for key in keys_for(t.shape):
|
|
what = f"{name}[{key!r}]"
|
|
try:
|
|
expected = t[key]
|
|
except Exception as e: # noqa: BLE001 - h5py refuses: so must we
|
|
with pytest.raises(type(e)):
|
|
o[key]
|
|
continue
|
|
assert_same(o[key], expected, what)
|
|
checked += 1
|
|
assert checked > 500
|
|
|
|
|
|
def test_vlen_big_endian(pair):
|
|
ours, _, _ = pair
|
|
ds = ours["vlen/f8_be"]
|
|
assert ds.dtype.metadata["vlen"] == np.dtype(">f8")
|
|
got = ds[()]
|
|
expected = [np.linspace(0, 1, 5), np.array([2.5]), np.array([])]
|
|
assert got.shape == (3,)
|
|
for g, e in zip(got, expected):
|
|
assert g.dtype == np.dtype(">f8")
|
|
np.testing.assert_array_equal(g, e)
|
|
np.testing.assert_array_equal(ds[1], [2.5])
|
|
|
|
|
|
def test_errors_match_h5py(h5py, pair):
|
|
ours, theirs, _ = pair
|
|
for name in ["num/le_i4_1d", "num/be_f8_2d_gzip", "str/vlen", "num/scalar_f8"]:
|
|
for key in ERROR_KEYS_1D:
|
|
try:
|
|
expected = theirs[name][key]
|
|
except Exception as e: # noqa: BLE001
|
|
with pytest.raises(type(e)):
|
|
ours[name][key]
|
|
else:
|
|
# h5py reads it, so must we (and the same values).
|
|
assert_same(ours[name][key], expected, f"{name}[{key!r}]")
|
|
|
|
|
|
def test_compound_fields_match_h5py(pair):
|
|
ours, theirs, _ = pair
|
|
for name in ["cmp/padded", "cmp/padded_chunked", "cmp/nested_2d"]:
|
|
t, o = theirs[name], ours[name]
|
|
for field in t.dtype.names:
|
|
assert_same(o[field], t[field], f"{name}[{field}]")
|
|
assert_same(o[field, 1:3], t[field, 1:3], f"{name}[{field}, 1:3]")
|
|
two = list(t.dtype.names[:2])
|
|
expected = t[tuple(two)]
|
|
got = o[tuple(two)]
|
|
assert got.dtype.names == expected.dtype.names
|
|
for field in two:
|
|
np.testing.assert_array_equal(got[field], expected[field])
|
|
with pytest.raises(ValueError):
|
|
ours["cmp/padded"]["no_such_field"]
|
|
with pytest.raises(ValueError):
|
|
ours["num/le_i4_1d"]["id"]
|
|
|
|
|
|
def test_numpy_asarray_and_len(pair):
|
|
ours, theirs, _ = pair
|
|
assert_same(np.asarray(ours["num/f8_3d"]), np.asarray(theirs["num/f8_3d"]))
|
|
assert len(ours["num/f8_3d"]) == len(theirs["num/f8_3d"])
|
|
with pytest.raises(TypeError):
|
|
len(ours["num/scalar_f8"])
|
|
|
|
|
|
def test_attributes_match_h5py(h5py, pair):
|
|
ours, theirs, _ = pair
|
|
for path in ["/", "num/le_f8_1d", "deep/er/est"]:
|
|
t = theirs[path].attrs
|
|
o = ours[path].attrs
|
|
assert list(o.keys()) == sorted(t.keys()), path
|
|
assert len(o) == len(t)
|
|
for k in t.keys():
|
|
assert k in o
|
|
assert_same(o[k], t[k], f"{path}.attrs[{k}]")
|
|
assert o.get("missing", 5) == 5
|
|
with pytest.raises(KeyError):
|
|
o["missing"]
|
|
assert [k for k, _ in o.items()] == list(o.keys())
|
|
|
|
|
|
def test_groups_match_h5py(h5py, pair):
|
|
ours, theirs, _ = pair
|
|
assert list(ours.keys()) == list(theirs.keys())
|
|
assert len(ours) == len(theirs)
|
|
for path in ["num", "deep", "deep/er", "deep/er/est", "empty_group"]:
|
|
o, t = ours[path], theirs[path]
|
|
assert isinstance(o, clawhdf5.Group)
|
|
assert list(o.keys()) == list(t.keys()), path
|
|
assert list(o) == list(t), path
|
|
assert len(o) == len(t), path
|
|
assert o.name == t.name
|
|
g = ours["deep/er"]
|
|
assert isinstance(g["est"], clawhdf5.Group)
|
|
assert isinstance(g["est/leaf"], clawhdf5.Dataset)
|
|
assert g["/deep/er/est/leaf"].name == "/deep/er/est/leaf"
|
|
assert "est/leaf" in g and "/num" in g and "nope" not in g
|
|
assert ours["/"].name == "/"
|
|
assert "deep/er/est/leaf" in ours
|
|
assert ours.get("nope") is None
|
|
with pytest.raises(KeyError):
|
|
ours["deep/nope"]
|
|
names = [k for k, v in ours["deep/er/est"].items()]
|
|
assert names == ["leaf"]
|
|
assert isinstance(ours["deep/er/est"].values()[0], clawhdf5.Dataset)
|
|
|
|
|
|
def test_unsupported_types_are_errors_not_data(pair):
|
|
ours, _, _ = pair
|
|
ds = ours["unsupported/refs"] # opening works
|
|
with pytest.raises(TypeError):
|
|
ds.dtype
|
|
with pytest.raises(TypeError):
|
|
ds[()]
|
|
with pytest.raises(TypeError):
|
|
ours["cmp/with_vlen"][()]
|
|
|
|
|
|
def test_boolean_masks_are_refused(pair):
|
|
ours, _, _ = pair
|
|
with pytest.raises(TypeError):
|
|
ours["num/le_i4_1d"][np.ones(37, dtype=bool)]
|
|
|
|
|
|
def test_reads_hand_numpy_the_rust_buffer(pair):
|
|
"""A fixed-size read is a view over the buffer the library filled, not a
|
|
copy of it."""
|
|
ours, _, _ = pair
|
|
arr = ours["num/le_f8_2d_gzip"][2:9, 1:4]
|
|
assert not arr.flags.owndata
|
|
assert arr.base is not None
|
|
assert arr.flags.aligned and arr.flags.c_contiguous
|
|
|
|
|
|
def test_only_the_selected_chunks_are_read(h5py, tmp_path):
|
|
"""Damage one chunk: a selection that avoids it still reads, one that
|
|
touches it fails. Reading everything and slicing afterwards (what the
|
|
bindings did before) failed both. (The library reads the whole dataset
|
|
anyway when a selection's bounding box covers more than half of it, so
|
|
the selections here stay below that.)"""
|
|
path = str(tmp_path / "damaged.h5")
|
|
data = np.arange(1000, dtype="<f8")
|
|
with h5py.File(path, "w") as f:
|
|
f.create_dataset("d", data=data, chunks=(100,), compression="gzip")
|
|
with h5py.File(path, "r") as f:
|
|
info = f["d"].id.get_chunk_info(9) # the last chunk
|
|
with open(path, "r+b") as fh:
|
|
fh.seek(info.byte_offset)
|
|
fh.write(b"\xff" * info.size)
|
|
with clawhdf5.File(path, "r") as f:
|
|
ds = f["d"]
|
|
np.testing.assert_array_equal(ds[0:400], data[0:400])
|
|
np.testing.assert_array_equal(ds[805:900], data[805:900])
|
|
np.testing.assert_array_equal(ds[5:450:7], data[5:450:7])
|
|
np.testing.assert_array_equal(ds[[3, 450, 899]], data[[3, 450, 899]])
|
|
with pytest.raises(Exception):
|
|
ds[950]
|
|
with pytest.raises(Exception):
|
|
ds[:]
|
|
|
|
|
|
def test_threads_read_the_same_file(pair):
|
|
"""Reads from many threads at once return exactly what h5py returns."""
|
|
ours, theirs, _ = pair
|
|
names = ["num/le_f8_2d_gzip", "num/i4_3d_shuffle", "str/vlen_2d_gzip", "cmp/padded_chunked", "num/be_i8_1d"]
|
|
expected = {n: theirs[n][()] for n in names}
|
|
errors = []
|
|
barrier = threading.Barrier(8)
|
|
|
|
def work(i):
|
|
barrier.wait()
|
|
for r in range(40):
|
|
n = names[(i + r) % len(names)]
|
|
got = ours[n][()]
|
|
try:
|
|
assert_same(got, expected[n], n)
|
|
except AssertionError as e:
|
|
errors.append(e)
|
|
|
|
with ThreadPoolExecutor(8) as pool:
|
|
list(pool.map(work, range(8)))
|
|
assert not errors, errors[:3]
|
|
|
|
|
|
def test_reads_release_the_gil(h5py, tmp_path):
|
|
"""While one thread is inside a long read, another Python thread keeps
|
|
running. With the GIL held for the read, the other thread would stall
|
|
for the whole read; the test measures its longest stall."""
|
|
import sys
|
|
import time
|
|
|
|
path = str(tmp_path / "gil.h5")
|
|
data = np.arange(2048 * 4096, dtype="<f4").reshape(2048, 4096)
|
|
with h5py.File(path, "w") as f:
|
|
f.create_dataset("d", data=data, chunks=(64, 4096), compression="gzip", compression_opts=1)
|
|
ds = clawhdf5.File(path, "r")["d"]
|
|
key = (slice(0, 1000), slice(None)) # under half: the uncached selection path
|
|
t0 = time.perf_counter()
|
|
ds[key]
|
|
one_read = time.perf_counter() - t0
|
|
assert one_read > 0.03, f"a read took only {one_read:.3f} s; too short to measure"
|
|
|
|
old = sys.getswitchinterval()
|
|
sys.setswitchinterval(0.001)
|
|
stop = threading.Event()
|
|
gaps = []
|
|
|
|
def spin():
|
|
last = time.perf_counter()
|
|
worst = 0.0
|
|
while not stop.is_set():
|
|
now = time.perf_counter()
|
|
worst = max(worst, now - last)
|
|
last = now
|
|
gaps.append(worst)
|
|
|
|
try:
|
|
t = threading.Thread(target=spin)
|
|
t.start()
|
|
time.sleep(0.01)
|
|
t0 = time.perf_counter()
|
|
for _ in range(2):
|
|
ds[key]
|
|
reading = time.perf_counter() - t0
|
|
stop.set()
|
|
t.join()
|
|
finally:
|
|
sys.setswitchinterval(old)
|
|
np.testing.assert_array_equal(ds[key], data[:1000])
|
|
# Held, the spinner would stall for about one read.
|
|
assert gaps[0] < one_read / 3, f"spinner stalled {gaps[0]:.3f} s during reads of {one_read:.3f} s ({reading:.3f} s)"
|
|
|
|
|
|
def _v4_index_fixture(h5py, path):
|
|
"""One 2-D dataset per v4 chunk index (HDF5 1.10+ layout, libver='latest')."""
|
|
data = (np.arange(37 * 23, dtype="<i4") * 7 - 1000).reshape(37, 23)
|
|
early = h5py.h5p.create(h5py.h5p.DATASET_CREATE)
|
|
early.set_alloc_time(h5py.h5d.ALLOC_TIME_EARLY)
|
|
with h5py.File(path, "w", libver="latest") as f:
|
|
f.create_dataset("implicit", data=data, chunks=(5, 4), dcpl=early)
|
|
f.create_dataset("fixed_array", data=data, chunks=(5, 4), compression="gzip")
|
|
f.create_dataset("extensible_array", data=data, chunks=(5, 4), maxshape=(None, 23), compression="gzip")
|
|
f.create_dataset("btree2", data=data, chunks=(5, 4), maxshape=(None, None), compression="gzip")
|
|
f.create_dataset("single_chunk", data=data, chunks=(37, 23), compression="gzip")
|
|
|
|
|
|
V4_KEYS = [
|
|
slice(0, 3), slice(0, 30), (slice(7, 16), slice(3, 9)), (36, 22), (slice(None, None, 3), slice(1, None, 4)),
|
|
(slice(1, None, 2), Ellipsis), (Ellipsis, slice(2, 22)), [0, 5, 6, 36], (slice(None), [0, 3, 22]), -1, (),
|
|
]
|
|
|
|
|
|
def test_every_v4_chunk_index_matches_h5py(h5py, tmp_path):
|
|
"""Partial reads of each v4 chunk index. The implicit index (early
|
|
allocation, no filters) used to panic in the library for any selection
|
|
covering more than half the dataset, e.g. ds[0:30]."""
|
|
path = str(tmp_path / "v4.h5")
|
|
_v4_index_fixture(h5py, path)
|
|
with h5py.File(path, "r") as theirs, clawhdf5.File(path, "r") as ours:
|
|
for name in theirs:
|
|
for key in V4_KEYS:
|
|
assert_same(ours[name][key], theirs[name][key], f"{name}[{key!r}]")
|
|
|
|
|
|
def test_a_library_panic_is_an_ordinary_exception():
|
|
"""PyO3 turns a Rust panic into PanicException, a BaseException that
|
|
`except Exception` does not catch. Every call into the library is
|
|
guarded, so a panic surfaces as clawhdf5.InternalError instead."""
|
|
assert issubclass(clawhdf5.InternalError, RuntimeError)
|
|
with pytest.raises(clawhdf5.InternalError, match="deliberate panic"):
|
|
clawhdf5._panic_for_test()
|
|
try:
|
|
clawhdf5._panic_for_test()
|
|
except Exception: # noqa: BLE001 - the point of the test
|
|
pass
|
|
|
|
|
|
def test_compound_padding_bytes_match_h5py(pair):
|
|
"""Every byte of a padded compound, padding included, is h5py's, for
|
|
index lists with many runs as well as slices. Joining the runs with
|
|
np.concatenate left the padding uninitialised: process memory ended up
|
|
in tobytes()."""
|
|
ours, theirs, _ = pair
|
|
keys = [[0, 3, 6], [1, 2, 5, 9], [0, 2, 4, 6, 8], slice(None), slice(1, 9, 3), 4, [9]]
|
|
for name in ["cmp/padded", "cmp/padded_chunked"]:
|
|
for _ in range(20): # garbage varies between runs; zeros do not
|
|
for key in keys:
|
|
assert ours[name][key].tobytes() == theirs[name][key].tobytes(), f"{name}[{key!r}]"
|
|
for key in [(slice(None), [0, 2]), ([0, 2, 3], slice(None)), ([1, 3], 1)]:
|
|
assert ours["cmp/nested_2d"][key].tobytes() == theirs["cmp/nested_2d"][key].tobytes(), key
|
|
|
|
|
|
def test_a_long_index_list_decodes_each_chunk_once(h5py, tmp_path):
|
|
"""An index list is read one group of chunks at a time, not one
|
|
hyperslab per run of indices: 5000 runs over 20 gzip chunks used to
|
|
decode the chunks 5000 times (8 s, against h5py's 0.014 s)."""
|
|
import time
|
|
|
|
path = str(tmp_path / "long_list.h5")
|
|
data = np.arange(200000, dtype="<f8")
|
|
grid = np.arange(400 * 3000, dtype="<i4").reshape(400, 3000)
|
|
with h5py.File(path, "w") as f:
|
|
f.create_dataset("d", data=data, chunks=(10000,), compression="gzip")
|
|
f.create_dataset("grid", data=grid, chunks=(50, 100), compression="gzip")
|
|
f.create_dataset("flat", data=data) # contiguous
|
|
rng = np.random.default_rng(3)
|
|
cases = [
|
|
("d", list(range(0, 200000, 40))),
|
|
("d", sorted(rng.choice(200000, 3000, replace=False).tolist())),
|
|
("flat", list(range(0, 200000, 40))),
|
|
("flat", [0, 7, 199999]),
|
|
("grid", (slice(None), list(range(0, 3000, 3)))),
|
|
("grid", (sorted(rng.choice(400, 150, replace=False).tolist()), slice(5, 2900, 7))),
|
|
("grid", (7, [0, 1, 2, 2000, 2999])),
|
|
]
|
|
with h5py.File(path, "r") as theirs, clawhdf5.File(path, "r") as ours:
|
|
for name, key in cases:
|
|
t0 = time.perf_counter()
|
|
got = ours[name][key]
|
|
took = time.perf_counter() - t0
|
|
assert_same(got, theirs[name][key], f"{name}[{len(key)}-key]")
|
|
assert took < 2.0, f"{name}: {took:.2f} s"
|
|
|
|
|
|
@pytest.mark.parametrize("libver", ["earliest", "latest"])
|
|
def test_big_groups_are_not_quadratic(h5py, tmp_path, libver):
|
|
"""A dataset or group remembers where its object is, and a group its
|
|
links, so reads and walks over a large group do not resolve every path
|
|
from the root again (it was O(n) per access: O(n^2) to visit a group)."""
|
|
import time
|
|
|
|
path = str(tmp_path / f"big_{libver}.h5")
|
|
n = 4000
|
|
with h5py.File(path, "w", libver=libver) as f:
|
|
g = f.create_group("g")
|
|
for i in range(n):
|
|
g.create_dataset(f"d{i:05d}", data=np.int32(i))
|
|
g.create_group("sub").create_dataset("leaf", data=np.arange(3))
|
|
with h5py.File(path, "r") as theirs, clawhdf5.File(path, "r") as ours:
|
|
t0 = time.perf_counter()
|
|
g = ours["g"]
|
|
assert list(g.keys()) == list(theirs["g"].keys())
|
|
total = sum(int(v[()]) for k, v in g.items() if k.startswith("d"))
|
|
assert total == n * (n - 1) // 2
|
|
seen = 0
|
|
for k in g:
|
|
if k.startswith("d"):
|
|
seen += int(g[k][()]) == int(k[1:])
|
|
assert seen == n
|
|
ds = g["d00007"]
|
|
assert all(ds[()] == 7 for _ in range(2000))
|
|
assert list(g["sub"]["leaf"][:]) == [0, 1, 2]
|
|
assert ours["/g/sub/leaf"][1] == 1 and g["/g/d00003"][()] == 3
|
|
took = time.perf_counter() - t0
|
|
assert took < 5.0, f"{took:.2f} s"
|