"""Every read through clawhdf5 compared against h5py (libhdf5) on files h5py
writes: dtypes, shapes, values and the type of what comes back (array,
numpy scalar, bytes, str, Empty), for every datatype the bindings map and a
spread of index expressions; plus the errors h5py gives for the same keys."""

import threading
from concurrent.futures import ThreadPoolExecutor

import numpy as np
import pytest

import clawhdf5

# ---------------------------------------------------------------------------
# The generated file
# ---------------------------------------------------------------------------

NUMERIC = [
    "<i1", "<i2", "<i4", "<i8", "<u1", "<u2", "<u4", "<u8",
    ">i2", ">i4", ">i8", ">u2", ">u4", ">u8",
    "<f2", "<f4", "<f8", ">f2", ">f4", ">f8",
]


def _values(dtype, shape, seed):
    rng = np.random.default_rng(seed)
    dt = np.dtype(dtype)
    n = int(np.prod(shape))
    if dt.kind == "f":
        return rng.standard_normal(n).astype(dt).reshape(shape)
    info = np.iinfo(dt)
    return rng.integers(info.min, info.max, size=n, dtype=dt.newbyteorder("=")).astype(dt).reshape(shape)


def _compound_dtype():
    return np.dtype(
        {
            "names": ["id", "pos", "label", "flag", "vec"],
            "formats": ["<i4", ">f8", "S6", "u1", ("<f4", (3,))],
            "offsets": [0, 8, 16, 22, 24],
            "itemsize": 40,
        }
    )


def _nested_dtype():
    inner = np.dtype([("a", "<i2"), ("b", "<f4")])
    return np.dtype([("x", "<u8"), ("inner", inner), ("c", "<c8")])


def _write_fixture(h5py, path):
    str_dt = h5py.string_dtype()
    ascii_dt = h5py.string_dtype("ascii")
    with h5py.File(path, "w") as f:
        # Numeric types, both byte orders, 1-D contiguous and 2-D chunked+gzip.
        for i, dt in enumerate(NUMERIC):
            name = dt.replace("<", "le_").replace(">", "be_")
            f.create_dataset(f"num/{name}_1d", data=_values(dt, (37,), i))
            f.create_dataset(
                f"num/{name}_2d_gzip",
                data=_values(dt, (13, 11), 100 + i),
                chunks=(4, 5),
                compression="gzip",
            )
        f.create_dataset("num/f8_3d", data=_values("<f8", (6, 7, 5), 7), chunks=(2, 3, 5))
        f.create_dataset(
            "num/i4_3d_shuffle",
            data=_values("<i4", (5, 9, 4), 8),
            chunks=(3, 4, 2),
            shuffle=True,
            fletcher32=True,
            compression="gzip",
        )
        f.create_dataset("num/scalar_f8", data=np.float64(3.25))
        f.create_dataset("num/scalar_i2_be", data=np.array(-7, dtype=">i2"))
        f.create_dataset("num/zero_size", shape=(0, 3), dtype="<f4")
        f.create_dataset("num/empty", data=h5py.Empty("<f8"))
        sparse = f.create_dataset("num/sparse_fill", shape=(40,), chunks=(8,), dtype="<i4", fillvalue=-3)
        sparse[10:14] = [1, 2, 3, 4]
        f.create_dataset("num/resizable", data=np.arange(12.0), maxshape=(None,), chunks=(5,))
        # Compact layout (low level: h5py's create_dataset cannot ask for it).
        dcpl = h5py.h5p.create(h5py.h5p.DATASET_CREATE)
        dcpl.set_layout(h5py.h5d.COMPACT)
        space = h5py.h5s.create_simple((9,))
        dsid = h5py.h5d.create(f.id, b"num/compact_u2", h5py.h5t.py_create(np.dtype("<u2")), space, dcpl=dcpl)
        dsid.write(h5py.h5s.ALL, h5py.h5s.ALL, np.arange(9, dtype="<u2") * 7)

        # bool, enum, complex.
        f.create_dataset("misc/bool", data=np.array([True, False, True, True, False]))
        enum_dt = h5py.enum_dtype({"RED": 0, "GREEN": 1, "BLUE": 42}, basetype="i2")
        f.create_dataset("misc/enum", data=np.array([0, 42, 1, 1, 0], dtype="<i2"), dtype=enum_dt)
        uenum_dt = h5py.enum_dtype({"LO": 0, "HI": 2**40}, basetype="u8")
        f.create_dataset("misc/enum_u8", data=np.array([0, 2**40], dtype="<u8"), dtype=uenum_dt)
        f.create_dataset("misc/c8", data=(np.arange(6) + 1j * np.arange(6)).astype("<c8"))
        f.create_dataset("misc/c16_2d", data=(np.arange(12) - 2j).reshape(3, 4).astype("<c16"))
        f.create_dataset("misc/opaque", data=np.array([b"\x00\x01\x02", b"\xff\xfe\xfd"], dtype="V3"))

        # Strings.
        f.create_dataset("str/fixed", data=np.array([b"alpha", b"be", b"", b"gamma!"], dtype="S6"))
        utf8_4 = h5py.string_dtype("utf-8", 4)
        f.create_dataset(
            "str/fixed_2d_utf8",
            data=np.array([["é".encode(), b"b"], [b"c", b"dd"]], dtype=utf8_4),
        )
        f.create_dataset("str/vlen", data=["", "one", "twø", "a" * 300, "ünïcödé"], dtype=str_dt)
        f.create_dataset("str/vlen_ascii", data=[b"x", b"yy", b"zzz"], dtype=ascii_dt)
        f.create_dataset(
            "str/vlen_2d_gzip",
            data=np.array([[f"r{r}c{c}" * (r + c) for c in range(5)] for r in range(6)], dtype=object),
            dtype=str_dt,
            chunks=(2, 2),
            compression="gzip",
        )
        f.create_dataset("str/vlen_scalar", data="just one", dtype=str_dt)

        # Variable-length sequences.
        vl_i = h5py.vlen_dtype(np.dtype("<i4"))
        seqs = np.empty(4, dtype=object)
        seqs[:] = [np.arange(3, dtype="<i4"), np.array([], dtype="<i4"), np.arange(10, dtype="<i4") * -1, np.array([7], dtype="<i4")]
        f.create_dataset("vlen/i4", data=seqs, dtype=vl_i)
        vl_f = h5py.vlen_dtype(np.dtype(">f8"))
        fseqs = np.empty(3, dtype=object)
        fseqs[:] = [np.linspace(0, 1, 5).astype(">f8"), np.array([2.5], dtype=">f8"), np.array([], dtype=">f8")]
        f.create_dataset("vlen/f8_be", data=fseqs, dtype=vl_f)

        # Compounds.
        cdt = _compound_dtype()
        rec = np.zeros(10, dtype=cdt)
        rec["id"] = np.arange(10) * 3
        rec["pos"] = np.linspace(-1, 1, 10)
        rec["label"] = [f"n{i}".encode() for i in range(10)]
        rec["flag"] = np.arange(10) % 2
        rec["vec"] = np.arange(30, dtype="<f4").reshape(10, 3)
        f.create_dataset("cmp/padded", data=rec)
        f.create_dataset("cmp/padded_chunked", data=rec, chunks=(3,), compression="gzip")
        ndt = _nested_dtype()
        nrec = np.zeros((4, 3), dtype=ndt)
        nrec["x"] = np.arange(12).reshape(4, 3)
        nrec["inner"]["a"] = -np.arange(12).reshape(4, 3)
        nrec["inner"]["b"] = np.arange(12).reshape(4, 3) / 4
        nrec["c"] = np.arange(12).reshape(4, 3) * (1 + 1j)
        f.create_dataset("cmp/nested_2d", data=nrec)
        vdt = np.dtype([("n", "<i4"), ("s", h5py.string_dtype())])
        vrec = np.array([(1, "a"), (2, "bb")], dtype=vdt)
        f.create_dataset("cmp/with_vlen", data=vrec)

        # A true HDF5 array datatype (h5py's high level would widen the shape).
        tid = h5py.h5t.array_create(h5py.h5t.py_create(np.dtype("<i4")), (2, 3))
        space = h5py.h5s.create_simple((4,))
        dsid = h5py.h5d.create(f.id, b"cmp/array_type", tid, space)
        dsid.write(h5py.h5s.ALL, h5py.h5s.ALL, np.arange(24, dtype="<i4").reshape(4, 2, 3), mtype=tid)

        # Unsupported: references.
        f.create_dataset("unsupported/refs", data=[f["num"].ref, f["misc"].ref], dtype=h5py.ref_dtype)

        # Groups and attributes.
        g = f.create_group("deep/er/est")
        g.create_dataset("leaf", data=np.arange(5))
        f.create_group("empty_group")
        f.attrs["i4"] = np.int32(-5)
        f.attrs["u8"] = np.uint64(2**63 + 1)
        f.attrs["f2"] = np.float16(1.5)
        f.attrs["f8_arr"] = np.array([1.0, 2.5, -3.0])
        f.attrs["f8_one"] = np.array([4.0])
        f.attrs["i2_2d_be"] = np.arange(6, dtype=">i2").reshape(2, 3)
        f.attrs["bool"] = True
        f.attrs["bool_arr"] = np.array([True, False])
        f.attrs["vstr"] = "héllo"
        f.attrs["vstr_arr"] = ["a", "bcd", ""]
        f.attrs["fstr"] = np.bytes_(b"fixed")
        f.attrs["fstr_arr"] = np.array([b"x", b"yz"], dtype="S2")
        f.attrs["empty"] = h5py.Empty("<i4")
        f.attrs["complex"] = np.complex128(1 - 2j)
        f.attrs["compound"] = np.array((7, 2.5), dtype=[("a", "<i2"), ("b", "<f8")])
        f.attrs["enum"] = np.array(42, dtype=enum_dt)
        f["num/le_f8_1d"].attrs["units"] = "m/s"
        f["num/le_f8_1d"].attrs["scale"] = np.float32(0.5)
        g.attrs["depth"] = np.int8(3)


@pytest.fixture(scope="module")
def pair(h5py, tmp_path_factory):
    path = str(tmp_path_factory.mktemp("h5") / "fixture.h5")
    _write_fixture(h5py, path)
    theirs = h5py.File(path, "r")
    ours = clawhdf5.File(path, "r")
    yield ours, theirs, path
    theirs.close()
    ours.close()


def _all_datasets(h5py, f):
    names = []
    f.visititems(lambda n, o: names.append(n) if isinstance(o, h5py.Dataset) else None)
    return sorted(names)


# ---------------------------------------------------------------------------
# Comparison helpers
# ---------------------------------------------------------------------------


def assert_same(ours, theirs, what=""):
    if type(theirs).__name__ == "Empty":
        assert isinstance(ours, clawhdf5.Empty), what
        assert ours.dtype == theirs.dtype, what
        return
    assert type(ours) is type(theirs), f"{what}: {type(ours)} vs {type(theirs)}"
    if isinstance(theirs, np.ndarray):
        assert ours.shape == theirs.shape, what
        assert ours.dtype == theirs.dtype, f"{what}: {ours.dtype} vs {theirs.dtype}"
        if theirs.dtype == object:
            for a, b in zip(ours.ravel(), theirs.ravel()):
                assert_same(a, b, what)
        elif theirs.dtype.kind == "V":
            # Structured and opaque: every byte, padding included (h5py's
            # padding is zero; uninitialised memory there would leak).
            if theirs.dtype.names is not None:
                np.testing.assert_array_equal(ours, theirs, err_msg=what)
            assert ours.tobytes() == theirs.tobytes(), f"{what}: bytes differ"
        else:
            np.testing.assert_array_equal(ours, theirs, err_msg=what)
    elif isinstance(theirs, np.generic):
        assert ours.dtype == theirs.dtype, what
        if theirs.dtype.names is not None:
            np.testing.assert_array_equal(np.asarray(ours), np.asarray(theirs), err_msg=what)
            assert ours.tobytes() == theirs.tobytes(), f"{what}: bytes differ"
        else:
            assert ours == theirs or (ours != ours and theirs != theirs), what
    else:
        assert ours == theirs, what


def keys_for(shape):
    if shape == ():
        return [(), Ellipsis]
    keys = [(), Ellipsis, 0, -1, slice(None), slice(None, None, 2), slice(1, None, 3), slice(0, 0), np.int64(0),
            np.array(0), np.array(-1, dtype="i1")]
    n0 = shape[0]
    if n0 == 0:
        return [(), Ellipsis, slice(None), slice(None, None, 2), slice(0, 0)]
    keys += [slice(n0 // 2, None), slice(-3, None), [0, n0 - 1] if n0 > 1 else [0], (Ellipsis,)]
    if n0 > 5:
        keys += [[1, 2, 3, 5], [0, 4, 5]]
    if len(shape) >= 2:
        n1 = shape[1]
        keys += [
            (0, 0),
            (-1, -1),
            (slice(1, 3), slice(None, None, 2)),
            (Ellipsis, 1),
            (1, Ellipsis),
            (slice(None), [0, n1 - 1] if n1 > 1 else [0]),
            (slice(None, None, 2), 1),
            (slice(0, 2), slice(3, 1)),
            (np.array(1) if n0 > 1 else np.array(0), slice(None)),
            (slice(None), np.array(n1 - 1, dtype="u2")),
        ]
    if len(shape) >= 3:
        keys += [(0, slice(None), -1), (slice(1, None, 2), 2, slice(None, None, 3)), (Ellipsis, 0, 0), (0, Ellipsis, 1)]
    return keys


# h5py 3.16 (HDF5 2.0) returns the elements of a variable-length sequence of
# big-endian floats unswapped (0.25 comes back as 2.6e-319), so these are
# checked against the values written instead (test_vlen_big_endian).
H5PY_MISREADS = {"vlen/f8_be"}

ERROR_KEYS_1D = [slice(None, None, -1), 10**6, -(10**6), None, (0, 0, 0, 0, 0), [3, 1], (Ellipsis, Ellipsis), 1.5, "nope"]


# ---------------------------------------------------------------------------
# Tests
# ---------------------------------------------------------------------------


def test_every_dataset_matches_h5py(h5py, pair):
    ours, theirs, _ = pair
    checked = 0
    for name in _all_datasets(h5py, theirs):
        if name.startswith("unsupported/") or name in H5PY_MISREADS or name == "cmp/with_vlen":
            continue
        t = theirs[name]
        o = ours[name]
        assert o.shape == t.shape, name
        assert o.dtype == t.dtype, f"{name}: {o.dtype} vs {t.dtype}"
        assert dict(o.dtype.metadata or {}) == dict(t.dtype.metadata or {}), name
        assert o.ndim == t.ndim and o.size == t.size, name
        assert o.maxshape == t.maxshape, name
        assert o.name == t.name, name
        if t.shape is None:
            assert_same(o[()], t[()], name)
            continue
        for key in keys_for(t.shape):
            what = f"{name}[{key!r}]"
            try:
                expected = t[key]
            except Exception as e:  # noqa: BLE001 - h5py refuses: so must we
                with pytest.raises(type(e)):
                    o[key]
                continue
            assert_same(o[key], expected, what)
            checked += 1
    assert checked > 500


def test_vlen_big_endian(pair):
    ours, _, _ = pair
    ds = ours["vlen/f8_be"]
    assert ds.dtype.metadata["vlen"] == np.dtype(">f8")
    got = ds[()]
    expected = [np.linspace(0, 1, 5), np.array([2.5]), np.array([])]
    assert got.shape == (3,)
    for g, e in zip(got, expected):
        assert g.dtype == np.dtype(">f8")
        np.testing.assert_array_equal(g, e)
    np.testing.assert_array_equal(ds[1], [2.5])


def test_errors_match_h5py(h5py, pair):
    ours, theirs, _ = pair
    for name in ["num/le_i4_1d", "num/be_f8_2d_gzip", "str/vlen", "num/scalar_f8"]:
        for key in ERROR_KEYS_1D:
            try:
                theirs[name][key]
            except Exception as e:  # noqa: BLE001
                with pytest.raises(type(e)):
                    ours[name][key]
            else:
                pass  # valid for this shape; covered by the value test


def test_compound_fields_match_h5py(pair):
    ours, theirs, _ = pair
    for name in ["cmp/padded", "cmp/padded_chunked", "cmp/nested_2d"]:
        t, o = theirs[name], ours[name]
        for field in t.dtype.names:
            assert_same(o[field], t[field], f"{name}[{field}]")
            assert_same(o[field, 1:3], t[field, 1:3], f"{name}[{field}, 1:3]")
        two = list(t.dtype.names[:2])
        expected = t[tuple(two)]
        got = o[tuple(two)]
        assert got.dtype.names == expected.dtype.names
        for field in two:
            np.testing.assert_array_equal(got[field], expected[field])
    with pytest.raises(ValueError):
        ours["cmp/padded"]["no_such_field"]
    with pytest.raises(ValueError):
        ours["num/le_i4_1d"]["id"]


def test_numpy_asarray_and_len(pair):
    ours, theirs, _ = pair
    assert_same(np.asarray(ours["num/f8_3d"]), np.asarray(theirs["num/f8_3d"]))
    assert len(ours["num/f8_3d"]) == len(theirs["num/f8_3d"])
    with pytest.raises(TypeError):
        len(ours["num/scalar_f8"])


def test_attributes_match_h5py(h5py, pair):
    ours, theirs, _ = pair
    for path in ["/", "num/le_f8_1d", "deep/er/est"]:
        t = theirs[path].attrs
        o = ours[path].attrs
        assert list(o.keys()) == sorted(t.keys()), path
        assert len(o) == len(t)
        for k in t.keys():
            assert k in o
            assert_same(o[k], t[k], f"{path}.attrs[{k}]")
        assert o.get("missing", 5) == 5
        with pytest.raises(KeyError):
            o["missing"]
        assert [k for k, _ in o.items()] == list(o.keys())


def test_groups_match_h5py(h5py, pair):
    ours, theirs, _ = pair
    assert list(ours.keys()) == list(theirs.keys())
    assert len(ours) == len(theirs)
    for path in ["num", "deep", "deep/er", "deep/er/est", "empty_group"]:
        o, t = ours[path], theirs[path]
        assert isinstance(o, clawhdf5.Group)
        assert list(o.keys()) == list(t.keys()), path
        assert list(o) == list(t), path
        assert len(o) == len(t), path
        assert o.name == t.name
    g = ours["deep/er"]
    assert isinstance(g["est"], clawhdf5.Group)
    assert isinstance(g["est/leaf"], clawhdf5.Dataset)
    assert g["/deep/er/est/leaf"].name == "/deep/er/est/leaf"
    assert "est/leaf" in g and "/num" in g and "nope" not in g
    assert ours["/"].name == "/"
    assert "deep/er/est/leaf" in ours
    assert ours.get("nope") is None
    with pytest.raises(KeyError):
        ours["deep/nope"]
    names = [k for k, v in ours["deep/er/est"].items()]
    assert names == ["leaf"]
    assert isinstance(ours["deep/er/est"].values()[0], clawhdf5.Dataset)


def test_unsupported_types_are_errors_not_data(pair):
    ours, _, _ = pair
    ds = ours["unsupported/refs"]  # opening works
    with pytest.raises(TypeError):
        ds.dtype
    with pytest.raises(TypeError):
        ds[()]
    with pytest.raises(TypeError):
        ours["cmp/with_vlen"][()]


def test_boolean_masks_are_refused(pair):
    ours, _, _ = pair
    with pytest.raises(TypeError):
        ours["num/le_i4_1d"][np.ones(37, dtype=bool)]


def test_reads_hand_numpy_the_rust_buffer(pair):
    """A fixed-size read is a view over the buffer the library filled, not a
    copy of it."""
    ours, _, _ = pair
    arr = ours["num/le_f8_2d_gzip"][2:9, 1:4]
    assert not arr.flags.owndata
    assert arr.base is not None
    assert arr.flags.aligned and arr.flags.c_contiguous


def test_only_the_selected_chunks_are_read(h5py, tmp_path):
    """Damage one chunk: a selection that avoids it still reads, one that
    touches it fails. Reading everything and slicing afterwards (what the
    bindings did before) failed both. (The library reads the whole dataset
    anyway when a selection's bounding box covers more than half of it, so
    the selections here stay below that.)"""
    path = str(tmp_path / "damaged.h5")
    data = np.arange(1000, dtype="<f8")
    with h5py.File(path, "w") as f:
        f.create_dataset("d", data=data, chunks=(100,), compression="gzip")
    with h5py.File(path, "r") as f:
        info = f["d"].id.get_chunk_info(9)  # the last chunk
    with open(path, "r+b") as fh:
        fh.seek(info.byte_offset)
        fh.write(b"\xff" * info.size)
    with clawhdf5.File(path, "r") as f:
        ds = f["d"]
        np.testing.assert_array_equal(ds[0:400], data[0:400])
        np.testing.assert_array_equal(ds[805:900], data[805:900])
        np.testing.assert_array_equal(ds[5:450:7], data[5:450:7])
        np.testing.assert_array_equal(ds[[3, 450, 899]], data[[3, 450, 899]])
        with pytest.raises(Exception):
            ds[950]
        with pytest.raises(Exception):
            ds[:]


def test_threads_read_the_same_file(pair):
    """Reads from many threads at once (the GIL is released during each
    read) return exactly what h5py returns."""
    ours, theirs, _ = pair
    names = ["num/le_f8_2d_gzip", "num/i4_3d_shuffle", "str/vlen_2d_gzip", "cmp/padded_chunked", "num/be_i8_1d"]
    expected = {n: theirs[n][()] for n in names}
    errors = []
    barrier = threading.Barrier(8)

    def work(i):
        barrier.wait()
        for r in range(40):
            n = names[(i + r) % len(names)]
            got = ours[n][()]
            try:
                assert_same(got, expected[n], n)
            except AssertionError as e:
                errors.append(e)

    with ThreadPoolExecutor(8) as pool:
        list(pool.map(work, range(8)))
    assert not errors, errors[:3]


def _v4_index_fixture(h5py, path):
    """One 2-D dataset per v4 chunk index (HDF5 1.10+ layout, libver='latest')."""
    data = (np.arange(37 * 23, dtype="<i4") * 7 - 1000).reshape(37, 23)
    early = h5py.h5p.create(h5py.h5p.DATASET_CREATE)
    early.set_alloc_time(h5py.h5d.ALLOC_TIME_EARLY)
    with h5py.File(path, "w", libver="latest") as f:
        f.create_dataset("implicit", data=data, chunks=(5, 4), dcpl=early)
        f.create_dataset("fixed_array", data=data, chunks=(5, 4), compression="gzip")
        f.create_dataset("extensible_array", data=data, chunks=(5, 4), maxshape=(None, 23), compression="gzip")
        f.create_dataset("btree2", data=data, chunks=(5, 4), maxshape=(None, None), compression="gzip")
        f.create_dataset("single_chunk", data=data, chunks=(37, 23), compression="gzip")


V4_KEYS = [
    slice(0, 3), slice(0, 30), (slice(7, 16), slice(3, 9)), (36, 22), (slice(None, None, 3), slice(1, None, 4)),
    (slice(1, None, 2), Ellipsis), (Ellipsis, slice(2, 22)), [0, 5, 6, 36], (slice(None), [0, 3, 22]), -1, (),
]


def test_every_v4_chunk_index_matches_h5py(h5py, tmp_path):
    """Partial reads of each v4 chunk index. The implicit index (early
    allocation, no filters) used to panic in the library for any selection
    covering more than half the dataset, e.g. ds[0:30]."""
    path = str(tmp_path / "v4.h5")
    _v4_index_fixture(h5py, path)
    with h5py.File(path, "r") as theirs, clawhdf5.File(path, "r") as ours:
        for name in theirs:
            for key in V4_KEYS:
                assert_same(ours[name][key], theirs[name][key], f"{name}[{key!r}]")


def test_a_library_panic_is_an_ordinary_exception():
    """PyO3 turns a Rust panic into PanicException, a BaseException that
    `except Exception` does not catch. Every call into the library is
    guarded, so a panic surfaces as clawhdf5.InternalError instead."""
    assert issubclass(clawhdf5.InternalError, RuntimeError)
    with pytest.raises(clawhdf5.InternalError, match="deliberate panic"):
        clawhdf5._panic_for_test()
    try:
        clawhdf5._panic_for_test()
    except Exception:  # noqa: BLE001 - the point of the test
        pass


def test_compound_padding_bytes_match_h5py(pair):
    """Every byte of a padded compound, padding included, is h5py's, for
    index lists with many runs as well as slices. Joining the runs with
    np.concatenate left the padding uninitialised: process memory ended up
    in tobytes()."""
    ours, theirs, _ = pair
    keys = [[0, 3, 6], [1, 2, 5, 9], [0, 2, 4, 6, 8], slice(None), slice(1, 9, 3), 4, [9]]
    for name in ["cmp/padded", "cmp/padded_chunked"]:
        for _ in range(20):  # garbage varies between runs; zeros do not
            for key in keys:
                assert ours[name][key].tobytes() == theirs[name][key].tobytes(), f"{name}[{key!r}]"
    for key in [(slice(None), [0, 2]), ([0, 2, 3], slice(None)), ([1, 3], 1)]:
        assert ours["cmp/nested_2d"][key].tobytes() == theirs["cmp/nested_2d"][key].tobytes(), key


def test_a_long_index_list_decodes_each_chunk_once(h5py, tmp_path):
    """An index list is read one group of chunks at a time, not one
    hyperslab per run of indices: 5000 runs over 20 gzip chunks used to
    decode the chunks 5000 times (8 s, against h5py's 0.014 s)."""
    import time

    path = str(tmp_path / "long_list.h5")
    data = np.arange(200000, dtype="<f8")
    grid = np.arange(400 * 3000, dtype="<i4").reshape(400, 3000)
    with h5py.File(path, "w") as f:
        f.create_dataset("d", data=data, chunks=(10000,), compression="gzip")
        f.create_dataset("grid", data=grid, chunks=(50, 100), compression="gzip")
        f.create_dataset("flat", data=data)  # contiguous
    rng = np.random.default_rng(3)
    cases = [
        ("d", list(range(0, 200000, 40))),
        ("d", sorted(rng.choice(200000, 3000, replace=False).tolist())),
        ("flat", list(range(0, 200000, 40))),
        ("flat", [0, 7, 199999]),
        ("grid", (slice(None), list(range(0, 3000, 3)))),
        ("grid", (sorted(rng.choice(400, 150, replace=False).tolist()), slice(5, 2900, 7))),
        ("grid", (7, [0, 1, 2, 2000, 2999])),
    ]
    with h5py.File(path, "r") as theirs, clawhdf5.File(path, "r") as ours:
        for name, key in cases:
            t0 = time.perf_counter()
            got = ours[name][key]
            took = time.perf_counter() - t0
            assert_same(got, theirs[name][key], f"{name}[{len(key)}-key]")
            assert took < 2.0, f"{name}: {took:.2f} s"


@pytest.mark.parametrize("libver", ["earliest", "latest"])
def test_big_groups_are_not_quadratic(h5py, tmp_path, libver):
    """A dataset or group remembers where its object is, and a group its
    links, so reads and walks over a large group do not resolve every path
    from the root again (it was O(n) per access: O(n^2) to visit a group)."""
    import time

    path = str(tmp_path / f"big_{libver}.h5")
    n = 4000
    with h5py.File(path, "w", libver=libver) as f:
        g = f.create_group("g")
        for i in range(n):
            g.create_dataset(f"d{i:05d}", data=np.int32(i))
        g.create_group("sub").create_dataset("leaf", data=np.arange(3))
    with h5py.File(path, "r") as theirs, clawhdf5.File(path, "r") as ours:
        t0 = time.perf_counter()
        g = ours["g"]
        assert list(g.keys()) == list(theirs["g"].keys())
        total = sum(int(v[()]) for k, v in g.items() if k.startswith("d"))
        assert total == n * (n - 1) // 2
        seen = 0
        for k in g:
            if k.startswith("d"):
                seen += int(g[k][()]) == int(k[1:])
        assert seen == n
        ds = g["d00007"]
        assert all(ds[()] == 7 for _ in range(2000))
        assert list(g["sub"]["leaf"][:]) == [0, 1, 2]
        assert ours["/g/sub/leaf"][1] == 1 and g["/g/d00003"][()] == 3
        took = time.perf_counter() - t0
        assert took < 5.0, f"{took:.2f} s"