perf(py): datasets and groups keep their address; groups their links

Every ds[...] and g[k] resolved the path from the root again, two or three times per open, and resolving a name in a large group scans its links: visiting a group was O(n^2). 4000 scalar datasets in one group took 39 s (v1 group) and 131 s (dense) to list, read and re-read; now 0.3 s each. A Dataset keeps its object address, a Group (and the file's root) its address and, after the first lookup, its link table. New facade API File::dataset_at(address), tested in integration_tests. Co-Authored-By: Claude Opus 5.5 (1M context) <[email protected]>
2026-09-26 09:01:54 -05:00
parent b43bd2e67f
commit f0ecae38b6
9 changed files with 349 additions and 163 deletions
@@ -563,3 +563,36 @@ def test_a_long_index_list_decodes_each_chunk_once(h5py, tmp_path):
            took = time.perf_counter() - t0
            assert_same(got, theirs[name][key], f"{name}[{len(key)}-key]")
            assert took < 2.0, f"{name}: {took:.2f} s"
+
+
+@pytest.mark.parametrize("libver", ["earliest", "latest"])
+def test_big_groups_are_not_quadratic(h5py, tmp_path, libver):
+    """A dataset or group remembers where its object is, and a group its
+    links, so reads and walks over a large group do not resolve every path
+    from the root again (it was O(n) per access: O(n^2) to visit a group)."""
+    import time
+
+    path = str(tmp_path / f"big_{libver}.h5")
+    n = 4000
+    with h5py.File(path, "w", libver=libver) as f:
+        g = f.create_group("g")
+        for i in range(n):
+            g.create_dataset(f"d{i:05d}", data=np.int32(i))
+        g.create_group("sub").create_dataset("leaf", data=np.arange(3))
+    with h5py.File(path, "r") as theirs, clawhdf5.File(path, "r") as ours:
+        t0 = time.perf_counter()
+        g = ours["g"]
+        assert list(g.keys()) == list(theirs["g"].keys())
+        total = sum(int(v[()]) for k, v in g.items() if k.startswith("d"))
+        assert total == n * (n - 1) // 2
+        seen = 0
+        for k in g:
+            if k.startswith("d"):
+                seen += int(g[k][()]) == int(k[1:])
+        assert seen == n
+        ds = g["d00007"]
+        assert all(ds[()] == 7 for _ in range(2000))
+        assert list(g["sub"]["leaf"][:]) == [0, 1, 2]
+        assert ours["/g/sub/leaf"][1] == 1 and g["/g/d00003"][()] == 3
+        took = time.perf_counter() - t0
+        assert took < 5.0, f"{took:.2f} s"