From 9e59499c56535a8d6252ee06b26658e57374785d Mon Sep 17 00:00:00 2001 From: osobh Date: Sat, 26 Sep 2026 10:13:11 -0500 Subject: [PATCH] conformance: fix three reference-probe artefacts ref.py and compare.py reported 13 files as mismatches or our-errors that were artefacts of the harness, not differences between the readers: - User-defined links (tall.h5, tudlink.h5, twithub*.h5, tmany.h5, ...): h5py's `get(name, getlink=True)` reports a user-defined link as a HardLink, so ref.py listed it as an object. Read the link type from H5Lget_info instead. - Objects h5py cannot open (cve-2019-8397/8398, cve-2021-46243, cve-2024-32618): the probe deduplicates by header address, ref.py by ObjectID, which an unopenable object does not have, so each extra hard link to it was listed again. Deduplicate those by link address. - Nested array types (tarray3.h5): h5py expands them into trailing dims; hash_values stripped one level and numpy broadcast every element into a whole subarray. Strip every level. compare.py no longer compares the attributes or links of an object h5py could not open at all (cve-2018-17438/17439, cve-2019-9151): h5py read none, so ours are neither extra nor errors against it. Co-Authored-By: Claude Opus 5.5 (1M context) --- conformance/compare.py | 8 +++++++- conformance/ref.py | 34 ++++++++++++++++++++++++---------- 2 files changed, 31 insertions(+), 11 deletions(-) diff --git a/conformance/compare.py b/conformance/compare.py index 8c14353..c1d65b2 100755 --- a/conformance/compare.py +++ b/conformance/compare.py @@ -146,7 +146,13 @@ for rel in files: if a.get("kind") != b.get("kind") and "error" not in b and "error" not in a: issues.append(("mismatch", f"{p}: kind {a.get('kind')} vs ours {b.get('kind')}", "kind", b)) ok = False + # h5py could not open the object at all: it read none of its + # attributes or links, so there is nothing to compare ours with + # (the object's own error is compared above and below). + ref_unopened = a.get("kind") == "unknown" and "error" in a for k in ("error", "list_error", "attrs_error"): + if ref_unopened and k != "error": + continue if k in b and k not in a: issues.append(("our-error", f"{p}: {k}: {b[k]}", b[k], b)) ok = False @@ -164,7 +170,7 @@ for rel in files: issues.append(("mismatch", f"{p}: values differ (h5py {a.get('dtype')} vs ours {b.get('dtype')})", "values", b | {"ref_head": a.get("head"), "ref_dtype": a.get("dtype")})) ok = False ra, oa = a.get("attrs") or {}, b.get("attrs") or {} - if "attrs_error" not in b and "attrs_error" not in a: + if "attrs_error" not in b and "attrs_error" not in a and not ref_unopened: for an in sorted(set(ra) | set(oa)): x, y = ra.get(an), oa.get(an) if x is None: diff --git a/conformance/ref.py b/conformance/ref.py index 573deeb..428b6a5 100755 --- a/conformance/ref.py +++ b/conformance/ref.py @@ -111,8 +111,11 @@ def note_conversion(tid, dt, rec): def hash_values(arr, dt, rec): - if dt.subdtype is not None: - # h5py expands an HDF5 array element type into trailing array dims + # h5py expands an HDF5 array element type into trailing array dims, a + # nested array type (an array of arrays) into all of them. Converting the + # expanded array back to the inner subarray type would broadcast every + # element into a whole subarray, so strip every level. + while dt.subdtype is not None: dt = dt.subdtype[0] arr = np.asarray(arr, dtype=dt) if simple(dt): @@ -173,9 +176,13 @@ def main(path): return objects = [] seen = set() - stack = [("/", None)] + # Objects h5py cannot open have no ObjectID to deduplicate by; they are + # deduplicated by the address their hard link points at instead, as the + # probe deduplicates every object by header address. + seen_unopenable = set() + stack = [("/", None, None)] while stack: - p, obj = stack.pop() + p, obj, link_addr = stack.pop() if len(objects) >= MAX_OBJECTS: top["truncated"] = True break @@ -185,6 +192,10 @@ def main(path): obj = f[p] key = hash(obj.id) # h5py ObjectID hash = (fileno, object address/token) except Exception as e: # noqa: BLE001 + if link_addr is not None: + if link_addr in seen_unopenable: + continue + seen_unopenable.add(link_addr) rec["kind"] = "unknown" rec["error"] = err(e) objects.append(rec) @@ -232,15 +243,18 @@ def main(path): base = "" if p == "/" else p kids = [] for n in names: + # The link's own type: `obj.get(n, getlink=True)` reports + # a user-defined link (type 64-255) as a HardLink. try: - link = obj.get(n, getlink=True) + info = obj.id.links.get_info(n.encode("utf-8", "surrogateescape")) except Exception: # noqa: BLE001 - link = None - if link is not None and not isinstance(link, h5py.HardLink): + info = None + if info is not None and info.type != h5py.h5l.TYPE_HARD: continue - kids.append(f"{base}/{n}") - for k in reversed(kids): - stack.append((k, None)) + addr = info.u if info is not None else None + kids.append((f"{base}/{n}", addr)) + for k, addr in reversed(kids): + stack.append((k, None, addr)) except Exception as e: # noqa: BLE001 rec["list_error"] = err(e) objects.append(rec)