diff --git a/conformance/compare.py b/conformance/compare.py index 8c14353..c1d65b2 100755 --- a/conformance/compare.py +++ b/conformance/compare.py @@ -146,7 +146,13 @@ for rel in files: if a.get("kind") != b.get("kind") and "error" not in b and "error" not in a: issues.append(("mismatch", f"{p}: kind {a.get('kind')} vs ours {b.get('kind')}", "kind", b)) ok = False + # h5py could not open the object at all: it read none of its + # attributes or links, so there is nothing to compare ours with + # (the object's own error is compared above and below). + ref_unopened = a.get("kind") == "unknown" and "error" in a for k in ("error", "list_error", "attrs_error"): + if ref_unopened and k != "error": + continue if k in b and k not in a: issues.append(("our-error", f"{p}: {k}: {b[k]}", b[k], b)) ok = False @@ -164,7 +170,7 @@ for rel in files: issues.append(("mismatch", f"{p}: values differ (h5py {a.get('dtype')} vs ours {b.get('dtype')})", "values", b | {"ref_head": a.get("head"), "ref_dtype": a.get("dtype")})) ok = False ra, oa = a.get("attrs") or {}, b.get("attrs") or {} - if "attrs_error" not in b and "attrs_error" not in a: + if "attrs_error" not in b and "attrs_error" not in a and not ref_unopened: for an in sorted(set(ra) | set(oa)): x, y = ra.get(an), oa.get(an) if x is None: diff --git a/conformance/ref.py b/conformance/ref.py index 573deeb..428b6a5 100755 --- a/conformance/ref.py +++ b/conformance/ref.py @@ -111,8 +111,11 @@ def note_conversion(tid, dt, rec): def hash_values(arr, dt, rec): - if dt.subdtype is not None: - # h5py expands an HDF5 array element type into trailing array dims + # h5py expands an HDF5 array element type into trailing array dims, a + # nested array type (an array of arrays) into all of them. Converting the + # expanded array back to the inner subarray type would broadcast every + # element into a whole subarray, so strip every level. + while dt.subdtype is not None: dt = dt.subdtype[0] arr = np.asarray(arr, dtype=dt) if simple(dt): @@ -173,9 +176,13 @@ def main(path): return objects = [] seen = set() - stack = [("/", None)] + # Objects h5py cannot open have no ObjectID to deduplicate by; they are + # deduplicated by the address their hard link points at instead, as the + # probe deduplicates every object by header address. + seen_unopenable = set() + stack = [("/", None, None)] while stack: - p, obj = stack.pop() + p, obj, link_addr = stack.pop() if len(objects) >= MAX_OBJECTS: top["truncated"] = True break @@ -185,6 +192,10 @@ def main(path): obj = f[p] key = hash(obj.id) # h5py ObjectID hash = (fileno, object address/token) except Exception as e: # noqa: BLE001 + if link_addr is not None: + if link_addr in seen_unopenable: + continue + seen_unopenable.add(link_addr) rec["kind"] = "unknown" rec["error"] = err(e) objects.append(rec) @@ -232,15 +243,18 @@ def main(path): base = "" if p == "/" else p kids = [] for n in names: + # The link's own type: `obj.get(n, getlink=True)` reports + # a user-defined link (type 64-255) as a HardLink. try: - link = obj.get(n, getlink=True) + info = obj.id.links.get_info(n.encode("utf-8", "surrogateescape")) except Exception: # noqa: BLE001 - link = None - if link is not None and not isinstance(link, h5py.HardLink): + info = None + if info is not None and info.type != h5py.h5l.TYPE_HARD: continue - kids.append(f"{base}/{n}") - for k in reversed(kids): - stack.append((k, None)) + addr = info.u if info is not None else None + kids.append((f"{base}/{n}", addr)) + for k, addr in reversed(kids): + stack.append((k, None, addr)) except Exception as e: # noqa: BLE001 rec["list_error"] = err(e) objects.append(rec)