conformance/compare.py

@@ -146,7 +146,13 @@ for rel in files:
             if a.get("kind") != b.get("kind") and "error" not in b and "error" not in a:
                 issues.append(("mismatch", f"{p}: kind {a.get('kind')} vs ours {b.get('kind')}", "kind", b))
                 ok = False
+            # h5py could not open the object at all: it read none of its
+            # attributes or links, so there is nothing to compare ours with
+            # (the object's own error is compared above and below).
+            ref_unopened = a.get("kind") == "unknown" and "error" in a
             for k in ("error", "list_error", "attrs_error"):
+                if ref_unopened and k != "error":
+                    continue
                 if k in b and k not in a:
                     issues.append(("our-error", f"{p}: {k}: {b[k]}", b[k], b))
                     ok = False
@@ -164,7 +170,7 @@ for rel in files:
                     issues.append(("mismatch", f"{p}: values differ (h5py {a.get('dtype')} vs ours {b.get('dtype')})", "values", b | {"ref_head": a.get("head"), "ref_dtype": a.get("dtype")}))
                     ok = False
             ra, oa = a.get("attrs") or {}, b.get("attrs") or {}
-            if "attrs_error" not in b and "attrs_error" not in a:
+            if "attrs_error" not in b and "attrs_error" not in a and not ref_unopened:
                 for an in sorted(set(ra) | set(oa)):
                     x, y = ra.get(an), oa.get(an)
                     if x is None:

conformance/ref.py

@@ -111,8 +111,11 @@ def note_conversion(tid, dt, rec):
 
 
 def hash_values(arr, dt, rec):
-    if dt.subdtype is not None:
+    # h5py expands an HDF5 array element type into trailing array dims, a
-        # h5py expands an HDF5 array element type into trailing array dims
+    # nested array type (an array of arrays) into all of them. Converting the
+    # expanded array back to the inner subarray type would broadcast every
+    # element into a whole subarray, so strip every level.
+    while dt.subdtype is not None:
         dt = dt.subdtype[0]
     arr = np.asarray(arr, dtype=dt)
     if simple(dt):
@@ -173,9 +176,13 @@ def main(path):
         return
     objects = []
     seen = set()
-    stack = [("/", None)]
+    # Objects h5py cannot open have no ObjectID to deduplicate by; they are
+    # deduplicated by the address their hard link points at instead, as the
+    # probe deduplicates every object by header address.
+    seen_unopenable = set()
+    stack = [("/", None, None)]
     while stack:
-        p, obj = stack.pop()
+        p, obj, link_addr = stack.pop()
         if len(objects) >= MAX_OBJECTS:
             top["truncated"] = True
             break
@@ -185,6 +192,10 @@ def main(path):
                 obj = f[p]
             key = hash(obj.id)  # h5py ObjectID hash = (fileno, object address/token)
         except Exception as e:  # noqa: BLE001
+            if link_addr is not None:
+                if link_addr in seen_unopenable:
+                    continue
+                seen_unopenable.add(link_addr)
             rec["kind"] = "unknown"
             rec["error"] = err(e)
             objects.append(rec)
@@ -232,15 +243,18 @@ def main(path):
                 base = "" if p == "/" else p
                 kids = []
                 for n in names:
+                    # The link's own type: `obj.get(n, getlink=True)` reports
+                    # a user-defined link (type 64-255) as a HardLink.
                     try:
-                        link = obj.get(n, getlink=True)
+                        info = obj.id.links.get_info(n.encode("utf-8", "surrogateescape"))
                     except Exception:  # noqa: BLE001
-                        link = None
+                        info = None
-                    if link is not None and not isinstance(link, h5py.HardLink):
+                    if info is not None and info.type != h5py.h5l.TYPE_HARD:
                         continue
-                    kids.append(f"{base}/{n}")
+                    addr = info.u if info is not None else None
-                for k in reversed(kids):
+                    kids.append((f"{base}/{n}", addr))
-                    stack.append((k, None))
+                for k, addr in reversed(kids):
+                    stack.append((k, None, addr))
             except Exception as e:  # noqa: BLE001
                 rec["list_error"] = err(e)
         objects.append(rec)