osobhandClaude Opus 5.5
5c44630ea2
The last 3 our-errors and 2 mismatches were documented as not ours but
still counted against us, on a heuristic (any big-endian VL mismatch) and
a fixed list.
- ref.py checks that the installed h5py returns big-endian VL elements
with the file's bytes under a little-endian dtype (writing and reading
a vlen('>f4') in memory) and, if so, relabels them with the file's byte
order before hashing, marking the object `ref_fix`. The values are now
compared: attr_datatypes.hdf5 /@vlen_uint64 and tcomplex_be.h5
/VariableLengthDatasetFloatComplex are identical to ours (h5dump 1.14.6
prints the same (1, 2), (3, 4, 5), (42)).
- ref_bugs.py re-reads each object h5py reads only through a libhdf5 bug
in six processes with different heaps (import order, MALLOC_PERTURB_).
Values the file determines are the same every time; these three change
(6, 6 and 3 distinct results), so they are over-read memory, not data
clawhdf5 could match. compare.py classifies a file `ref-bug` only when
every difference is such an object confirmed in the same run.
- report.py: the ref-bug class, the evidence table, the corrected
objects; test_ref.py covers both (run in the nightly job).
Co-Authored-By: Claude Opus 5.5 (1M context) <[email protected]>
326 lines
11 KiB
Python
Executable File
326 lines
11 KiB
Python
Executable File
#!/usr/bin/env python3
|
|
"""Reference probe: same JSON as the Rust `conformance-probe`, produced with h5py.
|
|
|
|
Walk: iterative DFS from '/', children in sorted (UTF-8 byte) name order, hard
|
|
links only, each object once (first path wins, deduplicated by object identity).
|
|
Canonical value encoding: see harness/src/main.rs.
|
|
"""
|
|
import hashlib
|
|
import json
|
|
import os
|
|
import struct
|
|
import sys
|
|
|
|
import numpy as np
|
|
import h5py
|
|
|
|
try:
|
|
import hdf5plugin # noqa: F401 registers blosc/lz4/zstd/bzip2/... filters
|
|
except Exception: # pragma: no cover
|
|
pass
|
|
|
|
MAX_BYTES = 200 * 1024 * 1024
|
|
MAX_OBJECTS = 200_000
|
|
|
|
|
|
def canon_str(b, out):
|
|
if isinstance(b, str):
|
|
b = b.encode("utf-8", "surrogateescape")
|
|
b = bytes(b)
|
|
cut = b.find(b"\x00")
|
|
if cut >= 0:
|
|
b = b[:cut]
|
|
b = b.rstrip(b" ")
|
|
out += b"S" + struct.pack("<I", len(b)) + b
|
|
|
|
|
|
def simple(dt):
|
|
if dt.fields:
|
|
return all(simple(dt.fields[n][0]) for n in dt.names)
|
|
if dt.subdtype:
|
|
return simple(dt.subdtype[0])
|
|
return dt.kind in "iufcbV"
|
|
|
|
|
|
def packed(dt):
|
|
if dt.fields:
|
|
return np.dtype([(n, packed(dt.fields[n][0])) for n in dt.names])
|
|
if dt.subdtype:
|
|
base, shape = dt.subdtype
|
|
return np.dtype((packed(base), shape))
|
|
if dt.kind in "iufcb":
|
|
return dt.newbyteorder("<")
|
|
return dt
|
|
|
|
|
|
# --- reference corrections ---------------------------------------------------
|
|
# Where h5py is known to return values the file does not hold, and the right
|
|
# values follow from what it returned, ref.py corrects them and records the
|
|
# correction on the object ("ref_fix"), so the comparison is still a real
|
|
# comparison and CONFORMANCE.md lists every corrected object. Each correction
|
|
# first checks that the installed h5py still has the bug.
|
|
|
|
# Corrections applied while encoding the current object.
|
|
FIXES = set()
|
|
_BE_VLEN_BUG = None
|
|
|
|
|
|
def be_vlen_bug():
|
|
"""h5py (3.16 / HDF5 2.0 at least) returns the elements of a
|
|
variable-length sequence whose base type is big-endian with the file's
|
|
big-endian bytes under a native (little-endian) dtype: a
|
|
`vlen_dtype('>f4')` dataset holding [1.0, 2.0] reads back as
|
|
[4.6e-41, 9.0e-44]. `h5dump` prints the file's values. Checked once per
|
|
process by writing and reading exactly that dataset in memory."""
|
|
global _BE_VLEN_BUG
|
|
if _BE_VLEN_BUG is None:
|
|
import io
|
|
try:
|
|
bio = io.BytesIO()
|
|
with h5py.File(bio, "w") as f:
|
|
d = f.create_dataset("v", (1,), dtype=h5py.vlen_dtype(np.dtype(">f4")))
|
|
d[0] = np.array([1.0, 2.0], dtype=">f4")
|
|
with h5py.File(bio, "r") as f:
|
|
got = np.asarray(f["v"][0])
|
|
_BE_VLEN_BUG = (got.dtype == np.dtype("<f4")
|
|
and got.view(">f4").tolist() == [1.0, 2.0]
|
|
and got.tolist() != [1.0, 2.0])
|
|
except Exception: # noqa: BLE001
|
|
_BE_VLEN_BUG = False
|
|
return _BE_VLEN_BUG
|
|
|
|
|
|
def unswapped(got, base):
|
|
"""`got` is `base` (big-endian somewhere) with every field in native
|
|
little-endian order instead: the shape of h5py's big-endian VL bug."""
|
|
return base.newbyteorder("<") == got and base != got
|
|
|
|
|
|
def canon_el(dt, val, out):
|
|
if dt.fields:
|
|
for n in dt.names:
|
|
canon_el(dt.fields[n][0], val[n], out)
|
|
return
|
|
if dt.subdtype:
|
|
base, _ = dt.subdtype
|
|
for x in np.asarray(val).reshape(-1):
|
|
canon_el(base, x, out)
|
|
return
|
|
k = dt.kind
|
|
if k in "iufcb":
|
|
out += np.asarray(val, dtype=dt).astype(dt.newbyteorder("<")).tobytes()
|
|
elif k == "V":
|
|
out += np.asarray(val, dtype=dt).tobytes()
|
|
elif k == "S":
|
|
canon_str(val, out)
|
|
elif k == "O":
|
|
if h5py.check_string_dtype(dt) is not None:
|
|
canon_str(val if val is not None else b"", out)
|
|
elif h5py.check_ref_dtype(dt) is not None:
|
|
out += b"R"
|
|
else:
|
|
base = h5py.check_vlen_dtype(dt)
|
|
if base is None:
|
|
raise TypeError(f"unhandled object dtype {dt!r}")
|
|
arr = np.asarray(val if val is not None else [])
|
|
if arr.dtype != base and be_vlen_bug() and unswapped(arr.dtype, base):
|
|
# h5py's big-endian VL bug (see be_vlen_bug): the bytes are
|
|
# the file's, the dtype label is wrong. Relabel, don't convert.
|
|
arr = arr.view(base)
|
|
FIXES.add("h5py-be-vlen")
|
|
arr = np.asarray(arr, dtype=base).reshape(-1)
|
|
out += b"V" + struct.pack("<I", arr.shape[0])
|
|
if simple(base):
|
|
out += arr.astype(packed(base)).tobytes()
|
|
else:
|
|
for x in arr:
|
|
canon_el(base, x, out)
|
|
elif k == "U":
|
|
canon_str(str(val), out)
|
|
else:
|
|
raise TypeError(f"unhandled dtype kind {k} ({dt!r})")
|
|
|
|
|
|
def has_obj(dt):
|
|
if dt.fields:
|
|
return any(has_obj(dt.fields[n][0]) for n in dt.names)
|
|
if dt.subdtype:
|
|
return has_obj(dt.subdtype[0])
|
|
return dt.kind == "O"
|
|
|
|
|
|
def note_conversion(tid, dt, rec):
|
|
"""h5py converts some file types (FP8, bfloat16, x87 long double, ...) to a
|
|
different-sized numpy type; then value bytes are not comparable."""
|
|
try:
|
|
if not has_obj(dt) and tid.get_size() != dt.itemsize:
|
|
rec["converted"] = f"file type size {tid.get_size()} -> numpy {dt} ({dt.itemsize})"
|
|
except Exception: # noqa: BLE001
|
|
pass
|
|
|
|
|
|
def hash_values(arr, dt, rec):
|
|
# h5py expands an HDF5 array element type into trailing array dims, a
|
|
# nested array type (an array of arrays) into all of them. Converting the
|
|
# expanded array back to the inner subarray type would broadcast every
|
|
# element into a whole subarray, so strip every level.
|
|
while dt.subdtype is not None:
|
|
dt = dt.subdtype[0]
|
|
arr = np.asarray(arr, dtype=dt)
|
|
FIXES.clear()
|
|
if simple(dt):
|
|
c = np.ascontiguousarray(arr).astype(packed(dt)).tobytes()
|
|
else:
|
|
out = bytearray()
|
|
for x in arr.reshape(-1):
|
|
canon_el(dt, x, out)
|
|
c = bytes(out)
|
|
if FIXES:
|
|
rec["ref_fix"] = sorted(FIXES)
|
|
rec["hash"] = hashlib.sha256(c).hexdigest()
|
|
rec["head"] = c[:48].hex()
|
|
|
|
|
|
def err(e):
|
|
s = f"{type(e).__name__}: {e}"
|
|
return s.splitlines()[0][:400] if s else type(e).__name__
|
|
|
|
|
|
def shape_of(s):
|
|
return "null" if s is None else list(s)
|
|
|
|
|
|
def n_bytes(shape, tid):
|
|
n = 1
|
|
for d in shape or ():
|
|
n *= d
|
|
return n * tid.get_size()
|
|
|
|
|
|
def read_attrs(obj):
|
|
out = {}
|
|
names = sorted(obj.attrs.keys(), key=lambda s: s.encode("utf-8", "surrogateescape"))
|
|
for name in names:
|
|
rec = {}
|
|
try:
|
|
aid = obj.attrs.get_id(name)
|
|
rec["dtype"] = str(aid.dtype)
|
|
rec["shape"] = shape_of(aid.shape)
|
|
note_conversion(aid.get_type(), aid.dtype, rec)
|
|
if aid.shape is None:
|
|
hash_values(np.empty((0,), dtype=aid.dtype), aid.dtype, rec)
|
|
else:
|
|
val = obj.attrs[name]
|
|
hash_values(val, aid.dtype, rec)
|
|
except Exception as e: # noqa: BLE001
|
|
rec = {"error": err(e)}
|
|
out[name] = rec
|
|
return out
|
|
|
|
|
|
def main(path):
|
|
top = {"file": path}
|
|
try:
|
|
f = h5py.File(path, "r")
|
|
except Exception as e: # noqa: BLE001
|
|
top["open_error"] = err(e)
|
|
print(json.dumps(top))
|
|
return
|
|
objects = []
|
|
seen = set()
|
|
# Objects h5py cannot open have no ObjectID to deduplicate by; they are
|
|
# deduplicated by the address their hard link points at instead, as the
|
|
# probe deduplicates every object by header address.
|
|
seen_unopenable = set()
|
|
stack = [("/", None, None)]
|
|
while stack:
|
|
p, obj, link_addr = stack.pop()
|
|
if len(objects) >= MAX_OBJECTS:
|
|
top["truncated"] = True
|
|
break
|
|
rec = {"path": p}
|
|
try:
|
|
if obj is None:
|
|
obj = f[p]
|
|
key = hash(obj.id) # h5py ObjectID hash = (fileno, object address/token)
|
|
except Exception as e: # noqa: BLE001
|
|
if link_addr is not None:
|
|
if link_addr in seen_unopenable:
|
|
continue
|
|
seen_unopenable.add(link_addr)
|
|
rec["kind"] = "unknown"
|
|
rec["error"] = err(e)
|
|
objects.append(rec)
|
|
continue
|
|
if key in seen:
|
|
continue
|
|
seen.add(key)
|
|
if isinstance(obj, h5py.Dataset):
|
|
kind = "dataset"
|
|
elif isinstance(obj, h5py.Group):
|
|
kind = "group"
|
|
elif isinstance(obj, h5py.Datatype):
|
|
kind = "datatype"
|
|
else:
|
|
kind = "unknown"
|
|
rec["kind"] = kind
|
|
if kind == "dataset":
|
|
try:
|
|
dt = obj.dtype
|
|
rec["dtype"] = str(dt)
|
|
rec["shape"] = shape_of(obj.shape)
|
|
note_conversion(obj.id.get_type(), dt, rec)
|
|
if obj.shape is None:
|
|
hash_values(np.empty((0,), dtype=dt), dt, rec)
|
|
elif n_bytes(obj.shape, obj.id.get_type()) > MAX_BYTES:
|
|
rec["skipped"] = "too large"
|
|
else:
|
|
arr = np.empty(obj.shape, dtype=dt)
|
|
if arr.size:
|
|
try:
|
|
obj.read_direct(arr)
|
|
except Exception: # noqa: BLE001
|
|
arr = obj[()]
|
|
hash_values(arr, dt, rec)
|
|
except Exception as e: # noqa: BLE001
|
|
rec["error"] = err(e)
|
|
if kind != "datatype":
|
|
try:
|
|
rec["attrs"] = read_attrs(obj)
|
|
except Exception as e: # noqa: BLE001
|
|
rec["attrs_error"] = err(e)
|
|
if kind == "group":
|
|
try:
|
|
names = sorted(obj.keys(), key=lambda s: s.encode("utf-8", "surrogateescape"))
|
|
base = "" if p == "/" else p
|
|
kids = []
|
|
for n in names:
|
|
# The link's own type: `obj.get(n, getlink=True)` reports
|
|
# a user-defined link (type 64-255) as a HardLink.
|
|
try:
|
|
info = obj.id.links.get_info(n.encode("utf-8", "surrogateescape"))
|
|
except Exception: # noqa: BLE001
|
|
info = None
|
|
if info is not None and info.type != h5py.h5l.TYPE_HARD:
|
|
continue
|
|
addr = info.u if info is not None else None
|
|
kids.append((f"{base}/{n}", addr))
|
|
for k, addr in reversed(kids):
|
|
stack.append((k, None, addr))
|
|
except Exception as e: # noqa: BLE001
|
|
rec["list_error"] = err(e)
|
|
objects.append(rec)
|
|
top["objects"] = objects
|
|
print(json.dumps(top), flush=True)
|
|
# Exit without tearing down the h5py objects: freeing them for some files
|
|
# that hold references (hdf5's h5repack_attr_refs.h5, cve-2024-32623.h5)
|
|
# makes libhdf5 2.0 abort with "free(): chunks in smallbin corrupted"
|
|
# about half the time. That happens after the reading is done, so it says
|
|
# nothing about what h5py read, but it flipped those files between ok and
|
|
# h5py-cannot-read from one run to the next.
|
|
os._exit(0)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main(sys.argv[1])
|