osobhandClaude Opus 5.5
9e59499c56
ref.py and compare.py reported 13 files as mismatches or our-errors that were artefacts of the harness, not differences between the readers: - User-defined links (tall.h5, tudlink.h5, twithub*.h5, tmany.h5, ...): h5py's `get(name, getlink=True)` reports a user-defined link as a HardLink, so ref.py listed it as an object. Read the link type from H5Lget_info instead. - Objects h5py cannot open (cve-2019-8397/8398, cve-2021-46243, cve-2024-32618): the probe deduplicates by header address, ref.py by ObjectID, which an unopenable object does not have, so each extra hard link to it was listed again. Deduplicate those by link address. - Nested array types (tarray3.h5): h5py expands them into trailing dims; hash_values stripped one level and numpy broadcast every element into a whole subarray. Strip every level. compare.py no longer compares the attributes or links of an object h5py could not open at all (cve-2018-17438/17439, cve-2019-9151): h5py read none, so ours are neither extra nor errors against it. Co-Authored-By: Claude Opus 5.5 (1M context) <[email protected]>
296 lines
13 KiB
Python
Executable File
296 lines
13 KiB
Python
Executable File
#!/usr/bin/env python3
|
|
"""compare.py <results_dir>: classify each file and group failures by root cause.
|
|
|
|
Writes <results_dir>/results.csv, results.json and summary.md.
|
|
File classes (first match wins):
|
|
hang, oom, crash, panic ours: timeout / allocation failure / signal / any panic (caught or not)
|
|
h5py-cannot-read libhdf5/h5py failed to open the file (or crashed/hung)
|
|
our-error we fail to open, list, or read something h5py reads
|
|
mismatch we read something with different shape/values, or a different object set
|
|
ok
|
|
"""
|
|
import collections
|
|
import csv
|
|
import json
|
|
import os
|
|
import re
|
|
import sys
|
|
|
|
R = sys.argv[1]
|
|
RUNS = os.path.join(R, "runs")
|
|
|
|
|
|
def load(d, name):
|
|
rc_p = os.path.join(d, name + ".rc")
|
|
if not os.path.exists(rc_p):
|
|
return None
|
|
rc = int(open(rc_p).read().strip() or -1)
|
|
err = open(os.path.join(d, name + ".err"), errors="replace").read()
|
|
js = None
|
|
try:
|
|
js = json.load(open(os.path.join(d, name + ".json")))
|
|
except Exception: # noqa: BLE001
|
|
pass
|
|
return {"rc": rc, "err": err, "json": js}
|
|
|
|
|
|
def proc_status(p):
|
|
"""-> (status, detail)"""
|
|
if p is None:
|
|
return "missing", ""
|
|
rc, err = p["rc"], p["err"]
|
|
first_panic = next((ln for ln in err.splitlines() if ln.startswith("PANIC:") or "panicked at" in ln), "")
|
|
if rc == 0 and p["json"] is not None:
|
|
return "ok", ""
|
|
if rc == 137 or rc == 124:
|
|
return "hang", f"timeout ({os.environ.get('TMO', '20')} s)"
|
|
if "memory allocation of" in err or "MemoryError" in err or "std::bad_alloc" in err:
|
|
m = re.search(r"memory allocation of \d+ bytes failed", err)
|
|
return "oom", m.group(0) if m else "allocation failure"
|
|
if "overflowed its stack" in err:
|
|
return "crash", "stack overflow"
|
|
if rc == 101:
|
|
return "panic", first_panic or (err.strip().splitlines() or [""])[-1]
|
|
if rc in (134, 139, 136, 135, 132) or rc > 128:
|
|
sig = {134: "SIGABRT", 139: "SIGSEGV", 136: "SIGFPE", 135: "SIGBUS", 132: "SIGILL"}.get(rc, f"signal {rc - 128}")
|
|
tail = [ln for ln in err.strip().splitlines() if ln.strip()][-1:]
|
|
return "crash", f"{sig}: {tail[0][:200] if tail else ''}"
|
|
tail = [ln for ln in err.strip().splitlines() if ln.strip()][-1:]
|
|
return "crash", f"rc={rc}: {tail[0][:200] if tail else ''}"
|
|
|
|
|
|
def norm(msg):
|
|
m = msg.split("\n")[0]
|
|
m = re.sub(r"0x[0-9a-fA-F]+", "X", m)
|
|
m = re.sub(r'"[^"]*"', '"…"', m)
|
|
m = re.sub(r"'[^']*'", "'…'", m)
|
|
m = re.sub(r"\d+", "N", m)
|
|
return m[:160]
|
|
|
|
|
|
def panic_head(msg):
|
|
"""First line + first clawhdf5 frame of a PANIC record."""
|
|
lines = msg.split("\n")
|
|
frame = next((ln.strip() for ln in lines[1:] if "clawhdf5_format" in ln), "")
|
|
return lines[0][:300], frame[:300]
|
|
|
|
|
|
def eq_shape(a, b):
|
|
return a == b
|
|
|
|
|
|
rows = []
|
|
issues_by_file = {}
|
|
root_causes = collections.defaultdict(lambda: {"files": set(), "count": 0, "examples": []})
|
|
mismatch_causes = collections.defaultdict(lambda: {"files": set(), "count": 0, "examples": []})
|
|
panics = []
|
|
ref_only_errors = collections.Counter()
|
|
incomparable = collections.Counter()
|
|
|
|
|
|
def add(bucket, key, file, example):
|
|
b = bucket[key]
|
|
b["count"] += 1
|
|
if file not in b["files"] and len(b["examples"]) < 6:
|
|
b["examples"].append(example)
|
|
b["files"].add(file)
|
|
|
|
|
|
files = [ln.strip() for ln in open(os.path.join(R, "files.txt")) if ln.strip()]
|
|
for rel in files:
|
|
d = os.path.join(RUNS, rel.replace("/", "__"))
|
|
corpus = rel.split("/")[0]
|
|
ours, ref = load(d, "ours"), load(d, "ref")
|
|
h5dump = load(d, "h5dump")
|
|
os_, od = proc_status(ours)
|
|
rs, rd = proc_status(ref)
|
|
oj = ours["json"] if ours else None
|
|
rj = ref["json"] if ref else None
|
|
issues = [] # (kind, detail)
|
|
caught_panics = []
|
|
|
|
def scan_err(path, what, msg):
|
|
if msg.startswith("PANIC:"):
|
|
caught_panics.append((path, what, msg))
|
|
|
|
if oj:
|
|
for o in oj.get("objects", []):
|
|
for k in ("error", "attrs_error", "list_error"):
|
|
if k in o:
|
|
scan_err(o["path"], k, o[k])
|
|
for an, av in (o.get("attrs") or {}).items():
|
|
if "error" in av:
|
|
scan_err(o["path"], f"attr {an}", av["error"])
|
|
if oj.get("open_error", "").startswith("PANIC:"):
|
|
caught_panics.append(("<open>", "open", oj["open_error"]))
|
|
|
|
ref_open_fail = rs != "ok" or (rj is not None and "open_error" in rj)
|
|
ours_open_err = oj.get("open_error") if oj else None
|
|
n_obj = n_ok = 0
|
|
if os_ == "ok" and rj and not ref_open_fail and not ours_open_err:
|
|
ro = {x["path"]: x for x in rj.get("objects", [])}
|
|
oo = {x["path"]: x for x in oj.get("objects", [])}
|
|
our_list_errors = [x for x in oo.values() if "list_error" in x]
|
|
for p in sorted(set(ro) | set(oo)):
|
|
a, b = ro.get(p), oo.get(p)
|
|
n_obj += 1
|
|
if a is None:
|
|
issues.append(("mismatch", f"extra object {p} (kind={b.get('kind')})", "extra-object", b))
|
|
continue
|
|
if b is None:
|
|
if our_list_errors:
|
|
continue # accounted for by the list_error
|
|
issues.append(("mismatch", f"missing object {p} (kind={a.get('kind')})", "missing-object", a))
|
|
continue
|
|
ok = True
|
|
if a.get("kind") != b.get("kind") and "error" not in b and "error" not in a:
|
|
issues.append(("mismatch", f"{p}: kind {a.get('kind')} vs ours {b.get('kind')}", "kind", b))
|
|
ok = False
|
|
# h5py could not open the object at all: it read none of its
|
|
# attributes or links, so there is nothing to compare ours with
|
|
# (the object's own error is compared above and below).
|
|
ref_unopened = a.get("kind") == "unknown" and "error" in a
|
|
for k in ("error", "list_error", "attrs_error"):
|
|
if ref_unopened and k != "error":
|
|
continue
|
|
if k in b and k not in a:
|
|
issues.append(("our-error", f"{p}: {k}: {b[k]}", b[k], b))
|
|
ok = False
|
|
elif k in a and k not in b and k == "error":
|
|
ref_only_errors[norm(a[k])] += 1
|
|
if a.get("kind") == "dataset" and "error" not in a and "error" not in b:
|
|
if "skipped" in a or "skipped" in b:
|
|
pass
|
|
elif a.get("converted"):
|
|
incomparable[f"dataset {a['converted']}"] += 1
|
|
elif a.get("shape") != b.get("shape"):
|
|
issues.append(("mismatch", f"{p}: shape {a.get('shape')} vs ours {b.get('shape')}", "shape", b))
|
|
ok = False
|
|
elif a.get("hash") != b.get("hash"):
|
|
issues.append(("mismatch", f"{p}: values differ (h5py {a.get('dtype')} vs ours {b.get('dtype')})", "values", b | {"ref_head": a.get("head"), "ref_dtype": a.get("dtype")}))
|
|
ok = False
|
|
ra, oa = a.get("attrs") or {}, b.get("attrs") or {}
|
|
if "attrs_error" not in b and "attrs_error" not in a and not ref_unopened:
|
|
for an in sorted(set(ra) | set(oa)):
|
|
x, y = ra.get(an), oa.get(an)
|
|
if x is None:
|
|
issues.append(("mismatch", f"{p}@{an}: extra attribute", "extra-attr", y or {}))
|
|
elif y is None:
|
|
issues.append(("mismatch", f"{p}@{an}: missing attribute", "missing-attr", x))
|
|
elif "error" in y and "error" not in x:
|
|
issues.append(("our-error", f"{p}@{an}: {y['error']}", y["error"], y))
|
|
elif "error" in x:
|
|
continue
|
|
elif x.get("converted"):
|
|
incomparable[f"attr {x['converted']}"] += 1
|
|
elif x.get("shape") != y.get("shape"):
|
|
issues.append(("mismatch", f"{p}@{an}: attr shape {x.get('shape')} vs ours {y.get('shape')}", "attr-shape", y | {"ref_dtype": x.get("dtype")}))
|
|
elif x.get("hash") != y.get("hash"):
|
|
issues.append(("mismatch", f"{p}@{an}: attr values differ (h5py {x.get('dtype')} vs ours {y.get('dtype')})", "attr-values", y | {"ref_head": x.get("head"), "ref_dtype": x.get("dtype")}))
|
|
if ok:
|
|
n_ok += 1
|
|
|
|
# classify
|
|
if os_ in ("hang", "oom", "crash", "panic"):
|
|
cls = os_
|
|
elif caught_panics:
|
|
cls = "panic"
|
|
elif ref_open_fail:
|
|
cls = "h5py-cannot-read"
|
|
elif ours_open_err:
|
|
cls = "our-error"
|
|
issues.append(("our-error", f"open: {ours_open_err}", ours_open_err, {}))
|
|
elif any(i[0] == "our-error" for i in issues):
|
|
cls = "our-error"
|
|
elif issues:
|
|
cls = "mismatch"
|
|
else:
|
|
cls = "ok"
|
|
|
|
if os_ in ("hang", "oom", "crash", "panic") or caught_panics:
|
|
panics.append({
|
|
"file": rel, "class": cls, "detail": od,
|
|
"stderr": (ours["err"] if ours else "")[:3000],
|
|
"caught": [(p, w, m[:2500]) for p, w, m in caught_panics[:3]],
|
|
"n_caught": len(caught_panics),
|
|
})
|
|
for kind, detail, key, rec in issues:
|
|
if kind == "our-error":
|
|
add(root_causes, norm(key), rel, detail[:300])
|
|
else:
|
|
if key in ("values", "attr-values", "shape", "attr-shape"):
|
|
mk = f"{key}: ours={rec.get('dtype')} h5py={rec.get('ref_dtype')} layout={rec.get('layout','-')} filters={rec.get('filters','-')}"
|
|
else:
|
|
mk = key
|
|
add(mismatch_causes, mk, rel, detail[:300] + (f" | ref_head={rec.get('ref_head')} our_head={rec.get('head')}" if rec.get("ref_head") else ""))
|
|
ref_detail = rd if rs != "ok" else ((rj or {}).get("open_error") or "")
|
|
h5d = ""
|
|
if h5dump:
|
|
rc = h5dump["rc"]
|
|
h5d = {0: "ok", 1: "error", 137: "hang", 124: "hang", 134: "SIGABRT", 139: "SIGSEGV", 136: "SIGFPE", 135: "SIGBUS"}.get(rc, f"rc={rc}")
|
|
if "memory allocation" in h5dump["err"] or "Cannot allocate" in h5dump["err"]:
|
|
h5d += "(oom)"
|
|
rows.append({
|
|
"file": rel, "corpus": corpus, "class": cls,
|
|
"ours": os_ if os_ != "ok" else ("open-error" if ours_open_err else ("panic" if caught_panics else "ok")),
|
|
"ours_detail": (od or ours_open_err or (caught_panics[0][2].split("\n")[0] if caught_panics else ""))[:300],
|
|
"ref": rs if rs != "ok" else ("open-error" if (rj or {}).get("open_error") else "ok"),
|
|
"ref_detail": ref_detail[:300],
|
|
"h5dump_1_14_6": h5d,
|
|
"h5dump_detail": ([ln for ln in h5dump["err"].splitlines() if ln.strip()][-1:] or [""])[0][:200] if h5dump else "",
|
|
"objects": n_obj, "objects_ok": n_ok,
|
|
"issues": len(issues), "first_issue": issues[0][1][:300] if issues else "",
|
|
"superblock": (oj or {}).get("superblock_version", ""),
|
|
})
|
|
# the first issues of each file, for report.py's known-cause matching
|
|
issues_by_file[rel] = [
|
|
{"kind": k, "key": key, "detail": det[:300], "ours_dtype": rec.get("dtype"), "ref_dtype": rec.get("ref_dtype")}
|
|
for k, det, key, rec in issues[:50]
|
|
]
|
|
|
|
with open(os.path.join(R, "results.csv"), "w", newline="") as fh:
|
|
w = csv.DictWriter(fh, fieldnames=list(rows[0].keys()))
|
|
w.writeheader()
|
|
w.writerows(rows)
|
|
|
|
|
|
def ser(b):
|
|
return {k: {"files": len(v["files"]), "count": v["count"], "examples": v["examples"], "file_list": sorted(v["files"])} for k, v in sorted(b.items(), key=lambda kv: -len(kv[1]["files"]))}
|
|
|
|
|
|
json.dump({"rows": rows, "issues": issues_by_file, "root_causes": ser(root_causes), "mismatch_causes": ser(mismatch_causes),
|
|
"panics": panics, "incomparable": incomparable.most_common(), "ref_only_errors": ref_only_errors.most_common()},
|
|
open(os.path.join(R, "results.json"), "w"), indent=1)
|
|
|
|
classes = ["ok", "our-error", "mismatch", "h5py-cannot-read", "hang", "panic", "crash", "oom"]
|
|
by_corpus = collections.defaultdict(collections.Counter)
|
|
for r in rows:
|
|
by_corpus[r["corpus"]][r["class"]] += 1
|
|
by_corpus["ALL"][r["class"]] += 1
|
|
lines = ["# Conformance sweep summary", "", "| corpus | files | " + " | ".join(classes) + " |", "|---" * (len(classes) + 2) + "|"]
|
|
for c in sorted(by_corpus, key=lambda k: (k == "ALL", k)):
|
|
cnt = by_corpus[c]
|
|
lines.append(f"| {c} | {sum(cnt.values())} | " + " | ".join(str(cnt.get(k, 0)) for k in classes) + " |")
|
|
lines += ["", "## Panics / hangs / crashes / OOM", ""]
|
|
for p in panics:
|
|
lines.append(f"- **{p['file']}** [{p['class']}] {p['detail']}")
|
|
for path, what, m in p["caught"][:1]:
|
|
lines.append(" ```\n " + f"{path} ({what}): " + m.replace("\n", "\n ")[:1500] + "\n ```")
|
|
if not p["caught"] and p["stderr"]:
|
|
lines.append(" ```\n " + p["stderr"].strip()[:1500].replace("\n", "\n ") + "\n ```")
|
|
lines += ["", "## Our-error root causes (files affected)", ""]
|
|
for k, v in ser(root_causes).items():
|
|
lines.append(f"- [{v['files']} files, {v['count']} objs] `{k}`")
|
|
for ex in v["examples"][:3]:
|
|
lines.append(f" - {ex}")
|
|
lines += ["", "## Mismatch root causes", ""]
|
|
for k, v in ser(mismatch_causes).items():
|
|
lines.append(f"- [{v['files']} files, {v['count']} objs] `{k}`")
|
|
for ex in v["examples"][:3]:
|
|
lines.append(f" - {ex}")
|
|
lines += ["", "## Objects h5py fails on but we read (top)", ""]
|
|
for k, n in ref_only_errors.most_common(15):
|
|
lines.append(f"- {n} x `{k}`")
|
|
open(os.path.join(R, "summary.md"), "w").write("\n".join(lines) + "\n")
|
|
print("\n".join(lines[:4 + len(by_corpus)]))
|