feat(conformance): in-repo, reproducible conformance sweep
conformance/run.sh fetches eight public HDF5 corpora pinned by commit (conformance/corpus.txt) into a gitignored cache, reads every file with clawhdf5 (conformance/probe, a crate outside the workspace) and with h5py/libhdf5 (ref.py), and the CVE files with h5dump, each under a timeout and an address-space limit; compare.py classifies the files, report.py writes CONFORMANCE.md and check.py gates on panics/hangs/crashes/OOM and on regressions against conformance/baseline.json. ~25 s once cached. Changes from the ad-hoc audit harness: - the probe compares non-IEEE-layout floats (N-Bit) and integers with a bit offset or reduced precision as the values libhdf5 converts them to, not raw file bytes: 8 files that showed as mismatches now read identically; - ref.py exits without tearing down h5py objects: libhdf5 2.0 aborts while freeing them for two files about half the time, which flipped them between ok and h5py-cannot-read from run to run; - the file list is defined (list_files.py): netCDF classic files are left out, 11 HDF5 files the ad-hoc sweep missed are in. Co-Authored-By: Claude Opus 5.5 (1M context) <[email protected]>
This commit is contained in:
osobh
Executable
+289
@@ -0,0 +1,289 @@
|
||||
#!/usr/bin/env python3
|
||||
"""compare.py <results_dir>: classify each file and group failures by root cause.
|
||||
|
||||
Writes <results_dir>/results.csv, results.json and summary.md.
|
||||
File classes (first match wins):
|
||||
hang, oom, crash, panic ours: timeout / allocation failure / signal / any panic (caught or not)
|
||||
h5py-cannot-read libhdf5/h5py failed to open the file (or crashed/hung)
|
||||
our-error we fail to open, list, or read something h5py reads
|
||||
mismatch we read something with different shape/values, or a different object set
|
||||
ok
|
||||
"""
|
||||
import collections
|
||||
import csv
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import sys
|
||||
|
||||
R = sys.argv[1]
|
||||
RUNS = os.path.join(R, "runs")
|
||||
|
||||
|
||||
def load(d, name):
|
||||
rc_p = os.path.join(d, name + ".rc")
|
||||
if not os.path.exists(rc_p):
|
||||
return None
|
||||
rc = int(open(rc_p).read().strip() or -1)
|
||||
err = open(os.path.join(d, name + ".err"), errors="replace").read()
|
||||
js = None
|
||||
try:
|
||||
js = json.load(open(os.path.join(d, name + ".json")))
|
||||
except Exception: # noqa: BLE001
|
||||
pass
|
||||
return {"rc": rc, "err": err, "json": js}
|
||||
|
||||
|
||||
def proc_status(p):
|
||||
"""-> (status, detail)"""
|
||||
if p is None:
|
||||
return "missing", ""
|
||||
rc, err = p["rc"], p["err"]
|
||||
first_panic = next((ln for ln in err.splitlines() if ln.startswith("PANIC:") or "panicked at" in ln), "")
|
||||
if rc == 0 and p["json"] is not None:
|
||||
return "ok", ""
|
||||
if rc == 137 or rc == 124:
|
||||
return "hang", f"timeout ({os.environ.get('TMO', '20')} s)"
|
||||
if "memory allocation of" in err or "MemoryError" in err or "std::bad_alloc" in err:
|
||||
m = re.search(r"memory allocation of \d+ bytes failed", err)
|
||||
return "oom", m.group(0) if m else "allocation failure"
|
||||
if "overflowed its stack" in err:
|
||||
return "crash", "stack overflow"
|
||||
if rc == 101:
|
||||
return "panic", first_panic or (err.strip().splitlines() or [""])[-1]
|
||||
if rc in (134, 139, 136, 135, 132) or rc > 128:
|
||||
sig = {134: "SIGABRT", 139: "SIGSEGV", 136: "SIGFPE", 135: "SIGBUS", 132: "SIGILL"}.get(rc, f"signal {rc - 128}")
|
||||
tail = [ln for ln in err.strip().splitlines() if ln.strip()][-1:]
|
||||
return "crash", f"{sig}: {tail[0][:200] if tail else ''}"
|
||||
tail = [ln for ln in err.strip().splitlines() if ln.strip()][-1:]
|
||||
return "crash", f"rc={rc}: {tail[0][:200] if tail else ''}"
|
||||
|
||||
|
||||
def norm(msg):
|
||||
m = msg.split("\n")[0]
|
||||
m = re.sub(r"0x[0-9a-fA-F]+", "X", m)
|
||||
m = re.sub(r'"[^"]*"', '"…"', m)
|
||||
m = re.sub(r"'[^']*'", "'…'", m)
|
||||
m = re.sub(r"\d+", "N", m)
|
||||
return m[:160]
|
||||
|
||||
|
||||
def panic_head(msg):
|
||||
"""First line + first clawhdf5 frame of a PANIC record."""
|
||||
lines = msg.split("\n")
|
||||
frame = next((ln.strip() for ln in lines[1:] if "clawhdf5_format" in ln), "")
|
||||
return lines[0][:300], frame[:300]
|
||||
|
||||
|
||||
def eq_shape(a, b):
|
||||
return a == b
|
||||
|
||||
|
||||
rows = []
|
||||
issues_by_file = {}
|
||||
root_causes = collections.defaultdict(lambda: {"files": set(), "count": 0, "examples": []})
|
||||
mismatch_causes = collections.defaultdict(lambda: {"files": set(), "count": 0, "examples": []})
|
||||
panics = []
|
||||
ref_only_errors = collections.Counter()
|
||||
incomparable = collections.Counter()
|
||||
|
||||
|
||||
def add(bucket, key, file, example):
|
||||
b = bucket[key]
|
||||
b["count"] += 1
|
||||
if file not in b["files"] and len(b["examples"]) < 6:
|
||||
b["examples"].append(example)
|
||||
b["files"].add(file)
|
||||
|
||||
|
||||
files = [ln.strip() for ln in open(os.path.join(R, "files.txt")) if ln.strip()]
|
||||
for rel in files:
|
||||
d = os.path.join(RUNS, rel.replace("/", "__"))
|
||||
corpus = rel.split("/")[0]
|
||||
ours, ref = load(d, "ours"), load(d, "ref")
|
||||
h5dump = load(d, "h5dump")
|
||||
os_, od = proc_status(ours)
|
||||
rs, rd = proc_status(ref)
|
||||
oj = ours["json"] if ours else None
|
||||
rj = ref["json"] if ref else None
|
||||
issues = [] # (kind, detail)
|
||||
caught_panics = []
|
||||
|
||||
def scan_err(path, what, msg):
|
||||
if msg.startswith("PANIC:"):
|
||||
caught_panics.append((path, what, msg))
|
||||
|
||||
if oj:
|
||||
for o in oj.get("objects", []):
|
||||
for k in ("error", "attrs_error", "list_error"):
|
||||
if k in o:
|
||||
scan_err(o["path"], k, o[k])
|
||||
for an, av in (o.get("attrs") or {}).items():
|
||||
if "error" in av:
|
||||
scan_err(o["path"], f"attr {an}", av["error"])
|
||||
if oj.get("open_error", "").startswith("PANIC:"):
|
||||
caught_panics.append(("<open>", "open", oj["open_error"]))
|
||||
|
||||
ref_open_fail = rs != "ok" or (rj is not None and "open_error" in rj)
|
||||
ours_open_err = oj.get("open_error") if oj else None
|
||||
n_obj = n_ok = 0
|
||||
if os_ == "ok" and rj and not ref_open_fail and not ours_open_err:
|
||||
ro = {x["path"]: x for x in rj.get("objects", [])}
|
||||
oo = {x["path"]: x for x in oj.get("objects", [])}
|
||||
our_list_errors = [x for x in oo.values() if "list_error" in x]
|
||||
for p in sorted(set(ro) | set(oo)):
|
||||
a, b = ro.get(p), oo.get(p)
|
||||
n_obj += 1
|
||||
if a is None:
|
||||
issues.append(("mismatch", f"extra object {p} (kind={b.get('kind')})", "extra-object", b))
|
||||
continue
|
||||
if b is None:
|
||||
if our_list_errors:
|
||||
continue # accounted for by the list_error
|
||||
issues.append(("mismatch", f"missing object {p} (kind={a.get('kind')})", "missing-object", a))
|
||||
continue
|
||||
ok = True
|
||||
if a.get("kind") != b.get("kind") and "error" not in b and "error" not in a:
|
||||
issues.append(("mismatch", f"{p}: kind {a.get('kind')} vs ours {b.get('kind')}", "kind", b))
|
||||
ok = False
|
||||
for k in ("error", "list_error", "attrs_error"):
|
||||
if k in b and k not in a:
|
||||
issues.append(("our-error", f"{p}: {k}: {b[k]}", b[k], b))
|
||||
ok = False
|
||||
elif k in a and k not in b and k == "error":
|
||||
ref_only_errors[norm(a[k])] += 1
|
||||
if a.get("kind") == "dataset" and "error" not in a and "error" not in b:
|
||||
if "skipped" in a or "skipped" in b:
|
||||
pass
|
||||
elif a.get("converted"):
|
||||
incomparable[f"dataset {a['converted']}"] += 1
|
||||
elif a.get("shape") != b.get("shape"):
|
||||
issues.append(("mismatch", f"{p}: shape {a.get('shape')} vs ours {b.get('shape')}", "shape", b))
|
||||
ok = False
|
||||
elif a.get("hash") != b.get("hash"):
|
||||
issues.append(("mismatch", f"{p}: values differ (h5py {a.get('dtype')} vs ours {b.get('dtype')})", "values", b | {"ref_head": a.get("head"), "ref_dtype": a.get("dtype")}))
|
||||
ok = False
|
||||
ra, oa = a.get("attrs") or {}, b.get("attrs") or {}
|
||||
if "attrs_error" not in b and "attrs_error" not in a:
|
||||
for an in sorted(set(ra) | set(oa)):
|
||||
x, y = ra.get(an), oa.get(an)
|
||||
if x is None:
|
||||
issues.append(("mismatch", f"{p}@{an}: extra attribute", "extra-attr", y or {}))
|
||||
elif y is None:
|
||||
issues.append(("mismatch", f"{p}@{an}: missing attribute", "missing-attr", x))
|
||||
elif "error" in y and "error" not in x:
|
||||
issues.append(("our-error", f"{p}@{an}: {y['error']}", y["error"], y))
|
||||
elif "error" in x:
|
||||
continue
|
||||
elif x.get("converted"):
|
||||
incomparable[f"attr {x['converted']}"] += 1
|
||||
elif x.get("shape") != y.get("shape"):
|
||||
issues.append(("mismatch", f"{p}@{an}: attr shape {x.get('shape')} vs ours {y.get('shape')}", "attr-shape", y | {"ref_dtype": x.get("dtype")}))
|
||||
elif x.get("hash") != y.get("hash"):
|
||||
issues.append(("mismatch", f"{p}@{an}: attr values differ (h5py {x.get('dtype')} vs ours {y.get('dtype')})", "attr-values", y | {"ref_head": x.get("head"), "ref_dtype": x.get("dtype")}))
|
||||
if ok:
|
||||
n_ok += 1
|
||||
|
||||
# classify
|
||||
if os_ in ("hang", "oom", "crash", "panic"):
|
||||
cls = os_
|
||||
elif caught_panics:
|
||||
cls = "panic"
|
||||
elif ref_open_fail:
|
||||
cls = "h5py-cannot-read"
|
||||
elif ours_open_err:
|
||||
cls = "our-error"
|
||||
issues.append(("our-error", f"open: {ours_open_err}", ours_open_err, {}))
|
||||
elif any(i[0] == "our-error" for i in issues):
|
||||
cls = "our-error"
|
||||
elif issues:
|
||||
cls = "mismatch"
|
||||
else:
|
||||
cls = "ok"
|
||||
|
||||
if os_ in ("hang", "oom", "crash", "panic") or caught_panics:
|
||||
panics.append({
|
||||
"file": rel, "class": cls, "detail": od,
|
||||
"stderr": (ours["err"] if ours else "")[:3000],
|
||||
"caught": [(p, w, m[:2500]) for p, w, m in caught_panics[:3]],
|
||||
"n_caught": len(caught_panics),
|
||||
})
|
||||
for kind, detail, key, rec in issues:
|
||||
if kind == "our-error":
|
||||
add(root_causes, norm(key), rel, detail[:300])
|
||||
else:
|
||||
if key in ("values", "attr-values", "shape", "attr-shape"):
|
||||
mk = f"{key}: ours={rec.get('dtype')} h5py={rec.get('ref_dtype')} layout={rec.get('layout','-')} filters={rec.get('filters','-')}"
|
||||
else:
|
||||
mk = key
|
||||
add(mismatch_causes, mk, rel, detail[:300] + (f" | ref_head={rec.get('ref_head')} our_head={rec.get('head')}" if rec.get("ref_head") else ""))
|
||||
ref_detail = rd if rs != "ok" else ((rj or {}).get("open_error") or "")
|
||||
h5d = ""
|
||||
if h5dump:
|
||||
rc = h5dump["rc"]
|
||||
h5d = {0: "ok", 1: "error", 137: "hang", 124: "hang", 134: "SIGABRT", 139: "SIGSEGV", 136: "SIGFPE", 135: "SIGBUS"}.get(rc, f"rc={rc}")
|
||||
if "memory allocation" in h5dump["err"] or "Cannot allocate" in h5dump["err"]:
|
||||
h5d += "(oom)"
|
||||
rows.append({
|
||||
"file": rel, "corpus": corpus, "class": cls,
|
||||
"ours": os_ if os_ != "ok" else ("open-error" if ours_open_err else ("panic" if caught_panics else "ok")),
|
||||
"ours_detail": (od or ours_open_err or (caught_panics[0][2].split("\n")[0] if caught_panics else ""))[:300],
|
||||
"ref": rs if rs != "ok" else ("open-error" if (rj or {}).get("open_error") else "ok"),
|
||||
"ref_detail": ref_detail[:300],
|
||||
"h5dump_1_14_6": h5d,
|
||||
"h5dump_detail": ([ln for ln in h5dump["err"].splitlines() if ln.strip()][-1:] or [""])[0][:200] if h5dump else "",
|
||||
"objects": n_obj, "objects_ok": n_ok,
|
||||
"issues": len(issues), "first_issue": issues[0][1][:300] if issues else "",
|
||||
"superblock": (oj or {}).get("superblock_version", ""),
|
||||
})
|
||||
# the first issues of each file, for report.py's known-cause matching
|
||||
issues_by_file[rel] = [
|
||||
{"kind": k, "key": key, "detail": det[:300], "ours_dtype": rec.get("dtype"), "ref_dtype": rec.get("ref_dtype")}
|
||||
for k, det, key, rec in issues[:50]
|
||||
]
|
||||
|
||||
with open(os.path.join(R, "results.csv"), "w", newline="") as fh:
|
||||
w = csv.DictWriter(fh, fieldnames=list(rows[0].keys()))
|
||||
w.writeheader()
|
||||
w.writerows(rows)
|
||||
|
||||
|
||||
def ser(b):
|
||||
return {k: {"files": len(v["files"]), "count": v["count"], "examples": v["examples"], "file_list": sorted(v["files"])} for k, v in sorted(b.items(), key=lambda kv: -len(kv[1]["files"]))}
|
||||
|
||||
|
||||
json.dump({"rows": rows, "issues": issues_by_file, "root_causes": ser(root_causes), "mismatch_causes": ser(mismatch_causes),
|
||||
"panics": panics, "incomparable": incomparable.most_common(), "ref_only_errors": ref_only_errors.most_common()},
|
||||
open(os.path.join(R, "results.json"), "w"), indent=1)
|
||||
|
||||
classes = ["ok", "our-error", "mismatch", "h5py-cannot-read", "hang", "panic", "crash", "oom"]
|
||||
by_corpus = collections.defaultdict(collections.Counter)
|
||||
for r in rows:
|
||||
by_corpus[r["corpus"]][r["class"]] += 1
|
||||
by_corpus["ALL"][r["class"]] += 1
|
||||
lines = ["# Conformance sweep summary", "", "| corpus | files | " + " | ".join(classes) + " |", "|---" * (len(classes) + 2) + "|"]
|
||||
for c in sorted(by_corpus, key=lambda k: (k == "ALL", k)):
|
||||
cnt = by_corpus[c]
|
||||
lines.append(f"| {c} | {sum(cnt.values())} | " + " | ".join(str(cnt.get(k, 0)) for k in classes) + " |")
|
||||
lines += ["", "## Panics / hangs / crashes / OOM", ""]
|
||||
for p in panics:
|
||||
lines.append(f"- **{p['file']}** [{p['class']}] {p['detail']}")
|
||||
for path, what, m in p["caught"][:1]:
|
||||
lines.append(" ```\n " + f"{path} ({what}): " + m.replace("\n", "\n ")[:1500] + "\n ```")
|
||||
if not p["caught"] and p["stderr"]:
|
||||
lines.append(" ```\n " + p["stderr"].strip()[:1500].replace("\n", "\n ") + "\n ```")
|
||||
lines += ["", "## Our-error root causes (files affected)", ""]
|
||||
for k, v in ser(root_causes).items():
|
||||
lines.append(f"- [{v['files']} files, {v['count']} objs] `{k}`")
|
||||
for ex in v["examples"][:3]:
|
||||
lines.append(f" - {ex}")
|
||||
lines += ["", "## Mismatch root causes", ""]
|
||||
for k, v in ser(mismatch_causes).items():
|
||||
lines.append(f"- [{v['files']} files, {v['count']} objs] `{k}`")
|
||||
for ex in v["examples"][:3]:
|
||||
lines.append(f" - {ex}")
|
||||
lines += ["", "## Objects h5py fails on but we read (top)", ""]
|
||||
for k, n in ref_only_errors.most_common(15):
|
||||
lines.append(f"- {n} x `{k}`")
|
||||
open(os.path.join(R, "summary.md"), "w").write("\n".join(lines) + "\n")
|
||||
print("\n".join(lines[:4 + len(by_corpus)]))
|
||||
Reference in New Issue
Block a user