osobhandClaude Opus 5.5
9179aa356e
conformance/run.sh fetches eight public HDF5 corpora pinned by commit (conformance/corpus.txt) into a gitignored cache, reads every file with clawhdf5 (conformance/probe, a crate outside the workspace) and with h5py/libhdf5 (ref.py), and the CVE files with h5dump, each under a timeout and an address-space limit; compare.py classifies the files, report.py writes CONFORMANCE.md and check.py gates on panics/hangs/crashes/OOM and on regressions against conformance/baseline.json. ~25 s once cached. Changes from the ad-hoc audit harness: - the probe compares non-IEEE-layout floats (N-Bit) and integers with a bit offset or reduced precision as the values libhdf5 converts them to, not raw file bytes: 8 files that showed as mismatches now read identically; - ref.py exits without tearing down h5py objects: libhdf5 2.0 aborts while freeing them for two files about half the time, which flipped them between ok and h5py-cannot-read from run to run; - the file list is defined (list_files.py): netCDF classic files are left out, 11 HDF5 files the ad-hoc sweep missed are in. Co-Authored-By: Claude Opus 5.5 (1M context) <[email protected]>
89 lines
3.3 KiB
Python
Executable File
89 lines
3.3 KiB
Python
Executable File
#!/usr/bin/env python3
|
|
"""check.py <results_dir> <baseline.json> [--update]
|
|
|
|
The conformance gate. Fails (exit 1) when
|
|
* clawhdf5 panicked, hung, crashed or ran out of memory on any file, or
|
|
* the ok count fell below the baseline's, or
|
|
* a file the baseline lists as ok is no longer ok (even if another file
|
|
became ok and the total held).
|
|
New ok files are reported so the baseline can be raised (--update rewrites it
|
|
from the results).
|
|
"""
|
|
import json
|
|
import os
|
|
import sys
|
|
|
|
FATAL = ("panic", "hang", "crash", "oom")
|
|
|
|
|
|
def main():
|
|
args = [a for a in sys.argv[1:] if not a.startswith("--")]
|
|
update = "--update" in sys.argv
|
|
res_dir, base_path = args
|
|
res = json.load(open(os.path.join(res_dir, "results.json")))
|
|
rows = res["rows"]
|
|
counts = {}
|
|
per_corpus = {}
|
|
for r in rows:
|
|
counts[r["class"]] = counts.get(r["class"], 0) + 1
|
|
pc = per_corpus.setdefault(r["corpus"], {})
|
|
pc[r["class"]] = pc.get(r["class"], 0) + 1
|
|
ok_files = sorted(r["file"] for r in rows if r["class"] == "ok")
|
|
|
|
if update:
|
|
meta = {}
|
|
mp = os.path.join(res_dir, "report-meta.json")
|
|
if os.path.exists(mp):
|
|
meta = json.load(open(mp))
|
|
base = {
|
|
"comment": "conformance/run.sh fails if the ok count drops below `ok` or a file in `ok_files` stops being ok. "
|
|
"Regenerate with `conformance/run.sh --update-baseline` after an intended change.",
|
|
"commit": meta.get("commit", ""),
|
|
"date": meta.get("date", ""),
|
|
"reference": meta.get("reference", ""),
|
|
"files": len(rows),
|
|
"ok": len(ok_files),
|
|
"counts": dict(sorted(counts.items())),
|
|
"per_corpus": {k: dict(sorted(v.items())) for k, v in sorted(per_corpus.items())},
|
|
"ok_files": ok_files,
|
|
}
|
|
with open(base_path, "w") as fh:
|
|
json.dump(base, fh, indent=1)
|
|
fh.write("\n")
|
|
print(f"baseline updated: {len(ok_files)} ok of {len(rows)} files -> {base_path}")
|
|
return 0
|
|
|
|
base = json.load(open(base_path))
|
|
failures = []
|
|
fatal = [r for r in rows if r["class"] in FATAL]
|
|
for r in fatal:
|
|
failures.append(f"{r['class']}: {r['file']}: {r['ours_detail'][:200]}")
|
|
if len(ok_files) < base["ok"]:
|
|
failures.append(f"ok count dropped: {len(ok_files)} < baseline {base['ok']}")
|
|
now_ok = set(ok_files)
|
|
by_file = {r["file"]: r for r in rows}
|
|
for f in base["ok_files"]:
|
|
if f not in now_ok:
|
|
r = by_file.get(f)
|
|
why = f"now {r['class']}: {(r['ours_detail'] or r['first_issue'])[:200]}" if r else "no longer in the corpus"
|
|
failures.append(f"regressed: {f}: {why}")
|
|
gained = sorted(now_ok - set(base["ok_files"]))
|
|
|
|
print(f"conformance: {len(ok_files)} ok of {len(rows)} files (baseline {base['ok']} of {base['files']}); "
|
|
+ ", ".join(f"{k} {v}" for k, v in sorted(counts.items())))
|
|
if gained:
|
|
print(f"{len(gained)} file(s) newly ok — raise the baseline with `conformance/run.sh --update-baseline`:")
|
|
for f in gained:
|
|
print(f" + {f}")
|
|
if failures:
|
|
print(f"CONFORMANCE GATE FAILED ({len(failures)}):")
|
|
for f in failures:
|
|
print(f" - {f}")
|
|
return 1
|
|
print("conformance gate passed")
|
|
return 0
|
|
|
|
|
|
if __name__ == "__main__":
|
|
sys.exit(main())
|