diff --git a/conformance/compare.py b/conformance/compare.py index b3c944a..ec568b5 100755 --- a/conformance/compare.py +++ b/conformance/compare.py @@ -239,7 +239,8 @@ for rel in files: "caught": [(p, w, m[:2500]) for p, w, m in caught_panics[:3]], "n_caught": len(caught_panics), }) - for kind, detail, key, rec in issues: + # A ref-bug file's differences are listed with the evidence instead. + for kind, detail, key, rec in (issues if cls != "ref-bug" else []): if kind == "our-error": add(root_causes, norm(key), rel, detail[:300]) else: diff --git a/conformance/report.py b/conformance/report.py index 6c4cb13..0770936 100644 --- a/conformance/report.py +++ b/conformance/report.py @@ -225,9 +225,8 @@ for c in sorted(by_corpus): w(f"| **all** | **{len(rows)}** | " + " | ".join(f"**{total.get(k, 0)}**" for k in CLASSES) + " |") w("") nonok = total.get("our-error", 0) + total.get("mismatch", 0) -w(f"**Our errors and mismatches: {nonok}.** " - + ("Every file clawhdf5 does not read like h5py is either unreadable by h5py or a confirmed libhdf5 " - "bug (*ref-bug*)." if nonok == 0 else "See the root causes below.") +w(f"**Our errors and mismatches: {nonok}.** Files not ok: " + + (", ".join(f"{total[c]} {c}" for c in CLASSES if c != "ok" and total.get(c)) or "none") + "." + (f" {len(ref_fixes)} object(s) were compared against h5py's values corrected for a known h5py bug" f" ({sum(1 for x in ref_fixes if x[3])} identical to clawhdf5's; see *Reference bugs*)." if ref_fixes else "")) w("") @@ -250,19 +249,25 @@ w("") w("## Our-error root causes") w("") -w("Grouped by normalised error message. *files* counts files whose class this cause affects.") -w("") -w("| files | objects | error | examples |") -w("|---:|---:|---|---|") -for k, v in res["root_causes"].items(): - w(f"| {v['files']} | {v['count']} | `{k.replace('|', '/')}` | {ex_list(v['file_list'])} |") +if res["root_causes"]: + w("Grouped by normalised error message. *files* counts files whose class this cause affects.") + w("") + w("| files | objects | error | examples |") + w("|---:|---:|---|---|") + for k, v in res["root_causes"].items(): + w(f"| {v['files']} | {v['count']} | `{k.replace('|', '/')}` | {ex_list(v['file_list'])} |") +else: + w("None.") w("") w("## Mismatch root causes") w("") -w("| files | objects | cause | examples |") -w("|---:|---:|---|---|") -for k, v in res["mismatch_causes"].items(): - w(f"| {v['files']} | {v['count']} | `{k.replace('|', '/')}` | {ex_list(v['file_list'])} |") +if res["mismatch_causes"]: + w("| files | objects | cause | examples |") + w("|---:|---:|---|---|") + for k, v in res["mismatch_causes"].items(): + w(f"| {v['files']} | {v['count']} | `{k.replace('|', '/')}` | {ex_list(v['file_list'])} |") +else: + w("None.") w("") w("## CVE corpus: clawhdf5 vs h5dump vs h5py")