clawhdf5/conformance/report.py

#!/usr/bin/env python3
"""report.py <results_dir> <CONFORMANCE.md> <corpus_dir>

Render the sweep's results (compare.py's results.json plus the raw per-side
runs) as CONFORMANCE.md, and write <results_dir>/report-meta.json (commit,
date, versions) for check.py --update.
"""
import collections
import datetime
import json
import os
import platform

import subprocess
import sys

import h5py
import numpy

try:
    import hdf5plugin
    HDF5PLUGIN = hdf5plugin.version
except Exception:  # noqa: BLE001
    HDF5PLUGIN = "not installed"

R, OUT_MD, CORPUS = sys.argv[1], sys.argv[2], sys.argv[3]
HERE = os.path.dirname(os.path.abspath(__file__))
ROOT = os.path.dirname(HERE)
CLASSES = ["ok", "our-error", "mismatch", "h5py-cannot-read", "ref-bug", "panic", "hang", "crash", "oom"]


def sh(*cmd, cwd=ROOT):
    try:
        return subprocess.run(cmd, cwd=cwd, capture_output=True, text=True, timeout=30).stdout.strip()
    except Exception:  # noqa: BLE001
        return ""


def cpu_model():
    try:
        for ln in open("/proc/cpuinfo"):
            if ln.startswith(("model name", "Model")):
                return ln.split(":", 1)[1].strip()
    except OSError:
        pass
    return platform.processor() or "unknown"


def mem_gib():
    try:
        for ln in open("/proc/meminfo"):
            if ln.startswith("MemTotal:"):
                return f"{int(ln.split()[1]) / 1048576:.0f} GiB"
    except OSError:
        pass
    return "?"


res = json.load(open(os.path.join(R, "results.json")))
meta_run = json.load(open(os.path.join(R, "meta.json"))) if os.path.exists(os.path.join(R, "meta.json")) else {}
rows = res["rows"]
issues = res.get("issues", {})

# safe.directory: a checkout owned by another user (a container) is still ours to read
commit = sh("git", "-c", "safe.directory=*", "rev-parse", "HEAD") or os.environ.get("GITHUB_SHA", "unknown")
lib_dirty = sh("git", "-c", "safe.directory=*", "status", "--porcelain", "--", "crates", "Cargo.toml")
h5dump_v = sh("h5dump", "--version").replace("h5dump: ", "")
meta = {
    "date": datetime.datetime.now(datetime.timezone.utc).strftime("%Y-%m-%d %H:%M UTC"),
    "commit": commit + (" (library sources modified)" if lib_dirty else ""),
    "reference": f"h5py {h5py.__version__} / HDF5 {h5py.version.hdf5_version}",
}
json.dump(meta, open(os.path.join(R, "report-meta.json"), "w"), indent=1)

pins = []
for ln in open(os.path.join(HERE, "corpus.txt")):
    if ln.strip() and not ln.lstrip().startswith("#"):
        name, url, rev, root, *_ = ln.split()
        pins.append((name, url, rev, root))

by_corpus = collections.defaultdict(collections.Counter)
for r in rows:
    by_corpus[r["corpus"]][r["class"]] += 1
total = collections.Counter(r["class"] for r in rows)


def ex_list(files, n=3):
    s = ", ".join(f"`{f}`" for f in files[:n])
    return s + (f" (+{len(files) - n} more)" if len(files) > n else "")


# --- reference bugs ---------------------------------------------------------
# ref_bugs.py's re-check of the objects h5py reads only through a libhdf5 bug
# (compare.py classifies on the confirmed ones), and the objects whose h5py
# values ref.py corrected (compare.py's ref_fixes).
try:
    ref_bugs = json.load(open(os.path.join(R, "ref_bugs.json")))["read_bugs"]
except (OSError, ValueError, KeyError):
    ref_bugs = []
ref_fixes = res.get("ref_fixes", [])


# --- the CVE corpus: clawhdf5 vs h5dump vs h5py ------------------------------
def side(run, name):
    p = os.path.join(R, "runs", run, name)
    if not os.path.exists(p + ".rc"):
        return None
    rc = int(open(p + ".rc").read().strip() or -1)
    err = open(p + ".err", errors="replace").read()
    try:
        j = json.load(open(p + ".json"))
    except Exception:  # noqa: BLE001
        j = None
    return rc, err, j


def outcome(s, rust=False):
    """-> (bucket, text). bucket in read / error / panic / crash / hang / oom."""
    if s is None:
        return "missing", "not run"
    rc, err, j = s
    if rc in (137, 124):
        return "hang", "hang (killed at timeout)"
    if "memory allocation of" in err or "MemoryError" in err or "bad_alloc" in err or "Cannot allocate" in err:
        return "oom", "out of memory"
    if rust and (rc == 101 or "PANIC:" in err):
        return "panic", "panic"
    if "overflowed its stack" in err:
        return "crash", "stack overflow"
    if rc == 139:
        return "crash", "SIGSEGV"
    if rc == 134:
        return "crash", "SIGABRT" + (" (heap corruption)" if ("corrupted" in err or "free()" in err) else "")
    if rc > 128:
        return "crash", f"signal {rc - 128}"
    if j is None:
        return ("error", "error exit") if rc in (0, 1) else ("crash", f"exit {rc}")
    if "open_error" in j:
        return "error", "open error"
    objs = j.get("objects", [])
    ne = sum(1 for o in objs for k in ("error", "attrs_error", "list_error") if k in o)
    ne += sum(1 for o in objs for a in (o.get("attrs") or {}).values() if "error" in a)
    return "read", f"read {len(objs)} obj" + (f", {ne} errors" if ne else "")


def h5dump_outcome(s):
    if s is None:
        return "missing", "not run"
    rc, err, _ = s
    if rc in (137, 124):
        return "hang", "hang (killed at timeout)"
    if "memory allocation" in err or "Cannot allocate" in err:
        return "oom", "out of memory"
    if rc == 139:
        return "crash", "SIGSEGV"
    if rc == 134:
        return "crash", "SIGABRT" + (" (heap corruption)" if ("corrupted" in err or "free()" in err) else "")
    if rc > 128:
        return "crash", f"signal {rc - 128}"
    return ("read", "ok") if rc == 0 else ("error", "error exit")


cve_rows = []
buckets = {"clawhdf5": collections.Counter(), "h5dump": collections.Counter(), "h5py": collections.Counter()}
ours_panic = {r["file"] for r in rows if r["class"] == "panic"}
for r in rows:
    if r["corpus"] != "cve_hdf5":
        continue
    run = r["file"].replace("/", "__")
    o = outcome(side(run, "ours"), rust=True)
    if o[0] == "read" and r["file"] in ours_panic:
        o = ("panic", "caught panic")
    p = outcome(side(run, "ref"))
    d = h5dump_outcome(side(run, "h5dump"))
    buckets["clawhdf5"][o[0]] += 1
    buckets["h5py"][p[0]] += 1
    buckets["h5dump"][d[0]] += 1
    cve_rows.append((r["file"].split("/", 1)[1], d[1], p[1], o[1], r["class"]))

# --- render -----------------------------------------------------------------
L = []
w = L.append
w("# clawhdf5 conformance report")
w("")
w("Every HDF5 file of eight public corpora (pinned by commit) is read twice — by")
w("clawhdf5 (`conformance/probe`, the same `clawhdf5-format` calls the facade")
w("makes) and by h5py/libhdf5 (`conformance/ref.py`) — and the two readings are")
w("compared object by object: the set of hard-linked objects, each dataset's and")
w("attribute's shape, and a SHA-256 of its values in a canonical encoding. The")
w("CVE corpus is also run through `h5dump`. Each side runs under a timeout and an")
w("address-space limit, so a hang, crash or runaway allocation is recorded, not")
w("fatal. This file is generated by `conformance/run.sh`; do not edit it by hand.")
w("")
w("## Run")
w("")
w("| | |")
w("|---|---|")
w(f"| date | {meta['date']} |")
w(f"| clawhdf5 commit | `{meta['commit']}` |")
w(f"| machine | `{platform.node()}`: {cpu_model()}, {os.cpu_count()} CPUs, {mem_gib()}, {platform.system()} {platform.release()} {platform.machine()} |")
w(f"| command | `{os.environ.get('CONFORMANCE_CMD', 'conformance/run.sh')}` |")
w(f"| rustc | {sh('rustc', '-V')} |")
w(f"| reference | h5py {h5py.__version__}, HDF5 {h5py.version.hdf5_version}, numpy {numpy.__version__}, hdf5plugin {HDF5PLUGIN}, Python {platform.python_version()} |")
w(f"| h5dump | {h5dump_v} (CVE corpus only) |")
if meta_run:
    w(f"| limits | {meta_run.get('timeout_s')} s timeout (SIGKILL), {int(meta_run.get('mem_kb', 0)) // 1024} MiB address space, per process; {meta_run.get('jobs')} files in parallel |")
    w(f"| runtime | {meta_run.get('probe_seconds')} s probing + comparing ({meta_run.get('build_seconds')} s fetch/build before it) |")
w("")
w("## Results")
w("")
w("A file's class is the first that applies:")
w("")
w("- **panic / hang / crash / oom** — clawhdf5 panicked (caught per object or not), hit the timeout, died on a signal, or failed an allocation. The CI gate fails on any of these.")
w("- **h5py-cannot-read** — libhdf5 could not open the file (or itself crashed or hung). Nothing to compare against; most are the deliberately malformed CVE reproducers.")
w("- **ref-bug** — every difference is an object clawhdf5 refuses that h5py reads only through a libhdf5 bug: the values h5py returns for it change with the reading process's heap, re-checked in every run (see *Reference bugs*).")
w("- **our-error** — clawhdf5 returned an error for something h5py reads.")
w("- **mismatch** — both read it, but the shapes, values, object set or attribute set differ.")
w("- **ok** — every object h5py reads, clawhdf5 reads identically.")
w("")
w("| corpus | files | " + " | ".join(CLASSES) + " |")
w("|---" * (len(CLASSES) + 2) + "|")
for c in sorted(by_corpus):
    cnt = by_corpus[c]
    w(f"| {c} | {sum(cnt.values())} | " + " | ".join(str(cnt.get(k, 0)) for k in CLASSES) + " |")
w(f"| **all** | **{len(rows)}** | " + " | ".join(f"**{total.get(k, 0)}**" for k in CLASSES) + " |")
w("")
nonok = total.get("our-error", 0) + total.get("mismatch", 0)
w(f"**Our errors and mismatches: {nonok}.** "
  + ("Every file clawhdf5 does not read like h5py is either unreadable by h5py or a confirmed libhdf5 "
     "bug (*ref-bug*)." if nonok == 0 else "See the root causes below.")
  + (f" {len(ref_fixes)} object(s) were compared against h5py's values corrected for a known h5py bug"
     f" ({sum(1 for x in ref_fixes if x[3])} identical to clawhdf5's; see *Reference bugs*)." if ref_fixes else ""))
w("")
w("Corpora (fetched by `conformance/fetch-corpus.sh` into the gitignored `conformance/.cache/`):")
w("")
w("| corpus | source | commit |")
w("|---|---|---|")
for name, url, rev, root in pins:
    w(f"| {name} | {url.removesuffix('.git')}" + ("" if root == "." else f" (`{root}`)") + f" | `{rev[:12]}` |")
w("")

w("## Panics, hangs, crashes, out-of-memory")
w("")
if not res["panics"]:
    w("None.")
else:
    for p in res["panics"]:
        w(f"- `{p['file']}` [{p['class']}] {p['detail']}")
w("")

w("## Our-error root causes")
w("")
w("Grouped by normalised error message. *files* counts files whose class this cause affects.")
w("")
w("| files | objects | error | examples |")
w("|---:|---:|---|---|")
for k, v in res["root_causes"].items():
    w(f"| {v['files']} | {v['count']} | `{k.replace('|', '/')}` | {ex_list(v['file_list'])} |")
w("")
w("## Mismatch root causes")
w("")
w("| files | objects | cause | examples |")
w("|---:|---:|---|---|")
for k, v in res["mismatch_causes"].items():
    w(f"| {v['files']} | {v['count']} | `{k.replace('|', '/')}` | {ex_list(v['file_list'])} |")
w("")

w("## CVE corpus: clawhdf5 vs h5dump vs h5py")
w("")
w(f"The {len(cve_rows)} files of [HDFGroup/cve_hdf5](https://github.com/HDFGroup/cve_hdf5) — reproducers for")
w("published libhdf5 CVEs and fuzzer finds. *read* = produced output (possibly with per-object")
w("errors), *error* = refused cleanly. h5dump exits non-zero on any error anywhere in a file, so")
w("its read/error split is not comparable with the other two rows; the panic, crash, hang and oom")
w("columns are.")
w("")
w("| tool | read | error | panic | crash | hang | oom |")
w("|---|---:|---:|---:|---:|---:|---:|")
for tool, label in (("clawhdf5", "clawhdf5"), ("h5dump", f"h5dump {h5dump_v.split()[-1] if h5dump_v else ''}"),
                    ("h5py", f"h5py {h5py.__version__} / HDF5 {h5py.version.hdf5_version}")):
    b = buckets[tool]
    w(f"| {label} | " + " | ".join(str(b.get(k, 0)) for k in ("read", "error", "panic", "crash", "hang", "oom")) + " |")
w("")
w("<details><summary>Per-file outcomes</summary>")
w("")
w("| file | h5dump | h5py | clawhdf5 | class |")
w("|---|---|---|---|---|")
for f, d, p, o, cls in cve_rows:
    w(f"| {f} | {d} | {p} | {o} | {cls} |")
w("")
w("</details>")
w("")

w("## Reference bugs")
w("")
w("### Objects h5py reads only through a libhdf5 bug (*ref-bug*)")
w("")
w("clawhdf5 refuses these objects; h5py 3.16 / HDF5 2.0 returns values for them. `conformance/ref_bugs.py`")
w("re-reads each with h5py in six fresh processes whose heaps differ (h5py imported before numpy, three")
w("times and twice more with `MALLOC_PERTURB_`, and numpy imported first). Values the file determines")
w("come out the same every time; these do not, so they are memory libhdf5 over-reads, not the file's")
w("data. A file is *ref-bug* only while every one of its differences is such an object confirmed in")
w("the same run; an object that reads the same every time goes back to *our-error*. Reproducer:")
w("`python conformance/ref_bugs.py conformance/.cache/corpus` (prints every read's outcome).")
w("")
w("| file | object | distinct results in 6 reads | confirmed | what goes wrong |")
w("|---|---|---:|---|---|")
for b in ref_bugs:
    n = "missing" if b.get("missing") else b.get("distinct", "?")
    w(f"| `{b['file']}` | `{b['object']}` | {n} | {'yes' if b.get('confirmed') else '**no**'} | {b['why']} |")
w("")
w("### Values corrected for a known h5py bug")
w("")
w("- **h5py big-endian variable-length sequences.** h5py returns the elements of a VL sequence")
w("  whose base type is big-endian with the file's big-endian bytes but a native (little-endian)")
w("  numpy dtype: a `h5py.vlen_dtype(np.dtype('>f4'))` dataset holding `[1.0, 2.0]` reads back as")
w("  `[4.6e-41, 9.0e-44]`; `h5dump` prints the file's values. `ref.py` checks that the installed")
w("  h5py still does this (by writing and reading exactly that dataset in memory) and, if so,")
w("  relabels such elements with the file's byte order before hashing, so the values are still")
w("  compared. Corrected objects: "
  + (", ".join(f"`{f}` `{p}` ({'same as clawhdf5' if same else '**differs from clawhdf5**'})"
               for f, p, _, same in ref_fixes) if ref_fixes else "none") + ".")
w("")
w("## Other comparison rules")
w("")
w("- **Non-IEEE floats and partial-precision integers (N-Bit).** libhdf5 converts a float whose")
w("  bit layout is not IEEE (e.g. `H5Tset_precision` for the N-Bit filter) or an integer with a")
w("  bit offset / reduced precision into the plain numpy type of the same size. The probe")
w("  compares such values as converted numbers, not raw file bytes (before 2026-09-25 it compared")
w("  raw bytes, which reported every N-Bit float dataset as a mismatch).")
if res["incomparable"]:
    w("- **Types h5py widens.** Where h5py reads a type into a numpy type of a different size")
    w("  (FP8 -> float16, bfloat16 -> float32, x87 long double -> float128) the values are not")
    w("  compared (shape and presence still are): "
      + ", ".join(f"{k} ({n}x)" for k, n in res["incomparable"]) + ".")
w("- **References** are compared by presence only (`R`), not by target.")
w("")
if res.get("ref_only_errors"):
    w("## Objects h5py fails on but clawhdf5 reads")
    w("")
    for k, n in res["ref_only_errors"][:15]:
        w(f"- {n} x `{k}`")
    w("")
w("## Reproduce")
w("")
w("```sh")
w("# needs: Rust, python3 with h5py numpy hdf5plugin (conformance/requirements.txt), h5dump (hdf5-tools), git")
w("CLAWHDF5_PYTHON=/path/to/venv/bin/python conformance/run.sh")
w("```")
w("")
w("The corpus (about 450 MB of sparse checkouts) is cached in `conformance/.cache/`; results for")
w("every file, both sides' raw JSON and stderr, are in `conformance/.cache/results/`.")
w("`conformance/baseline.json` holds the ok files the nightly CI job (`.gitea/workflows/conformance.yml`)")
w("must keep; `conformance/run.sh --update-baseline` rewrites it.")

with open(OUT_MD, "w") as fh:
    fh.write("\n".join(L) + "\n")