clawhdf5/conformance/report.py

#!/usr/bin/env python3
"""report.py <results_dir> <CONFORMANCE.md> <corpus_dir>

Render the sweep's results (compare.py's results.json plus the raw per-side
runs) as CONFORMANCE.md, and write <results_dir>/report-meta.json (commit,
date, versions) for check.py --update.
"""
import collections
import datetime
import json
import os
import platform

import subprocess
import sys

import h5py
import numpy

try:
    import hdf5plugin
    HDF5PLUGIN = hdf5plugin.version
except Exception:  # noqa: BLE001
    HDF5PLUGIN = "not installed"

R, OUT_MD, CORPUS = sys.argv[1], sys.argv[2], sys.argv[3]
HERE = os.path.dirname(os.path.abspath(__file__))
ROOT = os.path.dirname(HERE)
CLASSES = ["ok", "our-error", "mismatch", "h5py-cannot-read", "panic", "hang", "crash", "oom"]


def sh(*cmd, cwd=ROOT):
    try:
        return subprocess.run(cmd, cwd=cwd, capture_output=True, text=True, timeout=30).stdout.strip()
    except Exception:  # noqa: BLE001
        return ""


def cpu_model():
    try:
        for ln in open("/proc/cpuinfo"):
            if ln.startswith(("model name", "Model")):
                return ln.split(":", 1)[1].strip()
    except OSError:
        pass
    return platform.processor() or "unknown"


def mem_gib():
    try:
        for ln in open("/proc/meminfo"):
            if ln.startswith("MemTotal:"):
                return f"{int(ln.split()[1]) / 1048576:.0f} GiB"
    except OSError:
        pass
    return "?"


res = json.load(open(os.path.join(R, "results.json")))
meta_run = json.load(open(os.path.join(R, "meta.json"))) if os.path.exists(os.path.join(R, "meta.json")) else {}
rows = res["rows"]
issues = res.get("issues", {})

# safe.directory: a checkout owned by another user (a container) is still ours to read
commit = sh("git", "-c", "safe.directory=*", "rev-parse", "HEAD") or os.environ.get("GITHUB_SHA", "unknown")
lib_dirty = sh("git", "-c", "safe.directory=*", "status", "--porcelain", "--", "crates", "Cargo.toml")
h5dump_v = sh("h5dump", "--version").replace("h5dump: ", "")
meta = {
    "date": datetime.datetime.now(datetime.timezone.utc).strftime("%Y-%m-%d %H:%M UTC"),
    "commit": commit + (" (library sources modified)" if lib_dirty else ""),
    "reference": f"h5py {h5py.__version__} / HDF5 {h5py.version.hdf5_version}",
}
json.dump(meta, open(os.path.join(R, "report-meta.json"), "w"), indent=1)

pins = []
for ln in open(os.path.join(HERE, "corpus.txt")):
    if ln.strip() and not ln.lstrip().startswith("#"):
        name, url, rev, root, *_ = ln.split()
        pins.append((name, url, rev, root))

by_corpus = collections.defaultdict(collections.Counter)
for r in rows:
    by_corpus[r["corpus"]][r["class"]] += 1
total = collections.Counter(r["class"] for r in rows)


def ex_list(files, n=3):
    s = ", ".join(f"`{f}`" for f in files[:n])
    return s + (f" (+{len(files) - n} more)" if len(files) > n else "")


# --- known causes that are not clawhdf5 bugs --------------------------------
def is_h5py_be_vlen(i):
    """h5py returns the elements of a VL sequence of a big-endian base type
    with their file (big-endian) bytes but a native-endian dtype."""
    return (i["kind"] == "mismatch" and i["key"] in ("values", "attr-values")
            and (i.get("ref_dtype") == "object") and (i.get("ours_dtype") or "").startswith("vlen(")
            and ">" in (i.get("ours_dtype") or ""))


known = collections.defaultdict(list)
for r in rows:
    if r["class"] != "mismatch":
        continue
    iss = issues.get(r["file"], [])
    if iss and all(is_h5py_be_vlen(i) for i in iss):
        known["h5py-be-vlen"].append(r["file"])


# --- the CVE corpus: clawhdf5 vs h5dump vs h5py ------------------------------
def side(run, name):
    p = os.path.join(R, "runs", run, name)
    if not os.path.exists(p + ".rc"):
        return None
    rc = int(open(p + ".rc").read().strip() or -1)
    err = open(p + ".err", errors="replace").read()
    try:
        j = json.load(open(p + ".json"))
    except Exception:  # noqa: BLE001
        j = None
    return rc, err, j


def outcome(s, rust=False):
    """-> (bucket, text). bucket in read / error / panic / crash / hang / oom."""
    if s is None:
        return "missing", "not run"
    rc, err, j = s
    if rc in (137, 124):
        return "hang", "hang (killed at timeout)"
    if "memory allocation of" in err or "MemoryError" in err or "bad_alloc" in err or "Cannot allocate" in err:
        return "oom", "out of memory"
    if rust and (rc == 101 or "PANIC:" in err):
        return "panic", "panic"
    if "overflowed its stack" in err:
        return "crash", "stack overflow"
    if rc == 139:
        return "crash", "SIGSEGV"
    if rc == 134:
        return "crash", "SIGABRT" + (" (heap corruption)" if ("corrupted" in err or "free()" in err) else "")
    if rc > 128:
        return "crash", f"signal {rc - 128}"
    if j is None:
        return ("error", "error exit") if rc in (0, 1) else ("crash", f"exit {rc}")
    if "open_error" in j:
        return "error", "open error"
    objs = j.get("objects", [])
    ne = sum(1 for o in objs for k in ("error", "attrs_error", "list_error") if k in o)
    ne += sum(1 for o in objs for a in (o.get("attrs") or {}).values() if "error" in a)
    return "read", f"read {len(objs)} obj" + (f", {ne} errors" if ne else "")


def h5dump_outcome(s):
    if s is None:
        return "missing", "not run"
    rc, err, _ = s
    if rc in (137, 124):
        return "hang", "hang (killed at timeout)"
    if "memory allocation" in err or "Cannot allocate" in err:
        return "oom", "out of memory"
    if rc == 139:
        return "crash", "SIGSEGV"
    if rc == 134:
        return "crash", "SIGABRT" + (" (heap corruption)" if ("corrupted" in err or "free()" in err) else "")
    if rc > 128:
        return "crash", f"signal {rc - 128}"
    return ("read", "ok") if rc == 0 else ("error", "error exit")


cve_rows = []
buckets = {"clawhdf5": collections.Counter(), "h5dump": collections.Counter(), "h5py": collections.Counter()}
ours_panic = {r["file"] for r in rows if r["class"] == "panic"}
for r in rows:
    if r["corpus"] != "cve_hdf5":
        continue
    run = r["file"].replace("/", "__")
    o = outcome(side(run, "ours"), rust=True)
    if o[0] == "read" and r["file"] in ours_panic:
        o = ("panic", "caught panic")
    p = outcome(side(run, "ref"))
    d = h5dump_outcome(side(run, "h5dump"))
    buckets["clawhdf5"][o[0]] += 1
    buckets["h5py"][p[0]] += 1
    buckets["h5dump"][d[0]] += 1
    cve_rows.append((r["file"].split("/", 1)[1], d[1], p[1], o[1], r["class"]))

# --- render -----------------------------------------------------------------
L = []
w = L.append
w("# clawhdf5 conformance report")
w("")
w("Every HDF5 file of eight public corpora (pinned by commit) is read twice — by")
w("clawhdf5 (`conformance/probe`, the same `clawhdf5-format` calls the facade")
w("makes) and by h5py/libhdf5 (`conformance/ref.py`) — and the two readings are")
w("compared object by object: the set of hard-linked objects, each dataset's and")
w("attribute's shape, and a SHA-256 of its values in a canonical encoding. The")
w("CVE corpus is also run through `h5dump`. Each side runs under a timeout and an")
w("address-space limit, so a hang, crash or runaway allocation is recorded, not")
w("fatal. This file is generated by `conformance/run.sh`; do not edit it by hand.")
w("")
w("## Run")
w("")
w("| | |")
w("|---|---|")
w(f"| date | {meta['date']} |")
w(f"| clawhdf5 commit | `{meta['commit']}` |")
w(f"| machine | `{platform.node()}`: {cpu_model()}, {os.cpu_count()} CPUs, {mem_gib()}, {platform.system()} {platform.release()} {platform.machine()} |")
w(f"| command | `{os.environ.get('CONFORMANCE_CMD', 'conformance/run.sh')}` |")
w(f"| rustc | {sh('rustc', '-V')} |")
w(f"| reference | h5py {h5py.__version__}, HDF5 {h5py.version.hdf5_version}, numpy {numpy.__version__}, hdf5plugin {HDF5PLUGIN}, Python {platform.python_version()} |")
w(f"| h5dump | {h5dump_v} (CVE corpus only) |")
if meta_run:
    w(f"| limits | {meta_run.get('timeout_s')} s timeout (SIGKILL), {int(meta_run.get('mem_kb', 0)) // 1024} MiB address space, per process; {meta_run.get('jobs')} files in parallel |")
    w(f"| runtime | {meta_run.get('probe_seconds')} s probing + comparing ({meta_run.get('build_seconds')} s fetch/build before it) |")
w("")
w("## Results")
w("")
w("A file's class is the first that applies:")
w("")
w("- **panic / hang / crash / oom** — clawhdf5 panicked (caught per object or not), hit the timeout, died on a signal, or failed an allocation. The CI gate fails on any of these.")
w("- **h5py-cannot-read** — libhdf5 could not open the file (or itself crashed or hung). Nothing to compare against; most are the deliberately malformed CVE reproducers.")
w("- **our-error** — clawhdf5 returned an error for something h5py reads.")
w("- **mismatch** — both read it, but the shapes, values, object set or attribute set differ.")
w("- **ok** — every object h5py reads, clawhdf5 reads identically.")
w("")
w("| corpus | files | " + " | ".join(CLASSES) + " |")
w("|---" * (len(CLASSES) + 2) + "|")
for c in sorted(by_corpus):
    cnt = by_corpus[c]
    w(f"| {c} | {sum(cnt.values())} | " + " | ".join(str(cnt.get(k, 0)) for k in CLASSES) + " |")
w(f"| **all** | **{len(rows)}** | " + " | ".join(f"**{total.get(k, 0)}**" for k in CLASSES) + " |")
w("")
n_known = sum(len(v) for v in known.values())
if n_known:
    w(f"{n_known} of the {total.get('mismatch', 0)} mismatches are a known h5py bug, not ours (see *Known not-our-bug*).")
    w("")
w("Corpora (fetched by `conformance/fetch-corpus.sh` into the gitignored `conformance/.cache/`):")
w("")
w("| corpus | source | commit |")
w("|---|---|---|")
for name, url, rev, root in pins:
    w(f"| {name} | {url.removesuffix('.git')}" + ("" if root == "." else f" (`{root}`)") + f" | `{rev[:12]}` |")
w("")

w("## Panics, hangs, crashes, out-of-memory")
w("")
if not res["panics"]:
    w("None.")
else:
    for p in res["panics"]:
        w(f"- `{p['file']}` [{p['class']}] {p['detail']}")
w("")

w("## Our-error root causes")
w("")
w("Grouped by normalised error message. *files* counts files whose class this cause affects.")
w("")
w("| files | objects | error | examples |")
w("|---:|---:|---|---|")
for k, v in res["root_causes"].items():
    w(f"| {v['files']} | {v['count']} | `{k.replace('|', '/')}` | {ex_list(v['file_list'])} |")
w("")
w("## Mismatch root causes")
w("")
w("| files | objects | cause | examples |")
w("|---:|---:|---|---|")
for k, v in res["mismatch_causes"].items():
    w(f"| {v['files']} | {v['count']} | `{k.replace('|', '/')}` | {ex_list(v['file_list'])} |")
w("")

w("## CVE corpus: clawhdf5 vs h5dump vs h5py")
w("")
w(f"The {len(cve_rows)} files of [HDFGroup/cve_hdf5](https://github.com/HDFGroup/cve_hdf5) — reproducers for")
w("published libhdf5 CVEs and fuzzer finds. *read* = produced output (possibly with per-object")
w("errors), *error* = refused cleanly. h5dump exits non-zero on any error anywhere in a file, so")
w("its read/error split is not comparable with the other two rows; the panic, crash, hang and oom")
w("columns are.")
w("")
w("| tool | read | error | panic | crash | hang | oom |")
w("|---|---:|---:|---:|---:|---:|---:|")
for tool, label in (("clawhdf5", "clawhdf5"), ("h5dump", f"h5dump {h5dump_v.split()[-1] if h5dump_v else ''}"),
                    ("h5py", f"h5py {h5py.__version__} / HDF5 {h5py.version.hdf5_version}")):
    b = buckets[tool]
    w(f"| {label} | " + " | ".join(str(b.get(k, 0)) for k in ("read", "error", "panic", "crash", "hang", "oom")) + " |")
w("")
w("<details><summary>Per-file outcomes</summary>")
w("")
w("| file | h5dump | h5py | clawhdf5 | class |")
w("|---|---|---|---|---|")
for f, d, p, o, cls in cve_rows:
    w(f"| {f} | {d} | {p} | {o} | {cls} |")
w("")
w("</details>")
w("")

w("## Known not-our-bug")
w("")
w("- **h5py big-endian variable-length sequences.** h5py returns the elements of a VL sequence")
w("  whose base type is big-endian with the file's big-endian bytes but a native (little-endian)")
w("  numpy dtype, so the values it reports are byte-swapped garbage; `h5dump` prints the values")
w("  clawhdf5 reads. Reproducer: `h5py.vlen_dtype(np.dtype('>f4'))` dataset holding `[1.0, 2.0]`")
w("  reads back in h5py as `[4.6e-41, 9.0e-44]`. Affected here: "
  + (ex_list(sorted(known["h5py-be-vlen"]), 10) if known["h5py-be-vlen"] else "none") + ".")
w("- **Non-IEEE floats and partial-precision integers (N-Bit).** libhdf5 converts a float whose")
w("  bit layout is not IEEE (e.g. `H5Tset_precision` for the N-Bit filter) or an integer with a")
w("  bit offset / reduced precision into the plain numpy type of the same size. The probe")
w("  compares such values as converted numbers, not raw file bytes (before 2026-09-25 it compared")
w("  raw bytes, which reported every N-Bit float dataset as a mismatch).")
if res["incomparable"]:
    w("- **Types h5py widens.** Where h5py reads a type into a numpy type of a different size")
    w("  (FP8 -> float16, bfloat16 -> float32, x87 long double -> float128) the values are not")
    w("  compared (shape and presence still are): "
      + ", ".join(f"{k} ({n}x)" for k, n in res["incomparable"]) + ".")
w("- **References** are compared by presence only (`R`), not by target.")
w("")
if res.get("ref_only_errors"):
    w("## Objects h5py fails on but clawhdf5 reads")
    w("")
    for k, n in res["ref_only_errors"][:15]:
        w(f"- {n} x `{k}`")
    w("")
w("## Reproduce")
w("")
w("```sh")
w("# needs: Rust, python3 with h5py numpy hdf5plugin (conformance/requirements.txt), h5dump (hdf5-tools), git")
w("CLAWHDF5_PYTHON=/path/to/venv/bin/python conformance/run.sh")
w("```")
w("")
w("The corpus (about 450 MB of sparse checkouts) is cached in `conformance/.cache/`; results for")
w("every file, both sides' raw JSON and stderr, are in `conformance/.cache/results/`.")
w("`conformance/baseline.json` holds the ok files the nightly CI job (`.gitea/workflows/conformance.yml`)")
w("must keep; `conformance/run.sh --update-baseline` rewrites it.")

with open(OUT_MD, "w") as fh:
    fh.write("\n".join(L) + "\n")