diff --git a/conformance/.gitignore b/conformance/.gitignore new file mode 100644 index 0000000..9e01d95 --- /dev/null +++ b/conformance/.gitignore @@ -0,0 +1,3 @@ +/.cache/ +# pin the probe's dependencies (the workspace lock is not committed) +!/probe/Cargo.lock diff --git a/conformance/README.md b/conformance/README.md new file mode 100644 index 0000000..5f08558 --- /dev/null +++ b/conformance/README.md @@ -0,0 +1,39 @@ +# Conformance sweep + +Reads every HDF5 file of eight public corpora with clawhdf5 and with +h5py/libhdf5, compares the two readings object by object, and writes +[`CONFORMANCE.md`](../CONFORMANCE.md). + +```sh +CLAWHDF5_PYTHON=/path/to/venv/bin/python conformance/run.sh # ~30 s once the corpus is cached +conformance/run.sh --update-baseline # after an intended change in results +``` + +Needs Rust, `git`, `h5dump` (Debian/Ubuntu `hdf5-tools`), `libaec` (for the +probe's `szip` feature; `libaec-dev`), and a Python with the packages in +`requirements.txt`. The first run downloads about 450 MB of sparse checkouts. + +| file | role | +|---|---| +| `corpus.txt` | the corpora: git URL, pinned commit, swept root, sparse-checkout patterns | +| `fetch-corpus.sh` | shallow, sparse, blob-filtered checkout of each pinned commit into `.cache/src/` (gitignored); no-op when already there | +| `list_files.py` | which files are probed (HDF5/netCDF-4 extensions minus netCDF classic, plus the CVE reproducers) | +| `probe/` | the clawhdf5 side: a standalone crate (outside the workspace, so `cargo test --workspace` never builds it) that walks a file with `clawhdf5-format` and prints canonical JSON | +| `ref.py` | the h5py side: the same JSON from h5py | +| `run_one.sh` | runs both sides on one file (and `h5dump` on the CVE corpus) under a timeout and an address-space limit | +| `compare.py` | classifies each file (ok / our-error / mismatch / h5py-cannot-read / panic / hang / crash / oom) and groups root causes | +| `report.py` | writes `CONFORMANCE.md` | +| `check.py` | the gate: fails on any panic/hang/crash/oom, on an ok count below `baseline.json`, or on a baseline-ok file that is no longer ok | +| `baseline.json` | the ok files the gate holds the line on | +| `requirements.txt` | pinned h5py / numpy / hdf5plugin / netCDF4 | + +Results for every file (both sides' JSON and stderr, `results.csv`, +`results.json`, `summary.md`) are left in `.cache/results/`. + +The nightly job is `.gitea/workflows/conformance.yml`; it prints the report +into the job log. + +The canonical value encoding both sides hash is documented at the top of +`probe/src/main.rs`. Values are compared as libhdf5 presents them: a float +with a non-IEEE bit layout (N-Bit) or an integer with a bit offset is compared +as the converted number, not as raw file bytes. diff --git a/conformance/check.py b/conformance/check.py new file mode 100755 index 0000000..dea272f --- /dev/null +++ b/conformance/check.py @@ -0,0 +1,88 @@ +#!/usr/bin/env python3 +"""check.py [--update] + +The conformance gate. Fails (exit 1) when + * clawhdf5 panicked, hung, crashed or ran out of memory on any file, or + * the ok count fell below the baseline's, or + * a file the baseline lists as ok is no longer ok (even if another file + became ok and the total held). +New ok files are reported so the baseline can be raised (--update rewrites it +from the results). +""" +import json +import os +import sys + +FATAL = ("panic", "hang", "crash", "oom") + + +def main(): + args = [a for a in sys.argv[1:] if not a.startswith("--")] + update = "--update" in sys.argv + res_dir, base_path = args + res = json.load(open(os.path.join(res_dir, "results.json"))) + rows = res["rows"] + counts = {} + per_corpus = {} + for r in rows: + counts[r["class"]] = counts.get(r["class"], 0) + 1 + pc = per_corpus.setdefault(r["corpus"], {}) + pc[r["class"]] = pc.get(r["class"], 0) + 1 + ok_files = sorted(r["file"] for r in rows if r["class"] == "ok") + + if update: + meta = {} + mp = os.path.join(res_dir, "report-meta.json") + if os.path.exists(mp): + meta = json.load(open(mp)) + base = { + "comment": "conformance/run.sh fails if the ok count drops below `ok` or a file in `ok_files` stops being ok. " + "Regenerate with `conformance/run.sh --update-baseline` after an intended change.", + "commit": meta.get("commit", ""), + "date": meta.get("date", ""), + "reference": meta.get("reference", ""), + "files": len(rows), + "ok": len(ok_files), + "counts": dict(sorted(counts.items())), + "per_corpus": {k: dict(sorted(v.items())) for k, v in sorted(per_corpus.items())}, + "ok_files": ok_files, + } + with open(base_path, "w") as fh: + json.dump(base, fh, indent=1) + fh.write("\n") + print(f"baseline updated: {len(ok_files)} ok of {len(rows)} files -> {base_path}") + return 0 + + base = json.load(open(base_path)) + failures = [] + fatal = [r for r in rows if r["class"] in FATAL] + for r in fatal: + failures.append(f"{r['class']}: {r['file']}: {r['ours_detail'][:200]}") + if len(ok_files) < base["ok"]: + failures.append(f"ok count dropped: {len(ok_files)} < baseline {base['ok']}") + now_ok = set(ok_files) + by_file = {r["file"]: r for r in rows} + for f in base["ok_files"]: + if f not in now_ok: + r = by_file.get(f) + why = f"now {r['class']}: {(r['ours_detail'] or r['first_issue'])[:200]}" if r else "no longer in the corpus" + failures.append(f"regressed: {f}: {why}") + gained = sorted(now_ok - set(base["ok_files"])) + + print(f"conformance: {len(ok_files)} ok of {len(rows)} files (baseline {base['ok']} of {base['files']}); " + + ", ".join(f"{k} {v}" for k, v in sorted(counts.items()))) + if gained: + print(f"{len(gained)} file(s) newly ok — raise the baseline with `conformance/run.sh --update-baseline`:") + for f in gained: + print(f" + {f}") + if failures: + print(f"CONFORMANCE GATE FAILED ({len(failures)}):") + for f in failures: + print(f" - {f}") + return 1 + print("conformance gate passed") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/conformance/compare.py b/conformance/compare.py new file mode 100755 index 0000000..8c14353 --- /dev/null +++ b/conformance/compare.py @@ -0,0 +1,289 @@ +#!/usr/bin/env python3 +"""compare.py : classify each file and group failures by root cause. + +Writes /results.csv, results.json and summary.md. +File classes (first match wins): + hang, oom, crash, panic ours: timeout / allocation failure / signal / any panic (caught or not) + h5py-cannot-read libhdf5/h5py failed to open the file (or crashed/hung) + our-error we fail to open, list, or read something h5py reads + mismatch we read something with different shape/values, or a different object set + ok +""" +import collections +import csv +import json +import os +import re +import sys + +R = sys.argv[1] +RUNS = os.path.join(R, "runs") + + +def load(d, name): + rc_p = os.path.join(d, name + ".rc") + if not os.path.exists(rc_p): + return None + rc = int(open(rc_p).read().strip() or -1) + err = open(os.path.join(d, name + ".err"), errors="replace").read() + js = None + try: + js = json.load(open(os.path.join(d, name + ".json"))) + except Exception: # noqa: BLE001 + pass + return {"rc": rc, "err": err, "json": js} + + +def proc_status(p): + """-> (status, detail)""" + if p is None: + return "missing", "" + rc, err = p["rc"], p["err"] + first_panic = next((ln for ln in err.splitlines() if ln.startswith("PANIC:") or "panicked at" in ln), "") + if rc == 0 and p["json"] is not None: + return "ok", "" + if rc == 137 or rc == 124: + return "hang", f"timeout ({os.environ.get('TMO', '20')} s)" + if "memory allocation of" in err or "MemoryError" in err or "std::bad_alloc" in err: + m = re.search(r"memory allocation of \d+ bytes failed", err) + return "oom", m.group(0) if m else "allocation failure" + if "overflowed its stack" in err: + return "crash", "stack overflow" + if rc == 101: + return "panic", first_panic or (err.strip().splitlines() or [""])[-1] + if rc in (134, 139, 136, 135, 132) or rc > 128: + sig = {134: "SIGABRT", 139: "SIGSEGV", 136: "SIGFPE", 135: "SIGBUS", 132: "SIGILL"}.get(rc, f"signal {rc - 128}") + tail = [ln for ln in err.strip().splitlines() if ln.strip()][-1:] + return "crash", f"{sig}: {tail[0][:200] if tail else ''}" + tail = [ln for ln in err.strip().splitlines() if ln.strip()][-1:] + return "crash", f"rc={rc}: {tail[0][:200] if tail else ''}" + + +def norm(msg): + m = msg.split("\n")[0] + m = re.sub(r"0x[0-9a-fA-F]+", "X", m) + m = re.sub(r'"[^"]*"', '"…"', m) + m = re.sub(r"'[^']*'", "'…'", m) + m = re.sub(r"\d+", "N", m) + return m[:160] + + +def panic_head(msg): + """First line + first clawhdf5 frame of a PANIC record.""" + lines = msg.split("\n") + frame = next((ln.strip() for ln in lines[1:] if "clawhdf5_format" in ln), "") + return lines[0][:300], frame[:300] + + +def eq_shape(a, b): + return a == b + + +rows = [] +issues_by_file = {} +root_causes = collections.defaultdict(lambda: {"files": set(), "count": 0, "examples": []}) +mismatch_causes = collections.defaultdict(lambda: {"files": set(), "count": 0, "examples": []}) +panics = [] +ref_only_errors = collections.Counter() +incomparable = collections.Counter() + + +def add(bucket, key, file, example): + b = bucket[key] + b["count"] += 1 + if file not in b["files"] and len(b["examples"]) < 6: + b["examples"].append(example) + b["files"].add(file) + + +files = [ln.strip() for ln in open(os.path.join(R, "files.txt")) if ln.strip()] +for rel in files: + d = os.path.join(RUNS, rel.replace("/", "__")) + corpus = rel.split("/")[0] + ours, ref = load(d, "ours"), load(d, "ref") + h5dump = load(d, "h5dump") + os_, od = proc_status(ours) + rs, rd = proc_status(ref) + oj = ours["json"] if ours else None + rj = ref["json"] if ref else None + issues = [] # (kind, detail) + caught_panics = [] + + def scan_err(path, what, msg): + if msg.startswith("PANIC:"): + caught_panics.append((path, what, msg)) + + if oj: + for o in oj.get("objects", []): + for k in ("error", "attrs_error", "list_error"): + if k in o: + scan_err(o["path"], k, o[k]) + for an, av in (o.get("attrs") or {}).items(): + if "error" in av: + scan_err(o["path"], f"attr {an}", av["error"]) + if oj.get("open_error", "").startswith("PANIC:"): + caught_panics.append(("", "open", oj["open_error"])) + + ref_open_fail = rs != "ok" or (rj is not None and "open_error" in rj) + ours_open_err = oj.get("open_error") if oj else None + n_obj = n_ok = 0 + if os_ == "ok" and rj and not ref_open_fail and not ours_open_err: + ro = {x["path"]: x for x in rj.get("objects", [])} + oo = {x["path"]: x for x in oj.get("objects", [])} + our_list_errors = [x for x in oo.values() if "list_error" in x] + for p in sorted(set(ro) | set(oo)): + a, b = ro.get(p), oo.get(p) + n_obj += 1 + if a is None: + issues.append(("mismatch", f"extra object {p} (kind={b.get('kind')})", "extra-object", b)) + continue + if b is None: + if our_list_errors: + continue # accounted for by the list_error + issues.append(("mismatch", f"missing object {p} (kind={a.get('kind')})", "missing-object", a)) + continue + ok = True + if a.get("kind") != b.get("kind") and "error" not in b and "error" not in a: + issues.append(("mismatch", f"{p}: kind {a.get('kind')} vs ours {b.get('kind')}", "kind", b)) + ok = False + for k in ("error", "list_error", "attrs_error"): + if k in b and k not in a: + issues.append(("our-error", f"{p}: {k}: {b[k]}", b[k], b)) + ok = False + elif k in a and k not in b and k == "error": + ref_only_errors[norm(a[k])] += 1 + if a.get("kind") == "dataset" and "error" not in a and "error" not in b: + if "skipped" in a or "skipped" in b: + pass + elif a.get("converted"): + incomparable[f"dataset {a['converted']}"] += 1 + elif a.get("shape") != b.get("shape"): + issues.append(("mismatch", f"{p}: shape {a.get('shape')} vs ours {b.get('shape')}", "shape", b)) + ok = False + elif a.get("hash") != b.get("hash"): + issues.append(("mismatch", f"{p}: values differ (h5py {a.get('dtype')} vs ours {b.get('dtype')})", "values", b | {"ref_head": a.get("head"), "ref_dtype": a.get("dtype")})) + ok = False + ra, oa = a.get("attrs") or {}, b.get("attrs") or {} + if "attrs_error" not in b and "attrs_error" not in a: + for an in sorted(set(ra) | set(oa)): + x, y = ra.get(an), oa.get(an) + if x is None: + issues.append(("mismatch", f"{p}@{an}: extra attribute", "extra-attr", y or {})) + elif y is None: + issues.append(("mismatch", f"{p}@{an}: missing attribute", "missing-attr", x)) + elif "error" in y and "error" not in x: + issues.append(("our-error", f"{p}@{an}: {y['error']}", y["error"], y)) + elif "error" in x: + continue + elif x.get("converted"): + incomparable[f"attr {x['converted']}"] += 1 + elif x.get("shape") != y.get("shape"): + issues.append(("mismatch", f"{p}@{an}: attr shape {x.get('shape')} vs ours {y.get('shape')}", "attr-shape", y | {"ref_dtype": x.get("dtype")})) + elif x.get("hash") != y.get("hash"): + issues.append(("mismatch", f"{p}@{an}: attr values differ (h5py {x.get('dtype')} vs ours {y.get('dtype')})", "attr-values", y | {"ref_head": x.get("head"), "ref_dtype": x.get("dtype")})) + if ok: + n_ok += 1 + + # classify + if os_ in ("hang", "oom", "crash", "panic"): + cls = os_ + elif caught_panics: + cls = "panic" + elif ref_open_fail: + cls = "h5py-cannot-read" + elif ours_open_err: + cls = "our-error" + issues.append(("our-error", f"open: {ours_open_err}", ours_open_err, {})) + elif any(i[0] == "our-error" for i in issues): + cls = "our-error" + elif issues: + cls = "mismatch" + else: + cls = "ok" + + if os_ in ("hang", "oom", "crash", "panic") or caught_panics: + panics.append({ + "file": rel, "class": cls, "detail": od, + "stderr": (ours["err"] if ours else "")[:3000], + "caught": [(p, w, m[:2500]) for p, w, m in caught_panics[:3]], + "n_caught": len(caught_panics), + }) + for kind, detail, key, rec in issues: + if kind == "our-error": + add(root_causes, norm(key), rel, detail[:300]) + else: + if key in ("values", "attr-values", "shape", "attr-shape"): + mk = f"{key}: ours={rec.get('dtype')} h5py={rec.get('ref_dtype')} layout={rec.get('layout','-')} filters={rec.get('filters','-')}" + else: + mk = key + add(mismatch_causes, mk, rel, detail[:300] + (f" | ref_head={rec.get('ref_head')} our_head={rec.get('head')}" if rec.get("ref_head") else "")) + ref_detail = rd if rs != "ok" else ((rj or {}).get("open_error") or "") + h5d = "" + if h5dump: + rc = h5dump["rc"] + h5d = {0: "ok", 1: "error", 137: "hang", 124: "hang", 134: "SIGABRT", 139: "SIGSEGV", 136: "SIGFPE", 135: "SIGBUS"}.get(rc, f"rc={rc}") + if "memory allocation" in h5dump["err"] or "Cannot allocate" in h5dump["err"]: + h5d += "(oom)" + rows.append({ + "file": rel, "corpus": corpus, "class": cls, + "ours": os_ if os_ != "ok" else ("open-error" if ours_open_err else ("panic" if caught_panics else "ok")), + "ours_detail": (od or ours_open_err or (caught_panics[0][2].split("\n")[0] if caught_panics else ""))[:300], + "ref": rs if rs != "ok" else ("open-error" if (rj or {}).get("open_error") else "ok"), + "ref_detail": ref_detail[:300], + "h5dump_1_14_6": h5d, + "h5dump_detail": ([ln for ln in h5dump["err"].splitlines() if ln.strip()][-1:] or [""])[0][:200] if h5dump else "", + "objects": n_obj, "objects_ok": n_ok, + "issues": len(issues), "first_issue": issues[0][1][:300] if issues else "", + "superblock": (oj or {}).get("superblock_version", ""), + }) + # the first issues of each file, for report.py's known-cause matching + issues_by_file[rel] = [ + {"kind": k, "key": key, "detail": det[:300], "ours_dtype": rec.get("dtype"), "ref_dtype": rec.get("ref_dtype")} + for k, det, key, rec in issues[:50] + ] + +with open(os.path.join(R, "results.csv"), "w", newline="") as fh: + w = csv.DictWriter(fh, fieldnames=list(rows[0].keys())) + w.writeheader() + w.writerows(rows) + + +def ser(b): + return {k: {"files": len(v["files"]), "count": v["count"], "examples": v["examples"], "file_list": sorted(v["files"])} for k, v in sorted(b.items(), key=lambda kv: -len(kv[1]["files"]))} + + +json.dump({"rows": rows, "issues": issues_by_file, "root_causes": ser(root_causes), "mismatch_causes": ser(mismatch_causes), + "panics": panics, "incomparable": incomparable.most_common(), "ref_only_errors": ref_only_errors.most_common()}, + open(os.path.join(R, "results.json"), "w"), indent=1) + +classes = ["ok", "our-error", "mismatch", "h5py-cannot-read", "hang", "panic", "crash", "oom"] +by_corpus = collections.defaultdict(collections.Counter) +for r in rows: + by_corpus[r["corpus"]][r["class"]] += 1 + by_corpus["ALL"][r["class"]] += 1 +lines = ["# Conformance sweep summary", "", "| corpus | files | " + " | ".join(classes) + " |", "|---" * (len(classes) + 2) + "|"] +for c in sorted(by_corpus, key=lambda k: (k == "ALL", k)): + cnt = by_corpus[c] + lines.append(f"| {c} | {sum(cnt.values())} | " + " | ".join(str(cnt.get(k, 0)) for k in classes) + " |") +lines += ["", "## Panics / hangs / crashes / OOM", ""] +for p in panics: + lines.append(f"- **{p['file']}** [{p['class']}] {p['detail']}") + for path, what, m in p["caught"][:1]: + lines.append(" ```\n " + f"{path} ({what}): " + m.replace("\n", "\n ")[:1500] + "\n ```") + if not p["caught"] and p["stderr"]: + lines.append(" ```\n " + p["stderr"].strip()[:1500].replace("\n", "\n ") + "\n ```") +lines += ["", "## Our-error root causes (files affected)", ""] +for k, v in ser(root_causes).items(): + lines.append(f"- [{v['files']} files, {v['count']} objs] `{k}`") + for ex in v["examples"][:3]: + lines.append(f" - {ex}") +lines += ["", "## Mismatch root causes", ""] +for k, v in ser(mismatch_causes).items(): + lines.append(f"- [{v['files']} files, {v['count']} objs] `{k}`") + for ex in v["examples"][:3]: + lines.append(f" - {ex}") +lines += ["", "## Objects h5py fails on but we read (top)", ""] +for k, n in ref_only_errors.most_common(15): + lines.append(f"- {n} x `{k}`") +open(os.path.join(R, "summary.md"), "w").write("\n".join(lines) + "\n") +print("\n".join(lines[:4 + len(by_corpus)])) diff --git a/conformance/corpus.txt b/conformance/corpus.txt new file mode 100644 index 0000000..dfcbe8d --- /dev/null +++ b/conformance/corpus.txt @@ -0,0 +1,19 @@ +# Conformance corpora, pinned by commit. fetch-corpus.sh reads this file. +# +# name git-url commit root [sparse-checkout patterns...] +# +# `root` is the directory inside the checkout that is swept ("." = all of it). +# Patterns are git non-cone sparse-checkout patterns; none = whole repository. +# Every file under with an HDF5/netCDF-4 extension is probed; for +# cve_hdf5 the extension-less files in cvefiles/ and fuzzerfiles/ are too. +# Licences: each corpus keeps its upstream licence; nothing here is committed +# to this repository — the files are downloaded into the gitignored cache. +hdf5 https://github.com/HDFGroup/hdf5.git a3cf1ea82cc7a66e50029a688121e1b105a7ce88 . *.h5 *.he5 *.nc *.hdf5 *.h5f +cve_hdf5 https://github.com/HDFGroup/cve_hdf5.git 3fd1f5ae3869e01b8ae02b41d7108de7ffb1a374 . +netcdf-c https://github.com/Unidata/netcdf-c.git beb7b9585273c1548386231a59b809d906359033 . /nc_test4/*.nc /ncdump/*.nc /nc_test4/*.h5 /ncdump/*.h5 /h5_test/*.h5 /hdf5_test/*.h5 +NCAS-CMS_pyfive https://github.com/NCAS-CMS/pyfive.git 8cf07b8749133f41c5e30b8a4c604486f687fe74 . *.h5 *.hdf5 *.hdf *.nc *.he5 +usnistgov_h5wasm https://github.com/usnistgov/h5wasm.git 02f6336527d2812783fcedabfbf42127ec8d06d2 . *.h5 *.hdf5 *.hdf *.nc *.he5 +netcdf4-python https://github.com/Unidata/netcdf4-python.git 6e67576d39aef8091fb20bd767b4f1a52ddc1bec . *.nc *.h5 +xarray-data https://github.com/pydata/xarray-data.git a35297e9da2cc99c811014f0c8a4297345a5c28d . /basin_mask.nc /precipitation.nc4 /imerghh_730.hdf5 /eraint_uvz.nc /ROMS_example.nc /tiny.nc +# h5py 3.16.0 (tag 3.16.0), its test data files. +h5py_data https://github.com/h5py/h5py.git b2f0347c4200333acd89b43733f1caa0c115162f h5py/tests/data_files /h5py/tests/data_files/* diff --git a/conformance/fetch-corpus.sh b/conformance/fetch-corpus.sh new file mode 100755 index 0000000..fca2824 --- /dev/null +++ b/conformance/fetch-corpus.sh @@ -0,0 +1,39 @@ +#!/usr/bin/env bash +# fetch-corpus.sh [cache_dir] +# +# Download the corpora pinned in conformance/corpus.txt into the (gitignored) +# cache: /src/ is a shallow, sparse, blob-filtered checkout of the +# pinned commit and /corpus/ links to the swept root inside it. +# A corpus already checked out at its pinned commit is left alone, so a second +# run costs nothing and needs no network. +set -euo pipefail +HERE="$(cd "$(dirname "$0")" && pwd)" +CACHE="${1:-${CONFORMANCE_CACHE:-$HERE/.cache}}" +mkdir -p "$CACHE/src" "$CACHE/corpus" +CACHE="$(cd "$CACHE" && pwd)" + +retry() { local i; for i in 1 2 3 4; do "$@" && return 0; sleep $((i * 5)); done; return 1; } + +grep -v '^[[:space:]]*\(#\|$\)' "$HERE/corpus.txt" | while read -r name url commit root patterns; do + src="$CACHE/src/$name" + if [ -d "$src/.git" ] && [ "$(git -C "$src" rev-parse HEAD 2>/dev/null)" = "$commit" ]; then + echo "cached $name @ ${commit:0:12}" + else + echo "fetching $name @ ${commit:0:12} from $url" + rm -rf "$src" + git init -q "$src" + git -C "$src" remote add origin "$url" + git -C "$src" config advice.detachedHead false + if [ -n "$patterns" ]; then + git -C "$src" config core.sparseCheckout true + # no-cone patterns (globs); `set -f` keeps the shell from expanding them + (set -f; printf '%s\n' $patterns) > "$src/.git/info/sparse-checkout" + fi + retry git -C "$src" fetch -q --depth 1 --filter=blob:none origin "$commit" + retry git -C "$src" checkout -q FETCH_HEAD + got="$(git -C "$src" rev-parse HEAD)" + [ "$got" = "$commit" ] || { echo "error: $name checked out $got, expected $commit" >&2; exit 1; } + fi + ln -sfn "$src/$root" "$CACHE/corpus/$name" +done +echo "corpus ready in $CACHE/corpus" diff --git a/conformance/list_files.py b/conformance/list_files.py new file mode 100644 index 0000000..4104a90 --- /dev/null +++ b/conformance/list_files.py @@ -0,0 +1,48 @@ +#!/usr/bin/env python3 +"""list_files.py : print the files the sweep probes, one per line, +as / in byte order. + + * every file named *.h5 *.hdf5 *.he5 *.nc *.nc4 *.hdf *.h5f in each corpus, + except netCDF classic / 64-bit-offset / CDF5 files (magic "CDF"): they are + not HDF5, so neither side can read them and they say nothing; + * plus, for cve_hdf5, every file in cvefiles/ and fuzzerfiles/ except + .md/.c sources — the reproducers are mostly extension-less, and they are + kept whatever their bytes look like (that is their point). +""" +import os +import sys + +EXTS = (".h5", ".hdf5", ".he5", ".nc", ".nc4", ".hdf", ".h5f") + + +def walk(top): + for dirpath, dirnames, filenames in os.walk(top): + dirnames[:] = [d for d in dirnames if d != ".git"] + for fn in filenames: + p = os.path.join(dirpath, fn) + if os.path.isfile(p) and not os.path.islink(p): + yield os.path.relpath(p, top) + + +def main(root): + out = set() + for corpus in sorted(os.listdir(root)): + top = os.path.join(root, corpus) + if not os.path.isdir(top): + continue + for rel in walk(top): + path = os.path.join(top, rel) + if rel.lower().endswith(EXTS): + with open(path, "rb") as fh: + if fh.read(3) == b"CDF": + continue + out.add(f"{corpus}/{rel}") + elif corpus == "cve_hdf5" and rel.split(os.sep)[0] in ("cvefiles", "fuzzerfiles") \ + and not rel.endswith((".md", ".c")): + out.add(f"{corpus}/{rel}") + for f in sorted(out, key=lambda s: s.encode()): + print(f) + + +if __name__ == "__main__": + main(sys.argv[1]) diff --git a/conformance/probe/Cargo.lock b/conformance/probe/Cargo.lock new file mode 100644 index 0000000..0ffb1f7 --- /dev/null +++ b/conformance/probe/Cargo.lock @@ -0,0 +1,458 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "adler2" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "320119579fcad9c21884f5c4861d16174d0e06250625266f50fe6898340abefa" + +[[package]] +name = "better_io" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ef0a3155e943e341e557863e69a708999c94ede624e37865c8e2a91b94efa78f" + +[[package]] +name = "block-buffer" +version = "0.10.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71" +dependencies = [ + "generic-array", +] + +[[package]] +name = "byteorder" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" + +[[package]] +name = "cc" +version = "1.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f360145194ee8e21db5ee7f3fcd4fe52210864c75c985dae33218202c8bbe040" +dependencies = [ + "find-msvc-tools", + "jobserver", + "libc", + "shlex", +] + +[[package]] +name = "cfg-if" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4e7648175b45a9a48536d676f68d918270699102aa8dab5496df06904c914600" + +[[package]] +name = "clawhdf5-format" +version = "2.7.0" +dependencies = [ + "byteorder", + "flate2", + "libaec-sys", + "lz4_flex", + "pco", + "portable-atomic", + "sha2", + "zstd", +] + +[[package]] +name = "conformance-probe" +version = "0.1.0" +dependencies = [ + "clawhdf5-format", + "serde_json", + "sha2", +] + +[[package]] +name = "cpufeatures" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "59ed5838eebb26a2bb2e58f6d5b5316989ae9d08bab10e0e6d103e656d1b0280" +dependencies = [ + "libc", +] + +[[package]] +name = "crc32fast" +version = "1.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "01a7799fd6b852db0e61728dde9a204c423b44d689dbd432522543614b490e78" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "crunchy" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "460fbee9c2c2f33933d720630a6a0bac33ba7053db5344fac858d4b8952d77d5" + +[[package]] +name = "crypto-common" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "78c8292055d1c1df0cce5d180393dc8cce0abec0a7102adb6c7b1eef6016d60a" +dependencies = [ + "generic-array", + "typenum", +] + +[[package]] +name = "digest" +version = "0.10.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" +dependencies = [ + "block-buffer", + "crypto-common", +] + +[[package]] +name = "dtype_dispatch" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ab23e69df104e2fd85ee63a533a22d2132ef5975dc6b36f9f3e5a7305e4a8ed7" + +[[package]] +name = "find-msvc-tools" +version = "0.1.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aedcfb3409746eddb02b9e19ebda1c3394f759a152e48ee875a0844d1b955484" + +[[package]] +name = "flate2" +version = "1.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e634e2e0ebac1ee034020da1ca582e17ffe4e0f5e985823721e168928136dcb" +dependencies = [ + "crc32fast", + "miniz_oxide", + "zlib-rs", +] + +[[package]] +name = "generic-array" +version = "0.14.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a" +dependencies = [ + "typenum", + "version_check", +] + +[[package]] +name = "getrandom" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "300e883d756b2e4ec94e02791f39b04b522276138852cfc41d9fb7e904106099" +dependencies = [ + "cfg-if", + "libc", + "r-efi", +] + +[[package]] +name = "half" +version = "2.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ea2d84b969582b4b1864a92dc5d27cd2b77b622a8d79306834f1be5ba20d84b" +dependencies = [ + "cfg-if", + "crunchy", + "zerocopy", +] + +[[package]] +name = "itoa" +version = "1.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" + +[[package]] +name = "jobserver" +version = "0.1.35" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1c00acbd29eabad4a2392fa0e921c874934dbbf4194312ad20f04a0ed67a3cb3" +dependencies = [ + "getrandom", + "libc", +] + +[[package]] +name = "libaec-sys" +version = "0.1.0" +dependencies = [ + "pkg-config", +] + +[[package]] +name = "libc" +version = "0.2.189" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3eaf3ede3fee6db1a4c2ee091bf8a8b4dccdc6d17f656fb07896ee72867612f2" + +[[package]] +name = "lz4_flex" +version = "0.11.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "373f5eceeeab7925e0c1098212f2fbc4d416adec9d35051a6ab251e824c1854a" +dependencies = [ + "twox-hash", +] + +[[package]] +name = "memchr" +version = "2.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf8baf1c55e62ffcace7a9f06f4bd9cd3f0c4beb022d3b367256b91b87513d98" + +[[package]] +name = "miniz_oxide" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b63fbc4a50860e98e7b2aa7804ded1db5cbc3aff9193adaff57a6931bf7c4b4c" +dependencies = [ + "adler2", + "simd-adler32", +] + +[[package]] +name = "pco" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "386342cad4c6e97f081568e5d910ea7d871314c843aa8fc564f2a6b64cab9456" +dependencies = [ + "better_io", + "dtype_dispatch", + "half", + "rand_xoshiro", +] + +[[package]] +name = "pkg-config" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f6b464fbc74e149a392436b17d523f769e057cb6877f6a5c4618bc6f11800548" + +[[package]] +name = "portable-atomic" +version = "1.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05c8b63e8d9609db387f0324918f81d68fe27748f084ef092fb35954d0539a85" + +[[package]] +name = "proc-macro2" +version = "1.0.107" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "985e7ec9bb745e6ce6535b544d84d6cd6f7ad8bd711c398938ae983b91a766d9" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "quote" +version = "1.0.47" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fbf4db142a473a8d80c26bbf18454ed458bf8d26c8219c331daecfdbd079001" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "r-efi" +version = "6.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" + +[[package]] +name = "rand_core" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" + +[[package]] +name = "rand_xoshiro" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6f97cdb2a36ed4183de61b2f824cc45c9f1037f28afe0a322e9fff4c108b5aaa" +dependencies = [ + "rand_core", +] + +[[package]] +name = "serde" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4148590afebada386688f18773da617792bf2ef03ffc1e4cbd2b1d45b023e0ba" +dependencies = [ + "serde_core", +] + +[[package]] +name = "serde_core" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67dca2c9c51e58a4791a4b1ed58308b39c64224d349a935ab5039aa360942a48" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7a5d71263a5a7d47b41f6b3f06ba276f10cc18b0931f1799f710578e2309348" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.6", +] + +[[package]] +name = "serde_json" +version = "1.0.151" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c841b55ecdae098c80dcae9cf767f6f8a0c2cdb3416bbef72181df4d0fe73f14" +dependencies = [ + "itoa", + "memchr", + "serde", + "serde_core", + "zmij", +] + +[[package]] +name = "sha2" +version = "0.10.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a7507d819769d01a365ab707794a4084392c824f54a7a6a7862f8c3d0892b283" +dependencies = [ + "cfg-if", + "cpufeatures", + "digest", +] + +[[package]] +name = "shlex" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8fadd59c855ef2080decdef8ff161eb6661b86933c9d82e5ba29dc602a55aba" + +[[package]] +name = "simd-adler32" +version = "0.3.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3a219298ac11a56ea9a6d2120044824d6f01aeb034955e7af7bc16858527deea" + +[[package]] +name = "syn" +version = "2.0.119" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "872831b642d1a07999a962a351ed35b955ea2cfc8f3862091e2a240a84f17297" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "syn" +version = "3.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8593e8e72159ed2257d083c7a454a85cbf854f37a0966d8d483aff8c8a3ebcee" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "twox-hash" +version = "2.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5283634e518fe9e82c7b20520bb4bc209009fd16c82077c802f8111ecbb0117a" + +[[package]] +name = "typenum" +version = "1.20.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6f5e870be6c3b371b77fe0ee0bafb859fa4964b4404c27de1d380043c4dda20" + +[[package]] +name = "unicode-ident" +version = "1.0.26" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d245f478577f809a851594d02313b640fb437e0bb33866753cff937863096954" + +[[package]] +name = "version_check" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" + +[[package]] +name = "zerocopy" +version = "0.8.59" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6df92bf3d9227be3d53173901ddbffac2babc27ae50f397776ffd6dc33f800cb" +dependencies = [ + "zerocopy-derive", +] + +[[package]] +name = "zerocopy-derive" +version = "0.8.59" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac4f328cf2f05d084e496c3e9c3f33ed0a183656a16e1fcec4d464d8373aec82" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "zlib-rs" +version = "0.6.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b268e58e7c693d7c271f93ffc4ba3b380412554231c85bf61ca7af91042a4112" + +[[package]] +name = "zmij" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29666d0abbfad1e3dc4dcf6144730dd3a3ab225bbbdac83319345b1b44ccfc1b" + +[[package]] +name = "zstd" +version = "0.13.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e91ee311a569c327171651566e07972200e76fcfe2242a4fa446149a3881c08a" +dependencies = [ + "zstd-safe", +] + +[[package]] +name = "zstd-safe" +version = "7.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "64d80649ab6db9d9f6f9c80a40becd948eda4714a0a5ac8c4d157a32231c7882" +dependencies = [ + "zstd-sys", +] + +[[package]] +name = "zstd-sys" +version = "2.1.0+zstd.1.5.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ef0a8027ec3ee71300ab3bcbcd0393f434aa72b91ca6d635a39941deae8eea0" +dependencies = [ + "cc", + "pkg-config", +] diff --git a/conformance/probe/Cargo.toml b/conformance/probe/Cargo.toml new file mode 100644 index 0000000..f5c68af --- /dev/null +++ b/conformance/probe/Cargo.toml @@ -0,0 +1,25 @@ +[package] +name = "conformance-probe" +version = "0.1.0" +edition = "2024" +rust-version = "1.92" +publish = false +description = "Walks an HDF5 file with clawhdf5-format and prints a canonical JSON description (see conformance/README.md)" + +# Deliberately outside the main workspace: `cargo test --workspace` never +# builds it, and it links the optional C codecs (zstd, libaec) that the core +# crates' default build must not. +[workspace] + +[dependencies] +clawhdf5-format = { path = "../../crates/clawhdf5-format", features = ["lz4", "zstd", "szip", "pcodec"] } +serde_json = "1" +sha2 = "0.10" + +[profile.release] +# Keep panics catchable (the probe records them per object) and turn integer +# overflow into a reported panic instead of silent wraparound. +debug = 1 +overflow-checks = true +debug-assertions = true +panic = "unwind" diff --git a/conformance/probe/src/main.rs b/conformance/probe/src/main.rs new file mode 100644 index 0000000..3591dc5 --- /dev/null +++ b/conformance/probe/src/main.rs @@ -0,0 +1,854 @@ +//! Conformance probe: walks an HDF5 file with clawhdf5-format (the same calls +//! the `clawhdf5` facade makes) and prints a canonical JSON description: +//! every hard-linked object (sorted-name DFS, deduplicated by header address), +//! and for each dataset / attribute its shape plus the SHA-256 of its values +//! in a canonical encoding shared with `ref.py`. +//! +//! Canonical value encoding (per element, concatenated, row-major): +//! int / float / bitfield / enum / time : element bytes, little-endian +//! non-IEEE-layout float (e.g. N-Bit) : the IEEE float of the same size it converts to +//! int with bit offset / short precision: the full-width integer it converts to +//! opaque : raw bytes +//! compound : members in declaration order (padding dropped) +//! array : base elements row-major +//! string (fixed or VL) : b'S' + u32le len + bytes (cut at first NUL, trailing spaces stripped) +//! VL sequence : b'V' + u32le count + base elements +//! reference : b'R' (payload not compared) +//! +//! Every object is processed inside catch_unwind; a caught panic is recorded +//! with its message, location and the clawhdf5 frames of its backtrace. + +use std::cell::RefCell; +use std::collections::{HashMap, HashSet}; +use std::panic::{self, AssertUnwindSafe}; +use std::rc::Rc; + +use clawhdf5_format::attribute::extract_attributes_full; +use clawhdf5_format::data_layout::DataLayout; +use clawhdf5_format::data_read; +use clawhdf5_format::dataspace::{Dataspace, DataspaceType}; +use clawhdf5_format::datatype::{Datatype, DatatypeByteOrder}; +use clawhdf5_format::filter_pipeline::FilterPipeline; +use clawhdf5_format::global_heap::GlobalHeapCollection; +use clawhdf5_format::group_v1::{self, GroupEntry}; +use clawhdf5_format::group_v2; +use clawhdf5_format::message_type::MessageType; +use clawhdf5_format::object_header::ObjectHeader; +use clawhdf5_format::signature; +use clawhdf5_format::superblock::Superblock; +use clawhdf5_format::symbol_table::SymbolTableMessage; +use serde_json::{Map, Value, json}; +use sha2::{Digest, Sha256}; + +const MAX_BYTES: u64 = 200 * 1024 * 1024; +const MAX_OBJECTS: usize = 200_000; + +thread_local! { + static LAST_PANIC: RefCell> = const { RefCell::new(None) }; +} + +fn install_hook() { + panic::set_hook(Box::new(|info| { + let msg = if let Some(s) = info.payload().downcast_ref::<&str>() { + s.to_string() + } else if let Some(s) = info.payload().downcast_ref::() { + s.clone() + } else { + "".into() + }; + let loc = info + .location() + .map(|l| format!("{}:{}", l.file(), l.line())) + .unwrap_or_default(); + let bt = std::backtrace::Backtrace::force_capture().to_string(); + // keep only frames from clawhdf5 code + let mut frames = Vec::new(); + let lines: Vec<&str> = bt.lines().collect(); + for (i, l) in lines.iter().enumerate() { + let t = l.trim(); + if t.contains("clawhdf5_format::") || t.contains("conformance_probe::") { + let at = lines + .get(i + 1) + .map(|n| n.trim()) + .filter(|n| n.starts_with("at ")) + .map(|n| { + let n = n.trim_start_matches("at "); + match n.find("/crates/") { + Some(p) => n[p + 1..].to_string(), + None => n.to_string(), + } + }) + .unwrap_or_default(); + let name = t.split_once(": ").map(|x| x.1).unwrap_or(t); + frames.push(format!("{name} ({at})")); + if frames.len() >= 12 { + break; + } + } + } + let full = format!("PANIC: {msg} @ {loc}\n {}", frames.join("\n ")); + eprintln!("{full}"); + LAST_PANIC.with(|p| *p.borrow_mut() = Some(full)); + })); +} + +/// Run `f`, turning a panic into Err("PANIC: ..."). +fn guarded(f: impl FnOnce() -> Result) -> Result { + match panic::catch_unwind(AssertUnwindSafe(f)) { + Ok(r) => r, + Err(_) => Err(LAST_PANIC + .with(|p| p.borrow_mut().take()) + .unwrap_or_else(|| "PANIC: ".into())), + } +} + +fn e(x: E) -> String { + format!("{x:?}") +} + +struct Ctx<'a> { + data: &'a [u8], + os: u8, + ls: u8, + base_dir: std::path::PathBuf, + heaps: RefCell, String>>>, +} + +impl<'a> Ctx<'a> { + fn header(&self, addr: u64) -> Result { + ObjectHeader::parse(self.data, addr as usize, self.os, self.ls).map_err(e) + } + + fn payload(&self, h: &ObjectHeader, t: MessageType) -> Result>, String> { + match h.messages.iter().find(|m| m.msg_type == t) { + None => Ok(None), + Some(m) => { + clawhdf5_format::shared_message::message_data(self.data, m, self.os, self.ls) + .map(|c| Some(c.into_owned())) + .map_err(e) + } + } + } + + fn heap_obj(&self, addr: u64, idx: u32) -> Result, String> { + let coll = { + let mut cache = self.heaps.borrow_mut(); + cache + .entry(addr) + .or_insert_with(|| { + GlobalHeapCollection::parse(self.data, addr as usize, self.ls) + .map(Rc::new) + .map_err(e) + }) + .clone()? + }; + coll.get_object(idx as u16) + .map(|o| o.data.clone()) + .ok_or_else(|| { + format!("GlobalHeapObjectNotFound {{ collection_address: {addr}, index: {idx} }}") + }) + } + + fn read_offset(&self, b: &[u8]) -> u64 { + let mut v = 0u64; + for (i, x) in b.iter().take(self.os as usize).enumerate() { + v |= (*x as u64) << (8 * i); + } + v + } + + fn canon(&self, dt: &Datatype, b: &[u8], out: &mut Vec) -> Result<(), String> { + let size = dt.type_size() as usize; + if b.len() < size { + return Err(format!( + "canon: element slice {} < type size {size}", + b.len() + )); + } + match dt { + Datatype::FloatingPoint { .. } if !ieee_layout(dt) => { + canon_custom_float(dt, &b[..size], out)? + } + Datatype::FixedPoint { .. } if partial_int(dt) => { + canon_partial_int(dt, &b[..size], out)? + } + Datatype::FixedPoint { byte_order, .. } + | Datatype::BitField { byte_order, .. } + | Datatype::FloatingPoint { byte_order, .. } => match byte_order { + DatatypeByteOrder::LittleEndian => out.extend_from_slice(&b[..size]), + DatatypeByteOrder::BigEndian => out.extend(b[..size].iter().rev()), + DatatypeByteOrder::Vax => return Err("canon: VAX byte order".into()), + }, + Datatype::Time { .. } | Datatype::Opaque { .. } => out.extend_from_slice(&b[..size]), + Datatype::String { .. } => canon_str(&b[..size], out), + Datatype::Compound { members, .. } => { + for m in members { + let off = m.byte_offset as usize; + let ms = m.datatype.type_size() as usize; + if off.checked_add(ms).is_none_or(|end| end > size) { + return Err(format!("canon: member {} out of bounds", m.name)); + } + self.canon(&m.datatype, &b[off..off + ms], out)?; + } + } + Datatype::Reference { .. } => out.push(b'R'), + Datatype::Enumeration { base_type, .. } => self.canon(base_type, b, out)?, + Datatype::Array { + base_type, + dimensions, + } => { + let n: usize = dimensions.iter().map(|d| *d as usize).product(); + let bs = base_type.type_size() as usize; + for i in 0..n { + self.canon(base_type, &b[i * bs..], out)?; + } + } + Datatype::VariableLength { + is_string, + base_type, + .. + } => { + let len = u32::from_le_bytes([b[0], b[1], b[2], b[3]]) as usize; + let addr = self.read_offset(&b[4..]); + let idx_off = 4 + self.os as usize; + let idx = u32::from_le_bytes([ + b[idx_off], + b[idx_off + 1], + b[idx_off + 2], + b[idx_off + 3], + ]); + let obj = if len == 0 || addr == 0 || addr == u64::MAX >> (64 - 8 * self.os as u32) + { + Vec::new() + } else { + self.heap_obj(addr, idx)? + }; + if *is_string { + let l = len.min(obj.len()); + canon_str(&obj[..l], out); + } else { + let bs = base_type.type_size() as usize; + if bs == 0 { + return Err("canon: VL base size 0".into()); + } + let need = len.checked_mul(bs).ok_or("canon: VL overflow")?; + if len > 0 && obj.len() < need { + return Err(format!("canon: VL object {} < {need}", obj.len())); + } + out.push(b'V'); + out.extend_from_slice(&(len as u32).to_le_bytes()); + for i in 0..len { + self.canon(base_type, &obj[i * bs..], out)?; + } + } + } + } + Ok(()) + } + + /// Returns (shape json, n_elements) + fn shape(ds: &Dataspace) -> (Value, u64) { + match ds.space_type { + DataspaceType::Null => (Value::String("null".into()), 0), + DataspaceType::Scalar => (json!([]), 1), + DataspaceType::Simple => { + let n = ds.dimensions.iter().fold(1u64, |a, d| a.saturating_mul(*d)); + (json!(ds.dimensions), n) + } + } + } + + fn hash_values( + &self, + dt: &Datatype, + raw: &[u8], + n: u64, + rec: &mut Map, + ) -> Result<(), String> { + let size = dt.type_size() as usize; + let need = (n as usize).checked_mul(size).ok_or("n*size overflow")?; + if raw.len() != need { + return Err(format!( + "raw length {} != n_elements {n} * type_size {size}", + raw.len() + )); + } + let mut canon = Vec::with_capacity(need); + for i in 0..n as usize { + self.canon(dt, &raw[i * size..(i + 1) * size], &mut canon)?; + } + let h = Sha256::digest(&canon); + rec.insert("hash".into(), Value::String(hex(&h))); + rec.insert( + "head".into(), + Value::String(hex(&canon[..canon.len().min(48)])), + ); + Ok(()) + } + + fn read_dataset(&self, h: &ObjectHeader, rec: &mut Map) -> Result<(), String> { + let dtb = self + .payload(h, MessageType::Datatype)? + .ok_or("MissingMessage(Datatype)")?; + let (dt, _) = Datatype::parse(&dtb).map_err(e)?; + rec.insert("dtype".into(), Value::String(dtype_str(&dt))); + let dsb = self + .payload(h, MessageType::Dataspace)? + .ok_or("MissingMessage(Dataspace)")?; + let ds = Dataspace::parse(&dsb, self.ls).map_err(e)?; + let (shape, n) = Self::shape(&ds); + rec.insert("shape".into(), shape); + if n.saturating_mul(dt.type_size() as u64) > MAX_BYTES { + rec.insert("skipped".into(), Value::String("too large".into())); + return Ok(()); + } + let lm = h + .messages + .iter() + .find(|m| m.msg_type == MessageType::DataLayout) + .ok_or("MissingMessage(DataLayout)")?; + let dl = DataLayout::parse(&lm.data, self.os, self.ls).map_err(e)?; + rec.insert( + "layout".into(), + Value::String( + match &dl { + DataLayout::Compact { .. } => "compact", + DataLayout::Contiguous { .. } => "contiguous", + DataLayout::Chunked { .. } => "chunked", + DataLayout::Virtual { .. } => "virtual", + } + .into(), + ), + ); + let pipeline = match self.payload(h, MessageType::FilterPipeline)? { + Some(p) => Some(FilterPipeline::parse(&p).map_err(e)?), + None => None, + }; + if let Some(p) = &pipeline { + rec.insert( + "filters".into(), + json!(p.filters.iter().map(|f| f.filter_id).collect::>()), + ); + } + let raw = if matches!(dl, DataLayout::Virtual { .. }) { + let base = self.base_dir.clone(); + let resolver = move |name: &str| -> Option> { + let p = std::path::Path::new(name); + if p.is_absolute() + || p.components() + .any(|c| matches!(c, std::path::Component::ParentDir)) + { + return None; + } + std::fs::read(base.join(p)).ok() + }; + data_read::read_raw_data_full_with_resolver( + self.data, + &dl, + &ds, + &dt, + pipeline.as_ref(), + self.os, + self.ls, + Some(&resolver), + ) + .map_err(e)? + } else { + let cache = clawhdf5_format::chunk_cache::ChunkCache::new(); + clawhdf5_format::fill_value::read_full_with_fill::( + &h.messages, + self.data, + &dl, + &ds, + dt.type_size() as usize, + self.os, + self.ls, + || { + data_read::read_raw_data_cached( + self.data, + &dl, + &ds, + &dt, + pipeline.as_ref(), + self.os, + self.ls, + &cache, + ) + }, + ) + .map_err(e)? + }; + self.hash_values(&dt, &raw, n, rec) + } + + fn attrs(&self, h: &ObjectHeader) -> Result, String> { + let msgs = extract_attributes_full(self.data, h, self.os, self.ls).map_err(e)?; + let mut out = Map::new(); + for a in &msgs { + let r = guarded(|| { + let mut rec = Map::new(); + rec.insert("dtype".into(), Value::String(dtype_str(&a.datatype))); + let (shape, n) = Self::shape(&a.dataspace); + rec.insert("shape".into(), shape); + self.hash_values(&a.datatype, &a.raw_data, n, &mut rec)?; + Ok(rec) + }); + let v = match r { + Ok(rec) => Value::Object(rec), + Err(msg) => json!({ "error": msg }), + }; + out.insert(a.name.clone(), v); + } + Ok(out) + } + + fn entries(&self, h: &ObjectHeader) -> Result, String> { + let v1 = h + .messages + .iter() + .find(|m| m.msg_type == MessageType::SymbolTable); + if let Some(m) = v1 { + let stm = SymbolTableMessage::parse(&m.data, self.os).map_err(e)?; + group_v1::resolve_v1_group_entries(self.data, &stm, self.os, self.ls).map_err(e) + } else if h + .messages + .iter() + .any(|m| m.msg_type == MessageType::LinkInfo || m.msg_type == MessageType::Link) + { + group_v2::resolve_v2_group_entries(self.data, h, self.os, self.ls).map_err(e) + } else { + Ok(Vec::new()) + } + } +} + +/// Element bytes as an unsigned integer (at most 16 bytes), honouring byte order. +fn element_bits(b: &[u8], byte_order: &DatatypeByteOrder) -> Result { + if b.len() > 16 { + return Err(format!("canon: {}-byte numeric element", b.len())); + } + let mut v = 0u128; + match byte_order { + DatatypeByteOrder::LittleEndian => { + for (i, x) in b.iter().enumerate() { + v |= u128::from(*x) << (8 * i); + } + } + DatatypeByteOrder::BigEndian => { + for x in b { + v = (v << 8) | u128::from(*x); + } + } + DatatypeByteOrder::Vax => return Err("canon: VAX byte order".into()), + } + Ok(v) +} + +fn field(v: u128, pos: u32, len: u32) -> u128 { + if len == 0 || pos >= 128 { + return 0; + } + let v = v >> pos; + if len >= 128 { + v + } else { + v & ((1u128 << len) - 1) + } +} + +/// True when a float's bit fields are exactly IEEE 754 binary16/32/64 for its +/// size. h5py hands back such a type's bytes untouched; any other layout (an +/// N-Bit `H5Tset_precision` float, say) is *converted* by libhdf5 into the +/// numpy float of the same size, so comparing raw bytes would be meaningless. +fn ieee_layout(dt: &Datatype) -> bool { + let Datatype::FloatingPoint { + size, + bit_offset, + bit_precision, + exponent_location, + exponent_size, + mantissa_location, + mantissa_size, + exponent_bias, + .. + } = dt + else { + return true; + }; + let std = match size { + 2 => (16, 10, 5, 10, 15), + 4 => (32, 23, 8, 23, 127), + 8 => (64, 52, 11, 52, 1023), + _ => return true, // no same-size numpy float to convert to: compare raw + }; + *bit_offset == 0 + && ( + *bit_precision, + *exponent_location, + *exponent_size, + *mantissa_size, + *exponent_bias, + ) == (std.0, std.1, std.2, std.3, std.4) + && *mantissa_location == 0 +} + +/// Canonicalise a non-IEEE-layout float the way libhdf5's float->float +/// conversion presents it to h5py: as the IEEE float of the same size. +/// Assumes the implied-leading-one normalisation and the sign bit at the top +/// of the precision (what `H5Tset_precision` produces; the parser does not +/// keep either field). +fn canon_custom_float(dt: &Datatype, b: &[u8], out: &mut Vec) -> Result<(), String> { + let Datatype::FloatingPoint { + size, + byte_order, + bit_offset, + bit_precision, + exponent_location, + exponent_size, + mantissa_location, + mantissa_size, + exponent_bias, + } = dt + else { + unreachable!() + }; + let (esize, msize) = (u32::from(*exponent_size), u32::from(*mantissa_size)); + if esize == 0 || esize > 30 || msize > 64 { + return Err(format!("canon: unsupported float layout e{esize} m{msize}")); + } + let v = element_bits(b, byte_order)?; + let sign_pos = (u32::from(*bit_offset) + u32::from(*bit_precision)).saturating_sub(1); + let neg = field(v, sign_pos, 1) == 1; + let e = field(v, u32::from(*exponent_location), esize) as i64; + let m = field(v, u32::from(*mantissa_location), msize); + let emax = (1i64 << esize) - 1; + let bias = i64::from(*exponent_bias); + let mag = if e == emax { + if m == 0 { f64::INFINITY } else { f64::NAN } + } else if e == 0 { + (m as f64) * 2f64.powi((1 - bias - msize as i64) as i32) + } else { + ((1u128 << msize) as f64 + m as f64) * 2f64.powi((e - bias - msize as i64) as i32) + }; + let x = if neg { -mag } else { mag }; + match size { + 2 => out + .extend_from_slice(&clawhdf5_format::float16::f32_to_f16_bits(x as f32).to_le_bytes()), + 4 => out.extend_from_slice(&(x as f32).to_le_bytes()), + 8 => out.extend_from_slice(&x.to_le_bytes()), + _ => unreachable!("ieee_layout keeps other sizes raw"), + } + Ok(()) +} + +/// Integers stored with a bit offset or reduced precision (N-Bit): libhdf5 +/// converts them to the full-width integer of the same size, shifting the +/// value down and sign-extending from the top precision bit. +fn canon_partial_int(dt: &Datatype, b: &[u8], out: &mut Vec) -> Result<(), String> { + let Datatype::FixedPoint { + size, + byte_order, + signed, + bit_offset, + bit_precision, + } = dt + else { + unreachable!() + }; + let prec = u32::from(*bit_precision); + let v = element_bits(b, byte_order)?; + let mut x = field(v, u32::from(*bit_offset), prec); + if *signed && prec > 0 && prec < 128 && field(x, prec - 1, 1) == 1 { + x |= !0u128 << prec; + } + out.extend_from_slice(&x.to_le_bytes()[..*size as usize]); + Ok(()) +} + +fn partial_int(dt: &Datatype) -> bool { + matches!(dt, Datatype::FixedPoint { size, bit_offset, bit_precision, .. } + if *bit_offset != 0 || u32::from(*bit_precision) != size * 8) +} + +fn canon_str(b: &[u8], out: &mut Vec) { + let cut = b.iter().position(|&c| c == 0).unwrap_or(b.len()); + let mut s = &b[..cut]; + while let [rest @ .., b' '] = s { + s = rest; + } + out.push(b'S'); + out.extend_from_slice(&(s.len() as u32).to_le_bytes()); + out.extend_from_slice(s); +} + +fn hex(b: &[u8]) -> String { + b.iter().map(|x| format!("{x:02x}")).collect() +} + +fn dtype_str(dt: &Datatype) -> String { + match dt { + Datatype::FixedPoint { + size, + signed, + byte_order, + .. + } => { + format!( + "{}{}{}", + bo(byte_order), + if *signed { "i" } else { "u" }, + size + ) + } + Datatype::FloatingPoint { + size, byte_order, .. + } => format!("{}f{}", bo(byte_order), size), + Datatype::BitField { + size, byte_order, .. + } => format!("{}b{}", bo(byte_order), size), + Datatype::Time { size, .. } => format!("time{size}"), + Datatype::String { size, .. } => format!("S{size}"), + Datatype::Opaque { size, .. } => format!("V{size}"), + Datatype::Compound { size, members } => format!( + "{{{}}}{size}", + members + .iter() + .map(|m| format!("{}:{}", m.name, dtype_str(&m.datatype))) + .collect::>() + .join(",") + ), + Datatype::Reference { ref_type, .. } => format!("ref({ref_type:?})"), + Datatype::Enumeration { base_type, .. } => format!("enum({})", dtype_str(base_type)), + Datatype::VariableLength { + is_string: true, .. + } => "vlstr".into(), + Datatype::VariableLength { base_type, .. } => format!("vlen({})", dtype_str(base_type)), + Datatype::Array { + base_type, + dimensions, + } => format!("({}){dimensions:?}", dtype_str(base_type)), + } +} + +fn bo(b: &DatatypeByteOrder) -> &'static str { + match b { + DatatypeByteOrder::LittleEndian => "<", + DatatypeByteOrder::BigEndian => ">", + DatatypeByteOrder::Vax => "vax", + } +} + +fn is_group(h: &ObjectHeader) -> bool { + h.messages.iter().any(|m| { + matches!( + m.msg_type, + MessageType::LinkInfo | MessageType::Link | MessageType::SymbolTable + ) + }) +} + +fn main() { + install_hook(); + let path = std::env::args().nth(1).expect("usage: probe "); + let mut top = Map::new(); + top.insert("file".into(), Value::String(path.clone())); + let data = match std::fs::read(&path) { + Ok(d) => d, + Err(err) => { + top.insert("open_error".into(), Value::String(format!("Io({err})"))); + println!("{}", Value::Object(top)); + return; + } + }; + let sb = guarded(|| { + let off = signature::find_signature(&data).map_err(e)?; + Superblock::parse(&data, off).map_err(e) + }); + let sb = match sb { + Ok(sb) => sb, + Err(msg) => { + top.insert("open_error".into(), Value::String(msg)); + println!("{}", Value::Object(top)); + return; + } + }; + top.insert("superblock_version".into(), json!(sb.version)); + let ctx = Ctx { + data: &data, + os: sb.offset_size, + ls: sb.length_size, + base_dir: std::path::Path::new(&path) + .parent() + .map(|p| p.to_path_buf()) + .unwrap_or_default(), + heaps: RefCell::new(HashMap::new()), + }; + let mut objects: Vec = Vec::new(); + let mut visited = HashSet::new(); + let mut soft_v1 = 0u64; + // explicit DFS stack: (address, path) + let mut stack: Vec<(u64, String)> = vec![(sb.root_group_address, "/".to_string())]; + while let Some((addr, p)) = stack.pop() { + if objects.len() >= MAX_OBJECTS { + top.insert("truncated".into(), json!(true)); + break; + } + if !visited.insert(addr) { + continue; + } + let mut rec = Map::new(); + rec.insert("path".into(), Value::String(p.clone())); + let r = guarded(|| { + let h = ctx.header(addr)?; + Ok(h) + }); + let h = match r { + Ok(h) => h, + Err(msg) => { + rec.insert("kind".into(), Value::String("unknown".into())); + rec.insert("error".into(), Value::String(msg)); + objects.push(Value::Object(rec)); + continue; + } + }; + let is_ds = h + .messages + .iter() + .any(|m| m.msg_type == MessageType::DataLayout); + let kind = if is_ds { + "dataset" + } else if is_group(&h) || addr == sb.root_group_address { + "group" + } else if h + .messages + .iter() + .any(|m| m.msg_type == MessageType::Datatype) + { + "datatype" + } else { + "unknown" + }; + rec.insert("kind".into(), Value::String(kind.into())); + if kind == "dataset" + && let Err(msg) = guarded(|| ctx.read_dataset(&h, &mut rec)) + { + rec.insert("error".into(), Value::String(msg)); + } + if kind != "datatype" { + match guarded(|| ctx.attrs(&h)) { + Ok(m) => { + rec.insert("attrs".into(), Value::Object(m)); + } + Err(msg) => { + rec.insert("attrs_error".into(), Value::String(msg)); + } + } + } + if kind == "group" { + match guarded(|| ctx.entries(&h)) { + Ok(mut ents) => { + ents.retain(|en| { + if en.cache_type == 2 { + soft_v1 += 1; + false + } else { + true + } + }); + ents.sort_by(|a, b| a.name.cmp(&b.name)); + let base = if p == "/" { String::new() } else { p.clone() }; + for en in ents.into_iter().rev() { + stack.push((en.object_header_address, format!("{base}/{}", en.name))); + } + } + Err(msg) => { + rec.insert("list_error".into(), Value::String(msg)); + } + } + } + objects.push(Value::Object(rec)); + } + if soft_v1 > 0 { + top.insert("v1_soft_link_entries".into(), json!(soft_v1)); + } + top.insert("objects".into(), Value::Array(objects)); + println!("{}", Value::Object(top)); +} + +#[cfg(test)] +mod tests { + use super::*; + + /// The N-Bit float of libhdf5's `test/testfiles/le_data.h5` + /// (`Nbit_float_data_le`): offset 7, precision 20, sign bit 26, exponent + /// 20+6 (bias 31), mantissa 7+13. + fn nbit_f32(byte_order: DatatypeByteOrder) -> Datatype { + Datatype::FloatingPoint { + size: 4, + byte_order, + bit_offset: 7, + bit_precision: 20, + exponent_location: 20, + exponent_size: 6, + mantissa_location: 7, + mantissa_size: 13, + exponent_bias: 31, + } + } + + fn canon_one(dt: &Datatype, bytes: &[u8]) -> Vec { + let mut out = Vec::new(); + canon_custom_float(dt, bytes, &mut out).unwrap(); + out + } + + #[test] + fn nbit_float_canonicalises_to_the_value_libhdf5_returns() { + let le = nbit_f32(DatatypeByteOrder::LittleEndian); + let be = nbit_f32(DatatypeByteOrder::BigEndian); + assert!(!ieee_layout(&le)); + // 1.0: exponent = bias, mantissa 0 + let one: u32 = 31 << 20; + assert_eq!(canon_one(&le, &one.to_le_bytes()), 1.0f32.to_le_bytes()); + assert_eq!(canon_one(&be, &one.to_be_bytes()), 1.0f32.to_le_bytes()); + // -2.1999512 (h5py's reading of the file's -2.2): sign, e = 32, m = 819 + let v: u32 = (1 << 26) | (32 << 20) | (819 << 7); + assert_eq!( + canon_one(&le, &v.to_le_bytes()), + (-2.199_951_2f32).to_le_bytes() + ); + assert_eq!(canon_one(&le, &[0; 4]), 0.0f32.to_le_bytes()); + } + + #[test] + fn ieee_floats_keep_their_raw_bytes() { + let f32le = Datatype::FloatingPoint { + size: 4, + byte_order: DatatypeByteOrder::LittleEndian, + bit_offset: 0, + bit_precision: 32, + exponent_location: 23, + exponent_size: 8, + mantissa_location: 0, + mantissa_size: 23, + exponent_bias: 127, + }; + assert!(ieee_layout(&f32le)); + } + + #[test] + fn partial_precision_int_is_shifted_and_sign_extended() { + let dt = Datatype::FixedPoint { + size: 4, + byte_order: DatatypeByteOrder::BigEndian, + signed: true, + bit_offset: 4, + bit_precision: 17, + }; + assert!(partial_int(&dt)); + let stored = (((-5i32) as u32) & 0x1_FFFF) << 4; + let mut out = Vec::new(); + canon_partial_int(&dt, &stored.to_be_bytes(), &mut out).unwrap(); + assert_eq!(out, (-5i32).to_le_bytes()); + } +} diff --git a/conformance/ref.py b/conformance/ref.py new file mode 100755 index 0000000..573deeb --- /dev/null +++ b/conformance/ref.py @@ -0,0 +1,259 @@ +#!/usr/bin/env python3 +"""Reference probe: same JSON as the Rust `conformance-probe`, produced with h5py. + +Walk: iterative DFS from '/', children in sorted (UTF-8 byte) name order, hard +links only, each object once (first path wins, deduplicated by object identity). +Canonical value encoding: see harness/src/main.rs. +""" +import hashlib +import json +import os +import struct +import sys + +import numpy as np +import h5py + +try: + import hdf5plugin # noqa: F401 registers blosc/lz4/zstd/bzip2/... filters +except Exception: # pragma: no cover + pass + +MAX_BYTES = 200 * 1024 * 1024 +MAX_OBJECTS = 200_000 + + +def canon_str(b, out): + if isinstance(b, str): + b = b.encode("utf-8", "surrogateescape") + b = bytes(b) + cut = b.find(b"\x00") + if cut >= 0: + b = b[:cut] + b = b.rstrip(b" ") + out += b"S" + struct.pack(" numpy {dt} ({dt.itemsize})" + except Exception: # noqa: BLE001 + pass + + +def hash_values(arr, dt, rec): + if dt.subdtype is not None: + # h5py expands an HDF5 array element type into trailing array dims + dt = dt.subdtype[0] + arr = np.asarray(arr, dtype=dt) + if simple(dt): + c = np.ascontiguousarray(arr).astype(packed(dt)).tobytes() + else: + out = bytearray() + for x in arr.reshape(-1): + canon_el(dt, x, out) + c = bytes(out) + rec["hash"] = hashlib.sha256(c).hexdigest() + rec["head"] = c[:48].hex() + + +def err(e): + s = f"{type(e).__name__}: {e}" + return s.splitlines()[0][:400] if s else type(e).__name__ + + +def shape_of(s): + return "null" if s is None else list(s) + + +def n_bytes(shape, tid): + n = 1 + for d in shape or (): + n *= d + return n * tid.get_size() + + +def read_attrs(obj): + out = {} + names = sorted(obj.attrs.keys(), key=lambda s: s.encode("utf-8", "surrogateescape")) + for name in names: + rec = {} + try: + aid = obj.attrs.get_id(name) + rec["dtype"] = str(aid.dtype) + rec["shape"] = shape_of(aid.shape) + note_conversion(aid.get_type(), aid.dtype, rec) + if aid.shape is None: + hash_values(np.empty((0,), dtype=aid.dtype), aid.dtype, rec) + else: + val = obj.attrs[name] + hash_values(val, aid.dtype, rec) + except Exception as e: # noqa: BLE001 + rec = {"error": err(e)} + out[name] = rec + return out + + +def main(path): + top = {"file": path} + try: + f = h5py.File(path, "r") + except Exception as e: # noqa: BLE001 + top["open_error"] = err(e) + print(json.dumps(top)) + return + objects = [] + seen = set() + stack = [("/", None)] + while stack: + p, obj = stack.pop() + if len(objects) >= MAX_OBJECTS: + top["truncated"] = True + break + rec = {"path": p} + try: + if obj is None: + obj = f[p] + key = hash(obj.id) # h5py ObjectID hash = (fileno, object address/token) + except Exception as e: # noqa: BLE001 + rec["kind"] = "unknown" + rec["error"] = err(e) + objects.append(rec) + continue + if key in seen: + continue + seen.add(key) + if isinstance(obj, h5py.Dataset): + kind = "dataset" + elif isinstance(obj, h5py.Group): + kind = "group" + elif isinstance(obj, h5py.Datatype): + kind = "datatype" + else: + kind = "unknown" + rec["kind"] = kind + if kind == "dataset": + try: + dt = obj.dtype + rec["dtype"] = str(dt) + rec["shape"] = shape_of(obj.shape) + note_conversion(obj.id.get_type(), dt, rec) + if obj.shape is None: + hash_values(np.empty((0,), dtype=dt), dt, rec) + elif n_bytes(obj.shape, obj.id.get_type()) > MAX_BYTES: + rec["skipped"] = "too large" + else: + arr = np.empty(obj.shape, dtype=dt) + if arr.size: + try: + obj.read_direct(arr) + except Exception: # noqa: BLE001 + arr = obj[()] + hash_values(arr, dt, rec) + except Exception as e: # noqa: BLE001 + rec["error"] = err(e) + if kind != "datatype": + try: + rec["attrs"] = read_attrs(obj) + except Exception as e: # noqa: BLE001 + rec["attrs_error"] = err(e) + if kind == "group": + try: + names = sorted(obj.keys(), key=lambda s: s.encode("utf-8", "surrogateescape")) + base = "" if p == "/" else p + kids = [] + for n in names: + try: + link = obj.get(n, getlink=True) + except Exception: # noqa: BLE001 + link = None + if link is not None and not isinstance(link, h5py.HardLink): + continue + kids.append(f"{base}/{n}") + for k in reversed(kids): + stack.append((k, None)) + except Exception as e: # noqa: BLE001 + rec["list_error"] = err(e) + objects.append(rec) + top["objects"] = objects + print(json.dumps(top), flush=True) + # Exit without tearing down the h5py objects: freeing them for some files + # that hold references (hdf5's h5repack_attr_refs.h5, cve-2024-32623.h5) + # makes libhdf5 2.0 abort with "free(): chunks in smallbin corrupted" + # about half the time. That happens after the reading is done, so it says + # nothing about what h5py read, but it flipped those files between ok and + # h5py-cannot-read from one run to the next. + os._exit(0) + + +if __name__ == "__main__": + main(sys.argv[1]) diff --git a/conformance/report.py b/conformance/report.py new file mode 100644 index 0000000..dc82413 --- /dev/null +++ b/conformance/report.py @@ -0,0 +1,335 @@ +#!/usr/bin/env python3 +"""report.py + +Render the sweep's results (compare.py's results.json plus the raw per-side +runs) as CONFORMANCE.md, and write /report-meta.json (commit, +date, versions) for check.py --update. +""" +import collections +import datetime +import json +import os +import platform + +import subprocess +import sys + +import h5py +import numpy + +try: + import hdf5plugin + HDF5PLUGIN = hdf5plugin.version +except Exception: # noqa: BLE001 + HDF5PLUGIN = "not installed" + +R, OUT_MD, CORPUS = sys.argv[1], sys.argv[2], sys.argv[3] +HERE = os.path.dirname(os.path.abspath(__file__)) +ROOT = os.path.dirname(HERE) +CLASSES = ["ok", "our-error", "mismatch", "h5py-cannot-read", "panic", "hang", "crash", "oom"] + + +def sh(*cmd, cwd=ROOT): + try: + return subprocess.run(cmd, cwd=cwd, capture_output=True, text=True, timeout=30).stdout.strip() + except Exception: # noqa: BLE001 + return "" + + +def cpu_model(): + try: + for ln in open("/proc/cpuinfo"): + if ln.startswith(("model name", "Model")): + return ln.split(":", 1)[1].strip() + except OSError: + pass + return platform.processor() or "unknown" + + +def mem_gib(): + try: + for ln in open("/proc/meminfo"): + if ln.startswith("MemTotal:"): + return f"{int(ln.split()[1]) / 1048576:.0f} GiB" + except OSError: + pass + return "?" + + +res = json.load(open(os.path.join(R, "results.json"))) +meta_run = json.load(open(os.path.join(R, "meta.json"))) if os.path.exists(os.path.join(R, "meta.json")) else {} +rows = res["rows"] +issues = res.get("issues", {}) + +# safe.directory: a checkout owned by another user (a container) is still ours to read +commit = sh("git", "-c", "safe.directory=*", "rev-parse", "HEAD") or os.environ.get("GITHUB_SHA", "unknown") +lib_dirty = sh("git", "-c", "safe.directory=*", "status", "--porcelain", "--", "crates", "Cargo.toml") +h5dump_v = sh("h5dump", "--version").replace("h5dump: ", "") +meta = { + "date": datetime.datetime.now(datetime.timezone.utc).strftime("%Y-%m-%d %H:%M UTC"), + "commit": commit + (" (library sources modified)" if lib_dirty else ""), + "reference": f"h5py {h5py.__version__} / HDF5 {h5py.version.hdf5_version}", +} +json.dump(meta, open(os.path.join(R, "report-meta.json"), "w"), indent=1) + +pins = [] +for ln in open(os.path.join(HERE, "corpus.txt")): + if ln.strip() and not ln.lstrip().startswith("#"): + name, url, rev, root, *_ = ln.split() + pins.append((name, url, rev, root)) + +by_corpus = collections.defaultdict(collections.Counter) +for r in rows: + by_corpus[r["corpus"]][r["class"]] += 1 +total = collections.Counter(r["class"] for r in rows) + + +def ex_list(files, n=3): + s = ", ".join(f"`{f}`" for f in files[:n]) + return s + (f" (+{len(files) - n} more)" if len(files) > n else "") + + +# --- known causes that are not clawhdf5 bugs -------------------------------- +def is_h5py_be_vlen(i): + """h5py returns the elements of a VL sequence of a big-endian base type + with their file (big-endian) bytes but a native-endian dtype.""" + return (i["kind"] == "mismatch" and i["key"] in ("values", "attr-values") + and (i.get("ref_dtype") == "object") and (i.get("ours_dtype") or "").startswith("vlen(") + and ">" in (i.get("ours_dtype") or "")) + + +known = collections.defaultdict(list) +for r in rows: + if r["class"] != "mismatch": + continue + iss = issues.get(r["file"], []) + if iss and all(is_h5py_be_vlen(i) for i in iss): + known["h5py-be-vlen"].append(r["file"]) + + +# --- the CVE corpus: clawhdf5 vs h5dump vs h5py ------------------------------ +def side(run, name): + p = os.path.join(R, "runs", run, name) + if not os.path.exists(p + ".rc"): + return None + rc = int(open(p + ".rc").read().strip() or -1) + err = open(p + ".err", errors="replace").read() + try: + j = json.load(open(p + ".json")) + except Exception: # noqa: BLE001 + j = None + return rc, err, j + + +def outcome(s, rust=False): + """-> (bucket, text). bucket in read / error / panic / crash / hang / oom.""" + if s is None: + return "missing", "not run" + rc, err, j = s + if rc in (137, 124): + return "hang", "hang (killed at timeout)" + if "memory allocation of" in err or "MemoryError" in err or "bad_alloc" in err or "Cannot allocate" in err: + return "oom", "out of memory" + if rust and (rc == 101 or "PANIC:" in err): + return "panic", "panic" + if "overflowed its stack" in err: + return "crash", "stack overflow" + if rc == 139: + return "crash", "SIGSEGV" + if rc == 134: + return "crash", "SIGABRT" + (" (heap corruption)" if ("corrupted" in err or "free()" in err) else "") + if rc > 128: + return "crash", f"signal {rc - 128}" + if j is None: + return ("error", "error exit") if rc in (0, 1) else ("crash", f"exit {rc}") + if "open_error" in j: + return "error", "open error" + objs = j.get("objects", []) + ne = sum(1 for o in objs for k in ("error", "attrs_error", "list_error") if k in o) + ne += sum(1 for o in objs for a in (o.get("attrs") or {}).values() if "error" in a) + return "read", f"read {len(objs)} obj" + (f", {ne} errors" if ne else "") + + +def h5dump_outcome(s): + if s is None: + return "missing", "not run" + rc, err, _ = s + if rc in (137, 124): + return "hang", "hang (killed at timeout)" + if "memory allocation" in err or "Cannot allocate" in err: + return "oom", "out of memory" + if rc == 139: + return "crash", "SIGSEGV" + if rc == 134: + return "crash", "SIGABRT" + (" (heap corruption)" if ("corrupted" in err or "free()" in err) else "") + if rc > 128: + return "crash", f"signal {rc - 128}" + return ("read", "ok") if rc == 0 else ("error", "error exit") + + +cve_rows = [] +buckets = {"clawhdf5": collections.Counter(), "h5dump": collections.Counter(), "h5py": collections.Counter()} +ours_panic = {r["file"] for r in rows if r["class"] == "panic"} +for r in rows: + if r["corpus"] != "cve_hdf5": + continue + run = r["file"].replace("/", "__") + o = outcome(side(run, "ours"), rust=True) + if o[0] == "read" and r["file"] in ours_panic: + o = ("panic", "caught panic") + p = outcome(side(run, "ref")) + d = h5dump_outcome(side(run, "h5dump")) + buckets["clawhdf5"][o[0]] += 1 + buckets["h5py"][p[0]] += 1 + buckets["h5dump"][d[0]] += 1 + cve_rows.append((r["file"].split("/", 1)[1], d[1], p[1], o[1], r["class"])) + +# --- render ----------------------------------------------------------------- +L = [] +w = L.append +w("# clawhdf5 conformance report") +w("") +w("Every HDF5 file of eight public corpora (pinned by commit) is read twice — by") +w("clawhdf5 (`conformance/probe`, the same `clawhdf5-format` calls the facade") +w("makes) and by h5py/libhdf5 (`conformance/ref.py`) — and the two readings are") +w("compared object by object: the set of hard-linked objects, each dataset's and") +w("attribute's shape, and a SHA-256 of its values in a canonical encoding. The") +w("CVE corpus is also run through `h5dump`. Each side runs under a timeout and an") +w("address-space limit, so a hang, crash or runaway allocation is recorded, not") +w("fatal. This file is generated by `conformance/run.sh`; do not edit it by hand.") +w("") +w("## Run") +w("") +w("| | |") +w("|---|---|") +w(f"| date | {meta['date']} |") +w(f"| clawhdf5 commit | `{meta['commit']}` |") +w(f"| machine | `{platform.node()}`: {cpu_model()}, {os.cpu_count()} CPUs, {mem_gib()}, {platform.system()} {platform.release()} {platform.machine()} |") +w(f"| command | `{os.environ.get('CONFORMANCE_CMD', 'conformance/run.sh')}` |") +w(f"| rustc | {sh('rustc', '-V')} |") +w(f"| reference | h5py {h5py.__version__}, HDF5 {h5py.version.hdf5_version}, numpy {numpy.__version__}, hdf5plugin {HDF5PLUGIN}, Python {platform.python_version()} |") +w(f"| h5dump | {h5dump_v} (CVE corpus only) |") +if meta_run: + w(f"| limits | {meta_run.get('timeout_s')} s timeout (SIGKILL), {int(meta_run.get('mem_kb', 0)) // 1024} MiB address space, per process; {meta_run.get('jobs')} files in parallel |") + w(f"| runtime | {meta_run.get('probe_seconds')} s probing + comparing ({meta_run.get('build_seconds')} s fetch/build before it) |") +w("") +w("## Results") +w("") +w("A file's class is the first that applies:") +w("") +w("- **panic / hang / crash / oom** — clawhdf5 panicked (caught per object or not), hit the timeout, died on a signal, or failed an allocation. The CI gate fails on any of these.") +w("- **h5py-cannot-read** — libhdf5 could not open the file (or itself crashed or hung). Nothing to compare against; most are the deliberately malformed CVE reproducers.") +w("- **our-error** — clawhdf5 returned an error for something h5py reads.") +w("- **mismatch** — both read it, but the shapes, values, object set or attribute set differ.") +w("- **ok** — every object h5py reads, clawhdf5 reads identically.") +w("") +w("| corpus | files | " + " | ".join(CLASSES) + " |") +w("|---" * (len(CLASSES) + 2) + "|") +for c in sorted(by_corpus): + cnt = by_corpus[c] + w(f"| {c} | {sum(cnt.values())} | " + " | ".join(str(cnt.get(k, 0)) for k in CLASSES) + " |") +w(f"| **all** | **{len(rows)}** | " + " | ".join(f"**{total.get(k, 0)}**" for k in CLASSES) + " |") +w("") +n_known = sum(len(v) for v in known.values()) +if n_known: + w(f"{n_known} of the {total.get('mismatch', 0)} mismatches are a known h5py bug, not ours (see *Known not-our-bug*).") + w("") +w("Corpora (fetched by `conformance/fetch-corpus.sh` into the gitignored `conformance/.cache/`):") +w("") +w("| corpus | source | commit |") +w("|---|---|---|") +for name, url, rev, root in pins: + w(f"| {name} | {url.removesuffix('.git')}" + ("" if root == "." else f" (`{root}`)") + f" | `{rev[:12]}` |") +w("") + +w("## Panics, hangs, crashes, out-of-memory") +w("") +if not res["panics"]: + w("None.") +else: + for p in res["panics"]: + w(f"- `{p['file']}` [{p['class']}] {p['detail']}") +w("") + +w("## Our-error root causes") +w("") +w("Grouped by normalised error message. *files* counts files whose class this cause affects.") +w("") +w("| files | objects | error | examples |") +w("|---:|---:|---|---|") +for k, v in res["root_causes"].items(): + w(f"| {v['files']} | {v['count']} | `{k.replace('|', '/')}` | {ex_list(v['file_list'])} |") +w("") +w("## Mismatch root causes") +w("") +w("| files | objects | cause | examples |") +w("|---:|---:|---|---|") +for k, v in res["mismatch_causes"].items(): + w(f"| {v['files']} | {v['count']} | `{k.replace('|', '/')}` | {ex_list(v['file_list'])} |") +w("") + +w("## CVE corpus: clawhdf5 vs h5dump vs h5py") +w("") +w(f"The {len(cve_rows)} files of [HDFGroup/cve_hdf5](https://github.com/HDFGroup/cve_hdf5) — reproducers for") +w("published libhdf5 CVEs and fuzzer finds. *read* = produced output (possibly with per-object") +w("errors), *error* = refused cleanly. h5dump exits non-zero on any error anywhere in a file, so") +w("its read/error split is not comparable with the other two rows; the panic, crash, hang and oom") +w("columns are.") +w("") +w("| tool | read | error | panic | crash | hang | oom |") +w("|---|---:|---:|---:|---:|---:|---:|") +for tool, label in (("clawhdf5", "clawhdf5"), ("h5dump", f"h5dump {h5dump_v.split()[-1] if h5dump_v else ''}"), + ("h5py", f"h5py {h5py.__version__} / HDF5 {h5py.version.hdf5_version}")): + b = buckets[tool] + w(f"| {label} | " + " | ".join(str(b.get(k, 0)) for k in ("read", "error", "panic", "crash", "hang", "oom")) + " |") +w("") +w("
Per-file outcomes") +w("") +w("| file | h5dump | h5py | clawhdf5 | class |") +w("|---|---|---|---|---|") +for f, d, p, o, cls in cve_rows: + w(f"| {f} | {d} | {p} | {o} | {cls} |") +w("") +w("
") +w("") + +w("## Known not-our-bug") +w("") +w("- **h5py big-endian variable-length sequences.** h5py returns the elements of a VL sequence") +w(" whose base type is big-endian with the file's big-endian bytes but a native (little-endian)") +w(" numpy dtype, so the values it reports are byte-swapped garbage; `h5dump` prints the values") +w(" clawhdf5 reads. Reproducer: `h5py.vlen_dtype(np.dtype('>f4'))` dataset holding `[1.0, 2.0]`") +w(" reads back in h5py as `[4.6e-41, 9.0e-44]`. Affected here: " + + (ex_list(sorted(known["h5py-be-vlen"]), 10) if known["h5py-be-vlen"] else "none") + ".") +w("- **Non-IEEE floats and partial-precision integers (N-Bit).** libhdf5 converts a float whose") +w(" bit layout is not IEEE (e.g. `H5Tset_precision` for the N-Bit filter) or an integer with a") +w(" bit offset / reduced precision into the plain numpy type of the same size. The probe") +w(" compares such values as converted numbers, not raw file bytes (before 2026-09-25 it compared") +w(" raw bytes, which reported every N-Bit float dataset as a mismatch).") +if res["incomparable"]: + w("- **Types h5py widens.** Where h5py reads a type into a numpy type of a different size") + w(" (FP8 -> float16, bfloat16 -> float32, x87 long double -> float128) the values are not") + w(" compared (shape and presence still are): " + + ", ".join(f"{k} ({n}x)" for k, n in res["incomparable"]) + ".") +w("- **References** are compared by presence only (`R`), not by target.") +w("") +if res.get("ref_only_errors"): + w("## Objects h5py fails on but clawhdf5 reads") + w("") + for k, n in res["ref_only_errors"][:15]: + w(f"- {n} x `{k}`") + w("") +w("## Reproduce") +w("") +w("```sh") +w("# needs: Rust, python3 with h5py numpy hdf5plugin (conformance/requirements.txt), h5dump (hdf5-tools), git") +w("CLAWHDF5_PYTHON=/path/to/venv/bin/python conformance/run.sh") +w("```") +w("") +w("The corpus (about 450 MB of sparse checkouts) is cached in `conformance/.cache/`; results for") +w("every file, both sides' raw JSON and stderr, are in `conformance/.cache/results/`.") +w("`conformance/baseline.json` holds the ok files the nightly CI job (`.gitea/workflows/conformance.yml`)") +w("must keep; `conformance/run.sh --update-baseline` rewrites it.") + +with open(OUT_MD, "w") as fh: + fh.write("\n".join(L) + "\n") diff --git a/conformance/requirements.txt b/conformance/requirements.txt new file mode 100644 index 0000000..5cd36af --- /dev/null +++ b/conformance/requirements.txt @@ -0,0 +1,6 @@ +# The reference side of the conformance sweep. Pinned so the nightly job and a +# local run compare against the same libhdf5 (h5py wheels bundle it). +h5py==3.16.0 +numpy==2.5.3 +hdf5plugin==7.1.0 +netCDF4==1.7.4 diff --git a/conformance/run.sh b/conformance/run.sh new file mode 100755 index 0000000..5ceb635 --- /dev/null +++ b/conformance/run.sh @@ -0,0 +1,88 @@ +#!/usr/bin/env bash +# conformance/run.sh — the clawhdf5 conformance sweep, end to end. +# +# fetch the pinned corpora (cached) -> build the probe -> probe every file +# with clawhdf5 and with h5py (and h5dump for the CVE corpus), each under a +# timeout and a memory limit -> compare -> write CONFORMANCE.md -> check the +# result against conformance/baseline.json. +# +# Usage: conformance/run.sh [--no-fetch] [--no-report] [--update-baseline] +# +# Environment: +# CLAWHDF5_PYTHON python with h5py, numpy, hdf5plugin (default: repo .venv, then python3) +# CONFORMANCE_CACHE corpus / build / results cache (default: conformance/.cache) +# CONFORMANCE_OUT results directory (default: $CONFORMANCE_CACHE/results) +# CONFORMANCE_REPORT report path (default: CONFORMANCE.md at the repo root) +# JOBS parallel files (default: nproc) +# CONFORMANCE_PROBE use this prebuilt probe binary instead of building one +# TMO / MEM_KB per-process timeout in seconds (20) / address-space limit in KiB (4 GiB) +# +# Exit status: 0 = gate passed; 1 = a panic/hang/crash/oom in clawhdf5, or the +# ok count fell below the baseline, or a baseline-ok file regressed; 2 = setup error. +set -euo pipefail +HERE="$(cd "$(dirname "$0")" && pwd)" +ROOT="$(cd "$HERE/.." && pwd)" +FETCH=1 REPORT=1 UPDATE=0 +for a in "$@"; do + case "$a" in + --no-fetch) FETCH=0 ;; + --no-report) REPORT=0 ;; + --update-baseline) UPDATE=1 ;; + -h|--help) sed -n '2,23p' "$0"; exit 0 ;; + *) echo "unknown argument: $a" >&2; exit 2 ;; + esac +done + +export PATH="$HOME/.cargo/bin:$PATH" +CACHE="${CONFORMANCE_CACHE:-$HERE/.cache}" +mkdir -p "$CACHE"; CACHE="$(cd "$CACHE" && pwd)" +OUT="${CONFORMANCE_OUT:-$CACHE/results}" +REPORT_PATH="${CONFORMANCE_REPORT:-$ROOT/CONFORMANCE.md}" +JOBS="${JOBS:-$(nproc 2>/dev/null || echo 4)}" +if [ -n "${CLAWHDF5_PYTHON:-}" ]; then PY="$CLAWHDF5_PYTHON" +elif [ -x "$ROOT/.venv/bin/python" ]; then PY="$ROOT/.venv/bin/python" +else PY="$(command -v python3)"; fi +export PY TMO="${TMO:-20}" MEM_KB="${MEM_KB:-4194304}" +command -v h5dump >/dev/null || { echo "error: h5dump not found (install hdf5-tools)" >&2; exit 2; } +"$PY" -c 'import h5py, numpy, hdf5plugin' || { echo "error: $PY lacks h5py/numpy/hdf5plugin" >&2; exit 2; } + +t0=$(date +%s) +[ "$FETCH" = 1 ] && bash "$HERE/fetch-corpus.sh" "$CACHE" +C="$CACHE/corpus" +[ -d "$C" ] || { echo "error: no corpus in $C (run without --no-fetch)" >&2; exit 2; } + +if [ -n "${CONFORMANCE_PROBE:-}" ]; then + export PROBE="$CONFORMANCE_PROBE" # a prebuilt probe, e.g. an older one for a before/after +else + echo "== building the probe" + CARGO_TARGET_DIR="${CARGO_TARGET_DIR:-$CACHE/target}" \ + cargo build -q --release --manifest-path "$HERE/probe/Cargo.toml" + export PROBE="${CARGO_TARGET_DIR:-$CACHE/target}/release/conformance-probe" +fi +t1=$(date +%s) + +rm -rf "$OUT"; mkdir -p "$OUT" +"$PY" "$HERE/list_files.py" "$C" > "$OUT/files.txt" +echo "== probing $(wc -l <"$OUT/files.txt") files, $JOBS at a time (timeout ${TMO}s, limit $((MEM_KB / 1024)) MiB)" +export C OUT HERE +# The shell's "Segmentation fault (core dumped)" notices go to probe.log; the +# signals themselves are recorded in each side's .rc. +xargs -a "$OUT/files.txt" -d '\n' -P "$JOBS" -I{} bash -c ' + f="$1"; d="$OUT/runs/${f//\//__}" + case "$f" in cve_hdf5/*) export WITH_H5DUMP=1 ;; esac + "$HERE/run_one.sh" "$C/$f" "$d"' _ {} 2>"$OUT/probe.log" +echo "== comparing" +"$PY" "$HERE/compare.py" "$OUT" >/dev/null +t2=$(date +%s) +cat > "$OUT/meta.json" < +# +# Probe one file with clawhdf5 (PROBE) and with h5py (PY ref.py), and with +# h5dump too when WITH_H5DUMP is set. Each side runs under a timeout (TMO +# seconds, SIGKILL) and an address-space limit (MEM_KB), with core dumps off. +# Writes /.{json,err,rc}; rc 137 = killed by the timeout. +set -u +f="$1"; out="$2"; mkdir -p "$out" +HERE="$(cd "$(dirname "$0")" && pwd)" +: "${PROBE:?PROBE must name the conformance-probe binary}" +: "${PY:?PY must name a python with h5py}" +TMO="${TMO:-20}" +MEM_KB="${MEM_KB:-4194304}" +run() { # name cmd... + local name=$1; shift + ( ulimit -v "$MEM_KB"; ulimit -c 0; RUST_BACKTRACE=1 exec timeout -s KILL "$TMO" "$@" ) \ + >"$out/$name.json" 2>"$out/$name.err" + echo $? >"$out/$name.rc" +} +run ours "$PROBE" "$f" +run ref "$PY" "$HERE/ref.py" "$f" +if [ -n "${WITH_H5DUMP:-}" ]; then + run h5dump h5dump "$f" + : >"$out/h5dump.json" # h5dump's text dump is not compared, only its exit status +fi +exit 0