osobhandClaude Opus 5.5
9179aa356e
conformance/run.sh fetches eight public HDF5 corpora pinned by commit (conformance/corpus.txt) into a gitignored cache, reads every file with clawhdf5 (conformance/probe, a crate outside the workspace) and with h5py/libhdf5 (ref.py), and the CVE files with h5dump, each under a timeout and an address-space limit; compare.py classifies the files, report.py writes CONFORMANCE.md and check.py gates on panics/hangs/crashes/OOM and on regressions against conformance/baseline.json. ~25 s once cached. Changes from the ad-hoc audit harness: - the probe compares non-IEEE-layout floats (N-Bit) and integers with a bit offset or reduced precision as the values libhdf5 converts them to, not raw file bytes: 8 files that showed as mismatches now read identically; - ref.py exits without tearing down h5py objects: libhdf5 2.0 aborts while freeing them for two files about half the time, which flipped them between ok and h5py-cannot-read from run to run; - the file list is defined (list_files.py): netCDF classic files are left out, 11 HDF5 files the ad-hoc sweep missed are in. Co-Authored-By: Claude Opus 5.5 (1M context) <[email protected]>
49 lines
1.7 KiB
Python
49 lines
1.7 KiB
Python
#!/usr/bin/env python3
|
|
"""list_files.py <corpus_dir>: print the files the sweep probes, one per line,
|
|
as <corpus>/<path> in byte order.
|
|
|
|
* every file named *.h5 *.hdf5 *.he5 *.nc *.nc4 *.hdf *.h5f in each corpus,
|
|
except netCDF classic / 64-bit-offset / CDF5 files (magic "CDF"): they are
|
|
not HDF5, so neither side can read them and they say nothing;
|
|
* plus, for cve_hdf5, every file in cvefiles/ and fuzzerfiles/ except
|
|
.md/.c sources — the reproducers are mostly extension-less, and they are
|
|
kept whatever their bytes look like (that is their point).
|
|
"""
|
|
import os
|
|
import sys
|
|
|
|
EXTS = (".h5", ".hdf5", ".he5", ".nc", ".nc4", ".hdf", ".h5f")
|
|
|
|
|
|
def walk(top):
|
|
for dirpath, dirnames, filenames in os.walk(top):
|
|
dirnames[:] = [d for d in dirnames if d != ".git"]
|
|
for fn in filenames:
|
|
p = os.path.join(dirpath, fn)
|
|
if os.path.isfile(p) and not os.path.islink(p):
|
|
yield os.path.relpath(p, top)
|
|
|
|
|
|
def main(root):
|
|
out = set()
|
|
for corpus in sorted(os.listdir(root)):
|
|
top = os.path.join(root, corpus)
|
|
if not os.path.isdir(top):
|
|
continue
|
|
for rel in walk(top):
|
|
path = os.path.join(top, rel)
|
|
if rel.lower().endswith(EXTS):
|
|
with open(path, "rb") as fh:
|
|
if fh.read(3) == b"CDF":
|
|
continue
|
|
out.add(f"{corpus}/{rel}")
|
|
elif corpus == "cve_hdf5" and rel.split(os.sep)[0] in ("cvefiles", "fuzzerfiles") \
|
|
and not rel.endswith((".md", ".c")):
|
|
out.add(f"{corpus}/{rel}")
|
|
for f in sorted(out, key=lambda s: s.encode()):
|
|
print(f)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main(sys.argv[1])
|