osobhandClaude Opus 5.5
5c44630ea2
The last 3 our-errors and 2 mismatches were documented as not ours but
still counted against us, on a heuristic (any big-endian VL mismatch) and
a fixed list.
- ref.py checks that the installed h5py returns big-endian VL elements
with the file's bytes under a little-endian dtype (writing and reading
a vlen('>f4') in memory) and, if so, relabels them with the file's byte
order before hashing, marking the object `ref_fix`. The values are now
compared: attr_datatypes.hdf5 /@vlen_uint64 and tcomplex_be.h5
/VariableLengthDatasetFloatComplex are identical to ours (h5dump 1.14.6
prints the same (1, 2), (3, 4, 5), (42)).
- ref_bugs.py re-reads each object h5py reads only through a libhdf5 bug
in six processes with different heaps (import order, MALLOC_PERTURB_).
Values the file determines are the same every time; these three change
(6, 6 and 3 distinct results), so they are over-read memory, not data
clawhdf5 could match. compare.py classifies a file `ref-bug` only when
every difference is such an object confirmed in the same run.
- report.py: the ref-bug class, the evidence table, the corrected
objects; test_ref.py covers both (run in the nightly job).
Co-Authored-By: Claude Opus 5.5 (1M context) <[email protected]>
91 lines
4.0 KiB
Bash
Executable File
91 lines
4.0 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
# conformance/run.sh — the clawhdf5 conformance sweep, end to end.
|
|
#
|
|
# fetch the pinned corpora (cached) -> build the probe -> probe every file
|
|
# with clawhdf5 and with h5py (and h5dump for the CVE corpus), each under a
|
|
# timeout and a memory limit -> compare -> write CONFORMANCE.md -> check the
|
|
# result against conformance/baseline.json.
|
|
#
|
|
# Usage: conformance/run.sh [--no-fetch] [--no-report] [--update-baseline]
|
|
#
|
|
# Environment:
|
|
# CLAWHDF5_PYTHON python with h5py, numpy, hdf5plugin (default: repo .venv, then python3)
|
|
# CONFORMANCE_CACHE corpus / build / results cache (default: conformance/.cache)
|
|
# CONFORMANCE_OUT results directory (default: $CONFORMANCE_CACHE/results)
|
|
# CONFORMANCE_REPORT report path (default: CONFORMANCE.md at the repo root)
|
|
# JOBS parallel files (default: nproc)
|
|
# CONFORMANCE_PROBE use this prebuilt probe binary instead of building one
|
|
# TMO / MEM_KB per-process timeout in seconds (20) / address-space limit in KiB (4 GiB)
|
|
#
|
|
# Exit status: 0 = gate passed; 1 = a panic/hang/crash/oom in clawhdf5, or the
|
|
# ok count fell below the baseline, or a baseline-ok file regressed; 2 = setup error.
|
|
set -euo pipefail
|
|
HERE="$(cd "$(dirname "$0")" && pwd)"
|
|
ROOT="$(cd "$HERE/.." && pwd)"
|
|
FETCH=1 REPORT=1 UPDATE=0
|
|
for a in "$@"; do
|
|
case "$a" in
|
|
--no-fetch) FETCH=0 ;;
|
|
--no-report) REPORT=0 ;;
|
|
--update-baseline) UPDATE=1 ;;
|
|
-h|--help) sed -n '2,23p' "$0"; exit 0 ;;
|
|
*) echo "unknown argument: $a" >&2; exit 2 ;;
|
|
esac
|
|
done
|
|
|
|
export PATH="$HOME/.cargo/bin:$PATH"
|
|
CACHE="${CONFORMANCE_CACHE:-$HERE/.cache}"
|
|
mkdir -p "$CACHE"; CACHE="$(cd "$CACHE" && pwd)"
|
|
OUT="${CONFORMANCE_OUT:-$CACHE/results}"
|
|
REPORT_PATH="${CONFORMANCE_REPORT:-$ROOT/CONFORMANCE.md}"
|
|
JOBS="${JOBS:-$(nproc 2>/dev/null || echo 4)}"
|
|
if [ -n "${CLAWHDF5_PYTHON:-}" ]; then PY="$CLAWHDF5_PYTHON"
|
|
elif [ -x "$ROOT/.venv/bin/python" ]; then PY="$ROOT/.venv/bin/python"
|
|
else PY="$(command -v python3)"; fi
|
|
export PY TMO="${TMO:-20}" MEM_KB="${MEM_KB:-4194304}"
|
|
command -v h5dump >/dev/null || { echo "error: h5dump not found (install hdf5-tools)" >&2; exit 2; }
|
|
"$PY" -c 'import h5py, numpy, hdf5plugin' || { echo "error: $PY lacks h5py/numpy/hdf5plugin" >&2; exit 2; }
|
|
|
|
t0=$(date +%s)
|
|
[ "$FETCH" = 1 ] && bash "$HERE/fetch-corpus.sh" "$CACHE"
|
|
C="$CACHE/corpus"
|
|
[ -d "$C" ] || { echo "error: no corpus in $C (run without --no-fetch)" >&2; exit 2; }
|
|
|
|
if [ -n "${CONFORMANCE_PROBE:-}" ]; then
|
|
export PROBE="$CONFORMANCE_PROBE" # a prebuilt probe, e.g. an older one for a before/after
|
|
else
|
|
echo "== building the probe"
|
|
CARGO_TARGET_DIR="${CARGO_TARGET_DIR:-$CACHE/target}" \
|
|
cargo build -q --release --manifest-path "$HERE/probe/Cargo.toml"
|
|
export PROBE="${CARGO_TARGET_DIR:-$CACHE/target}/release/conformance-probe"
|
|
fi
|
|
t1=$(date +%s)
|
|
|
|
rm -rf "$OUT"; mkdir -p "$OUT"
|
|
"$PY" "$HERE/list_files.py" "$C" > "$OUT/files.txt"
|
|
echo "== probing $(wc -l <"$OUT/files.txt") files, $JOBS at a time (timeout ${TMO}s, limit $((MEM_KB / 1024)) MiB)"
|
|
export C OUT HERE
|
|
# The shell's "Segmentation fault (core dumped)" notices go to probe.log; the
|
|
# signals themselves are recorded in each side's .rc.
|
|
xargs -a "$OUT/files.txt" -d '\n' -P "$JOBS" -I{} bash -c '
|
|
f="$1"; d="$OUT/runs/${f//\//__}"
|
|
case "$f" in cve_hdf5/*) export WITH_H5DUMP=1 ;; esac
|
|
"$HERE/run_one.sh" "$C/$f" "$d"' _ {} 2>"$OUT/probe.log"
|
|
echo "== re-checking the objects h5py reads only through a libhdf5 bug"
|
|
"$PY" "$HERE/ref_bugs.py" "$C" > "$OUT/ref_bugs.json" 2> "$OUT/ref_bugs.err" || true
|
|
echo "== comparing"
|
|
"$PY" "$HERE/compare.py" "$OUT" >/dev/null
|
|
t2=$(date +%s)
|
|
cat > "$OUT/meta.json" <<EOF
|
|
{"build_seconds": $((t1 - t0)), "probe_seconds": $((t2 - t1)), "jobs": $JOBS, "timeout_s": $TMO, "mem_kb": $MEM_KB}
|
|
EOF
|
|
export CONFORMANCE_CMD="${CONFORMANCE_CMD:-conformance/run.sh${*:+ $*}}"
|
|
if [ "$REPORT" = 1 ]; then
|
|
"$PY" "$HERE/report.py" "$OUT" "$REPORT_PATH" "$C"
|
|
echo "== wrote $REPORT_PATH"
|
|
fi
|
|
if [ "$UPDATE" = 1 ]; then
|
|
"$PY" "$HERE/check.py" "$OUT" "$HERE/baseline.json" --update
|
|
fi
|
|
"$PY" "$HERE/check.py" "$OUT" "$HERE/baseline.json"
|