test(tools): h5rs sweeps over the conformance corpora
scripts/h5rs-fuzz.sh runs every h5rs subcommand over every file of a corpus (default: the HDF Group's CVE reproducers), optionally with byte-flipped copies (MUTATE=N), under a timeout and a memory limit, with a debug build so integer overflow panics instead of wrapping; any exit status above 2 (a caught panic, a timeout, a signal) fails it. It found size*8 overflows in the datatype names on cve-2021-46244.h5, cve-2024-29161.h5 and unknown-1.h5 (fixed in the crate before it landed). scripts/h5rs-check-ok-files.sh runs check (--data) over the conformance files that clawhdf5 and h5py both read in full; none may be flagged. Co-Authored-By: Claude Opus 5.5 (1M context) <[email protected]>
This commit is contained in:
osobh
Executable
+79
@@ -0,0 +1,79 @@
|
||||
#!/usr/bin/env bash
|
||||
# scripts/h5rs-check-ok-files.sh — run `h5rs check` over the files the
|
||||
# conformance sweep reads correctly and completely: the `ok_files` of
|
||||
# conformance/baseline.json (clawhdf5 and h5py agree on every object) whose
|
||||
# probe results record no error at all (some ok files are deliberately
|
||||
# corrupt test files that both readers refuse in the same way; those are
|
||||
# left out). A validator that flags a file libhdf5 and clawhdf5 both read
|
||||
# in full has a false positive — unless the file really is damaged in a way
|
||||
# readers tolerate, which the report must then show.
|
||||
#
|
||||
# Usage: scripts/h5rs-check-ok-files.sh [--data] [SAMPLE]
|
||||
# --data pass --data to check (decode every chunk)
|
||||
# SAMPLE check only every SAMPLE-th file (default 1: all of them)
|
||||
#
|
||||
# Environment: H5RS (binary; default builds release), CONFORMANCE_CACHE
|
||||
# (corpus cache, default conformance/.cache), CONFORMANCE_OUT (the sweep's
|
||||
# results, default $CONFORMANCE_CACHE/results; run conformance/run.sh first),
|
||||
# TMO (timeout, default 60).
|
||||
#
|
||||
# Exit status: 0 = no file flagged; 1 = some file flagged or failed (listed);
|
||||
# 2 = setup error.
|
||||
set -uo pipefail
|
||||
HERE="$(cd "$(dirname "$0")" && pwd)"
|
||||
ROOT="$(cd "$HERE/.." && pwd)"
|
||||
export PATH="$HOME/.cargo/bin:$PATH"
|
||||
DATA=()
|
||||
if [ "${1:-}" = "--data" ]; then DATA=(--data); shift; fi
|
||||
SAMPLE="${1:-1}"
|
||||
CACHE="${CONFORMANCE_CACHE:-$ROOT/conformance/.cache}"
|
||||
C="$CACHE/corpus"
|
||||
R="${CONFORMANCE_OUT:-$CACHE/results}"
|
||||
[ -d "$C" ] || { echo "error: no corpus in $C (run conformance/fetch-corpus.sh)" >&2; exit 2; }
|
||||
[ -d "$R/runs" ] || { echo "error: no sweep results in $R (run conformance/run.sh)" >&2; exit 2; }
|
||||
if [ -z "${H5RS:-}" ]; then
|
||||
cargo build -q --release -p clawhdf5-tools --manifest-path "$ROOT/Cargo.toml" || exit 2
|
||||
H5RS="${CARGO_TARGET_DIR:-$ROOT/target}/release/h5rs"
|
||||
fi
|
||||
TMO="${TMO:-60}"
|
||||
|
||||
mapfile -t FILES < <(python3 -c '
|
||||
import json, os, sys
|
||||
b = json.load(open(sys.argv[1]))
|
||||
runs, step = sys.argv[3], int(sys.argv[2])
|
||||
def clean(f):
|
||||
"""Both readers read every object, attribute and group listing."""
|
||||
for side in ("ours.json", "ref.json"):
|
||||
try:
|
||||
d = json.load(open(os.path.join(runs, f.replace("/", "__"), side)))
|
||||
except (OSError, ValueError):
|
||||
return False
|
||||
if "open_error" in d or d.get("truncated"):
|
||||
return False
|
||||
for o in d.get("objects", []):
|
||||
if "error" in o or "attrs_error" in o or "list_error" in o:
|
||||
return False
|
||||
if any(isinstance(a, dict) and "error" in a for a in o.get("attrs", {}).values()):
|
||||
return False
|
||||
return True
|
||||
files = [f for f in b["ok_files"] if clean(f)]
|
||||
total = len(b["ok_files"])
|
||||
print(f"{len(files)} of {total} ok files are read in full by both readers", file=sys.stderr)
|
||||
for i, f in enumerate(files):
|
||||
if i % step == 0:
|
||||
print(f)
|
||||
' "$ROOT/conformance/baseline.json" "$SAMPLE" "$R/runs")
|
||||
|
||||
flagged=0 checked=0
|
||||
for f in "${FILES[@]}"; do
|
||||
checked=$((checked + 1))
|
||||
out=$(timeout "$TMO" "$H5RS" check -q "${DATA[@]}" "$C/$f" 2>&1)
|
||||
rc=$?
|
||||
if [ $rc -ne 0 ]; then
|
||||
flagged=$((flagged + 1))
|
||||
echo "== rc=$rc $f"
|
||||
echo "$out" | head -5 | sed 's/^/ /'
|
||||
fi
|
||||
done
|
||||
echo "== checked $checked ok files: $flagged flagged"
|
||||
[ $flagged -eq 0 ]
|
||||
Executable
+115
@@ -0,0 +1,115 @@
|
||||
#!/usr/bin/env bash
|
||||
# scripts/h5rs-fuzz.sh — run every `h5rs` subcommand over every file in a
|
||||
# corpus of hostile HDF5 files, each under a timeout and a memory limit, and
|
||||
# fail on any panic, crash or hang.
|
||||
#
|
||||
# Usage: scripts/h5rs-fuzz.sh [CORPUS_DIR ...]
|
||||
# default corpus: conformance/.cache/corpus/cve_hdf5 (fetch it with
|
||||
# conformance/fetch-corpus.sh; the HDF Group's CVE reproducers)
|
||||
#
|
||||
# Environment:
|
||||
# H5RS h5rs binary to test (default: a debug build, for overflow checks)
|
||||
# TMO per-run timeout in seconds (default 60)
|
||||
# MEM_KB per-run address-space limit in KiB (default 4 GiB)
|
||||
# JOBS files in parallel (default nproc)
|
||||
# MUTATE also run on N byte-flipped copies of each file (default 0)
|
||||
# MAX_BYTES --max-bytes for the commands that read values (default 16 MiB)
|
||||
#
|
||||
# A run may exit 0 (fine), 1 (problems found / could not read something) or
|
||||
# 2 (error). Anything else fails the sweep: 3 is a caught panic (h5rs prints
|
||||
# "internal error"), 124 a timeout, 128+N a signal (crash, abort, OOM kill).
|
||||
#
|
||||
# Exit status: 0 = every run ended cleanly; 1 = at least one did not (listed);
|
||||
# 2 = setup error.
|
||||
set -uo pipefail
|
||||
HERE="$(cd "$(dirname "$0")" && pwd)"
|
||||
ROOT="$(cd "$HERE/.." && pwd)"
|
||||
export PATH="$HOME/.cargo/bin:$PATH"
|
||||
|
||||
CORPORA=("$@")
|
||||
[ ${#CORPORA[@]} -eq 0 ] && CORPORA=("$ROOT/conformance/.cache/corpus/cve_hdf5")
|
||||
for c in "${CORPORA[@]}"; do
|
||||
[ -d "$c" ] || { echo "error: no corpus at $c (run conformance/fetch-corpus.sh)" >&2; exit 2; }
|
||||
done
|
||||
|
||||
if [ -z "${H5RS:-}" ]; then
|
||||
# A debug build: overflow checks turn silent wraparound on hostile sizes
|
||||
# into a caught panic this sweep reports.
|
||||
echo "== building h5rs (debug, with overflow checks)"
|
||||
cargo build -q -p clawhdf5-tools --manifest-path "$ROOT/Cargo.toml" || exit 2
|
||||
TD="${CARGO_TARGET_DIR:-$ROOT/target}"
|
||||
H5RS="$TD/debug/h5rs"
|
||||
fi
|
||||
[ -x "$H5RS" ] || { echo "error: $H5RS is not executable" >&2; exit 2; }
|
||||
export H5RS TMO="${TMO:-60}" MEM_KB="${MEM_KB:-4194304}" MUTATE="${MUTATE:-0}"
|
||||
export MAX_BYTES="${MAX_BYTES:-16777216}"
|
||||
JOBS="${JOBS:-$(nproc 2>/dev/null || echo 4)}"
|
||||
|
||||
WORK="$(mktemp -d)"
|
||||
trap 'rm -rf "$WORK"' EXIT
|
||||
export WORK
|
||||
|
||||
# Every file in the corpora, whatever its extension (the reproducers often
|
||||
# have none), except the corpus's own scripts and docs.
|
||||
find -L "${CORPORA[@]}" -type f ! -name '*.md' ! -name '*.yml' ! -name '*.sh' \
|
||||
! -name '*.py' ! -name 'COPYING' ! -name '*.gif' | sort > "$WORK/files.txt"
|
||||
N=$(wc -l < "$WORK/files.txt")
|
||||
[ "$N" -gt 0 ] || { echo "error: no files in ${CORPORA[*]}" >&2; exit 2; }
|
||||
echo "== $N files, $JOBS at a time (timeout ${TMO}s, limit $((MEM_KB / 1024)) MiB, $MUTATE mutations each)"
|
||||
|
||||
one() {
|
||||
local f="$1" out="$WORK/fail.$$.$RANDOM"
|
||||
local inputs=("$f")
|
||||
if [ "$MUTATE" -gt 0 ]; then
|
||||
local i
|
||||
for ((i = 0; i < MUTATE; i++)); do
|
||||
local m="$WORK/mut.$$.$i"
|
||||
python3 - "$f" "$m" "$i" <<'PY' || continue
|
||||
import random, sys
|
||||
src, dst, seed = sys.argv[1], sys.argv[2], int(sys.argv[3])
|
||||
data = bytearray(open(src, "rb").read())
|
||||
if not data:
|
||||
sys.exit(1)
|
||||
rng = random.Random(f"{src}:{seed}")
|
||||
for _ in range(rng.randint(1, 8)):
|
||||
data[rng.randrange(len(data))] ^= 1 << rng.randrange(8)
|
||||
open(dst, "wb").write(data)
|
||||
PY
|
||||
inputs+=("$m")
|
||||
done
|
||||
fi
|
||||
local x
|
||||
for x in "${inputs[@]}"; do
|
||||
local cmd
|
||||
# --max-bytes bounds the work per run: a valid 1 GiB dataset (the libhdf5
|
||||
# test files have some) is not what this sweep is looking for.
|
||||
local mb="--max-bytes $MAX_BYTES"
|
||||
for cmd in "ls -r -v $mb" "dump $mb" "dump --json $mb" "dump -p -A" "stat" "check --data $mb" "diff SELF"; do
|
||||
local argv
|
||||
read -r -a argv <<< "$cmd"
|
||||
if [ "${argv[0]}" = diff ]; then argv=(diff --max-bytes "$MAX_BYTES" "$x"); fi
|
||||
( ulimit -v "$MEM_KB"; exec timeout -k 2 "$TMO" "$H5RS" "${argv[@]}" "$x" ) \
|
||||
>/dev/null 2>"$out.err"
|
||||
local rc=$?
|
||||
if [ $rc -gt 2 ]; then
|
||||
{
|
||||
echo "rc=$rc: h5rs ${argv[*]} $x"
|
||||
[ "$x" != "$f" ] && echo " (a mutation of $f)"
|
||||
grep -m3 'internal error' "$out.err" | sed 's/^/ /'
|
||||
} >> "$WORK/failures.txt.$$"
|
||||
fi
|
||||
done
|
||||
[ "$x" != "$f" ] && rm -f "$x"
|
||||
done
|
||||
rm -f "$out.err"
|
||||
}
|
||||
export -f one
|
||||
xargs -a "$WORK/files.txt" -d '\n' -P "$JOBS" -I{} bash -c 'one "$1"' _ {}
|
||||
|
||||
cat "$WORK"/failures.txt.* > "$WORK/failures.txt" 2>/dev/null || true
|
||||
if [ -s "$WORK/failures.txt" ]; then
|
||||
echo "== FAILED: $(grep -c '^rc=' "$WORK/failures.txt") run(s) panicked, crashed or hung:"
|
||||
cat "$WORK/failures.txt"
|
||||
exit 1
|
||||
fi
|
||||
echo "== ok: every subcommand ended cleanly on all $N files"
|
||||
Reference in New Issue
Block a user