osobhandClaude Opus 5.5
40968b3578
scripts/h5rs-fuzz.sh runs every h5rs subcommand over every file of a corpus (default: the HDF Group's CVE reproducers), optionally with byte-flipped copies (MUTATE=N), under a timeout and a memory limit, with a debug build so integer overflow panics instead of wrapping; any exit status above 2 (a caught panic, a timeout, a signal) fails it. It found size*8 overflows in the datatype names on cve-2021-46244.h5, cve-2024-29161.h5 and unknown-1.h5 (fixed in the crate before it landed). scripts/h5rs-check-ok-files.sh runs check (--data) over the conformance files that clawhdf5 and h5py both read in full; none may be flagged. Co-Authored-By: Claude Opus 5.5 (1M context) <[email protected]>
116 lines
4.5 KiB
Bash
Executable File
116 lines
4.5 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
# scripts/h5rs-fuzz.sh — run every `h5rs` subcommand over every file in a
|
|
# corpus of hostile HDF5 files, each under a timeout and a memory limit, and
|
|
# fail on any panic, crash or hang.
|
|
#
|
|
# Usage: scripts/h5rs-fuzz.sh [CORPUS_DIR ...]
|
|
# default corpus: conformance/.cache/corpus/cve_hdf5 (fetch it with
|
|
# conformance/fetch-corpus.sh; the HDF Group's CVE reproducers)
|
|
#
|
|
# Environment:
|
|
# H5RS h5rs binary to test (default: a debug build, for overflow checks)
|
|
# TMO per-run timeout in seconds (default 60)
|
|
# MEM_KB per-run address-space limit in KiB (default 4 GiB)
|
|
# JOBS files in parallel (default nproc)
|
|
# MUTATE also run on N byte-flipped copies of each file (default 0)
|
|
# MAX_BYTES --max-bytes for the commands that read values (default 16 MiB)
|
|
#
|
|
# A run may exit 0 (fine), 1 (problems found / could not read something) or
|
|
# 2 (error). Anything else fails the sweep: 3 is a caught panic (h5rs prints
|
|
# "internal error"), 124 a timeout, 128+N a signal (crash, abort, OOM kill).
|
|
#
|
|
# Exit status: 0 = every run ended cleanly; 1 = at least one did not (listed);
|
|
# 2 = setup error.
|
|
set -uo pipefail
|
|
HERE="$(cd "$(dirname "$0")" && pwd)"
|
|
ROOT="$(cd "$HERE/.." && pwd)"
|
|
export PATH="$HOME/.cargo/bin:$PATH"
|
|
|
|
CORPORA=("$@")
|
|
[ ${#CORPORA[@]} -eq 0 ] && CORPORA=("$ROOT/conformance/.cache/corpus/cve_hdf5")
|
|
for c in "${CORPORA[@]}"; do
|
|
[ -d "$c" ] || { echo "error: no corpus at $c (run conformance/fetch-corpus.sh)" >&2; exit 2; }
|
|
done
|
|
|
|
if [ -z "${H5RS:-}" ]; then
|
|
# A debug build: overflow checks turn silent wraparound on hostile sizes
|
|
# into a caught panic this sweep reports.
|
|
echo "== building h5rs (debug, with overflow checks)"
|
|
cargo build -q -p clawhdf5-tools --manifest-path "$ROOT/Cargo.toml" || exit 2
|
|
TD="${CARGO_TARGET_DIR:-$ROOT/target}"
|
|
H5RS="$TD/debug/h5rs"
|
|
fi
|
|
[ -x "$H5RS" ] || { echo "error: $H5RS is not executable" >&2; exit 2; }
|
|
export H5RS TMO="${TMO:-60}" MEM_KB="${MEM_KB:-4194304}" MUTATE="${MUTATE:-0}"
|
|
export MAX_BYTES="${MAX_BYTES:-16777216}"
|
|
JOBS="${JOBS:-$(nproc 2>/dev/null || echo 4)}"
|
|
|
|
WORK="$(mktemp -d)"
|
|
trap 'rm -rf "$WORK"' EXIT
|
|
export WORK
|
|
|
|
# Every file in the corpora, whatever its extension (the reproducers often
|
|
# have none), except the corpus's own scripts and docs.
|
|
find -L "${CORPORA[@]}" -type f ! -name '*.md' ! -name '*.yml' ! -name '*.sh' \
|
|
! -name '*.py' ! -name 'COPYING' ! -name '*.gif' | sort > "$WORK/files.txt"
|
|
N=$(wc -l < "$WORK/files.txt")
|
|
[ "$N" -gt 0 ] || { echo "error: no files in ${CORPORA[*]}" >&2; exit 2; }
|
|
echo "== $N files, $JOBS at a time (timeout ${TMO}s, limit $((MEM_KB / 1024)) MiB, $MUTATE mutations each)"
|
|
|
|
one() {
|
|
local f="$1" out="$WORK/fail.$$.$RANDOM"
|
|
local inputs=("$f")
|
|
if [ "$MUTATE" -gt 0 ]; then
|
|
local i
|
|
for ((i = 0; i < MUTATE; i++)); do
|
|
local m="$WORK/mut.$$.$i"
|
|
python3 - "$f" "$m" "$i" <<'PY' || continue
|
|
import random, sys
|
|
src, dst, seed = sys.argv[1], sys.argv[2], int(sys.argv[3])
|
|
data = bytearray(open(src, "rb").read())
|
|
if not data:
|
|
sys.exit(1)
|
|
rng = random.Random(f"{src}:{seed}")
|
|
for _ in range(rng.randint(1, 8)):
|
|
data[rng.randrange(len(data))] ^= 1 << rng.randrange(8)
|
|
open(dst, "wb").write(data)
|
|
PY
|
|
inputs+=("$m")
|
|
done
|
|
fi
|
|
local x
|
|
for x in "${inputs[@]}"; do
|
|
local cmd
|
|
# --max-bytes bounds the work per run: a valid 1 GiB dataset (the libhdf5
|
|
# test files have some) is not what this sweep is looking for.
|
|
local mb="--max-bytes $MAX_BYTES"
|
|
for cmd in "ls -r -v $mb" "dump $mb" "dump --json $mb" "dump -p -A" "stat" "check --data $mb" "diff SELF"; do
|
|
local argv
|
|
read -r -a argv <<< "$cmd"
|
|
if [ "${argv[0]}" = diff ]; then argv=(diff --max-bytes "$MAX_BYTES" "$x"); fi
|
|
( ulimit -v "$MEM_KB"; exec timeout -k 2 "$TMO" "$H5RS" "${argv[@]}" "$x" ) \
|
|
>/dev/null 2>"$out.err"
|
|
local rc=$?
|
|
if [ $rc -gt 2 ]; then
|
|
{
|
|
echo "rc=$rc: h5rs ${argv[*]} $x"
|
|
[ "$x" != "$f" ] && echo " (a mutation of $f)"
|
|
grep -m3 'internal error' "$out.err" | sed 's/^/ /'
|
|
} >> "$WORK/failures.txt.$$"
|
|
fi
|
|
done
|
|
[ "$x" != "$f" ] && rm -f "$x"
|
|
done
|
|
rm -f "$out.err"
|
|
}
|
|
export -f one
|
|
xargs -a "$WORK/files.txt" -d '\n' -P "$JOBS" -I{} bash -c 'one "$1"' _ {}
|
|
|
|
cat "$WORK"/failures.txt.* > "$WORK/failures.txt" 2>/dev/null || true
|
|
if [ -s "$WORK/failures.txt" ]; then
|
|
echo "== FAILED: $(grep -c '^rc=' "$WORK/failures.txt") run(s) panicked, crashed or hung:"
|
|
cat "$WORK/failures.txt"
|
|
exit 1
|
|
fi
|
|
echo "== ok: every subcommand ended cleanly on all $N files"
|