h5rs tools, browser reader, libhdf5 header checks, plugin filters, concurrency benchmark #14
Executable
+79
@@ -0,0 +1,79 @@
|
|||||||
|
#!/usr/bin/env bash
|
||||||
|
# scripts/h5rs-check-ok-files.sh — run `h5rs check` over the files the
|
||||||
|
# conformance sweep reads correctly and completely: the `ok_files` of
|
||||||
|
# conformance/baseline.json (clawhdf5 and h5py agree on every object) whose
|
||||||
|
# probe results record no error at all (some ok files are deliberately
|
||||||
|
# corrupt test files that both readers refuse in the same way; those are
|
||||||
|
# left out). A validator that flags a file libhdf5 and clawhdf5 both read
|
||||||
|
# in full has a false positive — unless the file really is damaged in a way
|
||||||
|
# readers tolerate, which the report must then show.
|
||||||
|
#
|
||||||
|
# Usage: scripts/h5rs-check-ok-files.sh [--data] [SAMPLE]
|
||||||
|
# --data pass --data to check (decode every chunk)
|
||||||
|
# SAMPLE check only every SAMPLE-th file (default 1: all of them)
|
||||||
|
#
|
||||||
|
# Environment: H5RS (binary; default builds release), CONFORMANCE_CACHE
|
||||||
|
# (corpus cache, default conformance/.cache), CONFORMANCE_OUT (the sweep's
|
||||||
|
# results, default $CONFORMANCE_CACHE/results; run conformance/run.sh first),
|
||||||
|
# TMO (timeout, default 60).
|
||||||
|
#
|
||||||
|
# Exit status: 0 = no file flagged; 1 = some file flagged or failed (listed);
|
||||||
|
# 2 = setup error.
|
||||||
|
set -uo pipefail
|
||||||
|
HERE="$(cd "$(dirname "$0")" && pwd)"
|
||||||
|
ROOT="$(cd "$HERE/.." && pwd)"
|
||||||
|
export PATH="$HOME/.cargo/bin:$PATH"
|
||||||
|
DATA=()
|
||||||
|
if [ "${1:-}" = "--data" ]; then DATA=(--data); shift; fi
|
||||||
|
SAMPLE="${1:-1}"
|
||||||
|
CACHE="${CONFORMANCE_CACHE:-$ROOT/conformance/.cache}"
|
||||||
|
C="$CACHE/corpus"
|
||||||
|
R="${CONFORMANCE_OUT:-$CACHE/results}"
|
||||||
|
[ -d "$C" ] || { echo "error: no corpus in $C (run conformance/fetch-corpus.sh)" >&2; exit 2; }
|
||||||
|
[ -d "$R/runs" ] || { echo "error: no sweep results in $R (run conformance/run.sh)" >&2; exit 2; }
|
||||||
|
if [ -z "${H5RS:-}" ]; then
|
||||||
|
cargo build -q --release -p clawhdf5-tools --manifest-path "$ROOT/Cargo.toml" || exit 2
|
||||||
|
H5RS="${CARGO_TARGET_DIR:-$ROOT/target}/release/h5rs"
|
||||||
|
fi
|
||||||
|
TMO="${TMO:-60}"
|
||||||
|
|
||||||
|
mapfile -t FILES < <(python3 -c '
|
||||||
|
import json, os, sys
|
||||||
|
b = json.load(open(sys.argv[1]))
|
||||||
|
runs, step = sys.argv[3], int(sys.argv[2])
|
||||||
|
def clean(f):
|
||||||
|
"""Both readers read every object, attribute and group listing."""
|
||||||
|
for side in ("ours.json", "ref.json"):
|
||||||
|
try:
|
||||||
|
d = json.load(open(os.path.join(runs, f.replace("/", "__"), side)))
|
||||||
|
except (OSError, ValueError):
|
||||||
|
return False
|
||||||
|
if "open_error" in d or d.get("truncated"):
|
||||||
|
return False
|
||||||
|
for o in d.get("objects", []):
|
||||||
|
if "error" in o or "attrs_error" in o or "list_error" in o:
|
||||||
|
return False
|
||||||
|
if any(isinstance(a, dict) and "error" in a for a in o.get("attrs", {}).values()):
|
||||||
|
return False
|
||||||
|
return True
|
||||||
|
files = [f for f in b["ok_files"] if clean(f)]
|
||||||
|
total = len(b["ok_files"])
|
||||||
|
print(f"{len(files)} of {total} ok files are read in full by both readers", file=sys.stderr)
|
||||||
|
for i, f in enumerate(files):
|
||||||
|
if i % step == 0:
|
||||||
|
print(f)
|
||||||
|
' "$ROOT/conformance/baseline.json" "$SAMPLE" "$R/runs")
|
||||||
|
|
||||||
|
flagged=0 checked=0
|
||||||
|
for f in "${FILES[@]}"; do
|
||||||
|
checked=$((checked + 1))
|
||||||
|
out=$(timeout "$TMO" "$H5RS" check -q "${DATA[@]}" "$C/$f" 2>&1)
|
||||||
|
rc=$?
|
||||||
|
if [ $rc -ne 0 ]; then
|
||||||
|
flagged=$((flagged + 1))
|
||||||
|
echo "== rc=$rc $f"
|
||||||
|
echo "$out" | head -5 | sed 's/^/ /'
|
||||||
|
fi
|
||||||
|
done
|
||||||
|
echo "== checked $checked ok files: $flagged flagged"
|
||||||
|
[ $flagged -eq 0 ]
|
||||||
Executable
+115
@@ -0,0 +1,115 @@
|
|||||||
|
#!/usr/bin/env bash
|
||||||
|
# scripts/h5rs-fuzz.sh — run every `h5rs` subcommand over every file in a
|
||||||
|
# corpus of hostile HDF5 files, each under a timeout and a memory limit, and
|
||||||
|
# fail on any panic, crash or hang.
|
||||||
|
#
|
||||||
|
# Usage: scripts/h5rs-fuzz.sh [CORPUS_DIR ...]
|
||||||
|
# default corpus: conformance/.cache/corpus/cve_hdf5 (fetch it with
|
||||||
|
# conformance/fetch-corpus.sh; the HDF Group's CVE reproducers)
|
||||||
|
#
|
||||||
|
# Environment:
|
||||||
|
# H5RS h5rs binary to test (default: a debug build, for overflow checks)
|
||||||
|
# TMO per-run timeout in seconds (default 60)
|
||||||
|
# MEM_KB per-run address-space limit in KiB (default 4 GiB)
|
||||||
|
# JOBS files in parallel (default nproc)
|
||||||
|
# MUTATE also run on N byte-flipped copies of each file (default 0)
|
||||||
|
# MAX_BYTES --max-bytes for the commands that read values (default 16 MiB)
|
||||||
|
#
|
||||||
|
# A run may exit 0 (fine), 1 (problems found / could not read something) or
|
||||||
|
# 2 (error). Anything else fails the sweep: 3 is a caught panic (h5rs prints
|
||||||
|
# "internal error"), 124 a timeout, 128+N a signal (crash, abort, OOM kill).
|
||||||
|
#
|
||||||
|
# Exit status: 0 = every run ended cleanly; 1 = at least one did not (listed);
|
||||||
|
# 2 = setup error.
|
||||||
|
set -uo pipefail
|
||||||
|
HERE="$(cd "$(dirname "$0")" && pwd)"
|
||||||
|
ROOT="$(cd "$HERE/.." && pwd)"
|
||||||
|
export PATH="$HOME/.cargo/bin:$PATH"
|
||||||
|
|
||||||
|
CORPORA=("$@")
|
||||||
|
[ ${#CORPORA[@]} -eq 0 ] && CORPORA=("$ROOT/conformance/.cache/corpus/cve_hdf5")
|
||||||
|
for c in "${CORPORA[@]}"; do
|
||||||
|
[ -d "$c" ] || { echo "error: no corpus at $c (run conformance/fetch-corpus.sh)" >&2; exit 2; }
|
||||||
|
done
|
||||||
|
|
||||||
|
if [ -z "${H5RS:-}" ]; then
|
||||||
|
# A debug build: overflow checks turn silent wraparound on hostile sizes
|
||||||
|
# into a caught panic this sweep reports.
|
||||||
|
echo "== building h5rs (debug, with overflow checks)"
|
||||||
|
cargo build -q -p clawhdf5-tools --manifest-path "$ROOT/Cargo.toml" || exit 2
|
||||||
|
TD="${CARGO_TARGET_DIR:-$ROOT/target}"
|
||||||
|
H5RS="$TD/debug/h5rs"
|
||||||
|
fi
|
||||||
|
[ -x "$H5RS" ] || { echo "error: $H5RS is not executable" >&2; exit 2; }
|
||||||
|
export H5RS TMO="${TMO:-60}" MEM_KB="${MEM_KB:-4194304}" MUTATE="${MUTATE:-0}"
|
||||||
|
export MAX_BYTES="${MAX_BYTES:-16777216}"
|
||||||
|
JOBS="${JOBS:-$(nproc 2>/dev/null || echo 4)}"
|
||||||
|
|
||||||
|
WORK="$(mktemp -d)"
|
||||||
|
trap 'rm -rf "$WORK"' EXIT
|
||||||
|
export WORK
|
||||||
|
|
||||||
|
# Every file in the corpora, whatever its extension (the reproducers often
|
||||||
|
# have none), except the corpus's own scripts and docs.
|
||||||
|
find -L "${CORPORA[@]}" -type f ! -name '*.md' ! -name '*.yml' ! -name '*.sh' \
|
||||||
|
! -name '*.py' ! -name 'COPYING' ! -name '*.gif' | sort > "$WORK/files.txt"
|
||||||
|
N=$(wc -l < "$WORK/files.txt")
|
||||||
|
[ "$N" -gt 0 ] || { echo "error: no files in ${CORPORA[*]}" >&2; exit 2; }
|
||||||
|
echo "== $N files, $JOBS at a time (timeout ${TMO}s, limit $((MEM_KB / 1024)) MiB, $MUTATE mutations each)"
|
||||||
|
|
||||||
|
one() {
|
||||||
|
local f="$1" out="$WORK/fail.$$.$RANDOM"
|
||||||
|
local inputs=("$f")
|
||||||
|
if [ "$MUTATE" -gt 0 ]; then
|
||||||
|
local i
|
||||||
|
for ((i = 0; i < MUTATE; i++)); do
|
||||||
|
local m="$WORK/mut.$$.$i"
|
||||||
|
python3 - "$f" "$m" "$i" <<'PY' || continue
|
||||||
|
import random, sys
|
||||||
|
src, dst, seed = sys.argv[1], sys.argv[2], int(sys.argv[3])
|
||||||
|
data = bytearray(open(src, "rb").read())
|
||||||
|
if not data:
|
||||||
|
sys.exit(1)
|
||||||
|
rng = random.Random(f"{src}:{seed}")
|
||||||
|
for _ in range(rng.randint(1, 8)):
|
||||||
|
data[rng.randrange(len(data))] ^= 1 << rng.randrange(8)
|
||||||
|
open(dst, "wb").write(data)
|
||||||
|
PY
|
||||||
|
inputs+=("$m")
|
||||||
|
done
|
||||||
|
fi
|
||||||
|
local x
|
||||||
|
for x in "${inputs[@]}"; do
|
||||||
|
local cmd
|
||||||
|
# --max-bytes bounds the work per run: a valid 1 GiB dataset (the libhdf5
|
||||||
|
# test files have some) is not what this sweep is looking for.
|
||||||
|
local mb="--max-bytes $MAX_BYTES"
|
||||||
|
for cmd in "ls -r -v $mb" "dump $mb" "dump --json $mb" "dump -p -A" "stat" "check --data $mb" "diff SELF"; do
|
||||||
|
local argv
|
||||||
|
read -r -a argv <<< "$cmd"
|
||||||
|
if [ "${argv[0]}" = diff ]; then argv=(diff --max-bytes "$MAX_BYTES" "$x"); fi
|
||||||
|
( ulimit -v "$MEM_KB"; exec timeout -k 2 "$TMO" "$H5RS" "${argv[@]}" "$x" ) \
|
||||||
|
>/dev/null 2>"$out.err"
|
||||||
|
local rc=$?
|
||||||
|
if [ $rc -gt 2 ]; then
|
||||||
|
{
|
||||||
|
echo "rc=$rc: h5rs ${argv[*]} $x"
|
||||||
|
[ "$x" != "$f" ] && echo " (a mutation of $f)"
|
||||||
|
grep -m3 'internal error' "$out.err" | sed 's/^/ /'
|
||||||
|
} >> "$WORK/failures.txt.$$"
|
||||||
|
fi
|
||||||
|
done
|
||||||
|
[ "$x" != "$f" ] && rm -f "$x"
|
||||||
|
done
|
||||||
|
rm -f "$out.err"
|
||||||
|
}
|
||||||
|
export -f one
|
||||||
|
xargs -a "$WORK/files.txt" -d '\n' -P "$JOBS" -I{} bash -c 'one "$1"' _ {}
|
||||||
|
|
||||||
|
cat "$WORK"/failures.txt.* > "$WORK/failures.txt" 2>/dev/null || true
|
||||||
|
if [ -s "$WORK/failures.txt" ]; then
|
||||||
|
echo "== FAILED: $(grep -c '^rc=' "$WORK/failures.txt") run(s) panicked, crashed or hung:"
|
||||||
|
cat "$WORK/failures.txt"
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
echo "== ok: every subcommand ended cleanly on all $N files"
|
||||||
Reference in New Issue
Block a user