osobhandClaude Opus 5.5
9179aa356e
conformance/run.sh fetches eight public HDF5 corpora pinned by commit (conformance/corpus.txt) into a gitignored cache, reads every file with clawhdf5 (conformance/probe, a crate outside the workspace) and with h5py/libhdf5 (ref.py), and the CVE files with h5dump, each under a timeout and an address-space limit; compare.py classifies the files, report.py writes CONFORMANCE.md and check.py gates on panics/hangs/crashes/OOM and on regressions against conformance/baseline.json. ~25 s once cached. Changes from the ad-hoc audit harness: - the probe compares non-IEEE-layout floats (N-Bit) and integers with a bit offset or reduced precision as the values libhdf5 converts them to, not raw file bytes: 8 files that showed as mismatches now read identically; - ref.py exits without tearing down h5py objects: libhdf5 2.0 aborts while freeing them for two files about half the time, which flipped them between ok and h5py-cannot-read from run to run; - the file list is defined (list_files.py): netCDF classic files are left out, 11 HDF5 files the ad-hoc sweep missed are in. Co-Authored-By: Claude Opus 5.5 (1M context) <[email protected]>
40 lines
1.7 KiB
Bash
Executable File
40 lines
1.7 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
# fetch-corpus.sh [cache_dir]
|
|
#
|
|
# Download the corpora pinned in conformance/corpus.txt into the (gitignored)
|
|
# cache: <cache>/src/<name> is a shallow, sparse, blob-filtered checkout of the
|
|
# pinned commit and <cache>/corpus/<name> links to the swept root inside it.
|
|
# A corpus already checked out at its pinned commit is left alone, so a second
|
|
# run costs nothing and needs no network.
|
|
set -euo pipefail
|
|
HERE="$(cd "$(dirname "$0")" && pwd)"
|
|
CACHE="${1:-${CONFORMANCE_CACHE:-$HERE/.cache}}"
|
|
mkdir -p "$CACHE/src" "$CACHE/corpus"
|
|
CACHE="$(cd "$CACHE" && pwd)"
|
|
|
|
retry() { local i; for i in 1 2 3 4; do "$@" && return 0; sleep $((i * 5)); done; return 1; }
|
|
|
|
grep -v '^[[:space:]]*\(#\|$\)' "$HERE/corpus.txt" | while read -r name url commit root patterns; do
|
|
src="$CACHE/src/$name"
|
|
if [ -d "$src/.git" ] && [ "$(git -C "$src" rev-parse HEAD 2>/dev/null)" = "$commit" ]; then
|
|
echo "cached $name @ ${commit:0:12}"
|
|
else
|
|
echo "fetching $name @ ${commit:0:12} from $url"
|
|
rm -rf "$src"
|
|
git init -q "$src"
|
|
git -C "$src" remote add origin "$url"
|
|
git -C "$src" config advice.detachedHead false
|
|
if [ -n "$patterns" ]; then
|
|
git -C "$src" config core.sparseCheckout true
|
|
# no-cone patterns (globs); `set -f` keeps the shell from expanding them
|
|
(set -f; printf '%s\n' $patterns) > "$src/.git/info/sparse-checkout"
|
|
fi
|
|
retry git -C "$src" fetch -q --depth 1 --filter=blob:none origin "$commit"
|
|
retry git -C "$src" checkout -q FETCH_HEAD
|
|
got="$(git -C "$src" rev-parse HEAD)"
|
|
[ "$got" = "$commit" ] || { echo "error: $name checked out $got, expected $commit" >&2; exit 1; }
|
|
fi
|
|
ln -sfn "$src/$root" "$CACHE/corpus/$name"
|
|
done
|
|
echo "corpus ready in $CACHE/corpus"
|