clawhdf5/scripts/ci-test.sh

#!/usr/bin/env bash
# CI test script — runs fmt, clippy (all targets + feature matrix), tests,
# Python interop suites, bench compilation, and no_std checks.
#
# Usage:
#   ./scripts/ci-test.sh
#
# Environment:
#   CLAWHDF5_REQUIRE_INTEROP=1  Fail (instead of skip) when python3 with
#                               h5py/netCDF4/xarray is missing, or without
#                               maturin/pytest for the Python package step.
#                               CI sets this.
#                               Unset locally, the interop steps are skipped
#                               if python3+h5py is not importable.
#   CLAWHDF5_FUZZ_SECONDS=N     Run each cargo-fuzz target for N seconds
#                               (needs nightly + cargo-fuzz). Default: skip.
#
# Exit codes:
#   0 — all checks passed
#   1 — one or more checks failed

set -uo pipefail

SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"

# Interop suites drive a Python interpreter. On a PEP 668 "externally managed"
# system h5py can only live in a virtualenv, so pick one up here — before any
# test step, since the non-ignored interop suites read the same variable.
if [ -z "${CLAWHDF5_PYTHON:-}" ] && [ -x "$SCRIPT_DIR/../.venv/bin/python" ]; then
    export CLAWHDF5_PYTHON="$SCRIPT_DIR/../.venv/bin/python"
fi
PASS=0
FAIL=0
STEPS=()

run_step() {
    local name="$1"
    shift
    echo ""
    echo "==> [$name]"
    if "$@" 2>&1; then
        echo "  ✓ PASS: $name"
        PASS=$((PASS + 1))
        STEPS+=("PASS: $name")
    else
        echo "  ✗ FAIL: $name"
        FAIL=$((FAIL + 1))
        STEPS+=("FAIL: $name")
    fi
}

# All steps always run so one failure doesn't hide the others; the summary
# and the exit code at the end are the verdict.

# 1. Format check
run_step "cargo fmt --check" cargo fmt --check

# 2. Clippy over every target (lib, bins, tests, benches, examples). Without
#    --all-targets, test and bench code is never linted. clawhdf5-py is
#    excluded here: PyO3 needs a Python interpreter to build, so it is
#    linted in the Python package step (5b) instead.
run_step "cargo clippy --all-targets" cargo clippy \
    --workspace \
    --exclude clawhdf5-py \
    --all-targets \
    -- -D warnings

# 3. Clippy over clawhdf5-format's optional features, which the default
#    workspace build never compiles (szip is left out: it needs libaec).
#    plugin-filters = bitshuffle, bzip2, blosc, blosc2, zfp (and the default-on lzf).
run_step "cargo clippy (format feature matrix)" cargo clippy \
    -p clawhdf5-format \
    --all-targets \
    --features parallel,lz4,zstd,pcodec,fast-checksum,plugin-filters \
    -- -D warnings

# Each plugin filter alone, so none of them leans on another's
# dependencies (bitshuffle and blosc share code).
plugin_filters_alone() {
    local f
    for f in bitshuffle bzip2 blosc blosc2 zfp; do
        echo "--- $f"
        cargo clippy -p clawhdf5-format --all-targets --features "$f" -- -D warnings || return 1
    done
}
run_step "cargo clippy (each plugin filter alone)" plugin_filters_alone

# The facade's plugin-filter interop tests and its parallel decoding tests
# only build with those features on (parallel_integration.rs stopped
# compiling unnoticed while nothing built it).
run_step "cargo clippy (facade plugin filters + parallel)" cargo clippy \
    -p clawhdf5 \
    --all-targets \
    --features plugin-filters,parallel \
    -- -D warnings

# The HNSW index's parallel bulk build is feature-gated too.
run_step "cargo clippy (ann parallel)" cargo clippy \
    -p clawhdf5-ann \
    --all-targets \
    --features parallel \
    -- -D warnings

# zlib-ng is opt-in (`fast-deflate`; the default is pure-Rust zlib-rs), so
# nothing above builds it. Keep it compiling and passing.
run_step "cargo clippy (fast-deflate / zlib-ng)" cargo clippy \
    -p clawhdf5-format -p clawhdf5-filters -p clawhdf5 \
    --all-targets \
    --features clawhdf5-format/fast-deflate,clawhdf5-filters/fast-deflate \
    -- -D warnings

# clawhdf5-remote's optional backends: object_store (in-memory and local
# stores in the tests), HTTPS through rustls, and the cloud stores.
run_step "cargo clippy (remote, all backends)" cargo clippy \
    -p clawhdf5-remote \
    --all-targets \
    --features object-store,https,s3,gcs,azure \
    -- -D warnings

# h5rs with URL arguments.
run_step "cargo clippy (h5rs remote)" cargo clippy \
    -p clawhdf5-tools \
    --all-targets \
    --features remote-https \
    -- -D warnings

# The README promises that the core crates build no C by default. Hold it to
# that: fail if a crate that compiles C (a *-sys crate, cc or cmake) enters the
# default dependency tree of any of them. clawhdf5-migrate (bundled SQLite),
# clawhdf5-napi (Node) and clawhdf5-gpu (graphics drivers) are exempt.
# js-sys (clawhdf5-wasm's bindings to JavaScript) builds no C.
# clawhdf5-remote is checked by default (plain HTTP) and with its
# object-store feature, and h5rs with URL support (remote); the https
# (ring) and s3/gcs/azure (aws-lc-rs) features build C and are opt-in. The
# Python bindings (clawhdf5-py, remote reads over plain HTTP) are checked too:
# their https/s3/gcs/azure features are opt-in for the same reason.
no_c_in_default_build() {
    local entry crate features found=0
    for entry in clawhdf5-format clawhdf5-io clawhdf5-filters clawhdf5 \
        clawhdf5-agent clawhdf5-ann clawhdf5-accel clawhdf5-netcdf4 clawhdf5-cli \
        clawhdf5-tools \
        clawhdf5-wasm \
        clawhdf5-remote clawhdf5-remote:object-store clawhdf5-tools:remote \
        clawhdf5-py; do
        crate=${entry%%:*}
        features=()
        [ "$entry" != "$crate" ] && features=(--features "${entry#*:}")
        local c_deps
        c_deps=$(cargo tree -q -p "$crate" "${features[@]}" -e normal,build --prefix none \
            | grep -E '^([a-z0-9_-]+-sys|cc|cmake) v' \
            | grep -v '^js-sys v' | sort -u)
        if [ -n "$c_deps" ]; then
            echo "$entry pulls in C by default:"
            echo "$c_deps" | sed 's/^/    /'
            found=1
        fi
    done
    return $found
}
run_step "no C in the default build (core crates)" no_c_in_default_build

# The reader in the browser: the facade's read path must build for
# wasm32-unknown-unknown (no mmap, no threads, no file system), and the
# wasm-bindgen crate must build and lint there. Needs
# `rustup target add wasm32-unknown-unknown`.
run_step "wasm32 build (clawhdf5, no default features)" cargo build \
    -p clawhdf5 \
    --target wasm32-unknown-unknown \
    --no-default-features
run_step "wasm32 clippy (clawhdf5-wasm)" cargo clippy \
    -p clawhdf5-wasm \
    --target wasm32-unknown-unknown \
    --all-targets \
    -- -D warnings
# A 64-bit file address must not wrap on a 32-bit target.
run_step "check-32bit-casts.sh" "$SCRIPT_DIR/check-32bit-casts.sh"

# The built wasm package, run under Node against h5py/netCDF4-written files,
# and the viewer page in headless Chromium when one is found.
# Needs node and the wasm-bindgen CLI, which the CI container does not have;
# the same expectations are checked natively by clawhdf5-wasm's h5py_interop
# test in the cargo test step.
if command -v node >/dev/null && command -v wasm-bindgen >/dev/null; then
    run_step "wasm package under Node (+ browser)" bash "$SCRIPT_DIR/../examples/wasm-viewer/test/run.sh"
else
    echo ""
    echo "==> [wasm package under Node] SKIPPED: needs node and wasm-bindgen"
    STEPS+=("SKIP: wasm package under Node")
fi

# The workspace declares a minimum Rust version (rust-version in Cargo.toml);
# check that it really builds there, so the README badge and the manifests
# cannot drift from the truth. Separate target dir: a different toolchain
# would otherwise invalidate the main build.
msrv_check() {
    local msrv
    msrv=$(sed -n 's/^rust-version = "\(.*\)"/\1/p' "$SCRIPT_DIR/../Cargo.toml")
    [ -n "$msrv" ] || { echo "no rust-version in Cargo.toml"; return 1; }
    rustup toolchain install "$msrv" --profile minimal >/dev/null || return 1
    echo "checking with Rust $msrv"
    CARGO_TARGET_DIR="$SCRIPT_DIR/../target/msrv" cargo "+$msrv" check \
        --workspace --exclude clawhdf5-py --all-targets
}
run_step "MSRV check" msrv_check

# 4. Tests (exclude clawhdf5-py)
run_step "cargo test" cargo test \
    --workspace \
    --exclude clawhdf5-py

run_step "cargo test (format feature matrix)" cargo test \
    -p clawhdf5-format \
    --features parallel,lz4,zstd,pcodec,fast-checksum,plugin-filters

run_step "cargo test (facade parallel)" cargo test \
    -p clawhdf5 \
    --features parallel

run_step "cargo test (remote, object_store backend, s3 URLs)" cargo test \
    -p clawhdf5-remote \
    --features object-store,s3

run_step "cargo test (h5rs on URLs)" cargo test \
    -p clawhdf5-tools \
    --features remote \
    --test remote

run_step "cargo test (ann parallel)" cargo test \
    -p clawhdf5-ann \
    --features parallel

run_step "cargo test (fast-deflate / zlib-ng)" cargo test \
    -p clawhdf5-format -p clawhdf5-filters -p clawhdf5 \
    --features clawhdf5-format/fast-deflate,clawhdf5-filters/fast-deflate

# 5. Python interop suites. The h5py writer tests are #[ignore]d so a plain
#    `cargo test` stays hermetic; run them explicitly here.
#    On a PEP 668 "externally managed" system h5py can only live in a
#    virtualenv, so honour CLAWHDF5_PYTHON (and a local .venv) rather than
#    skipping — the tests read the same variable.
PYTHON="${CLAWHDF5_PYTHON:-python3}"
if "$PYTHON" -c "import h5py" >/dev/null 2>&1 || [ "${CLAWHDF5_REQUIRE_INTEROP:-0}" = "1" ]; then
    # lz4/zstd so the hdf5plugin round-trips (our LZ4 and Zstd output read by
    # libhdf5's registered plugins) compile and run too.
    run_step "h5py interop (format, ignored tests)" cargo test \
        -p clawhdf5-format --features lz4,zstd --test writer_h5py_tests -- --include-ignored
    # LZF, bitshuffle, bzip2 and Blosc both ways against h5py + hdf5plugin;
    # Blosc2 and ZFP (read-only) against what h5py reads.
    run_step "h5py interop (plugin filters)" cargo test \
        -p clawhdf5 --features plugin-filters --test plugin_filters_interop --test zfp_interop
else
    echo ""
    echo "==> [h5py interop] SKIPPED: no h5py in $PYTHON"
    echo "    (set CLAWHDF5_PYTHON=/path/to/venv/bin/python, or create .venv;"
    echo "     CLAWHDF5_REQUIRE_INTEROP=1 makes this a failure instead)"
    STEPS+=("SKIP: h5py interop (format, ignored tests)")
fi

# 5b. The Python package (crates/clawhdf5-py): lint it, build the wheel with
#     maturin and run its pytest suite, which compares every read with h5py.
#     The wheel is unpacked under target/ and put on PYTHONPATH, so the
#     interpreter's environment is left as it was. Needs maturin and pytest
#     in $PYTHON (CI installs both); skipped without them, and a failure
#     instead when CLAWHDF5_REQUIRE_INTEROP=1.
python_package() {
    local root="$SCRIPT_DIR/.." out
    out="${CARGO_TARGET_DIR:-$root/target}/py-package"
    rm -rf "$out" && mkdir -p "$out/wheel" "$out/site" || return 1
    cargo clippy -p clawhdf5-py --all-targets -- -D warnings || return 1
    "$PYTHON" -m maturin build \
        -m "$root/crates/clawhdf5-py/Cargo.toml" \
        -i "$PYTHON" \
        --out "$out/wheel" || return 1
    "$PYTHON" -m pip install --quiet --no-deps --target "$out/site" "$out"/wheel/*.whl || return 1
    # The editing tests run `h5rs check` on every file they edit.
    cargo build -q -p clawhdf5-tools || return 1
    CLAWHDF5_H5RS="${CARGO_TARGET_DIR:-$root/target}/debug/h5rs" \
        PYTHONPATH="$out/site" "$PYTHON" -m pytest -q -p no:cacheprovider \
        "$root/crates/clawhdf5-py/tests"
}
if "$PYTHON" -m maturin --version >/dev/null 2>&1 && "$PYTHON" -c "import pytest" >/dev/null 2>&1; then
    run_step "Python package (maturin build + pytest vs h5py)" python_package
elif [ "${CLAWHDF5_REQUIRE_INTEROP:-0}" = "1" ]; then
    run_step "Python package (maturin build + pytest vs h5py)" \
        bash -c "echo \"maturin and pytest are required in $PYTHON (CLAWHDF5_REQUIRE_INTEROP=1)\"; exit 1"
else
    echo ""
    echo "==> [Python package] SKIPPED: needs maturin and pytest in $PYTHON"
    STEPS+=("SKIP: Python package (maturin build + pytest)")
fi

# 6. Benches must keep compiling (they are not run).
run_step "cargo bench --no-run" cargo bench \
    --workspace \
    --exclude clawhdf5-py \
    --no-run

# 7. no_std check
run_step "check-nostd.sh" "$SCRIPT_DIR/check-nostd.sh"

# 8. Optional fuzz smoke run
if [ -n "${CLAWHDF5_FUZZ_SECONDS:-}" ]; then
    fuzz_smoke() {
        local crate target
        for crate in clawhdf5-format clawhdf5-agent; do
            cd "$SCRIPT_DIR/../crates/$crate" || return 1
            for target in $(cargo +nightly fuzz list); do
                echo "--- fuzz: $crate/$target"
                cargo +nightly fuzz run "$target" -- \
                    -max_total_time="$CLAWHDF5_FUZZ_SECONDS" || return 1
            done
        done
    }
    run_step "fuzz smoke (${CLAWHDF5_FUZZ_SECONDS}s/target)" fuzz_smoke
fi

# Summary
echo ""
echo "========================================"
echo "  CI Summary"
echo "========================================"
for s in "${STEPS[@]}"; do
    echo "  $s"
done
echo "----------------------------------------"
echo "  $PASS passed, $FAIL failed"
echo "========================================"

if [ "$FAIL" -gt 0 ]; then
    exit 1
fi
exit 0