clawhdf5/scripts/ci-test.sh

#!/usr/bin/env bash
# CI test script — runs fmt, clippy (all targets + feature matrix), tests,
# Python interop suites, bench compilation, and no_std checks.
#
# Usage:
#   ./scripts/ci-test.sh
#
# Environment:
#   CLAWHDF5_REQUIRE_INTEROP=1  Fail (instead of skip) when python3 with
#                               h5py/netCDF4/xarray is missing. CI sets this.
#                               Unset locally, the interop steps are skipped
#                               if python3+h5py is not importable.
#   CLAWHDF5_FUZZ_SECONDS=N     Run each cargo-fuzz target for N seconds
#                               (needs nightly + cargo-fuzz). Default: skip.
#
# Exit codes:
#   0 — all checks passed
#   1 — one or more checks failed

set -uo pipefail

SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"

# Interop suites drive a Python interpreter. On a PEP 668 "externally managed"
# system h5py can only live in a virtualenv, so pick one up here — before any
# test step, since the non-ignored interop suites read the same variable.
if [ -z "${CLAWHDF5_PYTHON:-}" ] && [ -x "$SCRIPT_DIR/../.venv/bin/python" ]; then
    export CLAWHDF5_PYTHON="$SCRIPT_DIR/../.venv/bin/python"
fi
PASS=0
FAIL=0
STEPS=()

run_step() {
    local name="$1"
    shift
    echo ""
    echo "==> [$name]"
    if "$@" 2>&1; then
        echo "  ✓ PASS: $name"
        PASS=$((PASS + 1))
        STEPS+=("PASS: $name")
    else
        echo "  ✗ FAIL: $name"
        FAIL=$((FAIL + 1))
        STEPS+=("FAIL: $name")
    fi
}

# All steps always run so one failure doesn't hide the others; the summary
# and the exit code at the end are the verdict.

# 1. Format check
run_step "cargo fmt --check" cargo fmt --check

# 2. Clippy over every target (lib, bins, tests, benches, examples). Without
#    --all-targets, test and bench code is never linted. clawhdf5-py is
#    excluded because it needs PyO3/Python headers.
run_step "cargo clippy --all-targets" cargo clippy \
    --workspace \
    --exclude clawhdf5-py \
    --all-targets \
    -- -D warnings

# 3. Clippy over clawhdf5-format's optional features, which the default
#    workspace build never compiles (szip is left out: it needs libaec).
#    plugin-filters = bitshuffle, bzip2, blosc (and the default-on lzf).
run_step "cargo clippy (format feature matrix)" cargo clippy \
    -p clawhdf5-format \
    --all-targets \
    --features parallel,lz4,zstd,pcodec,fast-checksum,plugin-filters \
    -- -D warnings

# Each plugin filter alone, so none of them leans on another's
# dependencies (bitshuffle and blosc share code).
plugin_filters_alone() {
    local f
    for f in bitshuffle bzip2 blosc; do
        echo "--- $f"
        cargo clippy -p clawhdf5-format --all-targets --features "$f" -- -D warnings || return 1
    done
}
run_step "cargo clippy (each plugin filter alone)" plugin_filters_alone

# The facade's plugin-filter interop tests only build with the features on.
run_step "cargo clippy (facade plugin filters)" cargo clippy \
    -p clawhdf5 \
    --all-targets \
    --features plugin-filters \
    -- -D warnings

# The HNSW index's parallel bulk build is feature-gated too.
run_step "cargo clippy (ann parallel)" cargo clippy \
    -p clawhdf5-ann \
    --all-targets \
    --features parallel \
    -- -D warnings

# zlib-ng is opt-in (`fast-deflate`; the default is pure-Rust zlib-rs), so
# nothing above builds it. Keep it compiling and passing.
run_step "cargo clippy (fast-deflate / zlib-ng)" cargo clippy \
    -p clawhdf5-format -p clawhdf5-filters -p clawhdf5 \
    --all-targets \
    --features clawhdf5-format/fast-deflate,clawhdf5-filters/fast-deflate \
    -- -D warnings

# The README promises that the core crates build no C by default. Hold it to
# that: fail if a crate that compiles C (a *-sys crate, cc or cmake) enters the
# default dependency tree of any of them. clawhdf5-migrate (bundled SQLite),
# clawhdf5-napi (Node) and clawhdf5-gpu (graphics drivers) are exempt.
no_c_in_default_build() {
    local crate found=0
    for crate in clawhdf5-format clawhdf5-io clawhdf5-filters clawhdf5 \
        clawhdf5-agent clawhdf5-ann clawhdf5-accel clawhdf5-netcdf4 clawhdf5-cli; do
        local c_deps
        c_deps=$(cargo tree -q -p "$crate" -e normal,build --prefix none \
            | grep -E '^([a-z0-9_-]+-sys|cc|cmake) v' | sort -u)
        if [ -n "$c_deps" ]; then
            echo "$crate pulls in C by default:"
            echo "$c_deps" | sed 's/^/    /'
            found=1
        fi
    done
    return $found
}
run_step "no C in the default build (core crates)" no_c_in_default_build

# The workspace declares a minimum Rust version (rust-version in Cargo.toml);
# check that it really builds there, so the README badge and the manifests
# cannot drift from the truth. Separate target dir: a different toolchain
# would otherwise invalidate the main build.
msrv_check() {
    local msrv
    msrv=$(sed -n 's/^rust-version = "\(.*\)"/\1/p' "$SCRIPT_DIR/../Cargo.toml")
    [ -n "$msrv" ] || { echo "no rust-version in Cargo.toml"; return 1; }
    rustup toolchain install "$msrv" --profile minimal >/dev/null || return 1
    echo "checking with Rust $msrv"
    CARGO_TARGET_DIR="$SCRIPT_DIR/../target/msrv" cargo "+$msrv" check \
        --workspace --exclude clawhdf5-py --all-targets
}
run_step "MSRV check" msrv_check

# 4. Tests (exclude clawhdf5-py)
run_step "cargo test" cargo test \
    --workspace \
    --exclude clawhdf5-py

run_step "cargo test (format feature matrix)" cargo test \
    -p clawhdf5-format \
    --features parallel,lz4,zstd,pcodec,fast-checksum,plugin-filters

run_step "cargo test (ann parallel)" cargo test \
    -p clawhdf5-ann \
    --features parallel

run_step "cargo test (fast-deflate / zlib-ng)" cargo test \
    -p clawhdf5-format -p clawhdf5-filters -p clawhdf5 \
    --features clawhdf5-format/fast-deflate,clawhdf5-filters/fast-deflate

# 5. Python interop suites. The h5py writer tests are #[ignore]d so a plain
#    `cargo test` stays hermetic; run them explicitly here.
#    On a PEP 668 "externally managed" system h5py can only live in a
#    virtualenv, so honour CLAWHDF5_PYTHON (and a local .venv) rather than
#    skipping — the tests read the same variable.
PYTHON="${CLAWHDF5_PYTHON:-python3}"
if "$PYTHON" -c "import h5py" >/dev/null 2>&1 || [ "${CLAWHDF5_REQUIRE_INTEROP:-0}" = "1" ]; then
    # lz4/zstd so the hdf5plugin round-trips (our LZ4 and Zstd output read by
    # libhdf5's registered plugins) compile and run too.
    run_step "h5py interop (format, ignored tests)" cargo test \
        -p clawhdf5-format --features lz4,zstd --test writer_h5py_tests -- --include-ignored
    # LZF, bitshuffle, bzip2 and Blosc both ways against h5py + hdf5plugin.
    run_step "h5py interop (plugin filters)" cargo test \
        -p clawhdf5 --features plugin-filters --test plugin_filters_interop
else
    echo ""
    echo "==> [h5py interop] SKIPPED: no h5py in $PYTHON"
    echo "    (set CLAWHDF5_PYTHON=/path/to/venv/bin/python, or create .venv;"
    echo "     CLAWHDF5_REQUIRE_INTEROP=1 makes this a failure instead)"
    STEPS+=("SKIP: h5py interop (format, ignored tests)")
fi

# 6. Benches must keep compiling (they are not run).
run_step "cargo bench --no-run" cargo bench \
    --workspace \
    --exclude clawhdf5-py \
    --no-run

# 7. no_std check
run_step "check-nostd.sh" "$SCRIPT_DIR/check-nostd.sh"

# 8. Optional fuzz smoke run
if [ -n "${CLAWHDF5_FUZZ_SECONDS:-}" ]; then
    fuzz_smoke() {
        local crate target
        for crate in clawhdf5-format clawhdf5-agent; do
            cd "$SCRIPT_DIR/../crates/$crate" || return 1
            for target in $(cargo +nightly fuzz list); do
                echo "--- fuzz: $crate/$target"
                cargo +nightly fuzz run "$target" -- \
                    -max_total_time="$CLAWHDF5_FUZZ_SECONDS" || return 1
            done
        done
    }
    run_step "fuzz smoke (${CLAWHDF5_FUZZ_SECONDS}s/target)" fuzz_smoke
fi

# Summary
echo ""
echo "========================================"
echo "  CI Summary"
echo "========================================"
for s in "${STEPS[@]}"; do
    echo "  $s"
done
echo "----------------------------------------"
echo "  $PASS passed, $FAIL failed"
echo "========================================"

if [ "$FAIL" -gt 0 ]; then
    exit 1
fi
exit 0