Fast contiguous and concurrent reads, VL data, nested groups and links, Python bindings #15

Merged
osobh merged 41 commits from feat/p2-perf-coverage into main 2026-09-26 14:57:01 +00:00
3 changed files with 43 additions and 3 deletions
Showing only changes of commit f7d88bb4fb - Show all commits
+3 -1
View File
@@ -38,7 +38,9 @@ jobs:
# (clawhdf5-tools) interop tests compare against. # (clawhdf5-tools) interop tests compare against.
apt-get install -y --no-install-recommends python3 python3-venv cmake hdf5-tools apt-get install -y --no-install-recommends python3 python3-venv cmake hdf5-tools
python3 -m venv /opt/interop python3 -m venv /opt/interop
/opt/interop/bin/pip install --no-cache-dir h5py numpy netCDF4 xarray hdf5plugin # maturin + pytest: ci-test.sh builds the Python package
# (crates/clawhdf5-py) and runs its tests against h5py.
/opt/interop/bin/pip install --no-cache-dir h5py numpy netCDF4 xarray hdf5plugin maturin pytest
echo "/opt/interop/bin" >> "$GITHUB_PATH" echo "/opt/interop/bin" >> "$GITHUB_PATH"
- name: Show interop library versions - name: Show interop library versions
# h5dump's version too: the h5rs dump test requires its exact output # h5dump's version too: the h5rs dump test requires its exact output
+5
View File
@@ -41,6 +41,11 @@
h5py 3.16 (HDF5 2.0) on a file h5py writes. One difference is h5py's: h5py 3.16 (HDF5 2.0) on a file h5py writes. One difference is h5py's:
it returns variable-length sequences of big-endian floats unswapped; this it returns variable-length sequences of big-endian floats unswapped; this
package returns the stored values. package returns the stored values.
- **CI builds and tests the Python package.** It was excluded from CI.
`scripts/ci-test.sh` now lints `clawhdf5-py`, builds the wheel with
maturin, unpacks it under `target/` and runs the pytest suite; skipped
without maturin/pytest in `$CLAWHDF5_PYTHON`, a failure then under
`CLAWHDF5_REQUIRE_INTEROP=1`. The CI interop venv installs both.
### Plugin filters (2026-09-26) ### Plugin filters (2026-09-26)
- **LZF, bitshuffle, bzip2 and Blosc read and write, in pure Rust.** Files - **LZF, bitshuffle, bzip2 and Blosc read and write, in pure Rust.** Files
+35 -2
View File
@@ -7,7 +7,9 @@
# #
# Environment: # Environment:
# CLAWHDF5_REQUIRE_INTEROP=1 Fail (instead of skip) when python3 with # CLAWHDF5_REQUIRE_INTEROP=1 Fail (instead of skip) when python3 with
# h5py/netCDF4/xarray is missing. CI sets this. # h5py/netCDF4/xarray is missing, or without
# maturin/pytest for the Python package step.
# CI sets this.
# Unset locally, the interop steps are skipped # Unset locally, the interop steps are skipped
# if python3+h5py is not importable. # if python3+h5py is not importable.
# CLAWHDF5_FUZZ_SECONDS=N Run each cargo-fuzz target for N seconds # CLAWHDF5_FUZZ_SECONDS=N Run each cargo-fuzz target for N seconds
@@ -55,7 +57,8 @@ run_step "cargo fmt --check" cargo fmt --check
# 2. Clippy over every target (lib, bins, tests, benches, examples). Without # 2. Clippy over every target (lib, bins, tests, benches, examples). Without
# --all-targets, test and bench code is never linted. clawhdf5-py is # --all-targets, test and bench code is never linted. clawhdf5-py is
# excluded because it needs PyO3/Python headers. # excluded here: PyO3 needs a Python interpreter to build, so it is
# linted in the Python package step (5b) instead.
run_step "cargo clippy --all-targets" cargo clippy \ run_step "cargo clippy --all-targets" cargo clippy \
--workspace \ --workspace \
--exclude clawhdf5-py \ --exclude clawhdf5-py \
@@ -216,6 +219,36 @@ else
STEPS+=("SKIP: h5py interop (format, ignored tests)") STEPS+=("SKIP: h5py interop (format, ignored tests)")
fi fi
# 5b. The Python package (crates/clawhdf5-py): lint it, build the wheel with
# maturin and run its pytest suite, which compares every read with h5py.
# The wheel is unpacked under target/ and put on PYTHONPATH, so the
# interpreter's environment is left as it was. Needs maturin and pytest
# in $PYTHON (CI installs both); skipped without them, and a failure
# instead when CLAWHDF5_REQUIRE_INTEROP=1.
python_package() {
local root="$SCRIPT_DIR/.." out
out="${CARGO_TARGET_DIR:-$root/target}/py-package"
rm -rf "$out" && mkdir -p "$out/wheel" "$out/site" || return 1
cargo clippy -p clawhdf5-py --all-targets -- -D warnings || return 1
"$PYTHON" -m maturin build \
-m "$root/crates/clawhdf5-py/Cargo.toml" \
-i "$PYTHON" \
--out "$out/wheel" || return 1
"$PYTHON" -m pip install --quiet --no-deps --target "$out/site" "$out"/wheel/*.whl || return 1
PYTHONPATH="$out/site" "$PYTHON" -m pytest -q -p no:cacheprovider \
"$root/crates/clawhdf5-py/tests"
}
if "$PYTHON" -m maturin --version >/dev/null 2>&1 && "$PYTHON" -c "import pytest" >/dev/null 2>&1; then
run_step "Python package (maturin build + pytest vs h5py)" python_package
elif [ "${CLAWHDF5_REQUIRE_INTEROP:-0}" = "1" ]; then
run_step "Python package (maturin build + pytest vs h5py)" \
bash -c "echo \"maturin and pytest are required in $PYTHON (CLAWHDF5_REQUIRE_INTEROP=1)\"; exit 1"
else
echo ""
echo "==> [Python package] SKIPPED: needs maturin and pytest in $PYTHON"
STEPS+=("SKIP: Python package (maturin build + pytest)")
fi
# 6. Benches must keep compiling (they are not run). # 6. Benches must keep compiling (they are not run).
run_step "cargo bench --no-run" cargo bench \ run_step "cargo bench --no-run" cargo bench \
--workspace \ --workspace \