From a29c1b224b902fdb4985db52c04be6bf2bdab3cf Mon Sep 17 00:00:00 2001 From: osobh Date: Sat, 19 Sep 2026 20:44:54 -0700 Subject: [PATCH] test: let the interop suites find a Python that actually has h5py MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Every Python interop suite had stopped running on this machine: the h5py writer round-trips, the facade suite, netCDF4 and the reference files. `python3` is 3.14, nothing on the box has h5py, and PEP 668 refuses to install it into a system interpreter at all — so the availability probes all returned false and each suite skipped without failing. A silent skip here is exactly how the v5 compound-datatype bug reached a release, so the probes now read `CLAWHDF5_PYTHON` and `ci-test.sh` picks up `.venv/bin/python` on its own. The detection sits at the top of the script rather than beside the interop step, because the non-ignored suites run in the earlier `cargo test` step and would otherwise still miss it. `CLAWHDF5_REQUIRE_INTEROP=1` continues to turn a skip into a failure. Verified against a venv with h5py 3.16 / HDF5 2.0.0: 94 interop tests across the four suites, all passing. Co-Authored-By: Claude Opus 5 (1M context) --- .gitignore | 1 + CHANGELOG.md | 9 +++++++++ crates/clawhdf5-format/src/chunked_write.rs | 19 +++++++++++++++---- .../clawhdf5-format/tests/reference_tests.rs | 11 ++++++++++- .../tests/writer_h5py_tests.rs | 15 ++++++++++++--- .../clawhdf5-netcdf4/tests/interop_tests.rs | 15 ++++++++++++--- crates/clawhdf5/tests/h5py_interop_tests.rs | 15 ++++++++++++--- scripts/ci-test.sh | 17 +++++++++++++++-- 8 files changed, 86 insertions(+), 16 deletions(-) diff --git a/.gitignore b/.gitignore index 05c7db3..029ea7a 100644 --- a/.gitignore +++ b/.gitignore @@ -4,3 +4,4 @@ benchmarks/longmemeval/*.json # Local model weights (MiniLM etc.) — large, not committed weights/ +.venv diff --git a/CHANGELOG.md b/CHANGELOG.md index 65aea64..4d6091c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,15 @@ ## Unreleased +### Testing +- The Python interop suites honour **`CLAWHDF5_PYTHON`**, and `ci-test.sh` + picks up a `.venv/bin/python` automatically. On a PEP 668 "externally + managed" system h5py cannot be installed into the system interpreter at all, + so every interop suite — the h5py writer round-trips, the facade, netCDF4 + and the reference files — was skipping silently. A silent skip here is + exactly how the v5 compound-datatype bug reached a release. + `CLAWHDF5_REQUIRE_INTEROP=1` still turns a skip into a failure. + ### Memory - `clawhdf5-agent`: **`MemoryConfig::quantized_index`** stores the vector index's own copy of the embeddings as `i8` rather than `f32`, which at 100k diff --git a/crates/clawhdf5-format/src/chunked_write.rs b/crates/clawhdf5-format/src/chunked_write.rs index e6a9e44..6816546 100644 --- a/crates/clawhdf5-format/src/chunked_write.rs +++ b/crates/clawhdf5-format/src/chunked_write.rs @@ -15,7 +15,6 @@ use crate::filter_pipeline::{ FilterDescription, FilterPipeline, }; use crate::filters::compress_chunk; - /// Round a file offset up to the next cache-line boundary. /// /// This ensures chunk data starts at an address that is a multiple of the @@ -928,6 +927,7 @@ pub fn write_selection_to_buffer( #[cfg(test)] mod tests { + use super::*; use crate::chunked_read::read_chunked_data; use crate::data_layout::DataLayout; @@ -1512,9 +1512,20 @@ mod tests { // ---- h5py round-trip tests for chunked writes ---- + /// The Python interpreter to drive interop checks with. + /// + /// `CLAWHDF5_PYTHON` lets these run against a virtualenv holding h5py, + /// which on a PEP 668 "externally managed" system is the only place it + /// can be installed. Without it the suite silently skips, and a silent + /// skip here is how a datatype bug once reached a release. + #[cfg(feature = "std")] + fn python() -> String { + std::env::var("CLAWHDF5_PYTHON").unwrap_or_else(|_| "python3".to_string()) + } + #[cfg(feature = "std")] fn h5py_available() -> bool { - std::process::Command::new("python3") + std::process::Command::new(python()) .args(["-c", "import h5py"]) .output() .map(|o| o.status.success()) @@ -1526,10 +1537,10 @@ mod tests { if !h5py_available() { panic!("h5py not installed — skipping interop test"); } - let o = std::process::Command::new("python3") + let o = std::process::Command::new(python()) .args(["-c", script]) .output() - .expect("python3"); + .expect("python interpreter"); if !o.status.success() { panic!("h5py: {}", String::from_utf8_lossy(&o.stderr)); } diff --git a/crates/clawhdf5-format/tests/reference_tests.rs b/crates/clawhdf5-format/tests/reference_tests.rs index b1dcf40..8a2bbcb 100644 --- a/crates/clawhdf5-format/tests/reference_tests.rs +++ b/crates/clawhdf5-format/tests/reference_tests.rs @@ -2,6 +2,15 @@ use clawhdf5_format::data_read::{read_object_references, read_region_references}; use clawhdf5_format::datatype::{Datatype, ReferenceType}; +/// The Python interpreter to drive interop checks with. +/// +/// `CLAWHDF5_PYTHON` lets these run against a virtualenv holding h5py, which +/// on a PEP 668 "externally managed" system is the only place it can be +/// installed. Without it the suite silently skips, and a silent skip here is +/// how a datatype bug once reached a release. +fn python() -> String { + std::env::var("CLAWHDF5_PYTHON").unwrap_or_else(|_| "python3".to_string()) +} #[test] fn object_ref_single_valid() { @@ -173,7 +182,7 @@ print('ok') "#, path.display() ); - let output = std::process::Command::new("python3") + let output = std::process::Command::new(python()) .args(["-c", &script]) .output(); diff --git a/crates/clawhdf5-format/tests/writer_h5py_tests.rs b/crates/clawhdf5-format/tests/writer_h5py_tests.rs index 58d1660..9707d04 100644 --- a/crates/clawhdf5-format/tests/writer_h5py_tests.rs +++ b/crates/clawhdf5-format/tests/writer_h5py_tests.rs @@ -4,9 +4,18 @@ //! (and vice versa). They require python3 + h5py to be installed. use clawhdf5_format::file_writer::{AttrValue, CompoundTypeBuilder, EnumTypeBuilder, FileWriter}; +/// The Python interpreter to drive interop checks with. +/// +/// `CLAWHDF5_PYTHON` lets these run against a virtualenv holding h5py, which +/// on a PEP 668 "externally managed" system is the only place it can be +/// installed. Without it the suite silently skips, and a silent skip here is +/// how a datatype bug once reached a release. +fn python() -> String { + std::env::var("CLAWHDF5_PYTHON").unwrap_or_else(|_| "python3".to_string()) +} fn h5py_available() -> bool { - std::process::Command::new("python3") + std::process::Command::new(python()) .args(["-c", "import h5py"]) .output() .map(|o| o.status.success()) @@ -17,10 +26,10 @@ fn h5py_read(_path: &std::path::Path, script: &str) -> String { if !h5py_available() { panic!("h5py not installed — skipping interop test"); } - let o = std::process::Command::new("python3") + let o = std::process::Command::new(python()) .args(["-c", script]) .output() - .expect("python3"); + .expect("python interpreter"); if !o.status.success() { panic!("h5py: {}", String::from_utf8_lossy(&o.stderr)); } diff --git a/crates/clawhdf5-netcdf4/tests/interop_tests.rs b/crates/clawhdf5-netcdf4/tests/interop_tests.rs index 17c7373..af75136 100644 --- a/crates/clawhdf5-netcdf4/tests/interop_tests.rs +++ b/crates/clawhdf5-netcdf4/tests/interop_tests.rs @@ -9,6 +9,15 @@ use clawhdf5_netcdf4::{AttrValue, NetCDF4File}; // --------------------------------------------------------------------------- // Helpers // --------------------------------------------------------------------------- +/// The Python interpreter to drive interop checks with. +/// +/// `CLAWHDF5_PYTHON` lets these run against a virtualenv holding h5py, which +/// on a PEP 668 "externally managed" system is the only place it can be +/// installed. Without it the suite silently skips, and a silent skip here is +/// how a datatype bug once reached a release. +fn python() -> String { + std::env::var("CLAWHDF5_PYTHON").unwrap_or_else(|_| "python3".to_string()) +} /// When `CLAWHDF5_REQUIRE_INTEROP=1` (set in CI), a missing Python dependency /// is a test failure instead of a silent skip. @@ -17,7 +26,7 @@ fn interop_required() -> bool { } fn netcdf4_python_available() -> bool { - Command::new("python3") + Command::new(python()) .args(["-c", "import netCDF4; print(netCDF4.__version__)"]) .output() .map(|o| o.status.success()) @@ -25,7 +34,7 @@ fn netcdf4_python_available() -> bool { } fn xarray_available() -> bool { - Command::new("python3") + Command::new(python()) .args(["-c", "import xarray; print(xarray.__version__)"]) .output() .map(|o| o.status.success()) @@ -59,7 +68,7 @@ macro_rules! skip_if_no_xarray { } fn run_python(script: &str) { - let output = Command::new("python3") + let output = Command::new(python()) .args(["-c", script]) .output() .expect("failed to run python3"); diff --git a/crates/clawhdf5/tests/h5py_interop_tests.rs b/crates/clawhdf5/tests/h5py_interop_tests.rs index e177b72..c7ab11c 100644 --- a/crates/clawhdf5/tests/h5py_interop_tests.rs +++ b/crates/clawhdf5/tests/h5py_interop_tests.rs @@ -9,6 +9,15 @@ use clawhdf5::{AttrValue, CompoundTypeBuilder, DType, File, FileBuilder}; // --------------------------------------------------------------------------- // Helpers // --------------------------------------------------------------------------- +/// The Python interpreter to drive interop checks with. +/// +/// `CLAWHDF5_PYTHON` lets these run against a virtualenv holding h5py, which +/// on a PEP 668 "externally managed" system is the only place it can be +/// installed. Without it the suite silently skips, and a silent skip here is +/// how a datatype bug once reached a release. +fn python() -> String { + std::env::var("CLAWHDF5_PYTHON").unwrap_or_else(|_| "python3".to_string()) +} /// When `CLAWHDF5_REQUIRE_INTEROP=1` (set in CI), a missing Python dependency /// is a test failure instead of a silent skip. @@ -17,7 +26,7 @@ fn interop_required() -> bool { } fn python_available() -> bool { - Command::new("python3") + Command::new(python()) .args(["-c", "import h5py; print(h5py.__version__)"]) .output() .map(|o| o.status.success()) @@ -39,7 +48,7 @@ macro_rules! skip_if_no_python { /// Run a Python script and panic if it fails. fn run_python(script: &str) { - let output = Command::new("python3") + let output = Command::new(python()) .args(["-c", script]) .output() .expect("failed to run python3"); @@ -52,7 +61,7 @@ fn run_python(script: &str) { /// Run a Python script and return stdout as a trimmed string. fn run_python_output(script: &str) -> String { - let output = Command::new("python3") + let output = Command::new(python()) .args(["-c", script]) .output() .expect("failed to run python3"); diff --git a/scripts/ci-test.sh b/scripts/ci-test.sh index 03367a0..f04dd09 100755 --- a/scripts/ci-test.sh +++ b/scripts/ci-test.sh @@ -20,6 +20,13 @@ set -uo pipefail SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" + +# Interop suites drive a Python interpreter. On a PEP 668 "externally managed" +# system h5py can only live in a virtualenv, so pick one up here — before any +# test step, since the non-ignored interop suites read the same variable. +if [ -z "${CLAWHDF5_PYTHON:-}" ] && [ -x "$SCRIPT_DIR/../.venv/bin/python" ]; then + export CLAWHDF5_PYTHON="$SCRIPT_DIR/../.venv/bin/python" +fi PASS=0 FAIL=0 STEPS=() @@ -85,12 +92,18 @@ run_step "cargo test (ann parallel)" cargo test \ # 5. Python interop suites. The h5py writer tests are #[ignore]d so a plain # `cargo test` stays hermetic; run them explicitly here. -if python3 -c "import h5py" >/dev/null 2>&1 || [ "${CLAWHDF5_REQUIRE_INTEROP:-0}" = "1" ]; then +# On a PEP 668 "externally managed" system h5py can only live in a +# virtualenv, so honour CLAWHDF5_PYTHON (and a local .venv) rather than +# skipping — the tests read the same variable. +PYTHON="${CLAWHDF5_PYTHON:-python3}" +if "$PYTHON" -c "import h5py" >/dev/null 2>&1 || [ "${CLAWHDF5_REQUIRE_INTEROP:-0}" = "1" ]; then run_step "h5py interop (format, ignored tests)" cargo test \ -p clawhdf5-format --test writer_h5py_tests -- --include-ignored else echo "" - echo "==> [h5py interop] SKIPPED: python3 with h5py not available" + echo "==> [h5py interop] SKIPPED: no h5py in $PYTHON" + echo " (set CLAWHDF5_PYTHON=/path/to/venv/bin/python, or create .venv;" + echo " CLAWHDF5_REQUIRE_INTEROP=1 makes this a failure instead)" STEPS+=("SKIP: h5py interop (format, ignored tests)") fi