diff --git a/crates/clawhdf5/Cargo.toml b/crates/clawhdf5/Cargo.toml index 1eca373..9acc7e9 100644 --- a/crates/clawhdf5/Cargo.toml +++ b/crates/clawhdf5/Cargo.toml @@ -48,8 +48,10 @@ bitshuffle = ["clawhdf5-format/bitshuffle"] bzip2 = ["clawhdf5-format/bzip2"] blosc = ["clawhdf5-format/blosc"] blosc2 = ["clawhdf5-format/blosc2"] +# ZFP (32013), read-only. +zfp = ["clawhdf5-format/zfp"] # Every plugin filter. -plugin-filters = ["lzf", "bitshuffle", "bzip2", "blosc", "blosc2"] +plugin-filters = ["lzf", "bitshuffle", "bzip2", "blosc", "blosc2", "zfp"] # Dataset::verify_provenance() — recompute a dataset's SHA-256 and compare # against its stored _provenance_sha256 attribute. On by default, matching # clawhdf5-format's own default-on `provenance` feature. diff --git a/crates/clawhdf5/tests/plugin_filters_interop.rs b/crates/clawhdf5/tests/plugin_filters_interop.rs index 00911c6..2b30de4 100644 --- a/crates/clawhdf5/tests/plugin_filters_interop.rs +++ b/crates/clawhdf5/tests/plugin_filters_interop.rs @@ -495,16 +495,15 @@ fn lzf_written_by_clawhdf5_reads_in_h5py() { }); } -/// ZFP is not implemented, and Blosc2 is not in a build without the -/// `blosc2` feature: reading them must be a clear error naming the filter, -/// never data. +/// Blosc2 and ZFP are not in a build without their features: reading them +/// must be a clear error naming the filter, never data. #[test] -fn unimplemented_filters_are_a_clear_error() { +fn filters_left_out_of_the_build_are_a_clear_error() { if !have_python("h5py, hdf5plugin") { return; } let dir = tempfile::tempdir().unwrap(); - let path = dir.path().join("unimplemented.h5"); + let path = dir.path().join("left_out.h5"); run_python( r#" import sys @@ -517,7 +516,10 @@ with h5py.File(sys.argv[1], 'w') as f: &[path.to_str().unwrap()], ); let file = File::open(&path).unwrap(); - let mut missing = vec![("zfp", 32013u16, "ZFP")]; + let mut missing = Vec::new(); + if !cfg!(feature = "zfp") { + missing.push(("zfp", 32013u16, "ZFP")); + } if !cfg!(feature = "blosc2") { missing.push(("blosc2", 32026, "Blosc2")); } @@ -526,7 +528,7 @@ with h5py.File(sys.argv[1], 'w') as f: .dataset(name) .unwrap() .read_selection(&Selection::All) - .expect_err("an unimplemented filter must not read"); + .expect_err("a filter left out of the build must not read"); let msg = err.to_string(); assert!( msg.contains(&id.to_string()) && msg.contains(label), diff --git a/crates/clawhdf5/tests/zfp_interop.rs b/crates/clawhdf5/tests/zfp_interop.rs new file mode 100644 index 0000000..e6e0bdb --- /dev/null +++ b/crates/clawhdf5/tests/zfp_interop.rs @@ -0,0 +1,277 @@ +//! ZFP (H5Z-ZFP, filter 32013) against libhdf5 + libzfp. +//! +//! h5py with hdf5plugin (H5Z-ZFP 1.1.1, zfp 1.0.1) writes datasets in every +//! ZFP mode (fixed rate, precision and accuracy, reversible, expert) for +//! each type ZFP supports (int32, int64, float, double), in 1 to 4 +//! dimensions, with chunks that are partial at the dataset's edges, blocks +//! that are partial at the chunks' edges, and chunks with unit dimensions +//! (a lower-dimensional ZFP field). Next to each it stores what h5py reads +//! back, unfiltered, and clawhdf5 must read the ZFP dataset bit for bit +//! equal to that: the decoder is deterministic, so lossy modes have one +//! right answer. +//! +//! Skipped when python3 with h5py and hdf5plugin is unavailable, unless +//! `CLAWHDF5_REQUIRE_INTEROP=1`. +#![cfg(feature = "zfp")] + +use std::collections::BTreeMap; +use std::process::Command; + +use clawhdf5::File; +use clawhdf5_format::selection::Selection; + +fn python() -> String { + std::env::var("CLAWHDF5_PYTHON").unwrap_or_else(|_| "python3".to_string()) +} + +fn have_python() -> bool { + let ok = Command::new(python()) + .args(["-c", "import h5py, hdf5plugin"]) + .output() + .map(|o| o.status.success()) + .unwrap_or(false); + if ok { + return true; + } + assert!( + !std::env::var("CLAWHDF5_REQUIRE_INTEROP").is_ok_and(|v| v == "1"), + "CLAWHDF5_REQUIRE_INTEROP=1 but python3 with h5py and hdf5plugin is not available" + ); + eprintln!("SKIP: python3 with h5py and hdf5plugin not available"); + false +} + +fn run_python(script: &str, args: &[&str]) -> String { + let output = Command::new(python()) + .arg("-c") + .arg(script) + .args(args) + .output() + .expect("failed to run python"); + assert!( + output.status.success(), + "Python script failed:\nSTDOUT: {}\nSTDERR: {}", + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr) + ); + String::from_utf8_lossy(&output.stdout).trim().to_string() +} + +/// Writes `f{i}` (ZFP) and `r{i}` (h5py's reading of `f{i}`, unfiltered) +/// for every mode x dtype x shape x data kind H5Z-ZFP accepts; each `f{i}` +/// has a `case` attribute. Prints the number of pairs. +const GENERATE: &str = r#" +import sys +import numpy as np, h5py, hdf5plugin +path = sys.argv[1] +MODES = [ + ('rate 2.5', dict(rate=2.5)), + ('rate 8', dict(rate=8)), + ('rate 16', dict(rate=16)), + ('rate 31', dict(rate=31)), + ('rate 64', dict(rate=64)), + ('precision 6', dict(precision=6)), + ('precision 20', dict(precision=20)), + ('precision 64', dict(precision=64)), + ('accuracy 0.5', dict(accuracy=0.5)), + ('accuracy 1e-3', dict(accuracy=1e-3)), + ('accuracy 1e-12', dict(accuracy=1e-12)), + ('reversible', dict(reversible=True)), + ('expert', dict(minbits=24, maxbits=600, maxprec=30, minexp=-20)), + # maxbits past ZFP_MAX_BITS: H5Z-ZFP stores it as fixed precision. + ('expert maxbits 20000', dict(minbits=1, maxbits=20000, maxprec=40, minexp=-1074)), + # A budget of 10 bits a block: one bit left after a float's exponent. + # (Doubles are left out: 12 bits of exponent and flag overrun it, and + # libzfp's encoder then writes past its buffer.) + ('expert maxbits 10', dict(minbits=5, maxbits=10, maxprec=64, minexp=-100)), + ('expert minbits 900', dict(minbits=900, maxbits=4000, maxprec=64, minexp=-60)), +] +DTYPES = [' dtypes read + let mut seen: BTreeMap> = BTreeMap::new(); + let mut failures = Vec::new(); + for i in 0..n { + let ds = file.dataset(&format!("f{i}")).unwrap(); + let case = match ds.attrs().unwrap().get("case") { + Some(clawhdf5::AttrValue::String(s)) => s.clone(), + other => panic!("f{i}: case attribute {other:?}"), + }; + let want = file + .dataset(&format!("r{i}")) + .unwrap() + .read_selection(&Selection::All) + .unwrap(); + match ds.read_selection(&Selection::All) { + Ok(got) if got == want => { + let mut parts = case.split('|'); + let mode = parts.next().unwrap().to_string(); + let dt = parts.next().unwrap().to_string(); + let e = seen.entry(mode).or_default(); + if !e.contains(&dt) { + e.push(dt); + } + } + Ok(got) => { + let first = got.iter().zip(&want).position(|(a, b)| a != b); + failures.push(format!("f{i} {case}: differs from byte {first:?}")); + } + Err(e) => failures.push(format!("f{i} {case}: {e}")), + } + } + assert!( + failures.is_empty(), + "{} of {n} datasets:\n{}", + failures.len(), + failures.join("\n") + ); + // Every mode was exercised on every type H5Z-ZFP accepts it for. + for (mode, dts) in &seen { + assert!(dts.len() >= 2, "{mode}: only {dts:?}"); + } + assert_eq!(seen.len(), 16, "{:?}", seen.keys()); + eprintln!("{n} ZFP datasets bit-exact; modes and types: {seen:?}"); +} + +/// A header written on a big-endian machine: H5Z-ZFP finds the magic only +/// after byte-swapping the `cd_values`, and then byte-swaps the decoded +/// values, since the dataset's datatype is big-endian there. The file is +/// made by swapping the header words of a little-endian one in place, so +/// the datatype stays little-endian and libhdf5 reads the values swapped; +/// clawhdf5 must read the same bytes. +#[test] +fn big_endian_header_swaps_the_values() { + if !have_python() { + return; + } + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("zfp_be.h5"); + let n: usize = run_python( + r#" +import sys, struct +import numpy as np, h5py, hdf5plugin +path = sys.argv[1] +cases = [('{len(cd) - 1}I', *cd[1:]) + at = raw.find(le) + assert at > 0 and raw.find(le, at + 1) < 0, 'cd_values not found once' + raw[at:at + len(le)] = swapped +open(path, 'wb').write(raw) +with h5py.File(path, 'a') as f: + for k in range(len(cases)): + f.create_dataset(f'r{k}', data=f[f'f{k}'][()]) + f.create_dataset(f'o{k}', data=f[f'f{k}'][()].byteswap()) +print(len(cases)) +"#, + &[path.to_str().unwrap()], + ) + .parse() + .unwrap(); + let file = File::open(&path).unwrap(); + for k in 0..n { + let read = |name: String| { + file.dataset(&name) + .unwrap() + .read_selection(&Selection::All) + .unwrap_or_else(|e| panic!("{name}: {e}")) + }; + let got = read(format!("f{k}")); + assert!(got == read(format!("r{k}")), "f{k}: differs from h5py"); + // Byte-swapped back, the values are the right ones. + assert!(got != read(format!("o{k}")), "f{k}: not swapped"); + } +} diff --git a/scripts/ci-test.sh b/scripts/ci-test.sh index 3012cba..a3fa3b8 100755 --- a/scripts/ci-test.sh +++ b/scripts/ci-test.sh @@ -67,7 +67,7 @@ run_step "cargo clippy --all-targets" cargo clippy \ # 3. Clippy over clawhdf5-format's optional features, which the default # workspace build never compiles (szip is left out: it needs libaec). -# plugin-filters = bitshuffle, bzip2, blosc, blosc2 (and the default-on lzf). +# plugin-filters = bitshuffle, bzip2, blosc, blosc2, zfp (and the default-on lzf). run_step "cargo clippy (format feature matrix)" cargo clippy \ -p clawhdf5-format \ --all-targets \ @@ -78,7 +78,7 @@ run_step "cargo clippy (format feature matrix)" cargo clippy \ # dependencies (bitshuffle and blosc share code). plugin_filters_alone() { local f - for f in bitshuffle bzip2 blosc blosc2; do + for f in bitshuffle bzip2 blosc blosc2 zfp; do echo "--- $f" cargo clippy -p clawhdf5-format --all-targets --features "$f" -- -D warnings || return 1 done @@ -208,9 +208,10 @@ if "$PYTHON" -c "import h5py" >/dev/null 2>&1 || [ "${CLAWHDF5_REQUIRE_INTEROP:- # libhdf5's registered plugins) compile and run too. run_step "h5py interop (format, ignored tests)" cargo test \ -p clawhdf5-format --features lz4,zstd --test writer_h5py_tests -- --include-ignored - # LZF, bitshuffle, bzip2 and Blosc both ways against h5py + hdf5plugin. + # LZF, bitshuffle, bzip2 and Blosc both ways against h5py + hdf5plugin; + # Blosc2 and ZFP (read-only) against what h5py reads. run_step "h5py interop (plugin filters)" cargo test \ - -p clawhdf5 --features plugin-filters --test plugin_filters_interop + -p clawhdf5 --features plugin-filters --test plugin_filters_interop --test zfp_interop else echo "" echo "==> [h5py interop] SKIPPED: no h5py in $PYTHON"