clawhdf5/crates/clawhdf5/tests/vds_interop.rs

//! Virtual Dataset (VDS) reads checked against libhdf5 (through h5py).
//!
//! Each test has h5py build virtual datasets and their source files in a temp
//! directory, record what libhdf5 reads back (shape and values) next to them,
//! and then compares that with what clawhdf5 reads from the same files.
//!
//! Skipped when python3 or h5py is unavailable, unless
//! `CLAWHDF5_REQUIRE_INTEROP=1`.

use std::path::Path;
use std::process::Command;

use clawhdf5::File;

/// The Python interpreter to drive interop checks with (`CLAWHDF5_PYTHON`
/// lets these run against a virtualenv holding h5py).
fn python() -> String {
    std::env::var("CLAWHDF5_PYTHON").unwrap_or_else(|_| "python3".to_string())
}

/// When `CLAWHDF5_REQUIRE_INTEROP=1` (set in CI), a missing Python dependency
/// is a test failure instead of a silent skip.
fn interop_required() -> bool {
    std::env::var("CLAWHDF5_REQUIRE_INTEROP").is_ok_and(|v| v == "1")
}

fn python_available() -> bool {
    Command::new(python())
        .args(["-c", "import h5py, numpy"])
        .output()
        .map(|o| o.status.success())
        .unwrap_or(false)
}

macro_rules! skip_if_no_python {
    () => {
        if !python_available() {
            assert!(
                !interop_required(),
                "CLAWHDF5_REQUIRE_INTEROP=1 but python3 with h5py is not available"
            );
            eprintln!("SKIP: python3 with h5py not available");
            return;
        }
    };
}

/// Prelude for every generator script: `expect(file, dset, tag)` records what
/// libhdf5 reads for `file:dset` as `<tag>.expect` (shape line, values line).
const PRELUDE: &str = r#"
import h5py, numpy as np
def expect(fn, dset, tag):
    with h5py.File(fn, "r") as f:
        d = f[dset]
        a = d[...]
        with open(tag + ".expect", "w") as out:
            out.write(" ".join(str(n) for n in d.shape) + "\n")
            out.write(" ".join(repr(float(v)) for v in a.ravel()) + "\n")
"#;

/// Run `body` (after [`PRELUDE`]) with `dir` as the working directory, so
/// relative source file names land next to the virtual file.
fn generate(dir: &Path, body: &str) {
    let script = format!("{PRELUDE}\n{body}");
    let out = Command::new(python())
        .args(["-c", &script])
        .current_dir(dir)
        .output()
        .expect("failed to run python");
    assert!(
        out.status.success(),
        "generator failed:\nSTDOUT: {}\nSTDERR: {}",
        String::from_utf8_lossy(&out.stdout),
        String::from_utf8_lossy(&out.stderr)
    );
}

/// What libhdf5 read for `tag`: (shape, values as f64).
fn expected(dir: &Path, tag: &str) -> (Vec<u64>, Vec<f64>) {
    let text = std::fs::read_to_string(dir.join(format!("{tag}.expect"))).unwrap();
    let mut lines = text.lines();
    let parse_line = |l: Option<&str>| -> Vec<String> {
        l.unwrap_or("")
            .split_whitespace()
            .map(str::to_string)
            .collect()
    };
    let shape = parse_line(lines.next())
        .iter()
        .map(|s| s.parse().unwrap())
        .collect();
    let values = parse_line(lines.next())
        .iter()
        .map(|s| s.parse().unwrap())
        .collect();
    (shape, values)
}

/// Assert clawhdf5 reads `file:dset` exactly as libhdf5 did for `tag`.
fn assert_matches_libhdf5(dir: &Path, file: &str, dset: &str, tag: &str) {
    let (shape, values) = expected(dir, tag);
    let f = File::open(dir.join(file)).unwrap();
    let ds = f.dataset(dset).unwrap();
    assert_eq!(
        ds.shape().unwrap(),
        shape,
        "{tag}: shape differs from libhdf5"
    );
    let got = ds
        .read_f64()
        .unwrap_or_else(|e| panic!("{tag}: read failed: {e}"));
    assert_eq!(got.len(), values.len(), "{tag}: element count differs");
    for (i, (g, e)) in got.iter().zip(&values).enumerate() {
        assert!(
            g == e || (g.is_nan() && e.is_nan()),
            "{tag}: element {i} is {g}, libhdf5 reads {e}\n ours: {got:?}\n libhdf5: {values:?}"
        );
    }
}

// ---------------------------------------------------------------------------
// Selection encodings
// ---------------------------------------------------------------------------

/// Files written with the default (earliest) format bounds serialize every
/// VDS hyperslab as a version-1 *irregular* selection (4-byte block corners),
/// and a strided selection as many blocks. These were refused outright.
#[test]
fn vds_version1_irregular_hyperslab_selections() {
    skip_if_no_python!();
    let dir = tempfile::tempdir().unwrap();
    generate(
        dir.path(),
        r#"
with h5py.File("src.h5", "w") as s:
    s.create_dataset("a", data=np.arange(12.0))
    s.create_dataset("m", data=np.arange(20.0).reshape(4, 5))
with h5py.File("v1.h5", "w") as f:   # default libver: hyperslab version 1
    f.create_dataset("local", data=np.arange(10.0) * -1)
    lay = h5py.VirtualLayout(shape=(12,), dtype="f8")
    lay[0:4] = h5py.VirtualSource(".", "local", shape=(10,))[2:6]
    lay[4:10] = h5py.VirtualSource("src.h5", "a", shape=(12,))[::2]
    lay[10:12] = h5py.VirtualSource("src.h5", "a", shape=(12,))[10:12]
    f.create_virtual_dataset("strided", lay)
    lay = h5py.VirtualLayout(shape=(4, 6), dtype="f8")
    lay[:, 0:2] = h5py.VirtualSource("src.h5", "m", shape=(4, 5))[:, 3:5]
    lay[:, 2:6:2] = h5py.VirtualSource("src.h5", "m", shape=(4, 5))[:, 0:2]
    lay[:, 3:6:2] = h5py.VirtualSource("src.h5", "m", shape=(4, 5))[:, 2:4]
    f.create_virtual_dataset("grid", lay)
expect("v1.h5", "strided", "strided")
expect("v1.h5", "grid", "grid")
"#,
    );
    assert_matches_libhdf5(dir.path(), "v1.h5", "strided", "strided");
    assert_matches_libhdf5(dir.path(), "v1.h5", "grid", "grid");
}