clawhdf5/crates/clawhdf5-wasm/tests/h5py_interop.rs

//! The wasm reader's core against files written by h5py and netCDF4, with
//! the values libhdf5 reads back as the reference. The generator,
//! `examples/wasm-viewer/test/make_fixture.py`, is shared with the Node test
//! of the built wasm package, so both compare against the same expectations.
//!
//! Skipped when python3 with h5py/netCDF4 is missing, unless
//! `CLAWHDF5_REQUIRE_INTEROP=1`. `CLAWHDF5_PYTHON` names the interpreter.

use std::path::{Path, PathBuf};
use std::process::Command;

use clawhdf5::AttrValue;
use clawhdf5_wasm::core::{Data, Hyperslab, Kind, Reader};
use serde_json::Value;

fn python() -> String {
    std::env::var("CLAWHDF5_PYTHON").unwrap_or_else(|_| "python3".to_string())
}

fn interop_required() -> bool {
    std::env::var("CLAWHDF5_REQUIRE_INTEROP").is_ok_and(|v| v == "1")
}

fn python_available() -> bool {
    Command::new(python())
        .args(["-c", "import h5py, netCDF4, numpy"])
        .output()
        .map(|o| o.status.success())
        .unwrap_or(false)
}

fn generator() -> PathBuf {
    Path::new(env!("CARGO_MANIFEST_DIR")).join("../../examples/wasm-viewer/test/make_fixture.py")
}

/// Values as comparable strings: integers exactly, floats by their f64
/// value (an f32 widens exactly), strings as themselves.
fn data_strings(d: &Data) -> (&'static str, Vec<String>) {
    fn s<T: ToString>(v: &[T]) -> Vec<String> {
        v.iter().map(ToString::to_string).collect()
    }
    fn f<T: Copy + Into<f64>>(v: &[T]) -> Vec<String> {
        v.iter().map(|&x| format!("{:?}", x.into())).collect()
    }
    match d {
        Data::F32(v) => ("f32", f(v)),
        Data::F64(v) => ("f64", f(v)),
        Data::I8(v) => ("i8", s(v)),
        Data::I16(v) => ("i16", s(v)),
        Data::I32(v) => ("i32", s(v)),
        Data::I64(v) => ("i64", s(v)),
        Data::U8(v) => ("u8", s(v)),
        Data::U16(v) => ("u16", s(v)),
        Data::U32(v) => ("u32", s(v)),
        Data::U64(v) => ("u64", s(v)),
        Data::Strings(v) => ("strings", v.clone()),
    }
}

fn expected_strings(kind: &str, values: &Value) -> Vec<String> {
    values
        .as_array()
        .unwrap()
        .iter()
        .map(|v| match (kind, v) {
            ("f32" | "f64", Value::Number(n)) => format!("{:?}", n.as_f64().unwrap()),
            (_, Value::String(s)) => s.clone(),
            other => panic!("unexpected expected value {other:?}"),
        })
        .collect()
}

fn shape(v: &Value) -> Vec<u64> {
    v.as_array()
        .unwrap()
        .iter()
        .map(|x| x.as_u64().unwrap())
        .collect()
}

fn check_attr(file: &str, path: &str, name: &str, got: &AttrValue, want: &Value) {
    let ctx = format!("{file}:{path}@{name}");
    let ints = |v: &Value| -> Vec<String> {
        v.as_array()
            .unwrap()
            .iter()
            .map(|x| x.as_str().unwrap().to_string())
            .collect()
    };
    let scalar = want["scalar"].as_bool().unwrap_or(false);
    match got {
        AttrValue::String(s) => assert_eq!(want["string"].as_str(), Some(s.as_str()), "{ctx}"),
        AttrValue::StringArray(v) => {
            let w: Vec<&str> = want["strings"]
                .as_array()
                .unwrap_or_else(|| panic!("{ctx}: got {v:?}"))
                .iter()
                .map(|x| x.as_str().unwrap())
                .collect();
            assert_eq!(v, &w, "{ctx}");
        }
        AttrValue::I64(x) => {
            assert!(scalar, "{ctx}");
            assert_eq!(vec![x.to_string()], ints(&want["int"]), "{ctx}");
        }
        AttrValue::U64(x) => {
            assert!(scalar, "{ctx}");
            assert_eq!(vec![x.to_string()], ints(&want["int"]), "{ctx}");
        }
        AttrValue::I64Array(v) => assert_eq!(
            v.iter().map(ToString::to_string).collect::<Vec<_>>(),
            ints(&want["int"]),
            "{ctx}"
        ),
        AttrValue::U64Array(v) => assert_eq!(
            v.iter().map(ToString::to_string).collect::<Vec<_>>(),
            ints(&want["int"]),
            "{ctx}"
        ),
        AttrValue::F64(x) => {
            assert!(scalar, "{ctx}");
            assert_eq!(Some(*x), want["float"][0].as_f64(), "{ctx}");
        }
        AttrValue::F64Array(v) => {
            let w: Vec<f64> = want["float"]
                .as_array()
                .unwrap()
                .iter()
                .map(|x| x.as_f64().unwrap())
                .collect();
            assert_eq!(v, &w, "{ctx}");
        }
        AttrValue::Raw { datatype, .. } => {
            let want = want["raw"]
                .as_str()
                .unwrap_or_else(|| panic!("{ctx}: undecoded {datatype:?}"));
            let d = clawhdf5_wasm::core::describe(datatype);
            assert!(d.contains(want), "{ctx}: {d}");
        }
    }
}

fn check_file(dir: &Path, file: &str, exp: &Value) {
    let r = Reader::open(std::fs::read(dir.join(file)).unwrap()).unwrap();

    for (path, want) in exp["lists"].as_object().unwrap() {
        let list = r
            .list(path)
            .unwrap_or_else(|e| panic!("{file}:{path}: {e}"));
        for (kind, key) in [(Kind::Group, "groups"), (Kind::Dataset, "datasets")] {
            let mut got: Vec<&str> = list
                .iter()
                .filter(|c| c.kind == kind)
                .map(|c| c.name.as_str())
                .collect();
            got.sort();
            let w: Vec<&str> = want[key]
                .as_array()
                .unwrap()
                .iter()
                .map(|x| x.as_str().unwrap())
                .collect();
            assert_eq!(got, w, "{file}:{path} {key}");
        }
    }

    for (path, want) in exp["datasets"].as_object().unwrap() {
        let kind = want["kind"].as_str().unwrap();
        let a = match (r.read(path, None), want["unavailable"].as_str()) {
            (Ok(a), _) => a,
            // A filter this build may lack (zstd: the wasm build has it
            // off, a workspace build may unify it on) must fail clearly.
            (Err(e), Some(why)) => {
                assert!(e.contains(why), "{file}:{path}: {e}");
                continue;
            }
            (Err(e), None) => panic!("{file}:{path}: {e}"),
        };
        assert_eq!(a.shape, shape(&want["shape"]), "{file}:{path} shape");
        let (got_kind, got) = data_strings(&a.data);
        assert_eq!(got_kind, kind, "{file}:{path} kind");
        assert_eq!(
            got,
            expected_strings(kind, &want["values"]),
            "{file}:{path}"
        );

        if let Some(slab) = want.get("slab") {
            let h = Hyperslab {
                start: shape(&slab["start"]),
                count: shape(&slab["count"]),
                stride: Some(shape(&slab["stride"])),
                block: None,
            };
            let a = r
                .read(path, Some(&h))
                .unwrap_or_else(|e| panic!("{file}:{path} {h:?}: {e}"));
            assert_eq!(a.shape, shape(&slab["shape"]), "{file}:{path} slab shape");
            assert_eq!(
                data_strings(&a.data).1,
                expected_strings(kind, &slab["values"]),
                "{file}:{path} slab"
            );
        }
    }

    for (path, what) in exp["errors"].as_object().unwrap() {
        let e = r.read(path, None).expect_err(path);
        assert!(e.contains(what.as_str().unwrap()), "{file}:{path}: {e}");
    }

    let skip: Vec<&str> = exp["skip_attrs"]
        .as_array()
        .unwrap()
        .iter()
        .map(|x| x.as_str().unwrap())
        .collect();
    for (path, want) in exp["attrs"].as_object().unwrap() {
        let (attrs, errors) = r
            .attrs(path)
            .unwrap_or_else(|e| panic!("{file}:{path}: {e}"));
        assert!(errors.is_empty(), "{file}:{path}: {errors:?}");
        let want = want.as_object().unwrap();
        let mut compared = 0;
        for a in &attrs {
            if a.name.starts_with('_') || skip.contains(&a.name.as_str()) {
                continue;
            }
            let w = want
                .get(&a.name)
                .unwrap_or_else(|| panic!("{file}:{path}: unexpected attribute {}", a.name));
            check_attr(file, path, &a.name, &a.value, w);
            compared += 1;
        }
        assert_eq!(compared, want.len(), "{file}:{path}: attributes missing");
    }
}

#[test]
fn reads_what_h5py_and_netcdf4_wrote() {
    if !python_available() {
        assert!(
            !interop_required(),
            "CLAWHDF5_REQUIRE_INTEROP=1 but python with h5py, netCDF4 and numpy is not available"
        );
        eprintln!("SKIP: python with h5py/netCDF4 not available");
        return;
    }
    let dir = tempfile::tempdir().unwrap();
    let out = Command::new(python())
        .arg(generator())
        .arg(dir.path())
        .output()
        .expect("run python");
    assert!(
        out.status.success(),
        "fixture generator failed:\n{}",
        String::from_utf8_lossy(&out.stderr)
    );
    let exp: Value =
        serde_json::from_slice(&std::fs::read(dir.path().join("expected.json")).unwrap()).unwrap();
    for file in ["fixture.h5", "fixture.nc"] {
        check_file(dir.path(), file, &exp[file]);
    }
}