clawhdf5/crates/clawhdf5/tests/legacy_format_interop.rs

//! Files written by HDF5 1.4/1.6-era libraries: Data Layout message versions
//! 1 and 2, compound datatype version 1 array members, and version-1 shared
//! message references. The fixtures are HDF5's own test files (see
//! `clawhdf5-format/tests/fixtures/legacy/README.md`).
//!
//! The expected values were read with h5py 3.16 / HDF5 2.0; the interop test
//! re-checks every dataset byte for byte against h5py, and is skipped when
//! python3 with h5py is unavailable unless `CLAWHDF5_REQUIRE_INTEROP=1`.

use std::process::Command;

use clawhdf5::{DType, File};
use clawhdf5_format::selection::Selection;

const FIXTURES: &str = concat!(
    env!("CARGO_MANIFEST_DIR"),
    "/../clawhdf5-format/tests/fixtures/legacy"
);

fn open(name: &str) -> File {
    File::open(format!("{FIXTURES}/{name}")).unwrap()
}

fn python() -> String {
    std::env::var("CLAWHDF5_PYTHON").unwrap_or_else(|_| "python3".to_string())
}

fn interop_required() -> bool {
    std::env::var("CLAWHDF5_REQUIRE_INTEROP").is_ok_and(|v| v == "1")
}

fn python_available() -> bool {
    Command::new(python())
        .args(["-c", "import h5py, numpy"])
        .output()
        .map(|o| o.status.success())
        .unwrap_or(false)
}

/// Layout v1, chunked (50x50 chunks of a 100x200 dataset), deflate: every
/// read path goes through the version-1 B-tree chunk index.
#[test]
fn layout_v1_chunked_deflate() {
    let file = open("deflate.h5");
    let ds = file.dataset("Dataset1").unwrap();
    assert_eq!(ds.shape().unwrap(), [100, 200]);
    let expected: Vec<i32> = (0..100).flat_map(|_| (0..200).map(|j| j % 5)).collect();
    assert_eq!(ds.read_i32().unwrap(), expected);

    // A hyperslab that straddles four chunks.
    let slab = Selection::Hyperslab {
        start: vec![48, 48],
        stride: vec![1, 1],
        count: vec![4, 4],
        block: vec![1, 1],
    };
    let raw = ds.read_selection(&slab).unwrap();
    let got: Vec<i32> = raw
        .as_chunks::<4>()
        .0
        .iter()
        .map(|b| i32::from_le_bytes(*b))
        .collect();
    assert_eq!(got, [3, 4, 0, 1, 3, 4, 0, 1, 3, 4, 0, 1, 3, 4, 0, 1]);
}

/// Layout v2, contiguous: one dataset with storage, one never written (reads
/// as its fill value, 0).
#[test]
fn layout_v2_contiguous() {
    let file = open("h5ex_g_iterate.h5");
    assert_eq!(file.dataset("G1/DS2").unwrap().read_i32().unwrap(), [1]);
    assert_eq!(file.dataset("DS1").unwrap().read_i32().unwrap(), [0]);
}

/// Compound datatype version 1 members carrying legacy array dimensions
/// (HDF5 before 1.4 had no array class). h5py: `[('i', '<i2'), ('f', '<f4',
/// (4,)), ('l', '<i4', (4,)), ('d', '<f8')]`, itemsize 44.
#[test]
fn compound_v1_legacy_array_members() {
    let file = open("tarrold.h5");
    let ds = file.dataset("Dataset2").unwrap();
    assert_eq!(
        ds.dtype().unwrap(),
        DType::Compound(vec![
            ("i".into(), DType::I16),
            ("f".into(), DType::Array(Box::new(DType::F32), vec![4])),
            ("l".into(), DType::Array(Box::new(DType::I32), vec![4])),
            ("d".into(), DType::F64),
        ])
    );
    assert_eq!(ds.shape().unwrap(), [8, 9]);
    assert_eq!(
        ds.read_selection(&Selection::All).unwrap().len(),
        8 * 9 * 44
    );
}

/// Every dataset in every fixture, byte for byte against h5py.
#[test]
fn legacy_fixtures_match_h5py() {
    if !python_available() {
        assert!(
            !interop_required(),
            "CLAWHDF5_REQUIRE_INTEROP=1 but python3 with h5py is not available"
        );
        eprintln!("SKIP: python3 with h5py not available");
        return;
    }
    for (name, datasets) in [
        ("deflate.h5", &["Dataset1"][..]),
        ("h5ex_g_iterate.h5", &["DS1", "G1/DS2"][..]),
        ("tarrold.h5", &["Dataset1", "Dataset2"][..]),
    ] {
        let path = format!("{FIXTURES}/{name}");
        let script = format!(
            r#"
import h5py, numpy as np
f = h5py.File({path:?}, "r")
for n in {datasets:?}:
    print(n, np.ascontiguousarray(f[n][()]).tobytes().hex())
"#
        );
        let out = Command::new(python())
            .args(["-c", &script])
            .output()
            .unwrap();
        assert!(
            out.status.success(),
            "h5py: {}",
            String::from_utf8_lossy(&out.stderr)
        );
        let file = File::open(&path).unwrap();
        for line in String::from_utf8(out.stdout).unwrap().lines() {
            let (ds, hex) = line.split_once(' ').unwrap();
            let ours = file
                .dataset(ds)
                .unwrap()
                .read_selection(&Selection::All)
                .unwrap();
            let ours: String = ours.iter().map(|b| format!("{b:02x}")).collect();
            assert_eq!(ours, hex, "{name}:{ds}");
        }
    }
}