clawhdf5/crates/clawhdf5/tests/legacy_format_interop.rs

//! Files written by HDF5 1.4/1.6-era libraries: Data Layout message versions
//! 1 and 2, compound datatype version 1 array members, and version-1 shared
//! message references. The fixtures are HDF5's own test files (see
//! `clawhdf5-format/tests/fixtures/legacy/README.md`).
//!
//! The expected values were read with h5py 3.16 / HDF5 2.0; the interop test
//! re-checks every dataset byte for byte against h5py, and is skipped when
//! python3 with h5py is unavailable unless `CLAWHDF5_REQUIRE_INTEROP=1`.

use std::process::Command;

use clawhdf5::{DType, File};
use clawhdf5_format::selection::Selection;

const FIXTURES: &str = concat!(
    env!("CARGO_MANIFEST_DIR"),
    "/../clawhdf5-format/tests/fixtures/legacy"
);

fn open(name: &str) -> File {
    File::open(format!("{FIXTURES}/{name}")).unwrap()
}

fn python() -> String {
    std::env::var("CLAWHDF5_PYTHON").unwrap_or_else(|_| "python3".to_string())
}

fn interop_required() -> bool {
    std::env::var("CLAWHDF5_REQUIRE_INTEROP").is_ok_and(|v| v == "1")
}

fn python_available() -> bool {
    Command::new(python())
        .args(["-c", "import h5py, numpy"])
        .output()
        .map(|o| o.status.success())
        .unwrap_or(false)
}

/// Layout v1, chunked (50x50 chunks of a 100x200 dataset), deflate: every
/// read path goes through the version-1 B-tree chunk index.
#[test]
fn layout_v1_chunked_deflate() {
    let file = open("deflate.h5");
    let ds = file.dataset("Dataset1").unwrap();
    assert_eq!(ds.shape().unwrap(), [100, 200]);
    let expected: Vec<i32> = (0..100).flat_map(|_| (0..200).map(|j| j % 5)).collect();
    assert_eq!(ds.read_i32().unwrap(), expected);

    // A hyperslab that straddles four chunks.
    let slab = Selection::Hyperslab {
        start: vec![48, 48],
        stride: vec![1, 1],
        count: vec![4, 4],
        block: vec![1, 1],
    };
    let raw = ds.read_selection(&slab).unwrap();
    let got: Vec<i32> = raw
        .as_chunks::<4>()
        .0
        .iter()
        .map(|b| i32::from_le_bytes(*b))
        .collect();
    assert_eq!(got, [3, 4, 0, 1, 3, 4, 0, 1, 3, 4, 0, 1, 3, 4, 0, 1]);
}

/// Layout v2, contiguous: one dataset with storage, one never written (reads
/// as its fill value, 0).
#[test]
fn layout_v2_contiguous() {
    let file = open("h5ex_g_iterate.h5");
    assert_eq!(file.dataset("G1/DS2").unwrap().read_i32().unwrap(), [1]);
    assert_eq!(file.dataset("DS1").unwrap().read_i32().unwrap(), [0]);
}

/// Compound datatype version 1 members carrying legacy array dimensions
/// (HDF5 before 1.4 had no array class). h5py: `[('i', '<i2'), ('f', '<f4',
/// (4,)), ('l', '<i4', (4,)), ('d', '<f8')]`, itemsize 44.
#[test]
fn compound_v1_legacy_array_members() {
    let file = open("tarrold.h5");
    let ds = file.dataset("Dataset2").unwrap();
    assert_eq!(
        ds.dtype().unwrap(),
        DType::Compound(vec![
            ("i".into(), DType::I16),
            ("f".into(), DType::Array(Box::new(DType::F32), vec![4])),
            ("l".into(), DType::Array(Box::new(DType::I32), vec![4])),
            ("d".into(), DType::F64),
        ])
    );
    assert_eq!(ds.shape().unwrap(), [8, 9]);
    assert_eq!(
        ds.read_selection(&Selection::All).unwrap().len(),
        8 * 9 * 44
    );
}

/// Datasets whose committed datatype is referenced by a version-1 shared
/// message, whose object header address follows a link-name offset. Values
/// from h5py; the file is big-endian.
#[test]
fn shared_message_v1_committed_datatypes() {
    let file = open("tcompound.h5");
    let be_pairs = |name: &str| -> Vec<(i32, f32)> {
        file.dataset(name)
            .unwrap()
            .read_selection(&Selection::All)
            .unwrap()
            .as_chunks::<8>()
            .0
            .iter()
            .map(|b| {
                (
                    i32::from_be_bytes(b[..4].try_into().unwrap()),
                    f32::from_be_bytes(b[4..].try_into().unwrap()),
                )
            })
            .collect()
    };
    assert_eq!(
        be_pairs("group1/dset2"),
        [(0, 0.0), (1, 1.1), (2, 2.2), (3, 3.3), (4, 4.4)]
    );
    assert_eq!(
        be_pairs("group2/dset5"),
        [(0, 0.0), (1, 0.1), (2, 0.2), (3, 0.3), (4, 0.4)]
    );

    // `/type2`: { int_array: i32[4], float_array: f32[5][6] }, whose array
    // members are compound v1 legacy dimensions.
    let dset3 = file.dataset("group1/dset3").unwrap();
    assert_eq!(
        dset3.dtype().unwrap(),
        DType::Compound(vec![
            (
                "int_array".into(),
                DType::Array(Box::new(DType::I32), vec![4])
            ),
            (
                "float_array".into(),
                DType::Array(Box::new(DType::F32), vec![5, 6])
            ),
        ])
    );
    let raw = dset3.read_selection(&Selection::All).unwrap();
    assert_eq!(raw.len(), 3 * 6 * (16 + 120));
    assert_eq!(
        &raw[..16],
        &[0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 2, 0, 0, 0, 3]
    );
    let first: Vec<f32> = raw[16..16 + 120]
        .as_chunks::<4>()
        .0
        .iter()
        .map(|b| f32::from_be_bytes(*b))
        .collect();
    let expected: Vec<f32> = (0..5)
        .flat_map(|i| (0..6).map(move |j| (1 + i + j) as f32))
        .collect();
    assert_eq!(first, expected);
}

/// Every dataset in every fixture, byte for byte against h5py.
#[test]
fn legacy_fixtures_match_h5py() {
    if !python_available() {
        assert!(
            !interop_required(),
            "CLAWHDF5_REQUIRE_INTEROP=1 but python3 with h5py is not available"
        );
        eprintln!("SKIP: python3 with h5py not available");
        return;
    }
    for (name, datasets) in [
        ("deflate.h5", &["Dataset1"][..]),
        ("h5ex_g_iterate.h5", &["DS1", "G1/DS2"][..]),
        ("tarrold.h5", &["Dataset1", "Dataset2"][..]),
        (
            "tcompound.h5",
            &[
                "dset1",
                "group1/dset2",
                "group1/dset3",
                "group1/dset4",
                "group2/dset5",
            ][..],
        ),
    ] {
        let path = format!("{FIXTURES}/{name}");
        let script = format!(
            r#"
import h5py, numpy as np
f = h5py.File({path:?}, "r")
for n in {datasets:?}:
    print(n, np.ascontiguousarray(f[n][()]).tobytes().hex())
"#
        );
        let out = Command::new(python())
            .args(["-c", &script])
            .output()
            .unwrap();
        assert!(
            out.status.success(),
            "h5py: {}",
            String::from_utf8_lossy(&out.stderr)
        );
        let file = File::open(&path).unwrap();
        for line in String::from_utf8(out.stdout).unwrap().lines() {
            let (ds, hex) = line.split_once(' ').unwrap();
            let ours = file
                .dataset(ds)
                .unwrap()
                .read_selection(&Selection::All)
                .unwrap();
            let ours: String = ours.iter().map(|b| format!("{b:02x}")).collect();
            assert_eq!(ours, hex, "{name}:{ds}");
        }
    }
}