clawhdf5/crates/clawhdf5/tests/vl_offset4_interop.rs

//! Variable-length values in files with 4-byte offsets and lengths
//! (`sizeof_addr = 4`), checked against h5py/libhdf5 through the
//! `clawhdf5_format` decoders.
//!
//! These failed with `GlobalHeapObjectNotFound` or came back as
//! `AttrValue::Raw`: the VL datatype claimed 16-byte elements whatever the
//! file's offset size, and the global heap was read without the padding
//! libhdf5 puts after its collection and object headers. Skipped when
//! python3 with h5py is unavailable, unless `CLAWHDF5_REQUIRE_INTEROP=1`.

use std::process::Command;

use clawhdf5::{AttrValue, File, Selection};
use clawhdf5_format::data_read::{read_as_i64, read_compound_fields};
use clawhdf5_format::vl_data::{read_vl_bytes, read_vl_strings};

fn python() -> String {
    std::env::var("CLAWHDF5_PYTHON").unwrap_or_else(|_| "python3".to_string())
}

fn interop_required() -> bool {
    std::env::var("CLAWHDF5_REQUIRE_INTEROP").is_ok_and(|v| v == "1")
}

fn python_available() -> bool {
    Command::new(python())
        .args(["-c", "import h5py, numpy"])
        .output()
        .map(|o| o.status.success())
        .unwrap_or(false)
}

#[test]
fn vl_values_in_a_file_with_4_byte_offsets_read_like_h5py() {
    if !python_available() {
        assert!(
            !interop_required(),
            "CLAWHDF5_REQUIRE_INTEROP=1 but python3 with h5py is not available"
        );
        eprintln!("SKIP: python3 with h5py not available");
        return;
    }
    let dir = tempfile::tempdir().unwrap();
    let path = dir.path().join("offset4.h5");
    let script = format!(
        r#"
import h5py, numpy as np
S = h5py.string_dtype()
fcpl = h5py.h5p.create(h5py.h5p.FILE_CREATE); fcpl.set_sizes(4, 4)
with h5py.File(h5py.h5f.create({path:?}.encode(), h5py.h5f.ACC_TRUNC, fcpl=fcpl)) as f:
    f.attrs['vlattr'] = 'attr-value'
    f.attrs.create('vlattr_arr', np.array(['p', 'qq', ''], dtype=object), dtype=S)
    ct = np.dtype([('id', '<i4'), ('name', S), ('v', '<f8')])
    arr = np.zeros(3, dtype=ct); arr['id'] = [1, 2, 3]; arr['name'] = ['one', '', 'three']
    f.create_dataset('compound', data=arr)
    f.attrs.create('vlen_attr', np.array([np.array([1, 2], dtype='<i8'), np.array([3], dtype='<i8')],
                   dtype=object), dtype=h5py.vlen_dtype(np.dtype('<i8')))
with h5py.File({path:?}, 'r') as f:
    assert f.id.get_create_plist().get_sizes() == (4, 4)
    print(f.attrs['vlattr'])
    print(','.join(f.attrs['vlattr_arr']))
    print(','.join(s.decode() for s in f['compound']['name']))
    print(';'.join(' '.join(str(x) for x in s) for s in f.attrs['vlen_attr']))
"#,
        path = path.display().to_string()
    );
    let output = Command::new(python())
        .args(["-c", &script])
        .output()
        .expect("failed to run python");
    assert!(
        output.status.success(),
        "python failed:\n{}",
        String::from_utf8_lossy(&output.stderr)
    );
    let stdout = String::from_utf8(output.stdout).unwrap();
    let lines: Vec<&str> = stdout.lines().collect();
    let (vlattr, vlattr_arr, names, seqs) = (lines[0], lines[1], lines[2], lines[3]);

    let file = File::open(&path).unwrap();
    let sb = file.superblock();
    assert_eq!((sb.offset_size, sb.length_size), (4, 4));

    let attrs = file.root().attrs().unwrap();
    match &attrs["vlattr"] {
        AttrValue::String(s) => assert_eq!(s, vlattr),
        other => panic!("vlattr: {other:?}"),
    }
    match &attrs["vlattr_arr"] {
        AttrValue::StringArray(s) => assert_eq!(s.join(","), vlattr_arr),
        other => panic!("vlattr_arr: {other:?}"),
    }

    // The compound's VL string member.
    let ds = file.dataset("compound").unwrap();
    let dt = ds.raw_datatype().unwrap();
    assert_eq!(dt.type_size(), 24, "4 + 12-byte VL element + 8");
    let raw = ds.read_selection(&Selection::All).unwrap();
    let fields = read_compound_fields(&raw, &dt).unwrap();
    let name = fields.iter().find(|f| f.name == "name").unwrap();
    assert_eq!(name.datatype.type_size(), 12);
    let got = read_vl_strings(file.as_bytes(), &name.raw_data, 3, 4, 4).unwrap();
    assert_eq!(got.join(","), names);
    let id = fields.iter().find(|f| f.name == "id").unwrap();
    assert_eq!(
        read_as_i64(&id.raw_data, &id.datatype).unwrap(),
        vec![1, 2, 3]
    );

    // A VL sequence attribute.
    let AttrValue::Raw { datatype, data, .. } = &attrs["vlen_attr"] else {
        panic!("vlen_attr is Raw");
    };
    let clawhdf5_format::datatype::Datatype::VariableLength { base_type, .. } = datatype else {
        panic!("vlen_attr is VL");
    };
    let got: Vec<String> = read_vl_bytes(file.as_bytes(), data, 2, 4, 4)
        .unwrap()
        .iter()
        .map(|b| {
            let v = read_as_i64(b, base_type).unwrap();
            v.iter().map(i64::to_string).collect::<Vec<_>>().join(" ")
        })
        .collect();
    assert_eq!(got.join(";"), seqs);
}