clawhdf5/crates/clawhdf5-wasm/tests/vl_strings.rs

//! The wasm reader resolves VL strings with the library's `VlResolver`, so
//! it returns what `File::read_string` and h5py return: a string ends at
//! its first NUL, a null element is empty, a heap object of the wrong size
//! is an error, an element at the undefined heap address is an error, and a
//! VL datatype whose stored element size disagrees with the file's offset
//! size is refused. Checked with 8- and 4-byte offsets.
//!
//! Skipped when python3 with h5py is missing, unless
//! `CLAWHDF5_REQUIRE_INTEROP=1`. `CLAWHDF5_PYTHON` names the interpreter.

use std::process::Command;

use clawhdf5::File;
use clawhdf5_wasm::core::{Data, Reader};

fn python() -> String {
    std::env::var("CLAWHDF5_PYTHON").unwrap_or_else(|_| "python3".to_string())
}

fn h5py_available() -> bool {
    let ok = Command::new(python())
        .args(["-c", "import h5py, numpy"])
        .output()
        .map(|o| o.status.success())
        .unwrap_or(false);
    if !ok {
        assert!(
            std::env::var("CLAWHDF5_REQUIRE_INTEROP").as_deref() != Ok("1"),
            "CLAWHDF5_REQUIRE_INTEROP=1 but python with h5py is not available"
        );
        eprintln!("SKIP: python with h5py not available");
    }
    ok
}

/// For each offset size: `vl{8,4}.h5` with dataset `d` = "a\0b", "", null,
/// "zz" (patched: h5py writes neither a NUL nor a null element);
/// `bad{8,4}.h5` whose element 0 claims 3 bytes of a 6-byte heap object;
/// `size{8,4}.h5` whose VL datatype message stores a 24-byte element; and
/// `undef{8,4}.h5` whose element 1 has length 0 and the undefined heap
/// address.
/// Prints h5py's reading of each element as hex, or "error".
const SCRIPT: &str = r#"
import struct, sys, h5py, numpy as np
out = sys.argv[1]
S = h5py.string_dtype('utf-8')
def create(path, os_):
    if os_ == 8:
        return h5py.File(path, 'w', libver='earliest')
    # The earliest format, so the patched object header has no checksum.
    fcpl = h5py.h5p.create(h5py.h5p.FILE_CREATE); fcpl.set_sizes(4, 4)
    fapl = h5py.h5p.create(h5py.h5p.FILE_ACCESS)
    fapl.set_libver_bounds(h5py.h5f.LIBVER_EARLIEST, h5py.h5f.LIBVER_V18)
    return h5py.File(h5py.h5f.create(path.encode(), h5py.h5f.ACC_TRUNC, fcpl=fcpl, fapl=fapl))
def elem(length, addr, index, os_):
    return struct.pack('<I', length) + addr.to_bytes(os_, 'little') + struct.pack('<I', index)
def make(path, os_, values):
    with create(path, os_) as f:
        f.create_dataset('d', data=np.array(values, dtype=object), dtype=S)
        return f['d'].id.get_offset()
for os_ in (8, 4):
    es = 8 + os_
    p = '%s/vl%d.h5' % (out, os_)
    off = make(p, os_, ['aXb', '', 'ok', 'zz'])
    b = bytearray(open(p, 'rb').read())
    b[b.index(b'aXb') + 1] = 0
    b[off + 2 * es:off + 3 * es] = elem(2, 0, 1, os_)
    open(p, 'wb').write(bytes(b))
    p = '%s/bad%d.h5' % (out, os_)
    off = make(p, os_, ['cdefgh', 'ok'])
    b = bytearray(open(p, 'rb').read())
    struct.pack_into('<I', b, off, 3)
    open(p, 'wb').write(bytes(b))
    p = '%s/size%d.h5' % (out, os_)
    make(p, os_, ['x', 'yy'])
    b = bytearray(open(p, 'rb').read())
    # datatype message: version 1, class 9 (VL); string, null-terminated, UTF-8
    pat = bytes([0x19, 0x01, 0x01, 0x00]) + struct.pack('<I', es)
    assert b.count(pat) == 1, b.count(pat)
    i = b.index(pat)
    struct.pack_into('<I', b, i + 4, 24)
    open(p, 'wb').write(bytes(b))
    p = '%s/undef%d.h5' % (out, os_)
    off = make(p, os_, ['x', '', 'yz'])
    b = bytearray(open(p, 'rb').read())
    b[off + es:off + 2 * es] = elem(0, (1 << (8 * os_)) - 1, 1, os_)
    open(p, 'wb').write(bytes(b))
    for name in ('vl', 'bad', 'size', 'undef'):
        with h5py.File('%s/%s%d.h5' % (out, name, os_), 'r') as f:
            got = []
            for i in range(f['d'].shape[0]):
                try:
                    got.append(f['d'][i].hex())
                except OSError:
                    got.append('error')
            print('%s%d\t%s' % (name, os_, ','.join(got)))
"#;

#[test]
fn vl_strings_read_like_file_and_h5py() {
    if !h5py_available() {
        return;
    }
    let dir = tempfile::tempdir().unwrap();
    let out = Command::new(python())
        .args(["-c", SCRIPT])
        .arg(dir.path())
        .output()
        .expect("run python");
    assert!(
        out.status.success(),
        "{}",
        String::from_utf8_lossy(&out.stderr)
    );
    let h5py: std::collections::HashMap<String, String> = String::from_utf8(out.stdout)
        .unwrap()
        .lines()
        .map(|l| {
            let (k, v) = l.split_once('\t').unwrap();
            (k.to_string(), v.to_string())
        })
        .collect();
    let read = |name: &str| {
        let bytes = std::fs::read(dir.path().join(format!("{name}.h5"))).unwrap();
        let wasm = Reader::open(bytes).unwrap().read("/d", None);
        let file = File::open(dir.path().join(format!("{name}.h5")))
            .unwrap()
            .dataset("d")
            .unwrap()
            .read_string();
        (wasm, file)
    };
    for os in [8, 4] {
        // h5py: "a\0b" is "a"; the null element (address 0) is empty.
        assert_eq!(h5py[&format!("vl{os}")], "61,,,7a7a");
        let (wasm, file) = read(&format!("vl{os}"));
        let Data::Strings(wasm) = wasm.unwrap().data else {
            panic!("vl{os}: not strings")
        };
        assert_eq!(wasm, ["a", "", "", "zz"], "vl{os}");
        assert_eq!(wasm, file.unwrap(), "vl{os}");

        // h5py refuses the mis-sized element; so do both readers.
        assert_eq!(h5py[&format!("bad{os}")], "error,6f6b");
        let (wasm, file) = read(&format!("bad{os}"));
        assert!(wasm.unwrap_err().contains("holds 6 bytes"), "bad{os}");
        assert!(file.is_err(), "bad{os}");

        // libhdf5 ignores the stored element size and reads the values;
        // File refuses the datatype rather than guess its layout, and the
        // wasm reader now does the same (it read with the stored size).
        assert_eq!(h5py[&format!("size{os}")], "78,7979");
        let (wasm, file) = read(&format!("size{os}"));
        let e = wasm.unwrap_err();
        assert!(e.contains("stores 24-byte elements"), "size{os}: {e}");
        assert!(file.is_err(), "size{os}");

        // Length 0 at the undefined heap address: libhdf5 fails the read
        // ("addr undefined"); both readers returned "".
        assert_eq!(h5py[&format!("undef{os}")], "78,error,797a");
        let (wasm, file) = read(&format!("undef{os}"));
        let e = wasm.unwrap_err();
        assert!(
            e.contains("undefined global heap address"),
            "undef{os}: {e}"
        );
        assert!(file.is_err(), "undef{os}");
    }
}