clawhdf5/crates/clawhdf5-format/tests/writer_meta_tests.rs

//! Regression tests for writer metadata bugs that produced files libhdf5
//! refuses (or reads differently from us), plus the reader-side counterparts.
//!
//! The plain tests check the bytes we write with our own parser. The
//! `#[ignore]`d ones are the interop half: they open what we write in h5py
//! (`CLAWHDF5_PYTHON`, as in `writer_h5py_tests.rs`) and run `h5dump` over it.

use clawhdf5_format::data_layout::DataLayout;
use clawhdf5_format::file_writer::{AttrValue, FileWriter};
use clawhdf5_format::group_v2::resolve_path_any;
use clawhdf5_format::message_type::MessageType;
use clawhdf5_format::object_header::ObjectHeader;
use clawhdf5_format::signature;
use clawhdf5_format::superblock::Superblock;
use clawhdf5_format::type_builders::make_u8_type;

// ---- helpers ----

fn header_at(bytes: &[u8], path: &str) -> (Superblock, ObjectHeader) {
    let sig = signature::find_signature(bytes).unwrap();
    let sb = Superblock::parse(bytes, sig).unwrap();
    let addr = if path == "/" {
        sb.root_group_address
    } else {
        resolve_path_any(bytes, &sb, path).unwrap()
    };
    let oh = ObjectHeader::parse(bytes, addr as usize, sb.offset_size, sb.length_size).unwrap();
    (sb, oh)
}

fn layout_of(bytes: &[u8], path: &str) -> DataLayout {
    let (sb, oh) = header_at(bytes, path);
    let msg = oh
        .messages
        .iter()
        .find(|m| m.msg_type == MessageType::DataLayout)
        .unwrap();
    DataLayout::parse(&msg.data, sb.offset_size, sb.length_size).unwrap()
}

fn python() -> String {
    std::env::var("CLAWHDF5_PYTHON").unwrap_or_else(|_| "python3".to_string())
}

fn write_tmp(name: &str, bytes: &[u8]) -> std::path::PathBuf {
    let path = std::env::temp_dir().join(format!("clawhdf5_writer_meta_{name}.h5"));
    std::fs::write(&path, bytes).unwrap();
    path
}

/// Run `script` (with `path` bound to the file) under h5py; return stdout.
fn h5py(path: &std::path::Path, script: &str) -> String {
    let full = format!(
        "import h5py, numpy as np, json\npath = {:?}\n{script}",
        path.display().to_string()
    );
    let o = std::process::Command::new(python())
        .args(["-c", &full])
        .output()
        .expect("python interpreter");
    assert!(
        o.status.success(),
        "h5py failed: {}",
        String::from_utf8_lossy(&o.stderr)
    );
    String::from_utf8(o.stdout).unwrap().trim().to_string()
}

/// `h5dump` must read the whole file without error.
fn h5dump_ok(path: &std::path::Path) {
    let o = std::process::Command::new("h5dump")
        .arg(path)
        .output()
        .expect("h5dump");
    assert!(
        o.status.success(),
        "h5dump failed: {}{}",
        String::from_utf8_lossy(&o.stdout),
        String::from_utf8_lossy(&o.stderr)
    );
}

fn u8_ramp(n: usize) -> Vec<u8> {
    (0..n).map(|i| (i % 251) as u8).collect()
}

// ---- 1. object header message size limit ----

#[test]
fn attribute_too_big_for_a_header_message_is_an_error() {
    // Measured: a 70000-byte attribute was written with its message size
    // wrapped to 16 bits, and libhdf5 refused the whole root group.
    let mut fw = FileWriter::new();
    fw.set_root_attr(
        "a",
        AttrValue::Raw {
            datatype: make_u8_type(),
            shape: vec![70_000],
            data: u8_ramp(70_000),
        },
    );
    assert!(fw.finish().is_err());

    // 65500 bytes still fits and still works.
    let mut fw = FileWriter::new();
    fw.set_root_attr(
        "a",
        AttrValue::Raw {
            datatype: make_u8_type(),
            shape: vec![65_500],
            data: u8_ramp(65_500),
        },
    );
    let bytes = fw.finish().unwrap();
    let (sb, oh) = header_at(&bytes, "/");
    let attrs = clawhdf5_format::attribute::extract_attributes(&oh, sb.length_size).unwrap();
    assert_eq!(attrs[0].raw_data, u8_ramp(65_500));
}

#[test]
fn compact_layout_falls_back_to_contiguous_past_the_message_limit() {
    // Layout message = 4 bytes + data; data may be at most 65531 bytes.
    for (n, compact) in [(65_531, true), (65_532, false), (65_534, false)] {
        let mut fw = FileWriter::new();
        fw.create_dataset("d").with_u8_data(&u8_ramp(n)).compact();
        let bytes = fw.finish().unwrap();
        match layout_of(&bytes, "d") {
            DataLayout::Compact { data } => {
                assert!(compact, "{n} bytes must not be compact");
                assert_eq!(data, u8_ramp(n));
            }
            DataLayout::Contiguous { .. } => assert!(!compact, "{n} bytes should be compact"),
            other => panic!("unexpected layout {other:?}"),
        }
    }
}

#[test]
#[ignore = "requires Python h5py module and h5dump"]
fn h5py_reads_compact_datasets_at_the_limit() {
    for n in [65_531usize, 65_534] {
        let mut fw = FileWriter::new();
        fw.create_dataset("d").with_u8_data(&u8_ramp(n)).compact();
        let path = write_tmp(&format!("compact_{n}"), &fw.finish().unwrap());
        let out = h5py(
            &path,
            "f = h5py.File(path, 'r'); v = f['d'][()]\n\
             print(bool((v == (np.arange(v.size) % 251).astype(np.uint8)).all()), v.size)",
        );
        assert_eq!(out, format!("True {n}"));
        h5dump_ok(&path);
    }
}