clawhdf5/crates/clawhdf5/tests/header_validation_interop.rs

//! Corrupt files that libhdf5 refuses must be refused here too, not read.
//!
//! h5py writes a valid file, the script corrupts a copy the way a damaged or
//! malicious file would be, and records whether h5py (libhdf5) still opens
//! and reads the object. clawhdf5 must agree: read the valid file, refuse
//! each corrupt one. Skipped when python3 with h5py is unavailable, unless
//! `CLAWHDF5_REQUIRE_INTEROP=1`.

use std::path::Path;
use std::process::Command;

use clawhdf5::File;

fn python() -> String {
    std::env::var("CLAWHDF5_PYTHON").unwrap_or_else(|_| "python3".to_string())
}

fn interop_required() -> bool {
    std::env::var("CLAWHDF5_REQUIRE_INTEROP").is_ok_and(|v| v == "1")
}

fn python_available() -> bool {
    Command::new(python())
        .args(["-c", "import h5py, numpy"])
        .output()
        .map(|o| o.status.success())
        .unwrap_or(false)
}

macro_rules! skip_if_no_python {
    () => {
        if !python_available() {
            assert!(
                !interop_required(),
                "CLAWHDF5_REQUIRE_INTEROP=1 but python3 with h5py is not available"
            );
            eprintln!("SKIP: python3 with h5py not available");
            return;
        }
    };
}

/// Runs `body` (Python, with `h5py`, `numpy as np`, `struct` imported and
/// `d` the output directory) and then, for every `NAME.h5` it wrote,
/// prints `NAME ok` when h5py opens and reads dataset `d` and `NAME ERROR`
/// otherwise. Returns those lines, sorted.
fn h5py_verdicts(dir: &Path, body: &str) -> Vec<String> {
    let script = format!(
        r#"
import h5py, numpy as np, struct, os, glob
d = "{dir}"
{body}
for path in sorted(glob.glob(os.path.join(d, "*.h5"))):
    name = os.path.basename(path)[:-3]
    try:
        with h5py.File(path, "r") as f:
            f["d"][()]
        print(name, "ok")
    except Exception:
        print(name, "ERROR")
"#,
        dir = dir.display()
    );
    let out = Command::new(python())
        .args(["-c", &script])
        .output()
        .expect("failed to run python");
    assert!(
        out.status.success(),
        "python failed:\n{}",
        String::from_utf8_lossy(&out.stderr)
    );
    let mut lines: Vec<String> = String::from_utf8_lossy(&out.stdout)
        .lines()
        .map(str::to_owned)
        .collect();
    lines.sort();
    lines
}

/// Whether clawhdf5 opens and reads dataset `d` of `path` (as raw bytes of
/// whatever type it has).
fn clawhdf5_reads(path: &Path) -> Result<(), String> {
    let file = File::open(path).map_err(|e| format!("open: {e}"))?;
    let ds = file.dataset("d").map_err(|e| format!("dataset: {e}"))?;
    ds.dtype().map_err(|e| format!("dtype: {e}"))?;
    ds.shape().map_err(|e| format!("shape: {e}"))?;
    ds.read_i32().map(|_| ()).map_err(|e| format!("read: {e}"))
}

/// h5py's verdict for each file must be `expected`, and clawhdf5 must read
/// exactly the files h5py reads.
fn assert_agrees_with_h5py(dir: &Path, verdicts: &[String], expected: &[&str]) {
    assert_eq!(verdicts, expected, "h5py's view changed");
    for line in verdicts {
        let (name, verdict) = line.split_once(' ').unwrap();
        let ours = clawhdf5_reads(&dir.join(format!("{name}.h5")));
        match verdict {
            "ok" => assert!(ours.is_ok(), "{name}: h5py reads it, we fail: {ours:?}"),
            _ => assert!(ours.is_err(), "{name}: h5py refuses it, we read it"),
        }
    }
}

#[test]
fn chunk_dimensions_libhdf5_refuses_are_refused() {
    skip_if_no_python!();
    let dir = tempfile::tempdir().unwrap();
    // A chunked int32 dataset (chunk 37, element size 4 after it in the
    // layout message), with libver earliest (layout v3) and latest (v4).
    // The corrupt copies set the chunk dimension to 0 (which read as all
    // fill values), to 0x80000000 (an 8 GiB chunk) and to 38, which the
    // chunk index's offsets 37 and 74 are not multiples of (libhdf5: "bad
    // coordinate offset"; the chunks were read at the wrong place). A v4
    // layout indexes chunks by position, not offset, but both libraries
    // refuse the changed chunk grid there too.
    let verdicts = h5py_verdicts(
        dir.path(),
        r#"
for libver in ("earliest", "latest"):
    good = os.path.join(d, f"{libver}_good.h5")
    with h5py.File(good, "w", libver=libver) as f:
        f.create_dataset("d", data=np.arange(100, dtype="<i4"), chunks=(37,))
    data = bytearray(open(good, "rb").read())
    if libver == "earliest":
        at = data.find(struct.pack("<II", 37, 4))
        width = 4
    else:
        # v4: flags, ndims, bytes per dim, then the dims
        at = data.find(bytes([4, 2, 0, 2]))
        width = data[at + 4]
        at += 5
    assert at > 0
    for name, value in (("zero", 0), ("huge", 0x80000000), ("offgrid", 38)):
        bad = bytearray(data)
        if value >= 1 << (8 * width):
            continue
        bad[at:at + width] = value.to_bytes(width, "little")
        open(os.path.join(d, f"{libver}_{name}.h5"), "wb").write(bad)
"#,
    );
    assert_agrees_with_h5py(
        dir.path(),
        &verdicts,
        &[
            "earliest_good ok",
            "earliest_huge ERROR",
            "earliest_offgrid ERROR",
            "earliest_zero ERROR",
            "latest_good ok",
            "latest_offgrid ERROR",
            "latest_zero ERROR",
        ],
    );
}