fix(tools,wasm): resolve VL data through the library's VlResolver

h5rs (dump, ls, diff, check --data) kept its own lenient VL decoder: a heap object longer than its element was cut to the element's length (libhdf5 and h5py refuse it), a null string printed "" where h5dump prints NULL, the stored element size was trusted, and every heap collection was kept as an owned copy for the whole run. It now resolves each element with VlResolver::element / string_element (new: one element in place, borrowing from the file), and refuses a VL type whose stored element size is not 4 + offset size + 4, as File does. H5::heap_object and its cache are gone. h5diff compares a null VL string equal to an empty one; so does h5rs diff. clawhdf5-wasm already resolved VL strings with read_vl_strings; it now uses VlResolver and checks the stored element size before reading, as File::read_string does. Tests (h5py writes the files, patched for "a\0b", a null element and mis-sized heap objects, with 8- and 4-byte offsets): - h5rs_interop dump_prints_vl_data_like_h5dump: byte-identical to h5dump; - dump_json_vl_values_match_h5py: h5py's values, errors where h5py fails; - check_data_flags_mis_sized_vl_heap_objects; - clawhdf5-wasm tests/vl_strings.rs: wasm, File and h5py agree. All four fail before. check --data over the 150 cve_hdf5 CVE and fuzzer files now passes 15 (h5dump rejects 8 of them), was 16 and 9: the stored-size check flags cve-2024-32608. h5rs-check-ok-files.sh --data: 0 of 422 flagged; h5rs-fuzz.sh: clean on 180 files. Co-Authored-By: Claude Opus 5.5 (1M context) <[email protected]>
2026-09-26 09:04:21 -05:00
parent 41b7837d0a
commit d345ffbf80
11 changed files with 533 additions and 152 deletions
@@ -0,0 +1,150 @@
+//! The wasm reader resolves VL strings with the library's `VlResolver`, so
+//! it returns what `File::read_string` and h5py return: a string ends at
+//! its first NUL, a null element is empty, a heap object of the wrong size
+//! is an error, and a VL datatype whose stored element size disagrees with
+//! the file's offset size is refused. Checked with 8- and 4-byte offsets.
+//!
+//! Skipped when python3 with h5py is missing, unless
+//! `CLAWHDF5_REQUIRE_INTEROP=1`. `CLAWHDF5_PYTHON` names the interpreter.
+
+use std::process::Command;
+
+use clawhdf5::File;
+use clawhdf5_wasm::core::{Data, Reader};
+
+fn python() -> String {
+    std::env::var("CLAWHDF5_PYTHON").unwrap_or_else(|_| "python3".to_string())
+}
+
+fn h5py_available() -> bool {
+    let ok = Command::new(python())
+        .args(["-c", "import h5py, numpy"])
+        .output()
+        .map(|o| o.status.success())
+        .unwrap_or(false);
+    if !ok {
+        assert!(
+            std::env::var("CLAWHDF5_REQUIRE_INTEROP").as_deref() != Ok("1"),
+            "CLAWHDF5_REQUIRE_INTEROP=1 but python with h5py is not available"
+        );
+        eprintln!("SKIP: python with h5py not available");
+    }
+    ok
+}
+
+/// For each offset size: `vl{8,4}.h5` with dataset `d` = "a\0b", "", null,
+/// "zz" (patched: h5py writes neither a NUL nor a null element);
+/// `bad{8,4}.h5` whose element 0 claims 3 bytes of a 6-byte heap object;
+/// and `size{8,4}.h5` whose VL datatype message stores a 24-byte element.
+/// Prints h5py's reading of each element as hex, or "error".
+const SCRIPT: &str = r#"
+import struct, sys, h5py, numpy as np
+out = sys.argv[1]
+S = h5py.string_dtype('utf-8')
+def create(path, os_):
+    if os_ == 8:
+        return h5py.File(path, 'w', libver='earliest')
+    # The earliest format, so the patched object header has no checksum.
+    fcpl = h5py.h5p.create(h5py.h5p.FILE_CREATE); fcpl.set_sizes(4, 4)
+    fapl = h5py.h5p.create(h5py.h5p.FILE_ACCESS)
+    fapl.set_libver_bounds(h5py.h5f.LIBVER_EARLIEST, h5py.h5f.LIBVER_V18)
+    return h5py.File(h5py.h5f.create(path.encode(), h5py.h5f.ACC_TRUNC, fcpl=fcpl, fapl=fapl))
+def elem(length, addr, index, os_):
+    return struct.pack('<I', length) + addr.to_bytes(os_, 'little') + struct.pack('<I', index)
+def make(path, os_, values):
+    with create(path, os_) as f:
+        f.create_dataset('d', data=np.array(values, dtype=object), dtype=S)
+        return f['d'].id.get_offset()
+for os_ in (8, 4):
+    es = 8 + os_
+    p = '%s/vl%d.h5' % (out, os_)
+    off = make(p, os_, ['aXb', '', 'ok', 'zz'])
+    b = bytearray(open(p, 'rb').read())
+    b[b.index(b'aXb') + 1] = 0
+    b[off + 2 * es:off + 3 * es] = elem(2, 0, 1, os_)
+    open(p, 'wb').write(bytes(b))
+    p = '%s/bad%d.h5' % (out, os_)
+    off = make(p, os_, ['cdefgh', 'ok'])
+    b = bytearray(open(p, 'rb').read())
+    struct.pack_into('<I', b, off, 3)
+    open(p, 'wb').write(bytes(b))
+    p = '%s/size%d.h5' % (out, os_)
+    make(p, os_, ['x', 'yy'])
+    b = bytearray(open(p, 'rb').read())
+    # datatype message: version 1, class 9 (VL); string, null-terminated, UTF-8
+    pat = bytes([0x19, 0x01, 0x01, 0x00]) + struct.pack('<I', es)
+    assert b.count(pat) == 1, b.count(pat)
+    i = b.index(pat)
+    struct.pack_into('<I', b, i + 4, 24)
+    open(p, 'wb').write(bytes(b))
+    for name in ('vl', 'bad', 'size'):
+        with h5py.File('%s/%s%d.h5' % (out, name, os_), 'r') as f:
+            got = []
+            for i in range(f['d'].shape[0]):
+                try:
+                    got.append(f['d'][i].hex())
+                except OSError:
+                    got.append('error')
+            print('%s%d\t%s' % (name, os_, ','.join(got)))
+"#;
+
+#[test]
+fn vl_strings_read_like_file_and_h5py() {
+    if !h5py_available() {
+        return;
+    }
+    let dir = tempfile::tempdir().unwrap();
+    let out = Command::new(python())
+        .args(["-c", SCRIPT])
+        .arg(dir.path())
+        .output()
+        .expect("run python");
+    assert!(
+        out.status.success(),
+        "{}",
+        String::from_utf8_lossy(&out.stderr)
+    );
+    let h5py: std::collections::HashMap<String, String> = String::from_utf8(out.stdout)
+        .unwrap()
+        .lines()
+        .map(|l| {
+            let (k, v) = l.split_once('\t').unwrap();
+            (k.to_string(), v.to_string())
+        })
+        .collect();
+    let read = |name: &str| {
+        let bytes = std::fs::read(dir.path().join(format!("{name}.h5"))).unwrap();
+        let wasm = Reader::open(bytes).unwrap().read("/d", None);
+        let file = File::open(dir.path().join(format!("{name}.h5")))
+            .unwrap()
+            .dataset("d")
+            .unwrap()
+            .read_string();
+        (wasm, file)
+    };
+    for os in [8, 4] {
+        // h5py: "a\0b" is "a"; the null element (address 0) is empty.
+        assert_eq!(h5py[&format!("vl{os}")], "61,,,7a7a");
+        let (wasm, file) = read(&format!("vl{os}"));
+        let Data::Strings(wasm) = wasm.unwrap().data else {
+            panic!("vl{os}: not strings")
+        };
+        assert_eq!(wasm, ["a", "", "", "zz"], "vl{os}");
+        assert_eq!(wasm, file.unwrap(), "vl{os}");
+
+        // h5py refuses the mis-sized element; so do both readers.
+        assert_eq!(h5py[&format!("bad{os}")], "error,6f6b");
+        let (wasm, file) = read(&format!("bad{os}"));
+        assert!(wasm.unwrap_err().contains("holds 6 bytes"), "bad{os}");
+        assert!(file.is_err(), "bad{os}");
+
+        // libhdf5 ignores the stored element size and reads the values;
+        // File refuses the datatype rather than guess its layout, and the
+        // wasm reader now does the same (it read with the stored size).
+        assert_eq!(h5py[&format!("size{os}")], "78,7979");
+        let (wasm, file) = read(&format!("size{os}"));
+        let e = wasm.unwrap_err();
+        assert!(e.contains("stores 24-byte elements"), "size{os}: {e}");
+        assert!(file.is_err(), "size{os}");
+    }
+}