fix(format): a VL element at the undefined heap address is an error

libhdf5 fails to read a VL element whose global heap address is
undefined (all 0xff), even at length 0 ("addr undefined"); we returned
"" (or an empty sequence) in every reader. Checked with h5py first:
libhdf5 writes a null element with address 0, which still reads as
empty, and h5py writes "" as a zero-size heap object at a real address,
so no file they write relies on the old behaviour. read_vl_bytes now
treats address 0 as null whatever the length, as VlResolver does.

Tests, each failing before: vl_data unit test (8- and 4-byte offsets,
lengths 0 and 1); clawhdf5 vl_data_interop
a_vl_element_at_the_undefined_heap_address_fails_like_h5py (also checks
where h5py writes ""); h5rs dump --json and check --data on the patched
`undef` dataset; clawhdf5-wasm vl_strings.

Co-Authored-By: Claude Opus 5.5 (1M context) <[email protected]>
This commit is contained in:
osobh
2026-09-26 09:06:46 -05:00
co-authored by Claude Opus 5.5
parent d345ffbf80
commit 5a202f3791
7 changed files with 143 additions and 27 deletions
+23 -4
View File
@@ -1,8 +1,9 @@
//! The wasm reader resolves VL strings with the library's `VlResolver`, so
//! it returns what `File::read_string` and h5py return: a string ends at
//! its first NUL, a null element is empty, a heap object of the wrong size
//! is an error, and a VL datatype whose stored element size disagrees with
//! the file's offset size is refused. Checked with 8- and 4-byte offsets.
//! is an error, an element at the undefined heap address is an error, and a
//! VL datatype whose stored element size disagrees with the file's offset
//! size is refused. Checked with 8- and 4-byte offsets.
//!
//! Skipped when python3 with h5py is missing, unless
//! `CLAWHDF5_REQUIRE_INTEROP=1`. `CLAWHDF5_PYTHON` names the interpreter.
@@ -35,7 +36,9 @@ fn h5py_available() -> bool {
/// For each offset size: `vl{8,4}.h5` with dataset `d` = "a\0b", "", null,
/// "zz" (patched: h5py writes neither a NUL nor a null element);
/// `bad{8,4}.h5` whose element 0 claims 3 bytes of a 6-byte heap object;
/// and `size{8,4}.h5` whose VL datatype message stores a 24-byte element.
/// `size{8,4}.h5` whose VL datatype message stores a 24-byte element; and
/// `undef{8,4}.h5` whose element 1 has length 0 and the undefined heap
/// address.
/// Prints h5py's reading of each element as hex, or "error".
const SCRIPT: &str = r#"
import struct, sys, h5py, numpy as np
@@ -77,7 +80,12 @@ for os_ in (8, 4):
i = b.index(pat)
struct.pack_into('<I', b, i + 4, 24)
open(p, 'wb').write(bytes(b))
for name in ('vl', 'bad', 'size'):
p = '%s/undef%d.h5' % (out, os_)
off = make(p, os_, ['x', '', 'yz'])
b = bytearray(open(p, 'rb').read())
b[off + es:off + 2 * es] = elem(0, (1 << (8 * os_)) - 1, 1, os_)
open(p, 'wb').write(bytes(b))
for name in ('vl', 'bad', 'size', 'undef'):
with h5py.File('%s/%s%d.h5' % (out, name, os_), 'r') as f:
got = []
for i in range(f['d'].shape[0]):
@@ -146,5 +154,16 @@ fn vl_strings_read_like_file_and_h5py() {
let e = wasm.unwrap_err();
assert!(e.contains("stores 24-byte elements"), "size{os}: {e}");
assert!(file.is_err(), "size{os}");
// Length 0 at the undefined heap address: libhdf5 fails the read
// ("addr undefined"); both readers returned "".
assert_eq!(h5py[&format!("undef{os}")], "78,error,797a");
let (wasm, file) = read(&format!("undef{os}"));
let e = wasm.unwrap_err();
assert!(
e.contains("undefined global heap address"),
"undef{os}: {e}"
);
assert!(file.is_err(), "undef{os}");
}
}