fix(format): a VL element at the undefined heap address is an error
libhdf5 fails to read a VL element whose global heap address is
undefined (all 0xff), even at length 0 ("addr undefined"); we returned
"" (or an empty sequence) in every reader. Checked with h5py first:
libhdf5 writes a null element with address 0, which still reads as
empty, and h5py writes "" as a zero-size heap object at a real address,
so no file they write relies on the old behaviour. read_vl_bytes now
treats address 0 as null whatever the length, as VlResolver does.
Tests, each failing before: vl_data unit test (8- and 4-byte offsets,
lengths 0 and 1); clawhdf5 vl_data_interop
a_vl_element_at_the_undefined_heap_address_fails_like_h5py (also checks
where h5py writes ""); h5rs dump --json and check --data on the patched
`undef` dataset; clawhdf5-wasm vl_strings.
Co-Authored-By: Claude Opus 5.5 (1M context) <[email protected]>
This commit is contained in:
@@ -1,8 +1,9 @@
|
||||
//! The wasm reader resolves VL strings with the library's `VlResolver`, so
|
||||
//! it returns what `File::read_string` and h5py return: a string ends at
|
||||
//! its first NUL, a null element is empty, a heap object of the wrong size
|
||||
//! is an error, and a VL datatype whose stored element size disagrees with
|
||||
//! the file's offset size is refused. Checked with 8- and 4-byte offsets.
|
||||
//! is an error, an element at the undefined heap address is an error, and a
|
||||
//! VL datatype whose stored element size disagrees with the file's offset
|
||||
//! size is refused. Checked with 8- and 4-byte offsets.
|
||||
//!
|
||||
//! Skipped when python3 with h5py is missing, unless
|
||||
//! `CLAWHDF5_REQUIRE_INTEROP=1`. `CLAWHDF5_PYTHON` names the interpreter.
|
||||
@@ -35,7 +36,9 @@ fn h5py_available() -> bool {
|
||||
/// For each offset size: `vl{8,4}.h5` with dataset `d` = "a\0b", "", null,
|
||||
/// "zz" (patched: h5py writes neither a NUL nor a null element);
|
||||
/// `bad{8,4}.h5` whose element 0 claims 3 bytes of a 6-byte heap object;
|
||||
/// and `size{8,4}.h5` whose VL datatype message stores a 24-byte element.
|
||||
/// `size{8,4}.h5` whose VL datatype message stores a 24-byte element; and
|
||||
/// `undef{8,4}.h5` whose element 1 has length 0 and the undefined heap
|
||||
/// address.
|
||||
/// Prints h5py's reading of each element as hex, or "error".
|
||||
const SCRIPT: &str = r#"
|
||||
import struct, sys, h5py, numpy as np
|
||||
@@ -77,7 +80,12 @@ for os_ in (8, 4):
|
||||
i = b.index(pat)
|
||||
struct.pack_into('<I', b, i + 4, 24)
|
||||
open(p, 'wb').write(bytes(b))
|
||||
for name in ('vl', 'bad', 'size'):
|
||||
p = '%s/undef%d.h5' % (out, os_)
|
||||
off = make(p, os_, ['x', '', 'yz'])
|
||||
b = bytearray(open(p, 'rb').read())
|
||||
b[off + es:off + 2 * es] = elem(0, (1 << (8 * os_)) - 1, 1, os_)
|
||||
open(p, 'wb').write(bytes(b))
|
||||
for name in ('vl', 'bad', 'size', 'undef'):
|
||||
with h5py.File('%s/%s%d.h5' % (out, name, os_), 'r') as f:
|
||||
got = []
|
||||
for i in range(f['d'].shape[0]):
|
||||
@@ -146,5 +154,16 @@ fn vl_strings_read_like_file_and_h5py() {
|
||||
let e = wasm.unwrap_err();
|
||||
assert!(e.contains("stores 24-byte elements"), "size{os}: {e}");
|
||||
assert!(file.is_err(), "size{os}");
|
||||
|
||||
// Length 0 at the undefined heap address: libhdf5 fails the read
|
||||
// ("addr undefined"); both readers returned "".
|
||||
assert_eq!(h5py[&format!("undef{os}")], "78,error,797a");
|
||||
let (wasm, file) = read(&format!("undef{os}"));
|
||||
let e = wasm.unwrap_err();
|
||||
assert!(
|
||||
e.contains("undefined global heap address"),
|
||||
"undef{os}: {e}"
|
||||
);
|
||||
assert!(file.is_err(), "undef{os}");
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user