fix(format): a VL element at the undefined heap address is an error

libhdf5 fails to read a VL element whose global heap address is
undefined (all 0xff), even at length 0 ("addr undefined"); we returned
"" (or an empty sequence) in every reader. Checked with h5py first:
libhdf5 writes a null element with address 0, which still reads as
empty, and h5py writes "" as a zero-size heap object at a real address,
so no file they write relies on the old behaviour. read_vl_bytes now
treats address 0 as null whatever the length, as VlResolver does.

Tests, each failing before: vl_data unit test (8- and 4-byte offsets,
lengths 0 and 1); clawhdf5 vl_data_interop
a_vl_element_at_the_undefined_heap_address_fails_like_h5py (also checks
where h5py writes ""); h5rs dump --json and check --data on the patched
`undef` dataset; clawhdf5-wasm vl_strings.

Co-Authored-By: Claude Opus 5.5 (1M context) <[email protected]>
This commit is contained in:
osobh
2026-09-26 09:06:46 -05:00
co-authored by Claude Opus 5.5
parent d345ffbf80
commit 5a202f3791
7 changed files with 143 additions and 27 deletions
+50
View File
@@ -442,3 +442,53 @@ with h5py.File(path, 'r') as f:
assert!(one(0).is_err());
assert_eq!(one(1).unwrap(), vec![vec![4]]);
}
#[test]
fn a_vl_element_at_the_undefined_heap_address_fails_like_h5py() {
// libhdf5 writes a null element with heap address 0 (h5py reads it as
// b''), and an empty string as a real zero-size heap object; neither
// uses the undefined address. An element of length 0 at the undefined
// address fails in libhdf5 ("addr undefined"); we returned "".
skip_if_no_python!();
let dir = tempfile::tempdir().unwrap();
let path = dir.path().join("undef.h5");
let script = format!(
r#"
import struct, h5py, numpy as np
path = {path:?}
with h5py.File(path, 'w') as f:
f.create_dataset('d', data=np.array(['x', '', 'yz', ''], dtype=object), dtype=h5py.string_dtype())
off = f['d'].id.get_offset()
b = bytearray(open(path, 'rb').read())
# h5py's '' (element 3): length 0 at a real heap address, not 0 or all 0xff.
length, addr, _ = struct.unpack_from('<IQI', b, off + 48)
print('empty\t%d %d' % (length, addr not in (0, 2**64 - 1)))
struct.pack_into('<IQI', b, off + 16, 0, 2**64 - 1, 1)
open(path, 'wb').write(bytes(b))
with h5py.File(path, 'r') as f:
for i in range(4):
try:
print('d%d\t%s' % (i, f['d'][i].hex()))
except OSError as e:
print('d%d\terror %s' % (i, 'addr undefined' in str(e)))
"#,
path = path.display().to_string()
);
let expected = run_python(&script);
assert_eq!(expected["empty"], "0 1", "h5py writes '' at a real address");
assert_eq!(expected["d0"], "78");
assert_eq!(expected["d1"], "error True");
assert_eq!(expected["d2"], "797a");
assert_eq!(expected["d3"], "");
let file = File::open(&path).unwrap();
let d = file.dataset("d").unwrap();
let one = |i: u64| d.read_string_selection(&Selection::slice(&[i..i + 1]));
assert_eq!(one(0).unwrap(), vec!["x"]);
let e = one(1).unwrap_err().to_string();
assert!(e.contains("undefined global heap address"), "{e}");
assert_eq!(one(2).unwrap(), vec!["yz"]);
assert_eq!(one(3).unwrap(), vec![""]);
assert!(d.read_string().is_err());
assert!(d.read_string_bytes().is_err());
}