fix(format): a VL element at the undefined heap address is an error
libhdf5 fails to read a VL element whose global heap address is
undefined (all 0xff), even at length 0 ("addr undefined"); we returned
"" (or an empty sequence) in every reader. Checked with h5py first:
libhdf5 writes a null element with address 0, which still reads as
empty, and h5py writes "" as a zero-size heap object at a real address,
so no file they write relies on the old behaviour. read_vl_bytes now
treats address 0 as null whatever the length, as VlResolver does.
Tests, each failing before: vl_data unit test (8- and 4-byte offsets,
lengths 0 and 1); clawhdf5 vl_data_interop
a_vl_element_at_the_undefined_heap_address_fails_like_h5py (also checks
where h5py writes ""); h5rs dump --json and check --data on the patched
`undef` dataset; clawhdf5-wasm vl_strings.
Co-Authored-By: Claude Opus 5.5 (1M context) <[email protected]>
This commit is contained in:
@@ -442,3 +442,53 @@ with h5py.File(path, 'r') as f:
|
||||
assert!(one(0).is_err());
|
||||
assert_eq!(one(1).unwrap(), vec![vec![4]]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_vl_element_at_the_undefined_heap_address_fails_like_h5py() {
|
||||
// libhdf5 writes a null element with heap address 0 (h5py reads it as
|
||||
// b''), and an empty string as a real zero-size heap object; neither
|
||||
// uses the undefined address. An element of length 0 at the undefined
|
||||
// address fails in libhdf5 ("addr undefined"); we returned "".
|
||||
skip_if_no_python!();
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let path = dir.path().join("undef.h5");
|
||||
let script = format!(
|
||||
r#"
|
||||
import struct, h5py, numpy as np
|
||||
path = {path:?}
|
||||
with h5py.File(path, 'w') as f:
|
||||
f.create_dataset('d', data=np.array(['x', '', 'yz', ''], dtype=object), dtype=h5py.string_dtype())
|
||||
off = f['d'].id.get_offset()
|
||||
b = bytearray(open(path, 'rb').read())
|
||||
# h5py's '' (element 3): length 0 at a real heap address, not 0 or all 0xff.
|
||||
length, addr, _ = struct.unpack_from('<IQI', b, off + 48)
|
||||
print('empty\t%d %d' % (length, addr not in (0, 2**64 - 1)))
|
||||
struct.pack_into('<IQI', b, off + 16, 0, 2**64 - 1, 1)
|
||||
open(path, 'wb').write(bytes(b))
|
||||
with h5py.File(path, 'r') as f:
|
||||
for i in range(4):
|
||||
try:
|
||||
print('d%d\t%s' % (i, f['d'][i].hex()))
|
||||
except OSError as e:
|
||||
print('d%d\terror %s' % (i, 'addr undefined' in str(e)))
|
||||
"#,
|
||||
path = path.display().to_string()
|
||||
);
|
||||
let expected = run_python(&script);
|
||||
assert_eq!(expected["empty"], "0 1", "h5py writes '' at a real address");
|
||||
assert_eq!(expected["d0"], "78");
|
||||
assert_eq!(expected["d1"], "error True");
|
||||
assert_eq!(expected["d2"], "797a");
|
||||
assert_eq!(expected["d3"], "");
|
||||
|
||||
let file = File::open(&path).unwrap();
|
||||
let d = file.dataset("d").unwrap();
|
||||
let one = |i: u64| d.read_string_selection(&Selection::slice(&[i..i + 1]));
|
||||
assert_eq!(one(0).unwrap(), vec!["x"]);
|
||||
let e = one(1).unwrap_err().to_string();
|
||||
assert!(e.contains("undefined global heap address"), "{e}");
|
||||
assert_eq!(one(2).unwrap(), vec!["yz"]);
|
||||
assert_eq!(one(3).unwrap(), vec![""]);
|
||||
assert!(d.read_string().is_err());
|
||||
assert!(d.read_string_bytes().is_err());
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user