quantumclaw/clawhdf5
3
1
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 1 Wiki Activity

fix(format): a VL element at the undefined heap address is an error

Browse Source 
libhdf5 fails to read a VL element whose global heap address is
undefined (all 0xff), even at length 0 ("addr undefined"); we returned
"" (or an empty sequence) in every reader. Checked with h5py first:
libhdf5 writes a null element with address 0, which still reads as
empty, and h5py writes "" as a zero-size heap object at a real address,
so no file they write relies on the old behaviour. read_vl_bytes now
treats address 0 as null whatever the length, as VlResolver does.

Tests, each failing before: vl_data unit test (8- and 4-byte offsets,
lengths 0 and 1); clawhdf5 vl_data_interop
a_vl_element_at_the_undefined_heap_address_fails_like_h5py (also checks
where h5py writes ""); h5rs dump --json and check --data on the patched
`undef` dataset; clawhdf5-wasm vl_strings.

Co-Authored-By: Claude Opus 5.5 (1M context) <[email protected]>
This commit is contained in:
osobh
2026-09-26 09:06:46 -05:00
co-authored by Claude Opus 5.5
parent d345ffbf80
commit 5a202f3791
7 changed files with 143 additions and 27 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/clawhdf5-format/src/vl_data.rs
+32 -17
View File
@@ -239,9 +239,6 @@ impl<'a> VlResolver<'a> {
if addr == 0 {
return Ok(None);
}
if vl.length == 0 && is_undefined_address(addr, self.offset_size) {
return Ok(Some(&[]));
}
let data = self.object(vl)?;
let expected = (vl.length as usize)
.checked_mul(base_size)
@@ -372,10 +369,8 @@ pub fn read_vl_bytes(
let mut result = Vec::with_capacity(refs.len());
for vl in &refs {
if vl.length == 0
&& (is_undefined_address(vl.collection_address, offset_size)
|| vl.collection_address == 0)
{
// A heap address of 0 is a null element, as in VlResolver.
if vl.collection_address == 0 {
result.push(Vec::new());
continue;
}
@@ -394,6 +389,15 @@ impl<'a> VlResolver<'a> {
/// parsed on first use.
fn object(&mut self, vl: &VlElement) -> Result<&'a [u8], FormatError> {
let addr = vl.collection_address;
// libhdf5 writes a null element with address 0, never the undefined
// address, and fails to read one ("addr undefined") even when its
// length is 0; we returned an empty value.
if is_undefined_address(addr, self.offset_size) {
return Err(FormatError::VlDataError(format!(
"variable-length element (length {}) has the undefined global heap address",
vl.length
)));
}
if !self.cache.contains_key(&addr) {
let offset = usize::try_from(addr).map_err(|_| FormatError::UnexpectedEof {
expected: usize::MAX,
@@ -546,16 +550,27 @@ mod tests {
}
#[test]
fn null_vl_element_empty_string() {
// length=0, address=undefined
let mut raw = Vec::new();
raw.extend_from_slice(&0u32.to_le_bytes()); // length=0
raw.extend_from_slice(&u64::MAX.to_le_bytes()); // undefined address
raw.extend_from_slice(&0u32.to_le_bytes()); // index
let file_data = vec![0u8; 16];
let strings = read_vl_strings(&file_data, &raw, 1, 8, 8).unwrap();
assert_eq!(strings, vec![""]);
fn an_undefined_heap_address_is_an_error_even_at_length_0() {
// libhdf5 fails the read ("addr undefined"); h5py and libhdf5 write
// a null element with address 0. We returned "".
let mut file_data = vec![0u8; 256];
build_gcol_at(&mut file_data, 64, &[(1, b"x")]);
for (os, undef) in [(8u8, u64::MAX), (4, 0xFFFF_FFFF)] {
for length in [0, 1] {
let mut raw = element(1, 64, 1, os);
raw.extend(element(length, undef, 1, os));
let mut r = VlResolver::new(&file_data, os, 8);
let e = r.string_bytes(&raw).unwrap_err().to_string();
assert!(e.contains("undefined"), "{e}");
assert!(r.sequences(&raw, 1).is_err());
assert!(r.string_element(&raw[raw.len() / 2..]).is_err());
let n = 2;
assert!(read_vl_strings(&file_data, &raw, n, os, 8).is_err());
assert!(read_vl_bytes(&file_data, &raw, n, os, 8).is_err());
// The defined element alone still reads.
assert_eq!(r.strings(&raw[..raw.len() / 2]).unwrap(), ["x"]);
}
}
}
#[test]