quantumclaw/clawhdf5
3
1
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 1 Wiki Activity

fix(format): read VL values in files with 4-byte offsets

Browse Source 
In a file with sizeof_addr = 4, a VL string attribute came back as
AttrValue::Raw, a compound's VL member failed with
GlobalHeapObjectNotFound and VL datasets failed with a size mismatch.

Two bugs: Datatype::type_size() said 16 for every VL type, while the
element is 4 + offset size + 4 bytes (12 here); and the global heap was
parsed without the padding libhdf5 puts after its collection and object
headers (both round up to 8), so with 4-byte lengths every object was
looked up 4 bytes early. Datatype::VariableLength now carries the size
its datatype message stores, and writes it back.

Checked against h5py in tests/vl_offset4_interop.rs (fails with either
fix reverted). Conformance unchanged at 575 of 697; in cve-2024-32608 a
VL attribute whose datatype claims 524304-byte elements is now an error
(h5py cannot iterate those attributes at all).

Co-Authored-By: Claude Opus 5.5 (1M context) <[email protected]>
This commit is contained in:
osobh
2026-09-26 08:19:13 -05:00
co-authored by Claude Opus 5.5
parent 63648c7000
commit 10da8f0d09
5 changed files with 184 additions and 9 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/clawhdf5-format/src/datatype.rs
+26 -2
View File
@@ -125,6 +125,11 @@ pub enum Datatype {
},
/// Class 9: Variable-length type.
VariableLength {
/// Size of one element as stored in the file: a sequence length (4
/// bytes), a global heap collection address (the file's
/// `offset_size`) and an object index (4 bytes) — 16 in a file with
/// 8-byte offsets, 12 with 4-byte offsets.
size: u32,
is_string: bool,
padding: Option<StringPadding>,
charset: Option<CharacterSet>,
@@ -771,6 +776,7 @@ impl Datatype {
pos += consumed;
Ok((
Datatype::VariableLength {
size,
is_string,
padding,
charset,
@@ -1017,6 +1023,7 @@ impl Datatype {
Self::build_header(3, 1, [bf0, 0, 0], *size)
}
Datatype::VariableLength {
size,
is_string,
padding,
charset,
@@ -1039,7 +1046,7 @@ impl Datatype {
} else {
0
};
let mut buf = Self::build_header(9, 1, [bf0, bf1, 0], 16);
let mut buf = Self::build_header(9, 1, [bf0, bf1, 0], *size);
buf.extend_from_slice(&base_type.serialize());
buf
}
@@ -1208,7 +1215,7 @@ impl Datatype {
Datatype::Compound { size, .. } => *size,
Datatype::Reference { size, .. } => *size,
Datatype::Enumeration { size, .. } => *size,
Datatype::VariableLength { .. } => 16, // typically pointer + length
Datatype::VariableLength { size, .. } => *size,
Datatype::Array {
base_type,
dimensions,
@@ -1889,11 +1896,13 @@ mod tests {
let (dt, _) = Datatype::parse(&buf).unwrap();
match dt {
Datatype::VariableLength {
size,
is_string,
padding,
charset,
base_type,
} => {
assert_eq!(size, 16);
assert!(is_string);
assert_eq!(padding, Some(StringPadding::NullTerminate));
assert_eq!(charset, Some(CharacterSet::Utf8));
@@ -1914,11 +1923,13 @@ mod tests {
let (dt, _) = Datatype::parse(&buf).unwrap();
match dt {
Datatype::VariableLength {
size,
is_string,
padding,
charset,
base_type,
} => {
assert_eq!(size, 16);
assert!(!is_string);
assert_eq!(padding, None);
assert_eq!(charset, None);
@@ -1928,6 +1939,19 @@ mod tests {
}
}
#[test]
fn variable_length_size_is_the_stored_size() {
// A file with 4-byte offsets stores 12-byte VL elements (length 4 +
// address 4 + index 4); the type used to report 16 regardless, so
// every read laid the elements out 16 bytes apart.
let mut buf = build_dt_header(9, 1, [0x01, 0x00, 0], 12);
buf.extend_from_slice(&build_fixed_point(1, false, false, 0, 8));
let (dt, _) = Datatype::parse(&buf).unwrap();
assert_eq!(dt.type_size(), 12);
// And it is written back as stored.
assert_eq!(dt.serialize()[4..8], 12u32.to_le_bytes());
}
#[test]
fn test_array_2d() {
// Array [3][4] of i32 LE, version 3
crates/clawhdf5-format/src/global_heap.rs
+13 -6
View File
@@ -64,8 +64,11 @@ impl GlobalHeapCollection {
offset: usize,
length_size: u8,
) -> Result<GlobalHeapCollection, FormatError> {
// signature(4) + version(1) + reserved(3) + collection_size(length_size)
let header_size = 8 + length_size as usize;
// signature(4) + version(1) + reserved(3) + collection_size(length_size),
// padded to a multiple of 8 as libhdf5 lays it out (`H5HG_SIZEOF_HDR`).
// With 8-byte lengths the padding is 0; with 4-byte lengths it is 4,
// and reading without it put every object 4 bytes early.
let header_size = pad8(8 + length_size as usize);
ensure_len(file_data, offset, header_size)?;
if file_data[offset..offset + 4] != GCOL_SIGNATURE {
@@ -104,8 +107,9 @@ impl GlobalHeapCollection {
break;
}
// object_index(2) + reference_count(2) + reserved(4) + object_size(length_size)
let obj_header_size = 8 + length_size as usize;
// object_index(2) + reference_count(2) + reserved(4) +
// object_size(length_size), padded to 8 (`H5HG_SIZEOF_OBJHDR`).
let obj_header_size = pad8(8 + length_size as usize);
ensure_len(file_data, pos, obj_header_size)?;
let reference_count = u16::from_le_bytes([file_data[pos + 2], file_data[pos + 3]]);
@@ -149,10 +153,11 @@ mod tests {
let ls = length_size as usize;
// Calculate total size
let header_size = 8 + ls;
// libhdf5 pads both headers to a multiple of 8.
let header_size = pad8(8 + ls);
let mut obj_size_total = 0usize;
for (_, _, data) in objects {
let obj_header = 8 + ls;
let obj_header = pad8(8 + ls);
obj_size_total += obj_header + pad8(data.len());
}
// Free space marker (2 bytes for index 0)
@@ -170,6 +175,7 @@ mod tests {
8 => buf.extend_from_slice(&(collection_size as u64).to_le_bytes()),
_ => panic!("unsupported length_size"),
}
buf.resize(header_size, 0);
// Objects
for (index, ref_count, data) in objects {
@@ -181,6 +187,7 @@ mod tests {
8 => buf.extend_from_slice(&(data.len() as u64).to_le_bytes()),
_ => panic!("unsupported"),
}
buf.resize(buf.len() + (pad8(8 + ls) - (8 + ls)), 0);
buf.extend_from_slice(data);
// Pad to 8 bytes
let padded = pad8(data.len());