quantumclaw/clawhdf5
3
1
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 1 Wiki Activity

fix(format): measure compound members by their stored size

Browse Source 
The compound overlap check measured each earlier member with
Datatype::type_size, which is a fixed 16 for a variable-length type. On
disk a VL member takes 4 + offset size + 4 bytes, 12 in a file with
4-byte offsets, so a member right after one was refused as "member
overlaps with previous member" (and with the type, every attribute of
the object). libhdf5 measures members by their decoded, stored size
(times a v1 member's array dimensions); so does this now.

Reading VL values in such files is a separate, older gap, now recorded
in known-issues.

Co-Authored-By: Claude Opus 5.5 (1M context) <[email protected]>
This commit is contained in:
osobh
2026-09-26 01:24:17 -05:00
co-authored by Claude Opus 5.5
parent 17f09375ad
commit 9238605661
4 changed files with 78 additions and 10 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/clawhdf5-format/src/datatype.rs
+18 -10
View File
@@ -215,11 +215,6 @@ fn stored_type_size(data: &[u8], pos: usize) -> Result<u32, FormatError> {
Ok(LittleEndian::read_u32(&data[pos + 4..pos + 8]))
}
/// A compound member's size in the compound, as libhdf5 counts it.
fn member_size(dt: &Datatype) -> u64 {
u64::from(dt.type_size())
}
/// libhdf5 refuses an array type of more than `H5S_MAX_RANK` (32)
/// dimensions.
fn check_array_rank(ndims: usize) -> Result<(), FormatError> {
@@ -543,11 +538,17 @@ impl Datatype {
return Err(invalid("invalid number of members: 0"));
}
let mut members: Vec<CompoundMember> = Vec::with_capacity(num_members as usize);
// Each member's size in the compound as libhdf5 decodes it:
// its stored size, times a v1 member's array dimensions. A
// variable-length member takes 4 + offset size + 4 bytes on
// disk, not the 16 of `Datatype::type_size`.
let mut member_sizes: Vec<u64> = Vec::with_capacity(num_members as usize);
// libhdf5 checks each member as it is decoded: it must fit in
// the compound (by its own stored size, before a v1 member's
// array dimensions are applied), and must not repeat a name
// or overlap an earlier member (by its final size).
let check_member = |members: &[CompoundMember],
member_sizes: &[u64],
name: &str,
byte_offset: u64,
stored_size: u32,
@@ -565,9 +566,8 @@ impl Datatype {
)));
}
let end = byte_offset + final_size;
if members.iter().any(|m| {
let m_end = m.byte_offset + member_size(&m.datatype);
byte_offset < m_end && m.byte_offset < end
if members.iter().zip(member_sizes).any(|(m, &m_size)| {
byte_offset < m.byte_offset + m_size && m.byte_offset < end
}) {
return Err(invalid("member overlaps with previous member"));
}
@@ -588,13 +588,16 @@ impl Datatype {
let (member_dt, consumed) =
Self::parse_with_depth(&data[pos..], depth + 1)?;
pos += consumed;
let final_size = u64::from(stored_size);
check_member(
&members,
&member_sizes,
&name,
byte_offset,
stored_size,
member_size(&member_dt),
final_size,
)?;
member_sizes.push(final_size);
members.push(CompoundMember {
name,
byte_offset,
@@ -652,6 +655,9 @@ impl Datatype {
let (mut member_dt, consumed) =
Self::parse_with_depth(&data[pos..], depth + 1)?;
pos += consumed;
let final_size = array_dims.iter().fold(u64::from(stored_size), |a, &d| {
a.saturating_mul(u64::from(d))
});
if !array_dims.is_empty() {
member_dt = Datatype::Array {
base_type: Box::new(member_dt),
@@ -660,11 +666,13 @@ impl Datatype {
}
check_member(
&members,
&member_sizes,
&name,
byte_offset,
stored_size,
member_size(&member_dt),
final_size,
)?;
member_sizes.push(final_size);
members.push(CompoundMember {
name,
byte_offset,