quantumclaw/clawhdf5
3
1
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 1 Wiki Activity

fix(format): read version-1 shared message addresses after the heap offset

Browse Source 
A version-1 shared message (HDF5 1.6) embeds the target as a symbol-table
entry: after six reserved bytes comes a length-sized local-heap offset,
then the object header address. We read the heap offset as the address,
so datasets using a committed datatype in 1.6-era files (tcompound.h5,
tcompound2.h5) failed with InvalidObjectHeaderVersion. parse_shared_ref
now takes length_size and skips the offset, as libhdf5 does.

Resolving a reference also no longer falls back to the first message of
any type in the target header: a missing target message is
SharedMessageTargetMissing instead of garbage.

Fixture: tcompound.h5 from libhdf5's tools/test/testfiles (8 KiB).

Co-Authored-By: Claude Opus 5.5 (1M context) <[email protected]>
This commit is contained in:
osobh
2026-09-25 22:07:50 -05:00
co-authored by Claude Opus 5.5
parent efc2dc53c9
commit 0555794850
7 changed files with 178 additions and 35 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/clawhdf5-format/src/attribute.rs
+3 -2
View File
@@ -97,7 +97,7 @@ impl AttributeMessage {
return Ok(Cow::Borrowed(bytes));
}
let (file_data, offset_size) = file.ok_or(FormatError::UnresolvedSharedMessage)?;
let shared_ref = shared_message::parse_shared_ref(bytes, offset_size)?;
let shared_ref = shared_message::parse_shared_ref(bytes, offset_size, length_size)?;
shared_message::resolve_shared_message(
file_data,
&shared_ref,
@@ -407,7 +407,8 @@ pub fn extract_attributes_full(
if msg.msg_type == MessageType::Attribute {
if shared_message::is_shared(msg.flags) {
// Shared attribute: resolve the reference to get actual attribute data
let shared_ref = shared_message::parse_shared_ref(&msg.data, offset_size)?;
let shared_ref =
shared_message::parse_shared_ref(&msg.data, offset_size, length_size)?;
let resolved_data = shared_message::resolve_shared_message(
file_data,
&shared_ref,
crates/clawhdf5-format/src/error.rs
+8
View File
@@ -117,6 +117,9 @@ pub enum FormatError {
/// A message is marked shared but was parsed without access to the file,
/// so the reference to the real message could not be followed.
UnresolvedSharedMessage,
/// A shared-message reference points at an object header that holds no
/// (unshared) message of the referenced type (raw message type id).
SharedMessageTargetMissing(u16),
/// A selection does not fit the dataset it was applied to (wrong rank, or
/// it reaches past a dimension's extent).
SelectionOutOfBounds(String),
@@ -339,6 +342,11 @@ impl fmt::Display for FormatError {
FormatError::SelectionOutOfBounds(msg) => {
write!(f, "selection out of bounds: {msg}")
}
FormatError::SharedMessageTargetMissing(t) => write!(
f,
"shared message reference points at an object header with no message of type \
{t:#06x}"
),
FormatError::UnresolvedSharedMessage => write!(
f,
"message is shared but no file data was available to resolve it"
crates/clawhdf5-format/src/shared_message.rs
+34 -33
View File
@@ -154,13 +154,24 @@ pub fn is_shared(msg_flags: u8) -> bool {
///
/// When the shared flag is set on a message, the data contains a reference
/// instead of the actual message content.
pub fn parse_shared_ref(data: &[u8], offset_size: u8) -> Result<SharedMessageRef, FormatError> {
///
/// `length_size` is needed for version 1 references, which embed a
/// symbol-table-entry-shaped pointer whose first field is a length-sized
/// heap offset.
pub fn parse_shared_ref(
data: &[u8],
offset_size: u8,
length_size: u8,
) -> Result<SharedMessageRef, FormatError> {
ensure_len(data, 0, 2)?;
let version = data[0];
let ref_type = data[1];
// Layouts (HDF5 spec IV.A.2 "Shared Message", and libhdf5's decoder):
// v1: version, type, reserved(6), address — always "committed"
// v1: version, type, reserved(6), then the HDF5 1.6 "symbol table
// entry" encoding of the target: a length-sized local-heap name
// offset (unused, skipped by libhdf5) followed by the object
// header address — always "committed"
// v2: version, type, address — always "committed"
// v3: version, type, then a fractal-heap ID if type == SOHM, otherwise
// an address
@@ -177,7 +188,7 @@ pub fn parse_shared_ref(data: &[u8], offset_size: u8) -> Result<SharedMessageRef
})
};
match version {
1 => address_at(2 + 6),
1 => address_at(2 + 6 + length_size as usize),
2 => address_at(2),
3 if ref_type == SHARE_TYPE_SOHM => {
ensure_len(data, 2, FHEAP_ID_LEN)?;
@@ -434,7 +445,7 @@ pub fn message_data_with_sohm<'a>(
if !is_shared(msg.flags) {
return Ok(Cow::Borrowed(&msg.data));
}
let shared_ref = parse_shared_ref(&msg.data, offset_size)?;
let shared_ref = parse_shared_ref(&msg.data, offset_size, length_size)?;
let table = if shared_ref.heap_id.is_some() {
load_sohm_table(file_data, offset_size, length_size)?
} else {
@@ -514,7 +525,7 @@ pub fn message_data<'a>(
if !is_shared(msg.flags) {
return Ok(Cow::Borrowed(&msg.data));
}
let shared_ref = parse_shared_ref(&msg.data, offset_size)?;
let shared_ref = parse_shared_ref(&msg.data, offset_size, length_size)?;
resolve_shared_message(
file_data,
&shared_ref,
@@ -571,24 +582,13 @@ pub fn resolve_shared_message_with_sohm(
return Ok(msg.data.clone());
}
}
// The message at that OH address is the message itself
// In many cases with type 1, the entire OH at that address IS the shared message
// Try returning the first message of any type that isn't Nil
for msg in &target_header.messages {
if msg.msg_type == target_msg_type {
return Ok(msg.data.clone());
}
}
// Fall back to first non-nil message
for msg in &target_header.messages {
if msg.msg_type != MessageType::Nil {
return Ok(msg.data.clone());
}
}
Err(FormatError::UnexpectedEof {
expected: 1,
available: 0,
})
// The referenced header has no (unshared) message of the wanted
// type: the reference is wrong or the file is damaged. Handing
// back some other message's bytes — or another reference's —
// would decode as garbage, so refuse.
Err(FormatError::SharedMessageTargetMissing(
target_msg_type.to_u16(),
))
}
(None, Some(heap_id)) => {
let table = sohm_table.ok_or(FormatError::InvalidSharedMessageVersion(2))?;
@@ -627,7 +627,7 @@ mod tests {
data.push(SHARE_TYPE_COMMITTED); // message lives in another object header
data.extend_from_slice(&0x1234u64.to_le_bytes()); // address
let shared = parse_shared_ref(&data, 8).unwrap();
let shared = parse_shared_ref(&data, 8, 8).unwrap();
assert_eq!(shared.version, 3);
assert_eq!(shared.ref_type, SHARE_TYPE_COMMITTED);
assert_eq!(shared.object_header_address, Some(0x1234));
@@ -641,7 +641,7 @@ mod tests {
data.push(SHARE_TYPE_HERE); // stored here but sharable: an address
data.extend_from_slice(&0xABCDu64.to_le_bytes());
let shared = parse_shared_ref(&data, 8).unwrap();
let shared = parse_shared_ref(&data, 8, 8).unwrap();
assert_eq!(shared.version, 3);
assert_eq!(shared.ref_type, 3);
assert_eq!(shared.object_header_address, Some(0xABCD));
@@ -653,9 +653,10 @@ mod tests {
data.push(1); // version
data.push(0); // type
data.extend_from_slice(&[0u8; 6]); // reserved
data.extend_from_slice(&0x10u64.to_le_bytes()); // heap name offset
data.extend_from_slice(&0x5678u64.to_le_bytes());
let shared = parse_shared_ref(&data, 8).unwrap();
let shared = parse_shared_ref(&data, 8, 8).unwrap();
assert_eq!(shared.version, 1);
assert_eq!(shared.object_header_address, Some(0x5678));
}
@@ -668,7 +669,7 @@ mod tests {
data.push(SHARE_TYPE_COMMITTED);
data.extend_from_slice(&0x9000u32.to_le_bytes());
let shared = parse_shared_ref(&data, 4).unwrap();
let shared = parse_shared_ref(&data, 4, 4).unwrap();
assert_eq!(shared.version, 2);
assert_eq!(shared.object_header_address, Some(0x9000));
}
@@ -679,7 +680,7 @@ mod tests {
// as written by h5py 3.16 / HDF5 2.0 (libver='latest'): header flags
// 0x03 (shared), payload `02 02 <8-byte object header address>`.
let data = [0x02, 0x02, 0xb3, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00];
let shared = parse_shared_ref(&data, 8).unwrap();
let shared = parse_shared_ref(&data, 8, 8).unwrap();
assert_eq!(shared.object_header_address, Some(0xb3));
assert!(shared.heap_id.is_none());
}
@@ -691,7 +692,7 @@ mod tests {
data.push(SHARE_TYPE_SOHM); // message lives in the SOHM fractal heap
data.extend_from_slice(&[0xAA, 0xBB, 0xCC, 0xDD, 0x11, 0x22, 0x33, 0x44]);
let shared = parse_shared_ref(&data, 8).unwrap();
let shared = parse_shared_ref(&data, 8, 8).unwrap();
assert_eq!(shared.version, 3);
assert_eq!(shared.ref_type, SHARE_TYPE_SOHM);
assert_eq!(shared.object_header_address, None);
@@ -708,21 +709,21 @@ mod tests {
data.push(SHARE_TYPE_SOHM);
data.extend_from_slice(&[0xAA, 0xBB]); // only 2 bytes, need 8
let err = parse_shared_ref(&data, 8).unwrap_err();
let err = parse_shared_ref(&data, 8, 8).unwrap_err();
assert!(matches!(err, FormatError::UnexpectedEof { .. }));
}
#[test]
fn invalid_version() {
let data = vec![99, 0];
let err = parse_shared_ref(&data, 8).unwrap_err();
let err = parse_shared_ref(&data, 8, 8).unwrap_err();
assert_eq!(err, FormatError::InvalidSharedMessageVersion(99));
}
#[test]
fn truncated_data() {
let data = vec![3u8]; // too short
let err = parse_shared_ref(&data, 8).unwrap_err();
let err = parse_shared_ref(&data, 8, 8).unwrap_err();
assert!(matches!(err, FormatError::UnexpectedEof { .. }));
}
@@ -733,7 +734,7 @@ mod tests {
data.push(SHARE_TYPE_COMMITTED);
data.extend_from_slice(&0x1000u32.to_le_bytes());
let shared = parse_shared_ref(&data, 4).unwrap();
let shared = parse_shared_ref(&data, 4, 4).unwrap();
assert_eq!(shared.object_header_address, Some(0x1000));
}
crates/clawhdf5-format/tests/fixtures/tcompound.h5
Vendored
BIN
View File
Binary file not shown.