quantumclaw/clawhdf5
3
1
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 1 Wiki Activity

fix(format): read version-1 shared message addresses after the heap offset

Browse Source 
A version-1 shared message (HDF5 1.6) embeds the target as a symbol-table
entry: after six reserved bytes comes a length-sized local-heap offset,
then the object header address. We read the heap offset as the address,
so datasets using a committed datatype in 1.6-era files (tcompound.h5,
tcompound2.h5) failed with InvalidObjectHeaderVersion. parse_shared_ref
now takes length_size and skips the offset, as libhdf5 does.

Resolving a reference also no longer falls back to the first message of
any type in the target header: a missing target message is
SharedMessageTargetMissing instead of garbage.

Fixture: tcompound.h5 from libhdf5's tools/test/testfiles (8 KiB).

Co-Authored-By: Claude Opus 5.5 (1M context) <[email protected]>
This commit is contained in:
osobh
2026-09-25 22:07:50 -05:00
co-authored by Claude Opus 5.5
parent efc2dc53c9
commit 0555794850
7 changed files with 178 additions and 35 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/clawhdf5-format/src/attribute.rs
+3 -2
View File
@@ -97,7 +97,7 @@ impl AttributeMessage {
return Ok(Cow::Borrowed(bytes));
}
let (file_data, offset_size) = file.ok_or(FormatError::UnresolvedSharedMessage)?;
let shared_ref = shared_message::parse_shared_ref(bytes, offset_size)?;
let shared_ref = shared_message::parse_shared_ref(bytes, offset_size, length_size)?;
shared_message::resolve_shared_message(
file_data,
&shared_ref,
@@ -407,7 +407,8 @@ pub fn extract_attributes_full(
if msg.msg_type == MessageType::Attribute {
if shared_message::is_shared(msg.flags) {
// Shared attribute: resolve the reference to get actual attribute data
let shared_ref = shared_message::parse_shared_ref(&msg.data, offset_size)?;
let shared_ref =
shared_message::parse_shared_ref(&msg.data, offset_size, length_size)?;
let resolved_data = shared_message::resolve_shared_message(
file_data,
&shared_ref,
crates/clawhdf5-format/src/error.rs
+8
View File
@@ -117,6 +117,9 @@ pub enum FormatError {
/// A message is marked shared but was parsed without access to the file,
/// so the reference to the real message could not be followed.
UnresolvedSharedMessage,
/// A shared-message reference points at an object header that holds no
/// (unshared) message of the referenced type (raw message type id).
SharedMessageTargetMissing(u16),
/// A selection does not fit the dataset it was applied to (wrong rank, or
/// it reaches past a dimension's extent).
SelectionOutOfBounds(String),
@@ -339,6 +342,11 @@ impl fmt::Display for FormatError {
FormatError::SelectionOutOfBounds(msg) => {
write!(f, "selection out of bounds: {msg}")
}
FormatError::SharedMessageTargetMissing(t) => write!(
f,
"shared message reference points at an object header with no message of type \
{t:#06x}"
),
FormatError::UnresolvedSharedMessage => write!(
f,
"message is shared but no file data was available to resolve it"
crates/clawhdf5-format/src/shared_message.rs
+34 -33
View File
@@ -154,13 +154,24 @@ pub fn is_shared(msg_flags: u8) -> bool {
///
/// When the shared flag is set on a message, the data contains a reference
/// instead of the actual message content.
pub fn parse_shared_ref(data: &[u8], offset_size: u8) -> Result<SharedMessageRef, FormatError> {
///
/// `length_size` is needed for version 1 references, which embed a
/// symbol-table-entry-shaped pointer whose first field is a length-sized
/// heap offset.
pub fn parse_shared_ref(
data: &[u8],
offset_size: u8,
length_size: u8,
) -> Result<SharedMessageRef, FormatError> {
ensure_len(data, 0, 2)?;
let version = data[0];
let ref_type = data[1];
// Layouts (HDF5 spec IV.A.2 "Shared Message", and libhdf5's decoder):
// v1: version, type, reserved(6), address — always "committed"
// v1: version, type, reserved(6), then the HDF5 1.6 "symbol table
// entry" encoding of the target: a length-sized local-heap name
// offset (unused, skipped by libhdf5) followed by the object
// header address — always "committed"
// v2: version, type, address — always "committed"
// v3: version, type, then a fractal-heap ID if type == SOHM, otherwise
// an address
@@ -177,7 +188,7 @@ pub fn parse_shared_ref(data: &[u8], offset_size: u8) -> Result<SharedMessageRef
})
};
match version {
1 => address_at(2 + 6),
1 => address_at(2 + 6 + length_size as usize),
2 => address_at(2),
3 if ref_type == SHARE_TYPE_SOHM => {
ensure_len(data, 2, FHEAP_ID_LEN)?;
@@ -434,7 +445,7 @@ pub fn message_data_with_sohm<'a>(
if !is_shared(msg.flags) {
return Ok(Cow::Borrowed(&msg.data));
}
let shared_ref = parse_shared_ref(&msg.data, offset_size)?;
let shared_ref = parse_shared_ref(&msg.data, offset_size, length_size)?;
let table = if shared_ref.heap_id.is_some() {
load_sohm_table(file_data, offset_size, length_size)?
} else {
@@ -514,7 +525,7 @@ pub fn message_data<'a>(
if !is_shared(msg.flags) {
return Ok(Cow::Borrowed(&msg.data));
}
let shared_ref = parse_shared_ref(&msg.data, offset_size)?;
let shared_ref = parse_shared_ref(&msg.data, offset_size, length_size)?;
resolve_shared_message(
file_data,
&shared_ref,
@@ -571,24 +582,13 @@ pub fn resolve_shared_message_with_sohm(
return Ok(msg.data.clone());
}
}
// The message at that OH address is the message itself
// In many cases with type 1, the entire OH at that address IS the shared message
// Try returning the first message of any type that isn't Nil
for msg in &target_header.messages {
if msg.msg_type == target_msg_type {
return Ok(msg.data.clone());
}
}
// Fall back to first non-nil message
for msg in &target_header.messages {
if msg.msg_type != MessageType::Nil {
return Ok(msg.data.clone());
}
}
Err(FormatError::UnexpectedEof {
expected: 1,
available: 0,
})
// The referenced header has no (unshared) message of the wanted
// type: the reference is wrong or the file is damaged. Handing
// back some other message's bytes — or another reference's —
// would decode as garbage, so refuse.
Err(FormatError::SharedMessageTargetMissing(
target_msg_type.to_u16(),
))
}
(None, Some(heap_id)) => {
let table = sohm_table.ok_or(FormatError::InvalidSharedMessageVersion(2))?;
@@ -627,7 +627,7 @@ mod tests {
data.push(SHARE_TYPE_COMMITTED); // message lives in another object header
data.extend_from_slice(&0x1234u64.to_le_bytes()); // address
let shared = parse_shared_ref(&data, 8).unwrap();
let shared = parse_shared_ref(&data, 8, 8).unwrap();
assert_eq!(shared.version, 3);
assert_eq!(shared.ref_type, SHARE_TYPE_COMMITTED);
assert_eq!(shared.object_header_address, Some(0x1234));
@@ -641,7 +641,7 @@ mod tests {
data.push(SHARE_TYPE_HERE); // stored here but sharable: an address
data.extend_from_slice(&0xABCDu64.to_le_bytes());
let shared = parse_shared_ref(&data, 8).unwrap();
let shared = parse_shared_ref(&data, 8, 8).unwrap();
assert_eq!(shared.version, 3);
assert_eq!(shared.ref_type, 3);
assert_eq!(shared.object_header_address, Some(0xABCD));
@@ -653,9 +653,10 @@ mod tests {
data.push(1); // version
data.push(0); // type
data.extend_from_slice(&[0u8; 6]); // reserved
data.extend_from_slice(&0x10u64.to_le_bytes()); // heap name offset
data.extend_from_slice(&0x5678u64.to_le_bytes());
let shared = parse_shared_ref(&data, 8).unwrap();
let shared = parse_shared_ref(&data, 8, 8).unwrap();
assert_eq!(shared.version, 1);
assert_eq!(shared.object_header_address, Some(0x5678));
}
@@ -668,7 +669,7 @@ mod tests {
data.push(SHARE_TYPE_COMMITTED);
data.extend_from_slice(&0x9000u32.to_le_bytes());
let shared = parse_shared_ref(&data, 4).unwrap();
let shared = parse_shared_ref(&data, 4, 4).unwrap();
assert_eq!(shared.version, 2);
assert_eq!(shared.object_header_address, Some(0x9000));
}
@@ -679,7 +680,7 @@ mod tests {
// as written by h5py 3.16 / HDF5 2.0 (libver='latest'): header flags
// 0x03 (shared), payload `02 02 <8-byte object header address>`.
let data = [0x02, 0x02, 0xb3, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00];
let shared = parse_shared_ref(&data, 8).unwrap();
let shared = parse_shared_ref(&data, 8, 8).unwrap();
assert_eq!(shared.object_header_address, Some(0xb3));
assert!(shared.heap_id.is_none());
}
@@ -691,7 +692,7 @@ mod tests {
data.push(SHARE_TYPE_SOHM); // message lives in the SOHM fractal heap
data.extend_from_slice(&[0xAA, 0xBB, 0xCC, 0xDD, 0x11, 0x22, 0x33, 0x44]);
let shared = parse_shared_ref(&data, 8).unwrap();
let shared = parse_shared_ref(&data, 8, 8).unwrap();
assert_eq!(shared.version, 3);
assert_eq!(shared.ref_type, SHARE_TYPE_SOHM);
assert_eq!(shared.object_header_address, None);
@@ -708,21 +709,21 @@ mod tests {
data.push(SHARE_TYPE_SOHM);
data.extend_from_slice(&[0xAA, 0xBB]); // only 2 bytes, need 8
let err = parse_shared_ref(&data, 8).unwrap_err();
let err = parse_shared_ref(&data, 8, 8).unwrap_err();
assert!(matches!(err, FormatError::UnexpectedEof { .. }));
}
#[test]
fn invalid_version() {
let data = vec![99, 0];
let err = parse_shared_ref(&data, 8).unwrap_err();
let err = parse_shared_ref(&data, 8, 8).unwrap_err();
assert_eq!(err, FormatError::InvalidSharedMessageVersion(99));
}
#[test]
fn truncated_data() {
let data = vec![3u8]; // too short
let err = parse_shared_ref(&data, 8).unwrap_err();
let err = parse_shared_ref(&data, 8, 8).unwrap_err();
assert!(matches!(err, FormatError::UnexpectedEof { .. }));
}
@@ -733,7 +734,7 @@ mod tests {
data.push(SHARE_TYPE_COMMITTED);
data.extend_from_slice(&0x1000u32.to_le_bytes());
let shared = parse_shared_ref(&data, 4).unwrap();
let shared = parse_shared_ref(&data, 4, 4).unwrap();
assert_eq!(shared.object_header_address, Some(0x1000));
}
crates/clawhdf5-format/tests/fixtures/tcompound.h5
Vendored
BIN
View File
Binary file not shown.
crates/clawhdf5/tests/shared_message_v1.rs
+121
View File
@@ -0,0 +1,121 @@
//! Version-1 shared messages (HDF5 1.6 era). A dataset that uses a committed
//! datatype stores a *shared* datatype message pointing at the type's object
//! header. In version 1 that pointer is a 1.6 "symbol table entry": after six
//! reserved bytes comes a length-sized heap offset, *then* the address.
//!
//! Fixture: `tcompound.h5` from libhdf5's own tool tests
//! (`tools/test/testfiles/tcompound.h5`, HDF5 source tree, BSD-style
//! licence), 8 KiB. Its datasets use committed compound types through v1
//! shared messages. The expected types are what h5dump 1.14.6 and h5py 3.16
//! (HDF5 2.0) report; the h5py cross-check runs when python3 with h5py is
//! available (required with `CLAWHDF5_REQUIRE_INTEROP=1`).
use std::process::Command;
use clawhdf5::{DType, File};
const FIXTURE: &[u8] = include_bytes!("../../clawhdf5-format/tests/fixtures/tcompound.h5");
fn python() -> String {
std::env::var("CLAWHDF5_PYTHON").unwrap_or_else(|_| "python3".to_string())
}
fn interop_required() -> bool {
std::env::var("CLAWHDF5_REQUIRE_INTEROP").is_ok_and(|v| v == "1")
}
fn python_available() -> bool {
Command::new(python())
.args(["-c", "import h5py, numpy"])
.output()
.map(|o| o.status.success())
.unwrap_or(false)
}
fn compound(fields: &[(&str, DType)]) -> DType {
DType::Compound(
fields
.iter()
.map(|(n, t)| (n.to_string(), t.clone()))
.collect(),
)
}
fn expected() -> Vec<(&'static str, DType)> {
let int_float = |i: &str, f: &str| compound(&[(i, DType::I32), (f, DType::F32)]);
vec![
("group1/dset2", int_float("int_name", "float_name")),
(
"group1/dset3",
compound(&[
("int_array", DType::Array(Box::new(DType::I32), vec![4])),
(
"float_array",
DType::Array(Box::new(DType::F32), vec![5, 6]),
),
]),
),
("group1/dset4", int_float("int", "float")),
("group2/dset5", int_float("int", "float")),
]
}
#[test]
fn v1_shared_datatype_resolves_to_the_committed_type() {
// Reading the heap offset as the address used to land on the superblock
// and fail with InvalidObjectHeaderVersion.
let file = File::from_bytes(FIXTURE.to_vec()).unwrap();
for (path, dtype) in expected() {
assert_eq!(
file.dataset(path).unwrap().dtype().unwrap(),
dtype,
"{path}"
);
}
}
#[test]
fn v1_shared_datatype_field_names_match_h5py() {
if !python_available() {
assert!(
!interop_required(),
"CLAWHDF5_REQUIRE_INTEROP=1 but python3 with h5py is not available"
);
eprintln!("SKIP: python3 with h5py not available");
return;
}
let dir = tempfile::tempdir().unwrap();
let path = dir.path().join("tcompound.h5");
std::fs::write(&path, FIXTURE).unwrap();
let script = format!(
r#"
import h5py
with h5py.File("{path}", "r") as f:
for p in ("group1/dset2", "group1/dset3", "group1/dset4", "group2/dset5"):
print(p, *f[p].dtype.names)
"#,
path = path.display()
);
let out = Command::new(python())
.args(["-c", &script])
.output()
.unwrap();
assert!(
out.status.success(),
"{}",
String::from_utf8_lossy(&out.stderr)
);
let stdout = String::from_utf8_lossy(&out.stdout);
let theirs: Vec<&str> = stdout.lines().collect();
let ours: Vec<String> = expected()
.into_iter()
.map(|(p, t)| match t {
DType::Compound(fields) => {
let names: Vec<String> = fields.into_iter().map(|(n, _)| n).collect();
format!("{p} {}", names.join(" "))
}
other => panic!("{other:?}"),
})
.collect();
assert_eq!(theirs, ours);
}