quantumclaw/clawhdf5
3
1
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 1 Wiki Activity

fix(format): read datasets and attributes that use committed datatypes

Browse Source 
A dataset created from a committed (named) datatype stores only a shared-
message reference to it. The facade parsed those reference bytes as the
datatype itself, producing `Time { size: 0 }` and unreadable data, and an
attribute using a committed datatype was silently dropped.

- shared_message::parse_shared_ref had the encoding wrong: it skipped six
  reserved bytes for version 2 (only version 1 has them) and had the version 3
  types inverted (1 is the SOHM heap, 2 is "committed, in another object
  header"). Verified against h5py 3.16 / HDF5 2.0, which writes
  `02 02 <address>` under both default and latest libver bounds. Resolution
  now dispatches on which field the reference carries.
- New shared_message::message_data resolves a header message through the
  indirection; the reader, lazy and mmap facades use it for datatype,
  dataspace and filter-pipeline messages.
- AttributeMessage honours the v2/v3 flags (bit 0 datatype shared, bit 1
  dataspace shared) via the new parse_in_file, used everywhere file data is
  available. Parsing a shared attribute without file access is now
  FormatError::UnresolvedSharedMessage instead of a garbage datatype.
- h5py interop test covering both libver settings.

Co-Authored-By: Claude Fable 5.1 <[email protected]>
This commit is contained in:
osobh
2026-09-19 06:31:17 -07:00
co-authored by Claude Fable 5.1
parent 0eca8574f5
commit 81e8294048
7 changed files with 374 additions and 103 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/clawhdf5-format/src/shared_message.rs
+104 -64
View File
@@ -16,8 +16,12 @@
//! - SMLI list structure: simple list of shared message entries
//! - B-tree v2 type 7: indexed shared message entries
#[cfg(not(feature = "std"))]
use alloc::borrow::Cow;
#[cfg(not(feature = "std"))]
use alloc::vec::Vec;
#[cfg(feature = "std")]
use std::borrow::Cow;
use crate::btree_v2::{BTreeV2Header, collect_btree_v2_records};
use crate::error::FormatError;
@@ -28,6 +32,14 @@ use crate::object_header::ObjectHeader;
/// Fractal heap ID length for SOHM entries (fixed at 8 bytes).
const FHEAP_ID_LEN: usize = 8;
/// Shared-message `type` values (version 3 encoding).
/// The message is in the file's shared-message (SOHM) fractal heap.
const SHARE_TYPE_SOHM: u8 = 1;
/// The message is in another object's header (a committed/named datatype).
const SHARE_TYPE_COMMITTED: u8 = 2;
/// The message is stored here but is sharable.
const SHARE_TYPE_HERE: u8 = 3;
/// A resolved shared message reference.
#[derive(Debug, Clone)]
pub struct SharedMessageRef {
@@ -35,9 +47,10 @@ pub struct SharedMessageRef {
pub ref_type: u8,
/// Version of the shared message encoding.
pub version: u8,
/// Address of the object header containing the shared message (type 1, 3).
/// Address of the object header holding the message (committed). Set for
/// every v1/v2 reference and for v3 types 2 and 3.
pub object_header_address: Option<u64>,
/// Fractal heap ID for type 2 (SOHM) references.
/// Fractal heap ID for a v3 SOHM (type 1) reference.
pub heap_id: Option<[u8; FHEAP_ID_LEN]>,
}
@@ -146,48 +159,39 @@ pub fn parse_shared_ref(data: &[u8], offset_size: u8) -> Result<SharedMessageRef
let version = data[0];
let ref_type = data[1];
// Layouts (HDF5 spec IV.A.2 "Shared Message", and libhdf5's decoder):
// v1: version, type, reserved(6), address — always "committed"
// v2: version, type, address — always "committed"
// v3: version, type, then a fractal-heap ID if type == SOHM, otherwise
// an address
// Verified against h5py/HDF5 2.0 output, which writes `02 02 <address>`
// for a dataset using a committed datatype under both default and
// `latest` libver bounds.
let address_at = |pos: usize| -> Result<SharedMessageRef, FormatError> {
ensure_len(data, pos, offset_size as usize)?;
Ok(SharedMessageRef {
ref_type,
version,
object_header_address: Some(read_offset(data, pos, offset_size)?),
heap_id: None,
})
};
match version {
1 | 2 => {
// v1/v2: reserved(6) + address(offset_size)
let pos = 2 + 6; // skip reserved bytes
ensure_len(data, pos, offset_size as usize)?;
let addr = read_offset(data, pos, offset_size)?;
1 => address_at(2 + 6),
2 => address_at(2),
3 if ref_type == SHARE_TYPE_SOHM => {
ensure_len(data, 2, FHEAP_ID_LEN)?;
let mut id = [0u8; FHEAP_ID_LEN];
id.copy_from_slice(&data[2..2 + FHEAP_ID_LEN]);
Ok(SharedMessageRef {
ref_type,
version,
object_header_address: Some(addr),
heap_id: None,
object_header_address: None,
heap_id: Some(id),
})
}
3 => {
match ref_type {
1 | 3 => {
// type 1/3: message in another object header
// v3 layout: version(1) + type(1) + address(offset_size)
ensure_len(data, 2, offset_size as usize)?;
let addr = read_offset(data, 2, offset_size)?;
Ok(SharedMessageRef {
ref_type,
version,
object_header_address: Some(addr),
heap_id: None,
})
}
2 => {
// type 2: SOHM table (fractal heap ID)
ensure_len(data, 2, FHEAP_ID_LEN)?;
let mut id = [0u8; FHEAP_ID_LEN];
id.copy_from_slice(&data[2..2 + FHEAP_ID_LEN]);
Ok(SharedMessageRef {
ref_type,
version,
object_header_address: None,
heap_id: Some(id),
})
}
_ => Err(FormatError::InvalidSharedMessageVersion(ref_type)),
}
}
3 if ref_type == SHARE_TYPE_COMMITTED || ref_type == SHARE_TYPE_HERE => address_at(2),
3 => Err(FormatError::InvalidSharedMessageVersion(ref_type)),
_ => Err(FormatError::InvalidSharedMessageVersion(version)),
}
}
@@ -422,6 +426,35 @@ pub fn resolve_sohm_message(
fh_header.read_managed_object(file_data, heap_id, offset_size)
}
/// The payload of an object-header message, following the indirection if the
/// message is *shared* (header flag bit 1).
///
/// A shared message's bytes are not the message itself but a reference to
/// where it lives — e.g. a dataset created with a committed (named) datatype
/// stores only a pointer to that datatype's object header. Every reader of a
/// message that may be shared (datatype, dataspace, fill value, filter
/// pipeline, attribute) must go through this; parsing the reference bytes as
/// the message yields garbage rather than an error.
pub fn message_data<'a>(
file_data: &[u8],
msg: &'a crate::object_header::HeaderMessage,
offset_size: u8,
length_size: u8,
) -> Result<Cow<'a, [u8]>, FormatError> {
if !is_shared(msg.flags) {
return Ok(Cow::Borrowed(&msg.data));
}
let shared_ref = parse_shared_ref(&msg.data, offset_size)?;
resolve_shared_message(
file_data,
&shared_ref,
msg.msg_type,
offset_size,
length_size,
)
.map(Cow::Owned)
}
/// Resolve a shared message to its actual message data.
///
/// For type 1/3 (shared in another object header), reads the target object header
@@ -453,14 +486,14 @@ pub fn resolve_shared_message_with_sohm(
length_size: u8,
sohm_table: Option<&SohmTable>,
) -> Result<Vec<u8>, FormatError> {
match shared_ref.ref_type {
1 | 3 => {
let addr = shared_ref
.object_header_address
.ok_or(FormatError::UnexpectedEof {
expected: 1,
available: 0,
})?;
// Dispatch on what the reference carries rather than on `ref_type`: v1/v2
// references are always an object-header address whatever their type
// byte says.
match (
shared_ref.object_header_address,
shared_ref.heap_id.as_ref(),
) {
(Some(addr), _) => {
let target_header =
ObjectHeader::parse(file_data, addr as usize, offset_size, length_size)?;
for msg in &target_header.messages {
@@ -487,11 +520,7 @@ pub fn resolve_shared_message_with_sohm(
available: 0,
})
}
2 => {
let heap_id = shared_ref
.heap_id
.as_ref()
.ok_or(FormatError::InvalidSharedMessageVersion(2))?;
(None, Some(heap_id)) => {
let table = sohm_table.ok_or(FormatError::InvalidSharedMessageVersion(2))?;
resolve_sohm_message(
file_data,
@@ -502,7 +531,7 @@ pub fn resolve_shared_message_with_sohm(
length_size,
)
}
_ => Err(FormatError::InvalidSharedMessageVersion(
(None, None) => Err(FormatError::InvalidSharedMessageVersion(
shared_ref.ref_type,
)),
}
@@ -522,15 +551,15 @@ mod tests {
}
#[test]
fn parse_v3_type1_ref() {
fn parse_v3_committed_ref() {
let mut data = Vec::new();
data.push(3); // version
data.push(1); // type 1 = shared in another OH
data.push(SHARE_TYPE_COMMITTED); // message lives in another object header
data.extend_from_slice(&0x1234u64.to_le_bytes()); // address
let shared = parse_shared_ref(&data, 8).unwrap();
assert_eq!(shared.version, 3);
assert_eq!(shared.ref_type, 1);
assert_eq!(shared.ref_type, SHARE_TYPE_COMMITTED);
assert_eq!(shared.object_header_address, Some(0x1234));
assert!(shared.heap_id.is_none());
}
@@ -539,7 +568,7 @@ mod tests {
fn parse_v3_type3_ref() {
let mut data = Vec::new();
data.push(3); // version
data.push(3); // type 3 = shared in another OH (v3 encoding)
data.push(SHARE_TYPE_HERE); // stored here but sharable: an address
data.extend_from_slice(&0xABCDu64.to_le_bytes());
let shared = parse_shared_ref(&data, 8).unwrap();
@@ -563,10 +592,10 @@ mod tests {
#[test]
fn parse_v2_ref() {
// v2 dropped v1's six reserved bytes: the address follows the type.
let mut data = Vec::new();
data.push(2); // version
data.push(0); // type
data.extend_from_slice(&[0u8; 6]); // reserved
data.push(SHARE_TYPE_COMMITTED);
data.extend_from_slice(&0x9000u32.to_le_bytes());
let shared = parse_shared_ref(&data, 4).unwrap();
@@ -575,15 +604,26 @@ mod tests {
}
#[test]
fn parse_v3_type2_sohm() {
fn parse_v2_ref_from_hdf5_2_0() {
// Datatype message of a dataset created with a committed datatype,
// as written by h5py 3.16 / HDF5 2.0 (libver='latest'): header flags
// 0x03 (shared), payload `02 02 <8-byte object header address>`.
let data = [0x02, 0x02, 0xb3, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00];
let shared = parse_shared_ref(&data, 8).unwrap();
assert_eq!(shared.object_header_address, Some(0xb3));
assert!(shared.heap_id.is_none());
}
#[test]
fn parse_v3_sohm_ref() {
let mut data = Vec::new();
data.push(3); // version
data.push(2); // type 2 = SOHM heap
data.push(SHARE_TYPE_SOHM); // message lives in the SOHM fractal heap
data.extend_from_slice(&[0xAA, 0xBB, 0xCC, 0xDD, 0x11, 0x22, 0x33, 0x44]);
let shared = parse_shared_ref(&data, 8).unwrap();
assert_eq!(shared.version, 3);
assert_eq!(shared.ref_type, 2);
assert_eq!(shared.ref_type, SHARE_TYPE_SOHM);
assert_eq!(shared.object_header_address, None);
assert_eq!(
shared.heap_id,
@@ -592,10 +632,10 @@ mod tests {
}
#[test]
fn parse_v3_type2_too_short() {
fn parse_v3_sohm_too_short() {
let mut data = Vec::new();
data.push(3); // version
data.push(2); // type 2 = SOHM heap
data.push(SHARE_TYPE_SOHM);
data.extend_from_slice(&[0xAA, 0xBB]); // only 2 bytes, need 8
let err = parse_shared_ref(&data, 8).unwrap_err();
@@ -620,7 +660,7 @@ mod tests {
fn parse_four_byte_offsets() {
let mut data = Vec::new();
data.push(3); // version
data.push(1); // type 1
data.push(SHARE_TYPE_COMMITTED);
data.extend_from_slice(&0x1000u32.to_le_bytes());
let shared = parse_shared_ref(&data, 4).unwrap();