quantumclaw/clawhdf5
3
1
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 1 Wiki Activity

fix(format): read datasets and attributes that use committed datatypes

Browse Source 
A dataset created from a committed (named) datatype stores only a shared-
message reference to it. The facade parsed those reference bytes as the
datatype itself, producing `Time { size: 0 }` and unreadable data, and an
attribute using a committed datatype was silently dropped.

- shared_message::parse_shared_ref had the encoding wrong: it skipped six
  reserved bytes for version 2 (only version 1 has them) and had the version 3
  types inverted (1 is the SOHM heap, 2 is "committed, in another object
  header"). Verified against h5py 3.16 / HDF5 2.0, which writes
  `02 02 <address>` under both default and latest libver bounds. Resolution
  now dispatches on which field the reference carries.
- New shared_message::message_data resolves a header message through the
  indirection; the reader, lazy and mmap facades use it for datatype,
  dataspace and filter-pipeline messages.
- AttributeMessage honours the v2/v3 flags (bit 0 datatype shared, bit 1
  dataspace shared) via the new parse_in_file, used everywhere file data is
  available. Parsing a shared attribute without file access is now
  FormatError::UnresolvedSharedMessage instead of a garbage datatype.
- h5py interop test covering both libver settings.

Co-Authored-By: Claude Fable 5.1 <[email protected]>
This commit is contained in:
osobh
2026-09-19 06:31:17 -07:00
co-authored by Claude Fable 5.1
parent 0eca8574f5
commit 81e8294048
7 changed files with 374 additions and 103 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/clawhdf5-format/src/attribute.rs
+112 -12
View File
@@ -1,7 +1,9 @@
//! HDF5 Attribute message parsing (message type 0x000C).
#[cfg(not(feature = "std"))]
use alloc::{string::String, vec::Vec};
use alloc::{borrow::Cow, string::String, vec::Vec};
#[cfg(feature = "std")]
use std::borrow::Cow;
use crate::attribute_info::AttributeInfoMessage;
use crate::btree_v2::{BTreeV2Header, collect_btree_v2_records};
@@ -48,17 +50,64 @@ impl AttributeMessage {
///
/// `length_size` is needed for dataspace dimension parsing.
pub fn parse(data: &[u8], length_size: u8) -> Result<AttributeMessage, FormatError> {
Self::parse_impl(data, length_size, None)
}
/// [`AttributeMessage::parse`] with access to the rest of the file, which
/// is needed when the attribute's datatype or dataspace is *shared* (v2/v3
/// flag bits 0/1) — e.g. an attribute created with a committed datatype.
/// In that case the embedded bytes are a reference to the real message,
/// not the message. Without file access such an attribute is an error
/// rather than a garbage datatype.
pub fn parse_in_file(
data: &[u8],
file_data: &[u8],
offset_size: u8,
length_size: u8,
) -> Result<AttributeMessage, FormatError> {
Self::parse_impl(data, length_size, Some((file_data, offset_size)))
}
fn parse_impl(
data: &[u8],
length_size: u8,
file: Option<(&[u8], u8)>,
) -> Result<AttributeMessage, FormatError> {
ensure_len(data, 0, 2)?;
let version = data[0];
match version {
1 => Self::parse_v1(data, length_size),
2 => Self::parse_v2(data, length_size),
3 => Self::parse_v3(data, length_size),
2 => Self::parse_v2(data, length_size, file),
3 => Self::parse_v3(data, length_size, file),
_ => Err(FormatError::InvalidAttributeVersion(version)),
}
}
/// The bytes of an embedded datatype/dataspace message, following the
/// shared-message reference when `shared` is set.
fn embedded_message<'a>(
bytes: &'a [u8],
shared: bool,
msg_type: MessageType,
length_size: u8,
file: Option<(&[u8], u8)>,
) -> Result<Cow<'a, [u8]>, FormatError> {
if !shared {
return Ok(Cow::Borrowed(bytes));
}
let (file_data, offset_size) = file.ok_or(FormatError::UnresolvedSharedMessage)?;
let shared_ref = shared_message::parse_shared_ref(bytes, offset_size)?;
shared_message::resolve_shared_message(
file_data,
&shared_ref,
msg_type,
offset_size,
length_size,
)
.map(Cow::Owned)
}
fn parse_v1(data: &[u8], length_size: u8) -> Result<AttributeMessage, FormatError> {
// version(1) + reserved(1) + name_size(2) + datatype_size(2) + dataspace_size(2) = 8
ensure_len(data, 0, 8)?;
@@ -94,7 +143,13 @@ impl AttributeMessage {
})
}
fn parse_v2(data: &[u8], length_size: u8) -> Result<AttributeMessage, FormatError> {
fn parse_v2(
data: &[u8],
length_size: u8,
file: Option<(&[u8], u8)>,
) -> Result<AttributeMessage, FormatError> {
// Flags: bit 0 = datatype is shared, bit 1 = dataspace is shared.
let flags = data.get(1).copied().unwrap_or(0);
// version(1) + flags(1) + name_size(2) + datatype_size(2) + dataspace_size(2) = 8
ensure_len(data, 0, 8)?;
let name_size = u16::from_le_bytes([data[2], data[3]]) as usize;
@@ -110,12 +165,26 @@ impl AttributeMessage {
// Datatype (NO padding)
ensure_len(data, pos, datatype_size)?;
let (datatype, _) = Datatype::parse(&data[pos..pos + datatype_size])?;
let dt_bytes = Self::embedded_message(
&data[pos..pos + datatype_size],
flags & 0x01 != 0,
MessageType::Datatype,
length_size,
file,
)?;
let (datatype, _) = Datatype::parse(&dt_bytes)?;
pos += datatype_size;
// Dataspace (NO padding)
ensure_len(data, pos, dataspace_size)?;
let dataspace = Dataspace::parse(&data[pos..pos + dataspace_size], length_size)?;
let ds_bytes = Self::embedded_message(
&data[pos..pos + dataspace_size],
flags & 0x02 != 0,
MessageType::Dataspace,
length_size,
file,
)?;
let dataspace = Dataspace::parse(&ds_bytes, length_size)?;
pos += dataspace_size;
let raw_data = compute_raw_data(data, pos, &dataspace, &datatype);
@@ -128,7 +197,13 @@ impl AttributeMessage {
})
}
fn parse_v3(data: &[u8], length_size: u8) -> Result<AttributeMessage, FormatError> {
fn parse_v3(
data: &[u8],
length_size: u8,
file: Option<(&[u8], u8)>,
) -> Result<AttributeMessage, FormatError> {
// Flags: bit 0 = datatype is shared, bit 1 = dataspace is shared.
let flags = data.get(1).copied().unwrap_or(0);
// version(1) + flags(1) + name_size(2) + datatype_size(2) + dataspace_size(2) + encoding(1) = 9
ensure_len(data, 0, 9)?;
let name_size = u16::from_le_bytes([data[2], data[3]]) as usize;
@@ -145,12 +220,26 @@ impl AttributeMessage {
// Datatype (NO padding)
ensure_len(data, pos, datatype_size)?;
let (datatype, _) = Datatype::parse(&data[pos..pos + datatype_size])?;
let dt_bytes = Self::embedded_message(
&data[pos..pos + datatype_size],
flags & 0x01 != 0,
MessageType::Datatype,
length_size,
file,
)?;
let (datatype, _) = Datatype::parse(&dt_bytes)?;
pos += datatype_size;
// Dataspace (NO padding)
ensure_len(data, pos, dataspace_size)?;
let dataspace = Dataspace::parse(&data[pos..pos + dataspace_size], length_size)?;
let ds_bytes = Self::embedded_message(
&data[pos..pos + dataspace_size],
flags & 0x02 != 0,
MessageType::Dataspace,
length_size,
file,
)?;
let dataspace = Dataspace::parse(&ds_bytes, length_size)?;
pos += dataspace_size;
let raw_data = compute_raw_data(data, pos, &dataspace, &datatype);
@@ -326,10 +415,20 @@ pub fn extract_attributes_full(
offset_size,
length_size,
)?;
let attr = AttributeMessage::parse(&resolved_data, length_size)?;
let attr = AttributeMessage::parse_in_file(
&resolved_data,
file_data,
offset_size,
length_size,
)?;
attrs.push(attr);
} else {
let attr = AttributeMessage::parse(&msg.data, length_size)?;
let attr = AttributeMessage::parse_in_file(
&msg.data,
file_data,
offset_size,
length_size,
)?;
attrs.push(attr);
}
}
@@ -399,7 +498,8 @@ fn extract_dense_attributes(
let attr_data = fh.read_managed_object(file_data, id_bytes, offset_size)?;
// The data in the heap is a complete attribute message
let attr = AttributeMessage::parse(&attr_data, length_size)?;
let attr =
AttributeMessage::parse_in_file(&attr_data, file_data, offset_size, length_size)?;
attrs.push(attr);
}