quantumclaw/clawhdf5
3
1
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 1 Wiki Activity

fix(format): resolve shared fill value messages instead of zero-filling

Browse Source 
dataset_fill_value treated a shared Fill Value message as "no fill
value", so unwritten storage of a dataset whose fill value lives in the
file's shared-message (SOHM) heap read as zeros rather than its fill
value. libhdf5 shares fill values whenever the file has a SOHM index for
them.

- fill_value::dataset_fill_value_in follows the reference (another object
  header, or the SOHM heap); read_full_with_fill and the facade's
  selection read use it.
- dataset_fill_value, which has no file to follow a reference into, now
  returns UnresolvedSharedMessage for a shared message instead of None.
- shared_message::load_sohm_table / message_data_with_sohm load the SOHM
  table from the superblock extension on demand.
- parse_sohm_table skipped each index's leading version byte, reading
  every field one byte off; SOHM references could never resolve.

Fixture shared_fill_value.h5 (HDF5 2.0, gen_shared_fill.py): sohm_b read
[0,1,2,3,0,0,0,0] and now reads [0,1,2,3,-7,-7,-7,-7], as h5py does.

Co-Authored-By: Claude Opus 5.5 (1M context) <[email protected]>
This commit is contained in:
osobh
2026-09-25 21:18:12 -05:00
co-authored by Claude Opus 5.5
parent 57e938c438
commit 7c1968a34a
7 changed files with 268 additions and 12 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/clawhdf5-format/src/fill_value.rs
+42 -7
View File
@@ -98,15 +98,50 @@ pub fn parse_fill_value(msg: &HeaderMessage) -> Result<Option<Vec<u8>>, FormatEr
/// The fill value that applies to a dataset given its header messages. The new
/// message wins over the old one when both are present.
///
/// A *shared* fill value message holds only a reference to the real message,
/// which cannot be followed without the file: this returns
/// [`FormatError::UnresolvedSharedMessage`] for one (it used to answer "zeros").
/// Use [`dataset_fill_value_in`] when the file bytes are at hand.
pub fn dataset_fill_value(messages: &[HeaderMessage]) -> Result<Option<Vec<u8>>, FormatError> {
fill_value_from(messages, |_| Err(FormatError::UnresolvedSharedMessage))
}
/// [`dataset_fill_value`] for a dataset in `file_data`, following a shared
/// fill value message to where it lives: another object header, or the
/// file's shared-message (SOHM) heap, as libhdf5 writes it when the file has
/// a SOHM index for fill values.
pub fn dataset_fill_value_in(
file_data: &[u8],
messages: &[HeaderMessage],
offset_size: u8,
length_size: u8,
) -> Result<Option<Vec<u8>>, FormatError> {
fill_value_from(messages, |msg| {
crate::shared_message::message_data_with_sohm(file_data, msg, offset_size, length_size)
.map(|data| data.into_owned())
})
}
fn fill_value_from(
messages: &[HeaderMessage],
resolve_shared: impl Fn(&HeaderMessage) -> Result<Vec<u8>, FormatError>,
) -> Result<Option<Vec<u8>>, FormatError> {
for wanted in [MessageType::FillValue, MessageType::FillValueOld] {
if let Some(msg) = messages.iter().find(|m| m.msg_type == wanted) {
if crate::shared_message::is_shared(msg.flags) {
// A shared fill value is legal but vanishingly rare; treat it
// as the default rather than misparsing the reference.
return Ok(None);
}
if let Some(value) = parse_fill_value(msg)? {
let value = if crate::shared_message::is_shared(msg.flags) {
let data = resolve_shared(msg)?;
parse_fill_value(&HeaderMessage {
msg_type: msg.msg_type,
size: data.len(),
flags: msg.flags & !0x02,
creation_order: msg.creation_order,
data,
})?
} else {
parse_fill_value(msg)?
};
if let Some(value) = value {
return Ok(Some(value));
}
}
@@ -174,7 +209,7 @@ pub fn read_full_with_fill<E: From<FormatError>>(
{
return Err(FormatError::ExternalDataFilesUnsupported.into());
}
let fill = dataset_fill_value(messages)?;
let fill = dataset_fill_value_in(file_data, messages, offset_size, length_size)?;
if !has_storage(layout) {
return Ok(filled_dataset(dataspace, elem_size, fill.as_deref())?);
}
crates/clawhdf5-format/src/shared_message.rs
+75 -4
View File
@@ -225,9 +225,12 @@ pub fn parse_sohm_table_message(
/// Parse the SOHM table structure (signature "SMTB") from the file.
///
/// Each index entry: index_type(1) + mesg_types(2) + min_mesg_size(4) +
/// list_max(2) + btree_min(2) + num_messages(2) + index_addr(offset_size) +
/// heap_addr(offset_size)
/// Each index entry: version(1) + index_type(1) + mesg_types(2) +
/// min_mesg_size(4) + list_max(2) + btree_min(2) + num_messages(2) +
/// index_addr(offset_size) + heap_addr(offset_size)
///
/// The leading per-index version byte (0) was missing here, so every field
/// after it was read one byte off — verified against an HDF5 2.0 file.
pub fn parse_sohm_table(
file_data: &[u8],
table_addr: usize,
@@ -240,11 +243,16 @@ pub fn parse_sohm_table(
}
let mut pos = table_addr + 4;
let os = offset_size as usize;
let entry_size = 1 + 2 + 4 + 2 + 2 + 2 + os + os; // 13 + 2*offset_size
let entry_size = 1 + 1 + 2 + 4 + 2 + 2 + 2 + os + os; // 14 + 2*offset_size
let mut indexes = Vec::with_capacity(nindexes as usize);
for _ in 0..nindexes {
ensure_len(file_data, pos, entry_size)?;
let version = file_data[pos];
if version != 0 {
return Err(FormatError::InvalidSohmTableVersion(version));
}
pos += 1;
let index_type = file_data[pos];
pos += 1;
let mesg_types = u16::from_le_bytes([file_data[pos], file_data[pos + 1]]);
@@ -381,6 +389,68 @@ pub fn parse_sohm_btree_entries(
// ---- SOHM resolution ----
/// Find the SOHM index that handles the given message type.
/// Load a file's SOHM table: superblock → superblock extension → Shared
/// Message Table message → SMTB. `Ok(None)` when the file has no superblock
/// extension or no shared-message table.
pub fn load_sohm_table(
file_data: &[u8],
offset_size: u8,
length_size: u8,
) -> Result<Option<SohmTable>, FormatError> {
let sig = crate::signature::find_signature(file_data)?;
let sb = crate::superblock::Superblock::parse(file_data, sig)?;
let Some(ext_addr) = sb
.superblock_extension_address
.filter(|&a| !is_undefined(a, offset_size))
else {
return Ok(None);
};
let ext = ObjectHeader::parse(file_data, ext_addr as usize, offset_size, length_size)?;
let Some(msg) = ext
.messages
.iter()
.find(|m| m.msg_type == MessageType::SharedMessageTable)
else {
return Ok(None);
};
let table_msg = parse_sohm_table_message(&msg.data, offset_size)?;
parse_sohm_table(
file_data,
table_msg.table_address as usize,
table_msg.nindexes,
offset_size,
)
.map(Some)
}
/// Like [`message_data`], but also follows references into the file's SOHM
/// heap (shared object header messages), loading the SOHM table on demand.
pub fn message_data_with_sohm<'a>(
file_data: &[u8],
msg: &'a crate::object_header::HeaderMessage,
offset_size: u8,
length_size: u8,
) -> Result<Cow<'a, [u8]>, FormatError> {
if !is_shared(msg.flags) {
return Ok(Cow::Borrowed(&msg.data));
}
let shared_ref = parse_shared_ref(&msg.data, offset_size)?;
let table = if shared_ref.heap_id.is_some() {
load_sohm_table(file_data, offset_size, length_size)?
} else {
None
};
resolve_shared_message_with_sohm(
file_data,
&shared_ref,
msg.msg_type,
offset_size,
length_size,
table.as_ref(),
)
.map(Cow::Owned)
}
fn find_index_for_msg_type(table: &SohmTable, msg_type: MessageType) -> Option<&SohmIndex> {
let type_bit = 1u16 << msg_type.to_u16();
table
@@ -707,6 +777,7 @@ mod tests {
let mut buf = Vec::new();
buf.extend_from_slice(b"SMTB");
for idx in indexes {
buf.push(0); // version
buf.push(idx.index_type);
buf.extend_from_slice(&idx.mesg_types.to_le_bytes());
buf.extend_from_slice(&idx.min_mesg_size.to_le_bytes());