quantumclaw/clawhdf5
3
1
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 1 Wiki Activity

feat(format): read the other attributes when one cannot be read

Browse Source 
attrs() read every attribute of an object through extract_attributes_full,
so one attribute it could not read (a corrupt or unsupported attribute
message, or a heap object it could not locate) failed all of them — the
same shape as the huge-object bug, where one 8 KiB attribute hid every
attribute on a NetCDF file's root group.

- clawhdf5-format: new attribute::extract_attributes_tolerant returns the
  attributes it could read plus one error per attribute it could not.
  Errors in the attribute index itself (Attribute Info message, dense
  heap header, B-tree) still fail, since then it is unknown which
  attributes exist. extract_attributes_full is unchanged (strict); both
  share one implementation.
- clawhdf5: attrs() on Group/Dataset, MmapGroup/MmapDataset and
  LazyGroup/LazyDataset leaves an unreadable attribute out (documented),
  and the new attrs_with_errors() returns the map with the per-attribute
  errors. A value is either returned complete or not at all.

Regression test: one_unreadable_attribute_does_not_hide_the_others (h5py
writes 11 dense attributes; one message's version byte is corrupted;
before: attrs() failed with InvalidAttributeVersion(127), after: the 10
others come back with their values and one error is reported).

Co-Authored-By: Claude Opus 5.5 (1M context) <[email protected]>
This commit is contained in:
osobh
2026-09-25 22:03:09 -05:00
co-authored by Claude Opus 5.5
parent aadfd18d4c
commit d54a0f4737
8 changed files with 290 additions and 97 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/clawhdf5-format/src/attribute.rs
+90 -40
View File
@@ -394,42 +394,80 @@ pub fn find_attribute<'a>(
///
/// Use this instead of `extract_attributes` when reading files that may use dense storage
/// (e.g., objects with many attributes, typically >8).
///
/// Fails if any attribute cannot be read; see [`extract_attributes_tolerant`]
/// to read the others.
pub fn extract_attributes_full(
file_data: &[u8],
header: &ObjectHeader,
offset_size: u8,
length_size: u8,
) -> Result<Vec<AttributeMessage>, FormatError> {
extract_attributes_with(file_data, header, offset_size, length_size, &mut Err)
}
/// Like [`extract_attributes_full`], but an attribute that cannot be read
/// (a corrupt or unsupported attribute message, or a heap object that cannot
/// be located) is left out and its error returned alongside the attributes
/// that could be read, instead of failing them all.
///
/// Errors in the structures that index the attributes (the Attribute Info
/// message, the dense-storage heap header or B-tree) still fail the call:
/// then it is unknown which attributes exist at all.
pub fn extract_attributes_tolerant(
file_data: &[u8],
header: &ObjectHeader,
offset_size: u8,
length_size: u8,
) -> Result<(Vec<AttributeMessage>, Vec<FormatError>), FormatError> {
let mut errors = Vec::new();
let attrs = extract_attributes_with(file_data, header, offset_size, length_size, &mut |e| {
errors.push(e);
Ok(())
})?;
Ok((attrs, errors))
}
/// Read every attribute; each one that fails goes to `on_error`, which
/// either stops the read (returns the error) or skips that attribute.
fn extract_attributes_with(
file_data: &[u8],
header: &ObjectHeader,
offset_size: u8,
length_size: u8,
on_error: &mut dyn FnMut(FormatError) -> Result<(), FormatError>,
) -> Result<Vec<AttributeMessage>, FormatError> {
let mut attrs = Vec::new();
// Collect compact attributes (inline in OH)
for msg in &header.messages {
if msg.msg_type == MessageType::Attribute {
if shared_message::is_shared(msg.flags) {
let attr = if shared_message::is_shared(msg.flags) {
// Shared attribute: resolve the reference to get actual attribute data
let shared_ref = shared_message::parse_shared_ref(&msg.data, offset_size)?;
let resolved_data = shared_message::resolve_shared_message(
file_data,
&shared_ref,
MessageType::Attribute,
offset_size,
length_size,
)?;
let attr = AttributeMessage::parse_in_file(
&resolved_data,
file_data,
offset_size,
length_size,
)?;
attrs.push(attr);
shared_message::parse_shared_ref(&msg.data, offset_size)
.and_then(|shared_ref| {
shared_message::resolve_shared_message(
file_data,
&shared_ref,
MessageType::Attribute,
offset_size,
length_size,
)
})
.and_then(|resolved| {
AttributeMessage::parse_in_file(
&resolved,
file_data,
offset_size,
length_size,
)
})
} else {
let attr = AttributeMessage::parse_in_file(
&msg.data,
file_data,
offset_size,
length_size,
)?;
attrs.push(attr);
AttributeMessage::parse_in_file(&msg.data, file_data, offset_size, length_size)
};
match attr {
Ok(attr) => attrs.push(attr),
Err(e) => on_error(e)?,
}
}
}
@@ -439,9 +477,15 @@ pub fn extract_attributes_full(
if let Some(info) = attr_info
&& let Some(fh_addr) = info.fractal_heap_address
{
let dense_attrs =
extract_dense_attributes(file_data, &info, fh_addr, offset_size, length_size)?;
attrs.extend(dense_attrs);
extract_dense_attributes(
file_data,
&info,
fh_addr,
offset_size,
length_size,
&mut attrs,
on_error,
)?;
}
Ok(attrs)
@@ -468,7 +512,9 @@ fn extract_dense_attributes(
fh_addr: u64,
offset_size: u8,
length_size: u8,
) -> Result<Vec<AttributeMessage>, FormatError> {
attrs: &mut Vec<AttributeMessage>,
on_error: &mut dyn FnMut(FormatError) -> Result<(), FormatError>,
) -> Result<(), FormatError> {
// Parse fractal heap
let fh = FractalHeapHeader::parse(file_data, fh_addr as usize, offset_size, length_size)?;
@@ -482,28 +528,32 @@ fn extract_dense_attributes(
let btree_hdr = BTreeV2Header::parse(file_data, btree_addr as usize, offset_size, length_size)?;
let records = collect_btree_v2_records(file_data, &btree_hdr, offset_size, length_size)?;
let mut attrs = Vec::new();
for record in &records {
// Per HDF5 spec, both type 8 and type 9 records start with heap_id:
// Type 8: heap_id(8) + msg_flags(1) + creation_order(4) + hash(4)
// Type 9: heap_id(8) + msg_flags(1) + creation_order(4)
let id_offset = 0;
if record.data.len() < id_offset + fh.heap_id_length as usize {
let id_len = fh.heap_id_length as usize;
let Some(id_bytes) = record.data.get(..id_len) else {
on_error(FormatError::UnexpectedEof {
expected: id_len,
available: record.data.len(),
})?;
continue;
}
let id_bytes = &record.data[id_offset..id_offset + fh.heap_id_length as usize];
// Read attribute message from fractal heap
let attr_data = fh.read_managed_object(file_data, id_bytes, offset_size)?;
};
// The data in the heap is a complete attribute message
let attr =
AttributeMessage::parse_in_file(&attr_data, file_data, offset_size, length_size)?;
attrs.push(attr);
let attr = fh
.read_managed_object(file_data, id_bytes, offset_size)
.and_then(|attr_data| {
AttributeMessage::parse_in_file(&attr_data, file_data, offset_size, length_size)
});
match attr {
Ok(attr) => attrs.push(attr),
Err(e) => on_error(e)?,
}
}
Ok(attrs)
Ok(())
}
#[cfg(test)]