quantumclaw/clawhdf5
3
1
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 1 Wiki Activity

Merge branch 'fix/p1-attrs-links' into fix/p1-read-gaps

Browse Source 
# Conflicts:
#	crates/clawhdf5-format/src/attribute.rs
#	crates/clawhdf5-format/src/group_v1.rs
#	crates/clawhdf5/src/lazy.rs
#	crates/clawhdf5/src/mmap_file.rs
#	docs/known-issues.md
This commit is contained in:
osobh
2026-09-25 22:41:33 -05:00
parent 45720fe5a6 17fa783dce
commit 61f36516d7
17 changed files with 1568 additions and 378 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/clawhdf5-format/src/attribute.rs
+90 -41
View File
@@ -394,43 +394,80 @@ pub fn find_attribute<'a>(
///
/// Use this instead of `extract_attributes` when reading files that may use dense storage
/// (e.g., objects with many attributes, typically >8).
///
/// Fails if any attribute cannot be read; see [`extract_attributes_tolerant`]
/// to read the others.
pub fn extract_attributes_full(
file_data: &[u8],
header: &ObjectHeader,
offset_size: u8,
length_size: u8,
) -> Result<Vec<AttributeMessage>, FormatError> {
extract_attributes_with(file_data, header, offset_size, length_size, &mut Err)
}
/// Like [`extract_attributes_full`], but an attribute that cannot be read
/// (a corrupt or unsupported attribute message, or a heap object that cannot
/// be located) is left out and its error returned alongside the attributes
/// that could be read, instead of failing them all.
///
/// Errors in the structures that index the attributes (the Attribute Info
/// message, the dense-storage heap header or B-tree) still fail the call:
/// then it is unknown which attributes exist at all.
pub fn extract_attributes_tolerant(
file_data: &[u8],
header: &ObjectHeader,
offset_size: u8,
length_size: u8,
) -> Result<(Vec<AttributeMessage>, Vec<FormatError>), FormatError> {
let mut errors = Vec::new();
let attrs = extract_attributes_with(file_data, header, offset_size, length_size, &mut |e| {
errors.push(e);
Ok(())
})?;
Ok((attrs, errors))
}
/// Read every attribute; each one that fails goes to `on_error`, which
/// either stops the read (returns the error) or skips that attribute.
fn extract_attributes_with(
file_data: &[u8],
header: &ObjectHeader,
offset_size: u8,
length_size: u8,
on_error: &mut dyn FnMut(FormatError) -> Result<(), FormatError>,
) -> Result<Vec<AttributeMessage>, FormatError> {
let mut attrs = Vec::new();
// Collect compact attributes (inline in OH)
for msg in &header.messages {
if msg.msg_type == MessageType::Attribute {
if shared_message::is_shared(msg.flags) {
let attr = if shared_message::is_shared(msg.flags) {
// Shared attribute: resolve the reference to get actual attribute data
let shared_ref =
shared_message::parse_shared_ref_sized(&msg.data, offset_size, length_size)?;
let resolved_data = shared_message::resolve_shared_message(
file_data,
&shared_ref,
MessageType::Attribute,
offset_size,
length_size,
)?;
let attr = AttributeMessage::parse_in_file(
&resolved_data,
file_data,
offset_size,
length_size,
)?;
attrs.push(attr);
shared_message::parse_shared_ref_sized(&msg.data, offset_size, length_size)
.and_then(|shared_ref| {
shared_message::resolve_shared_message(
file_data,
&shared_ref,
MessageType::Attribute,
offset_size,
length_size,
)
})
.and_then(|resolved| {
AttributeMessage::parse_in_file(
&resolved,
file_data,
offset_size,
length_size,
)
})
} else {
let attr = AttributeMessage::parse_in_file(
&msg.data,
file_data,
offset_size,
length_size,
)?;
attrs.push(attr);
AttributeMessage::parse_in_file(&msg.data, file_data, offset_size, length_size)
};
match attr {
Ok(attr) => attrs.push(attr),
Err(e) => on_error(e)?,
}
}
}
@@ -440,9 +477,15 @@ pub fn extract_attributes_full(
if let Some(info) = attr_info
&& let Some(fh_addr) = info.fractal_heap_address
{
let dense_attrs =
extract_dense_attributes(file_data, &info, fh_addr, offset_size, length_size)?;
attrs.extend(dense_attrs);
extract_dense_attributes(
file_data,
&info,
fh_addr,
offset_size,
length_size,
&mut attrs,
on_error,
)?;
}
Ok(attrs)
@@ -469,7 +512,9 @@ fn extract_dense_attributes(
fh_addr: u64,
offset_size: u8,
length_size: u8,
) -> Result<Vec<AttributeMessage>, FormatError> {
attrs: &mut Vec<AttributeMessage>,
on_error: &mut dyn FnMut(FormatError) -> Result<(), FormatError>,
) -> Result<(), FormatError> {
// Parse fractal heap
let fh = FractalHeapHeader::parse(file_data, fh_addr as usize, offset_size, length_size)?;
@@ -483,28 +528,32 @@ fn extract_dense_attributes(
let btree_hdr = BTreeV2Header::parse(file_data, btree_addr as usize, offset_size, length_size)?;
let records = collect_btree_v2_records(file_data, &btree_hdr, offset_size, length_size)?;
let mut attrs = Vec::new();
for record in &records {
// Per HDF5 spec, both type 8 and type 9 records start with heap_id:
// Type 8: heap_id(8) + msg_flags(1) + creation_order(4) + hash(4)
// Type 9: heap_id(8) + msg_flags(1) + creation_order(4)
let id_offset = 0;
if record.data.len() < id_offset + fh.heap_id_length as usize {
let id_len = fh.heap_id_length as usize;
let Some(id_bytes) = record.data.get(..id_len) else {
on_error(FormatError::UnexpectedEof {
expected: id_len,
available: record.data.len(),
})?;
continue;
}
let id_bytes = &record.data[id_offset..id_offset + fh.heap_id_length as usize];
// Read attribute message from fractal heap
let attr_data = fh.read_managed_object(file_data, id_bytes, offset_size)?;
};
// The data in the heap is a complete attribute message
let attr =
AttributeMessage::parse_in_file(&attr_data, file_data, offset_size, length_size)?;
attrs.push(attr);
let attr = fh
.read_managed_object(file_data, id_bytes, offset_size)
.and_then(|attr_data| {
AttributeMessage::parse_in_file(&attr_data, file_data, offset_size, length_size)
});
match attr {
Ok(attr) => attrs.push(attr),
Err(e) => on_error(e)?,
}
}
Ok(attrs)
Ok(())
}
#[cfg(test)]