quantumclaw/clawhdf5
3
1
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 1 Wiki Activity

Merge branch 'fix/p1-attrs-links' into fix/p1-read-gaps

Browse Source 
# Conflicts:
#	crates/clawhdf5-format/src/attribute.rs
#	crates/clawhdf5-format/src/group_v1.rs
#	crates/clawhdf5/src/lazy.rs
#	crates/clawhdf5/src/mmap_file.rs
#	docs/known-issues.md
This commit is contained in:
osobh
2026-09-25 22:41:33 -05:00
parent 45720fe5a6 17fa783dce
commit 61f36516d7
17 changed files with 1568 additions and 378 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/clawhdf5-format/src/attribute.rs
+90 -41
View File
@@ -394,43 +394,80 @@ pub fn find_attribute<'a>(
///
/// Use this instead of `extract_attributes` when reading files that may use dense storage
/// (e.g., objects with many attributes, typically >8).
///
/// Fails if any attribute cannot be read; see [`extract_attributes_tolerant`]
/// to read the others.
pub fn extract_attributes_full(
file_data: &[u8],
header: &ObjectHeader,
offset_size: u8,
length_size: u8,
) -> Result<Vec<AttributeMessage>, FormatError> {
extract_attributes_with(file_data, header, offset_size, length_size, &mut Err)
}
/// Like [`extract_attributes_full`], but an attribute that cannot be read
/// (a corrupt or unsupported attribute message, or a heap object that cannot
/// be located) is left out and its error returned alongside the attributes
/// that could be read, instead of failing them all.
///
/// Errors in the structures that index the attributes (the Attribute Info
/// message, the dense-storage heap header or B-tree) still fail the call:
/// then it is unknown which attributes exist at all.
pub fn extract_attributes_tolerant(
file_data: &[u8],
header: &ObjectHeader,
offset_size: u8,
length_size: u8,
) -> Result<(Vec<AttributeMessage>, Vec<FormatError>), FormatError> {
let mut errors = Vec::new();
let attrs = extract_attributes_with(file_data, header, offset_size, length_size, &mut |e| {
errors.push(e);
Ok(())
})?;
Ok((attrs, errors))
}
/// Read every attribute; each one that fails goes to `on_error`, which
/// either stops the read (returns the error) or skips that attribute.
fn extract_attributes_with(
file_data: &[u8],
header: &ObjectHeader,
offset_size: u8,
length_size: u8,
on_error: &mut dyn FnMut(FormatError) -> Result<(), FormatError>,
) -> Result<Vec<AttributeMessage>, FormatError> {
let mut attrs = Vec::new();
// Collect compact attributes (inline in OH)
for msg in &header.messages {
if msg.msg_type == MessageType::Attribute {
if shared_message::is_shared(msg.flags) {
let attr = if shared_message::is_shared(msg.flags) {
// Shared attribute: resolve the reference to get actual attribute data
let shared_ref =
shared_message::parse_shared_ref_sized(&msg.data, offset_size, length_size)?;
let resolved_data = shared_message::resolve_shared_message(
file_data,
&shared_ref,
MessageType::Attribute,
offset_size,
length_size,
)?;
let attr = AttributeMessage::parse_in_file(
&resolved_data,
file_data,
offset_size,
length_size,
)?;
attrs.push(attr);
shared_message::parse_shared_ref_sized(&msg.data, offset_size, length_size)
.and_then(|shared_ref| {
shared_message::resolve_shared_message(
file_data,
&shared_ref,
MessageType::Attribute,
offset_size,
length_size,
)
})
.and_then(|resolved| {
AttributeMessage::parse_in_file(
&resolved,
file_data,
offset_size,
length_size,
)
})
} else {
let attr = AttributeMessage::parse_in_file(
&msg.data,
file_data,
offset_size,
length_size,
)?;
attrs.push(attr);
AttributeMessage::parse_in_file(&msg.data, file_data, offset_size, length_size)
};
match attr {
Ok(attr) => attrs.push(attr),
Err(e) => on_error(e)?,
}
}
}
@@ -440,9 +477,15 @@ pub fn extract_attributes_full(
if let Some(info) = attr_info
&& let Some(fh_addr) = info.fractal_heap_address
{
let dense_attrs =
extract_dense_attributes(file_data, &info, fh_addr, offset_size, length_size)?;
attrs.extend(dense_attrs);
extract_dense_attributes(
file_data,
&info,
fh_addr,
offset_size,
length_size,
&mut attrs,
on_error,
)?;
}
Ok(attrs)
@@ -469,7 +512,9 @@ fn extract_dense_attributes(
fh_addr: u64,
offset_size: u8,
length_size: u8,
) -> Result<Vec<AttributeMessage>, FormatError> {
attrs: &mut Vec<AttributeMessage>,
on_error: &mut dyn FnMut(FormatError) -> Result<(), FormatError>,
) -> Result<(), FormatError> {
// Parse fractal heap
let fh = FractalHeapHeader::parse(file_data, fh_addr as usize, offset_size, length_size)?;
@@ -483,28 +528,32 @@ fn extract_dense_attributes(
let btree_hdr = BTreeV2Header::parse(file_data, btree_addr as usize, offset_size, length_size)?;
let records = collect_btree_v2_records(file_data, &btree_hdr, offset_size, length_size)?;
let mut attrs = Vec::new();
for record in &records {
// Per HDF5 spec, both type 8 and type 9 records start with heap_id:
// Type 8: heap_id(8) + msg_flags(1) + creation_order(4) + hash(4)
// Type 9: heap_id(8) + msg_flags(1) + creation_order(4)
let id_offset = 0;
if record.data.len() < id_offset + fh.heap_id_length as usize {
let id_len = fh.heap_id_length as usize;
let Some(id_bytes) = record.data.get(..id_len) else {
on_error(FormatError::UnexpectedEof {
expected: id_len,
available: record.data.len(),
})?;
continue;
}
let id_bytes = &record.data[id_offset..id_offset + fh.heap_id_length as usize];
// Read attribute message from fractal heap
let attr_data = fh.read_managed_object(file_data, id_bytes, offset_size)?;
};
// The data in the heap is a complete attribute message
let attr =
AttributeMessage::parse_in_file(&attr_data, file_data, offset_size, length_size)?;
attrs.push(attr);
let attr = fh
.read_managed_object(file_data, id_bytes, offset_size)
.and_then(|attr_data| {
AttributeMessage::parse_in_file(&attr_data, file_data, offset_size, length_size)
});
match attr {
Ok(attr) => attrs.push(attr),
Err(e) => on_error(e)?,
}
}
Ok(attrs)
Ok(())
}
#[cfg(test)]
crates/clawhdf5-format/src/btree_v2.rs
+64 -40
View File
@@ -323,39 +323,21 @@ fn collect_internal_records(
let records_start = pos;
pos += records_total;
// Compute sizes for child pointers
// max_records at child depth - for variable-width nrec encoding
// Child pointer layout, as libhdf5 computes it (H5B2__hdr_init): the
// child's record count is always encoded in the width needed for a
// *leaf's* maximum, and — below the first internal level — the child
// subtree's total record count in the width needed for the most records
// a subtree of that depth can hold.
let child_depth = depth - 1;
let max_nrec_child = if child_depth == 0 {
max_leaf_nrec
} else {
// For internal nodes at child_depth, the true max_nrec depends on the
// node size, record size, and the recursive width of child pointer
// entries (which themselves depend on max_nrec at deeper levels).
// Computing the exact value requires iterating from the leaf level
// upward, as described in the HDF5 spec (III.A.2 "Computing the Size
// of B-tree Nodes").
//
// We use `max_leaf_nrec * 2` as a conservative upper bound. This
// over-estimates the nrec encoding width, which means we may read
// slightly more bytes per child pointer than strictly necessary, but
// never fewer. The over-read bytes are harmless because we only
// decode `num_records` entries (the actual count from the node header).
//
// Known limitation: for very deep trees (depth > 3) with small record
// sizes, the true max could exceed this estimate, causing us to
// under-allocate the nrec encoding width and misparse child pointers.
// In practice, HDF5 B-tree v2 depths rarely exceed 2-3.
max_leaf_nrec * 2
};
let nrec_width = bytes_for_max_records(max_nrec_child);
// Total records in subtree width (only if depth > 1)
let nrec_width = bytes_for_max_records(max_leaf_nrec);
let total_nrec_width = if depth > 1 {
// Width to hold total records in a subtree
// We compute max possible total records at this subtree depth
let max_total = header_max_total_records(max_leaf_nrec, depth - 1);
bytes_for_max_records(max_total)
bytes_for_max_records(cum_max_records(
node_size,
record_size,
offset_size,
max_leaf_nrec,
child_depth,
))
} else {
0
};
@@ -435,14 +417,36 @@ fn collect_internal_records(
Ok(())
}
/// Estimate maximum total records at a given depth (for variable-width encoding).
fn header_max_total_records(max_leaf_nrec: u64, depth: u16) -> u64 {
// Conservative: branching factor * max_leaf at each level
let mut total = max_leaf_nrec;
for _ in 0..depth {
total = total.saturating_mul(max_leaf_nrec.max(2));
/// Most records a subtree whose root is at `depth` can hold (libhdf5's
/// `cum_max_nrec`): a leaf holds `max_leaf_nrec`; an internal node at depth
/// `d` holds `max_nrec(d)` records and `max_nrec(d) + 1` subtrees of depth
/// `d - 1`, where `max_nrec(d)` is what fits in a node once each record is
/// paired with a child pointer of the width depth `d` needs.
fn cum_max_records(
node_size: u32,
record_size: u16,
offset_size: u8,
max_leaf_nrec: u64,
depth: u16,
) -> u64 {
// Internal node overhead: signature(4) + version(1) + type(1) + checksum(4).
const PREFIX: u64 = 10;
let nrec_width = bytes_for_max_records(max_leaf_nrec) as u64;
let mut cum = max_leaf_nrec;
let mut cum_width = 0u64;
for d in 1..=depth {
let ptr = u64::from(offset_size) + nrec_width + if d > 1 { cum_width } else { 0 };
let max_nrec = u64::from(node_size)
.saturating_sub(PREFIX)
.saturating_sub(ptr)
/ (u64::from(record_size) + ptr).max(1);
cum = max_nrec
.saturating_add(1)
.saturating_mul(cum)
.saturating_add(max_nrec);
cum_width = bytes_for_max_records(cum) as u64;
}
total
cum
}
#[cfg(test)]
@@ -512,9 +516,15 @@ mod tests {
child_nrec: u64,
) -> Vec<u8> {
let max_leaf = max_records_leaf(node_size, record_size);
let nrec_width = bytes_for_max_records(if depth == 1 { max_leaf } else { max_leaf * 2 });
let nrec_width = bytes_for_max_records(max_leaf);
let total_width = if depth > 1 {
bytes_for_max_records(header_max_total_records(max_leaf, depth - 1))
bytes_for_max_records(cum_max_records(
node_size,
record_size,
8,
max_leaf,
depth - 1,
))
} else {
0
};
@@ -673,4 +683,18 @@ mod tests {
let records = collect_btree_v2_records(&header, &hdr, 8, 8).unwrap();
assert!(records.is_empty());
}
#[test]
fn subtree_capacity_matches_libhdf5() {
// A link-name index (11-byte records, 512-byte nodes, 8-byte
// addresses): libhdf5's H5B2__hdr_init gives 45 records per leaf,
// then cum_max_nrec 1 149 at depth 1 and 26 449 at depth 2 — two
// bytes of subtree count in a depth-3 root's child pointers, where
// leaf_max^3 = 91 125 would need three.
let leaf = max_records_leaf(512, 11);
assert_eq!(leaf, 45);
assert_eq!(cum_max_records(512, 11, 8, leaf, 0), 45);
assert_eq!(cum_max_records(512, 11, 8, leaf, 1), 1_149);
assert_eq!(cum_max_records(512, 11, 8, leaf, 2), 26_449);
}
}
crates/clawhdf5-format/src/file_writer.rs
+6
View File
@@ -1947,6 +1947,12 @@ mod tests {
root_block_address: 0,
current_rows_in_root_indirect_block: 0,
managed_objects_count: 0,
huge_btree_address: u64::MAX,
filter_pipeline: None,
root_direct_block_filtered_size: 0,
root_direct_block_filter_mask: 0,
offset_size: 8,
length_size: 8,
};
let (off, len) = fh.decode_managed_id(&id).unwrap();
assert_eq!(off, 100);
crates/clawhdf5-format/src/fractal_heap.rs
+414 -100
View File
@@ -1,12 +1,14 @@
//! HDF5 Fractal Heap parsing for v2 group link storage.
#[cfg(not(feature = "std"))]
use alloc::vec::Vec;
use alloc::{format, vec::Vec};
#[cfg(feature = "checksum")]
use byteorder::{ByteOrder, LittleEndian};
use crate::btree_v2::{BTreeV2Header, collect_btree_v2_records};
use crate::error::FormatError;
use crate::filter_pipeline::FilterPipeline;
/// Parsed fractal heap header (signature "FRHP").
#[derive(Debug, Clone)]
@@ -33,6 +35,23 @@ pub struct FractalHeapHeader {
pub current_rows_in_root_indirect_block: u16,
/// Total number of managed objects.
pub managed_objects_count: u64,
/// Address of the v2 B-tree indexing "huge" objects (undefined address
/// when the heap has none). Huge objects are those larger than
/// `max_managed_object_size`; they live outside the heap's blocks.
pub huge_btree_address: u64,
/// The heap's I/O filter pipeline, if it has one. It applies to managed
/// direct blocks and to huge objects.
pub filter_pipeline: Option<FilterPipeline>,
/// Stored (filtered) size of the root direct block; meaningful only when
/// the heap is filtered and its root is a direct block.
pub root_direct_block_filtered_size: u64,
/// Filter mask of the root direct block (bit *i* set = filter *i*
/// skipped); meaningful only when the heap is filtered.
pub root_direct_block_filter_mask: u32,
/// Size of addresses in the file ("Size of Offsets").
pub offset_size: u8,
/// Size of lengths in the file ("Size of Lengths").
pub length_size: u8,
}
fn read_offset(data: &[u8], pos: usize, size: u8) -> Result<u64, FormatError> {
@@ -79,6 +98,38 @@ fn is_undefined(val: u64, offset_size: u8) -> bool {
}
}
/// Little-endian unsigned integer of up to 8 bytes.
fn le_uint(bytes: &[u8]) -> u64 {
bytes
.iter()
.take(8)
.enumerate()
.fold(0u64, |acc, (i, &b)| acc | (u64::from(b) << (i * 8)))
}
fn heap_error(msg: &str) -> FormatError {
FormatError::ChunkedReadError(format!("fractal heap: {msg}"))
}
/// Heap ID type, from bits 4-5 of an ID's first byte (libhdf5's
/// `H5HF_ID_TYPE_MASK`, 0x30); bits 6-7 are the ID version, which must be 0.
const HEAP_ID_MANAGED: u8 = 0;
const HEAP_ID_HUGE: u8 = 1;
const HEAP_ID_TINY: u8 = 2;
/// The type (0 managed, 1 huge, 2 tiny) of a heap ID from its first byte,
/// refusing an ID version other than 0.
fn heap_id_type(first: u8) -> Result<u8, FormatError> {
if first >> 6 != 0 {
return Err(heap_error("unsupported heap ID version"));
}
Ok((first >> 4) & 0x03)
}
/// v2 B-tree record types indexing a heap's huge objects.
const BTREE_HUGE_INDIRECT: u8 = 1;
const BTREE_HUGE_INDIRECT_FILTERED: u8 = 2;
impl FractalHeapHeader {
/// Parse a fractal heap header at the given offset.
pub fn parse(
@@ -122,11 +173,17 @@ impl FractalHeapHeader {
]);
pos += 4;
// Skip several fixed fields: next_huge_object_id(ls), btree_huge_objects_address(os),
// free_space_managed_blocks(ls), managed_block_free_space_manager_address(os),
// next_huge_object_id (length_size)
ensure_len(file_data, pos, ls)?;
pos += ls;
// btree_huge_objects_address (offset_size)
let huge_btree_address = read_offset(file_data, pos, offset_size)?;
pos += os;
// Skip: free_space_managed_blocks(ls), managed_block_free_space_manager_address(os),
// managed_space_in_heap(ls), allocated_managed_space_in_heap(ls),
// direct_block_allocation_iterator_offset(ls)
let skip_size = 5 * ls + 2 * os;
let skip_size = 4 * ls + os;
ensure_len(file_data, pos, skip_size)?;
pos += skip_size;
@@ -134,14 +191,9 @@ impl FractalHeapHeader {
let managed_objects_count = read_offset(file_data, pos, length_size)?;
pos += ls;
// huge_objects_size (length_size)
pos += ls;
// huge_objects_count (length_size)
pos += ls;
// tiny_objects_size (length_size)
pos += ls;
// tiny_objects_count (length_size)
pos += ls;
// huge_objects_size, huge_objects_count, tiny_objects_size,
// tiny_objects_count (length_size each)
pos += 4 * ls;
// table_width (2)
ensure_len(file_data, pos, 2)?;
@@ -175,16 +227,28 @@ impl FractalHeapHeader {
ensure_len(file_data, pos, 2)?;
let current_rows_in_root_indirect_block =
u16::from_le_bytes([file_data[pos], file_data[pos + 1]]);
#[allow(unused_variables, unused_mut, unused_assignments)]
let mut pos = pos + 2;
pos += 2;
// Skip IO filter encoded info if present
// With I/O filters: root direct block's filtered size (length_size),
// its filter mask (4), then the encoded filter pipeline message.
let mut filter_pipeline = None;
let mut root_direct_block_filtered_size = 0;
let mut root_direct_block_filter_mask = 0;
if io_filter_encoded_length > 0 {
// root_block_filter_info_size (length_size) + filter_mask (4)
#[allow(unused_assignments)]
{
pos += ls + 4;
}
root_direct_block_filtered_size = read_offset(file_data, pos, length_size)?;
pos += ls;
ensure_len(file_data, pos, 4)?;
root_direct_block_filter_mask = u32::from_le_bytes([
file_data[pos],
file_data[pos + 1],
file_data[pos + 2],
file_data[pos + 3],
]);
pos += 4;
let n = io_filter_encoded_length as usize;
ensure_len(file_data, pos, n)?;
filter_pipeline = Some(FilterPipeline::parse(&file_data[pos..pos + n])?);
pos += n;
}
// Validate header checksum
@@ -200,6 +264,8 @@ impl FractalHeapHeader {
});
}
}
#[cfg(not(feature = "checksum"))]
let _ = pos;
Ok(FractalHeapHeader {
heap_id_length,
@@ -213,13 +279,19 @@ impl FractalHeapHeader {
root_block_address,
current_rows_in_root_indirect_block,
managed_objects_count,
huge_btree_address,
filter_pipeline,
root_direct_block_filtered_size,
root_direct_block_filter_mask,
offset_size,
length_size,
})
}
/// Decode a managed heap ID into (offset_in_heap, object_length).
///
/// The heap ID layout for managed objects (type 0):
/// - Byte 0: bits 6-7 = type (0), bits 4-5 = version (0), bits 0-3 = reserved
/// - Byte 0: bits 6-7 = version (0), bits 4-5 = type (0), bits 0-3 = reserved
/// - Bytes 1+: offset (max_heap_size bits, LE) then length (remaining bits, LE)
pub fn decode_managed_id(&self, id_bytes: &[u8]) -> Result<(u64, u64), FormatError> {
if id_bytes.is_empty() {
@@ -229,8 +301,8 @@ impl FractalHeapHeader {
});
}
let id_type = (id_bytes[0] >> 6) & 0x03;
if id_type != 0 {
let id_type = heap_id_type(id_bytes[0])?;
if id_type != HEAP_ID_MANAGED {
return Err(FormatError::InvalidHeapIdType(id_type));
}
@@ -269,12 +341,183 @@ impl FractalHeapHeader {
Ok((heap_offset, length_val))
}
/// Read a managed object from the heap given its raw heap ID bytes.
/// Read any object from the heap given its raw heap ID bytes: managed
/// (stored in the heap's blocks), huge (stored outside them, found
/// directly from the ID or through the huge-object v2 B-tree, optionally
/// filtered) or tiny (stored in the ID itself).
///
/// Despite its name this accepts every ID type; `offset_size` must match
/// the one the header was parsed with.
pub fn read_managed_object(
&self,
file_data: &[u8],
id_bytes: &[u8],
offset_size: u8,
) -> Result<Vec<u8>, FormatError> {
let Some(&first) = id_bytes.first() else {
return Err(FormatError::UnexpectedEof {
expected: 1,
available: 0,
});
};
match heap_id_type(first)? {
HEAP_ID_MANAGED => self.read_heap_managed(file_data, id_bytes, offset_size),
HEAP_ID_HUGE => self.read_huge_object(file_data, id_bytes),
HEAP_ID_TINY => self.read_tiny_object(id_bytes),
other => Err(FormatError::InvalidHeapIdType(other)),
}
}
/// Whether a huge object's ID holds its address and length directly
/// (libhdf5 does this when they fit in the ID), rather than a key into
/// the huge-object B-tree.
fn huge_ids_direct(&self) -> bool {
let room = usize::from(self.heap_id_length).saturating_sub(1);
let os = usize::from(self.offset_size);
let ls = usize::from(self.length_size);
if self.filter_pipeline.is_some() {
room >= os + ls + 4 + ls
} else {
room >= os + ls
}
}
/// Read a huge object (heap ID type 1).
fn read_huge_object(&self, file_data: &[u8], id: &[u8]) -> Result<Vec<u8>, FormatError> {
let os = usize::from(self.offset_size);
let ls = usize::from(self.length_size);
// (address, stored length, filter mask, decoded length); the last two
// only matter for a filtered heap.
let (addr, stored_len, mask, mem_len) = if self.huge_ids_direct() {
let body = &id[1..];
let need = if self.filter_pipeline.is_some() {
os + ls + 4 + ls
} else {
os + ls
};
ensure_len(body, 0, need)?;
let addr = le_uint(&body[..os]);
let len = le_uint(&body[os..os + ls]);
if self.filter_pipeline.is_some() {
let mask = u32::from_le_bytes([
body[os + ls],
body[os + ls + 1],
body[os + ls + 2],
body[os + ls + 3],
]);
let mem = le_uint(&body[os + ls + 4..os + ls + 4 + ls]);
(addr, len, mask, mem)
} else {
(addr, len, 0, len)
}
} else {
let key_len = (usize::from(self.heap_id_length).saturating_sub(1)).min(8);
ensure_len(id, 1, key_len)?;
let key = le_uint(&id[1..1 + key_len]);
self.find_huge_record(file_data, key)?
};
let start = usize::try_from(addr).map_err(|_| heap_error("huge object address"))?;
let len = usize::try_from(stored_len).map_err(|_| heap_error("huge object length"))?;
ensure_len(file_data, start, len)?;
let stored = &file_data[start..start + len];
match &self.filter_pipeline {
None => Ok(stored.to_vec()),
Some(pipeline) => {
let mem = usize::try_from(mem_len).map_err(|_| heap_error("huge object size"))?;
let out = crate::filters::decompress_chunk_masked(stored, pipeline, mem, 1, mask)?;
if out.len() != mem {
return Err(heap_error("filtered huge object decoded to the wrong size"));
}
Ok(out)
}
}
}
/// Look up huge object `key` in the huge-object v2 B-tree, returning
/// (address, stored length, filter mask, decoded length).
fn find_huge_record(
&self,
file_data: &[u8],
key: u64,
) -> Result<(u64, u64, u32, u64), FormatError> {
if is_undefined(self.huge_btree_address, self.offset_size) {
return Err(heap_error(
"huge object ID but the heap has no huge-object index",
));
}
let hdr = BTreeV2Header::parse(
file_data,
self.huge_btree_address as usize,
self.offset_size,
self.length_size,
)?;
let os = usize::from(self.offset_size);
let ls = usize::from(self.length_size);
let filtered = self.filter_pipeline.is_some();
let (expected_type, rec_len) = if filtered {
(BTREE_HUGE_INDIRECT_FILTERED, os + ls + 4 + ls + ls)
} else {
(BTREE_HUGE_INDIRECT, os + ls + ls)
};
if hdr.tree_type != expected_type || usize::from(hdr.record_size) < rec_len {
return Err(heap_error("unexpected huge-object B-tree record type"));
}
let records =
collect_btree_v2_records(file_data, &hdr, self.offset_size, self.length_size)?;
for rec in &records {
let d = &rec.data;
if d.len() < rec_len {
continue;
}
let addr = le_uint(&d[..os]);
let len = le_uint(&d[os..os + ls]);
if filtered {
let mask = u32::from_le_bytes([
d[os + ls],
d[os + ls + 1],
d[os + ls + 2],
d[os + ls + 3],
]);
let mem = le_uint(&d[os + ls + 4..os + 2 * ls + 4]);
let id = le_uint(&d[os + 2 * ls + 4..os + 3 * ls + 4]);
if id == key {
return Ok((addr, len, mask, mem));
}
} else {
let id = le_uint(&d[os + ls..os + 2 * ls]);
if id == key {
return Ok((addr, len, 0, len));
}
}
}
Err(heap_error("huge object not found in its B-tree"))
}
/// Read a tiny object (heap ID type 2), stored in the ID itself.
fn read_tiny_object(&self, id: &[u8]) -> Result<Vec<u8>, FormatError> {
// libhdf5 uses a one-byte length (low 4 bits of byte 0) unless the ID
// is long enough to need 12 bits, which then borrow byte 1.
let extended = usize::from(self.heap_id_length).saturating_sub(1) > 17;
let (len, start) = if extended {
ensure_len(id, 0, 2)?;
(
((usize::from(id[0] & 0x0F)) << 8 | usize::from(id[1])) + 1,
2,
)
} else {
(usize::from(id[0] & 0x0F) + 1, 1)
};
ensure_len(id, start, len)?;
Ok(id[start..start + len].to_vec())
}
/// Read a managed object (heap ID type 0).
fn read_heap_managed(
&self,
file_data: &[u8],
id_bytes: &[u8],
offset_size: u8,
) -> Result<Vec<u8>, FormatError> {
let (heap_offset, obj_len) = self.decode_managed_id(id_bytes)?;
@@ -289,12 +532,15 @@ impl FractalHeapHeader {
// Root is a direct block
self.read_from_direct_block(
file_data,
self.root_block_address as usize,
self.starting_block_size,
0, // block offset in heap = 0 for root
DirectBlock {
addr: self.root_block_address as usize,
size: self.starting_block_size,
heap_offset: 0,
filtered_size: self.root_direct_block_filtered_size,
filter_mask: self.root_direct_block_filter_mask,
},
heap_offset,
obj_len as usize,
offset_size,
)
} else {
// Root is an indirect block — limit recursion to 64 levels
@@ -313,27 +559,41 @@ impl FractalHeapHeader {
/// Read an object from a direct block.
///
/// The heap offset is relative to the start of the block (including its header),
/// so we just add it to the block address minus the block's heap offset.
#[allow(clippy::too_many_arguments)]
/// The heap offset is relative to the start of the block (including its
/// header), so we just add it to the block address minus the block's heap
/// offset. A filtered heap stores each direct block (header included)
/// through its filter pipeline, so the block is decoded first.
fn read_from_direct_block(
&self,
file_data: &[u8],
block_addr: usize,
_block_size: u64,
block_heap_offset: u64,
block: DirectBlock,
target_offset: u64,
length: usize,
_offset_size: u8,
) -> Result<Vec<u8>, FormatError> {
if target_offset < block_heap_offset {
if target_offset < block.heap_offset {
return Err(FormatError::UnexpectedEof {
expected: block_heap_offset as usize,
expected: block.heap_offset as usize,
available: target_offset as usize,
});
}
let local_offset = (target_offset - block_heap_offset) as usize;
let pos = block_addr
let local_offset = (target_offset - block.heap_offset) as usize;
if let Some(pipeline) = &self.filter_pipeline {
let stored_len = usize::try_from(block.filtered_size)
.map_err(|_| heap_error("direct block size"))?;
let size = usize::try_from(block.size).map_err(|_| heap_error("direct block size"))?;
ensure_len(file_data, block.addr, stored_len)?;
let decoded = crate::filters::decompress_chunk_masked(
&file_data[block.addr..block.addr + stored_len],
pipeline,
size,
1,
block.filter_mask,
)?;
ensure_len(&decoded, local_offset, length)?;
return Ok(decoded[local_offset..local_offset + length].to_vec());
}
let pos = block
.addr
.checked_add(local_offset)
.ok_or(FormatError::UnexpectedEof {
expected: usize::MAX,
@@ -371,19 +631,13 @@ impl FractalHeapHeader {
let iblock_header = 5 + offset_size as usize + block_offset_bytes;
let mut pos = iblock_addr + iblock_header;
// Compute block sizes for each row using the doubling table
let tw = self.table_width as u64;
let nrows_usize = nrows as usize;
// Build table of (block_size, heap_offset) for each child entry
let mut current_heap_offset = iblock_heap_offset;
// Rows below max_direct_rows hold direct blocks; rows at/above hold
// child indirect blocks. (NOT the FRHP "starting rows" field.)
let start_indirect = self.max_direct_rows();
// Read child addresses for direct block rows
let max_direct_rows = nrows_usize.min(start_indirect);
for row in 0..max_direct_rows {
@@ -393,60 +647,74 @@ impl FractalHeapHeader {
let child_addr = read_offset(file_data, pos, offset_size)?;
pos += offset_size as usize;
if self.io_filter_encoded_length > 0 {
// filtered_size(length_size) + filter_mask(4)
// Skip for now - we don't handle filtered direct blocks in fractal heaps
pos += 4; // filter_mask - simplified
}
// A filtered heap stores each direct block's filtered size
// (length_size) and filter mask (4) after its address.
let (filtered_size, filter_mask) = if self.filter_pipeline.is_some() {
let size = read_offset(file_data, pos, self.length_size)?;
pos += usize::from(self.length_size);
ensure_len(file_data, pos, 4)?;
let mask = u32::from_le_bytes([
file_data[pos],
file_data[pos + 1],
file_data[pos + 2],
file_data[pos + 3],
]);
pos += 4;
(size, mask)
} else {
(0, 0)
};
if !is_undefined(child_addr, offset_size) {
let block_end = current_heap_offset + block_size;
if target_offset >= current_heap_offset && target_offset < block_end {
return self.read_from_direct_block(
file_data,
child_addr as usize,
block_size,
current_heap_offset,
target_offset,
length,
offset_size,
);
}
let block_end = current_heap_offset.saturating_add(block_size);
if !is_undefined(child_addr, offset_size)
&& target_offset >= current_heap_offset
&& target_offset < block_end
{
return self.read_from_direct_block(
file_data,
DirectBlock {
addr: child_addr as usize,
size: block_size,
heap_offset: current_heap_offset,
filtered_size,
filter_mask,
},
target_offset,
length,
);
}
current_heap_offset += block_size;
current_heap_offset = block_end;
}
}
// If we have indirect block rows
// Rows at and above `start_indirect` hold child indirect blocks. A
// child in row r spans exactly that row's block size of heap space,
// so it has as many rows as a table of that total size needs.
for row in start_indirect..nrows_usize {
let _block_size = self.block_size_for_row(row);
let child_nrows = row - start_indirect + 1;
let child_space = self.block_size_for_row(row);
let child_nrows = self.rows_for_size(child_space);
for _col in 0..tw {
let child_addr = read_offset(file_data, pos, offset_size)?;
pos += offset_size as usize;
if !is_undefined(child_addr, offset_size) {
// Calculate total heap space covered by this indirect block child
let total_child_space = self.indirect_block_heap_size(child_nrows);
let block_end = current_heap_offset + total_child_space;
if target_offset >= current_heap_offset && target_offset < block_end {
return self.read_from_indirect_block(
file_data,
child_addr as usize,
child_nrows as u16,
current_heap_offset,
target_offset,
length,
offset_size,
depth_remaining - 1,
);
}
current_heap_offset += total_child_space;
} else {
let total_child_space = self.indirect_block_heap_size(child_nrows);
current_heap_offset += total_child_space;
let block_end = current_heap_offset.saturating_add(child_space);
if !is_undefined(child_addr, offset_size)
&& target_offset >= current_heap_offset
&& target_offset < block_end
{
return self.read_from_indirect_block(
file_data,
child_addr as usize,
child_nrows,
current_heap_offset,
target_offset,
length,
offset_size,
depth_remaining - 1,
);
}
current_heap_offset = block_end;
}
}
@@ -475,25 +743,34 @@ impl FractalHeapHeader {
log2 + 2
}
/// Rows an indirect block needs to span `size` bytes of heap space:
/// `log2(size) - log2(starting_block_size * table_width) + 1`, as
/// libhdf5's `H5HF__dtable_size_to_rows`.
fn rows_for_size(&self, size: u64) -> u16 {
let log2 = |v: u64| 63u32.saturating_sub(v.max(1).leading_zeros());
let first_row_bits = log2(self.starting_block_size) + log2(u64::from(self.table_width));
(log2(size).saturating_sub(first_row_bits) + 1) as u16
}
/// Get block size for a given row in the doubling table.
fn block_size_for_row(&self, row: usize) -> u64 {
let sbs = self.starting_block_size;
if row <= 1 {
sbs
} else {
sbs * (1u64 << (row - 1))
sbs.saturating_mul(1u64.checked_shl((row - 1) as u32).unwrap_or(u64::MAX))
}
}
}
/// Total heap space covered by an indirect block with the given number of rows.
fn indirect_block_heap_size(&self, nrows: usize) -> u64 {
let tw = self.table_width as u64;
let mut total = 0u64;
for row in 0..nrows {
total += self.block_size_for_row(row) * tw;
}
total
}
/// A managed direct block's location, extent and (for a filtered heap) its
/// stored size and filter mask.
struct DirectBlock {
addr: usize,
size: u64,
heap_offset: u64,
filtered_size: u64,
filter_mask: u32,
}
#[cfg(test)]
@@ -641,7 +918,7 @@ mod tests {
let hdr = FractalHeapHeader::parse(&file_data, 0, 8, 8).unwrap();
// Build a managed heap ID:
// byte 0: type=0 (bits 6-7 = 00), version=0 (bits 4-5), reserved (bits 0-3)
// byte 0: version=0 (bits 6-7), type=0 (bits 4-5), reserved (bits 0-3)
// bytes 1-6: offset (max_heap_size=16 bits) then length (remaining bits)
// For offset=0, length=13:
// payload = offset | (length << 16) = 0 | (13 << 16) = 0x000D0000
@@ -705,9 +982,46 @@ mod tests {
fn invalid_heap_id_type() {
let (file_data, _) = build_simple_heap(8, 8);
let hdr = FractalHeapHeader::parse(&file_data, 0, 8, 8).unwrap();
// Type = 1 (tiny) in bits 6-7
let id = vec![0x40u8, 0, 0, 0, 0, 0, 0]; // bit 6 set = type 1
// Type = 1 (huge) in bits 4-5 is not a managed ID
let id = vec![0x10u8, 0, 0, 0, 0, 0, 0];
let err = hdr.decode_managed_id(&id).unwrap_err();
assert_eq!(err, FormatError::InvalidHeapIdType(1));
}
#[test]
fn tiny_object_is_read_from_the_id() {
let (file_data, _) = build_simple_heap(8, 8);
let hdr = FractalHeapHeader::parse(&file_data, 0, 8, 8).unwrap();
// Type 2 (0x20), length - 1 in the low 4 bits, data after.
let id = [0x20 | 2, b'a', b'b', b'c', 0, 0, 0];
assert_eq!(hdr.read_managed_object(&file_data, &id, 8).unwrap(), b"abc");
// A length running past the ID is an error, not a short read.
let id = [0x20 | 9, b'a', b'b', b'c', 0, 0, 0];
assert!(hdr.read_managed_object(&file_data, &id, 8).is_err());
}
#[test]
fn huge_object_with_a_direct_id() {
// With IDs long enough for an address and a length, libhdf5 stores
// huge objects' location in the ID instead of the huge-object B-tree.
let (mut file_data, _) = build_simple_heap(8, 8);
let mut hdr = FractalHeapHeader::parse(&file_data, 0, 8, 8).unwrap();
hdr.heap_id_length = 17;
file_data[900..905].copy_from_slice(b"huge!");
let mut id = vec![0x10u8];
id.extend_from_slice(&900u64.to_le_bytes());
id.extend_from_slice(&5u64.to_le_bytes());
assert_eq!(
hdr.read_managed_object(&file_data, &id, 8).unwrap(),
b"huge!"
);
}
#[test]
fn unknown_heap_id_version_is_refused() {
let (file_data, _) = build_simple_heap(8, 8);
let hdr = FractalHeapHeader::parse(&file_data, 0, 8, 8).unwrap();
let id = [0x40u8, 0, 0, 0, 0, 0, 0];
assert!(hdr.read_managed_object(&file_data, &id, 8).is_err());
}
}
crates/clawhdf5-format/src/group_v1.rs
+60 -5
View File
@@ -80,6 +80,53 @@ pub fn find_v1_soft_link(
offset_size: u8,
length_size: u8,
) -> Result<Option<String>, FormatError> {
let mut found = None;
for_each_v1_soft_link(
file_data,
sym_table_msg,
offset_size,
length_size,
|link_name| link_name == name,
|_, target| {
found = Some(target);
false
},
)?;
Ok(found)
}
/// Every soft link in a v1 group, as `(name, target path)`.
pub fn v1_soft_links(
file_data: &[u8],
sym_table_msg: &SymbolTableMessage,
offset_size: u8,
length_size: u8,
) -> Result<Vec<(String, String)>, FormatError> {
let mut links = Vec::new();
for_each_v1_soft_link(
file_data,
sym_table_msg,
offset_size,
length_size,
|_| true,
|name, target| {
links.push((String::from(name), target));
true
},
)?;
Ok(links)
}
/// Visit the soft links of a v1 group whose name passes `wanted`, with their
/// target paths, until `visit` returns false.
fn for_each_v1_soft_link(
file_data: &[u8],
sym_table_msg: &SymbolTableMessage,
offset_size: u8,
length_size: u8,
wanted: impl Fn(&str) -> bool,
mut visit: impl FnMut(&str, String) -> bool,
) -> Result<(), FormatError> {
let heap = LocalHeap::parse(
file_data,
sym_table_msg.local_heap_address as usize,
@@ -103,7 +150,8 @@ pub fn find_v1_soft_link(
heap.validate_free_list(file_data, length_size)?;
heap_checked = true;
}
if heap.read_string(file_data, entry.link_name_offset)? != name {
let name = heap.read_string(file_data, entry.link_name_offset)?;
if !wanted(&name) {
continue;
}
let value_offset = u32::from_le_bytes([
@@ -112,12 +160,19 @@ pub fn find_v1_soft_link(
entry.scratch_pad[2],
entry.scratch_pad[3],
]);
return heap
.read_string(file_data, u64::from(value_offset))
.map(Some);
let target = heap.read_string(file_data, u64::from(value_offset))?;
if !visit(&name, target) {
return Ok(());
}
}
}
Ok(None)
Ok(())
}
/// Whether a v1 symbol-table entry is a soft link (no object header of its
/// own; its target path is in the local heap).
pub fn is_v1_soft_link(entry: &GroupEntry) -> bool {
entry.cache_type == CACHE_TYPE_SOFT_LINK
}
/// Extract the SymbolTableMessage from an object header's messages.
crates/clawhdf5-format/src/group_v2.rs
+145 -21
View File
@@ -38,6 +38,24 @@ pub fn resolve_v2_group_entries(
}
}
/// First user-defined link type (HDF5 reserves 2-63; 64 is external).
const FIRST_USER_DEFINED_LINK_TYPE: u8 = 65;
/// Parse a Link message, or `None` for a user-defined link (type 65-255).
///
/// A user-defined link's target is only meaningful to the application that
/// registered its class, so, like libhdf5 without that class, we cannot
/// follow it. Leaving it out lets the rest of the group be listed and
/// resolved instead of one such link failing the whole group; reserved
/// types (2-63) are still an error.
fn parse_link(data: &[u8], offset_size: u8) -> Result<Option<LinkMessage>, FormatError> {
match LinkMessage::parse(data, offset_size) {
Ok(link) => Ok(Some(link)),
Err(FormatError::InvalidLinkType(t)) if t >= FIRST_USER_DEFINED_LINK_TYPE => Ok(None),
Err(e) => Err(e),
}
}
/// Extract link entries from Link messages directly in the object header (compact storage).
fn resolve_compact_entries(
object_header: &ObjectHeader,
@@ -46,7 +64,9 @@ fn resolve_compact_entries(
let mut entries = Vec::new();
for msg in &object_header.messages {
if msg.msg_type == MessageType::Link {
let link = LinkMessage::parse(&msg.data, offset_size)?;
let Some(link) = parse_link(&msg.data, offset_size)? else {
continue;
};
if let LinkTarget::Hard {
object_header_address,
} = link.link_target
@@ -98,7 +118,9 @@ fn for_each_dense_link(
// Read managed object from fractal heap
let link_data = fh.read_managed_object(file_data, id_bytes, offset_size)?;
visit(LinkMessage::parse(&link_data, offset_size)?);
if let Some(link) = parse_link(&link_data, offset_size)? {
visit(link);
}
}
Ok(())
}
@@ -178,7 +200,9 @@ fn find_symbolic_link(
} else {
for msg in &object_header.messages {
if msg.msg_type == MessageType::Link {
let link = LinkMessage::parse(&msg.data, offset_size)?;
let Some(link) = parse_link(&msg.data, offset_size)? else {
continue;
};
if link.name == name && is_symbolic(&link.link_target) {
found = Some(link.link_target);
}
@@ -231,32 +255,135 @@ pub fn resolve_path_any(
superblock: &Superblock,
path: &str,
) -> Result<u64, FormatError> {
resolve_path_following_links(file_data, superblock, path, 0)
resolve_path_following_links(
file_data,
superblock,
superblock.root_group_address,
path,
0,
)
}
/// Resolve `path` relative to the group at `group_address` (an absolute path
/// starts at the root group instead), following soft links. This is how a
/// relative soft link's target is resolved: from the group holding the link.
pub fn resolve_path_from(
file_data: &[u8],
superblock: &Superblock,
group_address: u64,
path: &str,
) -> Result<u64, FormatError> {
let start = if path.starts_with('/') {
superblock.root_group_address
} else {
group_address
};
resolve_path_following_links(file_data, superblock, start, path, 0)
}
/// The children of the group at `group_address` that can be opened, as h5py
/// lists them: hard links, and soft links resolved to the object they point
/// at (under the soft link's own name). Links that cannot be followed are
/// left out rather than failing the listing — a dangling or cyclic soft link
/// (h5py lists its name but cannot open it), an external link (another
/// file), and a user-defined link. An object header that is not a group has
/// no children.
///
/// Any other error, such as a corrupt structure met while resolving a soft
/// link, is returned.
pub fn resolve_group_children(
file_data: &[u8],
superblock: &Superblock,
group_address: u64,
) -> Result<Vec<GroupEntry>, FormatError> {
let os = superblock.offset_size;
let ls = superblock.length_size;
let header = ObjectHeader::parse(file_data, group_address as usize, os, ls)?;
let mut entries = Vec::new();
let mut soft = Vec::new();
if is_v1_group(&header) {
let sym_msg = header
.messages
.iter()
.find(|m| m.msg_type == MessageType::SymbolTable)
.ok_or_else(|| FormatError::PathNotFound(String::from("no symbol table message")))?;
let stm = SymbolTableMessage::parse(&sym_msg.data, os)?;
let all = group_v1::resolve_v1_group_entries(file_data, &stm, os, ls)?;
if all.iter().any(group_v1::is_v1_soft_link) {
soft = group_v1::v1_soft_links(file_data, &stm, os, ls)?;
}
entries.extend(all.into_iter().filter(|e| !group_v1::is_v1_soft_link(e)));
} else if is_v2_group(&header) {
let mut visit = |link: LinkMessage| match link.link_target {
LinkTarget::Hard {
object_header_address,
} => entries.push(GroupEntry {
name: link.name,
object_header_address,
cache_type: 0,
}),
LinkTarget::Soft { target_path } => soft.push((link.name, target_path)),
LinkTarget::External { .. } => {}
};
let link_info = find_link_info(&header, os)?;
if let Some(fh_addr) = link_info.fractal_heap_address {
for_each_dense_link(file_data, &link_info, fh_addr, os, ls, visit)?;
} else {
for msg in &header.messages {
if msg.msg_type == MessageType::Link
&& let Some(link) = parse_link(&msg.data, os)?
{
visit(link);
}
}
}
}
for (name, target) in soft {
match resolve_path_from(file_data, superblock, group_address, &target) {
Ok(object_header_address) => entries.push(GroupEntry {
name,
object_header_address,
cache_type: 0,
}),
// Dangling, cyclic, or ending in another file: not openable here.
Err(
FormatError::PathNotFound(_)
| FormatError::NestingDepthExceeded
| FormatError::ExternalLinkUnsupported { .. },
) => {}
Err(e) => return Err(e),
}
}
Ok(entries)
}
/// Soft links followed while resolving one path. Guards against link cycles
/// (`a -> b -> a`), which are legal to create.
const MAX_SOFT_LINK_DEPTH: u8 = 16;
/// Walk `path` from the group at `start`, following soft links.
fn resolve_path_following_links(
file_data: &[u8],
superblock: &Superblock,
start: u64,
path: &str,
depth: u8,
) -> Result<u64, FormatError> {
let components: Vec<&str> = path.split('/').filter(|s| !s.is_empty()).collect();
let components: Vec<&str> = path
.split('/')
.filter(|s| !s.is_empty() && *s != ".")
.collect();
if components.is_empty() {
return Ok(superblock.root_group_address);
return Ok(start);
}
let os = superblock.offset_size;
let ls = superblock.length_size;
let root_header =
ObjectHeader::parse(file_data, superblock.root_group_address as usize, os, ls)?;
let mut current_addr = superblock.root_group_address;
let mut current_header = root_header;
let mut current_addr = start;
let mut current_header = ObjectHeader::parse(file_data, start as usize, os, ls)?;
for (i, component) in components.iter().enumerate() {
let entries = resolve_group_entries(file_data, &current_header, os, ls)?;
@@ -280,20 +407,17 @@ fn resolve_path_following_links(
}
// A relative target is relative to the group holding
// the link; then the rest of the original path.
let mut full = String::new();
if !target_path.starts_with('/') {
for parent in &components[..i] {
full.push('/');
full.push_str(parent);
}
}
full.push('/');
full.push_str(&target_path);
let from = if target_path.starts_with('/') {
superblock.root_group_address
} else {
current_addr
};
let mut full = target_path;
for rest in &components[i + 1..] {
full.push('/');
full.push_str(rest);
}
resolve_path_following_links(file_data, superblock, &full, depth + 1)
resolve_path_following_links(file_data, superblock, from, &full, depth + 1)
}
Some(LinkTarget::External {
filename,
crates/clawhdf5-format/src/shared_message.rs
+8 -2
View File
@@ -545,7 +545,8 @@ pub fn message_data<'a>(
///
/// For type 1/3 (shared in another object header), reads the target object header
/// and finds the message of the specified type.
/// For type 2 (SOHM), uses the fractal heap from the SOHM table.
/// For type 2 (SOHM), uses the fractal heap from the file's SOHM table,
/// loaded from the superblock extension on demand.
pub fn resolve_shared_message(
file_data: &[u8],
shared_ref: &SharedMessageRef,
@@ -553,13 +554,18 @@ pub fn resolve_shared_message(
offset_size: u8,
length_size: u8,
) -> Result<Vec<u8>, FormatError> {
let table = if shared_ref.heap_id.is_some() {
load_sohm_table(file_data, offset_size, length_size)?
} else {
None
};
resolve_shared_message_with_sohm(
file_data,
shared_ref,
target_msg_type,
offset_size,
length_size,
None,
table.as_ref(),
)
}