quantumclaw/clawhdf5
3
1
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 1 Wiki Activity

read: look names up through the dense name indexes

Browse Source 
Finding one link or attribute by name read every entry: Group::dataset and
Group::group (File, MmapFile, LazyFile) listed the whole group per call, and
path resolution scanned each group's links. Opening every child of a
35 001-link group by name decoded ~1.2e9 links.

Now a dense group's v2 B-tree name index (type 5, lookup3 hash of the
name) is descended to the records with the name's hash
(btree_v2::find_btree_v2_records reads only the nodes whose key interval
overlaps), and only those links are read and compared; all hash-equal
records are compared, so libhdf5's tie order does not matter. Dense
attributes the same through their type 8 index
(attribute::find_attribute_in_file, facade attr(name)); huge heap objects
through their ID-ordered index. group_v2::resolve_child returns what the
listing has under a name (soft links followed, dangling/external ones not
found). Group::entries and File::group_at hand out a listing's addresses.

The lookup-stats feature counts heap objects read. Tests: one lookup in
an h5py-written 35 001-link group with colliding hashes reads at most two
links (before: 35 001, failing), attribute lookups likewise (before: 3 000,
failing), every child opens through all three readers and matches h5py,
every link kind resolves as h5py resolves it in dense and compact groups,
300 huge attributes are found, and a range search matches a full scan at
every tree depth.

Co-Authored-By: Claude Opus 5.5 (1M context) <[email protected]>
This commit is contained in:
osobh
2026-09-26 13:33:19 -05:00
co-authored by Claude Opus 5.5
parent 6248b411f0
commit 02e89c1d2d
15 changed files with 1245 additions and 159 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/clawhdf5-format/src/attribute.rs
+148 -33
View File
@@ -5,8 +5,10 @@ use alloc::{borrow::Cow, string::String, vec::Vec};
#[cfg(feature = "std")]
use std::borrow::Cow;
use crate::addr::to_usize;
use crate::attribute_info::AttributeInfoMessage;
use crate::btree_v2::{BTreeV2Header, collect_btree_v2_records};
use crate::btree_v2::{BTreeV2Header, collect_btree_v2_records, find_btree_v2_records};
use crate::checksum::jenkins_lookup3;
use crate::data_read;
use crate::dataspace::Dataspace;
use crate::datatype::Datatype;
@@ -341,7 +343,8 @@ fn compute_raw_data(
dataspace: &Dataspace,
datatype: &Datatype,
) -> Vec<u8> {
let num_elements = dataspace.num_elements() as usize;
// Saturating, like the product: the size is capped at what is there.
let num_elements = usize::try_from(dataspace.num_elements()).unwrap_or(usize::MAX);
let elem_size = datatype.type_size() as usize;
let expected_size = num_elements.saturating_mul(elem_size);
let available = data.len().saturating_sub(pos);
@@ -453,7 +456,145 @@ fn extract_attributes_with(
// Each attribute's creation order, where the file records one.
let mut orders: Vec<u32> = Vec::new();
// Collect compact attributes (inline in OH)
extract_compact_attributes(
file_data,
header,
offset_size,
length_size,
&mut attrs,
&mut orders,
on_error,
)?;
// Check for dense attributes via AttributeInfo message
let attr_info = find_attribute_info(header, offset_size)?;
if let Some(info) = &attr_info
&& let Some(fh_addr) = info.fractal_heap_address
{
extract_dense_attributes(
file_data,
info,
fh_addr,
offset_size,
length_size,
&mut attrs,
&mut orders,
on_error,
)?;
}
// An object that tracks attribute creation order lists its attributes
// in that order (h5py's `track_order=True`), as libhdf5 does; otherwise
// they come in storage order.
if attr_info.is_some_and(|i| i.max_creation_index.is_some()) {
let mut paired: Vec<(u32, AttributeMessage)> = orders.into_iter().zip(attrs).collect();
paired.sort_by_key(|(o, _)| *o);
attrs = paired.into_iter().map(|(_, a)| a).collect();
}
Ok(attrs)
}
/// B-tree v2 record type of dense attribute storage's name index.
const ATTRIBUTE_NAME_INDEX: u8 = 8;
/// The attribute called `name` on the object with header `header`: the
/// first one [`extract_attributes_tolerant`] returns under that name, or
/// `None` if it returns none (an attribute that cannot be read is not
/// returned there either).
///
/// Compact attributes are in the header and are scanned. Dense attributes
/// are found through the name index (a v2 B-tree of lookup3 name hashes,
/// record type 8): only the attributes whose names hash like `name` are read
/// from the heap, O(log n) instead of all of them. Errors in the structures
/// that index the attributes fail the call, as they fail a listing.
pub fn find_attribute_in_file(
file_data: &[u8],
header: &ObjectHeader,
name: &str,
offset_size: u8,
length_size: u8,
) -> Result<Option<AttributeMessage>, FormatError> {
let attr_info = find_attribute_info(header, offset_size)?;
let dense = attr_info
.as_ref()
.and_then(|i| Some((i.fractal_heap_address?, i.btree_name_index_address?)));
let Some((fh_addr, btree_addr)) = dense else {
// Compact only (or dense storage without a name index, which a
// listing reports): as a listing finds it.
return Ok(
extract_attributes_tolerant(file_data, header, offset_size, length_size)?
.0
.into_iter()
.find(|a| a.name == name),
);
};
let btree_hdr =
BTreeV2Header::parse(file_data, to_usize(btree_addr)?, offset_size, length_size)?;
let fh = FractalHeapHeader::parse(file_data, to_usize(fh_addr)?, offset_size, length_size)?;
if btree_hdr.tree_type != ATTRIBUTE_NAME_INDEX || btree_hdr.record_size < 4 {
return Ok(
extract_attributes_tolerant(file_data, header, offset_size, length_size)?
.0
.into_iter()
.find(|a| a.name == name),
);
}
// A listing has the compact attributes first.
let mut compact = Vec::new();
extract_compact_attributes(
file_data,
header,
offset_size,
length_size,
&mut compact,
&mut Vec::new(),
&mut |_| Ok(()),
)?;
if let Some(a) = compact.into_iter().find(|a| a.name == name) {
return Ok(Some(a));
}
// Record: heap ID + message flags(1) + creation order(4) + hash(4); the
// hash is the last field.
let hash = jenkins_lookup3(name.as_bytes());
let hash_at = usize::from(btree_hdr.record_size) - 4;
let records = find_btree_v2_records(file_data, &btree_hdr, offset_size, &mut |r| match r
.get(hash_at..hash_at + 4)
{
Some(h) => u32::from_le_bytes([h[0], h[1], h[2], h[3]]).cmp(&hash),
None => core::cmp::Ordering::Less,
})?;
let id_len = usize::from(fh.heap_id_length);
for record in &records {
let Some(id_bytes) = record.data.get(..id_len) else {
continue;
};
let attr = fh
.read_managed_object(file_data, id_bytes, offset_size)
.and_then(|d| AttributeMessage::parse_in_file(&d, file_data, offset_size, length_size));
// One that cannot be read is left out, as from a listing.
if let Ok(attr) = attr
&& attr.name == name
{
return Ok(Some(attr));
}
}
Ok(None)
}
/// The attributes stored in the object header itself (compact storage), and
/// each one's creation order into `orders`.
fn extract_compact_attributes(
file_data: &[u8],
header: &ObjectHeader,
offset_size: u8,
length_size: u8,
attrs: &mut Vec<AttributeMessage>,
orders: &mut Vec<u32>,
on_error: &mut dyn FnMut(FormatError) -> Result<(), FormatError>,
) -> Result<(), FormatError> {
for msg in &header.messages {
if msg.msg_type == MessageType::Attribute {
let attr = if shared_message::is_shared(msg.flags) {
@@ -489,34 +630,7 @@ fn extract_attributes_with(
}
}
}
// Check for dense attributes via AttributeInfo message
let attr_info = find_attribute_info(header, offset_size)?;
if let Some(info) = &attr_info
&& let Some(fh_addr) = info.fractal_heap_address
{
extract_dense_attributes(
file_data,
info,
fh_addr,
offset_size,
length_size,
&mut attrs,
&mut orders,
on_error,
)?;
}
// An object that tracks attribute creation order lists its attributes
// in that order (h5py's `track_order=True`), as libhdf5 does; otherwise
// they come in storage order.
if attr_info.is_some_and(|i| i.max_creation_index.is_some()) {
let mut paired: Vec<(u32, AttributeMessage)> = orders.into_iter().zip(attrs).collect();
paired.sort_by_key(|(o, _)| *o);
attrs = paired.into_iter().map(|(_, a)| a).collect();
}
Ok(attrs)
Ok(())
}
/// Find and parse the Attribute Info message from an object header.
@@ -547,7 +661,7 @@ fn extract_dense_attributes(
on_error: &mut dyn FnMut(FormatError) -> Result<(), FormatError>,
) -> Result<(), FormatError> {
// Parse fractal heap
let fh = FractalHeapHeader::parse(file_data, fh_addr as usize, offset_size, length_size)?;
let fh = FractalHeapHeader::parse(file_data, to_usize(fh_addr)?, offset_size, length_size)?;
// Parse B-tree v2 for name index (type 8)
let btree_addr = attr_info
@@ -556,7 +670,8 @@ fn extract_dense_attributes(
expected: 1,
available: 0,
})?;
let btree_hdr = BTreeV2Header::parse(file_data, btree_addr as usize, offset_size, length_size)?;
let btree_hdr =
BTreeV2Header::parse(file_data, to_usize(btree_addr)?, offset_size, length_size)?;
let records = collect_btree_v2_records(file_data, &btree_hdr, offset_size, length_size)?;
for record in &records {
crates/clawhdf5-format/src/btree_v2.rs
+179 -37
View File
@@ -2,10 +2,12 @@
#[cfg(not(feature = "std"))]
use alloc::vec::Vec;
use core::cmp::Ordering;
#[cfg(feature = "checksum")]
use byteorder::{ByteOrder, LittleEndian};
use crate::addr::to_usize;
use crate::error::FormatError;
/// Parsed B-tree v2 header (signature "BTHD").
@@ -216,7 +218,7 @@ pub fn collect_btree_v2_records(
// Root is a leaf
parse_leaf_records(
file_data,
header.root_node_address as usize,
to_usize(header.root_node_address)?,
header.num_records_in_root,
header.record_size,
)
@@ -225,7 +227,7 @@ pub fn collect_btree_v2_records(
let mut records = Vec::new();
collect_internal_records(
file_data,
header.root_node_address as usize,
to_usize(header.root_node_address)?,
header.num_records_in_root,
header.depth,
header.record_size,
@@ -289,9 +291,10 @@ fn parse_leaf_records(
Ok(records)
}
/// Recursively collect records from an internal node.
#[allow(clippy::too_many_arguments, clippy::only_used_in_recursion)]
fn collect_internal_records(
/// An internal node's layout: where its records start, and its children as
/// `(address, record count)`.
#[allow(clippy::too_many_arguments)]
fn read_internal_node(
file_data: &[u8],
offset: usize,
num_records: u16,
@@ -299,11 +302,8 @@ fn collect_internal_records(
record_size: u16,
node_size: u32,
offset_size: u8,
length_size: u8,
max_leaf_nrec: u64,
budget: &mut usize,
out: &mut Vec<BTreeV2Record>,
) -> Result<(), FormatError> {
) -> Result<(usize, Vec<(u64, u16)>), FormatError> {
// signature(4) + version(1) + type(1) = 6
ensure_len(file_data, offset, 6)?;
if &file_data[offset..offset + 4] != b"BTIN" {
@@ -314,7 +314,7 @@ fn collect_internal_records(
let rs = record_size as usize;
let mut pos = offset + 6;
// Read all records first
// Records first
let records_total = nr.checked_mul(rs).ok_or(FormatError::UnexpectedEof {
expected: usize::MAX,
available: file_data.len(),
@@ -346,7 +346,6 @@ fn collect_internal_records(
let child_ptr_size = offset_size as usize + nrec_width + total_nrec_width;
ensure_len(file_data, pos, num_children * child_ptr_size)?;
// Read child pointers
let mut children = Vec::with_capacity(num_children);
for _ in 0..num_children {
let addr = read_offset(file_data, pos, offset_size)?;
@@ -356,6 +355,61 @@ fn collect_internal_records(
pos += total_nrec_width; // skip total records in subtree
children.push((addr, child_nrec));
}
Ok((records_start, children))
}
/// Record `i` of an internal node whose records start at `records_start`.
fn internal_record(
file_data: &[u8],
records_start: usize,
i: usize,
rs: usize,
) -> Result<&[u8], FormatError> {
let overflow = || FormatError::UnexpectedEof {
expected: usize::MAX,
available: file_data.len(),
};
let rec_start = i
.checked_mul(rs)
.and_then(|o| records_start.checked_add(o))
.ok_or_else(overflow)?;
let rec_end = rec_start.checked_add(rs).ok_or_else(overflow)?;
file_data
.get(rec_start..rec_end)
.ok_or(FormatError::UnexpectedEof {
expected: rec_end,
available: file_data.len(),
})
}
/// Recursively collect records from an internal node.
#[allow(clippy::too_many_arguments, clippy::only_used_in_recursion)]
fn collect_internal_records(
file_data: &[u8],
offset: usize,
num_records: u16,
depth: u16,
record_size: u16,
node_size: u32,
offset_size: u8,
length_size: u8,
max_leaf_nrec: u64,
budget: &mut usize,
out: &mut Vec<BTreeV2Record>,
) -> Result<(), FormatError> {
let nr = num_records as usize;
let rs = record_size as usize;
let (records_start, children) = read_internal_node(
file_data,
offset,
num_records,
depth,
record_size,
node_size,
offset_size,
max_leaf_nrec,
)?;
let child_depth = depth - 1;
// Interleave: child[0], record[0], child[1], record[1], ..., child[nr]
// We collect child[0] records, then record[0], then child[1], etc.
@@ -364,12 +418,12 @@ fn collect_internal_records(
// Before parsing, so a refused tree is not also a large allocation.
spend(budget, usize::from(child_nrec))?;
let leaf_recs =
parse_leaf_records(file_data, child_addr as usize, child_nrec, record_size)?;
parse_leaf_records(file_data, to_usize(child_addr)?, child_nrec, record_size)?;
out.extend(leaf_recs);
} else {
collect_internal_records(
file_data,
child_addr as usize,
to_usize(child_addr)?,
child_nrec,
child_depth,
record_size,
@@ -384,32 +438,10 @@ fn collect_internal_records(
// Add record[i] (except after the last child)
if i < nr {
let rec_offset = i.checked_mul(rs).ok_or(FormatError::UnexpectedEof {
expected: usize::MAX,
available: file_data.len(),
})?;
let rec_start =
records_start
.checked_add(rec_offset)
.ok_or(FormatError::UnexpectedEof {
expected: usize::MAX,
available: file_data.len(),
})?;
let rec_end = rec_start
.checked_add(rs)
.ok_or(FormatError::UnexpectedEof {
expected: usize::MAX,
available: file_data.len(),
})?;
if rec_end > file_data.len() {
return Err(FormatError::UnexpectedEof {
expected: rec_end,
available: file_data.len(),
});
}
let data = internal_record(file_data, records_start, i, rs)?;
spend(budget, 1)?;
out.push(BTreeV2Record {
data: file_data[rec_start..rec_end].to_vec(),
data: data.to_vec(),
});
}
}
@@ -417,6 +449,116 @@ fn collect_internal_records(
Ok(())
}
/// The records of a B-tree v2 that fall in one key range, found by
/// descending the tree instead of reading all of it.
///
/// `cmp` places a record relative to the range: `Less` if the record sorts
/// before it, `Greater` if after, `Equal` if the record is in it. The tree
/// must be ordered consistently with `cmp`, as libhdf5 orders it (a link or
/// attribute name index by name hash, so all records with one hash form a
/// range whatever order their names are in). Only the nodes whose key
/// interval overlaps the range are read: O(depth) nodes plus those holding
/// the matches. Matches come in tree order.
pub fn find_btree_v2_records(
file_data: &[u8],
header: &BTreeV2Header,
offset_size: u8,
cmp: &mut dyn FnMut(&[u8]) -> Ordering,
) -> Result<Vec<BTreeV2Record>, FormatError> {
if header.total_records == 0 || header.num_records_in_root == 0 {
return Ok(Vec::new());
}
if header.depth > MAX_DEPTH {
return Err(FormatError::NestingDepthExceeded);
}
// As in `collect_btree_v2_records`: a valid tree cannot hold more
// records than the file has room for, however its children are shared.
let mut budget = file_data.len() / usize::from(header.record_size.max(1));
let max_leaf_nrec = max_records_leaf(header.node_size, header.record_size);
let mut out = Vec::new();
find_in_node(
file_data,
header,
to_usize(header.root_node_address)?,
header.num_records_in_root,
header.depth,
offset_size,
max_leaf_nrec,
cmp,
&mut budget,
&mut out,
)?;
Ok(out)
}
#[allow(clippy::too_many_arguments)]
fn find_in_node(
file_data: &[u8],
header: &BTreeV2Header,
offset: usize,
num_records: u16,
depth: u16,
offset_size: u8,
max_leaf_nrec: u64,
cmp: &mut dyn FnMut(&[u8]) -> Ordering,
budget: &mut usize,
out: &mut Vec<BTreeV2Record>,
) -> Result<(), FormatError> {
spend(budget, usize::from(num_records))?;
if depth == 0 {
let records = parse_leaf_records(file_data, offset, num_records, header.record_size)?;
out.extend(
records
.into_iter()
.filter(|r| cmp(&r.data) == Ordering::Equal),
);
return Ok(());
}
let rs = usize::from(header.record_size);
let (records_start, children) = read_internal_node(
file_data,
offset,
num_records,
depth,
header.record_size,
header.node_size,
offset_size,
max_leaf_nrec,
)?;
let nr = usize::from(num_records);
let mut order = Vec::with_capacity(nr);
for i in 0..nr {
order.push(cmp(internal_record(file_data, records_start, i, rs)?));
}
// Child `i` holds the keys between record `i - 1` and record `i`: it can
// hold a match unless the record before it is already past the range or
// the record after it is still before it.
for (i, &(child_addr, child_nrec)) in children.iter().enumerate() {
let after_left = i == 0 || order[i - 1] != Ordering::Greater;
let before_right = i == nr || order[i] != Ordering::Less;
if after_left && before_right {
find_in_node(
file_data,
header,
to_usize(child_addr)?,
child_nrec,
depth - 1,
offset_size,
max_leaf_nrec,
cmp,
budget,
out,
)?;
}
if i < nr && order[i] == Ordering::Equal {
out.push(BTreeV2Record {
data: internal_record(file_data, records_start, i, rs)?.to_vec(),
});
}
}
Ok(())
}
/// Most records a subtree whose root is at `depth` can hold (libhdf5's
/// `cum_max_nrec`). See [`node_info`].
fn cum_max_records(
crates/clawhdf5-format/src/btree_v2_write.rs
+53
View File
@@ -385,6 +385,59 @@ mod tests {
assert!(nodes > 0);
}
/// Descending to a key range finds exactly the records a full read
/// holds in it — runs of equal keys that straddle node boundaries
/// included — at every depth, and nothing for keys not in the tree.
#[test]
fn a_key_range_search_matches_a_full_scan() {
use crate::btree_v2::find_btree_v2_records;
use core::cmp::Ordering;
let rs = 11usize;
// Keys 0, 0, 0, 2, 2, 2, 4, ...: runs of three, odd keys missing.
for n in [1usize, 45, 46, 1150, 30_000] {
let mut recs = Vec::with_capacity(n * rs);
for i in 0..n {
let mut r = vec![0u8; rs];
r[..8].copy_from_slice(&((i / 3 * 2) as u64).to_be_bytes());
r[8..].copy_from_slice(&[(i % 3) as u8, 0, 0]);
recs.extend_from_slice(&r);
}
let base = 4096u64;
let tree = build_btree_v2(params(512, 11), &recs, base, 8, 8).unwrap();
let mut file = vec![0u8; base as usize];
file.extend_from_slice(&tree);
let hdr = BTreeV2Header::parse(&file, base as usize, 8, 8).unwrap();
let all = collect_btree_v2_records(&file, &hdr, 8, 8).unwrap();
let key = |r: &[u8]| u64::from_be_bytes(r[..8].try_into().unwrap());
let last = key(&all[n - 1].data);
let probes = (0..=last + 1).step_by(if n > 1000 { 37 } else { 1 });
for k in probes.chain([last, last + 1, u64::MAX]) {
let found =
find_btree_v2_records(&file, &hdr, 8, &mut |r: &[u8]| key(r).cmp(&k)).unwrap();
let want: Vec<&[u8]> = all
.iter()
.map(|r| r.data.as_slice())
.filter(|r| key(r) == k)
.collect();
let got: Vec<&[u8]> = found.iter().map(|r| r.data.as_slice()).collect();
assert_eq!(got, want, "n {n} key {k}");
assert_eq!(
got.len(),
if k % 2 == 0 && k <= last {
want.len()
} else {
0
}
);
}
// Every record, or none, when the whole tree is in or out of range.
let every = find_btree_v2_records(&file, &hdr, 8, &mut |_| Ordering::Equal).unwrap();
assert_eq!(every.len(), n);
let none = find_btree_v2_records(&file, &hdr, 8, &mut |_| Ordering::Less).unwrap();
assert!(none.is_empty());
}
}
#[test]
fn a_node_too_small_or_too_big_is_an_error() {
assert!(build_btree_v2(params(16, 11), &records(1, 11), 0, 8, 8).is_err());
crates/clawhdf5-format/src/fractal_heap.rs
+20 -14
View File
@@ -6,7 +6,8 @@ use alloc::{format, vec::Vec};
#[cfg(feature = "checksum")]
use byteorder::{ByteOrder, LittleEndian};
use crate::btree_v2::{BTreeV2Header, collect_btree_v2_records};
use crate::addr::to_usize;
use crate::btree_v2::{BTreeV2Header, find_btree_v2_records};
use crate::error::FormatError;
use crate::filter_pipeline::FilterPipeline;
@@ -354,6 +355,7 @@ impl FractalHeapHeader {
id_bytes: &[u8],
offset_size: u8,
) -> Result<Vec<u8>, FormatError> {
crate::lookup_stats::heap_object_read();
let Some(&first) = id_bytes.first() else {
return Err(FormatError::UnexpectedEof {
expected: 1,
@@ -448,7 +450,7 @@ impl FractalHeapHeader {
}
let hdr = BTreeV2Header::parse(
file_data,
self.huge_btree_address as usize,
to_usize(self.huge_btree_address)?,
self.offset_size,
self.length_size,
)?;
@@ -463,8 +465,12 @@ impl FractalHeapHeader {
if hdr.tree_type != expected_type || usize::from(hdr.record_size) < rec_len {
return Err(heap_error("unexpected huge-object B-tree record type"));
}
let records =
collect_btree_v2_records(file_data, &hdr, self.offset_size, self.length_size)?;
// Records are ordered by ID (the last field): descend to the ones
// equal to `key` instead of reading the whole index.
let id_at = rec_len - ls;
let records = find_btree_v2_records(file_data, &hdr, self.offset_size, &mut |r| {
le_uint(&r[id_at..id_at + ls]).cmp(&key)
})?;
for rec in &records {
let d = &rec.data;
if d.len() < rec_len {
@@ -533,24 +539,24 @@ impl FractalHeapHeader {
self.read_from_direct_block(
file_data,
DirectBlock {
addr: self.root_block_address as usize,
addr: to_usize(self.root_block_address)?,
size: self.starting_block_size,
heap_offset: 0,
filtered_size: self.root_direct_block_filtered_size,
filter_mask: self.root_direct_block_filter_mask,
},
heap_offset,
obj_len as usize,
to_usize(obj_len)?,
)
} else {
// Root is an indirect block — limit recursion to 64 levels
self.read_from_indirect_block(
file_data,
self.root_block_address as usize,
to_usize(self.root_block_address)?,
self.current_rows_in_root_indirect_block,
0, // block offset
heap_offset,
obj_len as usize,
to_usize(obj_len)?,
offset_size,
64, // max recursion depth
)
@@ -572,11 +578,11 @@ impl FractalHeapHeader {
) -> Result<Vec<u8>, FormatError> {
if target_offset < block.heap_offset {
return Err(FormatError::UnexpectedEof {
expected: block.heap_offset as usize,
available: target_offset as usize,
expected: to_usize(block.heap_offset)?,
available: to_usize(target_offset)?,
});
}
let local_offset = (target_offset - block.heap_offset) as usize;
let local_offset = to_usize(target_offset - block.heap_offset)?;
if let Some(pipeline) = &self.filter_pipeline {
let stored_len = usize::try_from(block.filtered_size)
.map_err(|_| heap_error("direct block size"))?;
@@ -673,7 +679,7 @@ impl FractalHeapHeader {
return self.read_from_direct_block(
file_data,
DirectBlock {
addr: child_addr as usize,
addr: to_usize(child_addr)?,
size: block_size,
heap_offset: current_heap_offset,
filtered_size,
@@ -705,7 +711,7 @@ impl FractalHeapHeader {
{
return self.read_from_indirect_block(
file_data,
child_addr as usize,
to_usize(child_addr)?,
child_nrows,
current_heap_offset,
target_offset,
@@ -719,7 +725,7 @@ impl FractalHeapHeader {
}
Err(FormatError::UnexpectedEof {
expected: target_offset as usize + length,
expected: to_usize(target_offset)?.saturating_add(length),
available: file_data.len(),
})
}
crates/clawhdf5-format/src/group_v2.rs
+185 -32
View File
@@ -6,7 +6,9 @@
#[cfg(not(feature = "std"))]
use alloc::{string::String, vec::Vec};
use crate::btree_v2::{BTreeV2Header, collect_btree_v2_records};
use crate::addr::to_usize;
use crate::btree_v2::{BTreeV2Header, collect_btree_v2_records, find_btree_v2_records};
use crate::checksum::jenkins_lookup3;
use crate::error::FormatError;
use crate::fractal_heap::FractalHeapHeader;
use crate::group_v1::{self, GroupEntry};
@@ -93,13 +95,14 @@ fn for_each_dense_link(
mut visit: impl FnMut(LinkMessage),
) -> Result<(), FormatError> {
// Parse fractal heap
let fh = FractalHeapHeader::parse(file_data, fh_addr as usize, offset_size, length_size)?;
let fh = FractalHeapHeader::parse(file_data, to_usize(fh_addr)?, offset_size, length_size)?;
// Parse B-tree v2 for name index
let btree_addr = link_info
.btree_name_index_address
.ok_or_else(|| FormatError::PathNotFound(String::from("no B-tree v2 name index")))?;
let btree_hdr = BTreeV2Header::parse(file_data, btree_addr as usize, offset_size, length_size)?;
let btree_hdr =
BTreeV2Header::parse(file_data, to_usize(btree_addr)?, offset_size, length_size)?;
let records = collect_btree_v2_records(file_data, &btree_hdr, offset_size, length_size)?;
for record in &records {
@@ -181,10 +184,55 @@ fn find_symbolic_link(
if !is_v2_group(object_header) {
return Ok(None);
}
let is_symbolic = |t: &LinkTarget| !matches!(t, LinkTarget::Hard { .. });
// As when all links were scanned: the last symbolic link of that name.
Ok(
links_named(file_data, object_header, name, offset_size, length_size)?
.into_iter()
.rev()
.map(|link| link.link_target)
.find(|t| !matches!(t, LinkTarget::Hard { .. })),
)
}
/// B-tree v2 record type of a dense group's link name index.
const LINK_NAME_INDEX: u8 = 5;
/// The links called `name` in a v2 group (a valid group has at most one).
///
/// In dense storage the link name index (a v2 B-tree of lookup3 name
/// hashes, record type 5) is descended to the records with the name's hash,
/// and only their links are read from the heap — O(log n) instead of every
/// link. libhdf5 orders records with equal hashes by name; all of them are
/// read and compared here, so that order does not matter. An index of
/// another type is scanned in full.
fn links_named(
file_data: &[u8],
object_header: &ObjectHeader,
name: &str,
offset_size: u8,
length_size: u8,
) -> Result<Vec<LinkMessage>, FormatError> {
let mut found = Vec::new();
let link_info = find_link_info(object_header, offset_size)?;
let mut found = None;
if let Some(fh_addr) = link_info.fractal_heap_address {
let Some(fh_addr) = link_info.fractal_heap_address else {
for msg in &object_header.messages {
if msg.msg_type == MessageType::Link
&& let Some(link) = parse_link(&msg.data, offset_size)?
&& link.name == name
{
found.push(link);
}
}
return Ok(found);
};
let fh = FractalHeapHeader::parse(file_data, to_usize(fh_addr)?, offset_size, length_size)?;
let btree_addr = link_info
.btree_name_index_address
.ok_or_else(|| FormatError::PathNotFound(String::from("no B-tree v2 name index")))?;
let btree_hdr =
BTreeV2Header::parse(file_data, to_usize(btree_addr)?, offset_size, length_size)?;
if btree_hdr.tree_type != LINK_NAME_INDEX {
for_each_dense_link(
file_data,
&link_info,
@@ -192,26 +240,134 @@ fn find_symbolic_link(
offset_size,
length_size,
|link| {
if link.name == name && is_symbolic(&link.link_target) {
found = Some(link.link_target);
if link.name == name {
found.push(link);
}
},
)?;
} else {
for msg in &object_header.messages {
if msg.msg_type == MessageType::Link {
let Some(link) = parse_link(&msg.data, offset_size)? else {
continue;
};
if link.name == name && is_symbolic(&link.link_target) {
found = Some(link.link_target);
}
}
return Ok(found);
}
// Record: hash(4) + heap ID.
let hash = jenkins_lookup3(name.as_bytes());
let records = find_btree_v2_records(file_data, &btree_hdr, offset_size, &mut |r| {
match r.get(..4) {
Some(h) => u32::from_le_bytes([h[0], h[1], h[2], h[3]]).cmp(&hash),
// Too short to hold a hash (a corrupt record size): never a match.
None => core::cmp::Ordering::Less,
}
})?;
let id_len = usize::from(fh.heap_id_length);
for record in &records {
let Some(id_bytes) = record.data.get(4..4 + id_len) else {
continue;
};
let link_data = fh.read_managed_object(file_data, id_bytes, offset_size)?;
if let Some(link) = parse_link(&link_data, offset_size)?
&& link.name == name
{
found.push(link);
}
}
Ok(found)
}
/// The link [`resolve_path_any`] follows for one path component `name` of
/// the group with header `object_header`: a hard link (as `Hard`), else a
/// soft or external link of that name, else `None`. Fails with
/// `PathNotFound` if the object is not a group.
fn lookup_link(
file_data: &[u8],
object_header: &ObjectHeader,
name: &str,
offset_size: u8,
length_size: u8,
) -> Result<Option<LinkTarget>, FormatError> {
if is_v1_group(object_header) {
let entries = resolve_group_entries(file_data, object_header, offset_size, length_size)?;
if let Some(e) = entries
.iter()
.find(|e| e.name == name && e.object_header_address != u64::MAX)
{
return Ok(Some(LinkTarget::Hard {
object_header_address: e.object_header_address,
}));
}
return find_symbolic_link(file_data, object_header, name, offset_size, length_size);
}
if !is_v2_group(object_header) {
return Err(FormatError::PathNotFound(String::from(
"object header is not a group",
)));
}
let links = links_named(file_data, object_header, name, offset_size, length_size)?;
if let Some(addr) = links.iter().find_map(|l| match l.link_target {
LinkTarget::Hard {
object_header_address,
} if object_header_address != u64::MAX => Some(object_header_address),
_ => None,
}) {
return Ok(Some(LinkTarget::Hard {
object_header_address: addr,
}));
}
Ok(links
.into_iter()
.rev()
.map(|link| link.link_target)
.find(|t| !matches!(t, LinkTarget::Hard { .. })))
}
/// The object header address of the child called `name` of the group at
/// `group_address`: the address [`resolve_group_children`] lists under that
/// name, or `PathNotFound` if it lists none.
///
/// A dense group's child is found through its link name index (see
/// [`links_named`]) and only the named link is read and, if it is a soft
/// link, followed — not every link in the group. A v1 group is listed.
pub fn resolve_child(
file_data: &[u8],
superblock: &Superblock,
group_address: u64,
name: &str,
) -> Result<u64, FormatError> {
let os = superblock.offset_size;
let ls = superblock.length_size;
let not_found = || FormatError::PathNotFound(String::from(name));
let header = ObjectHeader::parse(file_data, to_usize(group_address)?, os, ls)?;
if !is_v2_group(&header) || is_v1_group(&header) {
return resolve_group_children(file_data, superblock, group_address)?
.into_iter()
.find(|e| e.name == name)
.map(|e| e.object_header_address)
.ok_or_else(not_found);
}
let links = links_named(file_data, &header, name, os, ls)?;
// The listing puts hard links before resolved soft links.
if let Some(addr) = links.iter().find_map(|l| match l.link_target {
LinkTarget::Hard {
object_header_address,
} => Some(object_header_address),
_ => None,
}) {
return Ok(addr);
}
for link in links {
if let LinkTarget::Soft { target_path } = link.link_target {
return match resolve_path_from(file_data, superblock, group_address, &target_path) {
// Left out of the listing: dangling, cyclic, or in another file.
Err(
FormatError::PathNotFound(_)
| FormatError::NestingDepthExceeded
| FormatError::ExternalLinkUnsupported { .. },
) => Err(not_found()),
other => other,
};
}
}
Err(not_found())
}
/// Find and parse the Link Info message from an object header.
fn find_link_info(
object_header: &ObjectHeader,
@@ -298,7 +454,7 @@ pub fn resolve_group_children(
) -> Result<Vec<GroupEntry>, FormatError> {
let os = superblock.offset_size;
let ls = superblock.length_size;
let header = ObjectHeader::parse(file_data, group_address as usize, os, ls)?;
let header = ObjectHeader::parse(file_data, to_usize(group_address)?, os, ls)?;
let mut entries = Vec::new();
let mut soft = Vec::new();
@@ -383,24 +539,21 @@ fn resolve_path_following_links(
let ls = superblock.length_size;
let mut current_addr = start;
let mut current_header = ObjectHeader::parse(file_data, start as usize, os, ls)?;
let mut current_header = ObjectHeader::parse(file_data, to_usize(start)?, os, ls)?;
for (i, component) in components.iter().enumerate() {
let entries = resolve_group_entries(file_data, &current_header, os, ls)?;
let found = entries
.iter()
.find(|e| e.name == *component && e.object_header_address != u64::MAX);
match found {
Some(entry) => {
match lookup_link(file_data, &current_header, component, os, ls)? {
Some(LinkTarget::Hard {
object_header_address,
}) => {
if i == components.len() - 1 {
return Ok(entry.object_header_address);
return Ok(object_header_address);
}
current_addr = entry.object_header_address;
current_header = ObjectHeader::parse(file_data, current_addr as usize, os, ls)?;
current_addr = object_header_address;
current_header = ObjectHeader::parse(file_data, to_usize(current_addr)?, os, ls)?;
}
None => {
return match find_symbolic_link(file_data, &current_header, component, os, ls)? {
found => {
return match found {
Some(LinkTarget::Soft { target_path }) => {
if depth >= MAX_SOFT_LINK_DEPTH {
return Err(FormatError::NestingDepthExceeded);
crates/clawhdf5-format/src/lib.rs
+1
View File
@@ -108,6 +108,7 @@ pub mod lane_partition;
pub mod link_info;
pub mod link_message;
pub mod local_heap;
pub mod lookup_stats;
pub mod message_type;
pub mod metadata_cache;
pub mod metadata_index;
crates/clawhdf5-format/src/lookup_stats.rs
+31
View File
@@ -0,0 +1,31 @@
//! Work counters for tests of lookup cost (feature `lookup-stats`).
//!
//! Counts fractal-heap objects read — each is one link or attribute message
//! decoded out of a dense group or dense attribute storage — so a test can
//! check that finding one name reads a handful of them, not the whole group.
//! Per thread, so tests running in parallel do not see each other's reads.
//! Without the feature the counting compiles to nothing.
#[cfg(feature = "lookup-stats")]
std::thread_local! {
static HEAP_OBJECTS: core::cell::Cell<u64> = const { core::cell::Cell::new(0) };
}
/// Record one heap object read.
#[inline(always)]
pub(crate) fn heap_object_read() {
#[cfg(feature = "lookup-stats")]
HEAP_OBJECTS.with(|c| c.set(c.get() + 1));
}
/// Heap objects read on this thread since the last [`reset`].
#[cfg(feature = "lookup-stats")]
pub fn heap_objects_read() -> u64 {
HEAP_OBJECTS.with(core::cell::Cell::get)
}
/// Zero this thread's counters.
#[cfg(feature = "lookup-stats")]
pub fn reset() {
HEAP_OBJECTS.with(|c| c.set(0));
}