quantumclaw/clawhdf5
3
1
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 1 Wiki Activity

read: look names up through the dense name indexes

Browse Source 
Finding one link or attribute by name read every entry: Group::dataset and
Group::group (File, MmapFile, LazyFile) listed the whole group per call, and
path resolution scanned each group's links. Opening every child of a
35 001-link group by name decoded ~1.2e9 links.

Now a dense group's v2 B-tree name index (type 5, lookup3 hash of the
name) is descended to the records with the name's hash
(btree_v2::find_btree_v2_records reads only the nodes whose key interval
overlaps), and only those links are read and compared; all hash-equal
records are compared, so libhdf5's tie order does not matter. Dense
attributes the same through their type 8 index
(attribute::find_attribute_in_file, facade attr(name)); huge heap objects
through their ID-ordered index. group_v2::resolve_child returns what the
listing has under a name (soft links followed, dangling/external ones not
found). Group::entries and File::group_at hand out a listing's addresses.

The lookup-stats feature counts heap objects read. Tests: one lookup in
an h5py-written 35 001-link group with colliding hashes reads at most two
links (before: 35 001, failing), attribute lookups likewise (before: 3 000,
failing), every child opens through all three readers and matches h5py,
every link kind resolves as h5py resolves it in dense and compact groups,
300 huge attributes are found, and a range search matches a full scan at
every tree depth.

Co-Authored-By: Claude Opus 5.5 (1M context) <[email protected]>
This commit is contained in:
osobh
2026-09-26 13:33:19 -05:00
co-authored by Claude Opus 5.5
parent 6248b411f0
commit 02e89c1d2d
15 changed files with 1245 additions and 159 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/clawhdf5-format/src/attribute.rs
+148 -33
View File
@@ -5,8 +5,10 @@ use alloc::{borrow::Cow, string::String, vec::Vec};
#[cfg(feature = "std")]
use std::borrow::Cow;
use crate::addr::to_usize;
use crate::attribute_info::AttributeInfoMessage;
use crate::btree_v2::{BTreeV2Header, collect_btree_v2_records};
use crate::btree_v2::{BTreeV2Header, collect_btree_v2_records, find_btree_v2_records};
use crate::checksum::jenkins_lookup3;
use crate::data_read;
use crate::dataspace::Dataspace;
use crate::datatype::Datatype;
@@ -341,7 +343,8 @@ fn compute_raw_data(
dataspace: &Dataspace,
datatype: &Datatype,
) -> Vec<u8> {
let num_elements = dataspace.num_elements() as usize;
// Saturating, like the product: the size is capped at what is there.
let num_elements = usize::try_from(dataspace.num_elements()).unwrap_or(usize::MAX);
let elem_size = datatype.type_size() as usize;
let expected_size = num_elements.saturating_mul(elem_size);
let available = data.len().saturating_sub(pos);
@@ -453,7 +456,145 @@ fn extract_attributes_with(
// Each attribute's creation order, where the file records one.
let mut orders: Vec<u32> = Vec::new();
// Collect compact attributes (inline in OH)
extract_compact_attributes(
file_data,
header,
offset_size,
length_size,
&mut attrs,
&mut orders,
on_error,
)?;
// Check for dense attributes via AttributeInfo message
let attr_info = find_attribute_info(header, offset_size)?;
if let Some(info) = &attr_info
&& let Some(fh_addr) = info.fractal_heap_address
{
extract_dense_attributes(
file_data,
info,
fh_addr,
offset_size,
length_size,
&mut attrs,
&mut orders,
on_error,
)?;
}
// An object that tracks attribute creation order lists its attributes
// in that order (h5py's `track_order=True`), as libhdf5 does; otherwise
// they come in storage order.
if attr_info.is_some_and(|i| i.max_creation_index.is_some()) {
let mut paired: Vec<(u32, AttributeMessage)> = orders.into_iter().zip(attrs).collect();
paired.sort_by_key(|(o, _)| *o);
attrs = paired.into_iter().map(|(_, a)| a).collect();
}
Ok(attrs)
}
/// B-tree v2 record type of dense attribute storage's name index.
const ATTRIBUTE_NAME_INDEX: u8 = 8;
/// The attribute called `name` on the object with header `header`: the
/// first one [`extract_attributes_tolerant`] returns under that name, or
/// `None` if it returns none (an attribute that cannot be read is not
/// returned there either).
///
/// Compact attributes are in the header and are scanned. Dense attributes
/// are found through the name index (a v2 B-tree of lookup3 name hashes,
/// record type 8): only the attributes whose names hash like `name` are read
/// from the heap, O(log n) instead of all of them. Errors in the structures
/// that index the attributes fail the call, as they fail a listing.
pub fn find_attribute_in_file(
file_data: &[u8],
header: &ObjectHeader,
name: &str,
offset_size: u8,
length_size: u8,
) -> Result<Option<AttributeMessage>, FormatError> {
let attr_info = find_attribute_info(header, offset_size)?;
let dense = attr_info
.as_ref()
.and_then(|i| Some((i.fractal_heap_address?, i.btree_name_index_address?)));
let Some((fh_addr, btree_addr)) = dense else {
// Compact only (or dense storage without a name index, which a
// listing reports): as a listing finds it.
return Ok(
extract_attributes_tolerant(file_data, header, offset_size, length_size)?
.0
.into_iter()
.find(|a| a.name == name),
);
};
let btree_hdr =
BTreeV2Header::parse(file_data, to_usize(btree_addr)?, offset_size, length_size)?;
let fh = FractalHeapHeader::parse(file_data, to_usize(fh_addr)?, offset_size, length_size)?;
if btree_hdr.tree_type != ATTRIBUTE_NAME_INDEX || btree_hdr.record_size < 4 {
return Ok(
extract_attributes_tolerant(file_data, header, offset_size, length_size)?
.0
.into_iter()
.find(|a| a.name == name),
);
}
// A listing has the compact attributes first.
let mut compact = Vec::new();
extract_compact_attributes(
file_data,
header,
offset_size,
length_size,
&mut compact,
&mut Vec::new(),
&mut |_| Ok(()),
)?;
if let Some(a) = compact.into_iter().find(|a| a.name == name) {
return Ok(Some(a));
}
// Record: heap ID + message flags(1) + creation order(4) + hash(4); the
// hash is the last field.
let hash = jenkins_lookup3(name.as_bytes());
let hash_at = usize::from(btree_hdr.record_size) - 4;
let records = find_btree_v2_records(file_data, &btree_hdr, offset_size, &mut |r| match r
.get(hash_at..hash_at + 4)
{
Some(h) => u32::from_le_bytes([h[0], h[1], h[2], h[3]]).cmp(&hash),
None => core::cmp::Ordering::Less,
})?;
let id_len = usize::from(fh.heap_id_length);
for record in &records {
let Some(id_bytes) = record.data.get(..id_len) else {
continue;
};
let attr = fh
.read_managed_object(file_data, id_bytes, offset_size)
.and_then(|d| AttributeMessage::parse_in_file(&d, file_data, offset_size, length_size));
// One that cannot be read is left out, as from a listing.
if let Ok(attr) = attr
&& attr.name == name
{
return Ok(Some(attr));
}
}
Ok(None)
}
/// The attributes stored in the object header itself (compact storage), and
/// each one's creation order into `orders`.
fn extract_compact_attributes(
file_data: &[u8],
header: &ObjectHeader,
offset_size: u8,
length_size: u8,
attrs: &mut Vec<AttributeMessage>,
orders: &mut Vec<u32>,
on_error: &mut dyn FnMut(FormatError) -> Result<(), FormatError>,
) -> Result<(), FormatError> {
for msg in &header.messages {
if msg.msg_type == MessageType::Attribute {
let attr = if shared_message::is_shared(msg.flags) {
@@ -489,34 +630,7 @@ fn extract_attributes_with(
}
}
}
// Check for dense attributes via AttributeInfo message
let attr_info = find_attribute_info(header, offset_size)?;
if let Some(info) = &attr_info
&& let Some(fh_addr) = info.fractal_heap_address
{
extract_dense_attributes(
file_data,
info,
fh_addr,
offset_size,
length_size,
&mut attrs,
&mut orders,
on_error,
)?;
}
// An object that tracks attribute creation order lists its attributes
// in that order (h5py's `track_order=True`), as libhdf5 does; otherwise
// they come in storage order.
if attr_info.is_some_and(|i| i.max_creation_index.is_some()) {
let mut paired: Vec<(u32, AttributeMessage)> = orders.into_iter().zip(attrs).collect();
paired.sort_by_key(|(o, _)| *o);
attrs = paired.into_iter().map(|(_, a)| a).collect();
}
Ok(attrs)
Ok(())
}
/// Find and parse the Attribute Info message from an object header.
@@ -547,7 +661,7 @@ fn extract_dense_attributes(
on_error: &mut dyn FnMut(FormatError) -> Result<(), FormatError>,
) -> Result<(), FormatError> {
// Parse fractal heap
let fh = FractalHeapHeader::parse(file_data, fh_addr as usize, offset_size, length_size)?;
let fh = FractalHeapHeader::parse(file_data, to_usize(fh_addr)?, offset_size, length_size)?;
// Parse B-tree v2 for name index (type 8)
let btree_addr = attr_info
@@ -556,7 +670,8 @@ fn extract_dense_attributes(
expected: 1,
available: 0,
})?;
let btree_hdr = BTreeV2Header::parse(file_data, btree_addr as usize, offset_size, length_size)?;
let btree_hdr =
BTreeV2Header::parse(file_data, to_usize(btree_addr)?, offset_size, length_size)?;
let records = collect_btree_v2_records(file_data, &btree_hdr, offset_size, length_size)?;
for record in &records {