quantumclaw/clawhdf5
3
1
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 1 Wiki Activity

read: look names up through the dense name indexes

Browse Source 
Finding one link or attribute by name read every entry: Group::dataset and
Group::group (File, MmapFile, LazyFile) listed the whole group per call, and
path resolution scanned each group's links. Opening every child of a
35 001-link group by name decoded ~1.2e9 links.

Now a dense group's v2 B-tree name index (type 5, lookup3 hash of the
name) is descended to the records with the name's hash
(btree_v2::find_btree_v2_records reads only the nodes whose key interval
overlaps), and only those links are read and compared; all hash-equal
records are compared, so libhdf5's tie order does not matter. Dense
attributes the same through their type 8 index
(attribute::find_attribute_in_file, facade attr(name)); huge heap objects
through their ID-ordered index. group_v2::resolve_child returns what the
listing has under a name (soft links followed, dangling/external ones not
found). Group::entries and File::group_at hand out a listing's addresses.

The lookup-stats feature counts heap objects read. Tests: one lookup in
an h5py-written 35 001-link group with colliding hashes reads at most two
links (before: 35 001, failing), attribute lookups likewise (before: 3 000,
failing), every child opens through all three readers and matches h5py,
every link kind resolves as h5py resolves it in dense and compact groups,
300 huge attributes are found, and a range search matches a full scan at
every tree depth.

Co-Authored-By: Claude Opus 5.5 (1M context) <[email protected]>
This commit is contained in:
osobh
2026-09-26 13:33:19 -05:00
co-authored by Claude Opus 5.5
parent 6248b411f0
commit 02e89c1d2d
15 changed files with 1245 additions and 159 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/clawhdf5-format/src/group_v2.rs
+185 -32
View File
@@ -6,7 +6,9 @@
#[cfg(not(feature = "std"))]
use alloc::{string::String, vec::Vec};
use crate::btree_v2::{BTreeV2Header, collect_btree_v2_records};
use crate::addr::to_usize;
use crate::btree_v2::{BTreeV2Header, collect_btree_v2_records, find_btree_v2_records};
use crate::checksum::jenkins_lookup3;
use crate::error::FormatError;
use crate::fractal_heap::FractalHeapHeader;
use crate::group_v1::{self, GroupEntry};
@@ -93,13 +95,14 @@ fn for_each_dense_link(
mut visit: impl FnMut(LinkMessage),
) -> Result<(), FormatError> {
// Parse fractal heap
let fh = FractalHeapHeader::parse(file_data, fh_addr as usize, offset_size, length_size)?;
let fh = FractalHeapHeader::parse(file_data, to_usize(fh_addr)?, offset_size, length_size)?;
// Parse B-tree v2 for name index
let btree_addr = link_info
.btree_name_index_address
.ok_or_else(|| FormatError::PathNotFound(String::from("no B-tree v2 name index")))?;
let btree_hdr = BTreeV2Header::parse(file_data, btree_addr as usize, offset_size, length_size)?;
let btree_hdr =
BTreeV2Header::parse(file_data, to_usize(btree_addr)?, offset_size, length_size)?;
let records = collect_btree_v2_records(file_data, &btree_hdr, offset_size, length_size)?;
for record in &records {
@@ -181,10 +184,55 @@ fn find_symbolic_link(
if !is_v2_group(object_header) {
return Ok(None);
}
let is_symbolic = |t: &LinkTarget| !matches!(t, LinkTarget::Hard { .. });
// As when all links were scanned: the last symbolic link of that name.
Ok(
links_named(file_data, object_header, name, offset_size, length_size)?
.into_iter()
.rev()
.map(|link| link.link_target)
.find(|t| !matches!(t, LinkTarget::Hard { .. })),
)
}
/// B-tree v2 record type of a dense group's link name index.
const LINK_NAME_INDEX: u8 = 5;
/// The links called `name` in a v2 group (a valid group has at most one).
///
/// In dense storage the link name index (a v2 B-tree of lookup3 name
/// hashes, record type 5) is descended to the records with the name's hash,
/// and only their links are read from the heap — O(log n) instead of every
/// link. libhdf5 orders records with equal hashes by name; all of them are
/// read and compared here, so that order does not matter. An index of
/// another type is scanned in full.
fn links_named(
file_data: &[u8],
object_header: &ObjectHeader,
name: &str,
offset_size: u8,
length_size: u8,
) -> Result<Vec<LinkMessage>, FormatError> {
let mut found = Vec::new();
let link_info = find_link_info(object_header, offset_size)?;
let mut found = None;
if let Some(fh_addr) = link_info.fractal_heap_address {
let Some(fh_addr) = link_info.fractal_heap_address else {
for msg in &object_header.messages {
if msg.msg_type == MessageType::Link
&& let Some(link) = parse_link(&msg.data, offset_size)?
&& link.name == name
{
found.push(link);
}
}
return Ok(found);
};
let fh = FractalHeapHeader::parse(file_data, to_usize(fh_addr)?, offset_size, length_size)?;
let btree_addr = link_info
.btree_name_index_address
.ok_or_else(|| FormatError::PathNotFound(String::from("no B-tree v2 name index")))?;
let btree_hdr =
BTreeV2Header::parse(file_data, to_usize(btree_addr)?, offset_size, length_size)?;
if btree_hdr.tree_type != LINK_NAME_INDEX {
for_each_dense_link(
file_data,
&link_info,
@@ -192,26 +240,134 @@ fn find_symbolic_link(
offset_size,
length_size,
|link| {
if link.name == name && is_symbolic(&link.link_target) {
found = Some(link.link_target);
if link.name == name {
found.push(link);
}
},
)?;
} else {
for msg in &object_header.messages {
if msg.msg_type == MessageType::Link {
let Some(link) = parse_link(&msg.data, offset_size)? else {
continue;
};
if link.name == name && is_symbolic(&link.link_target) {
found = Some(link.link_target);
}
}
return Ok(found);
}
// Record: hash(4) + heap ID.
let hash = jenkins_lookup3(name.as_bytes());
let records = find_btree_v2_records(file_data, &btree_hdr, offset_size, &mut |r| {
match r.get(..4) {
Some(h) => u32::from_le_bytes([h[0], h[1], h[2], h[3]]).cmp(&hash),
// Too short to hold a hash (a corrupt record size): never a match.
None => core::cmp::Ordering::Less,
}
})?;
let id_len = usize::from(fh.heap_id_length);
for record in &records {
let Some(id_bytes) = record.data.get(4..4 + id_len) else {
continue;
};
let link_data = fh.read_managed_object(file_data, id_bytes, offset_size)?;
if let Some(link) = parse_link(&link_data, offset_size)?
&& link.name == name
{
found.push(link);
}
}
Ok(found)
}
/// The link [`resolve_path_any`] follows for one path component `name` of
/// the group with header `object_header`: a hard link (as `Hard`), else a
/// soft or external link of that name, else `None`. Fails with
/// `PathNotFound` if the object is not a group.
fn lookup_link(
file_data: &[u8],
object_header: &ObjectHeader,
name: &str,
offset_size: u8,
length_size: u8,
) -> Result<Option<LinkTarget>, FormatError> {
if is_v1_group(object_header) {
let entries = resolve_group_entries(file_data, object_header, offset_size, length_size)?;
if let Some(e) = entries
.iter()
.find(|e| e.name == name && e.object_header_address != u64::MAX)
{
return Ok(Some(LinkTarget::Hard {
object_header_address: e.object_header_address,
}));
}
return find_symbolic_link(file_data, object_header, name, offset_size, length_size);
}
if !is_v2_group(object_header) {
return Err(FormatError::PathNotFound(String::from(
"object header is not a group",
)));
}
let links = links_named(file_data, object_header, name, offset_size, length_size)?;
if let Some(addr) = links.iter().find_map(|l| match l.link_target {
LinkTarget::Hard {
object_header_address,
} if object_header_address != u64::MAX => Some(object_header_address),
_ => None,
}) {
return Ok(Some(LinkTarget::Hard {
object_header_address: addr,
}));
}
Ok(links
.into_iter()
.rev()
.map(|link| link.link_target)
.find(|t| !matches!(t, LinkTarget::Hard { .. })))
}
/// The object header address of the child called `name` of the group at
/// `group_address`: the address [`resolve_group_children`] lists under that
/// name, or `PathNotFound` if it lists none.
///
/// A dense group's child is found through its link name index (see
/// [`links_named`]) and only the named link is read and, if it is a soft
/// link, followed — not every link in the group. A v1 group is listed.
pub fn resolve_child(
file_data: &[u8],
superblock: &Superblock,
group_address: u64,
name: &str,
) -> Result<u64, FormatError> {
let os = superblock.offset_size;
let ls = superblock.length_size;
let not_found = || FormatError::PathNotFound(String::from(name));
let header = ObjectHeader::parse(file_data, to_usize(group_address)?, os, ls)?;
if !is_v2_group(&header) || is_v1_group(&header) {
return resolve_group_children(file_data, superblock, group_address)?
.into_iter()
.find(|e| e.name == name)
.map(|e| e.object_header_address)
.ok_or_else(not_found);
}
let links = links_named(file_data, &header, name, os, ls)?;
// The listing puts hard links before resolved soft links.
if let Some(addr) = links.iter().find_map(|l| match l.link_target {
LinkTarget::Hard {
object_header_address,
} => Some(object_header_address),
_ => None,
}) {
return Ok(addr);
}
for link in links {
if let LinkTarget::Soft { target_path } = link.link_target {
return match resolve_path_from(file_data, superblock, group_address, &target_path) {
// Left out of the listing: dangling, cyclic, or in another file.
Err(
FormatError::PathNotFound(_)
| FormatError::NestingDepthExceeded
| FormatError::ExternalLinkUnsupported { .. },
) => Err(not_found()),
other => other,
};
}
}
Err(not_found())
}
/// Find and parse the Link Info message from an object header.
fn find_link_info(
object_header: &ObjectHeader,
@@ -298,7 +454,7 @@ pub fn resolve_group_children(
) -> Result<Vec<GroupEntry>, FormatError> {
let os = superblock.offset_size;
let ls = superblock.length_size;
let header = ObjectHeader::parse(file_data, group_address as usize, os, ls)?;
let header = ObjectHeader::parse(file_data, to_usize(group_address)?, os, ls)?;
let mut entries = Vec::new();
let mut soft = Vec::new();
@@ -383,24 +539,21 @@ fn resolve_path_following_links(
let ls = superblock.length_size;
let mut current_addr = start;
let mut current_header = ObjectHeader::parse(file_data, start as usize, os, ls)?;
let mut current_header = ObjectHeader::parse(file_data, to_usize(start)?, os, ls)?;
for (i, component) in components.iter().enumerate() {
let entries = resolve_group_entries(file_data, &current_header, os, ls)?;
let found = entries
.iter()
.find(|e| e.name == *component && e.object_header_address != u64::MAX);
match found {
Some(entry) => {
match lookup_link(file_data, &current_header, component, os, ls)? {
Some(LinkTarget::Hard {
object_header_address,
}) => {
if i == components.len() - 1 {
return Ok(entry.object_header_address);
return Ok(object_header_address);
}
current_addr = entry.object_header_address;
current_header = ObjectHeader::parse(file_data, current_addr as usize, os, ls)?;
current_addr = object_header_address;
current_header = ObjectHeader::parse(file_data, to_usize(current_addr)?, os, ls)?;
}
None => {
return match find_symbolic_link(file_data, &current_header, component, os, ls)? {
found => {
return match found {
Some(LinkTarget::Soft { target_path }) => {
if depth >= MAX_SOFT_LINK_DEPTH {
return Err(FormatError::NestingDepthExceeded);