From 02e89c1d2d6cf65f3eae207cfb23c25a1bd9ae87 Mon Sep 17 00:00:00 2001 From: osobh Date: Sat, 26 Sep 2026 13:33:19 -0500 Subject: [PATCH] read: look names up through the dense name indexes Finding one link or attribute by name read every entry: Group::dataset and Group::group (File, MmapFile, LazyFile) listed the whole group per call, and path resolution scanned each group's links. Opening every child of a 35 001-link group by name decoded ~1.2e9 links. Now a dense group's v2 B-tree name index (type 5, lookup3 hash of the name) is descended to the records with the name's hash (btree_v2::find_btree_v2_records reads only the nodes whose key interval overlaps), and only those links are read and compared; all hash-equal records are compared, so libhdf5's tie order does not matter. Dense attributes the same through their type 8 index (attribute::find_attribute_in_file, facade attr(name)); huge heap objects through their ID-ordered index. group_v2::resolve_child returns what the listing has under a name (soft links followed, dangling/external ones not found). Group::entries and File::group_at hand out a listing's addresses. The lookup-stats feature counts heap objects read. Tests: one lookup in an h5py-written 35 001-link group with colliding hashes reads at most two links (before: 35 001, failing), attribute lookups likewise (before: 3 000, failing), every child opens through all three readers and matches h5py, every link kind resolves as h5py resolves it in dense and compact groups, 300 huge attributes are found, and a range search matches a full scan at every tree depth. Co-Authored-By: Claude Opus 5.5 (1M context) --- crates/clawhdf5-format/Cargo.toml | 3 + crates/clawhdf5-format/src/attribute.rs | 181 ++++++-- crates/clawhdf5-format/src/btree_v2.rs | 216 +++++++-- crates/clawhdf5-format/src/btree_v2_write.rs | 53 +++ crates/clawhdf5-format/src/fractal_heap.rs | 34 +- crates/clawhdf5-format/src/group_v2.rs | 217 +++++++-- crates/clawhdf5-format/src/lib.rs | 1 + crates/clawhdf5-format/src/lookup_stats.rs | 31 ++ crates/clawhdf5/Cargo.toml | 3 +- crates/clawhdf5/src/lazy.rs | 57 ++- crates/clawhdf5/src/mmap_file.rs | 57 ++- crates/clawhdf5/src/reader.rs | 88 +++- crates/clawhdf5/src/types.rs | 29 ++ .../clawhdf5/tests/dense_storage_interop.rs | 24 + .../clawhdf5/tests/indexed_lookup_interop.rs | 410 ++++++++++++++++++ 15 files changed, 1245 insertions(+), 159 deletions(-) create mode 100644 crates/clawhdf5-format/src/lookup_stats.rs create mode 100644 crates/clawhdf5/tests/indexed_lookup_interop.rs diff --git a/crates/clawhdf5-format/Cargo.toml b/crates/clawhdf5-format/Cargo.toml index c1ec6df..e6a1258 100644 --- a/crates/clawhdf5-format/Cargo.toml +++ b/crates/clawhdf5-format/Cargo.toml @@ -80,6 +80,9 @@ blosc = ["lz4_flex", "ruzstd", "snap", "deflate", "std"] blosc2 = ["blosc"] # Every plugin filter above. plugin-filters = ["lzf", "bitshuffle", "bzip2", "blosc", "blosc2"] +# Test instrumentation: per-thread counts of heap objects read (see +# `lookup_stats`), so tests can bound the cost of a name lookup. +lookup-stats = ["std"] [[bench]] name = "parallel_decompress_bench" diff --git a/crates/clawhdf5-format/src/attribute.rs b/crates/clawhdf5-format/src/attribute.rs index 6c4e9ae..d22a552 100644 --- a/crates/clawhdf5-format/src/attribute.rs +++ b/crates/clawhdf5-format/src/attribute.rs @@ -5,8 +5,10 @@ use alloc::{borrow::Cow, string::String, vec::Vec}; #[cfg(feature = "std")] use std::borrow::Cow; +use crate::addr::to_usize; use crate::attribute_info::AttributeInfoMessage; -use crate::btree_v2::{BTreeV2Header, collect_btree_v2_records}; +use crate::btree_v2::{BTreeV2Header, collect_btree_v2_records, find_btree_v2_records}; +use crate::checksum::jenkins_lookup3; use crate::data_read; use crate::dataspace::Dataspace; use crate::datatype::Datatype; @@ -341,7 +343,8 @@ fn compute_raw_data( dataspace: &Dataspace, datatype: &Datatype, ) -> Vec { - let num_elements = dataspace.num_elements() as usize; + // Saturating, like the product: the size is capped at what is there. + let num_elements = usize::try_from(dataspace.num_elements()).unwrap_or(usize::MAX); let elem_size = datatype.type_size() as usize; let expected_size = num_elements.saturating_mul(elem_size); let available = data.len().saturating_sub(pos); @@ -453,7 +456,145 @@ fn extract_attributes_with( // Each attribute's creation order, where the file records one. let mut orders: Vec = Vec::new(); - // Collect compact attributes (inline in OH) + extract_compact_attributes( + file_data, + header, + offset_size, + length_size, + &mut attrs, + &mut orders, + on_error, + )?; + + // Check for dense attributes via AttributeInfo message + let attr_info = find_attribute_info(header, offset_size)?; + if let Some(info) = &attr_info + && let Some(fh_addr) = info.fractal_heap_address + { + extract_dense_attributes( + file_data, + info, + fh_addr, + offset_size, + length_size, + &mut attrs, + &mut orders, + on_error, + )?; + } + + // An object that tracks attribute creation order lists its attributes + // in that order (h5py's `track_order=True`), as libhdf5 does; otherwise + // they come in storage order. + if attr_info.is_some_and(|i| i.max_creation_index.is_some()) { + let mut paired: Vec<(u32, AttributeMessage)> = orders.into_iter().zip(attrs).collect(); + paired.sort_by_key(|(o, _)| *o); + attrs = paired.into_iter().map(|(_, a)| a).collect(); + } + + Ok(attrs) +} + +/// B-tree v2 record type of dense attribute storage's name index. +const ATTRIBUTE_NAME_INDEX: u8 = 8; + +/// The attribute called `name` on the object with header `header`: the +/// first one [`extract_attributes_tolerant`] returns under that name, or +/// `None` if it returns none (an attribute that cannot be read is not +/// returned there either). +/// +/// Compact attributes are in the header and are scanned. Dense attributes +/// are found through the name index (a v2 B-tree of lookup3 name hashes, +/// record type 8): only the attributes whose names hash like `name` are read +/// from the heap, O(log n) instead of all of them. Errors in the structures +/// that index the attributes fail the call, as they fail a listing. +pub fn find_attribute_in_file( + file_data: &[u8], + header: &ObjectHeader, + name: &str, + offset_size: u8, + length_size: u8, +) -> Result, FormatError> { + let attr_info = find_attribute_info(header, offset_size)?; + let dense = attr_info + .as_ref() + .and_then(|i| Some((i.fractal_heap_address?, i.btree_name_index_address?))); + let Some((fh_addr, btree_addr)) = dense else { + // Compact only (or dense storage without a name index, which a + // listing reports): as a listing finds it. + return Ok( + extract_attributes_tolerant(file_data, header, offset_size, length_size)? + .0 + .into_iter() + .find(|a| a.name == name), + ); + }; + let btree_hdr = + BTreeV2Header::parse(file_data, to_usize(btree_addr)?, offset_size, length_size)?; + let fh = FractalHeapHeader::parse(file_data, to_usize(fh_addr)?, offset_size, length_size)?; + if btree_hdr.tree_type != ATTRIBUTE_NAME_INDEX || btree_hdr.record_size < 4 { + return Ok( + extract_attributes_tolerant(file_data, header, offset_size, length_size)? + .0 + .into_iter() + .find(|a| a.name == name), + ); + } + + // A listing has the compact attributes first. + let mut compact = Vec::new(); + extract_compact_attributes( + file_data, + header, + offset_size, + length_size, + &mut compact, + &mut Vec::new(), + &mut |_| Ok(()), + )?; + if let Some(a) = compact.into_iter().find(|a| a.name == name) { + return Ok(Some(a)); + } + + // Record: heap ID + message flags(1) + creation order(4) + hash(4); the + // hash is the last field. + let hash = jenkins_lookup3(name.as_bytes()); + let hash_at = usize::from(btree_hdr.record_size) - 4; + let records = find_btree_v2_records(file_data, &btree_hdr, offset_size, &mut |r| match r + .get(hash_at..hash_at + 4) + { + Some(h) => u32::from_le_bytes([h[0], h[1], h[2], h[3]]).cmp(&hash), + None => core::cmp::Ordering::Less, + })?; + let id_len = usize::from(fh.heap_id_length); + for record in &records { + let Some(id_bytes) = record.data.get(..id_len) else { + continue; + }; + let attr = fh + .read_managed_object(file_data, id_bytes, offset_size) + .and_then(|d| AttributeMessage::parse_in_file(&d, file_data, offset_size, length_size)); + // One that cannot be read is left out, as from a listing. + if let Ok(attr) = attr + && attr.name == name + { + return Ok(Some(attr)); + } + } + Ok(None) +} + +/// The attributes stored in the object header itself (compact storage), and +/// each one's creation order into `orders`. +fn extract_compact_attributes( + file_data: &[u8], + header: &ObjectHeader, + offset_size: u8, + length_size: u8, + attrs: &mut Vec, + orders: &mut Vec, + on_error: &mut dyn FnMut(FormatError) -> Result<(), FormatError>, +) -> Result<(), FormatError> { for msg in &header.messages { if msg.msg_type == MessageType::Attribute { let attr = if shared_message::is_shared(msg.flags) { @@ -489,34 +630,7 @@ fn extract_attributes_with( } } } - - // Check for dense attributes via AttributeInfo message - let attr_info = find_attribute_info(header, offset_size)?; - if let Some(info) = &attr_info - && let Some(fh_addr) = info.fractal_heap_address - { - extract_dense_attributes( - file_data, - info, - fh_addr, - offset_size, - length_size, - &mut attrs, - &mut orders, - on_error, - )?; - } - - // An object that tracks attribute creation order lists its attributes - // in that order (h5py's `track_order=True`), as libhdf5 does; otherwise - // they come in storage order. - if attr_info.is_some_and(|i| i.max_creation_index.is_some()) { - let mut paired: Vec<(u32, AttributeMessage)> = orders.into_iter().zip(attrs).collect(); - paired.sort_by_key(|(o, _)| *o); - attrs = paired.into_iter().map(|(_, a)| a).collect(); - } - - Ok(attrs) + Ok(()) } /// Find and parse the Attribute Info message from an object header. @@ -547,7 +661,7 @@ fn extract_dense_attributes( on_error: &mut dyn FnMut(FormatError) -> Result<(), FormatError>, ) -> Result<(), FormatError> { // Parse fractal heap - let fh = FractalHeapHeader::parse(file_data, fh_addr as usize, offset_size, length_size)?; + let fh = FractalHeapHeader::parse(file_data, to_usize(fh_addr)?, offset_size, length_size)?; // Parse B-tree v2 for name index (type 8) let btree_addr = attr_info @@ -556,7 +670,8 @@ fn extract_dense_attributes( expected: 1, available: 0, })?; - let btree_hdr = BTreeV2Header::parse(file_data, btree_addr as usize, offset_size, length_size)?; + let btree_hdr = + BTreeV2Header::parse(file_data, to_usize(btree_addr)?, offset_size, length_size)?; let records = collect_btree_v2_records(file_data, &btree_hdr, offset_size, length_size)?; for record in &records { diff --git a/crates/clawhdf5-format/src/btree_v2.rs b/crates/clawhdf5-format/src/btree_v2.rs index 9159253..7582c05 100644 --- a/crates/clawhdf5-format/src/btree_v2.rs +++ b/crates/clawhdf5-format/src/btree_v2.rs @@ -2,10 +2,12 @@ #[cfg(not(feature = "std"))] use alloc::vec::Vec; +use core::cmp::Ordering; #[cfg(feature = "checksum")] use byteorder::{ByteOrder, LittleEndian}; +use crate::addr::to_usize; use crate::error::FormatError; /// Parsed B-tree v2 header (signature "BTHD"). @@ -216,7 +218,7 @@ pub fn collect_btree_v2_records( // Root is a leaf parse_leaf_records( file_data, - header.root_node_address as usize, + to_usize(header.root_node_address)?, header.num_records_in_root, header.record_size, ) @@ -225,7 +227,7 @@ pub fn collect_btree_v2_records( let mut records = Vec::new(); collect_internal_records( file_data, - header.root_node_address as usize, + to_usize(header.root_node_address)?, header.num_records_in_root, header.depth, header.record_size, @@ -289,9 +291,10 @@ fn parse_leaf_records( Ok(records) } -/// Recursively collect records from an internal node. -#[allow(clippy::too_many_arguments, clippy::only_used_in_recursion)] -fn collect_internal_records( +/// An internal node's layout: where its records start, and its children as +/// `(address, record count)`. +#[allow(clippy::too_many_arguments)] +fn read_internal_node( file_data: &[u8], offset: usize, num_records: u16, @@ -299,11 +302,8 @@ fn collect_internal_records( record_size: u16, node_size: u32, offset_size: u8, - length_size: u8, max_leaf_nrec: u64, - budget: &mut usize, - out: &mut Vec, -) -> Result<(), FormatError> { +) -> Result<(usize, Vec<(u64, u16)>), FormatError> { // signature(4) + version(1) + type(1) = 6 ensure_len(file_data, offset, 6)?; if &file_data[offset..offset + 4] != b"BTIN" { @@ -314,7 +314,7 @@ fn collect_internal_records( let rs = record_size as usize; let mut pos = offset + 6; - // Read all records first + // Records first let records_total = nr.checked_mul(rs).ok_or(FormatError::UnexpectedEof { expected: usize::MAX, available: file_data.len(), @@ -346,7 +346,6 @@ fn collect_internal_records( let child_ptr_size = offset_size as usize + nrec_width + total_nrec_width; ensure_len(file_data, pos, num_children * child_ptr_size)?; - // Read child pointers let mut children = Vec::with_capacity(num_children); for _ in 0..num_children { let addr = read_offset(file_data, pos, offset_size)?; @@ -356,6 +355,61 @@ fn collect_internal_records( pos += total_nrec_width; // skip total records in subtree children.push((addr, child_nrec)); } + Ok((records_start, children)) +} + +/// Record `i` of an internal node whose records start at `records_start`. +fn internal_record( + file_data: &[u8], + records_start: usize, + i: usize, + rs: usize, +) -> Result<&[u8], FormatError> { + let overflow = || FormatError::UnexpectedEof { + expected: usize::MAX, + available: file_data.len(), + }; + let rec_start = i + .checked_mul(rs) + .and_then(|o| records_start.checked_add(o)) + .ok_or_else(overflow)?; + let rec_end = rec_start.checked_add(rs).ok_or_else(overflow)?; + file_data + .get(rec_start..rec_end) + .ok_or(FormatError::UnexpectedEof { + expected: rec_end, + available: file_data.len(), + }) +} + +/// Recursively collect records from an internal node. +#[allow(clippy::too_many_arguments, clippy::only_used_in_recursion)] +fn collect_internal_records( + file_data: &[u8], + offset: usize, + num_records: u16, + depth: u16, + record_size: u16, + node_size: u32, + offset_size: u8, + length_size: u8, + max_leaf_nrec: u64, + budget: &mut usize, + out: &mut Vec, +) -> Result<(), FormatError> { + let nr = num_records as usize; + let rs = record_size as usize; + let (records_start, children) = read_internal_node( + file_data, + offset, + num_records, + depth, + record_size, + node_size, + offset_size, + max_leaf_nrec, + )?; + let child_depth = depth - 1; // Interleave: child[0], record[0], child[1], record[1], ..., child[nr] // We collect child[0] records, then record[0], then child[1], etc. @@ -364,12 +418,12 @@ fn collect_internal_records( // Before parsing, so a refused tree is not also a large allocation. spend(budget, usize::from(child_nrec))?; let leaf_recs = - parse_leaf_records(file_data, child_addr as usize, child_nrec, record_size)?; + parse_leaf_records(file_data, to_usize(child_addr)?, child_nrec, record_size)?; out.extend(leaf_recs); } else { collect_internal_records( file_data, - child_addr as usize, + to_usize(child_addr)?, child_nrec, child_depth, record_size, @@ -384,32 +438,10 @@ fn collect_internal_records( // Add record[i] (except after the last child) if i < nr { - let rec_offset = i.checked_mul(rs).ok_or(FormatError::UnexpectedEof { - expected: usize::MAX, - available: file_data.len(), - })?; - let rec_start = - records_start - .checked_add(rec_offset) - .ok_or(FormatError::UnexpectedEof { - expected: usize::MAX, - available: file_data.len(), - })?; - let rec_end = rec_start - .checked_add(rs) - .ok_or(FormatError::UnexpectedEof { - expected: usize::MAX, - available: file_data.len(), - })?; - if rec_end > file_data.len() { - return Err(FormatError::UnexpectedEof { - expected: rec_end, - available: file_data.len(), - }); - } + let data = internal_record(file_data, records_start, i, rs)?; spend(budget, 1)?; out.push(BTreeV2Record { - data: file_data[rec_start..rec_end].to_vec(), + data: data.to_vec(), }); } } @@ -417,6 +449,116 @@ fn collect_internal_records( Ok(()) } +/// The records of a B-tree v2 that fall in one key range, found by +/// descending the tree instead of reading all of it. +/// +/// `cmp` places a record relative to the range: `Less` if the record sorts +/// before it, `Greater` if after, `Equal` if the record is in it. The tree +/// must be ordered consistently with `cmp`, as libhdf5 orders it (a link or +/// attribute name index by name hash, so all records with one hash form a +/// range whatever order their names are in). Only the nodes whose key +/// interval overlaps the range are read: O(depth) nodes plus those holding +/// the matches. Matches come in tree order. +pub fn find_btree_v2_records( + file_data: &[u8], + header: &BTreeV2Header, + offset_size: u8, + cmp: &mut dyn FnMut(&[u8]) -> Ordering, +) -> Result, FormatError> { + if header.total_records == 0 || header.num_records_in_root == 0 { + return Ok(Vec::new()); + } + if header.depth > MAX_DEPTH { + return Err(FormatError::NestingDepthExceeded); + } + // As in `collect_btree_v2_records`: a valid tree cannot hold more + // records than the file has room for, however its children are shared. + let mut budget = file_data.len() / usize::from(header.record_size.max(1)); + let max_leaf_nrec = max_records_leaf(header.node_size, header.record_size); + let mut out = Vec::new(); + find_in_node( + file_data, + header, + to_usize(header.root_node_address)?, + header.num_records_in_root, + header.depth, + offset_size, + max_leaf_nrec, + cmp, + &mut budget, + &mut out, + )?; + Ok(out) +} + +#[allow(clippy::too_many_arguments)] +fn find_in_node( + file_data: &[u8], + header: &BTreeV2Header, + offset: usize, + num_records: u16, + depth: u16, + offset_size: u8, + max_leaf_nrec: u64, + cmp: &mut dyn FnMut(&[u8]) -> Ordering, + budget: &mut usize, + out: &mut Vec, +) -> Result<(), FormatError> { + spend(budget, usize::from(num_records))?; + if depth == 0 { + let records = parse_leaf_records(file_data, offset, num_records, header.record_size)?; + out.extend( + records + .into_iter() + .filter(|r| cmp(&r.data) == Ordering::Equal), + ); + return Ok(()); + } + let rs = usize::from(header.record_size); + let (records_start, children) = read_internal_node( + file_data, + offset, + num_records, + depth, + header.record_size, + header.node_size, + offset_size, + max_leaf_nrec, + )?; + let nr = usize::from(num_records); + let mut order = Vec::with_capacity(nr); + for i in 0..nr { + order.push(cmp(internal_record(file_data, records_start, i, rs)?)); + } + // Child `i` holds the keys between record `i - 1` and record `i`: it can + // hold a match unless the record before it is already past the range or + // the record after it is still before it. + for (i, &(child_addr, child_nrec)) in children.iter().enumerate() { + let after_left = i == 0 || order[i - 1] != Ordering::Greater; + let before_right = i == nr || order[i] != Ordering::Less; + if after_left && before_right { + find_in_node( + file_data, + header, + to_usize(child_addr)?, + child_nrec, + depth - 1, + offset_size, + max_leaf_nrec, + cmp, + budget, + out, + )?; + } + if i < nr && order[i] == Ordering::Equal { + out.push(BTreeV2Record { + data: internal_record(file_data, records_start, i, rs)?.to_vec(), + }); + } + } + Ok(()) +} + /// Most records a subtree whose root is at `depth` can hold (libhdf5's /// `cum_max_nrec`). See [`node_info`]. fn cum_max_records( diff --git a/crates/clawhdf5-format/src/btree_v2_write.rs b/crates/clawhdf5-format/src/btree_v2_write.rs index 1a18984..795fb80 100644 --- a/crates/clawhdf5-format/src/btree_v2_write.rs +++ b/crates/clawhdf5-format/src/btree_v2_write.rs @@ -385,6 +385,59 @@ mod tests { assert!(nodes > 0); } + /// Descending to a key range finds exactly the records a full read + /// holds in it — runs of equal keys that straddle node boundaries + /// included — at every depth, and nothing for keys not in the tree. + #[test] + fn a_key_range_search_matches_a_full_scan() { + use crate::btree_v2::find_btree_v2_records; + use core::cmp::Ordering; + let rs = 11usize; + // Keys 0, 0, 0, 2, 2, 2, 4, ...: runs of three, odd keys missing. + for n in [1usize, 45, 46, 1150, 30_000] { + let mut recs = Vec::with_capacity(n * rs); + for i in 0..n { + let mut r = vec![0u8; rs]; + r[..8].copy_from_slice(&((i / 3 * 2) as u64).to_be_bytes()); + r[8..].copy_from_slice(&[(i % 3) as u8, 0, 0]); + recs.extend_from_slice(&r); + } + let base = 4096u64; + let tree = build_btree_v2(params(512, 11), &recs, base, 8, 8).unwrap(); + let mut file = vec![0u8; base as usize]; + file.extend_from_slice(&tree); + let hdr = BTreeV2Header::parse(&file, base as usize, 8, 8).unwrap(); + let all = collect_btree_v2_records(&file, &hdr, 8, 8).unwrap(); + let key = |r: &[u8]| u64::from_be_bytes(r[..8].try_into().unwrap()); + let last = key(&all[n - 1].data); + let probes = (0..=last + 1).step_by(if n > 1000 { 37 } else { 1 }); + for k in probes.chain([last, last + 1, u64::MAX]) { + let found = + find_btree_v2_records(&file, &hdr, 8, &mut |r: &[u8]| key(r).cmp(&k)).unwrap(); + let want: Vec<&[u8]> = all + .iter() + .map(|r| r.data.as_slice()) + .filter(|r| key(r) == k) + .collect(); + let got: Vec<&[u8]> = found.iter().map(|r| r.data.as_slice()).collect(); + assert_eq!(got, want, "n {n} key {k}"); + assert_eq!( + got.len(), + if k % 2 == 0 && k <= last { + want.len() + } else { + 0 + } + ); + } + // Every record, or none, when the whole tree is in or out of range. + let every = find_btree_v2_records(&file, &hdr, 8, &mut |_| Ordering::Equal).unwrap(); + assert_eq!(every.len(), n); + let none = find_btree_v2_records(&file, &hdr, 8, &mut |_| Ordering::Less).unwrap(); + assert!(none.is_empty()); + } + } + #[test] fn a_node_too_small_or_too_big_is_an_error() { assert!(build_btree_v2(params(16, 11), &records(1, 11), 0, 8, 8).is_err()); diff --git a/crates/clawhdf5-format/src/fractal_heap.rs b/crates/clawhdf5-format/src/fractal_heap.rs index 9bc4ba4..d896156 100644 --- a/crates/clawhdf5-format/src/fractal_heap.rs +++ b/crates/clawhdf5-format/src/fractal_heap.rs @@ -6,7 +6,8 @@ use alloc::{format, vec::Vec}; #[cfg(feature = "checksum")] use byteorder::{ByteOrder, LittleEndian}; -use crate::btree_v2::{BTreeV2Header, collect_btree_v2_records}; +use crate::addr::to_usize; +use crate::btree_v2::{BTreeV2Header, find_btree_v2_records}; use crate::error::FormatError; use crate::filter_pipeline::FilterPipeline; @@ -354,6 +355,7 @@ impl FractalHeapHeader { id_bytes: &[u8], offset_size: u8, ) -> Result, FormatError> { + crate::lookup_stats::heap_object_read(); let Some(&first) = id_bytes.first() else { return Err(FormatError::UnexpectedEof { expected: 1, @@ -448,7 +450,7 @@ impl FractalHeapHeader { } let hdr = BTreeV2Header::parse( file_data, - self.huge_btree_address as usize, + to_usize(self.huge_btree_address)?, self.offset_size, self.length_size, )?; @@ -463,8 +465,12 @@ impl FractalHeapHeader { if hdr.tree_type != expected_type || usize::from(hdr.record_size) < rec_len { return Err(heap_error("unexpected huge-object B-tree record type")); } - let records = - collect_btree_v2_records(file_data, &hdr, self.offset_size, self.length_size)?; + // Records are ordered by ID (the last field): descend to the ones + // equal to `key` instead of reading the whole index. + let id_at = rec_len - ls; + let records = find_btree_v2_records(file_data, &hdr, self.offset_size, &mut |r| { + le_uint(&r[id_at..id_at + ls]).cmp(&key) + })?; for rec in &records { let d = &rec.data; if d.len() < rec_len { @@ -533,24 +539,24 @@ impl FractalHeapHeader { self.read_from_direct_block( file_data, DirectBlock { - addr: self.root_block_address as usize, + addr: to_usize(self.root_block_address)?, size: self.starting_block_size, heap_offset: 0, filtered_size: self.root_direct_block_filtered_size, filter_mask: self.root_direct_block_filter_mask, }, heap_offset, - obj_len as usize, + to_usize(obj_len)?, ) } else { // Root is an indirect block — limit recursion to 64 levels self.read_from_indirect_block( file_data, - self.root_block_address as usize, + to_usize(self.root_block_address)?, self.current_rows_in_root_indirect_block, 0, // block offset heap_offset, - obj_len as usize, + to_usize(obj_len)?, offset_size, 64, // max recursion depth ) @@ -572,11 +578,11 @@ impl FractalHeapHeader { ) -> Result, FormatError> { if target_offset < block.heap_offset { return Err(FormatError::UnexpectedEof { - expected: block.heap_offset as usize, - available: target_offset as usize, + expected: to_usize(block.heap_offset)?, + available: to_usize(target_offset)?, }); } - let local_offset = (target_offset - block.heap_offset) as usize; + let local_offset = to_usize(target_offset - block.heap_offset)?; if let Some(pipeline) = &self.filter_pipeline { let stored_len = usize::try_from(block.filtered_size) .map_err(|_| heap_error("direct block size"))?; @@ -673,7 +679,7 @@ impl FractalHeapHeader { return self.read_from_direct_block( file_data, DirectBlock { - addr: child_addr as usize, + addr: to_usize(child_addr)?, size: block_size, heap_offset: current_heap_offset, filtered_size, @@ -705,7 +711,7 @@ impl FractalHeapHeader { { return self.read_from_indirect_block( file_data, - child_addr as usize, + to_usize(child_addr)?, child_nrows, current_heap_offset, target_offset, @@ -719,7 +725,7 @@ impl FractalHeapHeader { } Err(FormatError::UnexpectedEof { - expected: target_offset as usize + length, + expected: to_usize(target_offset)?.saturating_add(length), available: file_data.len(), }) } diff --git a/crates/clawhdf5-format/src/group_v2.rs b/crates/clawhdf5-format/src/group_v2.rs index b645bd6..e1bd9f5 100644 --- a/crates/clawhdf5-format/src/group_v2.rs +++ b/crates/clawhdf5-format/src/group_v2.rs @@ -6,7 +6,9 @@ #[cfg(not(feature = "std"))] use alloc::{string::String, vec::Vec}; -use crate::btree_v2::{BTreeV2Header, collect_btree_v2_records}; +use crate::addr::to_usize; +use crate::btree_v2::{BTreeV2Header, collect_btree_v2_records, find_btree_v2_records}; +use crate::checksum::jenkins_lookup3; use crate::error::FormatError; use crate::fractal_heap::FractalHeapHeader; use crate::group_v1::{self, GroupEntry}; @@ -93,13 +95,14 @@ fn for_each_dense_link( mut visit: impl FnMut(LinkMessage), ) -> Result<(), FormatError> { // Parse fractal heap - let fh = FractalHeapHeader::parse(file_data, fh_addr as usize, offset_size, length_size)?; + let fh = FractalHeapHeader::parse(file_data, to_usize(fh_addr)?, offset_size, length_size)?; // Parse B-tree v2 for name index let btree_addr = link_info .btree_name_index_address .ok_or_else(|| FormatError::PathNotFound(String::from("no B-tree v2 name index")))?; - let btree_hdr = BTreeV2Header::parse(file_data, btree_addr as usize, offset_size, length_size)?; + let btree_hdr = + BTreeV2Header::parse(file_data, to_usize(btree_addr)?, offset_size, length_size)?; let records = collect_btree_v2_records(file_data, &btree_hdr, offset_size, length_size)?; for record in &records { @@ -181,10 +184,55 @@ fn find_symbolic_link( if !is_v2_group(object_header) { return Ok(None); } - let is_symbolic = |t: &LinkTarget| !matches!(t, LinkTarget::Hard { .. }); + // As when all links were scanned: the last symbolic link of that name. + Ok( + links_named(file_data, object_header, name, offset_size, length_size)? + .into_iter() + .rev() + .map(|link| link.link_target) + .find(|t| !matches!(t, LinkTarget::Hard { .. })), + ) +} + +/// B-tree v2 record type of a dense group's link name index. +const LINK_NAME_INDEX: u8 = 5; + +/// The links called `name` in a v2 group (a valid group has at most one). +/// +/// In dense storage the link name index (a v2 B-tree of lookup3 name +/// hashes, record type 5) is descended to the records with the name's hash, +/// and only their links are read from the heap — O(log n) instead of every +/// link. libhdf5 orders records with equal hashes by name; all of them are +/// read and compared here, so that order does not matter. An index of +/// another type is scanned in full. +fn links_named( + file_data: &[u8], + object_header: &ObjectHeader, + name: &str, + offset_size: u8, + length_size: u8, +) -> Result, FormatError> { + let mut found = Vec::new(); let link_info = find_link_info(object_header, offset_size)?; - let mut found = None; - if let Some(fh_addr) = link_info.fractal_heap_address { + let Some(fh_addr) = link_info.fractal_heap_address else { + for msg in &object_header.messages { + if msg.msg_type == MessageType::Link + && let Some(link) = parse_link(&msg.data, offset_size)? + && link.name == name + { + found.push(link); + } + } + return Ok(found); + }; + + let fh = FractalHeapHeader::parse(file_data, to_usize(fh_addr)?, offset_size, length_size)?; + let btree_addr = link_info + .btree_name_index_address + .ok_or_else(|| FormatError::PathNotFound(String::from("no B-tree v2 name index")))?; + let btree_hdr = + BTreeV2Header::parse(file_data, to_usize(btree_addr)?, offset_size, length_size)?; + if btree_hdr.tree_type != LINK_NAME_INDEX { for_each_dense_link( file_data, &link_info, @@ -192,26 +240,134 @@ fn find_symbolic_link( offset_size, length_size, |link| { - if link.name == name && is_symbolic(&link.link_target) { - found = Some(link.link_target); + if link.name == name { + found.push(link); } }, )?; - } else { - for msg in &object_header.messages { - if msg.msg_type == MessageType::Link { - let Some(link) = parse_link(&msg.data, offset_size)? else { - continue; - }; - if link.name == name && is_symbolic(&link.link_target) { - found = Some(link.link_target); - } - } + return Ok(found); + } + + // Record: hash(4) + heap ID. + let hash = jenkins_lookup3(name.as_bytes()); + let records = find_btree_v2_records(file_data, &btree_hdr, offset_size, &mut |r| { + match r.get(..4) { + Some(h) => u32::from_le_bytes([h[0], h[1], h[2], h[3]]).cmp(&hash), + // Too short to hold a hash (a corrupt record size): never a match. + None => core::cmp::Ordering::Less, + } + })?; + let id_len = usize::from(fh.heap_id_length); + for record in &records { + let Some(id_bytes) = record.data.get(4..4 + id_len) else { + continue; + }; + let link_data = fh.read_managed_object(file_data, id_bytes, offset_size)?; + if let Some(link) = parse_link(&link_data, offset_size)? + && link.name == name + { + found.push(link); } } Ok(found) } +/// The link [`resolve_path_any`] follows for one path component `name` of +/// the group with header `object_header`: a hard link (as `Hard`), else a +/// soft or external link of that name, else `None`. Fails with +/// `PathNotFound` if the object is not a group. +fn lookup_link( + file_data: &[u8], + object_header: &ObjectHeader, + name: &str, + offset_size: u8, + length_size: u8, +) -> Result, FormatError> { + if is_v1_group(object_header) { + let entries = resolve_group_entries(file_data, object_header, offset_size, length_size)?; + if let Some(e) = entries + .iter() + .find(|e| e.name == name && e.object_header_address != u64::MAX) + { + return Ok(Some(LinkTarget::Hard { + object_header_address: e.object_header_address, + })); + } + return find_symbolic_link(file_data, object_header, name, offset_size, length_size); + } + if !is_v2_group(object_header) { + return Err(FormatError::PathNotFound(String::from( + "object header is not a group", + ))); + } + let links = links_named(file_data, object_header, name, offset_size, length_size)?; + if let Some(addr) = links.iter().find_map(|l| match l.link_target { + LinkTarget::Hard { + object_header_address, + } if object_header_address != u64::MAX => Some(object_header_address), + _ => None, + }) { + return Ok(Some(LinkTarget::Hard { + object_header_address: addr, + })); + } + Ok(links + .into_iter() + .rev() + .map(|link| link.link_target) + .find(|t| !matches!(t, LinkTarget::Hard { .. }))) +} + +/// The object header address of the child called `name` of the group at +/// `group_address`: the address [`resolve_group_children`] lists under that +/// name, or `PathNotFound` if it lists none. +/// +/// A dense group's child is found through its link name index (see +/// [`links_named`]) and only the named link is read and, if it is a soft +/// link, followed — not every link in the group. A v1 group is listed. +pub fn resolve_child( + file_data: &[u8], + superblock: &Superblock, + group_address: u64, + name: &str, +) -> Result { + let os = superblock.offset_size; + let ls = superblock.length_size; + let not_found = || FormatError::PathNotFound(String::from(name)); + let header = ObjectHeader::parse(file_data, to_usize(group_address)?, os, ls)?; + if !is_v2_group(&header) || is_v1_group(&header) { + return resolve_group_children(file_data, superblock, group_address)? + .into_iter() + .find(|e| e.name == name) + .map(|e| e.object_header_address) + .ok_or_else(not_found); + } + let links = links_named(file_data, &header, name, os, ls)?; + // The listing puts hard links before resolved soft links. + if let Some(addr) = links.iter().find_map(|l| match l.link_target { + LinkTarget::Hard { + object_header_address, + } => Some(object_header_address), + _ => None, + }) { + return Ok(addr); + } + for link in links { + if let LinkTarget::Soft { target_path } = link.link_target { + return match resolve_path_from(file_data, superblock, group_address, &target_path) { + // Left out of the listing: dangling, cyclic, or in another file. + Err( + FormatError::PathNotFound(_) + | FormatError::NestingDepthExceeded + | FormatError::ExternalLinkUnsupported { .. }, + ) => Err(not_found()), + other => other, + }; + } + } + Err(not_found()) +} + /// Find and parse the Link Info message from an object header. fn find_link_info( object_header: &ObjectHeader, @@ -298,7 +454,7 @@ pub fn resolve_group_children( ) -> Result, FormatError> { let os = superblock.offset_size; let ls = superblock.length_size; - let header = ObjectHeader::parse(file_data, group_address as usize, os, ls)?; + let header = ObjectHeader::parse(file_data, to_usize(group_address)?, os, ls)?; let mut entries = Vec::new(); let mut soft = Vec::new(); @@ -383,24 +539,21 @@ fn resolve_path_following_links( let ls = superblock.length_size; let mut current_addr = start; - let mut current_header = ObjectHeader::parse(file_data, start as usize, os, ls)?; + let mut current_header = ObjectHeader::parse(file_data, to_usize(start)?, os, ls)?; for (i, component) in components.iter().enumerate() { - let entries = resolve_group_entries(file_data, ¤t_header, os, ls)?; - - let found = entries - .iter() - .find(|e| e.name == *component && e.object_header_address != u64::MAX); - match found { - Some(entry) => { + match lookup_link(file_data, ¤t_header, component, os, ls)? { + Some(LinkTarget::Hard { + object_header_address, + }) => { if i == components.len() - 1 { - return Ok(entry.object_header_address); + return Ok(object_header_address); } - current_addr = entry.object_header_address; - current_header = ObjectHeader::parse(file_data, current_addr as usize, os, ls)?; + current_addr = object_header_address; + current_header = ObjectHeader::parse(file_data, to_usize(current_addr)?, os, ls)?; } - None => { - return match find_symbolic_link(file_data, ¤t_header, component, os, ls)? { + found => { + return match found { Some(LinkTarget::Soft { target_path }) => { if depth >= MAX_SOFT_LINK_DEPTH { return Err(FormatError::NestingDepthExceeded); diff --git a/crates/clawhdf5-format/src/lib.rs b/crates/clawhdf5-format/src/lib.rs index 2737c9b..345c923 100644 --- a/crates/clawhdf5-format/src/lib.rs +++ b/crates/clawhdf5-format/src/lib.rs @@ -108,6 +108,7 @@ pub mod lane_partition; pub mod link_info; pub mod link_message; pub mod local_heap; +pub mod lookup_stats; pub mod message_type; pub mod metadata_cache; pub mod metadata_index; diff --git a/crates/clawhdf5-format/src/lookup_stats.rs b/crates/clawhdf5-format/src/lookup_stats.rs new file mode 100644 index 0000000..0206729 --- /dev/null +++ b/crates/clawhdf5-format/src/lookup_stats.rs @@ -0,0 +1,31 @@ +//! Work counters for tests of lookup cost (feature `lookup-stats`). +//! +//! Counts fractal-heap objects read — each is one link or attribute message +//! decoded out of a dense group or dense attribute storage — so a test can +//! check that finding one name reads a handful of them, not the whole group. +//! Per thread, so tests running in parallel do not see each other's reads. +//! Without the feature the counting compiles to nothing. + +#[cfg(feature = "lookup-stats")] +std::thread_local! { + static HEAP_OBJECTS: core::cell::Cell = const { core::cell::Cell::new(0) }; +} + +/// Record one heap object read. +#[inline(always)] +pub(crate) fn heap_object_read() { + #[cfg(feature = "lookup-stats")] + HEAP_OBJECTS.with(|c| c.set(c.get() + 1)); +} + +/// Heap objects read on this thread since the last [`reset`]. +#[cfg(feature = "lookup-stats")] +pub fn heap_objects_read() -> u64 { + HEAP_OBJECTS.with(core::cell::Cell::get) +} + +/// Zero this thread's counters. +#[cfg(feature = "lookup-stats")] +pub fn reset() { + HEAP_OBJECTS.with(|c| c.set(0)); +} diff --git a/crates/clawhdf5/Cargo.toml b/crates/clawhdf5/Cargo.toml index 1eca373..4f68e58 100644 --- a/crates/clawhdf5/Cargo.toml +++ b/crates/clawhdf5/Cargo.toml @@ -19,7 +19,8 @@ rayon = { version = "1", optional = true } tempfile = { workspace = true } criterion = { workspace = true } clawhdf5-io = { path = "../clawhdf5-io", version = "2.7.0", features = ["mmap"] } -clawhdf5-format = { path = "../clawhdf5-format", version = "2.7.0", features = ["parallel", "fast-checksum"] } +clawhdf5-format = { path = "../clawhdf5-format", version = "2.7.0", features = ["parallel", "fast-checksum", "lookup-stats"] } +serde_json = "1" clawhdf5-filters = { path = "../clawhdf5-filters", version = "2.7.0" } [[bench]] diff --git a/crates/clawhdf5/src/lazy.rs b/crates/clawhdf5/src/lazy.rs index df41688..7ad7e78 100644 --- a/crates/clawhdf5/src/lazy.rs +++ b/crates/clawhdf5/src/lazy.rs @@ -28,7 +28,7 @@ use clawhdf5_format::superblock::Superblock; use clawhdf5_io::HDF5Read; use crate::error::Error; -use crate::types::{AttrValue, DType, classify_datatype, read_attrs}; +use crate::types::{AttrValue, DType, classify_datatype, read_attr, read_attrs}; /// A lazy HDF5 file handle that parses metadata on demand. /// @@ -304,12 +304,7 @@ impl<'f, R: HDF5Read> LazyGroup<'f, R> { /// Get a dataset within this group by name. pub fn dataset(&self, name: &str) -> Result, Error> { - let entries = self.children()?; - let entry = entries - .iter() - .find(|e| e.name == name) - .ok_or_else(|| Error::Format(FormatError::PathNotFound(name.to_string())))?; - let hdr = self.file.get_or_parse_header(entry.object_header_address)?; + let hdr = self.file.get_or_parse_header(self.child_address(name)?)?; if !has_message(&hdr, MessageType::DataLayout) { return Err(Error::NotADataset(name.to_string())); } @@ -322,17 +317,38 @@ impl<'f, R: HDF5Read> LazyGroup<'f, R> { /// Get a subgroup within this group by name. pub fn group(&self, name: &str) -> Result, Error> { - let entries = self.children()?; - let entry = entries - .iter() - .find(|e| e.name == name) - .ok_or_else(|| Error::Format(FormatError::PathNotFound(name.to_string())))?; Ok(LazyGroup { file: self.file, - address: entry.object_header_address, + address: self.child_address(name)?, }) } + /// The attribute called `name`, or `None` if it has none by that name + /// (or it cannot be read) — the value [`attrs`](Self::attrs) has under + /// that name, found without reading the other attributes when they are + /// stored densely. + pub fn attr(&self, name: &str) -> Result, Error> { + let hdr = self.file.get_or_parse_header(self.address)?; + let data = self.file.hdf5_bytes(); + read_attr( + data, + &hdr, + name, + self.file.offset_size(), + self.file.length_size(), + ) + } + + /// The object header address of the child called `name`: the entry of + /// the group's listing with that name, looked up through the group's + /// name index rather than by listing the group (see + /// [`group_v2::resolve_child`]). + fn child_address(&self, name: &str) -> Result { + let data = self.file.hdf5_bytes(); + group_v2::resolve_child(data, &self.file.superblock, self.address, name) + .map_err(Error::Format) + } + /// This group's links that can be opened: hard links, and soft links /// resolved to their targets (see /// [`group_v2::resolve_group_children`]); dangling, external and @@ -555,6 +571,21 @@ impl<'f, R: HDF5Read> LazyDataset<'f, R> { ) } + /// The attribute called `name`, or `None` if it has none by that name + /// (or it cannot be read) — the value [`attrs`](Self::attrs) has under + /// that name, found without reading the other attributes when they are + /// stored densely. + pub fn attr(&self, name: &str) -> Result, Error> { + let data = self.file.hdf5_bytes(); + read_attr( + data, + &self.header, + name, + self.file.offset_size(), + self.file.length_size(), + ) + } + /// A header message's payload, resolved through the shared-message /// indirection when needed (e.g. a committed datatype). See /// [`clawhdf5_format::shared_message::message_data`]. diff --git a/crates/clawhdf5/src/mmap_file.rs b/crates/clawhdf5/src/mmap_file.rs index b6df1d8..edb6dd4 100644 --- a/crates/clawhdf5/src/mmap_file.rs +++ b/crates/clawhdf5/src/mmap_file.rs @@ -23,7 +23,7 @@ use clawhdf5_format::superblock::Superblock; use clawhdf5_io::MmapReader; use crate::error::Error; -use crate::types::{AttrValue, DType, classify_datatype, read_attrs}; +use crate::types::{AttrValue, DType, classify_datatype, read_attr, read_attrs}; /// An HDF5 file opened via memory mapping. /// @@ -242,12 +242,7 @@ impl<'f> MmapGroup<'f> { /// Get a dataset within this group by name. pub fn dataset(&self, name: &str) -> Result, Error> { - let entries = self.children()?; - let entry = entries - .iter() - .find(|e| e.name == name) - .ok_or_else(|| Error::Format(FormatError::PathNotFound(name.to_string())))?; - let hdr = self.file.parse_header(entry.object_header_address)?; + let hdr = self.file.parse_header(self.child_address(name)?)?; if !has_message(&hdr, MessageType::DataLayout) { return Err(Error::NotADataset(name.to_string())); } @@ -260,17 +255,38 @@ impl<'f> MmapGroup<'f> { /// Get a subgroup within this group by name. pub fn group(&self, name: &str) -> Result, Error> { - let entries = self.children()?; - let entry = entries - .iter() - .find(|e| e.name == name) - .ok_or_else(|| Error::Format(FormatError::PathNotFound(name.to_string())))?; Ok(MmapGroup { file: self.file, - address: entry.object_header_address, + address: self.child_address(name)?, }) } + /// The attribute called `name`, or `None` if it has none by that name + /// (or it cannot be read) — the value [`attrs`](Self::attrs) has under + /// that name, found without reading the other attributes when they are + /// stored densely. + pub fn attr(&self, name: &str) -> Result, Error> { + let hdr = self.file.parse_header(self.address)?; + let data = self.file.hdf5_bytes(); + read_attr( + data, + &hdr, + name, + self.file.offset_size(), + self.file.length_size(), + ) + } + + /// The object header address of the child called `name`: the entry of + /// the group's listing with that name, looked up through the group's + /// name index rather than by listing the group (see + /// [`group_v2::resolve_child`]). + fn child_address(&self, name: &str) -> Result { + let data = self.file.meta()?; + group_v2::resolve_child(data, &self.file.superblock, self.address, name) + .map_err(Error::Format) + } + /// This group's links that can be opened: hard links, and soft links /// resolved to their targets (see /// [`group_v2::resolve_group_children`]); dangling, external and @@ -506,6 +522,21 @@ impl<'f> MmapDataset<'f> { ) } + /// The attribute called `name`, or `None` if it has none by that name + /// (or it cannot be read) — the value [`attrs`](Self::attrs) has under + /// that name, found without reading the other attributes when they are + /// stored densely. + pub fn attr(&self, name: &str) -> Result, Error> { + let data = self.file.hdf5_bytes(); + read_attr( + data, + &self.header, + name, + self.file.offset_size(), + self.file.length_size(), + ) + } + /// A header message's payload, resolved through the shared-message /// indirection when needed (e.g. a committed datatype). See /// [`clawhdf5_format::shared_message::message_data`]. diff --git a/crates/clawhdf5/src/reader.rs b/crates/clawhdf5/src/reader.rs index 1b20dd6..041a318 100644 --- a/crates/clawhdf5/src/reader.rs +++ b/crates/clawhdf5/src/reader.rs @@ -23,7 +23,7 @@ use clawhdf5_format::superblock::Superblock; use crate::cache_image::{self, ImageView}; use crate::error::Error; -use crate::types::{AttrValue, DType, classify_datatype, read_attrs}; +use crate::types::{AttrValue, DType, classify_datatype, read_attr, read_attrs}; // --------------------------------------------------------------------------- // FileData — internal storage for either owned bytes or an mmap @@ -230,9 +230,10 @@ impl File { /// A `Dataset` handle for the object header at `address` (an address /// from a group listing, or one kept from an earlier lookup), without - /// resolving a path. Resolving a path walks every group on it, which in - /// a large group costs a scan of its links; keep the address instead to - /// open the same dataset repeatedly. + /// resolving a path. Resolving a path looks each component up in its + /// group (through the name index of a dense group; a v1 group's entries + /// are scanned); keep the address instead to open the same dataset + /// repeatedly. pub fn dataset_at(&self, address: u64) -> Result, Error> { let hdr = self.parse_header(address)?; if !has_message(&hdr, MessageType::DataLayout) { @@ -245,6 +246,17 @@ impl File { .check_open() } + /// A `Group` handle for the object header at `address` (from + /// [`Group::entries`], or kept from an earlier lookup), without + /// resolving a path. Like [`group`](Self::group), the object is not + /// checked to be a group; a non-group has no children. + pub fn group_at(&self, address: u64) -> Group<'_> { + Group { + file: self, + address, + } + } + /// Resolve a path and return a `Group` handle. /// /// The path uses `/` separators (e.g., `"sensors"`). @@ -483,12 +495,7 @@ impl<'f> Group<'f> { /// Get a dataset within this group by name. pub fn dataset(&self, name: &str) -> Result, Error> { - let entries = self.children()?; - let entry = entries - .iter() - .find(|e| e.name == name) - .ok_or_else(|| Error::Format(FormatError::PathNotFound(name.to_string())))?; - let hdr = self.file.parse_header(entry.object_header_address)?; + let hdr = self.file.parse_header(self.child_address(name)?)?; if !has_message(&hdr, MessageType::DataLayout) { return Err(Error::NotADataset(name.to_string())); } @@ -501,17 +508,51 @@ impl<'f> Group<'f> { /// Get a subgroup within this group by name. pub fn group(&self, name: &str) -> Result, Error> { - let entries = self.children()?; - let entry = entries - .iter() - .find(|e| e.name == name) - .ok_or_else(|| Error::Format(FormatError::PathNotFound(name.to_string())))?; Ok(Group { file: self.file, - address: entry.object_header_address, + address: self.child_address(name)?, }) } + /// The attribute called `name`, or `None` if it has none by that name + /// (or it cannot be read) — the value [`attrs`](Self::attrs) has under + /// that name, found without reading the other attributes when they are + /// stored densely. + pub fn attr(&self, name: &str) -> Result, Error> { + let hdr = self.file.parse_header(self.address)?; + let data = self.file.data.as_bytes(); + read_attr( + data, + &hdr, + name, + self.file.offset_size(), + self.file.length_size(), + ) + } + + /// The object header address of the child called `name`: the entry of + /// the group's listing with that name, looked up through the group's + /// name index rather than by listing the group (see + /// [`group_v2::resolve_child`]). + fn child_address(&self, name: &str) -> Result { + let data = self.file.data.meta()?; + group_v2::resolve_child(data, &self.file.superblock, self.address, name) + .map_err(Error::Format) + } + + /// This group's children that can be opened, as `(name, object header + /// address)` in listing order — the entries [`datasets`](Self::datasets) + /// and [`groups`](Self::groups) are drawn from. Open one with + /// [`File::dataset_at`] or [`File::group_at`] to skip looking its name + /// up again, or keep the addresses to revisit the objects. + pub fn entries(&self) -> Result, Error> { + Ok(self + .children()? + .into_iter() + .map(|e| (e.name, e.object_header_address)) + .collect()) + } + /// This group's links that can be opened: hard links, and soft links /// resolved to their targets (see /// [`group_v2::resolve_group_children`]); dangling, external and @@ -1051,6 +1092,21 @@ impl<'f> Dataset<'f> { ) } + /// The attribute called `name`, or `None` if it has none by that name + /// (or it cannot be read) — the value [`attrs`](Self::attrs) has under + /// that name, found without reading the other attributes when they are + /// stored densely. + pub fn attr(&self, name: &str) -> Result, Error> { + let data = self.file.data.as_bytes(); + read_attr( + data, + &self.header, + name, + self.file.offset_size(), + self.file.length_size(), + ) + } + /// Verify this dataset's content against its stored provenance hash /// (`_provenance_sha256`, written automatically on save when a /// [`Provenance`](clawhdf5_format::provenance::Provenance) is set — see diff --git a/crates/clawhdf5/src/types.rs b/crates/clawhdf5/src/types.rs index 7023b14..acae280 100644 --- a/crates/clawhdf5/src/types.rs +++ b/crates/clawhdf5/src/types.rs @@ -182,6 +182,35 @@ pub(crate) fn read_attrs( )) } +/// The attribute called `name` on the object with header `header`, decoded +/// as [`read_attrs`] decodes it, or `None` (see +/// [`find_attribute_in_file`](clawhdf5_format::attribute::find_attribute_in_file)). +pub(crate) fn read_attr( + file_data: &[u8], + header: &clawhdf5_format::object_header::ObjectHeader, + name: &str, + offset_size: u8, + length_size: u8, +) -> Result, crate::Error> { + let Some(msg) = clawhdf5_format::attribute::find_attribute_in_file( + file_data, + header, + name, + offset_size, + length_size, + )? + else { + return Ok(None); + }; + Ok(attrs_to_map( + std::slice::from_ref(&msg), + file_data, + offset_size, + length_size, + ) + .remove(name)) +} + pub(crate) fn attrs_to_map( attrs: &[clawhdf5_format::attribute::AttributeMessage], file_data: &[u8], diff --git a/crates/clawhdf5/tests/dense_storage_interop.rs b/crates/clawhdf5/tests/dense_storage_interop.rs index b5aa8d1..3f741f6 100644 --- a/crates/clawhdf5/tests/dense_storage_interop.rs +++ b/crates/clawhdf5/tests/dense_storage_interop.rs @@ -211,6 +211,30 @@ fn dense_attribute_stored_as_a_huge_heap_object() { assert!(matches!(&attrs["bigger"], AttrValue::I64Array(v) if *v == bigger)); } +/// Enough huge attributes that the huge-object B-tree has internal nodes: +/// each one is found by descending it by heap ID (libhdf5 orders the +/// records of indirectly addressed huge objects by ID), and every value +/// matches what was written. +#[test] +fn many_huge_attributes_are_found_through_their_index() { + skip_if_no_python!(); + let (_dir, path) = h5py_file( + "d = f.create_dataset('d', data=[1.0])\n\ + for i in range(300):\n\ + \x20 d.attrs['h%03d' % i] = np.arange(600, dtype='i8') + i\n", + ); + let f = File::open(&path).unwrap(); + let d = f.dataset("d").unwrap(); + let attrs = d.attrs().unwrap(); + assert_eq!(attrs.len(), 300); + for i in 0..300i64 { + let name = format!("h{i:03}"); + let want: Vec = (0..600).map(|v| v + i).collect(); + assert!(matches!(&attrs[&name], AttrValue::I64Array(v) if *v == want), "{name}"); + assert!(matches!(d.attr(&name).unwrap(), Some(AttrValue::I64Array(v)) if v == want), "{name}"); + } +} + /// A group whose link heap has a deflate I/O filter (set on the group /// creation property list), with 3 000 links and one link whose message is /// larger than the heap's managed-object limit, so it is a huge object. diff --git a/crates/clawhdf5/tests/indexed_lookup_interop.rs b/crates/clawhdf5/tests/indexed_lookup_interop.rs new file mode 100644 index 0000000..518af1e --- /dev/null +++ b/crates/clawhdf5/tests/indexed_lookup_interop.rs @@ -0,0 +1,410 @@ +//! Looking one name up in a dense group (links in a fractal heap, indexed by +//! a v2 B-tree of name hashes) or in dense attribute storage reads the name +//! index, not every link: O(log n) index nodes and only the links whose +//! lookup3 hash equals the name's. Before, every lookup decoded all n links, +//! so opening each child of a 35 001-link group by name decoded ~1.2e9. +//! +//! The file is written by h5py (libhdf5 orders the index), with names whose +//! hashes collide, and every result is compared with what h5py reads. +//! +//! Skipped when python3 with h5py is unavailable, unless +//! `CLAWHDF5_REQUIRE_INTEROP=1`. + +use std::collections::{BTreeMap, HashMap}; +use std::process::Command; +use std::sync::OnceLock; +use std::time::{Duration, Instant}; + +use clawhdf5::{AttrValue, File, LazyFile, MmapFile}; +use clawhdf5_format::checksum::jenkins_lookup3; +use clawhdf5_format::error::FormatError; +use clawhdf5_format::lookup_stats; + +fn python() -> String { + std::env::var("CLAWHDF5_PYTHON").unwrap_or_else(|_| "python3".to_string()) +} + +fn interop_required() -> bool { + std::env::var("CLAWHDF5_REQUIRE_INTEROP").is_ok_and(|v| v == "1") +} + +fn python_available() -> bool { + Command::new(python()) + .args(["-c", "import h5py"]) + .output() + .map(|o| o.status.success()) + .unwrap_or(false) +} + +macro_rules! skip_if_no_python { + () => { + if !python_available() { + assert!( + !interop_required(), + "CLAWHDF5_REQUIRE_INTEROP=1 but python3 with h5py is not available" + ); + eprintln!("SKIP: python3 with h5py not available"); + return; + } + }; +} + +fn run_python(script: &str) -> String { + let output = Command::new(python()) + .args(["-c", script]) + .output() + .expect("failed to run python"); + assert!( + output.status.success(), + "Python script failed:\nSTDOUT: {}\nSTDERR: {}", + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr) + ); + String::from_utf8_lossy(&output.stdout).trim().to_string() +} + +/// Links in the big group, as libhdf5's `h5stat_newgrat.h5` has. +const LINKS: usize = 35_001; +/// Attributes on the dense-attribute dataset. +const ATTRS: usize = 3_000; + +/// Pairs of distinct names with equal lookup3 hashes, found by search (the +/// hash is fixed, so the pairs are too). +fn colliding_pairs(count: usize) -> Vec<(String, String)> { + let mut seen: HashMap = HashMap::new(); + let mut pairs = Vec::new(); + for i in 0.. { + let name = format!("c{i}"); + let h = jenkins_lookup3(name.as_bytes()); + if let Some(first) = seen.insert(h, name.clone()) { + pairs.push((first, name)); + if pairs.len() == count { + break; + } + } + } + pairs +} + +/// What h5py reads: link values, attribute values, and which of the +/// missing names it finds as links and as attributes (none). +type H5pyView = ( + BTreeMap, + BTreeMap, + Vec, + Vec, +); + +struct Fixture { + _dir: tempfile::TempDir, + path: String, + /// Names in the big group, with the value of the scalar dataset each + /// links to, as h5py reads them. + links: BTreeMap, + /// Names that are not links but hash like one that is. + missing_links: Vec, + /// Attributes of `/x`, as h5py reads them. + attrs: BTreeMap, + missing_attrs: Vec, +} + +fn fixture() -> &'static Fixture { + static FIXTURE: OnceLock = OnceLock::new(); + FIXTURE.get_or_init(|| { + let pairs = colliding_pairs(6); + for (a, b) in &pairs { + assert_ne!(a, b); + assert_eq!(jenkins_lookup3(a.as_bytes()), jenkins_lookup3(b.as_bytes())); + } + // Pairs 0-2 both present (either can be the one libhdf5 orders + // first), pairs 3-5 only the first: its partner must not be found. + // "k69209"/"k155448" is the pair the writer once misordered. + let mut present: Vec = vec!["k69209".into(), "k155448".into()]; + let mut missing: Vec = Vec::new(); + for (i, (a, b)) in pairs.into_iter().enumerate() { + present.push(a); + if i < 3 { + present.push(b); + } else { + missing.push(b); + } + } + missing.extend(["", "nope", "n35001x", "N1"].map(String::from)); + let mut links = present.clone(); + let mut i = 0; + while links.len() < LINKS { + links.push(format!("n{i}")); + i += 1; + } + let mut attrs = present.clone(); + attrs.extend((0..ATTRS - present.len()).map(|i| format!("a{i}"))); + + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("big.h5").display().to_string(); + // The names go through a file: 35 001 of them overflow an argument. + let names = dir.path().join("names.json"); + std::fs::write( + &names, + serde_json::to_string(&(&links, &attrs, &missing)).unwrap(), + ) + .unwrap(); + let names = names.display(); + let out = run_python(&format!( + "import h5py, json, numpy as np\n\ + links, attrs, missing = json.load(open(r'{names}'))\n\ + with h5py.File(r'{path}', 'w', libver='latest') as f:\n\ + \x20 g = f.create_group('g')\n\ + \x20 for i, n in enumerate(links):\n\ + \x20 g.create_dataset(n, data=np.int64(i))\n\ + \x20 x = f.create_dataset('x', data=np.int64(0))\n\ + \x20 for i, n in enumerate(attrs):\n\ + \x20 x.attrs[n] = np.int64(1000 + i)\n\ + with h5py.File(r'{path}', 'r') as f:\n\ + \x20 g, a = f['g'], f['x'].attrs\n\ + \x20 print(json.dumps([{{n: int(g[n][()]) for n in g}}, {{n: int(a[n]) for n in a}},\n\ + \x20 [n for n in missing if n and n in g], [n for n in missing if n and n in a]]))", + )); + let (links, attrs, found_links, found_attrs): H5pyView = + serde_json::from_str(&out).unwrap(); + assert_eq!(links.len(), LINKS); + assert_eq!(attrs.len(), ATTRS); + assert!(found_links.is_empty() && found_attrs.is_empty()); + Fixture { + _dir: dir, + path, + links, + missing_links: missing.clone(), + attrs, + missing_attrs: missing, + } + }) +} + +fn is_not_found(e: &clawhdf5::Error) -> bool { + matches!(e, clawhdf5::Error::Format(FormatError::PathNotFound(_))) +} + +#[test] +fn one_link_lookup_reads_the_index_not_every_link() { + skip_if_no_python!(); + let fx = fixture(); + let f = File::open(&fx.path).unwrap(); + let g = f.group("g").unwrap(); + for (name, value) in &fx.links { + lookup_stats::reset(); + let ds = g.dataset(name).unwrap(); + // One link decoded per lookup, two where hashes collide — not 35 001. + let read = lookup_stats::heap_objects_read(); + assert!(read <= 2, "looking up {name} read {read} heap objects"); + assert_eq!(ds.read_i64().unwrap(), vec![*value], "{name}"); + } + + for name in &fx.missing_links { + lookup_stats::reset(); + let err = g.dataset(name).unwrap_err(); + assert!(is_not_found(&err), "{name:?}: {err:?}"); + assert!(lookup_stats::heap_objects_read() <= 2, "{name:?}"); + } + + // A path resolves each component the same way. + for name in ["k155448", "n0", "n34000"] { + lookup_stats::reset(); + let ds = f.dataset(&format!("/g/{name}")).unwrap(); + assert!(lookup_stats::heap_objects_read() <= 2); + assert_eq!(ds.read_i64().unwrap(), vec![fx.links[name]]); + } +} + +#[test] +fn one_attribute_lookup_reads_the_index_not_every_attribute() { + skip_if_no_python!(); + let fx = fixture(); + let f = File::open(&fx.path).unwrap(); + let x = f.dataset("x").unwrap(); + let all = x.attrs().unwrap(); + assert_eq!(all.len(), ATTRS); + for (name, value) in &fx.attrs { + lookup_stats::reset(); + let got = x.attr(name).unwrap(); + assert!(lookup_stats::heap_objects_read() <= 2, "{name}"); + assert!( + matches!(got, Some(AttrValue::I64(v)) if v == *value), + "{name}: {got:?}" + ); + assert!(matches!(all.get(name), Some(AttrValue::I64(v)) if v == value)); + } + for name in &fx.missing_attrs { + lookup_stats::reset(); + assert!(x.attr(name).unwrap().is_none(), "{name:?}"); + assert!(lookup_stats::heap_objects_read() <= 2, "{name:?}"); + } + // Compact attributes (on the root group: none) and a group's attributes. + assert!(f.root().attr("k69209").unwrap().is_none()); +} + +/// Every child of the big group opened by name through each file type, +/// within `limit`: with a scan per lookup this is ~1.2e9 link decodes. +#[test] +fn opening_every_child_of_a_35001_link_group_by_name_is_quick() { + skip_if_no_python!(); + let fx = fixture(); + let limit = Duration::from_secs(120); + let started = Instant::now(); + let check_time = |n: usize| { + assert!( + started.elapsed() < limit, + "{n} lookups took {:?}", + started.elapsed() + ); + }; + + let f = File::open(&fx.path).unwrap(); + let g = f.group("g").unwrap(); + for (n, (name, value)) in fx.links.iter().enumerate() { + assert_eq!(g.dataset(name).unwrap().read_i64().unwrap(), vec![*value]); + check_time(n); + } + // The listing hands out entries: open each by address. + let entries = g.entries().unwrap(); + assert_eq!(entries.len(), LINKS); + for (name, address) in &entries { + let ds = f.dataset_at(*address).unwrap(); + assert_eq!(ds.read_i64().unwrap(), vec![fx.links[name]]); + } + assert!(f.group_at(g_address(&f)).dataset("n0").is_ok()); + + let m = MmapFile::open(&fx.path).unwrap(); + let mg = m.group("g").unwrap(); + for (n, (name, value)) in fx.links.iter().enumerate() { + assert_eq!(mg.dataset(name).unwrap().read_i64().unwrap(), vec![*value]); + check_time(n); + } + assert!(mg.group("nope").is_err_and(|e| is_not_found(&e))); + + let l = LazyFile::open_mmap(&fx.path).unwrap(); + let lg = l.group("g").unwrap(); + for (n, (name, value)) in fx.links.iter().enumerate() { + assert_eq!(lg.dataset(name).unwrap().read_i64().unwrap(), vec![*value]); + check_time(n); + } + let lx = l.dataset("x").unwrap(); + assert!( + matches!(lx.attr("k155448").unwrap(), Some(AttrValue::I64(v)) if v == fx.attrs["k155448"]) + ); + assert!( + lg.dataset(&fx.missing_links[0]) + .is_err_and(|e| is_not_found(&e)) + ); +} + +fn g_address(f: &File) -> u64 { + f.root() + .entries() + .unwrap() + .into_iter() + .find(|(n, _)| n == "g") + .unwrap() + .1 +} + +/// Every kind of link, looked up by name in a dense group (through the name +/// index) and in a compact one, opens what h5py opens and nothing it cannot: +/// hard links, soft links (absolute, relative, to a group), and not a +/// dangling soft link, an external link or a missing name. +#[test] +fn links_of_every_kind_resolve_by_name_as_in_h5py() { + skip_if_no_python!(); + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("links.h5").display().to_string(); + // For each group and name: "dataset ", "group", or "none" as + // h5py sees it. + let out = run_python(&format!( + "import h5py, json, numpy as np\n\ + with h5py.File(r'{path}', 'w', libver='latest') as f:\n\ + \x20 for gname, n in (('dense', 20), ('compact', 2)):\n\ + \x20 g = f.create_group(gname)\n\ + \x20 for i in range(n):\n\ + \x20 g.create_dataset(f'd{{i}}', data=np.int64(100 + i))\n\ + \x20 s = g.create_group('sub')\n\ + \x20 s.create_dataset('x', data=np.int64(7))\n\ + \x20 g['abs'] = h5py.SoftLink(f'/{{gname}}/d1')\n\ + \x20 g['rel'] = h5py.SoftLink('sub/x')\n\ + \x20 g['tosub'] = h5py.SoftLink('sub')\n\ + \x20 g['dangling'] = h5py.SoftLink('/nowhere')\n\ + \x20 g['ext'] = h5py.ExternalLink('other.h5', '/y')\n\ + names = ['d0', 'd1', 'sub', 'abs', 'rel', 'tosub', 'dangling', 'ext', 'nope', '']\n\ + seen = {{}}\n\ + with h5py.File(r'{path}', 'r') as f:\n\ + \x20 for gname in ('dense', 'compact'):\n\ + \x20 g = f[gname]\n\ + \x20 for n in names:\n\ + \x20 try:\n\ + \x20 o = g[n] if n else None\n\ + \x20 except (KeyError, OSError):\n\ + \x20 o = None\n\ + \x20 if isinstance(o, h5py.Dataset):\n\ + \x20 seen[f'{{gname}}/{{n}}'] = f'dataset {{int(o[()])}}'\n\ + \x20 elif isinstance(o, h5py.Group):\n\ + \x20 seen[f'{{gname}}/{{n}}'] = 'group'\n\ + \x20 else:\n\ + \x20 seen[f'{{gname}}/{{n}}'] = 'none'\n\ + print(json.dumps(seen))", + )); + let seen: BTreeMap = serde_json::from_str(&out).unwrap(); + assert_eq!(seen.len(), 20); + + let f = File::open(&path).unwrap(); + // The dense group's links are in a heap, the compact group's in its + // header. + for (gname, dense) in [("dense", true), ("compact", false)] { + let g = f.group(gname).unwrap(); + lookup_stats::reset(); + g.dataset("d0").unwrap(); + assert_eq!(lookup_stats::heap_objects_read() > 0, dense, "{gname}"); + } + let m = MmapFile::open(&path).unwrap(); + let l = LazyFile::open_mmap(&path).unwrap(); + for (key, want) in &seen { + let (gname, name) = key.split_once('/').unwrap(); + let got = { + let g = f.group(gname).unwrap(); + match (g.dataset(name), g.group(name)) { + (Ok(ds), _) => format!("dataset {}", ds.read_i64().unwrap()[0]), + (Err(clawhdf5::Error::NotADataset(_)), Ok(sub)) => { + // A group: it has the child `x` (checks the address). + assert!(sub.dataset("x").is_ok() || name == "sub" || name == "tosub"); + "group".to_string() + } + (Err(e), Err(e2)) => { + assert!(is_not_found(&e) && is_not_found(&e2), "{key}: {e:?} / {e2:?}"); + "none".to_string() + } + (Err(e), Ok(_)) => panic!("{key}: dataset {e:?} but group ok"), + } + }; + assert_eq!(&got, want, "{key}"); + // The other readers agree, and a path through the group resolves the + // same way. + let mg = m.group(gname).unwrap(); + let lg = l.group(gname).unwrap(); + match want.strip_prefix("dataset ") { + Some(v) => { + let v: i64 = v.parse().unwrap(); + assert_eq!(mg.dataset(name).unwrap().read_i64().unwrap(), vec![v]); + assert_eq!(lg.dataset(name).unwrap().read_i64().unwrap(), vec![v]); + let ds = f.dataset(&format!("/{gname}/{name}")).unwrap(); + assert_eq!(ds.read_i64().unwrap(), vec![v], "{key}"); + } + None if want == "group" => { + assert!(mg.group(name).unwrap().dataset("x").is_ok(), "{key}"); + assert!(lg.group(name).unwrap().dataset("x").is_ok(), "{key}"); + let ds = f.dataset(&format!("/{gname}/{name}/x")).unwrap(); + assert_eq!(ds.read_i64().unwrap(), vec![7]); + } + None => { + assert!(mg.dataset(name).is_err_and(|e| is_not_found(&e)), "{key}"); + assert!(lg.group(name).is_err_and(|e| is_not_found(&e)), "{key}"); + } + } + } +}