quantumclaw/clawhdf5
3
1
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 1 Wiki Activity

read: look names up through the dense name indexes

Browse Source 
Finding one link or attribute by name read every entry: Group::dataset and
Group::group (File, MmapFile, LazyFile) listed the whole group per call, and
path resolution scanned each group's links. Opening every child of a
35 001-link group by name decoded ~1.2e9 links.

Now a dense group's v2 B-tree name index (type 5, lookup3 hash of the
name) is descended to the records with the name's hash
(btree_v2::find_btree_v2_records reads only the nodes whose key interval
overlaps), and only those links are read and compared; all hash-equal
records are compared, so libhdf5's tie order does not matter. Dense
attributes the same through their type 8 index
(attribute::find_attribute_in_file, facade attr(name)); huge heap objects
through their ID-ordered index. group_v2::resolve_child returns what the
listing has under a name (soft links followed, dangling/external ones not
found). Group::entries and File::group_at hand out a listing's addresses.

The lookup-stats feature counts heap objects read. Tests: one lookup in
an h5py-written 35 001-link group with colliding hashes reads at most two
links (before: 35 001, failing), attribute lookups likewise (before: 3 000,
failing), every child opens through all three readers and matches h5py,
every link kind resolves as h5py resolves it in dense and compact groups,
300 huge attributes are found, and a range search matches a full scan at
every tree depth.

Co-Authored-By: Claude Opus 5.5 (1M context) <[email protected]>
This commit is contained in:
osobh
2026-09-26 13:33:19 -05:00
co-authored by Claude Opus 5.5
parent 6248b411f0
commit 02e89c1d2d
15 changed files with 1245 additions and 159 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/clawhdf5-format/Cargo.toml
+3
View File
@@ -80,6 +80,9 @@ blosc = ["lz4_flex", "ruzstd", "snap", "deflate", "std"]
blosc2 = ["blosc"]
# Every plugin filter above.
plugin-filters = ["lzf", "bitshuffle", "bzip2", "blosc", "blosc2"]
# Test instrumentation: per-thread counts of heap objects read (see
# `lookup_stats`), so tests can bound the cost of a name lookup.
lookup-stats = ["std"]
[[bench]]
name = "parallel_decompress_bench"
crates/clawhdf5-format/src/attribute.rs
+148 -33
View File
@@ -5,8 +5,10 @@ use alloc::{borrow::Cow, string::String, vec::Vec};
#[cfg(feature = "std")]
use std::borrow::Cow;
use crate::addr::to_usize;
use crate::attribute_info::AttributeInfoMessage;
use crate::btree_v2::{BTreeV2Header, collect_btree_v2_records};
use crate::btree_v2::{BTreeV2Header, collect_btree_v2_records, find_btree_v2_records};
use crate::checksum::jenkins_lookup3;
use crate::data_read;
use crate::dataspace::Dataspace;
use crate::datatype::Datatype;
@@ -341,7 +343,8 @@ fn compute_raw_data(
dataspace: &Dataspace,
datatype: &Datatype,
) -> Vec<u8> {
let num_elements = dataspace.num_elements() as usize;
// Saturating, like the product: the size is capped at what is there.
let num_elements = usize::try_from(dataspace.num_elements()).unwrap_or(usize::MAX);
let elem_size = datatype.type_size() as usize;
let expected_size = num_elements.saturating_mul(elem_size);
let available = data.len().saturating_sub(pos);
@@ -453,7 +456,145 @@ fn extract_attributes_with(
// Each attribute's creation order, where the file records one.
let mut orders: Vec<u32> = Vec::new();
// Collect compact attributes (inline in OH)
extract_compact_attributes(
file_data,
header,
offset_size,
length_size,
&mut attrs,
&mut orders,
on_error,
)?;
// Check for dense attributes via AttributeInfo message
let attr_info = find_attribute_info(header, offset_size)?;
if let Some(info) = &attr_info
&& let Some(fh_addr) = info.fractal_heap_address
{
extract_dense_attributes(
file_data,
info,
fh_addr,
offset_size,
length_size,
&mut attrs,
&mut orders,
on_error,
)?;
}
// An object that tracks attribute creation order lists its attributes
// in that order (h5py's `track_order=True`), as libhdf5 does; otherwise
// they come in storage order.
if attr_info.is_some_and(|i| i.max_creation_index.is_some()) {
let mut paired: Vec<(u32, AttributeMessage)> = orders.into_iter().zip(attrs).collect();
paired.sort_by_key(|(o, _)| *o);
attrs = paired.into_iter().map(|(_, a)| a).collect();
}
Ok(attrs)
}
/// B-tree v2 record type of dense attribute storage's name index.
const ATTRIBUTE_NAME_INDEX: u8 = 8;
/// The attribute called `name` on the object with header `header`: the
/// first one [`extract_attributes_tolerant`] returns under that name, or
/// `None` if it returns none (an attribute that cannot be read is not
/// returned there either).
///
/// Compact attributes are in the header and are scanned. Dense attributes
/// are found through the name index (a v2 B-tree of lookup3 name hashes,
/// record type 8): only the attributes whose names hash like `name` are read
/// from the heap, O(log n) instead of all of them. Errors in the structures
/// that index the attributes fail the call, as they fail a listing.
pub fn find_attribute_in_file(
file_data: &[u8],
header: &ObjectHeader,
name: &str,
offset_size: u8,
length_size: u8,
) -> Result<Option<AttributeMessage>, FormatError> {
let attr_info = find_attribute_info(header, offset_size)?;
let dense = attr_info
.as_ref()
.and_then(|i| Some((i.fractal_heap_address?, i.btree_name_index_address?)));
let Some((fh_addr, btree_addr)) = dense else {
// Compact only (or dense storage without a name index, which a
// listing reports): as a listing finds it.
return Ok(
extract_attributes_tolerant(file_data, header, offset_size, length_size)?
.0
.into_iter()
.find(|a| a.name == name),
);
};
let btree_hdr =
BTreeV2Header::parse(file_data, to_usize(btree_addr)?, offset_size, length_size)?;
let fh = FractalHeapHeader::parse(file_data, to_usize(fh_addr)?, offset_size, length_size)?;
if btree_hdr.tree_type != ATTRIBUTE_NAME_INDEX || btree_hdr.record_size < 4 {
return Ok(
extract_attributes_tolerant(file_data, header, offset_size, length_size)?
.0
.into_iter()
.find(|a| a.name == name),
);
}
// A listing has the compact attributes first.
let mut compact = Vec::new();
extract_compact_attributes(
file_data,
header,
offset_size,
length_size,
&mut compact,
&mut Vec::new(),
&mut |_| Ok(()),
)?;
if let Some(a) = compact.into_iter().find(|a| a.name == name) {
return Ok(Some(a));
}
// Record: heap ID + message flags(1) + creation order(4) + hash(4); the
// hash is the last field.
let hash = jenkins_lookup3(name.as_bytes());
let hash_at = usize::from(btree_hdr.record_size) - 4;
let records = find_btree_v2_records(file_data, &btree_hdr, offset_size, &mut |r| match r
.get(hash_at..hash_at + 4)
{
Some(h) => u32::from_le_bytes([h[0], h[1], h[2], h[3]]).cmp(&hash),
None => core::cmp::Ordering::Less,
})?;
let id_len = usize::from(fh.heap_id_length);
for record in &records {
let Some(id_bytes) = record.data.get(..id_len) else {
continue;
};
let attr = fh
.read_managed_object(file_data, id_bytes, offset_size)
.and_then(|d| AttributeMessage::parse_in_file(&d, file_data, offset_size, length_size));
// One that cannot be read is left out, as from a listing.
if let Ok(attr) = attr
&& attr.name == name
{
return Ok(Some(attr));
}
}
Ok(None)
}
/// The attributes stored in the object header itself (compact storage), and
/// each one's creation order into `orders`.
fn extract_compact_attributes(
file_data: &[u8],
header: &ObjectHeader,
offset_size: u8,
length_size: u8,
attrs: &mut Vec<AttributeMessage>,
orders: &mut Vec<u32>,
on_error: &mut dyn FnMut(FormatError) -> Result<(), FormatError>,
) -> Result<(), FormatError> {
for msg in &header.messages {
if msg.msg_type == MessageType::Attribute {
let attr = if shared_message::is_shared(msg.flags) {
@@ -489,34 +630,7 @@ fn extract_attributes_with(
}
}
}
// Check for dense attributes via AttributeInfo message
let attr_info = find_attribute_info(header, offset_size)?;
if let Some(info) = &attr_info
&& let Some(fh_addr) = info.fractal_heap_address
{
extract_dense_attributes(
file_data,
info,
fh_addr,
offset_size,
length_size,
&mut attrs,
&mut orders,
on_error,
)?;
}
// An object that tracks attribute creation order lists its attributes
// in that order (h5py's `track_order=True`), as libhdf5 does; otherwise
// they come in storage order.
if attr_info.is_some_and(|i| i.max_creation_index.is_some()) {
let mut paired: Vec<(u32, AttributeMessage)> = orders.into_iter().zip(attrs).collect();
paired.sort_by_key(|(o, _)| *o);
attrs = paired.into_iter().map(|(_, a)| a).collect();
}
Ok(attrs)
Ok(())
}
/// Find and parse the Attribute Info message from an object header.
@@ -547,7 +661,7 @@ fn extract_dense_attributes(
on_error: &mut dyn FnMut(FormatError) -> Result<(), FormatError>,
) -> Result<(), FormatError> {
// Parse fractal heap
let fh = FractalHeapHeader::parse(file_data, fh_addr as usize, offset_size, length_size)?;
let fh = FractalHeapHeader::parse(file_data, to_usize(fh_addr)?, offset_size, length_size)?;
// Parse B-tree v2 for name index (type 8)
let btree_addr = attr_info
@@ -556,7 +670,8 @@ fn extract_dense_attributes(
expected: 1,
available: 0,
})?;
let btree_hdr = BTreeV2Header::parse(file_data, btree_addr as usize, offset_size, length_size)?;
let btree_hdr =
BTreeV2Header::parse(file_data, to_usize(btree_addr)?, offset_size, length_size)?;
let records = collect_btree_v2_records(file_data, &btree_hdr, offset_size, length_size)?;
for record in &records {
crates/clawhdf5-format/src/btree_v2.rs
+179 -37
View File
@@ -2,10 +2,12 @@
#[cfg(not(feature = "std"))]
use alloc::vec::Vec;
use core::cmp::Ordering;
#[cfg(feature = "checksum")]
use byteorder::{ByteOrder, LittleEndian};
use crate::addr::to_usize;
use crate::error::FormatError;
/// Parsed B-tree v2 header (signature "BTHD").
@@ -216,7 +218,7 @@ pub fn collect_btree_v2_records(
// Root is a leaf
parse_leaf_records(
file_data,
header.root_node_address as usize,
to_usize(header.root_node_address)?,
header.num_records_in_root,
header.record_size,
)
@@ -225,7 +227,7 @@ pub fn collect_btree_v2_records(
let mut records = Vec::new();
collect_internal_records(
file_data,
header.root_node_address as usize,
to_usize(header.root_node_address)?,
header.num_records_in_root,
header.depth,
header.record_size,
@@ -289,9 +291,10 @@ fn parse_leaf_records(
Ok(records)
}
/// Recursively collect records from an internal node.
#[allow(clippy::too_many_arguments, clippy::only_used_in_recursion)]
fn collect_internal_records(
/// An internal node's layout: where its records start, and its children as
/// `(address, record count)`.
#[allow(clippy::too_many_arguments)]
fn read_internal_node(
file_data: &[u8],
offset: usize,
num_records: u16,
@@ -299,11 +302,8 @@ fn collect_internal_records(
record_size: u16,
node_size: u32,
offset_size: u8,
length_size: u8,
max_leaf_nrec: u64,
budget: &mut usize,
out: &mut Vec<BTreeV2Record>,
) -> Result<(), FormatError> {
) -> Result<(usize, Vec<(u64, u16)>), FormatError> {
// signature(4) + version(1) + type(1) = 6
ensure_len(file_data, offset, 6)?;
if &file_data[offset..offset + 4] != b"BTIN" {
@@ -314,7 +314,7 @@ fn collect_internal_records(
let rs = record_size as usize;
let mut pos = offset + 6;
// Read all records first
// Records first
let records_total = nr.checked_mul(rs).ok_or(FormatError::UnexpectedEof {
expected: usize::MAX,
available: file_data.len(),
@@ -346,7 +346,6 @@ fn collect_internal_records(
let child_ptr_size = offset_size as usize + nrec_width + total_nrec_width;
ensure_len(file_data, pos, num_children * child_ptr_size)?;
// Read child pointers
let mut children = Vec::with_capacity(num_children);
for _ in 0..num_children {
let addr = read_offset(file_data, pos, offset_size)?;
@@ -356,6 +355,61 @@ fn collect_internal_records(
pos += total_nrec_width; // skip total records in subtree
children.push((addr, child_nrec));
}
Ok((records_start, children))
}
/// Record `i` of an internal node whose records start at `records_start`.
fn internal_record(
file_data: &[u8],
records_start: usize,
i: usize,
rs: usize,
) -> Result<&[u8], FormatError> {
let overflow = || FormatError::UnexpectedEof {
expected: usize::MAX,
available: file_data.len(),
};
let rec_start = i
.checked_mul(rs)
.and_then(|o| records_start.checked_add(o))
.ok_or_else(overflow)?;
let rec_end = rec_start.checked_add(rs).ok_or_else(overflow)?;
file_data
.get(rec_start..rec_end)
.ok_or(FormatError::UnexpectedEof {
expected: rec_end,
available: file_data.len(),
})
}
/// Recursively collect records from an internal node.
#[allow(clippy::too_many_arguments, clippy::only_used_in_recursion)]
fn collect_internal_records(
file_data: &[u8],
offset: usize,
num_records: u16,
depth: u16,
record_size: u16,
node_size: u32,
offset_size: u8,
length_size: u8,
max_leaf_nrec: u64,
budget: &mut usize,
out: &mut Vec<BTreeV2Record>,
) -> Result<(), FormatError> {
let nr = num_records as usize;
let rs = record_size as usize;
let (records_start, children) = read_internal_node(
file_data,
offset,
num_records,
depth,
record_size,
node_size,
offset_size,
max_leaf_nrec,
)?;
let child_depth = depth - 1;
// Interleave: child[0], record[0], child[1], record[1], ..., child[nr]
// We collect child[0] records, then record[0], then child[1], etc.
@@ -364,12 +418,12 @@ fn collect_internal_records(
// Before parsing, so a refused tree is not also a large allocation.
spend(budget, usize::from(child_nrec))?;
let leaf_recs =
parse_leaf_records(file_data, child_addr as usize, child_nrec, record_size)?;
parse_leaf_records(file_data, to_usize(child_addr)?, child_nrec, record_size)?;
out.extend(leaf_recs);
} else {
collect_internal_records(
file_data,
child_addr as usize,
to_usize(child_addr)?,
child_nrec,
child_depth,
record_size,
@@ -384,32 +438,10 @@ fn collect_internal_records(
// Add record[i] (except after the last child)
if i < nr {
let rec_offset = i.checked_mul(rs).ok_or(FormatError::UnexpectedEof {
expected: usize::MAX,
available: file_data.len(),
})?;
let rec_start =
records_start
.checked_add(rec_offset)
.ok_or(FormatError::UnexpectedEof {
expected: usize::MAX,
available: file_data.len(),
})?;
let rec_end = rec_start
.checked_add(rs)
.ok_or(FormatError::UnexpectedEof {
expected: usize::MAX,
available: file_data.len(),
})?;
if rec_end > file_data.len() {
return Err(FormatError::UnexpectedEof {
expected: rec_end,
available: file_data.len(),
});
}
let data = internal_record(file_data, records_start, i, rs)?;
spend(budget, 1)?;
out.push(BTreeV2Record {
data: file_data[rec_start..rec_end].to_vec(),
data: data.to_vec(),
});
}
}
@@ -417,6 +449,116 @@ fn collect_internal_records(
Ok(())
}
/// The records of a B-tree v2 that fall in one key range, found by
/// descending the tree instead of reading all of it.
///
/// `cmp` places a record relative to the range: `Less` if the record sorts
/// before it, `Greater` if after, `Equal` if the record is in it. The tree
/// must be ordered consistently with `cmp`, as libhdf5 orders it (a link or
/// attribute name index by name hash, so all records with one hash form a
/// range whatever order their names are in). Only the nodes whose key
/// interval overlaps the range are read: O(depth) nodes plus those holding
/// the matches. Matches come in tree order.
pub fn find_btree_v2_records(
file_data: &[u8],
header: &BTreeV2Header,
offset_size: u8,
cmp: &mut dyn FnMut(&[u8]) -> Ordering,
) -> Result<Vec<BTreeV2Record>, FormatError> {
if header.total_records == 0 || header.num_records_in_root == 0 {
return Ok(Vec::new());
}
if header.depth > MAX_DEPTH {
return Err(FormatError::NestingDepthExceeded);
}
// As in `collect_btree_v2_records`: a valid tree cannot hold more
// records than the file has room for, however its children are shared.
let mut budget = file_data.len() / usize::from(header.record_size.max(1));
let max_leaf_nrec = max_records_leaf(header.node_size, header.record_size);
let mut out = Vec::new();
find_in_node(
file_data,
header,
to_usize(header.root_node_address)?,
header.num_records_in_root,
header.depth,
offset_size,
max_leaf_nrec,
cmp,
&mut budget,
&mut out,
)?;
Ok(out)
}
#[allow(clippy::too_many_arguments)]
fn find_in_node(
file_data: &[u8],
header: &BTreeV2Header,
offset: usize,
num_records: u16,
depth: u16,
offset_size: u8,
max_leaf_nrec: u64,
cmp: &mut dyn FnMut(&[u8]) -> Ordering,
budget: &mut usize,
out: &mut Vec<BTreeV2Record>,
) -> Result<(), FormatError> {
spend(budget, usize::from(num_records))?;
if depth == 0 {
let records = parse_leaf_records(file_data, offset, num_records, header.record_size)?;
out.extend(
records
.into_iter()
.filter(|r| cmp(&r.data) == Ordering::Equal),
);
return Ok(());
}
let rs = usize::from(header.record_size);
let (records_start, children) = read_internal_node(
file_data,
offset,
num_records,
depth,
header.record_size,
header.node_size,
offset_size,
max_leaf_nrec,
)?;
let nr = usize::from(num_records);
let mut order = Vec::with_capacity(nr);
for i in 0..nr {
order.push(cmp(internal_record(file_data, records_start, i, rs)?));
}
// Child `i` holds the keys between record `i - 1` and record `i`: it can
// hold a match unless the record before it is already past the range or
// the record after it is still before it.
for (i, &(child_addr, child_nrec)) in children.iter().enumerate() {
let after_left = i == 0 || order[i - 1] != Ordering::Greater;
let before_right = i == nr || order[i] != Ordering::Less;
if after_left && before_right {
find_in_node(
file_data,
header,
to_usize(child_addr)?,
child_nrec,
depth - 1,
offset_size,
max_leaf_nrec,
cmp,
budget,
out,
)?;
}
if i < nr && order[i] == Ordering::Equal {
out.push(BTreeV2Record {
data: internal_record(file_data, records_start, i, rs)?.to_vec(),
});
}
}
Ok(())
}
/// Most records a subtree whose root is at `depth` can hold (libhdf5's
/// `cum_max_nrec`). See [`node_info`].
fn cum_max_records(
crates/clawhdf5-format/src/btree_v2_write.rs
+53
View File
@@ -385,6 +385,59 @@ mod tests {
assert!(nodes > 0);
}
/// Descending to a key range finds exactly the records a full read
/// holds in it — runs of equal keys that straddle node boundaries
/// included — at every depth, and nothing for keys not in the tree.
#[test]
fn a_key_range_search_matches_a_full_scan() {
use crate::btree_v2::find_btree_v2_records;
use core::cmp::Ordering;
let rs = 11usize;
// Keys 0, 0, 0, 2, 2, 2, 4, ...: runs of three, odd keys missing.
for n in [1usize, 45, 46, 1150, 30_000] {
let mut recs = Vec::with_capacity(n * rs);
for i in 0..n {
let mut r = vec![0u8; rs];
r[..8].copy_from_slice(&((i / 3 * 2) as u64).to_be_bytes());
r[8..].copy_from_slice(&[(i % 3) as u8, 0, 0]);
recs.extend_from_slice(&r);
}
let base = 4096u64;
let tree = build_btree_v2(params(512, 11), &recs, base, 8, 8).unwrap();
let mut file = vec![0u8; base as usize];
file.extend_from_slice(&tree);
let hdr = BTreeV2Header::parse(&file, base as usize, 8, 8).unwrap();
let all = collect_btree_v2_records(&file, &hdr, 8, 8).unwrap();
let key = |r: &[u8]| u64::from_be_bytes(r[..8].try_into().unwrap());
let last = key(&all[n - 1].data);
let probes = (0..=last + 1).step_by(if n > 1000 { 37 } else { 1 });
for k in probes.chain([last, last + 1, u64::MAX]) {
let found =
find_btree_v2_records(&file, &hdr, 8, &mut |r: &[u8]| key(r).cmp(&k)).unwrap();
let want: Vec<&[u8]> = all
.iter()
.map(|r| r.data.as_slice())
.filter(|r| key(r) == k)
.collect();
let got: Vec<&[u8]> = found.iter().map(|r| r.data.as_slice()).collect();
assert_eq!(got, want, "n {n} key {k}");
assert_eq!(
got.len(),
if k % 2 == 0 && k <= last {
want.len()
} else {
0
}
);
}
// Every record, or none, when the whole tree is in or out of range.
let every = find_btree_v2_records(&file, &hdr, 8, &mut |_| Ordering::Equal).unwrap();
assert_eq!(every.len(), n);
let none = find_btree_v2_records(&file, &hdr, 8, &mut |_| Ordering::Less).unwrap();
assert!(none.is_empty());
}
}
#[test]
fn a_node_too_small_or_too_big_is_an_error() {
assert!(build_btree_v2(params(16, 11), &records(1, 11), 0, 8, 8).is_err());
crates/clawhdf5-format/src/fractal_heap.rs
+20 -14
View File
@@ -6,7 +6,8 @@ use alloc::{format, vec::Vec};
#[cfg(feature = "checksum")]
use byteorder::{ByteOrder, LittleEndian};
use crate::btree_v2::{BTreeV2Header, collect_btree_v2_records};
use crate::addr::to_usize;
use crate::btree_v2::{BTreeV2Header, find_btree_v2_records};
use crate::error::FormatError;
use crate::filter_pipeline::FilterPipeline;
@@ -354,6 +355,7 @@ impl FractalHeapHeader {
id_bytes: &[u8],
offset_size: u8,
) -> Result<Vec<u8>, FormatError> {
crate::lookup_stats::heap_object_read();
let Some(&first) = id_bytes.first() else {
return Err(FormatError::UnexpectedEof {
expected: 1,
@@ -448,7 +450,7 @@ impl FractalHeapHeader {
}
let hdr = BTreeV2Header::parse(
file_data,
self.huge_btree_address as usize,
to_usize(self.huge_btree_address)?,
self.offset_size,
self.length_size,
)?;
@@ -463,8 +465,12 @@ impl FractalHeapHeader {
if hdr.tree_type != expected_type || usize::from(hdr.record_size) < rec_len {
return Err(heap_error("unexpected huge-object B-tree record type"));
}
let records =
collect_btree_v2_records(file_data, &hdr, self.offset_size, self.length_size)?;
// Records are ordered by ID (the last field): descend to the ones
// equal to `key` instead of reading the whole index.
let id_at = rec_len - ls;
let records = find_btree_v2_records(file_data, &hdr, self.offset_size, &mut |r| {
le_uint(&r[id_at..id_at + ls]).cmp(&key)
})?;
for rec in &records {
let d = &rec.data;
if d.len() < rec_len {
@@ -533,24 +539,24 @@ impl FractalHeapHeader {
self.read_from_direct_block(
file_data,
DirectBlock {
addr: self.root_block_address as usize,
addr: to_usize(self.root_block_address)?,
size: self.starting_block_size,
heap_offset: 0,
filtered_size: self.root_direct_block_filtered_size,
filter_mask: self.root_direct_block_filter_mask,
},
heap_offset,
obj_len as usize,
to_usize(obj_len)?,
)
} else {
// Root is an indirect block — limit recursion to 64 levels
self.read_from_indirect_block(
file_data,
self.root_block_address as usize,
to_usize(self.root_block_address)?,
self.current_rows_in_root_indirect_block,
0, // block offset
heap_offset,
obj_len as usize,
to_usize(obj_len)?,
offset_size,
64, // max recursion depth
)
@@ -572,11 +578,11 @@ impl FractalHeapHeader {
) -> Result<Vec<u8>, FormatError> {
if target_offset < block.heap_offset {
return Err(FormatError::UnexpectedEof {
expected: block.heap_offset as usize,
available: target_offset as usize,
expected: to_usize(block.heap_offset)?,
available: to_usize(target_offset)?,
});
}
let local_offset = (target_offset - block.heap_offset) as usize;
let local_offset = to_usize(target_offset - block.heap_offset)?;
if let Some(pipeline) = &self.filter_pipeline {
let stored_len = usize::try_from(block.filtered_size)
.map_err(|_| heap_error("direct block size"))?;
@@ -673,7 +679,7 @@ impl FractalHeapHeader {
return self.read_from_direct_block(
file_data,
DirectBlock {
addr: child_addr as usize,
addr: to_usize(child_addr)?,
size: block_size,
heap_offset: current_heap_offset,
filtered_size,
@@ -705,7 +711,7 @@ impl FractalHeapHeader {
{
return self.read_from_indirect_block(
file_data,
child_addr as usize,
to_usize(child_addr)?,
child_nrows,
current_heap_offset,
target_offset,
@@ -719,7 +725,7 @@ impl FractalHeapHeader {
}
Err(FormatError::UnexpectedEof {
expected: target_offset as usize + length,
expected: to_usize(target_offset)?.saturating_add(length),
available: file_data.len(),
})
}
crates/clawhdf5-format/src/group_v2.rs
+185 -32
View File
@@ -6,7 +6,9 @@
#[cfg(not(feature = "std"))]
use alloc::{string::String, vec::Vec};
use crate::btree_v2::{BTreeV2Header, collect_btree_v2_records};
use crate::addr::to_usize;
use crate::btree_v2::{BTreeV2Header, collect_btree_v2_records, find_btree_v2_records};
use crate::checksum::jenkins_lookup3;
use crate::error::FormatError;
use crate::fractal_heap::FractalHeapHeader;
use crate::group_v1::{self, GroupEntry};
@@ -93,13 +95,14 @@ fn for_each_dense_link(
mut visit: impl FnMut(LinkMessage),
) -> Result<(), FormatError> {
// Parse fractal heap
let fh = FractalHeapHeader::parse(file_data, fh_addr as usize, offset_size, length_size)?;
let fh = FractalHeapHeader::parse(file_data, to_usize(fh_addr)?, offset_size, length_size)?;
// Parse B-tree v2 for name index
let btree_addr = link_info
.btree_name_index_address
.ok_or_else(|| FormatError::PathNotFound(String::from("no B-tree v2 name index")))?;
let btree_hdr = BTreeV2Header::parse(file_data, btree_addr as usize, offset_size, length_size)?;
let btree_hdr =
BTreeV2Header::parse(file_data, to_usize(btree_addr)?, offset_size, length_size)?;
let records = collect_btree_v2_records(file_data, &btree_hdr, offset_size, length_size)?;
for record in &records {
@@ -181,10 +184,55 @@ fn find_symbolic_link(
if !is_v2_group(object_header) {
return Ok(None);
}
let is_symbolic = |t: &LinkTarget| !matches!(t, LinkTarget::Hard { .. });
// As when all links were scanned: the last symbolic link of that name.
Ok(
links_named(file_data, object_header, name, offset_size, length_size)?
.into_iter()
.rev()
.map(|link| link.link_target)
.find(|t| !matches!(t, LinkTarget::Hard { .. })),
)
}
/// B-tree v2 record type of a dense group's link name index.
const LINK_NAME_INDEX: u8 = 5;
/// The links called `name` in a v2 group (a valid group has at most one).
///
/// In dense storage the link name index (a v2 B-tree of lookup3 name
/// hashes, record type 5) is descended to the records with the name's hash,
/// and only their links are read from the heap — O(log n) instead of every
/// link. libhdf5 orders records with equal hashes by name; all of them are
/// read and compared here, so that order does not matter. An index of
/// another type is scanned in full.
fn links_named(
file_data: &[u8],
object_header: &ObjectHeader,
name: &str,
offset_size: u8,
length_size: u8,
) -> Result<Vec<LinkMessage>, FormatError> {
let mut found = Vec::new();
let link_info = find_link_info(object_header, offset_size)?;
let mut found = None;
if let Some(fh_addr) = link_info.fractal_heap_address {
let Some(fh_addr) = link_info.fractal_heap_address else {
for msg in &object_header.messages {
if msg.msg_type == MessageType::Link
&& let Some(link) = parse_link(&msg.data, offset_size)?
&& link.name == name
{
found.push(link);
}
}
return Ok(found);
};
let fh = FractalHeapHeader::parse(file_data, to_usize(fh_addr)?, offset_size, length_size)?;
let btree_addr = link_info
.btree_name_index_address
.ok_or_else(|| FormatError::PathNotFound(String::from("no B-tree v2 name index")))?;
let btree_hdr =
BTreeV2Header::parse(file_data, to_usize(btree_addr)?, offset_size, length_size)?;
if btree_hdr.tree_type != LINK_NAME_INDEX {
for_each_dense_link(
file_data,
&link_info,
@@ -192,26 +240,134 @@ fn find_symbolic_link(
offset_size,
length_size,
|link| {
if link.name == name && is_symbolic(&link.link_target) {
found = Some(link.link_target);
if link.name == name {
found.push(link);
}
},
)?;
} else {
for msg in &object_header.messages {
if msg.msg_type == MessageType::Link {
let Some(link) = parse_link(&msg.data, offset_size)? else {
continue;
};
if link.name == name && is_symbolic(&link.link_target) {
found = Some(link.link_target);
}
}
return Ok(found);
}
// Record: hash(4) + heap ID.
let hash = jenkins_lookup3(name.as_bytes());
let records = find_btree_v2_records(file_data, &btree_hdr, offset_size, &mut |r| {
match r.get(..4) {
Some(h) => u32::from_le_bytes([h[0], h[1], h[2], h[3]]).cmp(&hash),
// Too short to hold a hash (a corrupt record size): never a match.
None => core::cmp::Ordering::Less,
}
})?;
let id_len = usize::from(fh.heap_id_length);
for record in &records {
let Some(id_bytes) = record.data.get(4..4 + id_len) else {
continue;
};
let link_data = fh.read_managed_object(file_data, id_bytes, offset_size)?;
if let Some(link) = parse_link(&link_data, offset_size)?
&& link.name == name
{
found.push(link);
}
}
Ok(found)
}
/// The link [`resolve_path_any`] follows for one path component `name` of
/// the group with header `object_header`: a hard link (as `Hard`), else a
/// soft or external link of that name, else `None`. Fails with
/// `PathNotFound` if the object is not a group.
fn lookup_link(
file_data: &[u8],
object_header: &ObjectHeader,
name: &str,
offset_size: u8,
length_size: u8,
) -> Result<Option<LinkTarget>, FormatError> {
if is_v1_group(object_header) {
let entries = resolve_group_entries(file_data, object_header, offset_size, length_size)?;
if let Some(e) = entries
.iter()
.find(|e| e.name == name && e.object_header_address != u64::MAX)
{
return Ok(Some(LinkTarget::Hard {
object_header_address: e.object_header_address,
}));
}
return find_symbolic_link(file_data, object_header, name, offset_size, length_size);
}
if !is_v2_group(object_header) {
return Err(FormatError::PathNotFound(String::from(
"object header is not a group",
)));
}
let links = links_named(file_data, object_header, name, offset_size, length_size)?;
if let Some(addr) = links.iter().find_map(|l| match l.link_target {
LinkTarget::Hard {
object_header_address,
} if object_header_address != u64::MAX => Some(object_header_address),
_ => None,
}) {
return Ok(Some(LinkTarget::Hard {
object_header_address: addr,
}));
}
Ok(links
.into_iter()
.rev()
.map(|link| link.link_target)
.find(|t| !matches!(t, LinkTarget::Hard { .. })))
}
/// The object header address of the child called `name` of the group at
/// `group_address`: the address [`resolve_group_children`] lists under that
/// name, or `PathNotFound` if it lists none.
///
/// A dense group's child is found through its link name index (see
/// [`links_named`]) and only the named link is read and, if it is a soft
/// link, followed — not every link in the group. A v1 group is listed.
pub fn resolve_child(
file_data: &[u8],
superblock: &Superblock,
group_address: u64,
name: &str,
) -> Result<u64, FormatError> {
let os = superblock.offset_size;
let ls = superblock.length_size;
let not_found = || FormatError::PathNotFound(String::from(name));
let header = ObjectHeader::parse(file_data, to_usize(group_address)?, os, ls)?;
if !is_v2_group(&header) || is_v1_group(&header) {
return resolve_group_children(file_data, superblock, group_address)?
.into_iter()
.find(|e| e.name == name)
.map(|e| e.object_header_address)
.ok_or_else(not_found);
}
let links = links_named(file_data, &header, name, os, ls)?;
// The listing puts hard links before resolved soft links.
if let Some(addr) = links.iter().find_map(|l| match l.link_target {
LinkTarget::Hard {
object_header_address,
} => Some(object_header_address),
_ => None,
}) {
return Ok(addr);
}
for link in links {
if let LinkTarget::Soft { target_path } = link.link_target {
return match resolve_path_from(file_data, superblock, group_address, &target_path) {
// Left out of the listing: dangling, cyclic, or in another file.
Err(
FormatError::PathNotFound(_)
| FormatError::NestingDepthExceeded
| FormatError::ExternalLinkUnsupported { .. },
) => Err(not_found()),
other => other,
};
}
}
Err(not_found())
}
/// Find and parse the Link Info message from an object header.
fn find_link_info(
object_header: &ObjectHeader,
@@ -298,7 +454,7 @@ pub fn resolve_group_children(
) -> Result<Vec<GroupEntry>, FormatError> {
let os = superblock.offset_size;
let ls = superblock.length_size;
let header = ObjectHeader::parse(file_data, group_address as usize, os, ls)?;
let header = ObjectHeader::parse(file_data, to_usize(group_address)?, os, ls)?;
let mut entries = Vec::new();
let mut soft = Vec::new();
@@ -383,24 +539,21 @@ fn resolve_path_following_links(
let ls = superblock.length_size;
let mut current_addr = start;
let mut current_header = ObjectHeader::parse(file_data, start as usize, os, ls)?;
let mut current_header = ObjectHeader::parse(file_data, to_usize(start)?, os, ls)?;
for (i, component) in components.iter().enumerate() {
let entries = resolve_group_entries(file_data, &current_header, os, ls)?;
let found = entries
.iter()
.find(|e| e.name == *component && e.object_header_address != u64::MAX);
match found {
Some(entry) => {
match lookup_link(file_data, &current_header, component, os, ls)? {
Some(LinkTarget::Hard {
object_header_address,
}) => {
if i == components.len() - 1 {
return Ok(entry.object_header_address);
return Ok(object_header_address);
}
current_addr = entry.object_header_address;
current_header = ObjectHeader::parse(file_data, current_addr as usize, os, ls)?;
current_addr = object_header_address;
current_header = ObjectHeader::parse(file_data, to_usize(current_addr)?, os, ls)?;
}
None => {
return match find_symbolic_link(file_data, &current_header, component, os, ls)? {
found => {
return match found {
Some(LinkTarget::Soft { target_path }) => {
if depth >= MAX_SOFT_LINK_DEPTH {
return Err(FormatError::NestingDepthExceeded);
crates/clawhdf5-format/src/lib.rs
+1
View File
@@ -108,6 +108,7 @@ pub mod lane_partition;
pub mod link_info;
pub mod link_message;
pub mod local_heap;
pub mod lookup_stats;
pub mod message_type;
pub mod metadata_cache;
pub mod metadata_index;
crates/clawhdf5-format/src/lookup_stats.rs
+31
View File
@@ -0,0 +1,31 @@
//! Work counters for tests of lookup cost (feature `lookup-stats`).
//!
//! Counts fractal-heap objects read — each is one link or attribute message
//! decoded out of a dense group or dense attribute storage — so a test can
//! check that finding one name reads a handful of them, not the whole group.
//! Per thread, so tests running in parallel do not see each other's reads.
//! Without the feature the counting compiles to nothing.
#[cfg(feature = "lookup-stats")]
std::thread_local! {
static HEAP_OBJECTS: core::cell::Cell<u64> = const { core::cell::Cell::new(0) };
}
/// Record one heap object read.
#[inline(always)]
pub(crate) fn heap_object_read() {
#[cfg(feature = "lookup-stats")]
HEAP_OBJECTS.with(|c| c.set(c.get() + 1));
}
/// Heap objects read on this thread since the last [`reset`].
#[cfg(feature = "lookup-stats")]
pub fn heap_objects_read() -> u64 {
HEAP_OBJECTS.with(core::cell::Cell::get)
}
/// Zero this thread's counters.
#[cfg(feature = "lookup-stats")]
pub fn reset() {
HEAP_OBJECTS.with(|c| c.set(0));
}
crates/clawhdf5/Cargo.toml
+2 -1
View File
@@ -19,7 +19,8 @@ rayon = { version = "1", optional = true }
tempfile = { workspace = true }
criterion = { workspace = true }
clawhdf5-io = { path = "../clawhdf5-io", version = "2.7.0", features = ["mmap"] }
clawhdf5-format = { path = "../clawhdf5-format", version = "2.7.0", features = ["parallel", "fast-checksum"] }
clawhdf5-format = { path = "../clawhdf5-format", version = "2.7.0", features = ["parallel", "fast-checksum", "lookup-stats"] }
serde_json = "1"
clawhdf5-filters = { path = "../clawhdf5-filters", version = "2.7.0" }
[[bench]]
crates/clawhdf5/src/lazy.rs
+44 -13
View File
@@ -28,7 +28,7 @@ use clawhdf5_format::superblock::Superblock;
use clawhdf5_io::HDF5Read;
use crate::error::Error;
use crate::types::{AttrValue, DType, classify_datatype, read_attrs};
use crate::types::{AttrValue, DType, classify_datatype, read_attr, read_attrs};
/// A lazy HDF5 file handle that parses metadata on demand.
///
@@ -304,12 +304,7 @@ impl<'f, R: HDF5Read> LazyGroup<'f, R> {
/// Get a dataset within this group by name.
pub fn dataset(&self, name: &str) -> Result<LazyDataset<'f, R>, Error> {
let entries = self.children()?;
let entry = entries
.iter()
.find(|e| e.name == name)
.ok_or_else(|| Error::Format(FormatError::PathNotFound(name.to_string())))?;
let hdr = self.file.get_or_parse_header(entry.object_header_address)?;
let hdr = self.file.get_or_parse_header(self.child_address(name)?)?;
if !has_message(&hdr, MessageType::DataLayout) {
return Err(Error::NotADataset(name.to_string()));
}
@@ -322,17 +317,38 @@ impl<'f, R: HDF5Read> LazyGroup<'f, R> {
/// Get a subgroup within this group by name.
pub fn group(&self, name: &str) -> Result<LazyGroup<'f, R>, Error> {
let entries = self.children()?;
let entry = entries
.iter()
.find(|e| e.name == name)
.ok_or_else(|| Error::Format(FormatError::PathNotFound(name.to_string())))?;
Ok(LazyGroup {
file: self.file,
address: entry.object_header_address,
address: self.child_address(name)?,
})
}
/// The attribute called `name`, or `None` if it has none by that name
/// (or it cannot be read) — the value [`attrs`](Self::attrs) has under
/// that name, found without reading the other attributes when they are
/// stored densely.
pub fn attr(&self, name: &str) -> Result<Option<AttrValue>, Error> {
let hdr = self.file.get_or_parse_header(self.address)?;
let data = self.file.hdf5_bytes();
read_attr(
data,
&hdr,
name,
self.file.offset_size(),
self.file.length_size(),
)
}
/// The object header address of the child called `name`: the entry of
/// the group's listing with that name, looked up through the group's
/// name index rather than by listing the group (see
/// [`group_v2::resolve_child`]).
fn child_address(&self, name: &str) -> Result<u64, Error> {
let data = self.file.hdf5_bytes();
group_v2::resolve_child(data, &self.file.superblock, self.address, name)
.map_err(Error::Format)
}
/// This group's links that can be opened: hard links, and soft links
/// resolved to their targets (see
/// [`group_v2::resolve_group_children`]); dangling, external and
@@ -555,6 +571,21 @@ impl<'f, R: HDF5Read> LazyDataset<'f, R> {
)
}
/// The attribute called `name`, or `None` if it has none by that name
/// (or it cannot be read) — the value [`attrs`](Self::attrs) has under
/// that name, found without reading the other attributes when they are
/// stored densely.
pub fn attr(&self, name: &str) -> Result<Option<AttrValue>, Error> {
let data = self.file.hdf5_bytes();
read_attr(
data,
&self.header,
name,
self.file.offset_size(),
self.file.length_size(),
)
}
/// A header message's payload, resolved through the shared-message
/// indirection when needed (e.g. a committed datatype). See
/// [`clawhdf5_format::shared_message::message_data`].
crates/clawhdf5/src/mmap_file.rs
+44 -13
View File
@@ -23,7 +23,7 @@ use clawhdf5_format::superblock::Superblock;
use clawhdf5_io::MmapReader;
use crate::error::Error;
use crate::types::{AttrValue, DType, classify_datatype, read_attrs};
use crate::types::{AttrValue, DType, classify_datatype, read_attr, read_attrs};
/// An HDF5 file opened via memory mapping.
///
@@ -242,12 +242,7 @@ impl<'f> MmapGroup<'f> {
/// Get a dataset within this group by name.
pub fn dataset(&self, name: &str) -> Result<MmapDataset<'f>, Error> {
let entries = self.children()?;
let entry = entries
.iter()
.find(|e| e.name == name)
.ok_or_else(|| Error::Format(FormatError::PathNotFound(name.to_string())))?;
let hdr = self.file.parse_header(entry.object_header_address)?;
let hdr = self.file.parse_header(self.child_address(name)?)?;
if !has_message(&hdr, MessageType::DataLayout) {
return Err(Error::NotADataset(name.to_string()));
}
@@ -260,17 +255,38 @@ impl<'f> MmapGroup<'f> {
/// Get a subgroup within this group by name.
pub fn group(&self, name: &str) -> Result<MmapGroup<'f>, Error> {
let entries = self.children()?;
let entry = entries
.iter()
.find(|e| e.name == name)
.ok_or_else(|| Error::Format(FormatError::PathNotFound(name.to_string())))?;
Ok(MmapGroup {
file: self.file,
address: entry.object_header_address,
address: self.child_address(name)?,
})
}
/// The attribute called `name`, or `None` if it has none by that name
/// (or it cannot be read) — the value [`attrs`](Self::attrs) has under
/// that name, found without reading the other attributes when they are
/// stored densely.
pub fn attr(&self, name: &str) -> Result<Option<AttrValue>, Error> {
let hdr = self.file.parse_header(self.address)?;
let data = self.file.hdf5_bytes();
read_attr(
data,
&hdr,
name,
self.file.offset_size(),
self.file.length_size(),
)
}
/// The object header address of the child called `name`: the entry of
/// the group's listing with that name, looked up through the group's
/// name index rather than by listing the group (see
/// [`group_v2::resolve_child`]).
fn child_address(&self, name: &str) -> Result<u64, Error> {
let data = self.file.meta()?;
group_v2::resolve_child(data, &self.file.superblock, self.address, name)
.map_err(Error::Format)
}
/// This group's links that can be opened: hard links, and soft links
/// resolved to their targets (see
/// [`group_v2::resolve_group_children`]); dangling, external and
@@ -506,6 +522,21 @@ impl<'f> MmapDataset<'f> {
)
}
/// The attribute called `name`, or `None` if it has none by that name
/// (or it cannot be read) — the value [`attrs`](Self::attrs) has under
/// that name, found without reading the other attributes when they are
/// stored densely.
pub fn attr(&self, name: &str) -> Result<Option<AttrValue>, Error> {
let data = self.file.hdf5_bytes();
read_attr(
data,
&self.header,
name,
self.file.offset_size(),
self.file.length_size(),
)
}
/// A header message's payload, resolved through the shared-message
/// indirection when needed (e.g. a committed datatype). See
/// [`clawhdf5_format::shared_message::message_data`].
crates/clawhdf5/src/reader.rs
+72 -16
View File
@@ -23,7 +23,7 @@ use clawhdf5_format::superblock::Superblock;
use crate::cache_image::{self, ImageView};
use crate::error::Error;
use crate::types::{AttrValue, DType, classify_datatype, read_attrs};
use crate::types::{AttrValue, DType, classify_datatype, read_attr, read_attrs};
// ---------------------------------------------------------------------------
// FileData — internal storage for either owned bytes or an mmap
@@ -230,9 +230,10 @@ impl File {
/// A `Dataset` handle for the object header at `address` (an address
/// from a group listing, or one kept from an earlier lookup), without
/// resolving a path. Resolving a path walks every group on it, which in
/// a large group costs a scan of its links; keep the address instead to
/// open the same dataset repeatedly.
/// resolving a path. Resolving a path looks each component up in its
/// group (through the name index of a dense group; a v1 group's entries
/// are scanned); keep the address instead to open the same dataset
/// repeatedly.
pub fn dataset_at(&self, address: u64) -> Result<Dataset<'_>, Error> {
let hdr = self.parse_header(address)?;
if !has_message(&hdr, MessageType::DataLayout) {
@@ -245,6 +246,17 @@ impl File {
.check_open()
}
/// A `Group` handle for the object header at `address` (from
/// [`Group::entries`], or kept from an earlier lookup), without
/// resolving a path. Like [`group`](Self::group), the object is not
/// checked to be a group; a non-group has no children.
pub fn group_at(&self, address: u64) -> Group<'_> {
Group {
file: self,
address,
}
}
/// Resolve a path and return a `Group` handle.
///
/// The path uses `/` separators (e.g., `"sensors"`).
@@ -483,12 +495,7 @@ impl<'f> Group<'f> {
/// Get a dataset within this group by name.
pub fn dataset(&self, name: &str) -> Result<Dataset<'f>, Error> {
let entries = self.children()?;
let entry = entries
.iter()
.find(|e| e.name == name)
.ok_or_else(|| Error::Format(FormatError::PathNotFound(name.to_string())))?;
let hdr = self.file.parse_header(entry.object_header_address)?;
let hdr = self.file.parse_header(self.child_address(name)?)?;
if !has_message(&hdr, MessageType::DataLayout) {
return Err(Error::NotADataset(name.to_string()));
}
@@ -501,17 +508,51 @@ impl<'f> Group<'f> {
/// Get a subgroup within this group by name.
pub fn group(&self, name: &str) -> Result<Group<'f>, Error> {
let entries = self.children()?;
let entry = entries
.iter()
.find(|e| e.name == name)
.ok_or_else(|| Error::Format(FormatError::PathNotFound(name.to_string())))?;
Ok(Group {
file: self.file,
address: entry.object_header_address,
address: self.child_address(name)?,
})
}
/// The attribute called `name`, or `None` if it has none by that name
/// (or it cannot be read) — the value [`attrs`](Self::attrs) has under
/// that name, found without reading the other attributes when they are
/// stored densely.
pub fn attr(&self, name: &str) -> Result<Option<AttrValue>, Error> {
let hdr = self.file.parse_header(self.address)?;
let data = self.file.data.as_bytes();
read_attr(
data,
&hdr,
name,
self.file.offset_size(),
self.file.length_size(),
)
}
/// The object header address of the child called `name`: the entry of
/// the group's listing with that name, looked up through the group's
/// name index rather than by listing the group (see
/// [`group_v2::resolve_child`]).
fn child_address(&self, name: &str) -> Result<u64, Error> {
let data = self.file.data.meta()?;
group_v2::resolve_child(data, &self.file.superblock, self.address, name)
.map_err(Error::Format)
}
/// This group's children that can be opened, as `(name, object header
/// address)` in listing order — the entries [`datasets`](Self::datasets)
/// and [`groups`](Self::groups) are drawn from. Open one with
/// [`File::dataset_at`] or [`File::group_at`] to skip looking its name
/// up again, or keep the addresses to revisit the objects.
pub fn entries(&self) -> Result<Vec<(String, u64)>, Error> {
Ok(self
.children()?
.into_iter()
.map(|e| (e.name, e.object_header_address))
.collect())
}
/// This group's links that can be opened: hard links, and soft links
/// resolved to their targets (see
/// [`group_v2::resolve_group_children`]); dangling, external and
@@ -1051,6 +1092,21 @@ impl<'f> Dataset<'f> {
)
}
/// The attribute called `name`, or `None` if it has none by that name
/// (or it cannot be read) — the value [`attrs`](Self::attrs) has under
/// that name, found without reading the other attributes when they are
/// stored densely.
pub fn attr(&self, name: &str) -> Result<Option<AttrValue>, Error> {
let data = self.file.data.as_bytes();
read_attr(
data,
&self.header,
name,
self.file.offset_size(),
self.file.length_size(),
)
}
/// Verify this dataset's content against its stored provenance hash
/// (`_provenance_sha256`, written automatically on save when a
/// [`Provenance`](clawhdf5_format::provenance::Provenance) is set — see
crates/clawhdf5/src/types.rs
+29
View File
@@ -182,6 +182,35 @@ pub(crate) fn read_attrs(
))
}
/// The attribute called `name` on the object with header `header`, decoded
/// as [`read_attrs`] decodes it, or `None` (see
/// [`find_attribute_in_file`](clawhdf5_format::attribute::find_attribute_in_file)).
pub(crate) fn read_attr(
file_data: &[u8],
header: &clawhdf5_format::object_header::ObjectHeader,
name: &str,
offset_size: u8,
length_size: u8,
) -> Result<Option<AttrValue>, crate::Error> {
let Some(msg) = clawhdf5_format::attribute::find_attribute_in_file(
file_data,
header,
name,
offset_size,
length_size,
)?
else {
return Ok(None);
};
Ok(attrs_to_map(
std::slice::from_ref(&msg),
file_data,
offset_size,
length_size,
)
.remove(name))
}
pub(crate) fn attrs_to_map(
attrs: &[clawhdf5_format::attribute::AttributeMessage],
file_data: &[u8],
crates/clawhdf5/tests/dense_storage_interop.rs
+24
View File
@@ -211,6 +211,30 @@ fn dense_attribute_stored_as_a_huge_heap_object() {
assert!(matches!(&attrs["bigger"], AttrValue::I64Array(v) if *v == bigger));
}
/// Enough huge attributes that the huge-object B-tree has internal nodes:
/// each one is found by descending it by heap ID (libhdf5 orders the
/// records of indirectly addressed huge objects by ID), and every value
/// matches what was written.
#[test]
fn many_huge_attributes_are_found_through_their_index() {
skip_if_no_python!();
let (_dir, path) = h5py_file(
"d = f.create_dataset('d', data=[1.0])\n\
for i in range(300):\n\
\x20 d.attrs['h%03d' % i] = np.arange(600, dtype='i8') + i\n",
);
let f = File::open(&path).unwrap();
let d = f.dataset("d").unwrap();
let attrs = d.attrs().unwrap();
assert_eq!(attrs.len(), 300);
for i in 0..300i64 {
let name = format!("h{i:03}");
let want: Vec<i64> = (0..600).map(|v| v + i).collect();
assert!(matches!(&attrs[&name], AttrValue::I64Array(v) if *v == want), "{name}");
assert!(matches!(d.attr(&name).unwrap(), Some(AttrValue::I64Array(v)) if v == want), "{name}");
}
}
/// A group whose link heap has a deflate I/O filter (set on the group
/// creation property list), with 3 000 links and one link whose message is
/// larger than the heap's managed-object limit, so it is a huge object.
crates/clawhdf5/tests/indexed_lookup_interop.rs
+410
View File
@@ -0,0 +1,410 @@
//! Looking one name up in a dense group (links in a fractal heap, indexed by
//! a v2 B-tree of name hashes) or in dense attribute storage reads the name
//! index, not every link: O(log n) index nodes and only the links whose
//! lookup3 hash equals the name's. Before, every lookup decoded all n links,
//! so opening each child of a 35 001-link group by name decoded ~1.2e9.
//!
//! The file is written by h5py (libhdf5 orders the index), with names whose
//! hashes collide, and every result is compared with what h5py reads.
//!
//! Skipped when python3 with h5py is unavailable, unless
//! `CLAWHDF5_REQUIRE_INTEROP=1`.
use std::collections::{BTreeMap, HashMap};
use std::process::Command;
use std::sync::OnceLock;
use std::time::{Duration, Instant};
use clawhdf5::{AttrValue, File, LazyFile, MmapFile};
use clawhdf5_format::checksum::jenkins_lookup3;
use clawhdf5_format::error::FormatError;
use clawhdf5_format::lookup_stats;
fn python() -> String {
std::env::var("CLAWHDF5_PYTHON").unwrap_or_else(|_| "python3".to_string())
}
fn interop_required() -> bool {
std::env::var("CLAWHDF5_REQUIRE_INTEROP").is_ok_and(|v| v == "1")
}
fn python_available() -> bool {
Command::new(python())
.args(["-c", "import h5py"])
.output()
.map(|o| o.status.success())
.unwrap_or(false)
}
macro_rules! skip_if_no_python {
() => {
if !python_available() {
assert!(
!interop_required(),
"CLAWHDF5_REQUIRE_INTEROP=1 but python3 with h5py is not available"
);
eprintln!("SKIP: python3 with h5py not available");
return;
}
};
}
fn run_python(script: &str) -> String {
let output = Command::new(python())
.args(["-c", script])
.output()
.expect("failed to run python");
assert!(
output.status.success(),
"Python script failed:\nSTDOUT: {}\nSTDERR: {}",
String::from_utf8_lossy(&output.stdout),
String::from_utf8_lossy(&output.stderr)
);
String::from_utf8_lossy(&output.stdout).trim().to_string()
}
/// Links in the big group, as libhdf5's `h5stat_newgrat.h5` has.
const LINKS: usize = 35_001;
/// Attributes on the dense-attribute dataset.
const ATTRS: usize = 3_000;
/// Pairs of distinct names with equal lookup3 hashes, found by search (the
/// hash is fixed, so the pairs are too).
fn colliding_pairs(count: usize) -> Vec<(String, String)> {
let mut seen: HashMap<u32, String> = HashMap::new();
let mut pairs = Vec::new();
for i in 0.. {
let name = format!("c{i}");
let h = jenkins_lookup3(name.as_bytes());
if let Some(first) = seen.insert(h, name.clone()) {
pairs.push((first, name));
if pairs.len() == count {
break;
}
}
}
pairs
}
/// What h5py reads: link values, attribute values, and which of the
/// missing names it finds as links and as attributes (none).
type H5pyView = (
BTreeMap<String, i64>,
BTreeMap<String, i64>,
Vec<String>,
Vec<String>,
);
struct Fixture {
_dir: tempfile::TempDir,
path: String,
/// Names in the big group, with the value of the scalar dataset each
/// links to, as h5py reads them.
links: BTreeMap<String, i64>,
/// Names that are not links but hash like one that is.
missing_links: Vec<String>,
/// Attributes of `/x`, as h5py reads them.
attrs: BTreeMap<String, i64>,
missing_attrs: Vec<String>,
}
fn fixture() -> &'static Fixture {
static FIXTURE: OnceLock<Fixture> = OnceLock::new();
FIXTURE.get_or_init(|| {
let pairs = colliding_pairs(6);
for (a, b) in &pairs {
assert_ne!(a, b);
assert_eq!(jenkins_lookup3(a.as_bytes()), jenkins_lookup3(b.as_bytes()));
}
// Pairs 0-2 both present (either can be the one libhdf5 orders
// first), pairs 3-5 only the first: its partner must not be found.
// "k69209"/"k155448" is the pair the writer once misordered.
let mut present: Vec<String> = vec!["k69209".into(), "k155448".into()];
let mut missing: Vec<String> = Vec::new();
for (i, (a, b)) in pairs.into_iter().enumerate() {
present.push(a);
if i < 3 {
present.push(b);
} else {
missing.push(b);
}
}
missing.extend(["", "nope", "n35001x", "N1"].map(String::from));
let mut links = present.clone();
let mut i = 0;
while links.len() < LINKS {
links.push(format!("n{i}"));
i += 1;
}
let mut attrs = present.clone();
attrs.extend((0..ATTRS - present.len()).map(|i| format!("a{i}")));
let dir = tempfile::tempdir().unwrap();
let path = dir.path().join("big.h5").display().to_string();
// The names go through a file: 35 001 of them overflow an argument.
let names = dir.path().join("names.json");
std::fs::write(
&names,
serde_json::to_string(&(&links, &attrs, &missing)).unwrap(),
)
.unwrap();
let names = names.display();
let out = run_python(&format!(
"import h5py, json, numpy as np\n\
links, attrs, missing = json.load(open(r'{names}'))\n\
with h5py.File(r'{path}', 'w', libver='latest') as f:\n\
\x20 g = f.create_group('g')\n\
\x20 for i, n in enumerate(links):\n\
\x20 g.create_dataset(n, data=np.int64(i))\n\
\x20 x = f.create_dataset('x', data=np.int64(0))\n\
\x20 for i, n in enumerate(attrs):\n\
\x20 x.attrs[n] = np.int64(1000 + i)\n\
with h5py.File(r'{path}', 'r') as f:\n\
\x20 g, a = f['g'], f['x'].attrs\n\
\x20 print(json.dumps([{{n: int(g[n][()]) for n in g}}, {{n: int(a[n]) for n in a}},\n\
\x20 [n for n in missing if n and n in g], [n for n in missing if n and n in a]]))",
));
let (links, attrs, found_links, found_attrs): H5pyView =
serde_json::from_str(&out).unwrap();
assert_eq!(links.len(), LINKS);
assert_eq!(attrs.len(), ATTRS);
assert!(found_links.is_empty() && found_attrs.is_empty());
Fixture {
_dir: dir,
path,
links,
missing_links: missing.clone(),
attrs,
missing_attrs: missing,
}
})
}
fn is_not_found(e: &clawhdf5::Error) -> bool {
matches!(e, clawhdf5::Error::Format(FormatError::PathNotFound(_)))
}
#[test]
fn one_link_lookup_reads_the_index_not_every_link() {
skip_if_no_python!();
let fx = fixture();
let f = File::open(&fx.path).unwrap();
let g = f.group("g").unwrap();
for (name, value) in &fx.links {
lookup_stats::reset();
let ds = g.dataset(name).unwrap();
// One link decoded per lookup, two where hashes collide — not 35 001.
let read = lookup_stats::heap_objects_read();
assert!(read <= 2, "looking up {name} read {read} heap objects");
assert_eq!(ds.read_i64().unwrap(), vec![*value], "{name}");
}
for name in &fx.missing_links {
lookup_stats::reset();
let err = g.dataset(name).unwrap_err();
assert!(is_not_found(&err), "{name:?}: {err:?}");
assert!(lookup_stats::heap_objects_read() <= 2, "{name:?}");
}
// A path resolves each component the same way.
for name in ["k155448", "n0", "n34000"] {
lookup_stats::reset();
let ds = f.dataset(&format!("/g/{name}")).unwrap();
assert!(lookup_stats::heap_objects_read() <= 2);
assert_eq!(ds.read_i64().unwrap(), vec![fx.links[name]]);
}
}
#[test]
fn one_attribute_lookup_reads_the_index_not_every_attribute() {
skip_if_no_python!();
let fx = fixture();
let f = File::open(&fx.path).unwrap();
let x = f.dataset("x").unwrap();
let all = x.attrs().unwrap();
assert_eq!(all.len(), ATTRS);
for (name, value) in &fx.attrs {
lookup_stats::reset();
let got = x.attr(name).unwrap();
assert!(lookup_stats::heap_objects_read() <= 2, "{name}");
assert!(
matches!(got, Some(AttrValue::I64(v)) if v == *value),
"{name}: {got:?}"
);
assert!(matches!(all.get(name), Some(AttrValue::I64(v)) if v == value));
}
for name in &fx.missing_attrs {
lookup_stats::reset();
assert!(x.attr(name).unwrap().is_none(), "{name:?}");
assert!(lookup_stats::heap_objects_read() <= 2, "{name:?}");
}
// Compact attributes (on the root group: none) and a group's attributes.
assert!(f.root().attr("k69209").unwrap().is_none());
}
/// Every child of the big group opened by name through each file type,
/// within `limit`: with a scan per lookup this is ~1.2e9 link decodes.
#[test]
fn opening_every_child_of_a_35001_link_group_by_name_is_quick() {
skip_if_no_python!();
let fx = fixture();
let limit = Duration::from_secs(120);
let started = Instant::now();
let check_time = |n: usize| {
assert!(
started.elapsed() < limit,
"{n} lookups took {:?}",
started.elapsed()
);
};
let f = File::open(&fx.path).unwrap();
let g = f.group("g").unwrap();
for (n, (name, value)) in fx.links.iter().enumerate() {
assert_eq!(g.dataset(name).unwrap().read_i64().unwrap(), vec![*value]);
check_time(n);
}
// The listing hands out entries: open each by address.
let entries = g.entries().unwrap();
assert_eq!(entries.len(), LINKS);
for (name, address) in &entries {
let ds = f.dataset_at(*address).unwrap();
assert_eq!(ds.read_i64().unwrap(), vec![fx.links[name]]);
}
assert!(f.group_at(g_address(&f)).dataset("n0").is_ok());
let m = MmapFile::open(&fx.path).unwrap();
let mg = m.group("g").unwrap();
for (n, (name, value)) in fx.links.iter().enumerate() {
assert_eq!(mg.dataset(name).unwrap().read_i64().unwrap(), vec![*value]);
check_time(n);
}
assert!(mg.group("nope").is_err_and(|e| is_not_found(&e)));
let l = LazyFile::open_mmap(&fx.path).unwrap();
let lg = l.group("g").unwrap();
for (n, (name, value)) in fx.links.iter().enumerate() {
assert_eq!(lg.dataset(name).unwrap().read_i64().unwrap(), vec![*value]);
check_time(n);
}
let lx = l.dataset("x").unwrap();
assert!(
matches!(lx.attr("k155448").unwrap(), Some(AttrValue::I64(v)) if v == fx.attrs["k155448"])
);
assert!(
lg.dataset(&fx.missing_links[0])
.is_err_and(|e| is_not_found(&e))
);
}
fn g_address(f: &File) -> u64 {
f.root()
.entries()
.unwrap()
.into_iter()
.find(|(n, _)| n == "g")
.unwrap()
.1
}
/// Every kind of link, looked up by name in a dense group (through the name
/// index) and in a compact one, opens what h5py opens and nothing it cannot:
/// hard links, soft links (absolute, relative, to a group), and not a
/// dangling soft link, an external link or a missing name.
#[test]
fn links_of_every_kind_resolve_by_name_as_in_h5py() {
skip_if_no_python!();
let dir = tempfile::tempdir().unwrap();
let path = dir.path().join("links.h5").display().to_string();
// For each group and name: "dataset <value>", "group", or "none" as
// h5py sees it.
let out = run_python(&format!(
"import h5py, json, numpy as np\n\
with h5py.File(r'{path}', 'w', libver='latest') as f:\n\
\x20 for gname, n in (('dense', 20), ('compact', 2)):\n\
\x20 g = f.create_group(gname)\n\
\x20 for i in range(n):\n\
\x20 g.create_dataset(f'd{{i}}', data=np.int64(100 + i))\n\
\x20 s = g.create_group('sub')\n\
\x20 s.create_dataset('x', data=np.int64(7))\n\
\x20 g['abs'] = h5py.SoftLink(f'/{{gname}}/d1')\n\
\x20 g['rel'] = h5py.SoftLink('sub/x')\n\
\x20 g['tosub'] = h5py.SoftLink('sub')\n\
\x20 g['dangling'] = h5py.SoftLink('/nowhere')\n\
\x20 g['ext'] = h5py.ExternalLink('other.h5', '/y')\n\
names = ['d0', 'd1', 'sub', 'abs', 'rel', 'tosub', 'dangling', 'ext', 'nope', '']\n\
seen = {{}}\n\
with h5py.File(r'{path}', 'r') as f:\n\
\x20 for gname in ('dense', 'compact'):\n\
\x20 g = f[gname]\n\
\x20 for n in names:\n\
\x20 try:\n\
\x20 o = g[n] if n else None\n\
\x20 except (KeyError, OSError):\n\
\x20 o = None\n\
\x20 if isinstance(o, h5py.Dataset):\n\
\x20 seen[f'{{gname}}/{{n}}'] = f'dataset {{int(o[()])}}'\n\
\x20 elif isinstance(o, h5py.Group):\n\
\x20 seen[f'{{gname}}/{{n}}'] = 'group'\n\
\x20 else:\n\
\x20 seen[f'{{gname}}/{{n}}'] = 'none'\n\
print(json.dumps(seen))",
));
let seen: BTreeMap<String, String> = serde_json::from_str(&out).unwrap();
assert_eq!(seen.len(), 20);
let f = File::open(&path).unwrap();
// The dense group's links are in a heap, the compact group's in its
// header.
for (gname, dense) in [("dense", true), ("compact", false)] {
let g = f.group(gname).unwrap();
lookup_stats::reset();
g.dataset("d0").unwrap();
assert_eq!(lookup_stats::heap_objects_read() > 0, dense, "{gname}");
}
let m = MmapFile::open(&path).unwrap();
let l = LazyFile::open_mmap(&path).unwrap();
for (key, want) in &seen {
let (gname, name) = key.split_once('/').unwrap();
let got = {
let g = f.group(gname).unwrap();
match (g.dataset(name), g.group(name)) {
(Ok(ds), _) => format!("dataset {}", ds.read_i64().unwrap()[0]),
(Err(clawhdf5::Error::NotADataset(_)), Ok(sub)) => {
// A group: it has the child `x` (checks the address).
assert!(sub.dataset("x").is_ok() || name == "sub" || name == "tosub");
"group".to_string()
}
(Err(e), Err(e2)) => {
assert!(is_not_found(&e) && is_not_found(&e2), "{key}: {e:?} / {e2:?}");
"none".to_string()
}
(Err(e), Ok(_)) => panic!("{key}: dataset {e:?} but group ok"),
}
};
assert_eq!(&got, want, "{key}");
// The other readers agree, and a path through the group resolves the
// same way.
let mg = m.group(gname).unwrap();
let lg = l.group(gname).unwrap();
match want.strip_prefix("dataset ") {
Some(v) => {
let v: i64 = v.parse().unwrap();
assert_eq!(mg.dataset(name).unwrap().read_i64().unwrap(), vec![v]);
assert_eq!(lg.dataset(name).unwrap().read_i64().unwrap(), vec![v]);
let ds = f.dataset(&format!("/{gname}/{name}")).unwrap();
assert_eq!(ds.read_i64().unwrap(), vec![v], "{key}");
}
None if want == "group" => {
assert!(mg.group(name).unwrap().dataset("x").is_ok(), "{key}");
assert!(lg.group(name).unwrap().dataset("x").is_ok(), "{key}");
let ds = f.dataset(&format!("/{gname}/{name}/x")).unwrap();
assert_eq!(ds.read_i64().unwrap(), vec![7]);
}
None => {
assert!(mg.dataset(name).is_err_and(|e| is_not_found(&e)), "{key}");
assert!(lg.group(name).is_err_and(|e| is_not_found(&e)), "{key}");
}
}
}
}