quantumclaw/clawhdf5
3
1
Fork 0
Code Issues Pull Requests 4 Actions 5 Packages Projects Releases 1 Wiki Activity

Merge branch 'perf/wasm-listing-passes' into feat/listing-header-last-files

Browse Source 
# Conflicts:
#	CHANGELOG.md
This commit is contained in:
osobh
2026-09-27 23:16:27 -05:00
parent 4f90ce02a7 2e5b059530
commit bf5a163dcf
16 changed files with 1039 additions and 134 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/clawhdf5-format/src/btree_v1.rs
+30 -12
View File
@@ -92,6 +92,8 @@ impl BTreeV1Node {
// + left_sibling(offset_size) + right_sibling(offset_size)
let os = offset_size as usize;
let header_size = 8 + os * 2;
// The body is read once the header says how long it is.
file.hint(offset, NODE_HINT_LEN);
let header = read_exact_at(file, offset, header_size)?;
let file_data: &[u8] = &header;
// The header's read checked that `offset + header_size` fits.
@@ -156,7 +158,18 @@ impl BTreeV1Node {
}
/// Maximum recursion depth for B-tree traversal (malformed data protection).
const MAX_BTREE_DEPTH: usize = 64;
pub(crate) const MAX_BTREE_DEPTH: usize = 64;
/// What a symbol table node takes with libhdf5's default group leaf K (4):
/// its 8-byte header and 2K entries of 40 bytes (8-byte offsets). Hinted
/// before one is read ([`Storage::hint`]); a node of another size is read
/// all the same.
const SNOD_HINT_LEN: usize = 8 + 8 * 40;
/// What a group B-tree node takes with libhdf5's default internal K (16):
/// its header (24 bytes with 8-byte offsets), 2K + 1 keys and 2K children
/// of 8 bytes. Hinted before one is read.
const NODE_HINT_LEN: usize = 24 + (2 * 16 + 1 + 2 * 16) * 8;
/// Collect all leaf-level child addresses (SNOD addresses) by traversing the B-tree.
pub fn collect_symbol_table_nodes(
@@ -196,20 +209,22 @@ fn collect_symbol_table_nodes_inner<S: Storage + ?Sized>(
}
if node.node_level == 0 {
// Leaf: children are SNOD addresses
// Leaf: children are SNOD addresses, read next (see
// `Storage::hint`).
for &snod in &node.children {
file.hint(snod, SNOD_HINT_LEN);
}
Ok(node.children)
} else {
// Internal: recurse into children. After the first child that
// fails, the others are only read (as `storage::touch` does), not
// descended into; that error is returned.
// Internal: recurse into children. A child that fails does not
// stop the walk: the others are still descended into (reading, not
// using, what they hold), then the first error is returned. The
// result and the error are those of stopping at the first failure;
// a storage that records what it lacks (see `storage::touch`)
// learns every node the walk can reach in one attempt.
let mut result = Vec::new();
let mut failed = None;
for &child_addr in &node.children {
if failed.is_some() {
// Parsing reads the node's header, then its body.
let _ = BTreeV1Node::parse_in(file, child_addr, offset_size, length_size);
continue;
}
match collect_symbol_table_nodes_inner(
file,
child_addr,
@@ -217,8 +232,11 @@ fn collect_symbol_table_nodes_inner<S: Storage + ?Sized>(
length_size,
depth + 1,
) {
Ok(child_snods) => result.extend(child_snods),
Err(e) => failed = Some(e),
Ok(child_snods) if failed.is_none() => result.extend(child_snods),
Ok(_) => {}
Err(e) => {
failed.get_or_insert(e);
}
}
}
match failed {
crates/clawhdf5-format/src/btree_v2.rs
+22 -13
View File
@@ -194,10 +194,17 @@ const MAX_DEPTH: u16 = 64;
/// Take `n` records from the traversal's budget, or refuse the tree.
fn spend(budget: &mut usize, n: usize) -> Result<(), FormatError> {
*budget = budget
.checked_sub(n)
.ok_or(FormatError::NestingDepthExceeded)?;
Ok(())
match budget.checked_sub(n) {
Some(left) => {
*budget = left;
Ok(())
}
None => {
// Spent: a walk that goes on after a failure stops here.
*budget = 0;
Err(FormatError::NestingDepthExceeded)
}
}
}
/// Collect all records from a B-tree v2 by traversing from the root.
@@ -504,16 +511,18 @@ fn collect_internal_records<S: Storage + ?Sized>(
// Interleave: child[0], record[0], child[1], record[1], ..., child[nr]
// We collect child[0] records, then record[0], then child[1], etc.
// After the first child that fails, the others are only touched (see
// `storage::touch`); that error is returned.
// A child that fails does not stop the walk: the others are still
// descended into (their records are dropped with the result), then the
// first error is returned, as when stopping there. A storage that
// records what it lacks (see `storage::touch`) so learns every node the
// walk can reach in one attempt. The record budget is spent as before,
// so the walk is no longer than a successful one.
let mut failed = None;
for (i, &(child_addr, child_nrec)) in node.children.iter().enumerate() {
if failed.is_some() {
let len = usize::try_from(node_size)
.unwrap_or(usize::MAX)
.min(1 << 16);
crate::storage::touch(file, child_addr, len);
continue;
if failed.is_some() && *budget == 0 {
// The record budget is spent: the tree is refused, and a walk
// over what is left could be as long as the one it bounds.
break;
}
if let Err(e) = (|| -> Result<(), FormatError> {
if child_depth == 0 {
@@ -553,7 +562,7 @@ fn collect_internal_records<S: Storage + ?Sized>(
}
Ok(())
})() {
failed = Some(e);
failed.get_or_insert(e);
}
}
crates/clawhdf5-format/src/fractal_heap.rs
+166 -32
View File
@@ -10,7 +10,7 @@ use crate::addr::to_usize;
use crate::btree_v2::{BTreeV2Header, find_btree_v2_records_in};
use crate::error::FormatError;
use crate::filter_pipeline::FilterPipeline;
use crate::storage::{Storage, Window, len_usize, read_exact_at};
use crate::storage::{Storage, Window, len_usize, read_exact_at, read_upto};
/// Parsed fractal heap header (signature "FRHP").
#[derive(Debug, Clone)]
@@ -132,6 +132,9 @@ fn heap_id_type(first: u8) -> Result<u8, FormatError> {
const BTREE_HUGE_INDIRECT: u8 = 1;
const BTREE_HUGE_INDIRECT_FILTERED: u8 = 2;
/// Most child entries [`FractalHeapHeader::hint_managed_blocks`] walks.
const MAX_HINTED_BLOCKS: usize = 4096;
impl FractalHeapHeader {
/// Parse a fractal heap header at the given offset.
pub fn parse(
@@ -667,15 +670,8 @@ impl FractalHeapHeader {
"fractal heap: maximum recursion depth exceeded".into(),
));
}
let block_offset_bytes = (self.max_heap_size as usize).div_ceil(8);
let iblock_header = 5 + offset_size as usize + block_offset_bytes;
let nrows_usize = nrows as usize;
// Rows below max_direct_rows hold direct blocks; rows at/above hold
// child indirect blocks. (NOT the FRHP "starting rows" field.)
let start_indirect = self.max_direct_rows();
let max_direct_rows = nrows_usize.min(start_indirect);
// The block up to its last child entry. The walk below reads
// entries in order and stops at the one covering the target, which
// the geometry alone locates, so the first window ends there: a
@@ -685,32 +681,11 @@ impl FractalHeapHeader {
// over the whole block. Either window holds what it was asked for or
// ends at the end of the file, so its bounds checks are the
// whole-file ones.
let direct_entry = usize::from(offset_size)
+ if self.filter_pipeline.is_some() {
usize::from(self.length_size) + 4
} else {
0
};
let direct_entries = max_direct_rows.saturating_mul(usize::from(self.table_width));
let entries_len = |n: usize| {
n.min(direct_entries)
.saturating_mul(direct_entry)
.saturating_add(
n.saturating_sub(direct_entries)
.saturating_mul(usize::from(offset_size)),
)
};
let all_entries = direct_entries.saturating_add(
nrows_usize
.saturating_sub(start_indirect)
.saturating_mul(usize::from(self.table_width)),
);
let block_len = iblock_header.saturating_add(entries_len(all_entries));
let layout = self.indirect_layout(nrows_usize, offset_size);
let block_len = layout.len_upto(layout.all_entries);
let target_entry = self.indirect_entry_for(nrows_usize, iblock_heap_offset, target_offset);
let first_len = target_entry.map_or(block_len, |i| {
iblock_header
.saturating_add(entries_len(i.saturating_add(1)))
.min(block_len)
layout.len_upto(i.saturating_add(1)).min(block_len)
});
let mut next = self.walk_indirect_block(
&Window::read(file, iblock_addr as u64, first_len)?,
@@ -755,6 +730,140 @@ impl FractalHeapHeader {
}
}
/// Where the child entries of an indirect block of `nrows` rows are.
fn indirect_layout(&self, nrows: usize, offset_size: u8) -> IndirectLayout {
let block_offset_bytes = (self.max_heap_size as usize).div_ceil(8);
// Rows below max_direct_rows hold direct blocks; rows at/above hold
// child indirect blocks. (NOT the FRHP "starting rows" field.)
let start_indirect = self.max_direct_rows();
let direct_entries = nrows
.min(start_indirect)
.saturating_mul(usize::from(self.table_width));
IndirectLayout {
header: 5 + usize::from(offset_size) + block_offset_bytes,
direct_entry: usize::from(offset_size)
+ if self.filter_pipeline.is_some() {
usize::from(self.length_size) + 4
} else {
0
},
direct_entries,
indirect_entry: usize::from(offset_size),
all_entries: direct_entries.saturating_add(
nrows
.saturating_sub(start_indirect)
.saturating_mul(usize::from(self.table_width)),
),
}
}
/// Hint the heap's root block (see [`Storage::hint`]): every managed
/// object is read through it, and a storage that fetches between
/// attempts can fetch it along with whatever else the attempt missed
/// (the name index read before any object, say).
pub fn hint_root_block<S: Storage + ?Sized>(&self, file: &S) {
if is_undefined(self.root_block_address, self.offset_size) {
return;
}
let len = if self.current_rows_in_root_indirect_block == 0 {
if self.filter_pipeline.is_some() {
self.root_direct_block_filtered_size
} else {
self.starting_block_size
}
} else {
let layout = self.indirect_layout(
usize::from(self.current_rows_in_root_indirect_block),
self.offset_size,
);
layout.len_upto(layout.all_entries) as u64
};
file.hint(
self.root_block_address,
usize::try_from(len).unwrap_or(usize::MAX),
);
}
/// Hint every managed direct block of the heap (see
/// [`Storage::hint`]), for a caller about to read all of its objects (a
/// dense group's listing). The indirect blocks leading to them are read
/// here, as every object read goes through them, a few levels deep and
/// up to [`MAX_HINTED_BLOCKS`] entries; direct blocks are only hinted.
/// Nothing is returned and no error: a storage that has the file in
/// memory skips it, and the objects are read (and checked) as before.
pub fn hint_managed_blocks<S: Storage + ?Sized>(&self, file: &S) {
if file.as_contiguous().is_some()
|| is_undefined(self.root_block_address, self.offset_size)
|| self.current_rows_in_root_indirect_block == 0
{
// A direct root block is what `hint_root_block` hints.
return;
}
let mut budget = MAX_HINTED_BLOCKS;
self.hint_indirect_block(
file,
self.root_block_address,
usize::from(self.current_rows_in_root_indirect_block),
0,
&mut budget,
);
}
fn hint_indirect_block<S: Storage + ?Sized>(
&self,
file: &S,
addr: u64,
nrows: usize,
depth: usize,
budget: &mut usize,
) {
let os = self.offset_size;
let layout = self.indirect_layout(nrows, os);
let len = layout.len_upto(layout.all_entries);
if depth > 4 || len > 1 << 20 {
return;
}
let Ok(bytes) = read_upto(file, addr, len) else {
return;
};
if bytes.len() < len || bytes.get(..4) != Some(b"FHIB".as_slice()) {
return;
}
let start_indirect = self.max_direct_rows();
let mut pos = layout.header;
for row in 0..nrows {
for _ in 0..self.table_width {
let Some(left) = budget.checked_sub(1) else {
return;
};
*budget = left;
let Ok(child) = read_offset(&bytes, pos, os) else {
return;
};
if row < start_indirect {
let size = if self.filter_pipeline.is_some() {
match read_offset(&bytes, pos + usize::from(os), self.length_size) {
Ok(n) => n,
Err(_) => return,
}
} else {
self.block_size_for_row(row)
};
pos += layout.direct_entry;
if !is_undefined(child, os) {
file.hint(child, usize::try_from(size).unwrap_or(usize::MAX));
}
} else {
pos += layout.indirect_entry;
if !is_undefined(child, os) {
let rows = self.rows_for_size(self.block_size_for_row(row));
self.hint_indirect_block(file, child, usize::from(rows), depth + 1, budget);
}
}
}
}
}
/// Which child entry of an indirect block (numbered in walk order:
/// direct rows, then indirect rows) covers `target_offset`, from the
/// doubling-table geometry alone — the entry
@@ -939,6 +1048,31 @@ impl FractalHeapHeader {
}
}
/// Where an indirect block's child entries are: after its header, the
/// direct blocks' entries (address, and for a filtered heap the stored
/// size and filter mask), then the child indirect blocks' (address).
struct IndirectLayout {
header: usize,
direct_entry: usize,
direct_entries: usize,
indirect_entry: usize,
all_entries: usize,
}
impl IndirectLayout {
/// Bytes from the block's start to the end of its first `n` entries.
fn len_upto(&self, n: usize) -> usize {
let direct = n.min(self.direct_entries);
self.header
.saturating_add(direct.saturating_mul(self.direct_entry))
.saturating_add(
n.min(self.all_entries)
.saturating_sub(direct)
.saturating_mul(self.indirect_entry),
)
}
}
/// A managed direct block's location, extent and (for a filtered heap) its
/// stored size and filter mask.
/// The child of an indirect block that covers a heap offset.
crates/clawhdf5-format/src/group_v1.rs
+117 -6
View File
@@ -4,7 +4,7 @@
use alloc::{string::String, vec::Vec};
use crate::addr::checked_addr;
use crate::btree_v1::collect_symbol_table_nodes_in;
use crate::btree_v1::{BTreeV1Node, collect_symbol_table_nodes_in};
use crate::error::FormatError;
use crate::local_heap::LocalHeap;
use crate::message_type::MessageType;
@@ -48,7 +48,7 @@ pub fn resolve_v1_group_entries_in<S: Storage + ?Sized>(
offset_size: u8,
length_size: u8,
) -> Result<Vec<GroupEntry>, FormatError> {
let entries = v1_group_entries(file_data, sym_table_msg, offset_size, length_size)?;
let entries = v1_group_entries(file_data, sym_table_msg, offset_size, length_size, true)?;
if entries.iter().any(|e| e.name.is_empty()) {
return Err(FormatError::InvalidLinkName);
}
@@ -57,11 +57,16 @@ pub fn resolve_v1_group_entries_in<S: Storage + ?Sized>(
/// Every entry of a v1 group, empty names included — for looking a name up,
/// which never matches an empty name.
///
/// With `hint_headers` (a listing, whose children are usually opened
/// next), each entry's object header is hinted (see
/// [`Storage::hint`]) as soon as its symbol table node is read.
pub(crate) fn v1_group_entries<S: Storage + ?Sized>(
file_data: &S,
sym_table_msg: &SymbolTableMessage,
offset_size: u8,
length_size: u8,
hint_headers: bool,
) -> Result<Vec<GroupEntry>, FormatError> {
// Parse local heap
let heap = LocalHeap::parse_in(
@@ -93,12 +98,23 @@ pub(crate) fn v1_group_entries<S: Storage + ?Sized>(
// `storage::touch` does); that error is returned.
let mut failed = None;
for snod_addr in snod_addrs {
let snod = checked_addr(snod_addr)
.and_then(|a| SymbolTableNode::parse_in(file_data, a, offset_size));
if hint_headers && let Ok(snod) = &snod {
for entry in &snod.entries {
if entry.object_header_address != u64::MAX {
file_data.hint(
entry.object_header_address,
crate::object_header::OBJECT_HEADER_HINT_LEN,
);
}
}
}
if failed.is_some() {
let _ = SymbolTableNode::parse_in(file_data, snod_addr, offset_size);
continue;
}
let mut node = || -> Result<(), FormatError> {
let snod = SymbolTableNode::parse_in(file_data, checked_addr(snod_addr)?, offset_size)?;
let node = || -> Result<(), FormatError> {
let snod = snod?;
for entry in &snod.entries {
// Like libhdf5, look at the heap's free list only once a name
// is needed: an empty group with a damaged heap still lists.
@@ -126,6 +142,95 @@ pub(crate) fn v1_group_entries<S: Storage + ?Sized>(
}
}
/// The entry called `name` in a v1 group, looked up as libhdf5 looks it up
/// (`H5G__stab_lookup`: `H5B_find` down the group's B-tree, then
/// `H5G__node_found` in one symbol table node) instead of by reading every
/// entry: at each node, the child whose key interval holds the name
/// (left key < name <= right key, keys being names in the local heap,
/// compared bytewise as `strcmp` does) is found by binary search. Reads
/// O(depth) nodes and names, where listing reads the whole group.
///
/// `Ok(None)` when the search does not lead to the name. In a group whose
/// B-tree is out of name order (damaged, or made by hand) that does not
/// prove it absent, so callers then fall back to reading every entry;
/// libhdf5 would report it missing.
pub(crate) fn find_v1_entry<S: Storage + ?Sized>(
file_data: &S,
sym_table_msg: &SymbolTableMessage,
name: &str,
offset_size: u8,
length_size: u8,
) -> Result<Option<GroupEntry>, FormatError> {
let heap = LocalHeap::parse_in(
file_data,
checked_addr(sym_table_msg.local_heap_address)?,
offset_size,
length_size,
)?;
// The keys and names are read from the heap's data segment one at a
// time: hint it (see `Storage::hint`).
file_data.hint(
heap.data_segment_address,
usize::try_from(heap.data_segment_size).map_or(1 << 20, |n| n.min(1 << 20)),
);
// As in listing: the heap's free list is checked before a name is used.
let mut heap_checked = false;
let mut name_at = |offset: u64| -> Result<String, FormatError> {
if !heap_checked {
heap.validate_free_list_in(file_data, length_size)?;
heap_checked = true;
}
heap.read_string_in(file_data, offset)
};
let want = name.as_bytes();
let mut address = sym_table_msg.btree_address;
for _ in 0..=crate::btree_v1::MAX_BTREE_DEPTH {
let node =
BTreeV1Node::parse_in(file_data, checked_addr(address)?, offset_size, length_size)?;
if node.node_type != 0 {
return Err(FormatError::InvalidBTreeNodeType(node.node_type));
}
// H5B_find's binary search with H5G__node_cmp3: go left when the
// name sorts at or before the left key, right when after the right
// key; otherwise this child holds it.
let (mut lo, mut hi) = (0usize, node.children.len());
let mut child = None;
while lo < hi {
let i = lo + (hi - lo) / 2;
let (Some(&left), Some(&right)) = (node.keys.get(i), node.keys.get(i + 1)) else {
return Ok(None);
};
if want <= name_at(left)?.as_bytes() {
hi = i;
} else if want > name_at(right)?.as_bytes() {
lo = i + 1;
} else {
child = Some(node.children[i]);
break;
}
}
let Some(child) = child else {
return Ok(None);
};
if node.node_level > 0 {
address = child;
continue;
}
let snod = SymbolTableNode::parse_in(file_data, checked_addr(child)?, offset_size)?;
for entry in &snod.entries {
if name_at(entry.link_name_offset)?.as_bytes() == want {
return Ok(Some(GroupEntry {
name: String::from(name),
object_header_address: entry.object_header_address,
cache_type: entry.cache_type,
}));
}
}
return Ok(None);
}
Err(FormatError::NestingDepthExceeded)
}
/// Symbol table cache type for a soft link: the scratch pad's first four bytes
/// are the local-heap offset of the link's target path, and the entry's object
/// header address is undefined.
@@ -298,7 +403,13 @@ pub fn resolve_path_in<S: Storage + ?Sized>(
let mut current_sym_table = root_sym_table.clone();
for (i, component) in components.iter().enumerate() {
let entries = v1_group_entries(file_data, &current_sym_table, offset_size, length_size)?;
let entries = v1_group_entries(
file_data,
&current_sym_table,
offset_size,
length_size,
false,
)?;
let found = entries.iter().find(|e| e.name == *component);
match found {
crates/clawhdf5-format/src/group_v2.rs
+176 -11
View File
@@ -101,18 +101,50 @@ fn resolve_compact_entries(
Ok(entries)
}
/// What a version-2 B-tree header takes with 8-byte offsets and lengths
/// (22 bytes of fields, the root node's address and record count, and the
/// checksum), rounded up: hinted before one is read.
const BTREE_V2_HEADER_HINT_LEN: usize = 64;
/// The fractal heap of a dense group. The name index's header and the
/// heap's root block are read next, whatever the lookup: they are hinted
/// (see [`Storage::hint`]) so that a storage fetching between attempts
/// gets them in the same round trip as the heap's header.
fn dense_heap<S: Storage + ?Sized>(
file_data: &S,
link_info: &LinkInfoMessage,
fh_addr: u64,
offset_size: u8,
length_size: u8,
) -> Result<FractalHeapHeader, FormatError> {
if let Some(btree_addr) = link_info.btree_name_index_address {
file_data.hint(btree_addr, BTREE_V2_HEADER_HINT_LEN);
}
let fh =
FractalHeapHeader::parse_in(file_data, checked_addr(fh_addr)?, offset_size, length_size)?;
fh.hint_root_block(file_data);
Ok(fh)
}
/// Visit every link in dense storage (fractal heap + B-tree v2 name index).
///
/// With `hint_headers` (a listing, whose children are usually opened
/// next), the object header of every hard link is hinted (see
/// [`Storage::hint`]) as soon as the link is read, even after a failure.
fn for_each_dense_link<S: Storage + ?Sized>(
file_data: &S,
link_info: &LinkInfoMessage,
fh_addr: u64,
offset_size: u8,
length_size: u8,
hint_headers: bool,
mut visit: impl FnMut(LinkMessage),
) -> Result<(), FormatError> {
// Parse fractal heap
let fh =
FractalHeapHeader::parse_in(file_data, checked_addr(fh_addr)?, offset_size, length_size)?;
let fh = dense_heap(file_data, link_info, fh_addr, offset_size, length_size)?;
if hint_headers {
// Every link is read: so is every block of the heap.
fh.hint_managed_blocks(file_data);
}
// Parse B-tree v2 for name index
let btree_addr = link_info
@@ -144,11 +176,27 @@ fn for_each_dense_link<S: Storage + ?Sized>(
let id_bytes = &record.data[id_offset..id_offset + fh.heap_id_length as usize];
// Read managed object from fractal heap
let link_data = fh.read_managed_object_in(file_data, id_bytes, offset_size);
let link = fh
.read_managed_object_in(file_data, id_bytes, offset_size)
.and_then(|d| parse_link(&d, offset_size));
if hint_headers
&& let Ok(Some(LinkMessage {
link_target:
LinkTarget::Hard {
object_header_address,
},
..
})) = &link
{
file_data.hint(
*object_header_address,
crate::object_header::OBJECT_HEADER_HINT_LEN,
);
}
if failed.is_some() {
continue;
}
match link_data.and_then(|d| parse_link(&d, offset_size)) {
match link {
Ok(Some(link)) => visit(link),
Ok(None) => {}
Err(e) => failed = Some(e),
@@ -175,6 +223,7 @@ fn resolve_dense_entries<S: Storage + ?Sized>(
fh_addr,
offset_size,
length_size,
true,
|link| {
if let LinkTarget::Hard {
object_header_address,
@@ -247,8 +296,7 @@ fn links_named<S: Storage + ?Sized>(
return Ok(found);
};
let fh =
FractalHeapHeader::parse_in(file_data, checked_addr(fh_addr)?, offset_size, length_size)?;
let fh = dense_heap(file_data, &link_info, fh_addr, offset_size, length_size)?;
let btree_addr = link_info
.btree_name_index_address
.ok_or_else(|| FormatError::PathNotFound(String::from("no B-tree v2 name index")))?;
@@ -265,6 +313,7 @@ fn links_named<S: Storage + ?Sized>(
fh_addr,
offset_size,
length_size,
false,
|link| {
if link.name == name {
found.push(link);
@@ -336,6 +385,27 @@ fn lookup_link<S: Storage + ?Sized>(
length_size: u8,
) -> Result<Option<LinkTarget>, FormatError> {
if is_v1_group(object_header) {
// Down the group's B-tree, as libhdf5 looks a name up; only when
// that does not find a hard link of that name is every entry read
// (a soft link, a group whose B-tree is out of order). A storage
// error (a read a restartable storage has not fetched yet) is
// returned as is: reading every entry would not get further.
if let Some(sym_msg) = object_header
.messages
.iter()
.find(|m| m.msg_type == MessageType::SymbolTable)
{
let stm = SymbolTableMessage::parse(&sym_msg.data, offset_size)?;
match group_v1::find_v1_entry(file_data, &stm, name, offset_size, length_size) {
Ok(Some(e)) if e.object_header_address != u64::MAX => {
return Ok(Some(LinkTarget::Hard {
object_header_address: e.object_header_address,
}));
}
Err(e @ FormatError::Storage(_)) => return Err(e),
_ => {}
}
}
let entries = resolve_group_entries(file_data, object_header, offset_size, length_size)?;
if let Some(e) = entries
.iter()
@@ -409,7 +479,7 @@ fn resolve_child_core<S: Storage + ?Sized>(
let not_found = || FormatError::PathNotFound(String::from(name));
let header = ObjectHeader::parse_in(file_data, checked_addr(group_address)?, os, ls)?;
if !is_v2_group(&header) || is_v1_group(&header) {
return resolve_group_children_in(file_data, superblock, group_address)?
return group_children(file_data, superblock, group_address, false)?
.into_iter()
.find(|e| e.name == name)
.map(|e| e.object_header_address)
@@ -575,6 +645,18 @@ fn resolve_group_children_core<S: Storage + ?Sized>(
file_data: &S,
superblock: &Superblock,
group_address: u64,
) -> Result<Vec<GroupEntry>, FormatError> {
group_children(file_data, superblock, group_address, true)
}
/// [`resolve_group_children`]; with `hint_headers`, every child's object
/// header is hinted (see [`Storage::hint`]) as soon as its address is
/// known, for a listing whose children are opened next.
fn group_children<S: Storage + ?Sized>(
file_data: &S,
superblock: &Superblock,
group_address: u64,
hint_headers: bool,
) -> Result<Vec<GroupEntry>, FormatError> {
let os = superblock.offset_size;
let ls = superblock.length_size;
@@ -589,7 +671,10 @@ fn resolve_group_children_core<S: Storage + ?Sized>(
.find(|m| m.msg_type == MessageType::SymbolTable)
.ok_or_else(|| FormatError::PathNotFound(String::from("no symbol table message")))?;
let stm = SymbolTableMessage::parse(&sym_msg.data, os)?;
let all = group_v1::resolve_v1_group_entries_in(file_data, &stm, os, ls)?;
let all = group_v1::v1_group_entries(file_data, &stm, os, ls, hint_headers)?;
if all.iter().any(|e| e.name.is_empty()) {
return Err(FormatError::InvalidLinkName);
}
if all.iter().any(group_v1::is_v1_soft_link) {
soft = group_v1::v1_soft_links_in(file_data, &stm, os, ls)?;
}
@@ -615,7 +700,7 @@ fn resolve_group_children_core<S: Storage + ?Sized>(
};
let link_info = find_link_info(&header, os)?;
if let Some(fh_addr) = link_info.fractal_heap_address {
for_each_dense_link(file_data, &link_info, fh_addr, os, ls, visit)?;
for_each_dense_link(file_data, &link_info, fh_addr, os, ls, hint_headers, visit)?;
} else {
for msg in &header.messages {
if msg.msg_type == MessageType::Link
@@ -737,7 +822,7 @@ fn resolve_group_entries<S: Storage + ?Sized>(
let stm = SymbolTableMessage::parse(&sym_msg.data, offset_size)?;
// A lookup: an entry with an empty name (which fails a listing) is
// skipped by the name comparison, as in libhdf5.
group_v1::v1_group_entries(file_data, &stm, offset_size, length_size)
group_v1::v1_group_entries(file_data, &stm, offset_size, length_size, false)
} else if is_v2_group(object_header) {
resolve_v2_group_entries_in(file_data, object_header, offset_size, length_size)
} else {
@@ -920,6 +1005,86 @@ mod tests {
assert_eq!(values, vec![22.5, 23.1, 21.8]);
}
/// `v1_groups_400.h5` from its superblock on (it has a user block).
fn v1_groups_400() -> (Vec<u8>, Superblock) {
let all: &[u8] = include_bytes!("../tests/fixtures/v1_groups_400.h5");
let data = all[signature::find_signature(all).unwrap()..].to_vec();
let sb = Superblock::parse(&data, 0).unwrap();
(data, sb)
}
/// Every child of a v1 group resolves by name, down the group's B-tree,
/// to the address the listing gives, reading a small part of what the
/// listing reads; a name the group does not hold is not found.
#[test]
fn v1_lookup_down_the_btree_agrees_with_the_listing() {
let (data, sb) = v1_groups_400();
let children = resolve_group_children(&data, &sb, sb.root_group_address).unwrap();
assert_eq!(children.len(), 401);
for c in &children {
let path = format!("/{}", c.name);
assert_eq!(
resolve_path_any(&data, &sb, &path).unwrap(),
c.object_header_address,
"{path}"
);
}
for missing in ["/g0400", "/a", "/g", "/g00000", "/zz", "/x0"] {
assert!(
matches!(
resolve_path_any(&data, &sb, missing),
Err(FormatError::PathNotFound(_))
),
"{missing}"
);
}
let st = crate::storage::CountingStorage::new(data.clone());
resolve_group_children_in(&st, &sb, sb.root_group_address).unwrap();
let listing = st.bytes_read();
st.reset();
let last = children.last().unwrap();
assert_eq!(
resolve_path_any_in(&st, &sb, &format!("/{}", last.name)).unwrap(),
last.object_header_address
);
assert!(
st.bytes_read() * 8 < listing,
"lookup read {} bytes, listing {listing}",
st.bytes_read()
);
}
/// A v1 group whose B-tree is out of name order (a name changed in the
/// heap so that it sorts past every key) is still looked up by reading
/// every entry, as before the lookup went down the B-tree.
#[test]
fn v1_lookup_falls_back_when_the_btree_is_out_of_order() {
let (mut data, sb) = v1_groups_400();
let at: Vec<usize> = data
.windows(6)
.enumerate()
.filter(|(_, w)| *w == b"g0200\0")
.map(|(i, _)| i)
.collect();
assert_eq!(at.len(), 1, "one heap string");
data[at[0]] = b'~';
let children = resolve_group_children(&data, &sb, sb.root_group_address).unwrap();
let moved = children.iter().find(|c| c.name == "~0200").unwrap();
assert_eq!(
resolve_path_any(&data, &sb, "/~0200").unwrap(),
moved.object_header_address
);
assert!(resolve_path_any(&data, &sb, "/g0200").is_err());
for c in &children {
let path = format!("/{}", c.name);
assert_eq!(
resolve_path_any(&data, &sb, &path).unwrap(),
c.object_header_address,
"{path}"
);
}
}
#[test]
fn path_not_found_v2() {
let file_data: &[u8] = include_bytes!("../tests/fixtures/v2_groups.h5");
crates/clawhdf5-format/src/object_header.rs
+35
View File
@@ -163,6 +163,12 @@ impl ObjectHeader {
offset_size: u8,
length_size: u8,
) -> Result<ObjectHeader, FormatError> {
// The first chunk is read once the prefix says how long it is: say
// so (see `Storage::hint`), for a storage that fetches between
// attempts.
if file.as_contiguous().is_none() {
file.hint(offset, OBJECT_HEADER_HINT_LEN);
}
// The longest prefix of either version, in one read. It holds the
// whole prefix or ends at the end of the file, so its bounds checks
// are the whole-file ones.
@@ -279,10 +285,20 @@ impl ObjectHeader {
let mut spans = ChunkSpans::new(file.len(), offset, length)?;
let mut chunk0_count = 0usize;
let mut next = 0usize;
let hints = file.as_contiguous().is_none();
while let Some((chunk_offset, chunk_length)) = spans.get(next) {
let chunk = read_exact_at(file, chunk_offset, chunk_length)?;
let known = spans.len;
let count =
Self::parse_v1_messages(&chunk, offset_size, length_size, messages, &mut spans)?;
// The continuation chunks this one names are read next.
if hints {
for i in known..spans.len {
if let Some((o, l)) = spans.get(i) {
file.hint(o, l);
}
}
}
// Only the first chunk's messages are held to the prefix count.
if next == 0 {
chunk0_count = count;
@@ -483,8 +499,17 @@ impl ObjectHeader {
// whenever a message no longer fits), up to the same bound as a
// version-1 header.
let mut spans = ChunkSpans::new(file.len(), base as u64, chunk0_msg_end.saturating_add(4))?;
// The continuation chunks a chunk names are read next (see
// `Storage::hint`).
let hints = file.as_contiguous().is_none();
if hints {
for &(o, l) in &continuations {
file.hint(o as u64, l);
}
}
while let Some((cont_offset, cont_length)) = continuations.pop() {
spans.add(cont_offset as u64, cont_length)?;
let known = continuations.len();
Self::parse_v2_continuation(
file,
cont_offset as u64,
@@ -495,6 +520,11 @@ impl ObjectHeader {
&mut messages,
&mut continuations,
)?;
if hints {
for &(o, l) in &continuations[known..] {
file.hint(o as u64, l);
}
}
}
Ok(ObjectHeader {
@@ -640,6 +670,11 @@ impl ObjectHeader {
}
}
/// What an object header is hinted to take before its prefix is read (see
/// [`Storage::hint`]): the first chunk of a typical dataset's header. A
/// longer header is read all the same.
pub(crate) const OBJECT_HEADER_HINT_LEN: usize = 512;
/// Longest version-2 object header prefix: signature(4) + version(1) +
/// flags(1) + times(16) + attribute phase change(4) + chunk-0 size(8).
const V2_PREFIX_MAX: usize = 34;
crates/clawhdf5-format/src/storage.rs
+30
View File
@@ -89,6 +89,21 @@ pub trait Storage {
fn as_contiguous(&self) -> Option<&[u8]> {
None
}
/// A hint that `[offset, offset + len)` is about to be read by the
/// same operation: a parser that has just learnt where the next
/// structures are (a node's children, a structure's body) says so
/// before it reads them one at a time. Nothing is read and nothing
/// fails. The default ignores it, as does every backend that reads
/// when asked; the browser's restartable reader, which fetches over
/// the network between attempts, fetches hinted bytes along with the
/// bytes an attempt actually missed, so structures a parser only
/// reaches after a miss arrive in the same round trip. Results never
/// depend on hints.
#[inline]
fn hint(&self, offset: u64, len: usize) {
let _ = (offset, len);
}
}
impl Storage for [u8] {
@@ -148,6 +163,11 @@ impl<T: Storage + ?Sized> Storage for &T {
fn as_contiguous(&self) -> Option<&[u8]> {
(**self).as_contiguous()
}
#[inline]
fn hint(&self, offset: u64, len: usize) {
(**self).hint(offset, len)
}
}
impl<T: Storage + ?Sized> Storage for Box<T> {
@@ -170,6 +190,11 @@ impl<T: Storage + ?Sized> Storage for Box<T> {
fn as_contiguous(&self) -> Option<&[u8]> {
(**self).as_contiguous()
}
#[inline]
fn hint(&self, offset: u64, len: usize) {
(**self).hint(offset, len)
}
}
#[cfg(feature = "std")]
@@ -193,6 +218,11 @@ impl<T: Storage + ?Sized> Storage for std::sync::Arc<T> {
fn as_contiguous(&self) -> Option<&[u8]> {
(**self).as_contiguous()
}
#[inline]
fn hint(&self, offset: u64, len: usize) {
(**self).hint(offset, len)
}
}
/// `storage.len()` as the `usize` the parsers' end-of-file errors report
crates/clawhdf5-format/src/symbol_table.rs
+4
View File
@@ -90,6 +90,10 @@ impl SymbolTableNode {
offset: u64,
offset_size: u8,
) -> Result<SymbolTableNode, FormatError> {
// The entries are read once the header says how many there are:
// say so (see `Storage::hint`), for libhdf5's default node size
// (group leaf K = 4: 8 entries of 40 bytes with 8-byte offsets).
file.hint(offset, 8 + 8 * (2 * usize::from(offset_size) + 24));
// signature(4) + version(1) + reserved(1) + number_of_symbols(2) = 8
let header = read_exact_at(file, offset, 8)?;