osobhandClaude Opus 5.5
e553153e48
Listing a group read every child's object header and stopped at the
first that was not fetched yet, and so did the traversals of the group's
index (v1 B-tree and symbol table nodes, the local heap's names, v2
B-tree nodes and fractal heap objects). Over openUrl's restartable
reader each block cost its own pass and round trip: 184 serial requests
to list 3000 datasets at 1 MiB blocks, 536 at 64 KiB.
- core::Reader::list reads every child's header before returning the
first error (the same error, in listing order, Group::groups/datasets
return), classifying them as those do.
- clawhdf5-format: after the first sibling that fails, the B-tree v1
and v2 collectors, the symbol table node loop and the dense-link loop
go on reading (not using) the remaining siblings, then return that
first error: results and errors are unchanged, only failing
traversals read more, and in memory that is free (storage::touch).
A v1 group's local heap segment (names) is read at once, up to 1 MiB.
- LazyStorage no longer fills a one-block hole that is already cached
(it was fetched again: 215 MB fetched from a 198 MB file).
Measured with tests/lazy.rs listing_cost_of_a_given_file on the
reviewer's file (h5py, 3000 datasets of 64 KiB, 198 MB), list('/'):
libver earliest, 1 MiB blocks: 185 passes/184 requests -> 6/73
libver earliest, 64 KiB: 537/536 -> 8/531 (6 in flight)
libver latest, 1 MiB: 189/188 -> 9/98
libver latest, 64 KiB: 453/452 -> 11/452
Bytes fetched are unchanged (the headers are spread through the file).
New test listing_a_large_group_takes_a_few_passes (512-byte blocks):
FileBuilder 600 children 102 -> 5 passes; h5py earliest/latest 2000
children 8 and 11 passes. Conformance 600 of 697 (baseline 600);
check-32bit-casts, check-nostd and h5rs-fuzz over the CVE corpus clean.
Co-Authored-By: Claude Opus 5.5 (1M context) <[email protected]>
1031 lines
34 KiB
Rust
1031 lines
34 KiB
Rust
//! HDF5 B-tree v2 parsing.
|
|
|
|
#[cfg(not(feature = "std"))]
|
|
use alloc::vec::Vec;
|
|
use core::cmp::Ordering;
|
|
|
|
#[cfg(feature = "checksum")]
|
|
use byteorder::{ByteOrder, LittleEndian};
|
|
|
|
use crate::addr::to_usize;
|
|
use crate::error::FormatError;
|
|
use crate::storage::{Storage, Window, len_usize};
|
|
|
|
/// Parsed B-tree v2 header (signature "BTHD").
|
|
#[derive(Debug, Clone)]
|
|
pub struct BTreeV2Header {
|
|
/// B-tree type: 5=links indexed by name, 6=links indexed by creation order, etc.
|
|
pub tree_type: u8,
|
|
/// Node size in bytes.
|
|
pub node_size: u32,
|
|
/// Record size in bytes.
|
|
pub record_size: u16,
|
|
/// Depth of the tree (0 = root is a leaf).
|
|
pub depth: u16,
|
|
/// Address of root node.
|
|
pub root_node_address: u64,
|
|
/// Number of records in the root node.
|
|
pub num_records_in_root: u16,
|
|
/// Total number of records in all nodes.
|
|
pub total_records: u64,
|
|
}
|
|
|
|
/// A single record from a B-tree v2 node.
|
|
#[derive(Debug, Clone)]
|
|
pub struct BTreeV2Record {
|
|
/// Raw record bytes (record_size bytes).
|
|
pub data: Vec<u8>,
|
|
}
|
|
|
|
fn read_offset(data: &[u8], pos: usize, size: u8) -> Result<u64, FormatError> {
|
|
let s = size as usize;
|
|
if pos.checked_add(s).is_none_or(|end| end > data.len()) {
|
|
return Err(FormatError::UnexpectedEof {
|
|
expected: pos.saturating_add(s),
|
|
available: data.len(),
|
|
});
|
|
}
|
|
Ok(match size {
|
|
2 => u16::from_le_bytes([data[pos], data[pos + 1]]) as u64,
|
|
4 => u32::from_le_bytes([data[pos], data[pos + 1], data[pos + 2], data[pos + 3]]) as u64,
|
|
8 => u64::from_le_bytes([
|
|
data[pos],
|
|
data[pos + 1],
|
|
data[pos + 2],
|
|
data[pos + 3],
|
|
data[pos + 4],
|
|
data[pos + 5],
|
|
data[pos + 6],
|
|
data[pos + 7],
|
|
]),
|
|
_ => return Err(FormatError::InvalidOffsetSize(size)),
|
|
})
|
|
}
|
|
|
|
fn ensure_len(data: &[u8], pos: usize, needed: usize) -> Result<(), FormatError> {
|
|
match pos.checked_add(needed) {
|
|
Some(end) if end <= data.len() => Ok(()),
|
|
_ => Err(FormatError::UnexpectedEof {
|
|
expected: pos.saturating_add(needed),
|
|
available: data.len(),
|
|
}),
|
|
}
|
|
}
|
|
|
|
/// Compute the number of bytes needed to represent a count, using variable-width encoding.
|
|
/// B-tree v2 uses this for the number of records fields in internal nodes.
|
|
pub(crate) fn bytes_for_max_records(max_nrec: u64) -> usize {
|
|
if max_nrec == 0 {
|
|
return 1;
|
|
}
|
|
let bits = 64 - max_nrec.leading_zeros() as usize;
|
|
bits.div_ceil(8)
|
|
}
|
|
|
|
/// Read a variable-width unsigned integer (1-8 bytes, LE).
|
|
fn read_var_uint(data: &[u8], pos: usize, width: usize) -> Result<u64, FormatError> {
|
|
ensure_len(data, pos, width)?;
|
|
let mut val = 0u64;
|
|
for i in 0..width {
|
|
val |= (data[pos + i] as u64) << (i * 8);
|
|
}
|
|
Ok(val)
|
|
}
|
|
|
|
impl BTreeV2Header {
|
|
/// Parse a B-tree v2 header at the given offset.
|
|
pub fn parse(
|
|
file_data: &[u8],
|
|
offset: usize,
|
|
offset_size: u8,
|
|
length_size: u8,
|
|
) -> Result<BTreeV2Header, FormatError> {
|
|
Self::parse_in(file_data, offset as u64, offset_size, length_size)
|
|
}
|
|
|
|
/// [`Self::parse`] over any [`Storage`]: one bounded read of the
|
|
/// header.
|
|
pub fn parse_in<S: Storage + ?Sized>(
|
|
file: &S,
|
|
offset: u64,
|
|
offset_size: u8,
|
|
length_size: u8,
|
|
) -> Result<BTreeV2Header, FormatError> {
|
|
// Every field and the checksum; the window holds all of it or ends
|
|
// at the end of the file, so its bounds checks are the whole-file
|
|
// ones.
|
|
let full = 16 + usize::from(offset_size) + 2 + usize::from(length_size) + 4;
|
|
let w = Window::read(file, offset, full)?;
|
|
let d = &w.bytes;
|
|
w.ensure(0, 4)?;
|
|
if &d[..4] != b"BTHD" {
|
|
return Err(FormatError::InvalidBTreeV2Signature);
|
|
}
|
|
|
|
w.ensure(0, 4 + 1 + 1 + 4 + 2 + 2 + 1 + 1)?;
|
|
let version = d[4];
|
|
if version != 0 {
|
|
return Err(FormatError::InvalidBTreeV2Version(version));
|
|
}
|
|
|
|
let tree_type = d[5];
|
|
let node_size = u32::from_le_bytes([d[6], d[7], d[8], d[9]]);
|
|
let record_size = u16::from_le_bytes([d[10], d[11]]);
|
|
let depth = u16::from_le_bytes([d[12], d[13]]);
|
|
let _split_percent = d[14];
|
|
let _merge_percent = d[15];
|
|
|
|
let mut pos = 16;
|
|
w.ensure(pos, usize::from(offset_size))?;
|
|
let root_node_address = read_offset(d, pos, offset_size)?;
|
|
pos += offset_size as usize;
|
|
|
|
w.ensure(pos, 2)?;
|
|
let num_records_in_root = u16::from_le_bytes([d[pos], d[pos + 1]]);
|
|
pos += 2;
|
|
|
|
w.ensure(pos, usize::from(length_size))?;
|
|
let total_records = read_offset(d, pos, length_size)?;
|
|
#[allow(unused_assignments)]
|
|
{
|
|
pos += length_size as usize;
|
|
}
|
|
|
|
// Validate header checksum
|
|
#[cfg(feature = "checksum")]
|
|
{
|
|
w.ensure(pos, 4)?;
|
|
let stored = LittleEndian::read_u32(&d[pos..pos + 4]);
|
|
let computed = crate::checksum::jenkins_lookup3(&d[..pos]);
|
|
if computed != stored {
|
|
return Err(FormatError::ChecksumMismatch {
|
|
expected: stored,
|
|
computed,
|
|
});
|
|
}
|
|
}
|
|
|
|
Ok(BTreeV2Header {
|
|
tree_type,
|
|
node_size,
|
|
record_size,
|
|
depth,
|
|
root_node_address,
|
|
num_records_in_root,
|
|
total_records,
|
|
})
|
|
}
|
|
}
|
|
|
|
/// Compute maximum records per node for a given depth level.
|
|
/// leaf: (node_size - overhead) / record_size
|
|
/// internal: depends on pointers
|
|
pub(crate) fn max_records_leaf(node_size: u32, record_size: u16) -> u64 {
|
|
// Leaf overhead: signature(4) + version(1) + type(1) + checksum(4) = 10
|
|
let overhead = 10u32;
|
|
if node_size <= overhead || record_size == 0 {
|
|
return 0;
|
|
}
|
|
((node_size - overhead) / record_size as u32) as u64
|
|
}
|
|
|
|
/// Deepest B-tree v2 accepted. See [`collect_btree_v2_records`].
|
|
const MAX_DEPTH: u16 = 64;
|
|
|
|
/// Take `n` records from the traversal's budget, or refuse the tree.
|
|
fn spend(budget: &mut usize, n: usize) -> Result<(), FormatError> {
|
|
*budget = budget
|
|
.checked_sub(n)
|
|
.ok_or(FormatError::NestingDepthExceeded)?;
|
|
Ok(())
|
|
}
|
|
|
|
/// Collect all records from a B-tree v2 by traversing from the root.
|
|
pub fn collect_btree_v2_records(
|
|
file_data: &[u8],
|
|
header: &BTreeV2Header,
|
|
offset_size: u8,
|
|
length_size: u8,
|
|
) -> Result<Vec<BTreeV2Record>, FormatError> {
|
|
collect_btree_v2_records_in(file_data, header, offset_size, length_size)
|
|
}
|
|
|
|
/// [`collect_btree_v2_records`] over any [`Storage`]: one bounded read per
|
|
/// node.
|
|
pub fn collect_btree_v2_records_in<S: Storage + ?Sized>(
|
|
file: &S,
|
|
header: &BTreeV2Header,
|
|
offset_size: u8,
|
|
length_size: u8,
|
|
) -> Result<Vec<BTreeV2Record>, FormatError> {
|
|
if header.total_records == 0 || header.num_records_in_root == 0 {
|
|
return Ok(Vec::new());
|
|
}
|
|
// Recursion is one frame per level, and the depth is read from the file:
|
|
// a crafted header claiming 65 535 levels over a node that is its own
|
|
// child overflowed the stack. 64 matches the fractal heap's guard, and no
|
|
// real tree comes close — even at the minimum fan-out of two it would
|
|
// hold more than 2^64 records.
|
|
if header.depth > MAX_DEPTH {
|
|
return Err(FormatError::NestingDepthExceeded);
|
|
}
|
|
// A valid tree stores each record once, in its own bytes, so it cannot
|
|
// hold more records than the file has room for. Children are addresses,
|
|
// though, and nothing makes them distinct: levels whose children all
|
|
// point at one shared node below reach it fan-out^depth times, which is
|
|
// millions of records from a few kilobytes. Counting against what the
|
|
// file could physically contain bounds that without trusting the
|
|
// header's own `total_records`.
|
|
let mut budget = len_usize(file) / usize::from(header.record_size.max(1));
|
|
|
|
let max_leaf_nrec = max_records_leaf(header.node_size, header.record_size);
|
|
|
|
if header.depth == 0 {
|
|
// Root is a leaf
|
|
parse_leaf_records(
|
|
file,
|
|
to_usize(header.root_node_address)?,
|
|
header.num_records_in_root,
|
|
header.record_size,
|
|
header.node_size,
|
|
)
|
|
} else {
|
|
// Root is internal; traverse recursively
|
|
let mut records = Vec::new();
|
|
collect_internal_records(
|
|
file,
|
|
to_usize(header.root_node_address)?,
|
|
header.num_records_in_root,
|
|
header.depth,
|
|
header.record_size,
|
|
header.node_size,
|
|
offset_size,
|
|
length_size,
|
|
max_leaf_nrec,
|
|
&mut budget,
|
|
&mut records,
|
|
)?;
|
|
Ok(records)
|
|
}
|
|
}
|
|
|
|
/// A node's bytes: `want` bytes at `offset` (fewer only at the end of the
|
|
/// file), after checking its 4-byte signature. A node is read in one piece
|
|
/// when it fits in `node_size` (every valid node does); a larger claimed
|
|
/// extent — record counts from a damaged parent — is first checked against
|
|
/// the end of the file, so it costs a read only of bytes the file has.
|
|
/// Bounds errors are the whole-file ones: the signature check needs the
|
|
/// first 6 bytes, then `checks` — `(position, length)` pairs relative to
|
|
/// the node, in the order the parser checks them — must lie in the file.
|
|
fn read_node<'a, S: Storage + ?Sized>(
|
|
file: &'a S,
|
|
offset: usize,
|
|
want: usize,
|
|
node_size: u32,
|
|
signature: &[u8; 4],
|
|
checks: &[(usize, usize)],
|
|
) -> Result<Window<'a>, FormatError> {
|
|
let one_read = usize::try_from(node_size).unwrap_or(usize::MAX).max(6);
|
|
let w = Window::read(file, offset as u64, want.min(one_read))?;
|
|
w.ensure(0, 6)?;
|
|
if &w.bytes[..4] != signature {
|
|
return Err(FormatError::InvalidBTreeV2Signature);
|
|
}
|
|
if want <= one_read {
|
|
return Ok(w);
|
|
}
|
|
for &(rel, len) in checks {
|
|
Window::check_extent(file, offset as u64, rel, len)?;
|
|
}
|
|
Window::read(file, offset as u64, want)
|
|
}
|
|
|
|
/// Parse records from a leaf node (signature "BTLF").
|
|
fn parse_leaf_records<S: Storage + ?Sized>(
|
|
file: &S,
|
|
offset: usize,
|
|
num_records: u16,
|
|
record_size: u16,
|
|
node_size: u32,
|
|
) -> Result<Vec<BTreeV2Record>, FormatError> {
|
|
// signature(4) + version(1) + type(1) = 6 bytes header
|
|
let pos = 6;
|
|
let rs = record_size as usize;
|
|
let total = (num_records as usize)
|
|
.checked_mul(rs)
|
|
.ok_or(FormatError::UnexpectedEof {
|
|
expected: usize::MAX,
|
|
available: len_usize(file),
|
|
})?;
|
|
let w = read_node(
|
|
file,
|
|
offset,
|
|
pos + total + 4,
|
|
node_size,
|
|
b"BTLF",
|
|
&[(pos, total)],
|
|
)?;
|
|
let d = &w.bytes;
|
|
w.ensure(pos, total)?;
|
|
|
|
// Validate checksum: 4 bytes after records + padding
|
|
#[cfg(feature = "checksum")]
|
|
{
|
|
let checksum_pos = pos + total;
|
|
if d.len() >= checksum_pos + 4 {
|
|
let stored = LittleEndian::read_u32(&d[checksum_pos..checksum_pos + 4]);
|
|
let computed = crate::checksum::jenkins_lookup3(&d[..checksum_pos]);
|
|
if computed != stored {
|
|
return Err(FormatError::ChecksumMismatch {
|
|
expected: stored,
|
|
computed,
|
|
});
|
|
}
|
|
}
|
|
}
|
|
|
|
let mut records = Vec::with_capacity(num_records as usize);
|
|
for i in 0..num_records as usize {
|
|
let start = pos + i * rs;
|
|
records.push(BTreeV2Record {
|
|
data: d[start..start + rs].to_vec(),
|
|
});
|
|
}
|
|
Ok(records)
|
|
}
|
|
|
|
/// An internal node read from the file: its bytes (from the signature on),
|
|
/// where its records start, and its children as `(address, record count)`.
|
|
struct InternalNode<'a> {
|
|
node: Window<'a>,
|
|
records_start: usize,
|
|
children: Vec<(u64, u16)>,
|
|
}
|
|
|
|
impl InternalNode<'_> {
|
|
/// Record `i`, `rs` bytes long.
|
|
fn record(&self, i: usize, rs: usize) -> Result<&[u8], FormatError> {
|
|
let overflow = || FormatError::UnexpectedEof {
|
|
expected: usize::MAX,
|
|
available: usize::MAX,
|
|
};
|
|
let rec_start = i
|
|
.checked_mul(rs)
|
|
.and_then(|o| self.records_start.checked_add(o))
|
|
.ok_or_else(overflow)?;
|
|
self.node.ensure(rec_start, rs)?;
|
|
Ok(&self.node.bytes[rec_start..rec_start + rs])
|
|
}
|
|
}
|
|
|
|
/// An internal node's layout: where its records start, and its children as
|
|
/// `(address, record count)`.
|
|
#[allow(clippy::too_many_arguments)]
|
|
fn read_internal_node<S: Storage + ?Sized>(
|
|
file: &S,
|
|
offset: usize,
|
|
num_records: u16,
|
|
depth: u16,
|
|
record_size: u16,
|
|
node_size: u32,
|
|
offset_size: u8,
|
|
max_leaf_nrec: u64,
|
|
) -> Result<InternalNode<'_>, FormatError> {
|
|
let nr = num_records as usize;
|
|
let rs = record_size as usize;
|
|
|
|
// Records first
|
|
let records_total = nr.checked_mul(rs).ok_or(FormatError::UnexpectedEof {
|
|
expected: usize::MAX,
|
|
available: len_usize(file),
|
|
})?;
|
|
|
|
// Child pointer layout, as libhdf5 computes it (H5B2__hdr_init): the
|
|
// child's record count is always encoded in the width needed for a
|
|
// *leaf's* maximum, and — below the first internal level — the child
|
|
// subtree's total record count in the width needed for the most records
|
|
// a subtree of that depth can hold.
|
|
let child_depth = depth - 1;
|
|
let nrec_width = bytes_for_max_records(max_leaf_nrec);
|
|
let total_nrec_width = if depth > 1 {
|
|
bytes_for_max_records(cum_max_records(
|
|
node_size,
|
|
record_size,
|
|
offset_size,
|
|
max_leaf_nrec,
|
|
child_depth,
|
|
))
|
|
} else {
|
|
0
|
|
};
|
|
|
|
let num_children = nr + 1;
|
|
let child_ptr_size = offset_size as usize + nrec_width + total_nrec_width;
|
|
let pointers = num_children * child_ptr_size;
|
|
|
|
// signature(4) + version(1) + type(1) = 6, records, pointers, checksum.
|
|
let w = read_node(
|
|
file,
|
|
offset,
|
|
6 + records_total + pointers + 4,
|
|
node_size,
|
|
b"BTIN",
|
|
&[(6, records_total), (6 + records_total, pointers)],
|
|
)?;
|
|
let d = &w.bytes;
|
|
let mut pos = 6;
|
|
w.ensure(pos, records_total)?;
|
|
let records_start = pos;
|
|
pos += records_total;
|
|
|
|
w.ensure(pos, pointers)?;
|
|
|
|
let mut children = Vec::with_capacity(num_children);
|
|
for _ in 0..num_children {
|
|
let addr = read_offset(d, pos, offset_size)?;
|
|
pos += offset_size as usize;
|
|
let child_nrec = read_var_uint(d, pos, nrec_width)? as u16;
|
|
pos += nrec_width;
|
|
pos += total_nrec_width; // skip total records in subtree
|
|
children.push((addr, child_nrec));
|
|
}
|
|
|
|
// The checksum follows the child pointers and covers the node up to it.
|
|
// Lookups prune children by the keys in this node, so an unverified
|
|
// internal node could hide a record without any error: libhdf5 refuses
|
|
// a mismatch here, and so does this.
|
|
#[cfg(feature = "checksum")]
|
|
{
|
|
w.ensure(pos, 4)?;
|
|
let stored = LittleEndian::read_u32(&d[pos..pos + 4]);
|
|
let computed = crate::checksum::jenkins_lookup3(&d[..pos]);
|
|
if computed != stored {
|
|
return Err(FormatError::ChecksumMismatch {
|
|
expected: stored,
|
|
computed,
|
|
});
|
|
}
|
|
}
|
|
Ok(InternalNode {
|
|
node: w,
|
|
records_start,
|
|
children,
|
|
})
|
|
}
|
|
|
|
/// Recursively collect records from an internal node.
|
|
#[allow(clippy::too_many_arguments, clippy::only_used_in_recursion)]
|
|
fn collect_internal_records<S: Storage + ?Sized>(
|
|
file: &S,
|
|
offset: usize,
|
|
num_records: u16,
|
|
depth: u16,
|
|
record_size: u16,
|
|
node_size: u32,
|
|
offset_size: u8,
|
|
length_size: u8,
|
|
max_leaf_nrec: u64,
|
|
budget: &mut usize,
|
|
out: &mut Vec<BTreeV2Record>,
|
|
) -> Result<(), FormatError> {
|
|
let nr = num_records as usize;
|
|
let rs = record_size as usize;
|
|
let node = read_internal_node(
|
|
file,
|
|
offset,
|
|
num_records,
|
|
depth,
|
|
record_size,
|
|
node_size,
|
|
offset_size,
|
|
max_leaf_nrec,
|
|
)?;
|
|
let child_depth = depth - 1;
|
|
|
|
// Interleave: child[0], record[0], child[1], record[1], ..., child[nr]
|
|
// We collect child[0] records, then record[0], then child[1], etc.
|
|
// After the first child that fails, the others are only touched (see
|
|
// `storage::touch`); that error is returned.
|
|
let mut failed = None;
|
|
for (i, &(child_addr, child_nrec)) in node.children.iter().enumerate() {
|
|
if failed.is_some() {
|
|
let len = usize::try_from(node_size)
|
|
.unwrap_or(usize::MAX)
|
|
.min(1 << 16);
|
|
crate::storage::touch(file, child_addr, len);
|
|
continue;
|
|
}
|
|
if let Err(e) = (|| -> Result<(), FormatError> {
|
|
if child_depth == 0 {
|
|
// Before parsing, so a refused tree is not also a large allocation.
|
|
spend(budget, usize::from(child_nrec))?;
|
|
let leaf_recs = parse_leaf_records(
|
|
file,
|
|
to_usize(child_addr)?,
|
|
child_nrec,
|
|
record_size,
|
|
node_size,
|
|
)?;
|
|
out.extend(leaf_recs);
|
|
} else {
|
|
collect_internal_records(
|
|
file,
|
|
to_usize(child_addr)?,
|
|
child_nrec,
|
|
child_depth,
|
|
record_size,
|
|
node_size,
|
|
offset_size,
|
|
length_size,
|
|
max_leaf_nrec,
|
|
budget,
|
|
out,
|
|
)?;
|
|
}
|
|
|
|
// Add record[i] (except after the last child)
|
|
if i < nr {
|
|
let data = node.record(i, rs)?;
|
|
spend(budget, 1)?;
|
|
out.push(BTreeV2Record {
|
|
data: data.to_vec(),
|
|
});
|
|
}
|
|
Ok(())
|
|
})() {
|
|
failed = Some(e);
|
|
}
|
|
}
|
|
|
|
match failed {
|
|
Some(e) => Err(e),
|
|
None => Ok(()),
|
|
}
|
|
}
|
|
|
|
/// The records of a B-tree v2 that fall in one key range, found by
|
|
/// descending the tree instead of reading all of it.
|
|
///
|
|
/// `cmp` places a record relative to the range: `Less` if the record sorts
|
|
/// before it, `Greater` if after, `Equal` if the record is in it. The tree
|
|
/// must be ordered consistently with `cmp`, as libhdf5 orders it (a link or
|
|
/// attribute name index by name hash, so all records with one hash form a
|
|
/// range whatever order their names are in). Only the nodes whose key
|
|
/// interval overlaps the range are read: O(depth) nodes plus those holding
|
|
/// the matches. Matches come in tree order.
|
|
pub fn find_btree_v2_records(
|
|
file_data: &[u8],
|
|
header: &BTreeV2Header,
|
|
offset_size: u8,
|
|
cmp: &mut dyn FnMut(&[u8]) -> Ordering,
|
|
) -> Result<Vec<BTreeV2Record>, FormatError> {
|
|
find_btree_v2_records_in(file_data, header, offset_size, cmp)
|
|
}
|
|
|
|
/// [`find_btree_v2_records`] over any [`Storage`]: one bounded read per
|
|
/// node visited.
|
|
pub fn find_btree_v2_records_in<S: Storage + ?Sized>(
|
|
file: &S,
|
|
header: &BTreeV2Header,
|
|
offset_size: u8,
|
|
cmp: &mut dyn FnMut(&[u8]) -> Ordering,
|
|
) -> Result<Vec<BTreeV2Record>, FormatError> {
|
|
if header.total_records == 0 || header.num_records_in_root == 0 {
|
|
return Ok(Vec::new());
|
|
}
|
|
if header.depth > MAX_DEPTH {
|
|
return Err(FormatError::NestingDepthExceeded);
|
|
}
|
|
// As in `collect_btree_v2_records`: a valid tree cannot hold more
|
|
// records than the file has room for, however its children are shared.
|
|
let mut budget = len_usize(file) / usize::from(header.record_size.max(1));
|
|
let max_leaf_nrec = max_records_leaf(header.node_size, header.record_size);
|
|
let mut out = Vec::new();
|
|
find_in_node(
|
|
file,
|
|
header,
|
|
to_usize(header.root_node_address)?,
|
|
header.num_records_in_root,
|
|
header.depth,
|
|
offset_size,
|
|
max_leaf_nrec,
|
|
cmp,
|
|
&mut budget,
|
|
&mut out,
|
|
)?;
|
|
Ok(out)
|
|
}
|
|
|
|
#[allow(clippy::too_many_arguments)]
|
|
fn find_in_node<S: Storage + ?Sized>(
|
|
file: &S,
|
|
header: &BTreeV2Header,
|
|
offset: usize,
|
|
num_records: u16,
|
|
depth: u16,
|
|
offset_size: u8,
|
|
max_leaf_nrec: u64,
|
|
cmp: &mut dyn FnMut(&[u8]) -> Ordering,
|
|
budget: &mut usize,
|
|
out: &mut Vec<BTreeV2Record>,
|
|
) -> Result<(), FormatError> {
|
|
spend(budget, usize::from(num_records))?;
|
|
if depth == 0 {
|
|
let records = parse_leaf_records(
|
|
file,
|
|
offset,
|
|
num_records,
|
|
header.record_size,
|
|
header.node_size,
|
|
)?;
|
|
out.extend(
|
|
records
|
|
.into_iter()
|
|
.filter(|r| cmp(&r.data) == Ordering::Equal),
|
|
);
|
|
return Ok(());
|
|
}
|
|
let rs = usize::from(header.record_size);
|
|
let node = read_internal_node(
|
|
file,
|
|
offset,
|
|
num_records,
|
|
depth,
|
|
header.record_size,
|
|
header.node_size,
|
|
offset_size,
|
|
max_leaf_nrec,
|
|
)?;
|
|
let nr = usize::from(num_records);
|
|
let mut order = Vec::with_capacity(nr);
|
|
for i in 0..nr {
|
|
order.push(cmp(node.record(i, rs)?));
|
|
}
|
|
// Child `i` holds the keys between record `i - 1` and record `i`: it can
|
|
// hold a match unless the record before it is already past the range or
|
|
// the record after it is still before it.
|
|
for (i, &(child_addr, child_nrec)) in node.children.iter().enumerate() {
|
|
let after_left = i == 0 || order[i - 1] != Ordering::Greater;
|
|
let before_right = i == nr || order[i] != Ordering::Less;
|
|
if after_left && before_right {
|
|
find_in_node(
|
|
file,
|
|
header,
|
|
to_usize(child_addr)?,
|
|
child_nrec,
|
|
depth - 1,
|
|
offset_size,
|
|
max_leaf_nrec,
|
|
cmp,
|
|
budget,
|
|
out,
|
|
)?;
|
|
}
|
|
if i < nr && order[i] == Ordering::Equal {
|
|
out.push(BTreeV2Record {
|
|
data: node.record(i, rs)?.to_vec(),
|
|
});
|
|
}
|
|
}
|
|
Ok(())
|
|
}
|
|
|
|
/// Most records a subtree whose root is at `depth` can hold (libhdf5's
|
|
/// `cum_max_nrec`). See [`node_info`].
|
|
fn cum_max_records(
|
|
node_size: u32,
|
|
record_size: u16,
|
|
offset_size: u8,
|
|
max_leaf_nrec: u64,
|
|
depth: u16,
|
|
) -> u64 {
|
|
node_info_from_leaf(node_size, record_size, offset_size, max_leaf_nrec, depth)
|
|
.last()
|
|
.map_or(max_leaf_nrec, |n| n.cum_max_nrec)
|
|
}
|
|
|
|
/// Capacity of a B-tree v2 node at one depth, as libhdf5 computes it
|
|
/// (`H5B2__hdr_init`'s `node_info`).
|
|
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
|
pub(crate) struct NodeInfo {
|
|
/// Most records one node at this depth holds.
|
|
pub(crate) max_nrec: u64,
|
|
/// Most records a subtree rooted at this depth holds.
|
|
pub(crate) cum_max_nrec: u64,
|
|
/// Bytes a subtree's total record count takes in a pointer to a node
|
|
/// at this depth (0 for a leaf, whose count is its own).
|
|
pub(crate) cum_max_nrec_size: usize,
|
|
}
|
|
|
|
/// Node capacities for depths `0..=depth` (entry `d` for depth `d`): a leaf
|
|
/// holds `max_nrec(0)` records; an internal node at depth `d` holds
|
|
/// `max_nrec(d)` records and `max_nrec(d) + 1` subtrees of depth `d - 1`,
|
|
/// where `max_nrec(d)` is what fits in a node once each record is paired
|
|
/// with a child pointer of the width depth `d` needs (address, the child's
|
|
/// record count in the width a *leaf's* maximum needs, and below the first
|
|
/// internal level the child subtree's total in the width its maximum
|
|
/// needs), with one pointer more than records.
|
|
pub(crate) fn node_info(
|
|
node_size: u32,
|
|
record_size: u16,
|
|
offset_size: u8,
|
|
depth: u16,
|
|
) -> Vec<NodeInfo> {
|
|
let max_leaf = max_records_leaf(node_size, record_size);
|
|
node_info_from_leaf(node_size, record_size, offset_size, max_leaf, depth)
|
|
}
|
|
|
|
fn node_info_from_leaf(
|
|
node_size: u32,
|
|
record_size: u16,
|
|
offset_size: u8,
|
|
max_leaf_nrec: u64,
|
|
depth: u16,
|
|
) -> Vec<NodeInfo> {
|
|
// Internal node overhead: signature(4) + version(1) + type(1) + checksum(4).
|
|
const PREFIX: u64 = 10;
|
|
let nrec_width = bytes_for_max_records(max_leaf_nrec) as u64;
|
|
let mut info = Vec::with_capacity(usize::from(depth) + 1);
|
|
info.push(NodeInfo {
|
|
max_nrec: max_leaf_nrec,
|
|
cum_max_nrec: max_leaf_nrec,
|
|
cum_max_nrec_size: 0,
|
|
});
|
|
for d in 1..=depth {
|
|
let below = info[usize::from(d) - 1];
|
|
let ptr = u64::from(offset_size)
|
|
+ nrec_width
|
|
+ if d > 1 {
|
|
below.cum_max_nrec_size as u64
|
|
} else {
|
|
0
|
|
};
|
|
let max_nrec = u64::from(node_size)
|
|
.saturating_sub(PREFIX)
|
|
.saturating_sub(ptr)
|
|
/ (u64::from(record_size) + ptr).max(1);
|
|
let cum = max_nrec
|
|
.saturating_add(1)
|
|
.saturating_mul(below.cum_max_nrec)
|
|
.saturating_add(max_nrec);
|
|
info.push(NodeInfo {
|
|
max_nrec,
|
|
cum_max_nrec: cum,
|
|
cum_max_nrec_size: bytes_for_max_records(cum),
|
|
});
|
|
}
|
|
info
|
|
}
|
|
|
|
#[cfg(test)]
|
|
mod tests {
|
|
use super::*;
|
|
|
|
#[allow(clippy::too_many_arguments)]
|
|
fn build_btree_v2_header(
|
|
tree_type: u8,
|
|
node_size: u32,
|
|
record_size: u16,
|
|
depth: u16,
|
|
root_addr: u64,
|
|
num_records_root: u16,
|
|
total_records: u64,
|
|
offset_size: u8,
|
|
length_size: u8,
|
|
) -> Vec<u8> {
|
|
let mut buf = Vec::new();
|
|
buf.extend_from_slice(b"BTHD");
|
|
buf.push(0); // version
|
|
buf.push(tree_type);
|
|
buf.extend_from_slice(&node_size.to_le_bytes());
|
|
buf.extend_from_slice(&record_size.to_le_bytes());
|
|
buf.extend_from_slice(&depth.to_le_bytes());
|
|
buf.push(85); // split_percent
|
|
buf.push(40); // merge_percent
|
|
match offset_size {
|
|
4 => buf.extend_from_slice(&(root_addr as u32).to_le_bytes()),
|
|
8 => buf.extend_from_slice(&root_addr.to_le_bytes()),
|
|
_ => {}
|
|
}
|
|
buf.extend_from_slice(&num_records_root.to_le_bytes());
|
|
match length_size {
|
|
4 => buf.extend_from_slice(&(total_records as u32).to_le_bytes()),
|
|
8 => buf.extend_from_slice(&total_records.to_le_bytes()),
|
|
_ => {}
|
|
}
|
|
let checksum = crate::checksum::jenkins_lookup3(&buf);
|
|
buf.extend_from_slice(&checksum.to_le_bytes());
|
|
buf
|
|
}
|
|
|
|
fn build_leaf_node(tree_type: u8, records: &[&[u8]]) -> Vec<u8> {
|
|
let mut buf = Vec::new();
|
|
buf.extend_from_slice(b"BTLF");
|
|
buf.push(0); // version
|
|
buf.push(tree_type);
|
|
for rec in records {
|
|
buf.extend_from_slice(rec);
|
|
}
|
|
let checksum = crate::checksum::jenkins_lookup3(&buf);
|
|
buf.extend_from_slice(&checksum.to_le_bytes());
|
|
buf
|
|
}
|
|
|
|
/// An internal node laid out exactly as `collect_internal_records` will
|
|
/// read it at `depth`: `records` zeroed records, then `children` pointers,
|
|
/// all to `child_addr` claiming `child_nrec` records.
|
|
fn internal_node(
|
|
depth: u16,
|
|
node_size: u32,
|
|
record_size: u16,
|
|
records: usize,
|
|
children: usize,
|
|
child_addr: u64,
|
|
child_nrec: u64,
|
|
) -> Vec<u8> {
|
|
let max_leaf = max_records_leaf(node_size, record_size);
|
|
let nrec_width = bytes_for_max_records(max_leaf);
|
|
let total_width = if depth > 1 {
|
|
bytes_for_max_records(cum_max_records(
|
|
node_size,
|
|
record_size,
|
|
8,
|
|
max_leaf,
|
|
depth - 1,
|
|
))
|
|
} else {
|
|
0
|
|
};
|
|
let mut buf = b"BTIN".to_vec();
|
|
buf.extend_from_slice(&[0, 5]);
|
|
buf.resize(buf.len() + records * record_size as usize, 0);
|
|
for _ in 0..children {
|
|
buf.extend_from_slice(&child_addr.to_le_bytes());
|
|
buf.extend_from_slice(&child_nrec.to_le_bytes()[..nrec_width]);
|
|
buf.resize(buf.len() + total_width, 0);
|
|
}
|
|
let sum = crate::checksum::jenkins_lookup3(&buf);
|
|
buf.extend_from_slice(&sum.to_le_bytes());
|
|
buf
|
|
}
|
|
|
|
fn header(depth: u16, root: u64, root_nrec: u16, total: u64) -> BTreeV2Header {
|
|
BTreeV2Header {
|
|
tree_type: 5,
|
|
node_size: 512,
|
|
record_size: 8,
|
|
depth,
|
|
root_node_address: root,
|
|
num_records_in_root: root_nrec,
|
|
total_records: total,
|
|
}
|
|
}
|
|
|
|
#[test]
|
|
fn a_node_that_is_its_own_child_is_rejected_not_recursed() {
|
|
// One internal node whose two children are itself, under a header
|
|
// claiming the deepest tree a u16 allows. The layout stops depending
|
|
// on depth once the subtree-total width saturates, so every level
|
|
// parses cleanly and recursion runs ~65 000 frames deep: before the
|
|
// cap this overflowed the stack and aborted the process, from a file
|
|
// of under 100 bytes.
|
|
let mut data = internal_node(u16::MAX, 512, 8, 1, 2, 0, 1);
|
|
data.resize(4096, 0);
|
|
let result = collect_btree_v2_records(&data, &header(u16::MAX, 0, 1, 1), 8, 8);
|
|
assert!(result.is_err(), "{result:?}");
|
|
}
|
|
|
|
#[test]
|
|
fn a_shared_subtree_cannot_multiply_the_work() {
|
|
// A chain of distinct levels, each node's children all pointing at the
|
|
// single node below, ending in a real leaf. Every node parses and
|
|
// nothing is cyclic, yet the leaf is reached fan-out^depth times: 62
|
|
// children over 4 levels is ~15 million leaf visits from a few
|
|
// kilobytes. A valid tree cannot hold more records than the file has
|
|
// room for, so that bounds the traversal instead.
|
|
let (node_size, record_size) = (512u32, 8u16);
|
|
let fanout = 62usize;
|
|
let depth = 4u16;
|
|
let leaf = build_leaf_node(5, &[&[0u8; 8][..]]);
|
|
|
|
// Lay out root first, then each lower level, then the leaf.
|
|
let mut nodes: Vec<Vec<u8>> = Vec::new();
|
|
let mut addrs = Vec::new();
|
|
let mut at = 0u64;
|
|
let mut sizes = Vec::new();
|
|
for d in (1..=depth).rev() {
|
|
let n = internal_node(d, node_size, record_size, fanout - 1, fanout, 0, 0);
|
|
sizes.push(n.len());
|
|
}
|
|
for size in &sizes {
|
|
addrs.push(at);
|
|
at += *size as u64;
|
|
}
|
|
let leaf_addr = at;
|
|
for (i, d) in (1..=depth).rev().enumerate() {
|
|
let (child, child_nrec) = if d == 1 {
|
|
(leaf_addr, 1)
|
|
} else {
|
|
(addrs[i + 1], fanout as u64 - 1)
|
|
};
|
|
nodes.push(internal_node(
|
|
d,
|
|
node_size,
|
|
record_size,
|
|
fanout - 1,
|
|
fanout,
|
|
child,
|
|
child_nrec,
|
|
));
|
|
}
|
|
let mut data: Vec<u8> = nodes.concat();
|
|
data.extend_from_slice(&leaf);
|
|
data.resize(data.len() + 64, 0);
|
|
|
|
let started = std::time::Instant::now();
|
|
let result =
|
|
collect_btree_v2_records(&data, &header(depth, 0, fanout as u16 - 1, u64::MAX), 8, 8);
|
|
assert!(
|
|
result.is_err(),
|
|
"expected a refusal, got {} records",
|
|
result.map_or(0, |r| r.len())
|
|
);
|
|
assert!(
|
|
started.elapsed() < std::time::Duration::from_secs(2),
|
|
"took {:?}",
|
|
started.elapsed()
|
|
);
|
|
}
|
|
|
|
#[test]
|
|
fn parse_header() {
|
|
let data = build_btree_v2_header(5, 512, 11, 0, 0x1000, 3, 3, 8, 8);
|
|
let hdr = BTreeV2Header::parse(&data, 0, 8, 8).unwrap();
|
|
assert_eq!(hdr.tree_type, 5);
|
|
assert_eq!(hdr.node_size, 512);
|
|
assert_eq!(hdr.record_size, 11);
|
|
assert_eq!(hdr.depth, 0);
|
|
assert_eq!(hdr.root_node_address, 0x1000);
|
|
assert_eq!(hdr.num_records_in_root, 3);
|
|
assert_eq!(hdr.total_records, 3);
|
|
}
|
|
|
|
#[test]
|
|
fn parse_leaf_with_2_records() {
|
|
let rec1 = [1u8, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11];
|
|
let rec2 = [11u8, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21];
|
|
let leaf = build_leaf_node(5, &[&rec1, &rec2]);
|
|
|
|
let leaf_offset = 256usize;
|
|
let header = build_btree_v2_header(5, 512, 11, 0, leaf_offset as u64, 2, 2, 8, 8);
|
|
|
|
let mut file_data = vec![0u8; 512];
|
|
file_data[..header.len()].copy_from_slice(&header);
|
|
file_data[leaf_offset..leaf_offset + leaf.len()].copy_from_slice(&leaf);
|
|
|
|
let hdr = BTreeV2Header::parse(&file_data, 0, 8, 8).unwrap();
|
|
let records = collect_btree_v2_records(&file_data, &hdr, 8, 8).unwrap();
|
|
assert_eq!(records.len(), 2);
|
|
assert_eq!(records[0].data, rec1.to_vec());
|
|
assert_eq!(records[1].data, rec2.to_vec());
|
|
}
|
|
|
|
#[test]
|
|
fn invalid_signature() {
|
|
let mut data = build_btree_v2_header(5, 512, 11, 0, 0, 0, 0, 8, 8);
|
|
data[0] = b'X';
|
|
let err = BTreeV2Header::parse(&data, 0, 8, 8).unwrap_err();
|
|
assert_eq!(err, FormatError::InvalidBTreeV2Signature);
|
|
}
|
|
|
|
#[test]
|
|
fn invalid_version() {
|
|
let mut data = build_btree_v2_header(5, 512, 11, 0, 0, 0, 0, 8, 8);
|
|
data[4] = 1; // bad version
|
|
let err = BTreeV2Header::parse(&data, 0, 8, 8).unwrap_err();
|
|
assert_eq!(err, FormatError::InvalidBTreeV2Version(1));
|
|
}
|
|
|
|
#[test]
|
|
fn empty_tree() {
|
|
let header = build_btree_v2_header(5, 512, 11, 0, 0, 0, 0, 8, 8);
|
|
let hdr = BTreeV2Header::parse(&header, 0, 8, 8).unwrap();
|
|
let records = collect_btree_v2_records(&header, &hdr, 8, 8).unwrap();
|
|
assert!(records.is_empty());
|
|
}
|
|
|
|
#[test]
|
|
fn subtree_capacity_matches_libhdf5() {
|
|
// A link-name index (11-byte records, 512-byte nodes, 8-byte
|
|
// addresses): libhdf5's H5B2__hdr_init gives 45 records per leaf,
|
|
// then cum_max_nrec 1 149 at depth 1 and 26 449 at depth 2 — two
|
|
// bytes of subtree count in a depth-3 root's child pointers, where
|
|
// leaf_max^3 = 91 125 would need three.
|
|
let leaf = max_records_leaf(512, 11);
|
|
assert_eq!(leaf, 45);
|
|
assert_eq!(cum_max_records(512, 11, 8, leaf, 0), 45);
|
|
assert_eq!(cum_max_records(512, 11, 8, leaf, 1), 1_149);
|
|
assert_eq!(cum_max_records(512, 11, 8, leaf, 2), 26_449);
|
|
}
|
|
}
|