osobhandClaude Opus 5.5
d63c76e7ab
Dense link and attribute indexes and the chunk index for several unlimited dimensions were single leaves, capping them at 65 535 records. btree_v2_write builds trees of any depth, with node capacities and pointer widths from libhdf5's H5B2__hdr_init arithmetic (now shared with the reader as btree_v2::node_info) and libhdf5's node sizes (512 dense, 2048 chunks). Indexes that fit the old one-leaf layout are written byte for byte as before (compared for 10..65 535 links, attrs and chunks, tracked and filtered). Tests: 100 000 links (short names; long names with creation order), 70 000 attributes, 200 000 chunks (and 80 000 deflated), read by h5py, h5dump and clawhdf5 and edited by h5py r+; h5rs check on the same shapes, asserting depths 2-3. Co-Authored-By: Claude Opus 5.5 (1M context) <[email protected]>
756 lines
26 KiB
Rust
756 lines
26 KiB
Rust
//! HDF5 B-tree v2 parsing.
|
|
|
|
#[cfg(not(feature = "std"))]
|
|
use alloc::vec::Vec;
|
|
|
|
#[cfg(feature = "checksum")]
|
|
use byteorder::{ByteOrder, LittleEndian};
|
|
|
|
use crate::error::FormatError;
|
|
|
|
/// Parsed B-tree v2 header (signature "BTHD").
|
|
#[derive(Debug, Clone)]
|
|
pub struct BTreeV2Header {
|
|
/// B-tree type: 5=links indexed by name, 6=links indexed by creation order, etc.
|
|
pub tree_type: u8,
|
|
/// Node size in bytes.
|
|
pub node_size: u32,
|
|
/// Record size in bytes.
|
|
pub record_size: u16,
|
|
/// Depth of the tree (0 = root is a leaf).
|
|
pub depth: u16,
|
|
/// Address of root node.
|
|
pub root_node_address: u64,
|
|
/// Number of records in the root node.
|
|
pub num_records_in_root: u16,
|
|
/// Total number of records in all nodes.
|
|
pub total_records: u64,
|
|
}
|
|
|
|
/// A single record from a B-tree v2 node.
|
|
#[derive(Debug, Clone)]
|
|
pub struct BTreeV2Record {
|
|
/// Raw record bytes (record_size bytes).
|
|
pub data: Vec<u8>,
|
|
}
|
|
|
|
fn read_offset(data: &[u8], pos: usize, size: u8) -> Result<u64, FormatError> {
|
|
let s = size as usize;
|
|
if pos.checked_add(s).is_none_or(|end| end > data.len()) {
|
|
return Err(FormatError::UnexpectedEof {
|
|
expected: pos.saturating_add(s),
|
|
available: data.len(),
|
|
});
|
|
}
|
|
Ok(match size {
|
|
2 => u16::from_le_bytes([data[pos], data[pos + 1]]) as u64,
|
|
4 => u32::from_le_bytes([data[pos], data[pos + 1], data[pos + 2], data[pos + 3]]) as u64,
|
|
8 => u64::from_le_bytes([
|
|
data[pos],
|
|
data[pos + 1],
|
|
data[pos + 2],
|
|
data[pos + 3],
|
|
data[pos + 4],
|
|
data[pos + 5],
|
|
data[pos + 6],
|
|
data[pos + 7],
|
|
]),
|
|
_ => return Err(FormatError::InvalidOffsetSize(size)),
|
|
})
|
|
}
|
|
|
|
fn ensure_len(data: &[u8], pos: usize, needed: usize) -> Result<(), FormatError> {
|
|
match pos.checked_add(needed) {
|
|
Some(end) if end <= data.len() => Ok(()),
|
|
_ => Err(FormatError::UnexpectedEof {
|
|
expected: pos.saturating_add(needed),
|
|
available: data.len(),
|
|
}),
|
|
}
|
|
}
|
|
|
|
/// Compute the number of bytes needed to represent a count, using variable-width encoding.
|
|
/// B-tree v2 uses this for the number of records fields in internal nodes.
|
|
pub(crate) fn bytes_for_max_records(max_nrec: u64) -> usize {
|
|
if max_nrec == 0 {
|
|
return 1;
|
|
}
|
|
let bits = 64 - max_nrec.leading_zeros() as usize;
|
|
bits.div_ceil(8)
|
|
}
|
|
|
|
/// Read a variable-width unsigned integer (1-8 bytes, LE).
|
|
fn read_var_uint(data: &[u8], pos: usize, width: usize) -> Result<u64, FormatError> {
|
|
ensure_len(data, pos, width)?;
|
|
let mut val = 0u64;
|
|
for i in 0..width {
|
|
val |= (data[pos + i] as u64) << (i * 8);
|
|
}
|
|
Ok(val)
|
|
}
|
|
|
|
impl BTreeV2Header {
|
|
/// Parse a B-tree v2 header at the given offset.
|
|
pub fn parse(
|
|
file_data: &[u8],
|
|
offset: usize,
|
|
offset_size: u8,
|
|
length_size: u8,
|
|
) -> Result<BTreeV2Header, FormatError> {
|
|
ensure_len(file_data, offset, 4)?;
|
|
if &file_data[offset..offset + 4] != b"BTHD" {
|
|
return Err(FormatError::InvalidBTreeV2Signature);
|
|
}
|
|
|
|
ensure_len(file_data, offset, 4 + 1 + 1 + 4 + 2 + 2 + 1 + 1)?;
|
|
let version = file_data[offset + 4];
|
|
if version != 0 {
|
|
return Err(FormatError::InvalidBTreeV2Version(version));
|
|
}
|
|
|
|
let tree_type = file_data[offset + 5];
|
|
let node_size = u32::from_le_bytes([
|
|
file_data[offset + 6],
|
|
file_data[offset + 7],
|
|
file_data[offset + 8],
|
|
file_data[offset + 9],
|
|
]);
|
|
let record_size = u16::from_le_bytes([file_data[offset + 10], file_data[offset + 11]]);
|
|
let depth = u16::from_le_bytes([file_data[offset + 12], file_data[offset + 13]]);
|
|
let _split_percent = file_data[offset + 14];
|
|
let _merge_percent = file_data[offset + 15];
|
|
|
|
let mut pos = offset + 16;
|
|
let root_node_address = read_offset(file_data, pos, offset_size)?;
|
|
pos += offset_size as usize;
|
|
|
|
ensure_len(file_data, pos, 2)?;
|
|
let num_records_in_root = u16::from_le_bytes([file_data[pos], file_data[pos + 1]]);
|
|
pos += 2;
|
|
|
|
let total_records = read_offset(file_data, pos, length_size)?;
|
|
#[allow(unused_assignments)]
|
|
{
|
|
pos += length_size as usize;
|
|
}
|
|
|
|
// Validate header checksum
|
|
#[cfg(feature = "checksum")]
|
|
{
|
|
ensure_len(file_data, pos, 4)?;
|
|
let stored = LittleEndian::read_u32(&file_data[pos..pos + 4]);
|
|
let computed = crate::checksum::jenkins_lookup3(&file_data[offset..pos]);
|
|
if computed != stored {
|
|
return Err(FormatError::ChecksumMismatch {
|
|
expected: stored,
|
|
computed,
|
|
});
|
|
}
|
|
}
|
|
|
|
Ok(BTreeV2Header {
|
|
tree_type,
|
|
node_size,
|
|
record_size,
|
|
depth,
|
|
root_node_address,
|
|
num_records_in_root,
|
|
total_records,
|
|
})
|
|
}
|
|
}
|
|
|
|
/// Compute maximum records per node for a given depth level.
|
|
/// leaf: (node_size - overhead) / record_size
|
|
/// internal: depends on pointers
|
|
pub(crate) fn max_records_leaf(node_size: u32, record_size: u16) -> u64 {
|
|
// Leaf overhead: signature(4) + version(1) + type(1) + checksum(4) = 10
|
|
let overhead = 10u32;
|
|
if node_size <= overhead || record_size == 0 {
|
|
return 0;
|
|
}
|
|
((node_size - overhead) / record_size as u32) as u64
|
|
}
|
|
|
|
/// Deepest B-tree v2 accepted. See [`collect_btree_v2_records`].
|
|
const MAX_DEPTH: u16 = 64;
|
|
|
|
/// Take `n` records from the traversal's budget, or refuse the tree.
|
|
fn spend(budget: &mut usize, n: usize) -> Result<(), FormatError> {
|
|
*budget = budget
|
|
.checked_sub(n)
|
|
.ok_or(FormatError::NestingDepthExceeded)?;
|
|
Ok(())
|
|
}
|
|
|
|
/// Collect all records from a B-tree v2 by traversing from the root.
|
|
pub fn collect_btree_v2_records(
|
|
file_data: &[u8],
|
|
header: &BTreeV2Header,
|
|
offset_size: u8,
|
|
length_size: u8,
|
|
) -> Result<Vec<BTreeV2Record>, FormatError> {
|
|
if header.total_records == 0 || header.num_records_in_root == 0 {
|
|
return Ok(Vec::new());
|
|
}
|
|
// Recursion is one frame per level, and the depth is read from the file:
|
|
// a crafted header claiming 65 535 levels over a node that is its own
|
|
// child overflowed the stack. 64 matches the fractal heap's guard, and no
|
|
// real tree comes close — even at the minimum fan-out of two it would
|
|
// hold more than 2^64 records.
|
|
if header.depth > MAX_DEPTH {
|
|
return Err(FormatError::NestingDepthExceeded);
|
|
}
|
|
// A valid tree stores each record once, in its own bytes, so it cannot
|
|
// hold more records than the file has room for. Children are addresses,
|
|
// though, and nothing makes them distinct: levels whose children all
|
|
// point at one shared node below reach it fan-out^depth times, which is
|
|
// millions of records from a few kilobytes. Counting against what the
|
|
// file could physically contain bounds that without trusting the
|
|
// header's own `total_records`.
|
|
let mut budget = file_data.len() / usize::from(header.record_size.max(1));
|
|
|
|
let max_leaf_nrec = max_records_leaf(header.node_size, header.record_size);
|
|
|
|
if header.depth == 0 {
|
|
// Root is a leaf
|
|
parse_leaf_records(
|
|
file_data,
|
|
header.root_node_address as usize,
|
|
header.num_records_in_root,
|
|
header.record_size,
|
|
)
|
|
} else {
|
|
// Root is internal; traverse recursively
|
|
let mut records = Vec::new();
|
|
collect_internal_records(
|
|
file_data,
|
|
header.root_node_address as usize,
|
|
header.num_records_in_root,
|
|
header.depth,
|
|
header.record_size,
|
|
header.node_size,
|
|
offset_size,
|
|
length_size,
|
|
max_leaf_nrec,
|
|
&mut budget,
|
|
&mut records,
|
|
)?;
|
|
Ok(records)
|
|
}
|
|
}
|
|
|
|
/// Parse records from a leaf node (signature "BTLF").
|
|
fn parse_leaf_records(
|
|
file_data: &[u8],
|
|
offset: usize,
|
|
num_records: u16,
|
|
record_size: u16,
|
|
) -> Result<Vec<BTreeV2Record>, FormatError> {
|
|
// signature(4) + version(1) + type(1) = 6 bytes header
|
|
ensure_len(file_data, offset, 6)?;
|
|
if &file_data[offset..offset + 4] != b"BTLF" {
|
|
return Err(FormatError::InvalidBTreeV2Signature);
|
|
}
|
|
|
|
let pos = offset + 6;
|
|
let rs = record_size as usize;
|
|
let total = (num_records as usize)
|
|
.checked_mul(rs)
|
|
.ok_or(FormatError::UnexpectedEof {
|
|
expected: usize::MAX,
|
|
available: file_data.len(),
|
|
})?;
|
|
ensure_len(file_data, pos, total)?;
|
|
|
|
// Validate checksum: 4 bytes after records + padding
|
|
#[cfg(feature = "checksum")]
|
|
{
|
|
let checksum_pos = pos + total;
|
|
if file_data.len() >= checksum_pos + 4 {
|
|
let stored = LittleEndian::read_u32(&file_data[checksum_pos..checksum_pos + 4]);
|
|
let computed = crate::checksum::jenkins_lookup3(&file_data[offset..checksum_pos]);
|
|
if computed != stored {
|
|
return Err(FormatError::ChecksumMismatch {
|
|
expected: stored,
|
|
computed,
|
|
});
|
|
}
|
|
}
|
|
}
|
|
|
|
let mut records = Vec::with_capacity(num_records as usize);
|
|
for i in 0..num_records as usize {
|
|
let start = pos + i * rs;
|
|
records.push(BTreeV2Record {
|
|
data: file_data[start..start + rs].to_vec(),
|
|
});
|
|
}
|
|
Ok(records)
|
|
}
|
|
|
|
/// Recursively collect records from an internal node.
|
|
#[allow(clippy::too_many_arguments, clippy::only_used_in_recursion)]
|
|
fn collect_internal_records(
|
|
file_data: &[u8],
|
|
offset: usize,
|
|
num_records: u16,
|
|
depth: u16,
|
|
record_size: u16,
|
|
node_size: u32,
|
|
offset_size: u8,
|
|
length_size: u8,
|
|
max_leaf_nrec: u64,
|
|
budget: &mut usize,
|
|
out: &mut Vec<BTreeV2Record>,
|
|
) -> Result<(), FormatError> {
|
|
// signature(4) + version(1) + type(1) = 6
|
|
ensure_len(file_data, offset, 6)?;
|
|
if &file_data[offset..offset + 4] != b"BTIN" {
|
|
return Err(FormatError::InvalidBTreeV2Signature);
|
|
}
|
|
|
|
let nr = num_records as usize;
|
|
let rs = record_size as usize;
|
|
let mut pos = offset + 6;
|
|
|
|
// Read all records first
|
|
let records_total = nr.checked_mul(rs).ok_or(FormatError::UnexpectedEof {
|
|
expected: usize::MAX,
|
|
available: file_data.len(),
|
|
})?;
|
|
ensure_len(file_data, pos, records_total)?;
|
|
let records_start = pos;
|
|
pos += records_total;
|
|
|
|
// Child pointer layout, as libhdf5 computes it (H5B2__hdr_init): the
|
|
// child's record count is always encoded in the width needed for a
|
|
// *leaf's* maximum, and — below the first internal level — the child
|
|
// subtree's total record count in the width needed for the most records
|
|
// a subtree of that depth can hold.
|
|
let child_depth = depth - 1;
|
|
let nrec_width = bytes_for_max_records(max_leaf_nrec);
|
|
let total_nrec_width = if depth > 1 {
|
|
bytes_for_max_records(cum_max_records(
|
|
node_size,
|
|
record_size,
|
|
offset_size,
|
|
max_leaf_nrec,
|
|
child_depth,
|
|
))
|
|
} else {
|
|
0
|
|
};
|
|
|
|
let num_children = nr + 1;
|
|
let child_ptr_size = offset_size as usize + nrec_width + total_nrec_width;
|
|
ensure_len(file_data, pos, num_children * child_ptr_size)?;
|
|
|
|
// Read child pointers
|
|
let mut children = Vec::with_capacity(num_children);
|
|
for _ in 0..num_children {
|
|
let addr = read_offset(file_data, pos, offset_size)?;
|
|
pos += offset_size as usize;
|
|
let child_nrec = read_var_uint(file_data, pos, nrec_width)? as u16;
|
|
pos += nrec_width;
|
|
pos += total_nrec_width; // skip total records in subtree
|
|
children.push((addr, child_nrec));
|
|
}
|
|
|
|
// Interleave: child[0], record[0], child[1], record[1], ..., child[nr]
|
|
// We collect child[0] records, then record[0], then child[1], etc.
|
|
for (i, &(child_addr, child_nrec)) in children.iter().enumerate() {
|
|
if child_depth == 0 {
|
|
// Before parsing, so a refused tree is not also a large allocation.
|
|
spend(budget, usize::from(child_nrec))?;
|
|
let leaf_recs =
|
|
parse_leaf_records(file_data, child_addr as usize, child_nrec, record_size)?;
|
|
out.extend(leaf_recs);
|
|
} else {
|
|
collect_internal_records(
|
|
file_data,
|
|
child_addr as usize,
|
|
child_nrec,
|
|
child_depth,
|
|
record_size,
|
|
node_size,
|
|
offset_size,
|
|
length_size,
|
|
max_leaf_nrec,
|
|
budget,
|
|
out,
|
|
)?;
|
|
}
|
|
|
|
// Add record[i] (except after the last child)
|
|
if i < nr {
|
|
let rec_offset = i.checked_mul(rs).ok_or(FormatError::UnexpectedEof {
|
|
expected: usize::MAX,
|
|
available: file_data.len(),
|
|
})?;
|
|
let rec_start =
|
|
records_start
|
|
.checked_add(rec_offset)
|
|
.ok_or(FormatError::UnexpectedEof {
|
|
expected: usize::MAX,
|
|
available: file_data.len(),
|
|
})?;
|
|
let rec_end = rec_start
|
|
.checked_add(rs)
|
|
.ok_or(FormatError::UnexpectedEof {
|
|
expected: usize::MAX,
|
|
available: file_data.len(),
|
|
})?;
|
|
if rec_end > file_data.len() {
|
|
return Err(FormatError::UnexpectedEof {
|
|
expected: rec_end,
|
|
available: file_data.len(),
|
|
});
|
|
}
|
|
spend(budget, 1)?;
|
|
out.push(BTreeV2Record {
|
|
data: file_data[rec_start..rec_end].to_vec(),
|
|
});
|
|
}
|
|
}
|
|
|
|
Ok(())
|
|
}
|
|
|
|
/// Most records a subtree whose root is at `depth` can hold (libhdf5's
|
|
/// `cum_max_nrec`). See [`node_info`].
|
|
fn cum_max_records(
|
|
node_size: u32,
|
|
record_size: u16,
|
|
offset_size: u8,
|
|
max_leaf_nrec: u64,
|
|
depth: u16,
|
|
) -> u64 {
|
|
node_info_from_leaf(node_size, record_size, offset_size, max_leaf_nrec, depth)
|
|
.last()
|
|
.map_or(max_leaf_nrec, |n| n.cum_max_nrec)
|
|
}
|
|
|
|
/// Capacity of a B-tree v2 node at one depth, as libhdf5 computes it
|
|
/// (`H5B2__hdr_init`'s `node_info`).
|
|
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
|
pub(crate) struct NodeInfo {
|
|
/// Most records one node at this depth holds.
|
|
pub(crate) max_nrec: u64,
|
|
/// Most records a subtree rooted at this depth holds.
|
|
pub(crate) cum_max_nrec: u64,
|
|
/// Bytes a subtree's total record count takes in a pointer to a node
|
|
/// at this depth (0 for a leaf, whose count is its own).
|
|
pub(crate) cum_max_nrec_size: usize,
|
|
}
|
|
|
|
/// Node capacities for depths `0..=depth` (entry `d` for depth `d`): a leaf
|
|
/// holds `max_nrec(0)` records; an internal node at depth `d` holds
|
|
/// `max_nrec(d)` records and `max_nrec(d) + 1` subtrees of depth `d - 1`,
|
|
/// where `max_nrec(d)` is what fits in a node once each record is paired
|
|
/// with a child pointer of the width depth `d` needs (address, the child's
|
|
/// record count in the width a *leaf's* maximum needs, and below the first
|
|
/// internal level the child subtree's total in the width its maximum
|
|
/// needs), with one pointer more than records.
|
|
pub(crate) fn node_info(
|
|
node_size: u32,
|
|
record_size: u16,
|
|
offset_size: u8,
|
|
depth: u16,
|
|
) -> Vec<NodeInfo> {
|
|
let max_leaf = max_records_leaf(node_size, record_size);
|
|
node_info_from_leaf(node_size, record_size, offset_size, max_leaf, depth)
|
|
}
|
|
|
|
fn node_info_from_leaf(
|
|
node_size: u32,
|
|
record_size: u16,
|
|
offset_size: u8,
|
|
max_leaf_nrec: u64,
|
|
depth: u16,
|
|
) -> Vec<NodeInfo> {
|
|
// Internal node overhead: signature(4) + version(1) + type(1) + checksum(4).
|
|
const PREFIX: u64 = 10;
|
|
let nrec_width = bytes_for_max_records(max_leaf_nrec) as u64;
|
|
let mut info = Vec::with_capacity(usize::from(depth) + 1);
|
|
info.push(NodeInfo {
|
|
max_nrec: max_leaf_nrec,
|
|
cum_max_nrec: max_leaf_nrec,
|
|
cum_max_nrec_size: 0,
|
|
});
|
|
for d in 1..=depth {
|
|
let below = info[usize::from(d) - 1];
|
|
let ptr = u64::from(offset_size)
|
|
+ nrec_width
|
|
+ if d > 1 {
|
|
below.cum_max_nrec_size as u64
|
|
} else {
|
|
0
|
|
};
|
|
let max_nrec = u64::from(node_size)
|
|
.saturating_sub(PREFIX)
|
|
.saturating_sub(ptr)
|
|
/ (u64::from(record_size) + ptr).max(1);
|
|
let cum = max_nrec
|
|
.saturating_add(1)
|
|
.saturating_mul(below.cum_max_nrec)
|
|
.saturating_add(max_nrec);
|
|
info.push(NodeInfo {
|
|
max_nrec,
|
|
cum_max_nrec: cum,
|
|
cum_max_nrec_size: bytes_for_max_records(cum),
|
|
});
|
|
}
|
|
info
|
|
}
|
|
|
|
#[cfg(test)]
|
|
mod tests {
|
|
use super::*;
|
|
|
|
#[allow(clippy::too_many_arguments)]
|
|
fn build_btree_v2_header(
|
|
tree_type: u8,
|
|
node_size: u32,
|
|
record_size: u16,
|
|
depth: u16,
|
|
root_addr: u64,
|
|
num_records_root: u16,
|
|
total_records: u64,
|
|
offset_size: u8,
|
|
length_size: u8,
|
|
) -> Vec<u8> {
|
|
let mut buf = Vec::new();
|
|
buf.extend_from_slice(b"BTHD");
|
|
buf.push(0); // version
|
|
buf.push(tree_type);
|
|
buf.extend_from_slice(&node_size.to_le_bytes());
|
|
buf.extend_from_slice(&record_size.to_le_bytes());
|
|
buf.extend_from_slice(&depth.to_le_bytes());
|
|
buf.push(85); // split_percent
|
|
buf.push(40); // merge_percent
|
|
match offset_size {
|
|
4 => buf.extend_from_slice(&(root_addr as u32).to_le_bytes()),
|
|
8 => buf.extend_from_slice(&root_addr.to_le_bytes()),
|
|
_ => {}
|
|
}
|
|
buf.extend_from_slice(&num_records_root.to_le_bytes());
|
|
match length_size {
|
|
4 => buf.extend_from_slice(&(total_records as u32).to_le_bytes()),
|
|
8 => buf.extend_from_slice(&total_records.to_le_bytes()),
|
|
_ => {}
|
|
}
|
|
let checksum = crate::checksum::jenkins_lookup3(&buf);
|
|
buf.extend_from_slice(&checksum.to_le_bytes());
|
|
buf
|
|
}
|
|
|
|
fn build_leaf_node(tree_type: u8, records: &[&[u8]]) -> Vec<u8> {
|
|
let mut buf = Vec::new();
|
|
buf.extend_from_slice(b"BTLF");
|
|
buf.push(0); // version
|
|
buf.push(tree_type);
|
|
for rec in records {
|
|
buf.extend_from_slice(rec);
|
|
}
|
|
let checksum = crate::checksum::jenkins_lookup3(&buf);
|
|
buf.extend_from_slice(&checksum.to_le_bytes());
|
|
buf
|
|
}
|
|
|
|
/// An internal node laid out exactly as `collect_internal_records` will
|
|
/// read it at `depth`: `records` zeroed records, then `children` pointers,
|
|
/// all to `child_addr` claiming `child_nrec` records.
|
|
fn internal_node(
|
|
depth: u16,
|
|
node_size: u32,
|
|
record_size: u16,
|
|
records: usize,
|
|
children: usize,
|
|
child_addr: u64,
|
|
child_nrec: u64,
|
|
) -> Vec<u8> {
|
|
let max_leaf = max_records_leaf(node_size, record_size);
|
|
let nrec_width = bytes_for_max_records(max_leaf);
|
|
let total_width = if depth > 1 {
|
|
bytes_for_max_records(cum_max_records(
|
|
node_size,
|
|
record_size,
|
|
8,
|
|
max_leaf,
|
|
depth - 1,
|
|
))
|
|
} else {
|
|
0
|
|
};
|
|
let mut buf = b"BTIN".to_vec();
|
|
buf.extend_from_slice(&[0, 5]);
|
|
buf.resize(buf.len() + records * record_size as usize, 0);
|
|
for _ in 0..children {
|
|
buf.extend_from_slice(&child_addr.to_le_bytes());
|
|
buf.extend_from_slice(&child_nrec.to_le_bytes()[..nrec_width]);
|
|
buf.resize(buf.len() + total_width, 0);
|
|
}
|
|
buf
|
|
}
|
|
|
|
fn header(depth: u16, root: u64, root_nrec: u16, total: u64) -> BTreeV2Header {
|
|
BTreeV2Header {
|
|
tree_type: 5,
|
|
node_size: 512,
|
|
record_size: 8,
|
|
depth,
|
|
root_node_address: root,
|
|
num_records_in_root: root_nrec,
|
|
total_records: total,
|
|
}
|
|
}
|
|
|
|
#[test]
|
|
fn a_node_that_is_its_own_child_is_rejected_not_recursed() {
|
|
// One internal node whose two children are itself, under a header
|
|
// claiming the deepest tree a u16 allows. The layout stops depending
|
|
// on depth once the subtree-total width saturates, so every level
|
|
// parses cleanly and recursion runs ~65 000 frames deep: before the
|
|
// cap this overflowed the stack and aborted the process, from a file
|
|
// of under 100 bytes.
|
|
let mut data = internal_node(u16::MAX, 512, 8, 1, 2, 0, 1);
|
|
data.resize(4096, 0);
|
|
let result = collect_btree_v2_records(&data, &header(u16::MAX, 0, 1, 1), 8, 8);
|
|
assert!(result.is_err(), "{result:?}");
|
|
}
|
|
|
|
#[test]
|
|
fn a_shared_subtree_cannot_multiply_the_work() {
|
|
// A chain of distinct levels, each node's children all pointing at the
|
|
// single node below, ending in a real leaf. Every node parses and
|
|
// nothing is cyclic, yet the leaf is reached fan-out^depth times: 62
|
|
// children over 4 levels is ~15 million leaf visits from a few
|
|
// kilobytes. A valid tree cannot hold more records than the file has
|
|
// room for, so that bounds the traversal instead.
|
|
let (node_size, record_size) = (512u32, 8u16);
|
|
let fanout = 62usize;
|
|
let depth = 4u16;
|
|
let leaf = build_leaf_node(5, &[&[0u8; 8][..]]);
|
|
|
|
// Lay out root first, then each lower level, then the leaf.
|
|
let mut nodes: Vec<Vec<u8>> = Vec::new();
|
|
let mut addrs = Vec::new();
|
|
let mut at = 0u64;
|
|
let mut sizes = Vec::new();
|
|
for d in (1..=depth).rev() {
|
|
let n = internal_node(d, node_size, record_size, fanout - 1, fanout, 0, 0);
|
|
sizes.push(n.len());
|
|
}
|
|
for size in &sizes {
|
|
addrs.push(at);
|
|
at += *size as u64;
|
|
}
|
|
let leaf_addr = at;
|
|
for (i, d) in (1..=depth).rev().enumerate() {
|
|
let (child, child_nrec) = if d == 1 {
|
|
(leaf_addr, 1)
|
|
} else {
|
|
(addrs[i + 1], fanout as u64 - 1)
|
|
};
|
|
nodes.push(internal_node(
|
|
d,
|
|
node_size,
|
|
record_size,
|
|
fanout - 1,
|
|
fanout,
|
|
child,
|
|
child_nrec,
|
|
));
|
|
}
|
|
let mut data: Vec<u8> = nodes.concat();
|
|
data.extend_from_slice(&leaf);
|
|
data.resize(data.len() + 64, 0);
|
|
|
|
let started = std::time::Instant::now();
|
|
let result =
|
|
collect_btree_v2_records(&data, &header(depth, 0, fanout as u16 - 1, u64::MAX), 8, 8);
|
|
assert!(
|
|
result.is_err(),
|
|
"expected a refusal, got {} records",
|
|
result.map_or(0, |r| r.len())
|
|
);
|
|
assert!(
|
|
started.elapsed() < std::time::Duration::from_secs(2),
|
|
"took {:?}",
|
|
started.elapsed()
|
|
);
|
|
}
|
|
|
|
#[test]
|
|
fn parse_header() {
|
|
let data = build_btree_v2_header(5, 512, 11, 0, 0x1000, 3, 3, 8, 8);
|
|
let hdr = BTreeV2Header::parse(&data, 0, 8, 8).unwrap();
|
|
assert_eq!(hdr.tree_type, 5);
|
|
assert_eq!(hdr.node_size, 512);
|
|
assert_eq!(hdr.record_size, 11);
|
|
assert_eq!(hdr.depth, 0);
|
|
assert_eq!(hdr.root_node_address, 0x1000);
|
|
assert_eq!(hdr.num_records_in_root, 3);
|
|
assert_eq!(hdr.total_records, 3);
|
|
}
|
|
|
|
#[test]
|
|
fn parse_leaf_with_2_records() {
|
|
let rec1 = [1u8, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11];
|
|
let rec2 = [11u8, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21];
|
|
let leaf = build_leaf_node(5, &[&rec1, &rec2]);
|
|
|
|
let leaf_offset = 256usize;
|
|
let header = build_btree_v2_header(5, 512, 11, 0, leaf_offset as u64, 2, 2, 8, 8);
|
|
|
|
let mut file_data = vec![0u8; 512];
|
|
file_data[..header.len()].copy_from_slice(&header);
|
|
file_data[leaf_offset..leaf_offset + leaf.len()].copy_from_slice(&leaf);
|
|
|
|
let hdr = BTreeV2Header::parse(&file_data, 0, 8, 8).unwrap();
|
|
let records = collect_btree_v2_records(&file_data, &hdr, 8, 8).unwrap();
|
|
assert_eq!(records.len(), 2);
|
|
assert_eq!(records[0].data, rec1.to_vec());
|
|
assert_eq!(records[1].data, rec2.to_vec());
|
|
}
|
|
|
|
#[test]
|
|
fn invalid_signature() {
|
|
let mut data = build_btree_v2_header(5, 512, 11, 0, 0, 0, 0, 8, 8);
|
|
data[0] = b'X';
|
|
let err = BTreeV2Header::parse(&data, 0, 8, 8).unwrap_err();
|
|
assert_eq!(err, FormatError::InvalidBTreeV2Signature);
|
|
}
|
|
|
|
#[test]
|
|
fn invalid_version() {
|
|
let mut data = build_btree_v2_header(5, 512, 11, 0, 0, 0, 0, 8, 8);
|
|
data[4] = 1; // bad version
|
|
let err = BTreeV2Header::parse(&data, 0, 8, 8).unwrap_err();
|
|
assert_eq!(err, FormatError::InvalidBTreeV2Version(1));
|
|
}
|
|
|
|
#[test]
|
|
fn empty_tree() {
|
|
let header = build_btree_v2_header(5, 512, 11, 0, 0, 0, 0, 8, 8);
|
|
let hdr = BTreeV2Header::parse(&header, 0, 8, 8).unwrap();
|
|
let records = collect_btree_v2_records(&header, &hdr, 8, 8).unwrap();
|
|
assert!(records.is_empty());
|
|
}
|
|
|
|
#[test]
|
|
fn subtree_capacity_matches_libhdf5() {
|
|
// A link-name index (11-byte records, 512-byte nodes, 8-byte
|
|
// addresses): libhdf5's H5B2__hdr_init gives 45 records per leaf,
|
|
// then cum_max_nrec 1 149 at depth 1 and 26 449 at depth 2 — two
|
|
// bytes of subtree count in a depth-3 root's child pointers, where
|
|
// leaf_max^3 = 91 125 would need three.
|
|
let leaf = max_records_leaf(512, 11);
|
|
assert_eq!(leaf, 45);
|
|
assert_eq!(cum_max_records(512, 11, 8, leaf, 0), 45);
|
|
assert_eq!(cum_max_records(512, 11, 8, leaf, 1), 1_149);
|
|
assert_eq!(cum_max_records(512, 11, 8, leaf, 2), 26_449);
|
|
}
|
|
}
|