quantumclaw/clawhdf5
3
1
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 1 Wiki Activity

format: v2 B-trees, dense groups and group listings over Storage

Browse Source 
BTreeV2Header::parse_in, collect_btree_v2_records_in and
find_btree_v2_records_in read one bounded window per node (its size is
known from the parent before the node is read; a count stretched past
node_size is checked against the end of the file first), with the
whole-file bounds errors unchanged. With them, dense attributes, a SOHM
B-tree index and huge fractal-heap objects no longer answer
ContiguousStorageRequired, and group_v1/group_v2 listings, lookups and
path resolution get *_in cores (resolve_group_children_in,
resolve_child_in, resolve_path_any_in, ...). The &[u8] functions are
thin wrappers, as in M1.

The equivalence harness now fails on any ContiguousStorageRequired and
compares v2 B-tree headers, records and descents, group listings, child
lookups and paths; a unit test compares a two-level tree through a
read_at-only storage truncated at every length and with every node byte
flipped.

Co-Authored-By: Claude Opus 5.5 (1M context) <[email protected]>
This commit is contained in:
osobh
2026-09-26 16:13:01 -05:00
co-authored by Claude Opus 5.5
parent 8f59b2e1c2
commit 42894bf93b
9 changed files with 544 additions and 297 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/clawhdf5-format/src/btree_v2.rs
+201 -105
View File
@@ -9,6 +9,7 @@ use byteorder::{ByteOrder, LittleEndian};
use crate::addr::to_usize;
use crate::error::FormatError;
use crate::storage::{Storage, Window, len_usize};
/// Parsed B-tree v2 header (signature "BTHD").
#[derive(Debug, Clone)]
@@ -99,38 +100,52 @@ impl BTreeV2Header {
offset_size: u8,
length_size: u8,
) -> Result<BTreeV2Header, FormatError> {
ensure_len(file_data, offset, 4)?;
if &file_data[offset..offset + 4] != b"BTHD" {
Self::parse_in(file_data, offset as u64, offset_size, length_size)
}
/// [`Self::parse`] over any [`Storage`]: one bounded read of the
/// header.
pub fn parse_in<S: Storage + ?Sized>(
file: &S,
offset: u64,
offset_size: u8,
length_size: u8,
) -> Result<BTreeV2Header, FormatError> {
// Every field and the checksum; the window holds all of it or ends
// at the end of the file, so its bounds checks are the whole-file
// ones.
let full = 16 + usize::from(offset_size) + 2 + usize::from(length_size) + 4;
let w = Window::read(file, offset, full)?;
let d = &w.bytes;
w.ensure(0, 4)?;
if &d[..4] != b"BTHD" {
return Err(FormatError::InvalidBTreeV2Signature);
}
ensure_len(file_data, offset, 4 + 1 + 1 + 4 + 2 + 2 + 1 + 1)?;
let version = file_data[offset + 4];
w.ensure(0, 4 + 1 + 1 + 4 + 2 + 2 + 1 + 1)?;
let version = d[4];
if version != 0 {
return Err(FormatError::InvalidBTreeV2Version(version));
}
let tree_type = file_data[offset + 5];
let node_size = u32::from_le_bytes([
file_data[offset + 6],
file_data[offset + 7],
file_data[offset + 8],
file_data[offset + 9],
]);
let record_size = u16::from_le_bytes([file_data[offset + 10], file_data[offset + 11]]);
let depth = u16::from_le_bytes([file_data[offset + 12], file_data[offset + 13]]);
let _split_percent = file_data[offset + 14];
let _merge_percent = file_data[offset + 15];
let tree_type = d[5];
let node_size = u32::from_le_bytes([d[6], d[7], d[8], d[9]]);
let record_size = u16::from_le_bytes([d[10], d[11]]);
let depth = u16::from_le_bytes([d[12], d[13]]);
let _split_percent = d[14];
let _merge_percent = d[15];
let mut pos = offset + 16;
let root_node_address = read_offset(file_data, pos, offset_size)?;
let mut pos = 16;
w.ensure(pos, usize::from(offset_size))?;
let root_node_address = read_offset(d, pos, offset_size)?;
pos += offset_size as usize;
ensure_len(file_data, pos, 2)?;
let num_records_in_root = u16::from_le_bytes([file_data[pos], file_data[pos + 1]]);
w.ensure(pos, 2)?;
let num_records_in_root = u16::from_le_bytes([d[pos], d[pos + 1]]);
pos += 2;
let total_records = read_offset(file_data, pos, length_size)?;
w.ensure(pos, usize::from(length_size))?;
let total_records = read_offset(d, pos, length_size)?;
#[allow(unused_assignments)]
{
pos += length_size as usize;
@@ -139,9 +154,9 @@ impl BTreeV2Header {
// Validate header checksum
#[cfg(feature = "checksum")]
{
ensure_len(file_data, pos, 4)?;
let stored = LittleEndian::read_u32(&file_data[pos..pos + 4]);
let computed = crate::checksum::jenkins_lookup3(&file_data[offset..pos]);
w.ensure(pos, 4)?;
let stored = LittleEndian::read_u32(&d[pos..pos + 4]);
let computed = crate::checksum::jenkins_lookup3(&d[..pos]);
if computed != stored {
return Err(FormatError::ChecksumMismatch {
expected: stored,
@@ -191,6 +206,17 @@ pub fn collect_btree_v2_records(
header: &BTreeV2Header,
offset_size: u8,
length_size: u8,
) -> Result<Vec<BTreeV2Record>, FormatError> {
collect_btree_v2_records_in(file_data, header, offset_size, length_size)
}
/// [`collect_btree_v2_records`] over any [`Storage`]: one bounded read per
/// node.
pub fn collect_btree_v2_records_in<S: Storage + ?Sized>(
file: &S,
header: &BTreeV2Header,
offset_size: u8,
length_size: u8,
) -> Result<Vec<BTreeV2Record>, FormatError> {
if header.total_records == 0 || header.num_records_in_root == 0 {
return Ok(Vec::new());
@@ -210,23 +236,24 @@ pub fn collect_btree_v2_records(
// millions of records from a few kilobytes. Counting against what the
// file could physically contain bounds that without trusting the
// header's own `total_records`.
let mut budget = file_data.len() / usize::from(header.record_size.max(1));
let mut budget = len_usize(file) / usize::from(header.record_size.max(1));
let max_leaf_nrec = max_records_leaf(header.node_size, header.record_size);
if header.depth == 0 {
// Root is a leaf
parse_leaf_records(
file_data,
file,
to_usize(header.root_node_address)?,
header.num_records_in_root,
header.record_size,
header.node_size,
)
} else {
// Root is internal; traverse recursively
let mut records = Vec::new();
collect_internal_records(
file_data,
file,
to_usize(header.root_node_address)?,
header.num_records_in_root,
header.depth,
@@ -242,36 +269,72 @@ pub fn collect_btree_v2_records(
}
}
/// A node's bytes: `want` bytes at `offset` (fewer only at the end of the
/// file), after checking its 4-byte signature. A node is read in one piece
/// when it fits in `node_size` (every valid node does); a larger claimed
/// extent — record counts from a damaged parent — is first checked against
/// the end of the file, so it costs a read only of bytes the file has.
/// Bounds errors are the whole-file ones: the signature check needs the
/// first 6 bytes, then `checks` — `(position, length)` pairs relative to
/// the node, in the order the parser checks them — must lie in the file.
fn read_node<'a, S: Storage + ?Sized>(
file: &'a S,
offset: usize,
want: usize,
node_size: u32,
signature: &[u8; 4],
checks: &[(usize, usize)],
) -> Result<Window<'a>, FormatError> {
let one_read = usize::try_from(node_size).unwrap_or(usize::MAX).max(6);
let w = Window::read(file, offset as u64, want.min(one_read))?;
w.ensure(0, 6)?;
if &w.bytes[..4] != signature {
return Err(FormatError::InvalidBTreeV2Signature);
}
if want <= one_read {
return Ok(w);
}
for &(rel, len) in checks {
Window::check_extent(file, offset as u64, rel, len)?;
}
Window::read(file, offset as u64, want)
}
/// Parse records from a leaf node (signature "BTLF").
fn parse_leaf_records(
file_data: &[u8],
fn parse_leaf_records<S: Storage + ?Sized>(
file: &S,
offset: usize,
num_records: u16,
record_size: u16,
node_size: u32,
) -> Result<Vec<BTreeV2Record>, FormatError> {
// signature(4) + version(1) + type(1) = 6 bytes header
ensure_len(file_data, offset, 6)?;
if &file_data[offset..offset + 4] != b"BTLF" {
return Err(FormatError::InvalidBTreeV2Signature);
}
let pos = offset + 6;
let pos = 6;
let rs = record_size as usize;
let total = (num_records as usize)
.checked_mul(rs)
.ok_or(FormatError::UnexpectedEof {
expected: usize::MAX,
available: file_data.len(),
available: len_usize(file),
})?;
ensure_len(file_data, pos, total)?;
let w = read_node(
file,
offset,
pos + total + 4,
node_size,
b"BTLF",
&[(pos, total)],
)?;
let d = &w.bytes;
w.ensure(pos, total)?;
// Validate checksum: 4 bytes after records + padding
#[cfg(feature = "checksum")]
{
let checksum_pos = pos + total;
if file_data.len() >= checksum_pos + 4 {
let stored = LittleEndian::read_u32(&file_data[checksum_pos..checksum_pos + 4]);
let computed = crate::checksum::jenkins_lookup3(&file_data[offset..checksum_pos]);
if d.len() >= checksum_pos + 4 {
let stored = LittleEndian::read_u32(&d[checksum_pos..checksum_pos + 4]);
let computed = crate::checksum::jenkins_lookup3(&d[..checksum_pos]);
if computed != stored {
return Err(FormatError::ChecksumMismatch {
expected: stored,
@@ -285,17 +348,41 @@ fn parse_leaf_records(
for i in 0..num_records as usize {
let start = pos + i * rs;
records.push(BTreeV2Record {
data: file_data[start..start + rs].to_vec(),
data: d[start..start + rs].to_vec(),
});
}
Ok(records)
}
/// An internal node read from the file: its bytes (from the signature on),
/// where its records start, and its children as `(address, record count)`.
struct InternalNode<'a> {
node: Window<'a>,
records_start: usize,
children: Vec<(u64, u16)>,
}
impl InternalNode<'_> {
/// Record `i`, `rs` bytes long.
fn record(&self, i: usize, rs: usize) -> Result<&[u8], FormatError> {
let overflow = || FormatError::UnexpectedEof {
expected: usize::MAX,
available: usize::MAX,
};
let rec_start = i
.checked_mul(rs)
.and_then(|o| self.records_start.checked_add(o))
.ok_or_else(overflow)?;
self.node.ensure(rec_start, rs)?;
Ok(&self.node.bytes[rec_start..rec_start + rs])
}
}
/// An internal node's layout: where its records start, and its children as
/// `(address, record count)`.
#[allow(clippy::too_many_arguments)]
fn read_internal_node(
file_data: &[u8],
fn read_internal_node<S: Storage + ?Sized>(
file: &S,
offset: usize,
num_records: u16,
depth: u16,
@@ -303,25 +390,15 @@ fn read_internal_node(
node_size: u32,
offset_size: u8,
max_leaf_nrec: u64,
) -> Result<(usize, Vec<(u64, u16)>), FormatError> {
// signature(4) + version(1) + type(1) = 6
ensure_len(file_data, offset, 6)?;
if &file_data[offset..offset + 4] != b"BTIN" {
return Err(FormatError::InvalidBTreeV2Signature);
}
) -> Result<InternalNode<'_>, FormatError> {
let nr = num_records as usize;
let rs = record_size as usize;
let mut pos = offset + 6;
// Records first
let records_total = nr.checked_mul(rs).ok_or(FormatError::UnexpectedEof {
expected: usize::MAX,
available: file_data.len(),
available: len_usize(file),
})?;
ensure_len(file_data, pos, records_total)?;
let records_start = pos;
pos += records_total;
// Child pointer layout, as libhdf5 computes it (H5B2__hdr_init): the
// child's record count is always encoded in the width needed for a
@@ -344,13 +421,30 @@ fn read_internal_node(
let num_children = nr + 1;
let child_ptr_size = offset_size as usize + nrec_width + total_nrec_width;
ensure_len(file_data, pos, num_children * child_ptr_size)?;
let pointers = num_children * child_ptr_size;
// signature(4) + version(1) + type(1) = 6, records, pointers, checksum.
let w = read_node(
file,
offset,
6 + records_total + pointers + 4,
node_size,
b"BTIN",
&[(6, records_total), (6 + records_total, pointers)],
)?;
let d = &w.bytes;
let mut pos = 6;
w.ensure(pos, records_total)?;
let records_start = pos;
pos += records_total;
w.ensure(pos, pointers)?;
let mut children = Vec::with_capacity(num_children);
for _ in 0..num_children {
let addr = read_offset(file_data, pos, offset_size)?;
let addr = read_offset(d, pos, offset_size)?;
pos += offset_size as usize;
let child_nrec = read_var_uint(file_data, pos, nrec_width)? as u16;
let child_nrec = read_var_uint(d, pos, nrec_width)? as u16;
pos += nrec_width;
pos += total_nrec_width; // skip total records in subtree
children.push((addr, child_nrec));
@@ -362,9 +456,9 @@ fn read_internal_node(
// a mismatch here, and so does this.
#[cfg(feature = "checksum")]
{
ensure_len(file_data, pos, 4)?;
let stored = LittleEndian::read_u32(&file_data[pos..pos + 4]);
let computed = crate::checksum::jenkins_lookup3(&file_data[offset..pos]);
w.ensure(pos, 4)?;
let stored = LittleEndian::read_u32(&d[pos..pos + 4]);
let computed = crate::checksum::jenkins_lookup3(&d[..pos]);
if computed != stored {
return Err(FormatError::ChecksumMismatch {
expected: stored,
@@ -372,37 +466,17 @@ fn read_internal_node(
});
}
}
Ok((records_start, children))
}
/// Record `i` of an internal node whose records start at `records_start`.
fn internal_record(
file_data: &[u8],
records_start: usize,
i: usize,
rs: usize,
) -> Result<&[u8], FormatError> {
let overflow = || FormatError::UnexpectedEof {
expected: usize::MAX,
available: file_data.len(),
};
let rec_start = i
.checked_mul(rs)
.and_then(|o| records_start.checked_add(o))
.ok_or_else(overflow)?;
let rec_end = rec_start.checked_add(rs).ok_or_else(overflow)?;
file_data
.get(rec_start..rec_end)
.ok_or(FormatError::UnexpectedEof {
expected: rec_end,
available: file_data.len(),
})
Ok(InternalNode {
node: w,
records_start,
children,
})
}
/// Recursively collect records from an internal node.
#[allow(clippy::too_many_arguments, clippy::only_used_in_recursion)]
fn collect_internal_records(
file_data: &[u8],
fn collect_internal_records<S: Storage + ?Sized>(
file: &S,
offset: usize,
num_records: u16,
depth: u16,
@@ -416,8 +490,8 @@ fn collect_internal_records(
) -> Result<(), FormatError> {
let nr = num_records as usize;
let rs = record_size as usize;
let (records_start, children) = read_internal_node(
file_data,
let node = read_internal_node(
file,
offset,
num_records,
depth,
@@ -430,16 +504,21 @@ fn collect_internal_records(
// Interleave: child[0], record[0], child[1], record[1], ..., child[nr]
// We collect child[0] records, then record[0], then child[1], etc.
for (i, &(child_addr, child_nrec)) in children.iter().enumerate() {
for (i, &(child_addr, child_nrec)) in node.children.iter().enumerate() {
if child_depth == 0 {
// Before parsing, so a refused tree is not also a large allocation.
spend(budget, usize::from(child_nrec))?;
let leaf_recs =
parse_leaf_records(file_data, to_usize(child_addr)?, child_nrec, record_size)?;
let leaf_recs = parse_leaf_records(
file,
to_usize(child_addr)?,
child_nrec,
record_size,
node_size,
)?;
out.extend(leaf_recs);
} else {
collect_internal_records(
file_data,
file,
to_usize(child_addr)?,
child_nrec,
child_depth,
@@ -455,7 +534,7 @@ fn collect_internal_records(
// Add record[i] (except after the last child)
if i < nr {
let data = internal_record(file_data, records_start, i, rs)?;
let data = node.record(i, rs)?;
spend(budget, 1)?;
out.push(BTreeV2Record {
data: data.to_vec(),
@@ -481,6 +560,17 @@ pub fn find_btree_v2_records(
header: &BTreeV2Header,
offset_size: u8,
cmp: &mut dyn FnMut(&[u8]) -> Ordering,
) -> Result<Vec<BTreeV2Record>, FormatError> {
find_btree_v2_records_in(file_data, header, offset_size, cmp)
}
/// [`find_btree_v2_records`] over any [`Storage`]: one bounded read per
/// node visited.
pub fn find_btree_v2_records_in<S: Storage + ?Sized>(
file: &S,
header: &BTreeV2Header,
offset_size: u8,
cmp: &mut dyn FnMut(&[u8]) -> Ordering,
) -> Result<Vec<BTreeV2Record>, FormatError> {
if header.total_records == 0 || header.num_records_in_root == 0 {
return Ok(Vec::new());
@@ -490,11 +580,11 @@ pub fn find_btree_v2_records(
}
// As in `collect_btree_v2_records`: a valid tree cannot hold more
// records than the file has room for, however its children are shared.
let mut budget = file_data.len() / usize::from(header.record_size.max(1));
let mut budget = len_usize(file) / usize::from(header.record_size.max(1));
let max_leaf_nrec = max_records_leaf(header.node_size, header.record_size);
let mut out = Vec::new();
find_in_node(
file_data,
file,
header,
to_usize(header.root_node_address)?,
header.num_records_in_root,
@@ -509,8 +599,8 @@ pub fn find_btree_v2_records(
}
#[allow(clippy::too_many_arguments)]
fn find_in_node(
file_data: &[u8],
fn find_in_node<S: Storage + ?Sized>(
file: &S,
header: &BTreeV2Header,
offset: usize,
num_records: u16,
@@ -523,7 +613,13 @@ fn find_in_node(
) -> Result<(), FormatError> {
spend(budget, usize::from(num_records))?;
if depth == 0 {
let records = parse_leaf_records(file_data, offset, num_records, header.record_size)?;
let records = parse_leaf_records(
file,
offset,
num_records,
header.record_size,
header.node_size,
)?;
out.extend(
records
.into_iter()
@@ -532,8 +628,8 @@ fn find_in_node(
return Ok(());
}
let rs = usize::from(header.record_size);
let (records_start, children) = read_internal_node(
file_data,
let node = read_internal_node(
file,
offset,
num_records,
depth,
@@ -545,17 +641,17 @@ fn find_in_node(
let nr = usize::from(num_records);
let mut order = Vec::with_capacity(nr);
for i in 0..nr {
order.push(cmp(internal_record(file_data, records_start, i, rs)?));
order.push(cmp(node.record(i, rs)?));
}
// Child `i` holds the keys between record `i - 1` and record `i`: it can
// hold a match unless the record before it is already past the range or
// the record after it is still before it.
for (i, &(child_addr, child_nrec)) in children.iter().enumerate() {
for (i, &(child_addr, child_nrec)) in node.children.iter().enumerate() {
let after_left = i == 0 || order[i - 1] != Ordering::Greater;
let before_right = i == nr || order[i] != Ordering::Less;
if after_left && before_right {
find_in_node(
file_data,
file,
header,
to_usize(child_addr)?,
child_nrec,
@@ -569,7 +665,7 @@ fn find_in_node(
}
if i < nr && order[i] == Ordering::Equal {
out.push(BTreeV2Record {
data: internal_record(file_data, records_start, i, rs)?.to_vec(),
data: node.record(i, rs)?.to_vec(),
});
}
}