//! HDF5 B-tree v2 parsing. #[cfg(not(feature = "std"))] use alloc::vec::Vec; #[cfg(feature = "checksum")] use byteorder::{ByteOrder, LittleEndian}; use crate::error::FormatError; /// Parsed B-tree v2 header (signature "BTHD"). #[derive(Debug, Clone)] pub struct BTreeV2Header { /// B-tree type: 5=links indexed by name, 6=links indexed by creation order, etc. pub tree_type: u8, /// Node size in bytes. pub node_size: u32, /// Record size in bytes. pub record_size: u16, /// Depth of the tree (0 = root is a leaf). pub depth: u16, /// Address of root node. pub root_node_address: u64, /// Number of records in the root node. pub num_records_in_root: u16, /// Total number of records in all nodes. pub total_records: u64, } /// A single record from a B-tree v2 node. #[derive(Debug, Clone)] pub struct BTreeV2Record { /// Raw record bytes (record_size bytes). pub data: Vec, } fn read_offset(data: &[u8], pos: usize, size: u8) -> Result { let s = size as usize; if pos.checked_add(s).is_none_or(|end| end > data.len()) { return Err(FormatError::UnexpectedEof { expected: pos.saturating_add(s), available: data.len(), }); } Ok(match size { 2 => u16::from_le_bytes([data[pos], data[pos + 1]]) as u64, 4 => u32::from_le_bytes([data[pos], data[pos + 1], data[pos + 2], data[pos + 3]]) as u64, 8 => u64::from_le_bytes([ data[pos], data[pos + 1], data[pos + 2], data[pos + 3], data[pos + 4], data[pos + 5], data[pos + 6], data[pos + 7], ]), _ => return Err(FormatError::InvalidOffsetSize(size)), }) } fn ensure_len(data: &[u8], pos: usize, needed: usize) -> Result<(), FormatError> { match pos.checked_add(needed) { Some(end) if end <= data.len() => Ok(()), _ => Err(FormatError::UnexpectedEof { expected: pos.saturating_add(needed), available: data.len(), }), } } /// Compute the number of bytes needed to represent a count, using variable-width encoding. /// B-tree v2 uses this for the number of records fields in internal nodes. fn bytes_for_max_records(max_nrec: u64) -> usize { if max_nrec == 0 { return 1; } let bits = 64 - max_nrec.leading_zeros() as usize; bits.div_ceil(8) } /// Read a variable-width unsigned integer (1-8 bytes, LE). fn read_var_uint(data: &[u8], pos: usize, width: usize) -> Result { ensure_len(data, pos, width)?; let mut val = 0u64; for i in 0..width { val |= (data[pos + i] as u64) << (i * 8); } Ok(val) } impl BTreeV2Header { /// Parse a B-tree v2 header at the given offset. pub fn parse( file_data: &[u8], offset: usize, offset_size: u8, length_size: u8, ) -> Result { ensure_len(file_data, offset, 4)?; if &file_data[offset..offset + 4] != b"BTHD" { return Err(FormatError::InvalidBTreeV2Signature); } ensure_len(file_data, offset, 4 + 1 + 1 + 4 + 2 + 2 + 1 + 1)?; let version = file_data[offset + 4]; if version != 0 { return Err(FormatError::InvalidBTreeV2Version(version)); } let tree_type = file_data[offset + 5]; let node_size = u32::from_le_bytes([ file_data[offset + 6], file_data[offset + 7], file_data[offset + 8], file_data[offset + 9], ]); let record_size = u16::from_le_bytes([file_data[offset + 10], file_data[offset + 11]]); let depth = u16::from_le_bytes([file_data[offset + 12], file_data[offset + 13]]); let _split_percent = file_data[offset + 14]; let _merge_percent = file_data[offset + 15]; let mut pos = offset + 16; let root_node_address = read_offset(file_data, pos, offset_size)?; pos += offset_size as usize; ensure_len(file_data, pos, 2)?; let num_records_in_root = u16::from_le_bytes([file_data[pos], file_data[pos + 1]]); pos += 2; let total_records = read_offset(file_data, pos, length_size)?; #[allow(unused_assignments)] { pos += length_size as usize; } // Validate header checksum #[cfg(feature = "checksum")] { ensure_len(file_data, pos, 4)?; let stored = LittleEndian::read_u32(&file_data[pos..pos + 4]); let computed = crate::checksum::jenkins_lookup3(&file_data[offset..pos]); if computed != stored { return Err(FormatError::ChecksumMismatch { expected: stored, computed, }); } } Ok(BTreeV2Header { tree_type, node_size, record_size, depth, root_node_address, num_records_in_root, total_records, }) } } /// Compute maximum records per node for a given depth level. /// leaf: (node_size - overhead) / record_size /// internal: depends on pointers fn max_records_leaf(node_size: u32, record_size: u16) -> u64 { // Leaf overhead: signature(4) + version(1) + type(1) + checksum(4) = 10 let overhead = 10u32; if node_size <= overhead || record_size == 0 { return 0; } ((node_size - overhead) / record_size as u32) as u64 } /// Deepest B-tree v2 accepted. See [`collect_btree_v2_records`]. const MAX_DEPTH: u16 = 64; /// Take `n` records from the traversal's budget, or refuse the tree. fn spend(budget: &mut usize, n: usize) -> Result<(), FormatError> { *budget = budget .checked_sub(n) .ok_or(FormatError::NestingDepthExceeded)?; Ok(()) } /// Collect all records from a B-tree v2 by traversing from the root. pub fn collect_btree_v2_records( file_data: &[u8], header: &BTreeV2Header, offset_size: u8, length_size: u8, ) -> Result, FormatError> { if header.total_records == 0 || header.num_records_in_root == 0 { return Ok(Vec::new()); } // Recursion is one frame per level, and the depth is read from the file: // a crafted header claiming 65 535 levels over a node that is its own // child overflowed the stack. 64 matches the fractal heap's guard, and no // real tree comes close — even at the minimum fan-out of two it would // hold more than 2^64 records. if header.depth > MAX_DEPTH { return Err(FormatError::NestingDepthExceeded); } // A valid tree stores each record once, in its own bytes, so it cannot // hold more records than the file has room for. Children are addresses, // though, and nothing makes them distinct: levels whose children all // point at one shared node below reach it fan-out^depth times, which is // millions of records from a few kilobytes. Counting against what the // file could physically contain bounds that without trusting the // header's own `total_records`. let mut budget = file_data.len() / usize::from(header.record_size.max(1)); let max_leaf_nrec = max_records_leaf(header.node_size, header.record_size); if header.depth == 0 { // Root is a leaf parse_leaf_records( file_data, header.root_node_address as usize, header.num_records_in_root, header.record_size, ) } else { // Root is internal; traverse recursively let mut records = Vec::new(); collect_internal_records( file_data, header.root_node_address as usize, header.num_records_in_root, header.depth, header.record_size, header.node_size, offset_size, length_size, max_leaf_nrec, &mut budget, &mut records, )?; Ok(records) } } /// Parse records from a leaf node (signature "BTLF"). fn parse_leaf_records( file_data: &[u8], offset: usize, num_records: u16, record_size: u16, ) -> Result, FormatError> { // signature(4) + version(1) + type(1) = 6 bytes header ensure_len(file_data, offset, 6)?; if &file_data[offset..offset + 4] != b"BTLF" { return Err(FormatError::InvalidBTreeV2Signature); } let pos = offset + 6; let rs = record_size as usize; let total = (num_records as usize) .checked_mul(rs) .ok_or(FormatError::UnexpectedEof { expected: usize::MAX, available: file_data.len(), })?; ensure_len(file_data, pos, total)?; // Validate checksum: 4 bytes after records + padding #[cfg(feature = "checksum")] { let checksum_pos = pos + total; if file_data.len() >= checksum_pos + 4 { let stored = LittleEndian::read_u32(&file_data[checksum_pos..checksum_pos + 4]); let computed = crate::checksum::jenkins_lookup3(&file_data[offset..checksum_pos]); if computed != stored { return Err(FormatError::ChecksumMismatch { expected: stored, computed, }); } } } let mut records = Vec::with_capacity(num_records as usize); for i in 0..num_records as usize { let start = pos + i * rs; records.push(BTreeV2Record { data: file_data[start..start + rs].to_vec(), }); } Ok(records) } /// Recursively collect records from an internal node. #[allow(clippy::too_many_arguments, clippy::only_used_in_recursion)] fn collect_internal_records( file_data: &[u8], offset: usize, num_records: u16, depth: u16, record_size: u16, node_size: u32, offset_size: u8, length_size: u8, max_leaf_nrec: u64, budget: &mut usize, out: &mut Vec, ) -> Result<(), FormatError> { // signature(4) + version(1) + type(1) = 6 ensure_len(file_data, offset, 6)?; if &file_data[offset..offset + 4] != b"BTIN" { return Err(FormatError::InvalidBTreeV2Signature); } let nr = num_records as usize; let rs = record_size as usize; let mut pos = offset + 6; // Read all records first let records_total = nr.checked_mul(rs).ok_or(FormatError::UnexpectedEof { expected: usize::MAX, available: file_data.len(), })?; ensure_len(file_data, pos, records_total)?; let records_start = pos; pos += records_total; // Compute sizes for child pointers // max_records at child depth - for variable-width nrec encoding let child_depth = depth - 1; let max_nrec_child = if child_depth == 0 { max_leaf_nrec } else { // For internal nodes at child_depth, the true max_nrec depends on the // node size, record size, and the recursive width of child pointer // entries (which themselves depend on max_nrec at deeper levels). // Computing the exact value requires iterating from the leaf level // upward, as described in the HDF5 spec (III.A.2 "Computing the Size // of B-tree Nodes"). // // We use `max_leaf_nrec * 2` as a conservative upper bound. This // over-estimates the nrec encoding width, which means we may read // slightly more bytes per child pointer than strictly necessary, but // never fewer. The over-read bytes are harmless because we only // decode `num_records` entries (the actual count from the node header). // // Known limitation: for very deep trees (depth > 3) with small record // sizes, the true max could exceed this estimate, causing us to // under-allocate the nrec encoding width and misparse child pointers. // In practice, HDF5 B-tree v2 depths rarely exceed 2-3. max_leaf_nrec * 2 }; let nrec_width = bytes_for_max_records(max_nrec_child); // Total records in subtree width (only if depth > 1) let total_nrec_width = if depth > 1 { // Width to hold total records in a subtree // We compute max possible total records at this subtree depth let max_total = header_max_total_records(max_leaf_nrec, depth - 1); bytes_for_max_records(max_total) } else { 0 }; let num_children = nr + 1; let child_ptr_size = offset_size as usize + nrec_width + total_nrec_width; ensure_len(file_data, pos, num_children * child_ptr_size)?; // Read child pointers let mut children = Vec::with_capacity(num_children); for _ in 0..num_children { let addr = read_offset(file_data, pos, offset_size)?; pos += offset_size as usize; let child_nrec = read_var_uint(file_data, pos, nrec_width)? as u16; pos += nrec_width; pos += total_nrec_width; // skip total records in subtree children.push((addr, child_nrec)); } // Interleave: child[0], record[0], child[1], record[1], ..., child[nr] // We collect child[0] records, then record[0], then child[1], etc. for (i, &(child_addr, child_nrec)) in children.iter().enumerate() { if child_depth == 0 { // Before parsing, so a refused tree is not also a large allocation. spend(budget, usize::from(child_nrec))?; let leaf_recs = parse_leaf_records(file_data, child_addr as usize, child_nrec, record_size)?; out.extend(leaf_recs); } else { collect_internal_records( file_data, child_addr as usize, child_nrec, child_depth, record_size, node_size, offset_size, length_size, max_leaf_nrec, budget, out, )?; } // Add record[i] (except after the last child) if i < nr { let rec_offset = i.checked_mul(rs).ok_or(FormatError::UnexpectedEof { expected: usize::MAX, available: file_data.len(), })?; let rec_start = records_start .checked_add(rec_offset) .ok_or(FormatError::UnexpectedEof { expected: usize::MAX, available: file_data.len(), })?; let rec_end = rec_start .checked_add(rs) .ok_or(FormatError::UnexpectedEof { expected: usize::MAX, available: file_data.len(), })?; if rec_end > file_data.len() { return Err(FormatError::UnexpectedEof { expected: rec_end, available: file_data.len(), }); } spend(budget, 1)?; out.push(BTreeV2Record { data: file_data[rec_start..rec_end].to_vec(), }); } } Ok(()) } /// Estimate maximum total records at a given depth (for variable-width encoding). fn header_max_total_records(max_leaf_nrec: u64, depth: u16) -> u64 { // Conservative: branching factor * max_leaf at each level let mut total = max_leaf_nrec; for _ in 0..depth { total = total.saturating_mul(max_leaf_nrec.max(2)); } total } #[cfg(test)] mod tests { use super::*; #[allow(clippy::too_many_arguments)] fn build_btree_v2_header( tree_type: u8, node_size: u32, record_size: u16, depth: u16, root_addr: u64, num_records_root: u16, total_records: u64, offset_size: u8, length_size: u8, ) -> Vec { let mut buf = Vec::new(); buf.extend_from_slice(b"BTHD"); buf.push(0); // version buf.push(tree_type); buf.extend_from_slice(&node_size.to_le_bytes()); buf.extend_from_slice(&record_size.to_le_bytes()); buf.extend_from_slice(&depth.to_le_bytes()); buf.push(85); // split_percent buf.push(40); // merge_percent match offset_size { 4 => buf.extend_from_slice(&(root_addr as u32).to_le_bytes()), 8 => buf.extend_from_slice(&root_addr.to_le_bytes()), _ => {} } buf.extend_from_slice(&num_records_root.to_le_bytes()); match length_size { 4 => buf.extend_from_slice(&(total_records as u32).to_le_bytes()), 8 => buf.extend_from_slice(&total_records.to_le_bytes()), _ => {} } let checksum = crate::checksum::jenkins_lookup3(&buf); buf.extend_from_slice(&checksum.to_le_bytes()); buf } fn build_leaf_node(tree_type: u8, records: &[&[u8]]) -> Vec { let mut buf = Vec::new(); buf.extend_from_slice(b"BTLF"); buf.push(0); // version buf.push(tree_type); for rec in records { buf.extend_from_slice(rec); } let checksum = crate::checksum::jenkins_lookup3(&buf); buf.extend_from_slice(&checksum.to_le_bytes()); buf } /// An internal node laid out exactly as `collect_internal_records` will /// read it at `depth`: `records` zeroed records, then `children` pointers, /// all to `child_addr` claiming `child_nrec` records. fn internal_node( depth: u16, node_size: u32, record_size: u16, records: usize, children: usize, child_addr: u64, child_nrec: u64, ) -> Vec { let max_leaf = max_records_leaf(node_size, record_size); let nrec_width = bytes_for_max_records(if depth == 1 { max_leaf } else { max_leaf * 2 }); let total_width = if depth > 1 { bytes_for_max_records(header_max_total_records(max_leaf, depth - 1)) } else { 0 }; let mut buf = b"BTIN".to_vec(); buf.extend_from_slice(&[0, 5]); buf.resize(buf.len() + records * record_size as usize, 0); for _ in 0..children { buf.extend_from_slice(&child_addr.to_le_bytes()); buf.extend_from_slice(&child_nrec.to_le_bytes()[..nrec_width]); buf.resize(buf.len() + total_width, 0); } buf } fn header(depth: u16, root: u64, root_nrec: u16, total: u64) -> BTreeV2Header { BTreeV2Header { tree_type: 5, node_size: 512, record_size: 8, depth, root_node_address: root, num_records_in_root: root_nrec, total_records: total, } } #[test] fn a_node_that_is_its_own_child_is_rejected_not_recursed() { // One internal node whose two children are itself, under a header // claiming the deepest tree a u16 allows. The layout stops depending // on depth once the subtree-total width saturates, so every level // parses cleanly and recursion runs ~65 000 frames deep: before the // cap this overflowed the stack and aborted the process, from a file // of under 100 bytes. let mut data = internal_node(u16::MAX, 512, 8, 1, 2, 0, 1); data.resize(4096, 0); let result = collect_btree_v2_records(&data, &header(u16::MAX, 0, 1, 1), 8, 8); assert!(result.is_err(), "{result:?}"); } #[test] fn a_shared_subtree_cannot_multiply_the_work() { // A chain of distinct levels, each node's children all pointing at the // single node below, ending in a real leaf. Every node parses and // nothing is cyclic, yet the leaf is reached fan-out^depth times: 62 // children over 4 levels is ~15 million leaf visits from a few // kilobytes. A valid tree cannot hold more records than the file has // room for, so that bounds the traversal instead. let (node_size, record_size) = (512u32, 8u16); let fanout = 62usize; let depth = 4u16; let leaf = build_leaf_node(5, &[&[0u8; 8][..]]); // Lay out root first, then each lower level, then the leaf. let mut nodes: Vec> = Vec::new(); let mut addrs = Vec::new(); let mut at = 0u64; let mut sizes = Vec::new(); for d in (1..=depth).rev() { let n = internal_node(d, node_size, record_size, fanout - 1, fanout, 0, 0); sizes.push(n.len()); } for size in &sizes { addrs.push(at); at += *size as u64; } let leaf_addr = at; for (i, d) in (1..=depth).rev().enumerate() { let (child, child_nrec) = if d == 1 { (leaf_addr, 1) } else { (addrs[i + 1], fanout as u64 - 1) }; nodes.push(internal_node( d, node_size, record_size, fanout - 1, fanout, child, child_nrec, )); } let mut data: Vec = nodes.concat(); data.extend_from_slice(&leaf); data.resize(data.len() + 64, 0); let started = std::time::Instant::now(); let result = collect_btree_v2_records(&data, &header(depth, 0, fanout as u16 - 1, u64::MAX), 8, 8); assert!( result.is_err(), "expected a refusal, got {} records", result.map_or(0, |r| r.len()) ); assert!( started.elapsed() < std::time::Duration::from_secs(2), "took {:?}", started.elapsed() ); } #[test] fn parse_header() { let data = build_btree_v2_header(5, 512, 11, 0, 0x1000, 3, 3, 8, 8); let hdr = BTreeV2Header::parse(&data, 0, 8, 8).unwrap(); assert_eq!(hdr.tree_type, 5); assert_eq!(hdr.node_size, 512); assert_eq!(hdr.record_size, 11); assert_eq!(hdr.depth, 0); assert_eq!(hdr.root_node_address, 0x1000); assert_eq!(hdr.num_records_in_root, 3); assert_eq!(hdr.total_records, 3); } #[test] fn parse_leaf_with_2_records() { let rec1 = [1u8, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11]; let rec2 = [11u8, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21]; let leaf = build_leaf_node(5, &[&rec1, &rec2]); let leaf_offset = 256usize; let header = build_btree_v2_header(5, 512, 11, 0, leaf_offset as u64, 2, 2, 8, 8); let mut file_data = vec![0u8; 512]; file_data[..header.len()].copy_from_slice(&header); file_data[leaf_offset..leaf_offset + leaf.len()].copy_from_slice(&leaf); let hdr = BTreeV2Header::parse(&file_data, 0, 8, 8).unwrap(); let records = collect_btree_v2_records(&file_data, &hdr, 8, 8).unwrap(); assert_eq!(records.len(), 2); assert_eq!(records[0].data, rec1.to_vec()); assert_eq!(records[1].data, rec2.to_vec()); } #[test] fn invalid_signature() { let mut data = build_btree_v2_header(5, 512, 11, 0, 0, 0, 0, 8, 8); data[0] = b'X'; let err = BTreeV2Header::parse(&data, 0, 8, 8).unwrap_err(); assert_eq!(err, FormatError::InvalidBTreeV2Signature); } #[test] fn invalid_version() { let mut data = build_btree_v2_header(5, 512, 11, 0, 0, 0, 0, 8, 8); data[4] = 1; // bad version let err = BTreeV2Header::parse(&data, 0, 8, 8).unwrap_err(); assert_eq!(err, FormatError::InvalidBTreeV2Version(1)); } #[test] fn empty_tree() { let header = build_btree_v2_header(5, 512, 11, 0, 0, 0, 0, 8, 8); let hdr = BTreeV2Header::parse(&header, 0, 8, 8).unwrap(); let records = collect_btree_v2_records(&header, &hdr, 8, 8).unwrap(); assert!(records.is_empty()); } }