CI / test (push) Failing after 14s
- Fix version skew: clawhdf5-py (pyproject.toml 1.93.0 -> 2.1.0) and packages/clawhdf5-node (package.json 2.0.0 -> 2.1.0) were both behind the actual crate version. - Correct stale ROADMAP.md claims: the TypeScript bridge already has a complete napi-rs package (not "no package.json"); CI/CD is now wired up via .gitea/workflows/ci.yml. - Fix CLAUDE.md: clawhdf5-gpu uses wgpu with hand-written WGSL compute shaders, not CubeCL. - chunked_read.rs: drop 12 unnecessary chunk_dimensions[..rank].to_vec() allocations — all three callees already accept &[u32]. - btree_v1.rs: add an overflow-safe ensure_len(data, offset, needed) helper (checked_add) and use it at the two plain-arithmetic bounds guards, closing a usize-overflow edge case reachable from a crafted near-usize::MAX B-tree offset. Add a regression test. - Clarify that the integrity hashes in clawhdf5-agent/provenance.rs (FNV-1a) and clawhdf5-format/provenance.rs (SHA-256) are unkeyed and only detect accidental corruption, not tampering — doc-only change. - README.md: document that the mpi-io feature's read/write paths are root-read+broadcast / gather-to-rank-0, not true collective I/O.
321 lines
10 KiB
Rust
321 lines
10 KiB
Rust
//! HDF5 B-tree v1 parsing (type 0 for groups).
|
|
|
|
#[cfg(not(feature = "std"))]
|
|
use alloc::vec::Vec;
|
|
|
|
use crate::error::FormatError;
|
|
|
|
/// A parsed B-tree v1 node.
|
|
#[derive(Debug, Clone)]
|
|
pub struct BTreeV1Node {
|
|
/// Node type: 0=group, 1=raw data chunks.
|
|
pub node_type: u8,
|
|
/// Node level: 0=leaf, >0=internal.
|
|
pub node_level: u8,
|
|
/// Number of entries used.
|
|
pub entries_used: u16,
|
|
/// Left sibling address, or None if undefined.
|
|
pub left_sibling: Option<u64>,
|
|
/// Right sibling address, or None if undefined.
|
|
pub right_sibling: Option<u64>,
|
|
/// Keys (entries_used + 1 values).
|
|
pub keys: Vec<u64>,
|
|
/// Child addresses (entries_used values).
|
|
pub children: Vec<u64>,
|
|
}
|
|
|
|
/// Checks that `[offset, offset + needed)` fits within `data`, guarding the
|
|
/// addition against `usize` overflow from a crafted near-`usize::MAX` offset.
|
|
fn ensure_len(data: &[u8], offset: usize, needed: usize) -> Result<(), FormatError> {
|
|
if offset
|
|
.checked_add(needed)
|
|
.is_none_or(|end| end > data.len())
|
|
{
|
|
return Err(FormatError::UnexpectedEof {
|
|
expected: offset.saturating_add(needed),
|
|
available: data.len(),
|
|
});
|
|
}
|
|
Ok(())
|
|
}
|
|
|
|
fn read_offset(data: &[u8], pos: usize, size: u8) -> Result<u64, FormatError> {
|
|
let s = size as usize;
|
|
if pos.checked_add(s).is_none_or(|end| end > data.len()) {
|
|
return Err(FormatError::UnexpectedEof {
|
|
expected: pos.saturating_add(s),
|
|
available: data.len(),
|
|
});
|
|
}
|
|
let slice = &data[pos..pos + s];
|
|
Ok(match size {
|
|
2 => u16::from_le_bytes([slice[0], slice[1]]) as u64,
|
|
4 => u32::from_le_bytes([slice[0], slice[1], slice[2], slice[3]]) as u64,
|
|
8 => u64::from_le_bytes([
|
|
slice[0], slice[1], slice[2], slice[3], slice[4], slice[5], slice[6], slice[7],
|
|
]),
|
|
_ => return Err(FormatError::InvalidOffsetSize(size)),
|
|
})
|
|
}
|
|
|
|
fn is_undefined(data: &[u8], pos: usize, size: u8) -> bool {
|
|
let s = size as usize;
|
|
if ensure_len(data, pos, s).is_err() {
|
|
return false;
|
|
}
|
|
data[pos..pos + s].iter().all(|&b| b == 0xFF)
|
|
}
|
|
|
|
impl BTreeV1Node {
|
|
/// Parse a B-tree v1 node at the given offset in the file data.
|
|
///
|
|
/// For type 0 (group) nodes, keys are offset_size bytes each (heap name offsets).
|
|
pub fn parse(
|
|
file_data: &[u8],
|
|
offset: usize,
|
|
offset_size: u8,
|
|
_length_size: u8,
|
|
) -> Result<BTreeV1Node, FormatError> {
|
|
// signature(4) + node_type(1) + node_level(1) + entries_used(2) = 8
|
|
// + left_sibling(offset_size) + right_sibling(offset_size)
|
|
let os = offset_size as usize;
|
|
let header_size = 8 + os * 2;
|
|
ensure_len(file_data, offset, header_size)?;
|
|
|
|
if &file_data[offset..offset + 4] != b"TREE" {
|
|
return Err(FormatError::InvalidBTreeSignature);
|
|
}
|
|
|
|
let node_type = file_data[offset + 4];
|
|
let node_level = file_data[offset + 5];
|
|
let entries_used = u16::from_le_bytes([file_data[offset + 6], file_data[offset + 7]]);
|
|
|
|
let mut pos = offset + 8;
|
|
let left_sibling = if is_undefined(file_data, pos, offset_size) {
|
|
None
|
|
} else {
|
|
Some(read_offset(file_data, pos, offset_size)?)
|
|
};
|
|
pos += os;
|
|
let right_sibling = if is_undefined(file_data, pos, offset_size) {
|
|
None
|
|
} else {
|
|
Some(read_offset(file_data, pos, offset_size)?)
|
|
};
|
|
pos += os;
|
|
|
|
// For type 0: keys are offset_size bytes, children are offset_size bytes
|
|
// Layout: key[0], child[0], key[1], child[1], ..., key[N-1], child[N-1], key[N]
|
|
let eu = entries_used as usize;
|
|
let key_size = os; // For type 0, key = offset_size
|
|
let needed = eu * (key_size + os) + key_size; // eu children + (eu+1) keys
|
|
ensure_len(file_data, pos, needed)?;
|
|
|
|
let mut keys = Vec::with_capacity(eu + 1);
|
|
let mut children = Vec::with_capacity(eu);
|
|
|
|
for _i in 0..eu {
|
|
// key[i]
|
|
let key = read_offset(file_data, pos, offset_size)?;
|
|
keys.push(key);
|
|
pos += key_size;
|
|
// child[i]
|
|
let child = read_offset(file_data, pos, offset_size)?;
|
|
children.push(child);
|
|
pos += os;
|
|
}
|
|
// final key
|
|
let key = read_offset(file_data, pos, offset_size)?;
|
|
keys.push(key);
|
|
|
|
Ok(BTreeV1Node {
|
|
node_type,
|
|
node_level,
|
|
entries_used,
|
|
left_sibling,
|
|
right_sibling,
|
|
keys,
|
|
children,
|
|
})
|
|
}
|
|
}
|
|
|
|
/// Maximum recursion depth for B-tree traversal (malformed data protection).
|
|
const MAX_BTREE_DEPTH: usize = 64;
|
|
|
|
/// Collect all leaf-level child addresses (SNOD addresses) by traversing the B-tree.
|
|
pub fn collect_symbol_table_nodes(
|
|
file_data: &[u8],
|
|
btree_address: u64,
|
|
offset_size: u8,
|
|
length_size: u8,
|
|
) -> Result<Vec<u64>, FormatError> {
|
|
collect_symbol_table_nodes_inner(file_data, btree_address, offset_size, length_size, 0)
|
|
}
|
|
|
|
fn collect_symbol_table_nodes_inner(
|
|
file_data: &[u8],
|
|
btree_address: u64,
|
|
offset_size: u8,
|
|
length_size: u8,
|
|
depth: usize,
|
|
) -> Result<Vec<u64>, FormatError> {
|
|
if depth > MAX_BTREE_DEPTH {
|
|
return Err(FormatError::NestingDepthExceeded);
|
|
}
|
|
|
|
let node = BTreeV1Node::parse(file_data, btree_address as usize, offset_size, length_size)?;
|
|
|
|
if node.node_type != 0 {
|
|
return Err(FormatError::InvalidBTreeNodeType(node.node_type));
|
|
}
|
|
|
|
if node.node_level == 0 {
|
|
// Leaf: children are SNOD addresses
|
|
Ok(node.children)
|
|
} else {
|
|
// Internal: recurse into children
|
|
let mut result = Vec::new();
|
|
for &child_addr in &node.children {
|
|
let child_snods = collect_symbol_table_nodes_inner(
|
|
file_data,
|
|
child_addr,
|
|
offset_size,
|
|
length_size,
|
|
depth + 1,
|
|
)?;
|
|
result.extend(child_snods);
|
|
}
|
|
Ok(result)
|
|
}
|
|
}
|
|
|
|
#[cfg(test)]
|
|
mod tests {
|
|
use super::*;
|
|
|
|
fn write_offset(buf: &mut Vec<u8>, val: u64, size: u8) {
|
|
match size {
|
|
4 => buf.extend_from_slice(&(val as u32).to_le_bytes()),
|
|
8 => buf.extend_from_slice(&val.to_le_bytes()),
|
|
_ => panic!("test"),
|
|
}
|
|
}
|
|
|
|
fn build_btree_node(
|
|
node_type: u8,
|
|
level: u8,
|
|
keys: &[u64],
|
|
children: &[u64],
|
|
left: Option<u64>,
|
|
right: Option<u64>,
|
|
offset_size: u8,
|
|
) -> Vec<u8> {
|
|
assert_eq!(keys.len(), children.len() + 1);
|
|
let entries_used = children.len() as u16;
|
|
let mut buf = Vec::new();
|
|
buf.extend_from_slice(b"TREE");
|
|
buf.push(node_type);
|
|
buf.push(level);
|
|
buf.extend_from_slice(&entries_used.to_le_bytes());
|
|
let undef: u64 = if offset_size == 4 {
|
|
0xFFFFFFFF
|
|
} else {
|
|
0xFFFFFFFFFFFFFFFF
|
|
};
|
|
write_offset(&mut buf, left.unwrap_or(undef), offset_size);
|
|
write_offset(&mut buf, right.unwrap_or(undef), offset_size);
|
|
for i in 0..children.len() {
|
|
write_offset(&mut buf, keys[i], offset_size);
|
|
write_offset(&mut buf, children[i], offset_size);
|
|
}
|
|
write_offset(&mut buf, *keys.last().unwrap(), offset_size);
|
|
buf
|
|
}
|
|
|
|
#[test]
|
|
fn parse_leaf_node() {
|
|
let data = build_btree_node(0, 0, &[0, 5, 10], &[0x100, 0x200], None, None, 8);
|
|
let node = BTreeV1Node::parse(&data, 0, 8, 8).unwrap();
|
|
assert_eq!(node.node_type, 0);
|
|
assert_eq!(node.node_level, 0);
|
|
assert_eq!(node.entries_used, 2);
|
|
assert_eq!(node.keys, vec![0, 5, 10]);
|
|
assert_eq!(node.children, vec![0x100, 0x200]);
|
|
assert_eq!(node.left_sibling, None);
|
|
assert_eq!(node.right_sibling, None);
|
|
}
|
|
|
|
#[test]
|
|
fn parse_near_usize_max_offset_rejected_without_overflow() {
|
|
let data = build_btree_node(0, 0, &[0, 5, 10], &[0x100, 0x200], None, None, 8);
|
|
let result = BTreeV1Node::parse(&data, usize::MAX - 4, 8, 8);
|
|
assert!(
|
|
matches!(result, Err(FormatError::UnexpectedEof { .. })),
|
|
"expected a clean UnexpectedEof, got {result:?}"
|
|
);
|
|
}
|
|
|
|
#[test]
|
|
fn parse_with_siblings_none() {
|
|
let data = build_btree_node(0, 0, &[0, 8], &[0x300], None, None, 8);
|
|
let node = BTreeV1Node::parse(&data, 0, 8, 8).unwrap();
|
|
assert_eq!(node.left_sibling, None);
|
|
assert_eq!(node.right_sibling, None);
|
|
}
|
|
|
|
#[test]
|
|
fn parse_internal_node_and_collect() {
|
|
// Build a 2-level tree: one internal node pointing to two leaf nodes
|
|
let os: u8 = 8;
|
|
let leaf1_offset: usize = 0;
|
|
let leaf2_offset: usize = 256;
|
|
let internal_offset: usize = 512;
|
|
|
|
let leaf1 = build_btree_node(0, 0, &[0, 5], &[0xA00], None, None, os);
|
|
let leaf2 = build_btree_node(0, 0, &[5, 10], &[0xB00], None, None, os);
|
|
let internal = build_btree_node(
|
|
0,
|
|
1,
|
|
&[0, 5, 10],
|
|
&[leaf1_offset as u64, leaf2_offset as u64],
|
|
None,
|
|
None,
|
|
os,
|
|
);
|
|
|
|
let mut file = vec![0u8; 1024];
|
|
file[leaf1_offset..leaf1_offset + leaf1.len()].copy_from_slice(&leaf1);
|
|
file[leaf2_offset..leaf2_offset + leaf2.len()].copy_from_slice(&leaf2);
|
|
file[internal_offset..internal_offset + internal.len()].copy_from_slice(&internal);
|
|
|
|
let snods = collect_symbol_table_nodes(&file, internal_offset as u64, os, os).unwrap();
|
|
assert_eq!(snods, vec![0xA00, 0xB00]);
|
|
}
|
|
|
|
#[test]
|
|
fn invalid_signature() {
|
|
let mut data = build_btree_node(0, 0, &[0, 1], &[0x100], None, None, 8);
|
|
data[0] = b'X';
|
|
let err = BTreeV1Node::parse(&data, 0, 8, 8).unwrap_err();
|
|
assert_eq!(err, FormatError::InvalidBTreeSignature);
|
|
}
|
|
|
|
#[test]
|
|
fn collect_wrong_node_type() {
|
|
let data = build_btree_node(1, 0, &[0, 1], &[0x100], None, None, 8);
|
|
let mut file = vec![0u8; 512];
|
|
file[..data.len()].copy_from_slice(&data);
|
|
let err = collect_symbol_table_nodes(&file, 0, 8, 8).unwrap_err();
|
|
assert_eq!(err, FormatError::InvalidBTreeNodeType(1));
|
|
}
|
|
|
|
#[test]
|
|
fn parse_4byte_offsets() {
|
|
let data = build_btree_node(0, 0, &[0, 4], &[0x50], None, None, 4);
|
|
let node = BTreeV1Node::parse(&data, 0, 4, 4).unwrap();
|
|
assert_eq!(node.entries_used, 1);
|
|
assert_eq!(node.children, vec![0x50]);
|
|
}
|
|
}
|