quantumclaw/clawhdf5
3
1
Fork 0
Code Issues Pull Requests 3 Actions Packages Projects Releases 1 Wiki Activity

format: v2 B-trees, dense groups and group listings over Storage

Browse Source 
BTreeV2Header::parse_in, collect_btree_v2_records_in and
find_btree_v2_records_in read one bounded window per node (its size is
known from the parent before the node is read; a count stretched past
node_size is checked against the end of the file first), with the
whole-file bounds errors unchanged. With them, dense attributes, a SOHM
B-tree index and huge fractal-heap objects no longer answer
ContiguousStorageRequired, and group_v1/group_v2 listings, lookups and
path resolution get *_in cores (resolve_group_children_in,
resolve_child_in, resolve_path_any_in, ...). The &[u8] functions are
thin wrappers, as in M1.

The equivalence harness now fails on any ContiguousStorageRequired and
compares v2 B-tree headers, records and descents, group listings, child
lookups and paths; a unit test compares a two-level tree through a
read_at-only storage truncated at every length and with every node byte
flipped.

Co-Authored-By: Claude Opus 5.5 (1M context) <[email protected]>
This commit is contained in:
osobh
2026-09-26 16:13:01 -05:00
co-authored by Claude Opus 5.5
parent 8f59b2e1c2
commit 42894bf93b
9 changed files with 544 additions and 297 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/clawhdf5-format/src/group_v2.rs
+99 -39
View File
@@ -11,8 +11,8 @@ use alloc::collections::BTreeSet;
#[cfg(feature = "std")]
use std::collections::BTreeSet;
use crate::addr::to_usize;
use crate::btree_v2::{BTreeV2Header, collect_btree_v2_records, find_btree_v2_records};
use crate::addr::checked_addr;
use crate::btree_v2::{BTreeV2Header, collect_btree_v2_records_in, find_btree_v2_records_in};
use crate::checksum::jenkins_lookup3;
use crate::error::FormatError;
use crate::fractal_heap::FractalHeapHeader;
@@ -21,6 +21,7 @@ use crate::link_info::LinkInfoMessage;
use crate::link_message::{LinkMessage, LinkTarget};
use crate::message_type::MessageType;
use crate::object_header::ObjectHeader;
use crate::storage::Storage;
use crate::superblock::Superblock;
use crate::symbol_table::SymbolTableMessage;
@@ -32,6 +33,16 @@ pub fn resolve_v2_group_entries(
object_header: &ObjectHeader,
offset_size: u8,
length_size: u8,
) -> Result<Vec<GroupEntry>, FormatError> {
resolve_v2_group_entries_in(file_data, object_header, offset_size, length_size)
}
/// [`resolve_v2_group_entries`] over any [`Storage`].
pub fn resolve_v2_group_entries_in<S: Storage + ?Sized>(
file_data: &S,
object_header: &ObjectHeader,
offset_size: u8,
length_size: u8,
) -> Result<Vec<GroupEntry>, FormatError> {
// Look for Link Info message to determine storage type
let link_info = find_link_info(object_header, offset_size)?;
@@ -91,8 +102,8 @@ fn resolve_compact_entries(
}
/// Visit every link in dense storage (fractal heap + B-tree v2 name index).
fn for_each_dense_link(
file_data: &[u8],
fn for_each_dense_link<S: Storage + ?Sized>(
file_data: &S,
link_info: &LinkInfoMessage,
fh_addr: u64,
offset_size: u8,
@@ -100,15 +111,20 @@ fn for_each_dense_link(
mut visit: impl FnMut(LinkMessage),
) -> Result<(), FormatError> {
// Parse fractal heap
let fh = FractalHeapHeader::parse(file_data, to_usize(fh_addr)?, offset_size, length_size)?;
let fh =
FractalHeapHeader::parse_in(file_data, checked_addr(fh_addr)?, offset_size, length_size)?;
// Parse B-tree v2 for name index
let btree_addr = link_info
.btree_name_index_address
.ok_or_else(|| FormatError::PathNotFound(String::from("no B-tree v2 name index")))?;
let btree_hdr =
BTreeV2Header::parse(file_data, to_usize(btree_addr)?, offset_size, length_size)?;
let records = collect_btree_v2_records(file_data, &btree_hdr, offset_size, length_size)?;
let btree_hdr = BTreeV2Header::parse_in(
file_data,
checked_addr(btree_addr)?,
offset_size,
length_size,
)?;
let records = collect_btree_v2_records_in(file_data, &btree_hdr, offset_size, length_size)?;
for record in &records {
// For type 5 (name index): hash(4) + heap_id(heap_id_length)
@@ -125,7 +141,7 @@ fn for_each_dense_link(
let id_bytes = &record.data[id_offset..id_offset + fh.heap_id_length as usize];
// Read managed object from fractal heap
let link_data = fh.read_managed_object(file_data, id_bytes, offset_size)?;
let link_data = fh.read_managed_object_in(file_data, id_bytes, offset_size)?;
if let Some(link) = parse_link(&link_data, offset_size)? {
visit(link);
}
@@ -134,8 +150,8 @@ fn for_each_dense_link(
}
/// Resolve entries from dense storage (fractal heap + B-tree v2).
fn resolve_dense_entries(
file_data: &[u8],
fn resolve_dense_entries<S: Storage + ?Sized>(
file_data: &S,
link_info: &LinkInfoMessage,
fh_addr: u64,
offset_size: u8,
@@ -167,8 +183,8 @@ fn resolve_dense_entries(
/// The soft link called `name` in a v1 (symbol table) group, if there is
/// one. Hard links are what `resolve_group_entries` returns; this is
/// consulted only when a path component isn't among them.
fn find_v1_symbolic_link(
file_data: &[u8],
fn find_v1_symbolic_link<S: Storage + ?Sized>(
file_data: &S,
object_header: &ObjectHeader,
name: &str,
offset_size: u8,
@@ -182,7 +198,7 @@ fn find_v1_symbolic_link(
return Ok(None);
};
let stm = SymbolTableMessage::parse(&sym_msg.data, offset_size)?;
group_v1::find_v1_soft_link(file_data, &stm, name, offset_size, length_size)
group_v1::find_v1_soft_link_in(file_data, &stm, name, offset_size, length_size)
.map(|target| target.map(|target_path| LinkTarget::Soft { target_path }))
}
@@ -199,8 +215,8 @@ const LINK_NAME_INDEX: u8 = 5;
/// link. libhdf5 orders records with equal hashes by name; all of them are
/// read and compared here, so that order does not matter. An index of
/// another type is scanned in full.
fn links_named(
file_data: &[u8],
fn links_named<S: Storage + ?Sized>(
file_data: &S,
object_header: &ObjectHeader,
name: &str,
offset_size: u8,
@@ -220,12 +236,17 @@ fn links_named(
return Ok(found);
};
let fh = FractalHeapHeader::parse(file_data, to_usize(fh_addr)?, offset_size, length_size)?;
let fh =
FractalHeapHeader::parse_in(file_data, checked_addr(fh_addr)?, offset_size, length_size)?;
let btree_addr = link_info
.btree_name_index_address
.ok_or_else(|| FormatError::PathNotFound(String::from("no B-tree v2 name index")))?;
let btree_hdr =
BTreeV2Header::parse(file_data, to_usize(btree_addr)?, offset_size, length_size)?;
let btree_hdr = BTreeV2Header::parse_in(
file_data,
checked_addr(btree_addr)?,
offset_size,
length_size,
)?;
if btree_hdr.tree_type != LINK_NAME_INDEX {
for_each_dense_link(
file_data,
@@ -244,7 +265,7 @@ fn links_named(
// Record: hash(4) + heap ID.
let hash = jenkins_lookup3(name.as_bytes());
let records = find_btree_v2_records(file_data, &btree_hdr, offset_size, &mut |r| {
let records = find_btree_v2_records_in(file_data, &btree_hdr, offset_size, &mut |r| {
match r.get(..4) {
Some(h) => u32::from_le_bytes([h[0], h[1], h[2], h[3]]).cmp(&hash),
// Too short to hold a hash (a corrupt record size): never a match.
@@ -256,7 +277,7 @@ fn links_named(
let Some(id_bytes) = record.data.get(4..4 + id_len) else {
continue;
};
let link_data = fh.read_managed_object(file_data, id_bytes, offset_size)?;
let link_data = fh.read_managed_object_in(file_data, id_bytes, offset_size)?;
if let Some(link) = parse_link(&link_data, offset_size)?
&& link.name == name
{
@@ -278,8 +299,8 @@ fn links_named(
/// and may land on another of several exact duplicates. The listing
/// ([`resolve_group_children`]), [`resolve_child`] and path resolution all
/// apply this rule, so they agree.
fn first_link_named(
file_data: &[u8],
fn first_link_named<S: Storage + ?Sized>(
file_data: &S,
object_header: &ObjectHeader,
name: &str,
offset_size: u8,
@@ -296,8 +317,8 @@ fn first_link_named(
/// the group with header `object_header`: a hard link (as `Hard`), else a
/// soft or external link of that name, else `None`. Fails with
/// `PathNotFound` if the object is not a group.
fn lookup_link(
file_data: &[u8],
fn lookup_link<S: Storage + ?Sized>(
file_data: &S,
object_header: &ObjectHeader,
name: &str,
offset_size: u8,
@@ -346,13 +367,23 @@ pub fn resolve_child(
superblock: &Superblock,
group_address: u64,
name: &str,
) -> Result<u64, FormatError> {
resolve_child_in(file_data, superblock, group_address, name)
}
/// [`resolve_child`] over any [`Storage`].
pub fn resolve_child_in<S: Storage + ?Sized>(
file_data: &S,
superblock: &Superblock,
group_address: u64,
name: &str,
) -> Result<u64, FormatError> {
let os = superblock.offset_size;
let ls = superblock.length_size;
let not_found = || FormatError::PathNotFound(String::from(name));
let header = ObjectHeader::parse(file_data, to_usize(group_address)?, os, ls)?;
let header = ObjectHeader::parse_in(file_data, checked_addr(group_address)?, os, ls)?;
if !is_v2_group(&header) || is_v1_group(&header) {
return resolve_group_children(file_data, superblock, group_address)?
return resolve_group_children_in(file_data, superblock, group_address)?
.into_iter()
.find(|e| e.name == name)
.map(|e| e.object_header_address)
@@ -365,7 +396,7 @@ pub fn resolve_child(
object_header_address,
}) => Ok(object_header_address),
Some(LinkTarget::Soft { target_path }) => {
match resolve_path_from(file_data, superblock, group_address, &target_path) {
match resolve_path_from_in(file_data, superblock, group_address, &target_path) {
// Left out of the listing: dangling, cyclic, or in another file.
Err(
FormatError::PathNotFound(_)
@@ -421,6 +452,15 @@ pub fn resolve_path_any(
file_data: &[u8],
superblock: &Superblock,
path: &str,
) -> Result<u64, FormatError> {
resolve_path_any_in(file_data, superblock, path)
}
/// [`resolve_path_any`] over any [`Storage`].
pub fn resolve_path_any_in<S: Storage + ?Sized>(
file_data: &S,
superblock: &Superblock,
path: &str,
) -> Result<u64, FormatError> {
resolve_path_following_links(
file_data,
@@ -439,6 +479,16 @@ pub fn resolve_path_from(
superblock: &Superblock,
group_address: u64,
path: &str,
) -> Result<u64, FormatError> {
resolve_path_from_in(file_data, superblock, group_address, path)
}
/// [`resolve_path_from`] over any [`Storage`].
pub fn resolve_path_from_in<S: Storage + ?Sized>(
file_data: &S,
superblock: &Superblock,
group_address: u64,
path: &str,
) -> Result<u64, FormatError> {
let start = if path.starts_with('/') {
superblock.root_group_address
@@ -462,10 +512,19 @@ pub fn resolve_group_children(
file_data: &[u8],
superblock: &Superblock,
group_address: u64,
) -> Result<Vec<GroupEntry>, FormatError> {
resolve_group_children_in(file_data, superblock, group_address)
}
/// [`resolve_group_children`] over any [`Storage`].
pub fn resolve_group_children_in<S: Storage + ?Sized>(
file_data: &S,
superblock: &Superblock,
group_address: u64,
) -> Result<Vec<GroupEntry>, FormatError> {
let os = superblock.offset_size;
let ls = superblock.length_size;
let header = ObjectHeader::parse(file_data, to_usize(group_address)?, os, ls)?;
let header = ObjectHeader::parse_in(file_data, checked_addr(group_address)?, os, ls)?;
let mut entries = Vec::new();
let mut soft = Vec::new();
@@ -476,9 +535,9 @@ pub fn resolve_group_children(
.find(|m| m.msg_type == MessageType::SymbolTable)
.ok_or_else(|| FormatError::PathNotFound(String::from("no symbol table message")))?;
let stm = SymbolTableMessage::parse(&sym_msg.data, os)?;
let all = group_v1::resolve_v1_group_entries(file_data, &stm, os, ls)?;
let all = group_v1::resolve_v1_group_entries_in(file_data, &stm, os, ls)?;
if all.iter().any(group_v1::is_v1_soft_link) {
soft = group_v1::v1_soft_links(file_data, &stm, os, ls)?;
soft = group_v1::v1_soft_links_in(file_data, &stm, os, ls)?;
}
entries.extend(all.into_iter().filter(|e| !group_v1::is_v1_soft_link(e)));
} else if is_v2_group(&header) {
@@ -515,7 +574,7 @@ pub fn resolve_group_children(
}
for (name, target) in soft {
match resolve_path_from(file_data, superblock, group_address, &target) {
match resolve_path_from_in(file_data, superblock, group_address, &target) {
Ok(object_header_address) => entries.push(GroupEntry {
name,
object_header_address,
@@ -538,8 +597,8 @@ pub fn resolve_group_children(
const MAX_SOFT_LINK_DEPTH: u8 = 16;
/// Walk `path` from the group at `start`, following soft links.
fn resolve_path_following_links(
file_data: &[u8],
fn resolve_path_following_links<S: Storage + ?Sized>(
file_data: &S,
superblock: &Superblock,
start: u64,
path: &str,
@@ -557,7 +616,7 @@ fn resolve_path_following_links(
let ls = superblock.length_size;
let mut current_addr = start;
let mut current_header = ObjectHeader::parse(file_data, to_usize(start)?, os, ls)?;
let mut current_header = ObjectHeader::parse_in(file_data, checked_addr(start)?, os, ls)?;
for (i, component) in components.iter().enumerate() {
match lookup_link(file_data, &current_header, component, os, ls)? {
@@ -568,7 +627,8 @@ fn resolve_path_following_links(
return Ok(object_header_address);
}
current_addr = object_header_address;
current_header = ObjectHeader::parse(file_data, to_usize(current_addr)?, os, ls)?;
current_header =
ObjectHeader::parse_in(file_data, checked_addr(current_addr)?, os, ls)?;
}
found => {
return match found {
@@ -607,8 +667,8 @@ fn resolve_path_following_links(
}
/// Resolve group entries from an object header, auto-detecting v1 vs v2.
fn resolve_group_entries(
file_data: &[u8],
fn resolve_group_entries<S: Storage + ?Sized>(
file_data: &S,
object_header: &ObjectHeader,
offset_size: u8,
length_size: u8,
@@ -625,7 +685,7 @@ fn resolve_group_entries(
// skipped by the name comparison, as in libhdf5.
group_v1::v1_group_entries(file_data, &stm, offset_size, length_size)
} else if is_v2_group(object_header) {
resolve_v2_group_entries(file_data, object_header, offset_size, length_size)
resolve_v2_group_entries_in(file_data, object_header, offset_size, length_size)
} else {
Err(FormatError::PathNotFound(String::from(
"object header is not a group",