quantumclaw/clawhdf5
3
1
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 1 Wiki Activity

format: v2 B-trees, dense groups and group listings over Storage

Browse Source 
BTreeV2Header::parse_in, collect_btree_v2_records_in and
find_btree_v2_records_in read one bounded window per node (its size is
known from the parent before the node is read; a count stretched past
node_size is checked against the end of the file first), with the
whole-file bounds errors unchanged. With them, dense attributes, a SOHM
B-tree index and huge fractal-heap objects no longer answer
ContiguousStorageRequired, and group_v1/group_v2 listings, lookups and
path resolution get *_in cores (resolve_group_children_in,
resolve_child_in, resolve_path_any_in, ...). The &[u8] functions are
thin wrappers, as in M1.

The equivalence harness now fails on any ContiguousStorageRequired and
compares v2 B-tree headers, records and descents, group listings, child
lookups and paths; a unit test compares a two-level tree through a
read_at-only storage truncated at every length and with every node byte
flipped.

Co-Authored-By: Claude Opus 5.5 (1M context) <[email protected]>
This commit is contained in:
osobh
2026-09-26 16:13:01 -05:00
co-authored by Claude Opus 5.5
parent 8f59b2e1c2
commit 42894bf93b
9 changed files with 544 additions and 297 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/clawhdf5-format/src/attribute.rs
+29 -26
View File
@@ -7,7 +7,7 @@ use std::borrow::Cow;
use crate::addr::to_usize;
use crate::attribute_info::AttributeInfoMessage;
use crate::btree_v2::{BTreeV2Header, collect_btree_v2_records, find_btree_v2_records};
use crate::btree_v2::{BTreeV2Header, collect_btree_v2_records_in, find_btree_v2_records_in};
use crate::checksum::jenkins_lookup3;
use crate::data_read;
use crate::dataspace::Dataspace;
@@ -17,7 +17,7 @@ use crate::fractal_heap::FractalHeapHeader;
use crate::message_type::MessageType;
use crate::object_header::ObjectHeader;
use crate::shared_message;
use crate::storage::{Storage, require_contiguous};
use crate::storage::Storage;
use crate::vl_data;
/// A parsed HDF5 attribute message.
@@ -578,9 +578,12 @@ pub fn find_attribute_in<S: Storage + ?Sized>(
.find(|a| a.name == name),
);
};
let contiguous = require_contiguous(file_data, "dense attribute storage (a v2 B-tree)")?;
let btree_hdr =
BTreeV2Header::parse(contiguous, to_usize(btree_addr)?, offset_size, length_size)?;
let btree_hdr = BTreeV2Header::parse_in(
file_data,
to_usize(btree_addr)? as u64,
offset_size,
length_size,
)?;
let fh = FractalHeapHeader::parse_in(file_data, fh_addr, offset_size, length_size)?;
if btree_hdr.tree_type != ATTRIBUTE_NAME_INDEX || btree_hdr.record_size < 4 {
return Ok(
@@ -610,7 +613,7 @@ pub fn find_attribute_in<S: Storage + ?Sized>(
// hash is the last field.
let hash = jenkins_lookup3(name.as_bytes());
let hash_at = usize::from(btree_hdr.record_size) - 4;
let records = find_btree_v2_records(contiguous, &btree_hdr, offset_size, &mut |r| match r
let records = find_btree_v2_records_in(file_data, &btree_hdr, offset_size, &mut |r| match r
.get(hash_at..hash_at + 4)
{
Some(h) => u32::from_le_bytes([h[0], h[1], h[2], h[3]]).cmp(&hash),
@@ -722,10 +725,13 @@ fn extract_dense_attributes<S: Storage + ?Sized>(
expected: 1,
available: 0,
})?;
let contiguous = require_contiguous(file_data, "dense attribute storage (a v2 B-tree)")?;
let btree_hdr =
BTreeV2Header::parse(contiguous, to_usize(btree_addr)?, offset_size, length_size)?;
let records = collect_btree_v2_records(contiguous, &btree_hdr, offset_size, length_size)?;
let btree_hdr = BTreeV2Header::parse_in(
file_data,
to_usize(btree_addr)? as u64,
offset_size,
length_size,
)?;
let records = collect_btree_v2_records_in(file_data, &btree_hdr, offset_size, length_size)?;
for record in &records {
// Per HDF5 spec, both type 8 and type 9 records start with heap_id:
@@ -1156,11 +1162,9 @@ mod tests {
}
/// Every object's attributes in h5py-written files read identically
/// through a read_at-only CountingStorage — compact ones, shared ones
/// and those behind an Attribute Info message — except dense storage,
/// whose v2 B-tree index is not read over Storage yet: that is the clean
/// ContiguousStorageRequired error, never a partial list. Through a
/// slice as Storage every object matches.
/// through a read_at-only CountingStorage — compact ones, shared ones,
/// those behind an Attribute Info message and dense storage (its v2
/// B-tree name index included) — and through a slice as Storage.
#[test]
fn storage_reads_match_slice_reads() {
use crate::storage::CountingStorage;
@@ -1206,18 +1210,17 @@ mod tests {
.unwrap()
.is_some_and(|i| i.fractal_heap_address.is_some());
if is_dense {
let e = FormatError::ContiguousStorageRequired(
"dense attribute storage (a v2 B-tree)",
);
assert_eq!(got.unwrap_err(), e, "{name}");
assert_eq!(got_t.unwrap_err(), e, "{name}");
dense += 1;
} else {
attrs += want.as_ref().map_or(0, Vec::len);
assert_eq!(format!("{got:?}"), format!("{want:?}"), "{name}");
let want_t = extract_attributes_tolerant(file, &header, os, ls);
assert_eq!(format!("{got_t:?}"), format!("{want_t:?}"), "{name}");
same += 1;
}
attrs += want.as_ref().map_or(0, Vec::len);
assert_eq!(format!("{got:?}"), format!("{want:?}"), "{name}");
let want_t = extract_attributes_tolerant(file, &header, os, ls);
assert_eq!(format!("{got_t:?}"), format!("{want_t:?}"), "{name}");
same += 1;
for a in want.iter().flatten() {
let one = find_attribute_in(&storage, &header, &a.name, os, ls);
let want_one = find_attribute_in_file(file, &header, &a.name, os, ls);
assert_eq!(format!("{one:?}"), format!("{want_one:?}"), "{name}");
}
}
}