quantumclaw/clawhdf5
3
1
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 1 Wiki Activity

format: v2 B-trees, dense groups and group listings over Storage

Browse Source 
BTreeV2Header::parse_in, collect_btree_v2_records_in and
find_btree_v2_records_in read one bounded window per node (its size is
known from the parent before the node is read; a count stretched past
node_size is checked against the end of the file first), with the
whole-file bounds errors unchanged. With them, dense attributes, a SOHM
B-tree index and huge fractal-heap objects no longer answer
ContiguousStorageRequired, and group_v1/group_v2 listings, lookups and
path resolution get *_in cores (resolve_group_children_in,
resolve_child_in, resolve_path_any_in, ...). The &[u8] functions are
thin wrappers, as in M1.

The equivalence harness now fails on any ContiguousStorageRequired and
compares v2 B-tree headers, records and descents, group listings, child
lookups and paths; a unit test compares a two-level tree through a
read_at-only storage truncated at every length and with every node byte
flipped.

Co-Authored-By: Claude Opus 5.5 (1M context) <[email protected]>
This commit is contained in:
osobh
2026-09-26 16:13:01 -05:00
co-authored by Claude Opus 5.5
parent 8f59b2e1c2
commit 42894bf93b
9 changed files with 544 additions and 297 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/clawhdf5-format/tests/storage_equivalence.rs
+66 -72
View File
@@ -1,24 +1,19 @@
//! Equivalence harness for the range-read migration
//! (`docs/design/range-reads.md`, milestone M1).
//! (`docs/design/range-reads.md`, milestones M1 and M2).
//!
//! Every metadata parser converted to [`Storage`] must give exactly what its
//! `&[u8]` form gives. This walks real files — the fixtures, files h5py
//! writes to exercise the less common structures, and optionally the
//! conformance corpus — and, for every object, runs each converted parser
//! twice: over the file as a slice, and over a [`CountingStorage`] that
//! serves the same bytes through `read_at` only (`as_contiguous()` is
//! `None`, so no parser can fall back to the whole slice). The results must
//! be identical, value for value and error for error.
//! Every parser converted to [`Storage`] must give exactly what its `&[u8]`
//! form gives. This walks real files — the fixtures, files h5py writes to
//! exercise the less common structures, and optionally the conformance
//! corpus — and, for every object, runs each converted parser twice: over
//! the file as a slice, and over a [`CountingStorage`] that serves the same
//! bytes through `read_at` only (`as_contiguous()` is `None`, so no parser
//! can fall back to the whole slice). The results must be identical, value
//! for value and error for error.
//!
//! The one allowed difference is [`FormatError::ContiguousStorageRequired`]
//! from the storage path, and only from the structures still indexed by a v2
//! B-tree (dense attributes, a SOHM B-tree index, huge fractal-heap objects;
//! see `CONTIGUOUS_REQUIRED`), which fail cleanly instead of reading the
//! whole file. Those are counted; the error from any other site or check
//! fails the harness.
//!
//! Milestones M2/M3 extend `check_object` with the raw-data and group
//! parsers as they are converted.
//! Nothing may answer [`FormatError::ContiguousStorageRequired`] any more:
//! since milestone M2 every read path works over `read_at` alone, the v2
//! B-tree structures (dense groups and attributes, a SOHM B-tree index,
//! huge fractal-heap objects) included.
//!
//! - `CLAWHDF5_STORAGE_CORPUS=dir[:dir...]` adds every `.h5`/`.hdf5`/`.he5`/
//! `.nc`/`.h5ad` file under those directories (the conformance corpus is
@@ -38,7 +33,10 @@ use clawhdf5_format::attribute::{
};
use clawhdf5_format::attribute_info::AttributeInfoMessage;
use clawhdf5_format::btree_v1::{collect_symbol_table_nodes, collect_symbol_table_nodes_in};
use clawhdf5_format::btree_v2::{BTreeV2Header, collect_btree_v2_records};
use clawhdf5_format::btree_v2::{
BTreeV2Header, collect_btree_v2_records, collect_btree_v2_records_in, find_btree_v2_records,
find_btree_v2_records_in,
};
use clawhdf5_format::data_layout::DataLayout;
use clawhdf5_format::dataspace::Dataspace;
use clawhdf5_format::datatype::Datatype;
@@ -51,6 +49,7 @@ use clawhdf5_format::fixed_array::{
FixedArrayHeader, read_fixed_array_chunks, read_fixed_array_chunks_in,
};
use clawhdf5_format::fractal_heap::FractalHeapHeader;
use clawhdf5_format::group_v2;
use clawhdf5_format::link_info::LinkInfoMessage;
use clawhdf5_format::local_heap::LocalHeap;
use clawhdf5_format::message_type::MessageType;
@@ -73,44 +72,11 @@ use clawhdf5_format::symbol_table::{SymbolTableMessage, SymbolTableNode};
const MAX_OBJECTS: usize = 1500;
const MAX_HEAP_IDS: usize = 200;
/// The structures that still need the whole file in memory, because they
/// are found through a version-2 B-tree (not converted yet), and the checks
/// that can reach each of them. Anything else answering
/// [`FormatError::ContiguousStorageRequired`] is a converted parser falling
/// back to the whole file, and fails the harness.
const CONTIGUOUS_REQUIRED: &[(&str, &[&str])] = &[
(
"dense attribute storage (a v2 B-tree)",
&["attributes", "attributes (tolerant)"],
),
(
"a shared-message B-tree index",
&[
"SOHM B-tree",
"shared message",
"fill value",
"attributes",
"attributes (tolerant)",
],
),
(
"a huge fractal-heap object's B-tree",
&["heap object", "attributes", "attributes (tolerant)"],
),
];
fn may_require_contiguous(check: &str, site: &str) -> bool {
CONTIGUOUS_REQUIRED
.iter()
.any(|(s, checks)| *s == site && checks.contains(&check))
}
#[derive(Default, Debug)]
struct Tally {
files: usize,
objects: usize,
checks: usize,
contiguous_required: usize,
reads: u64,
bytes: u64,
/// Chunk indexes (fixed and extensible arrays) read, and the most bytes
@@ -127,8 +93,8 @@ struct Walk<'a> {
}
impl Walk<'_> {
/// The storage result must equal the slice result, or be the clean
/// "needs the whole file" error.
/// The storage result must equal the slice result; no parser may ask
/// for the whole file.
fn same<T: Debug>(
&mut self,
what: &str,
@@ -137,14 +103,11 @@ impl Walk<'_> {
) {
self.tally.checks += 1;
if let Err(FormatError::ContiguousStorageRequired(site)) = got {
assert!(
may_require_contiguous(what, site),
"{}: {what} fell back to the whole file ({site}), which only the \
v2-B-tree-indexed structures may do",
panic!(
"{}: {what} fell back to the whole file ({site}); every read path \
must work through read_at",
self.name
);
self.tally.contiguous_required += 1;
return;
}
let (w, g) = (format!("{want:?}"), format!("{got:?}"));
assert!(
@@ -206,13 +169,34 @@ impl Walk<'_> {
}
self.tally.objects += 1;
self.check_object(&sb, addr);
// Traversal only (group lookups are milestone M0/M3 work).
if let Ok(children) =
clawhdf5_format::group_v2::resolve_group_children(slice, &sb, addr)
{
let children = group_v2::resolve_group_children(slice, &sb, addr);
let got = group_v2::resolve_group_children_in(self.st(), &sb, addr);
self.same("group listing", &children, &got);
if let Ok(children) = children {
for c in children.iter().take(MAX_HEAP_IDS) {
let want = group_v2::resolve_child(slice, &sb, addr, &c.name);
let got = group_v2::resolve_child_in(self.st(), &sb, addr, &c.name);
self.same("child lookup", &want, &got);
}
// A name no group has: the lookup's not-found path.
let want = group_v2::resolve_child(slice, &sb, addr, "no such child");
let got = group_v2::resolve_child_in(self.st(), &sb, addr, "no such child");
self.same("child lookup (missing)", &want, &got);
queue.extend(children.iter().map(|c| c.object_header_address));
}
}
// Paths: every listed name from the root, and one that is missing.
if let Ok(children) = group_v2::resolve_group_children(slice, &sb, sb.root_group_address) {
for c in children.iter().take(MAX_HEAP_IDS) {
let path = format!("/{}", c.name);
let want = group_v2::resolve_path_any(slice, &sb, &path);
let got = group_v2::resolve_path_any_in(self.st(), &sb, &path);
self.same("path", &want, &got);
}
}
let want = group_v2::resolve_path_any(slice, &sb, "/no/such/path");
let got = group_v2::resolve_path_any_in(self.st(), &sb, "/no/such/path");
self.same("path (missing)", &want, &got);
}
fn check_object(&mut self, sb: &Superblock, addr: u64) {
@@ -336,12 +320,24 @@ impl Walk<'_> {
let (Ok(fh), Some(index)) = (fh, index) else {
return;
};
let Ok(bt) = BTreeV2Header::parse(slice, index as usize, os, ls) else {
return;
};
let Ok(records) = collect_btree_v2_records(slice, &bt, os, ls) else {
return;
};
let bt = BTreeV2Header::parse(slice, index as usize, os, ls);
self.same(
"v2 B-tree header",
&bt,
&BTreeV2Header::parse_in(self.st(), index, os, ls),
);
let Ok(bt) = bt else { return };
let records = collect_btree_v2_records(slice, &bt, os, ls);
let got = collect_btree_v2_records_in(self.st(), &bt, os, ls);
self.same("v2 B-tree records", &records, &got);
let Ok(records) = records else { return };
// Descents to single records (by their bytes), as name lookups do.
for rec in records.iter().take(8) {
let key = rec.data.clone();
let want = find_btree_v2_records(slice, &bt, os, &mut |r| r.cmp(&key[..]));
let got = find_btree_v2_records_in(self.st(), &bt, os, &mut |r| r.cmp(&key[..]));
self.same("v2 B-tree descent", &want, &got);
}
let id_len = fh.heap_id_length as usize;
for rec in records.iter().take(MAX_HEAP_IDS) {
let Some(id) = rec.data.get(id_at..id_at + id_len) else {
@@ -696,6 +692,4 @@ fn h5py_files_parse_identically_through_storage() {
}
eprintln!("h5py files: {tally:?}");
assert!(tally.objects >= 700, "{tally:?}");
// Dense attributes and the SOHM B-tree are the known clean errors.
assert!(tally.contiguous_required > 0, "{tally:?}");
}