quantumclaw/clawhdf5
3
1
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases 1 Wiki Activity

format: monomorphise the Storage parsers so local files stay as fast

Browse Source 
Every `*_in` core and the read helpers take `file: &S` with
`S: Storage + ?Sized` instead of `&dyn Storage`, and the `&[u8]`
wrappers pass the slice itself, so they compile to a `[u8]` instance:
`as_contiguous()` inlines to `Some(self)` and each structure read is the
slice code's bounds check again, with no indirect call. `&dyn Storage`
still works (`S = dyn Storage`); there is one parser implementation.

Also, so the structure reads cost no more than the slice checks did:
- ObjectHeader::parse_in reads the prefix once (signature included)
  instead of the signature and then the prefix: two reads for a
  one-chunk header instead of three on a range backend;
- the symbol-table node and group B-tree (v1) loops walk their entries
  with chunks_exact over the bytes read, and the node's redundant second
  bounds check is gone (the entries' read is the check, same error);
- a version-1 header's message list is sized from its (capped) count.
Same results and errors; the unit and equivalence tests are unchanged.

New Criterion bench `clawhdf5/benches/local_metadata_bench.rs` over a
400-group version-1 file written by h5py (new fixture
`v1_groups_400.h5`): ObjectHeader::parse, symbol-table nodes, the group
B-tree walk and a facade listing, using only APIs that exist at f2ff2c4
so it builds there for an A/B.

Provisional A/B against f2ff2c4 (busy machine, not for docs): both
builds linked into one binary and timed in alternation, 200 rounds;
median ratio new/old: facade listing -0.5% to -3.5% (was +14%),
ObjectHeader::parse +1% to +2% (was +25%), symbol-table nodes -18%,
group B-tree walk -18%, local-heap names and resolve_group_children
within +-1.5%. An old-vs-old-copy run shows +-2% from code layout alone.

Co-Authored-By: Claude Opus 5.5 (1M context) <[email protected]>
This commit is contained in:
osobh
2026-09-26 14:22:37 -05:00
co-authored by Claude Opus 5.5
parent d2b25f154f
commit 052098bf36
18 changed files with 315 additions and 219 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/clawhdf5-format/src/object_header.rs
+37 -38
View File
@@ -7,7 +7,7 @@ use byteorder::{ByteOrder, LittleEndian};
use crate::error::FormatError;
use crate::message_type::MessageType;
use crate::storage::{Storage, len_usize, read_exact_at, read_upto};
use crate::storage::{Storage, Window, len_usize, read_exact_at};
/// OHDR signature for v2 object headers.
const OHDR_SIGNATURE: [u8; 4] = *b"OHDR";
@@ -119,36 +119,43 @@ impl ObjectHeader {
offset_size: u8,
length_size: u8,
) -> Result<ObjectHeader, FormatError> {
Self::parse_in(&data, offset as u64, offset_size, length_size)
Self::parse_in(data, offset as u64, offset_size, length_size)
}
/// [`Self::parse`] over any [`Storage`].
///
/// Reads the signature, the prefix (at most [`V2_PREFIX_MAX`] bytes),
/// then each chunk as one bounded read, continuation chunks included.
pub fn parse_in(
file: &dyn Storage,
/// Reads the prefix (at most [`V2_PREFIX_MAX`] bytes, signature
/// included), then each chunk as one bounded read, continuation chunks
/// included.
pub fn parse_in<S: Storage + ?Sized>(
file: &S,
offset: u64,
offset_size: u8,
length_size: u8,
) -> Result<ObjectHeader, FormatError> {
let sig = read_exact_at(file, offset, 4)?;
if *sig == OHDR_SIGNATURE {
Self::parse_v2(file, offset, offset_size, length_size)
// The longest prefix of either version, in one read. It holds the
// whole prefix or ends at the end of the file, so its bounds checks
// are the whole-file ones.
let prefix = Window::read(file, offset, V2_PREFIX_MAX)?;
prefix.ensure(0, 4)?;
if prefix.bytes[..4] == OHDR_SIGNATURE {
Self::parse_v2(file, offset, &prefix, offset_size, length_size)
} else {
Self::parse_v1(file, offset, offset_size, length_size)
Self::parse_v1(file, offset, &prefix, offset_size, length_size)
}
}
fn parse_v1(
file: &dyn Storage,
fn parse_v1<S: Storage + ?Sized>(
file: &S,
offset: u64,
prefix: &Window<'_>,
offset_size: u8,
length_size: u8,
) -> Result<ObjectHeader, FormatError> {
// version(1) + reserved(1) + num_messages(2) + ref_count(4) + header_size(4) = 12
// then pad to 8-byte alignment from start of header
let prefix = read_exact_at(file, offset, 12)?;
prefix.ensure(0, 12)?;
let prefix = &prefix.bytes[..12];
let version = prefix[0];
if version != 1 {
@@ -179,7 +186,9 @@ impl ObjectHeader {
})?;
// parse_v1_chunk reads the chunk, with the bounds check that was here.
let mut messages = Vec::new();
// The prefix's count (NIL messages included, capped: it is untrusted)
// sizes the list once instead of growing it message by message.
let mut messages = Vec::with_capacity(num_messages.min(64));
let chunk0_count = Self::parse_v1_chunk(
file,
msg_start,
@@ -221,8 +230,8 @@ impl ObjectHeader {
/// end of the chunk, or leftover bytes too few for a message header (a
/// "gap", which only version 2 allows).
#[allow(clippy::too_many_arguments)]
fn parse_v1_chunk(
file: &dyn Storage,
fn parse_v1_chunk<S: Storage + ?Sized>(
file: &S,
offset: u64,
length: usize,
offset_size: u8,
@@ -295,31 +304,21 @@ impl ObjectHeader {
Ok(count)
}
fn parse_v2(
file: &dyn Storage,
fn parse_v2<S: Storage + ?Sized>(
file: &S,
offset: u64,
prefix: &Window<'_>,
offset_size: u8,
length_size: u8,
) -> Result<ObjectHeader, FormatError> {
// The prefix, read as one window. The window holds the whole prefix
// or ends at the end of the file, so a position past the window is
// past the end of the file: `ensure_len` checks positions relative
// to the header against it and reports them as the whole-file check
// did, with absolute positions and the file's length.
let window = read_upto(file, offset, V2_PREFIX_MAX)?;
let data: &[u8] = &window;
// `ensure_len` checks positions relative to the header against the
// prefix window and reports them as the whole-file check did, with
// absolute positions and the file's length.
let data: &[u8] = &prefix.bytes;
let file_len = len_usize(file);
let base = usize::try_from(offset).unwrap_or(usize::MAX);
let abs = |rel: usize| base.saturating_add(rel);
let ensure_len = |_: &[u8], rel: usize, needed: usize| -> Result<(), FormatError> {
match rel.checked_add(needed) {
Some(end) if end <= data.len() => Ok(()),
_ => Err(FormatError::UnexpectedEof {
expected: abs(rel).saturating_add(needed),
available: file_len,
}),
}
};
let ensure_len = |_: &[u8], rel: usize, needed: usize| prefix.ensure(rel, needed);
let offset = 0usize;
// signature(4) + version(1) + flags(1) = 6
ensure_len(data, offset, 6)?;
@@ -533,8 +532,8 @@ impl ObjectHeader {
}
#[allow(clippy::too_many_arguments)]
fn parse_v2_continuation(
file: &dyn Storage,
fn parse_v2_continuation<S: Storage + ?Sized>(
file: &S,
offset: u64,
length: usize,
has_creation_order: bool,
@@ -1328,7 +1327,7 @@ mod tests {
/// Every header, and every truncation of it, parses to the same result
/// (or the same error) through a `read_at`-only storage as from a slice;
/// a header in one chunk takes three reads (signature, prefix, chunk).
/// a header in one chunk takes two reads (prefix, chunk).
#[test]
fn parse_in_matches_slice_parse() {
use crate::storage::CountingStorage;
@@ -1375,6 +1374,6 @@ mod tests {
let one_chunk = build_v2_header(0x00, &[(0x01, &[42], 0)], None);
let storage = CountingStorage::new(one_chunk);
ObjectHeader::parse_in(&storage, 0, 8, 8).unwrap();
assert_eq!(storage.reads(), 3);
assert_eq!(storage.reads(), 2);
}
}