quantumclaw/clawhdf5
3
1
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases 1 Wiki Activity

clawhdf5: parse local files through the slice, and cache the contiguous view

Browse Source 
Since M2 the facade handed the metadata parsers its FileData view, so a
local file ran the parsers monomorphised for FileData, whose Storage impl
worked out contiguous() (patched and overlay checks, two range conversions)
on every structure read. A metadata walk of h5stat_newgrat.h5 (35,001
groups: open, entries and attrs of each) was about 4.5% slower than at
8f59b2e.

- FileData works out its contiguous slice once at open (a borrow of its own
  heap/mapped buffer, kept as a pointer; see the SAFETY notes).
- with_bytes! hands the in-memory slice to the format parsers when the file
  has one (header parsing, attributes, group listings and lookups, path
  resolution, shared messages, VL decoding), so local files run the [u8]
  parsers as before; storage-backed files still get FileData.

Provisional A/B on tank (load 5-11), best of 30, 5 alternating rounds:
walk 26.07-26.37 ms at 8f59b2e, 27.24-28.03 ms before this commit,
26.52-26.98 ms after. Caching alone did not move it (27.02-27.47 ms); the
dispatch did. File::open read_f32 on 32M f32 (contiguous, chunked, gzip)
stays within noise of 8f59b2e.

Co-Authored-By: Claude Opus 5.5 (1M context) <[email protected]>
This commit is contained in:
osobh
2026-09-26 18:41:10 -05:00
co-authored by Claude Opus 5.5
parent b086dc3c2b
commit 89e7977943
1 changed files with 135 additions and 53 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/clawhdf5/src/reader.rs
+135 -53
View File
@@ -64,6 +64,24 @@ impl Backing {
}
}
/// Evaluate `$body` with `$d` bound to the bytes the `clawhdf5_format`
/// parsers read: the in-memory slice when the file has one (a `Vec`, an
/// mmap), so local files run the parsers monomorphised for `[u8]` — the
/// slice code, as before the range-read migration — and otherwise the
/// [`FileData`] itself, whose reads go to the storage.
macro_rules! with_bytes {
($data:expr, |$d:ident| $body:expr) => {{
let data: &FileData = $data;
match data.contiguous() {
Some($d) => $body,
None => {
let $d = data;
$body
}
}
}};
}
/// The file's bytes, viewed from the superblock on and up to the end of
/// file the superblock records. A file may start with a user block (the
/// superblock at 512, 1024, …); every HDF5 address is relative to the
@@ -93,8 +111,28 @@ struct FileData {
/// then); every object lookup here fails with this error, and no
/// metadata is read from the file's own, possibly stale, bytes.
image_error: Option<FormatError>,
/// [`Self::contiguous`], worked out once at open: every structure a
/// parser reads asks for it, and the patched/overlay checks and range
/// conversions behind it cost a local metadata walk a few percent.
contiguous: Option<WholeView>,
}
/// A borrow of the HDF5 data held by a [`FileData`]'s own `backing` or
/// `patched` buffer (see [`FileData::contiguous`]).
#[derive(Clone, Copy)]
struct WholeView {
ptr: *const u8,
len: usize,
}
// SAFETY: a `WholeView` is only a borrow of bytes owned (through `backing`
// or `patched`) by the `FileData` holding it, which is `Send + Sync`: the
// bytes are never written after open, so sharing the pointer across
// threads is sharing a `&[u8]`.
unsafe impl Send for WholeView {}
// SAFETY: as above.
unsafe impl Sync for WholeView {}
impl FileData {
/// Locate the superblock and parse it. A truncated file is refused, and
/// bytes past the recorded end of file are not read, as in libhdf5.
@@ -127,17 +165,17 @@ impl FileData {
ImageView::Patched(p) => (Some(p), None),
ImageView::Unloadable(e) => (None, Some(e)),
};
Ok((
Self {
backing,
base: base as u64,
end: end as u64,
patched,
overlay: Vec::new(),
image_error,
},
superblock,
))
let mut data = Self {
backing,
base: base as u64,
end: end as u64,
patched,
overlay: Vec::new(),
image_error,
contiguous: None,
};
data.contiguous = data.find_contiguous();
Ok((data, superblock))
}
/// [`Self::new`] for a [`Storage`] backend: the same checks, through
@@ -152,7 +190,11 @@ impl FileData {
patched: None,
overlay: Vec::new(),
image_error: None,
contiguous: None,
};
// Worked out again below, once the end of file and any cache image
// are known.
data.contiguous = data.find_contiguous();
let superblock = Superblock::parse_in(&data, 0)?;
data.end = base + superblock.data_end(base, file_len)?;
match superblock_ext::cache_image_state_in(&data, &superblock)? {
@@ -167,13 +209,37 @@ impl FileData {
.collect();
}
}
data.contiguous = data.find_contiguous();
Ok((data, superblock))
}
/// The HDF5 data as one slice, when the file is in memory (a `Vec`, an
/// mmap, or a storage that holds it all and has no cache image to lay
/// over it).
#[inline]
fn contiguous(&self) -> Option<&[u8]> {
// SAFETY: `find_contiguous` borrowed these bytes from `backing` or
// `patched`, which this `FileData` owns and never changes after
// open. They live on the heap or in a mapping (a `Vec`'s buffer, an
// mmap, a private copy, or a buffer inside the `Arc`'d storage), so
// they stay put when the `FileData` moves, and they live as long as
// `self`.
self.contiguous
.map(|v| unsafe { core::slice::from_raw_parts(v.ptr, v.len) })
}
/// [`Self::contiguous`], worked out from `backing` and `patched`.
fn find_contiguous(&self) -> Option<WholeView> {
let bytes = self.compute_contiguous()?;
Some(WholeView {
ptr: bytes.as_ptr(),
len: bytes.len(),
})
}
/// The HDF5 data as one slice, from `backing` and `patched` (see
/// [`Self::contiguous`]).
fn compute_contiguous(&self) -> Option<&[u8]> {
if let Some(p) = &self.patched {
return p.get(usize::try_from(self.base).ok()?..usize::try_from(self.end).ok()?);
}
@@ -224,6 +290,7 @@ impl FileData {
}
impl Storage for FileData {
#[inline]
fn read_at(&self, offset: u64, len: usize) -> Result<Cow<'_, [u8]>, FormatError> {
if let Some(all) = self.contiguous() {
return all.read_at(offset, len);
@@ -261,6 +328,7 @@ impl Storage for FileData {
.collect())
}
#[inline]
fn as_contiguous(&self) -> Option<&[u8]> {
self.contiguous()
}
@@ -400,8 +468,11 @@ impl File {
///
/// The path uses `/` separators (e.g., `"group1/values"`).
pub fn dataset(&self, path: &str) -> Result<Dataset<'_>, Error> {
let data = self.data.meta()?;
let addr = group_v2::resolve_path_any_in(data, &self.superblock, path)?;
let addr = with_bytes!(self.data.meta()?, |d| group_v2::resolve_path_any_in(
d,
&self.superblock,
path
))?;
let hdr = self.parse_header(addr)?;
if !has_message(&hdr, MessageType::DataLayout) {
return Err(Error::NotADataset(path.to_string()));
@@ -447,8 +518,11 @@ impl File {
/// The path uses `/` separators (e.g., `"sensors"`).
/// Use `"/"` or `""` for the root group.
pub fn group(&self, path: &str) -> Result<Group<'_>, Error> {
let data = self.data.meta()?;
let addr = group_v2::resolve_path_any_in(data, &self.superblock, path)?;
let addr = with_bytes!(self.data.meta()?, |d| group_v2::resolve_path_any_in(
d,
&self.superblock,
path
))?;
Ok(Group {
file: self,
address: addr,
@@ -560,13 +634,13 @@ impl File {
/// [`AttrValue::Raw`] attribute. Variable-length strings are resolved in
/// this file's global heap; see [`Dataset::read_string`] for the values.
pub fn decode_strings(&self, datatype: &Datatype, raw: &[u8]) -> Result<Vec<String>, Error> {
crate::vlen::decode_strings(
&self.data,
with_bytes!(&self.data, |d| crate::vlen::decode_strings(
d,
datatype,
raw,
self.offset_size(),
self.length_size(),
)
))
}
/// Like [`decode_strings`](Self::decode_strings) for variable-length
@@ -577,13 +651,13 @@ impl File {
datatype: &Datatype,
raw: &[u8],
) -> Result<Vec<Vec<u8>>, Error> {
crate::vlen::decode_string_bytes(
self.data.meta()?,
with_bytes!(self.data.meta()?, |d| crate::vlen::decode_string_bytes(
d,
datatype,
raw,
self.offset_size(),
self.length_size(),
)
))
}
/// Decode the variable-length sequences in `raw`, a buffer of elements
@@ -595,22 +669,22 @@ impl File {
datatype: &Datatype,
raw: &[u8],
) -> Result<Vec<Vec<T>>, Error> {
crate::vlen::decode_vlen(
self.data.meta()?,
with_bytes!(self.data.meta()?, |d| crate::vlen::decode_vlen(
d,
datatype,
raw,
self.offset_size(),
self.length_size(),
)
))
}
fn parse_header(&self, address: u64) -> Result<ObjectHeader, FormatError> {
ObjectHeader::parse_in(
self.data.meta()?,
with_bytes!(self.data.meta()?, |d| ObjectHeader::parse_in(
d,
address,
self.superblock.offset_size,
self.superblock.length_size,
)
))
}
fn offset_size(&self) -> u8 {
@@ -688,8 +762,12 @@ impl<'f> Group<'f> {
&self,
) -> Result<(HashMap<String, AttrValue>, Vec<FormatError>), Error> {
let hdr = self.file.parse_header(self.address)?;
let data = &self.file.data;
read_attrs(data, &hdr, self.file.offset_size(), self.file.length_size())
with_bytes!(&self.file.data, |d| read_attrs(
d,
&hdr,
self.file.offset_size(),
self.file.length_size()
))
}
/// Get a dataset within this group by name.
@@ -719,14 +797,13 @@ impl<'f> Group<'f> {
/// stored densely.
pub fn attr(&self, name: &str) -> Result<Option<AttrValue>, Error> {
let hdr = self.file.parse_header(self.address)?;
let data = &self.file.data;
read_attr(
data,
with_bytes!(&self.file.data, |d| read_attr(
d,
&hdr,
name,
self.file.offset_size(),
self.file.length_size(),
)
))
}
/// The object header address of the child called `name`: the entry of
@@ -734,9 +811,13 @@ impl<'f> Group<'f> {
/// name index rather than by listing the group (see
/// [`group_v2::resolve_child`]).
fn child_address(&self, name: &str) -> Result<u64, Error> {
let data = self.file.data.meta()?;
group_v2::resolve_child_in(data, &self.file.superblock, self.address, name)
.map_err(Error::Format)
with_bytes!(self.file.data.meta()?, |d| group_v2::resolve_child_in(
d,
&self.file.superblock,
self.address,
name
))
.map_err(Error::Format)
}
/// This group's children that can be opened, as `(name, object header
@@ -757,9 +838,10 @@ impl<'f> Group<'f> {
/// [`group_v2::resolve_group_children`]); dangling, external and
/// user-defined links are left out.
fn children(&self) -> Result<Vec<GroupEntry>, Error> {
let data = self.file.data.meta()?;
group_v2::resolve_group_children_in(data, &self.file.superblock, self.address)
.map_err(Error::Format)
with_bytes!(self.file.data.meta()?, |d| {
group_v2::resolve_group_children_in(d, &self.file.superblock, self.address)
})
.map_err(Error::Format)
}
}
@@ -1332,13 +1414,12 @@ impl<'f> Dataset<'f> {
pub fn attrs_with_errors(
&self,
) -> Result<(HashMap<String, AttrValue>, Vec<FormatError>), Error> {
let data = &self.file.data;
read_attrs(
data,
with_bytes!(&self.file.data, |d| read_attrs(
d,
&self.header,
self.file.offset_size(),
self.file.length_size(),
)
))
}
/// The attribute called `name`, or `None` if it has none by that name
@@ -1346,14 +1427,13 @@ impl<'f> Dataset<'f> {
/// that name, found without reading the other attributes when they are
/// stored densely.
pub fn attr(&self, name: &str) -> Result<Option<AttrValue>, Error> {
let data = &self.file.data;
read_attr(
data,
with_bytes!(&self.file.data, |d| read_attr(
d,
&self.header,
name,
self.file.offset_size(),
self.file.length_size(),
)
))
}
/// Verify this dataset's content against its stored provenance hash
@@ -1393,12 +1473,14 @@ impl<'f> Dataset<'f> {
.iter()
.find(|m| m.msg_type == msg_type)
.map(|msg| {
clawhdf5_format::shared_message::message_data_in(
&self.file.data,
msg,
self.file.offset_size(),
self.file.length_size(),
)
with_bytes!(&self.file.data, |d| {
clawhdf5_format::shared_message::message_data_in(
d,
msg,
self.file.offset_size(),
self.file.length_size(),
)
})
.map_err(Error::Format)
})
.transpose()