quantumclaw/clawhdf5
3
1
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 1 Wiki Activity

perf(py): datasets and groups keep their address; groups their links

Browse Source 
Every ds[...] and g[k] resolved the path from the root again, two or
three times per open, and resolving a name in a large group scans its
links: visiting a group was O(n^2). 4000 scalar datasets in one group
took 39 s (v1 group) and 131 s (dense) to list, read and re-read; now
0.3 s each. A Dataset keeps its object address, a Group (and the file's
root) its address and, after the first lookup, its link table.

New facade API File::dataset_at(address), tested in integration_tests.

Co-Authored-By: Claude Opus 5.5 (1M context) <[email protected]>
This commit is contained in:
osobh
2026-09-26 09:01:54 -05:00
co-authored by Claude Opus 5.5
parent b43bd2e67f
commit f0ecae38b6
9 changed files with 349 additions and 163 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/clawhdf5-py/src/node.rs
+44 -30
View File
@@ -33,24 +33,40 @@ pub(crate) fn name(path: &str) -> String {
format!("/{path}")
}
/// The object header of the object at `path`.
pub(crate) fn header(file: &clawhdf5_rs::File, path: &str) -> PyResult<ObjectHeader> {
/// The address of the object at `path`, resolved from the root group.
pub(crate) fn address(file: &clawhdf5_rs::File, path: &str) -> PyResult<u64> {
resolve_from(file, file.superblock().root_group_address, path, path)
}
/// The address of `rel` resolved from the group at `group` (`full` is the
/// resulting path, for the error message).
pub(crate) fn resolve_from(
file: &clawhdf5_rs::File,
group: u64,
rel: &str,
full: &str,
) -> PyResult<u64> {
if rel.is_empty() {
return Ok(group);
}
crate::no_panic(|| {
let sb = file.superblock();
let data = file.as_bytes();
let addr = if path.is_empty() {
sb.root_group_address
} else {
clawhdf5_format::group_v2::resolve_path_any(data, sb, path).map_err(|e| {
clawhdf5_format::group_v2::resolve_path_from(file.as_bytes(), file.superblock(), group, rel)
.map_err(|e| {
PyKeyError::new_err(format!(
"Unable to open object (object '{}' doesn't exist): {e}",
name(path)
name(full)
))
})?
};
let addr = usize::try_from(addr)
})
})
}
/// The object header at `addr` (the object at `path`).
pub(crate) fn header_at(file: &clawhdf5_rs::File, addr: u64, path: &str) -> PyResult<ObjectHeader> {
crate::no_panic(|| {
let sb = file.superblock();
let at = usize::try_from(addr)
.map_err(|_| PyValueError::new_err(format!("{}: address out of range", name(path))))?;
ObjectHeader::parse(data, addr, sb.offset_size, sb.length_size)
ObjectHeader::parse(file.as_bytes(), at, sb.offset_size, sb.length_size)
.map_err(|e| PyValueError::new_err(format!("{}: {e}", name(path))))
})
}
@@ -80,19 +96,21 @@ pub(crate) fn kind(hdr: &ObjectHeader) -> Option<Kind> {
}
}
/// Open the object at `path` as a `Dataset` or `Group`.
/// Open the object at `addr` (whose path is `path`) as a `Dataset` or
/// `Group`. Both keep the address, so later reads resolve nothing.
pub(crate) fn open(
py: Python<'_>,
file: &Arc<clawhdf5_rs::File>,
path: String,
addr: u64,
) -> PyResult<Py<PyAny>> {
let hdr = header(file, &path)?;
let hdr = header_at(file, addr, &path)?;
match kind(&hdr) {
Some(Kind::Dataset) => Ok(PyDataset::open(py, Arc::clone(file), path)?
Some(Kind::Dataset) => Ok(PyDataset::open(py, Arc::clone(file), path, addr, &hdr)?
.into_pyobject(py)?
.into_any()
.unbind()),
Some(Kind::Group) => Ok(PyGroup::from_read(Arc::clone(file), path)
Some(Kind::Group) => Ok(PyGroup::from_read(Arc::clone(file), path, addr)
.into_pyobject(py)?
.into_any()
.unbind()),
@@ -107,14 +125,6 @@ pub(crate) fn open(
}
}
/// Whether `path` names a dataset or group.
pub(crate) fn exists(file: &clawhdf5_rs::File, path: &str) -> bool {
header(file, path)
.ok()
.and_then(|h| kind(&h))
.is_some_and(|k| k != Kind::Datatype)
}
/// The dataspace message of an object header.
pub(crate) fn dataspace(file: &clawhdf5_rs::File, hdr: &ObjectHeader) -> PyResult<Dataspace> {
crate::no_panic(|| {
@@ -166,12 +176,16 @@ pub(crate) fn is_null(space: &Dataspace) -> bool {
space.space_type == DataspaceType::Null
}
/// The attributes of the object at `path`, sorted by name (h5py's order).
/// Attributes whose messages cannot be parsed are left out, as the facade's
/// `attrs()` does.
pub(crate) fn attributes(file: &clawhdf5_rs::File, path: &str) -> PyResult<Vec<AttributeMessage>> {
/// The attributes of the object at `addr` (whose path is `path`), sorted by
/// name (h5py's order). Attributes whose messages cannot be parsed are left
/// out, as the facade's `attrs()` does.
pub(crate) fn attributes(
file: &clawhdf5_rs::File,
addr: u64,
path: &str,
) -> PyResult<Vec<AttributeMessage>> {
let hdr = header_at(file, addr, path)?;
crate::no_panic(|| {
let hdr = header(file, path)?;
let sb = file.superblock();
let (mut attrs, _errors) = clawhdf5_format::attribute::extract_attributes_tolerant(
file.as_bytes(),