perf(py): datasets and groups keep their address; groups their links

Every ds[...] and g[k] resolved the path from the root again, two or
three times per open, and resolving a name in a large group scans its
links: visiting a group was O(n^2). 4000 scalar datasets in one group
took 39 s (v1 group) and 131 s (dense) to list, read and re-read; now
0.3 s each. A Dataset keeps its object address, a Group (and the file's
root) its address and, after the first lookup, its link table.

New facade API File::dataset_at(address), tested in integration_tests.

Co-Authored-By: Claude Opus 5.5 (1M context) <[email protected]>
This commit is contained in:
osobh
2026-09-26 09:01:54 -05:00
co-authored by Claude Opus 5.5
parent b43bd2e67f
commit f0ecae38b6
9 changed files with 349 additions and 163 deletions
+14 -8
View File
@@ -9,6 +9,7 @@
use std::sync::Arc;
use clawhdf5_format::datatype::Datatype;
use clawhdf5_format::object_header::ObjectHeader;
use pyo3::exceptions::{PyTypeError, PyValueError};
use pyo3::prelude::*;
use pyo3::types::{PyList, PyTuple};
@@ -29,6 +30,9 @@ use crate::{PyEmpty, node, to_py_err};
pub struct PyDataset {
file: Arc<clawhdf5_rs::File>,
path: String,
/// Where the dataset's object header is: reads open it from here rather
/// than resolve `path` again.
addr: u64,
/// `None` for a dataset with a null dataspace (h5py's `Empty`).
shape: Option<Vec<u64>>,
/// The chunk shape, for a chunked dataset.
@@ -43,12 +47,13 @@ impl PyDataset {
py: Python<'_>,
file: Arc<clawhdf5_rs::File>,
path: String,
addr: u64,
hdr: &ObjectHeader,
) -> PyResult<Self> {
crate::no_panic(|| {
let hdr = node::header(&file, &path)?;
let null = node::is_null(&node::dataspace(&file, &hdr)?);
let null = node::is_null(&node::dataspace(&file, hdr)?);
let (shape, datatype) = {
let ds = file.dataset(&path).map_err(to_py_err)?;
let ds = file.dataset_at(addr).map_err(to_py_err)?;
let shape = if null {
None
} else {
@@ -60,10 +65,11 @@ impl PyDataset {
.map_err(|e| e.value(py).to_string());
let chunks = shape
.as_ref()
.and_then(|s| node::chunk_shape(&file, &hdr, s.len()));
.and_then(|s| node::chunk_shape(&file, hdr, s.len()));
Ok(Self {
file,
path,
addr,
shape,
chunks,
datatype,
@@ -96,10 +102,10 @@ impl PyDataset {
let (reads, list_axis) = plan.reads(dims, chunk_len, elem_size);
let read_shape = plan.read_shape();
let file = &*self.file;
let path = self.path.as_str();
let addr = self.addr;
// Everything below touches only Rust data: release the GIL.
let read = || -> Result<Elements, ReadError> {
let ds = file.dataset(path)?;
let ds = file.dataset_at(addr)?;
let mut blocks = Vec::with_capacity(reads.len());
for read in reads {
let raw = ds.read_selection(&read.sel)?;
@@ -250,7 +256,7 @@ impl PyDataset {
};
let max = self
.file
.dataset(&self.path)
.dataset_at(self.addr)
.and_then(|ds| ds.max_dimensions())
.map_err(to_py_err)?
.unwrap_or_else(|| shape.clone());
@@ -288,7 +294,7 @@ impl PyDataset {
/// The dataset's attributes (read-only, dict-like).
#[getter]
fn attrs(&self) -> PyResult<PyAttrs> {
PyAttrs::read(Arc::clone(&self.file), &self.path)
PyAttrs::read(Arc::clone(&self.file), self.addr, &self.path)
}
/// Read with h5py indexing: integers, slices with positive steps,