perf(py): datasets and groups keep their address; groups their links
Every ds[...] and g[k] resolved the path from the root again, two or three times per open, and resolving a name in a large group scans its links: visiting a group was O(n^2). 4000 scalar datasets in one group took 39 s (v1 group) and 131 s (dense) to list, read and re-read; now 0.3 s each. A Dataset keeps its object address, a Group (and the file's root) its address and, after the first lookup, its link table. New facade API File::dataset_at(address), tested in integration_tests. Co-Authored-By: Claude Opus 5.5 (1M context) <[email protected]>
This commit is contained in:
@@ -9,6 +9,7 @@
|
||||
use std::sync::Arc;
|
||||
|
||||
use clawhdf5_format::datatype::Datatype;
|
||||
use clawhdf5_format::object_header::ObjectHeader;
|
||||
use pyo3::exceptions::{PyTypeError, PyValueError};
|
||||
use pyo3::prelude::*;
|
||||
use pyo3::types::{PyList, PyTuple};
|
||||
@@ -29,6 +30,9 @@ use crate::{PyEmpty, node, to_py_err};
|
||||
pub struct PyDataset {
|
||||
file: Arc<clawhdf5_rs::File>,
|
||||
path: String,
|
||||
/// Where the dataset's object header is: reads open it from here rather
|
||||
/// than resolve `path` again.
|
||||
addr: u64,
|
||||
/// `None` for a dataset with a null dataspace (h5py's `Empty`).
|
||||
shape: Option<Vec<u64>>,
|
||||
/// The chunk shape, for a chunked dataset.
|
||||
@@ -43,12 +47,13 @@ impl PyDataset {
|
||||
py: Python<'_>,
|
||||
file: Arc<clawhdf5_rs::File>,
|
||||
path: String,
|
||||
addr: u64,
|
||||
hdr: &ObjectHeader,
|
||||
) -> PyResult<Self> {
|
||||
crate::no_panic(|| {
|
||||
let hdr = node::header(&file, &path)?;
|
||||
let null = node::is_null(&node::dataspace(&file, &hdr)?);
|
||||
let null = node::is_null(&node::dataspace(&file, hdr)?);
|
||||
let (shape, datatype) = {
|
||||
let ds = file.dataset(&path).map_err(to_py_err)?;
|
||||
let ds = file.dataset_at(addr).map_err(to_py_err)?;
|
||||
let shape = if null {
|
||||
None
|
||||
} else {
|
||||
@@ -60,10 +65,11 @@ impl PyDataset {
|
||||
.map_err(|e| e.value(py).to_string());
|
||||
let chunks = shape
|
||||
.as_ref()
|
||||
.and_then(|s| node::chunk_shape(&file, &hdr, s.len()));
|
||||
.and_then(|s| node::chunk_shape(&file, hdr, s.len()));
|
||||
Ok(Self {
|
||||
file,
|
||||
path,
|
||||
addr,
|
||||
shape,
|
||||
chunks,
|
||||
datatype,
|
||||
@@ -96,10 +102,10 @@ impl PyDataset {
|
||||
let (reads, list_axis) = plan.reads(dims, chunk_len, elem_size);
|
||||
let read_shape = plan.read_shape();
|
||||
let file = &*self.file;
|
||||
let path = self.path.as_str();
|
||||
let addr = self.addr;
|
||||
// Everything below touches only Rust data: release the GIL.
|
||||
let read = || -> Result<Elements, ReadError> {
|
||||
let ds = file.dataset(path)?;
|
||||
let ds = file.dataset_at(addr)?;
|
||||
let mut blocks = Vec::with_capacity(reads.len());
|
||||
for read in reads {
|
||||
let raw = ds.read_selection(&read.sel)?;
|
||||
@@ -250,7 +256,7 @@ impl PyDataset {
|
||||
};
|
||||
let max = self
|
||||
.file
|
||||
.dataset(&self.path)
|
||||
.dataset_at(self.addr)
|
||||
.and_then(|ds| ds.max_dimensions())
|
||||
.map_err(to_py_err)?
|
||||
.unwrap_or_else(|| shape.clone());
|
||||
@@ -288,7 +294,7 @@ impl PyDataset {
|
||||
/// The dataset's attributes (read-only, dict-like).
|
||||
#[getter]
|
||||
fn attrs(&self) -> PyResult<PyAttrs> {
|
||||
PyAttrs::read(Arc::clone(&self.file), &self.path)
|
||||
PyAttrs::read(Arc::clone(&self.file), self.addr, &self.path)
|
||||
}
|
||||
|
||||
/// Read with h5py indexing: integers, slices with positive steps,
|
||||
|
||||
Reference in New Issue
Block a user