feat(py): h5py-style reads of only the selected elements, GIL released

ds[key] read the whole dataset and sliced it in numpy, and knew six
dtypes. Keys (ints, positive-step slices, Ellipsis, one increasing index
list, compound field names) now map onto hyperslab selections, and the
facade's read_selection bytes become the numpy buffer without a copy
(PyArray::from_vec viewed as the dtype). dtype mapping follows h5py for
all integer/IEEE float widths and byte orders, bool, enum, complex, fixed
and variable-length strings, vlen sequences, opaque, array types and
(nested, padded) compounds; anything it cannot describe exactly is a
TypeError. Attributes return what h5py returns; groups and files gain
the rest of the h5py mapping interface. Reads run under py.detach.

tests/test_read_vs_h5py.py compares >500 reads with h5py 3.16 on an
h5py-written file, checks errors match, that a damaged chunk outside the
selection is never touched, and 8 threads reading at once.

Co-Authored-By: Claude Opus 5.5 (1M context) <[email protected]>
This commit is contained in:
osobh
2026-09-26 08:19:48 -05:00
co-authored by Claude Opus 5.5
parent 006bf3b131
commit 2d4b211523
12 changed files with 2305 additions and 385 deletions
+54 -2
View File
@@ -10,9 +10,12 @@
//! ```
mod attrs;
mod convert;
mod dataset;
mod file;
mod group;
mod node;
mod select;
use pyo3::prelude::*;
@@ -46,6 +49,54 @@ pub(crate) fn to_py_err(e: clawhdf5_rs::Error) -> PyErr {
}
}
/// The value of a dataset or attribute with a null dataspace: a type but no
/// data. Mirrors `h5py.Empty`.
#[pyclass(name = "Empty", frozen)]
pub struct PyEmpty {
dtype: Py<PyAny>,
}
impl PyEmpty {
pub(crate) fn new(dtype: Py<PyAny>) -> Self {
Self { dtype }
}
}
#[pymethods]
impl PyEmpty {
#[new]
fn py_new(py: Python<'_>, dtype: &Bound<'_, PyAny>) -> PyResult<Self> {
let dtype = py.import("numpy")?.getattr("dtype")?.call1((dtype,))?;
Ok(Self::new(dtype.unbind()))
}
#[getter]
fn dtype(&self, py: Python<'_>) -> Py<PyAny> {
self.dtype.clone_ref(py)
}
#[getter]
fn shape(&self, py: Python<'_>) -> Py<PyAny> {
py.None()
}
#[getter]
fn size(&self, py: Python<'_>) -> Py<PyAny> {
py.None()
}
fn __eq__(&self, py: Python<'_>, other: &Bound<'_, PyAny>) -> PyResult<bool> {
match other.cast::<PyEmpty>() {
Ok(o) => self.dtype.bind(py).eq(o.get().dtype.bind(py)),
Err(_) => Ok(false),
}
}
fn __repr__(&self, py: Python<'_>) -> PyResult<String> {
Ok(format!("Empty(dtype={})", self.dtype.bind(py).repr()?))
}
}
/// The data payload for a dataset being written.
#[derive(Clone)]
pub(crate) enum DatasetData {
@@ -224,6 +275,7 @@ fn clawhdf5(m: &Bound<'_, PyModule>) -> PyResult<()> {
m.add_class::<PyDataset>()?;
m.add_class::<PyGroup>()?;
m.add_class::<PyAttrs>()?;
m.add_class::<PyEmpty>()?;
Ok(())
}
@@ -233,9 +285,9 @@ mod tests {
#[test]
fn owned_attr_value_roundtrip() {
let val = OwnedAttrValue::F64(3.14);
let val = OwnedAttrValue::F64(2.5);
let attr: clawhdf5_rs::AttrValue = val.into();
assert!(matches!(attr, clawhdf5_rs::AttrValue::F64(v) if (v - 3.14).abs() < 1e-10));
assert!(matches!(attr, clawhdf5_rs::AttrValue::F64(v) if (v - 2.5).abs() < 1e-10));
}
#[test]