feat(py): h5py-style reads of only the selected elements, GIL released
ds[key] read the whole dataset and sliced it in numpy, and knew six dtypes. Keys (ints, positive-step slices, Ellipsis, one increasing index list, compound field names) now map onto hyperslab selections, and the facade's read_selection bytes become the numpy buffer without a copy (PyArray::from_vec viewed as the dtype). dtype mapping follows h5py for all integer/IEEE float widths and byte orders, bool, enum, complex, fixed and variable-length strings, vlen sequences, opaque, array types and (nested, padded) compounds; anything it cannot describe exactly is a TypeError. Attributes return what h5py returns; groups and files gain the rest of the h5py mapping interface. Reads run under py.detach. tests/test_read_vs_h5py.py compares >500 reads with h5py 3.16 on an h5py-written file, checks errors match, that a damaged chunk outside the selection is never touched, and 8 threads reading at once. Co-Authored-By: Claude Opus 5.5 (1M context) <[email protected]>
This commit is contained in:
@@ -10,9 +10,12 @@
|
||||
//! ```
|
||||
|
||||
mod attrs;
|
||||
mod convert;
|
||||
mod dataset;
|
||||
mod file;
|
||||
mod group;
|
||||
mod node;
|
||||
mod select;
|
||||
|
||||
use pyo3::prelude::*;
|
||||
|
||||
@@ -46,6 +49,54 @@ pub(crate) fn to_py_err(e: clawhdf5_rs::Error) -> PyErr {
|
||||
}
|
||||
}
|
||||
|
||||
/// The value of a dataset or attribute with a null dataspace: a type but no
|
||||
/// data. Mirrors `h5py.Empty`.
|
||||
#[pyclass(name = "Empty", frozen)]
|
||||
pub struct PyEmpty {
|
||||
dtype: Py<PyAny>,
|
||||
}
|
||||
|
||||
impl PyEmpty {
|
||||
pub(crate) fn new(dtype: Py<PyAny>) -> Self {
|
||||
Self { dtype }
|
||||
}
|
||||
}
|
||||
|
||||
#[pymethods]
|
||||
impl PyEmpty {
|
||||
#[new]
|
||||
fn py_new(py: Python<'_>, dtype: &Bound<'_, PyAny>) -> PyResult<Self> {
|
||||
let dtype = py.import("numpy")?.getattr("dtype")?.call1((dtype,))?;
|
||||
Ok(Self::new(dtype.unbind()))
|
||||
}
|
||||
|
||||
#[getter]
|
||||
fn dtype(&self, py: Python<'_>) -> Py<PyAny> {
|
||||
self.dtype.clone_ref(py)
|
||||
}
|
||||
|
||||
#[getter]
|
||||
fn shape(&self, py: Python<'_>) -> Py<PyAny> {
|
||||
py.None()
|
||||
}
|
||||
|
||||
#[getter]
|
||||
fn size(&self, py: Python<'_>) -> Py<PyAny> {
|
||||
py.None()
|
||||
}
|
||||
|
||||
fn __eq__(&self, py: Python<'_>, other: &Bound<'_, PyAny>) -> PyResult<bool> {
|
||||
match other.cast::<PyEmpty>() {
|
||||
Ok(o) => self.dtype.bind(py).eq(o.get().dtype.bind(py)),
|
||||
Err(_) => Ok(false),
|
||||
}
|
||||
}
|
||||
|
||||
fn __repr__(&self, py: Python<'_>) -> PyResult<String> {
|
||||
Ok(format!("Empty(dtype={})", self.dtype.bind(py).repr()?))
|
||||
}
|
||||
}
|
||||
|
||||
/// The data payload for a dataset being written.
|
||||
#[derive(Clone)]
|
||||
pub(crate) enum DatasetData {
|
||||
@@ -224,6 +275,7 @@ fn clawhdf5(m: &Bound<'_, PyModule>) -> PyResult<()> {
|
||||
m.add_class::<PyDataset>()?;
|
||||
m.add_class::<PyGroup>()?;
|
||||
m.add_class::<PyAttrs>()?;
|
||||
m.add_class::<PyEmpty>()?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
@@ -233,9 +285,9 @@ mod tests {
|
||||
|
||||
#[test]
|
||||
fn owned_attr_value_roundtrip() {
|
||||
let val = OwnedAttrValue::F64(3.14);
|
||||
let val = OwnedAttrValue::F64(2.5);
|
||||
let attr: clawhdf5_rs::AttrValue = val.into();
|
||||
assert!(matches!(attr, clawhdf5_rs::AttrValue::F64(v) if (v - 3.14).abs() < 1e-10));
|
||||
assert!(matches!(attr, clawhdf5_rs::AttrValue::F64(v) if (v - 2.5).abs() < 1e-10));
|
||||
}
|
||||
|
||||
#[test]
|
||||
|
||||
Reference in New Issue
Block a user