feat(py): h5py-style reads of only the selected elements, GIL released
ds[key] read the whole dataset and sliced it in numpy, and knew six dtypes. Keys (ints, positive-step slices, Ellipsis, one increasing index list, compound field names) now map onto hyperslab selections, and the facade's read_selection bytes become the numpy buffer without a copy (PyArray::from_vec viewed as the dtype). dtype mapping follows h5py for all integer/IEEE float widths and byte orders, bool, enum, complex, fixed and variable-length strings, vlen sequences, opaque, array types and (nested, padded) compounds; anything it cannot describe exactly is a TypeError. Attributes return what h5py returns; groups and files gain the rest of the h5py mapping interface. Reads run under py.detach. tests/test_read_vs_h5py.py compares >500 reads with h5py 3.16 on an h5py-written file, checks errors match, that a damaged chunk outside the selection is never touched, and 8 threads reading at once. Co-Authored-By: Claude Opus 5.5 (1M context) <[email protected]>
This commit is contained in:
osobh
+301
-201
@@ -1,241 +1,341 @@
|
||||
//! PyDataset — read access to HDF5 datasets with numpy integration.
|
||||
//! PyDataset — h5py-style read access to HDF5 datasets.
|
||||
//!
|
||||
//! `ds[key]` parses the key into hyperslab selections (see `select`) and
|
||||
//! reads only those elements through the facade's `read_selection`; the
|
||||
//! bytes it returns become the numpy array's buffer without a copy (see
|
||||
//! `convert`). All file access and decoding runs with the GIL released, so
|
||||
//! Python threads reading the same or different datasets run in parallel.
|
||||
|
||||
use std::sync::Arc;
|
||||
|
||||
use numpy::PyArrayDyn;
|
||||
use numpy::ndarray::{ArrayD, IxDyn};
|
||||
use clawhdf5_format::datatype::Datatype;
|
||||
use pyo3::exceptions::{PyTypeError, PyValueError};
|
||||
use pyo3::prelude::*;
|
||||
use pyo3::types::PyList;
|
||||
|
||||
use clawhdf5_rs::DType;
|
||||
use pyo3::types::{PyList, PyTuple};
|
||||
|
||||
use crate::attrs::PyAttrs;
|
||||
use crate::to_py_err;
|
||||
use crate::convert::{Converter, Elements, resolve_vl};
|
||||
use crate::select::{self, Plan};
|
||||
use crate::{PyEmpty, node, to_py_err};
|
||||
|
||||
/// A handle to an HDF5 dataset (read mode).
|
||||
/// A dataset in a file opened for reading.
|
||||
///
|
||||
/// Supports numpy-style indexing via `__getitem__`:
|
||||
/// ```python
|
||||
/// ds = f['dataset_name']
|
||||
/// data = ds[:] # read all data as numpy array
|
||||
/// shape = ds.shape
|
||||
/// dtype = ds.dtype
|
||||
/// ds = f['group/dataset']
|
||||
/// ds.shape, ds.dtype, ds.attrs['units']
|
||||
/// block = ds[10:20, ::2] # reads only the selected elements
|
||||
/// ```
|
||||
#[pyclass(name = "Dataset")]
|
||||
pub struct PyDataset {
|
||||
file: Arc<clawhdf5_rs::File>,
|
||||
path: String,
|
||||
cached_shape: Vec<u64>,
|
||||
cached_dtype: DType,
|
||||
/// `None` for a dataset with a null dataspace (h5py's `Empty`).
|
||||
shape: Option<Vec<u64>>,
|
||||
datatype: Datatype,
|
||||
/// Why the datatype cannot be read into numpy, if it cannot.
|
||||
conv: Result<Converter, String>,
|
||||
}
|
||||
|
||||
impl PyDataset {
|
||||
pub fn new(file: Arc<clawhdf5_rs::File>, path: String) -> PyResult<Self> {
|
||||
let ds = file.dataset(&path).map_err(to_py_err)?;
|
||||
let cached_shape = ds.shape().map_err(to_py_err)?;
|
||||
let cached_dtype = ds.dtype().map_err(to_py_err)?;
|
||||
pub(crate) fn open(
|
||||
py: Python<'_>,
|
||||
file: Arc<clawhdf5_rs::File>,
|
||||
path: String,
|
||||
) -> PyResult<Self> {
|
||||
let hdr = node::header(&file, &path)?;
|
||||
let null = node::is_null(&node::dataspace(&file, &hdr)?);
|
||||
let (shape, datatype) = {
|
||||
let ds = file.dataset(&path).map_err(to_py_err)?;
|
||||
let shape = if null {
|
||||
None
|
||||
} else {
|
||||
Some(ds.shape().map_err(to_py_err)?)
|
||||
};
|
||||
(shape, ds.raw_datatype().map_err(to_py_err)?)
|
||||
};
|
||||
let conv = Converter::new(py, &datatype, file.superblock().offset_size)
|
||||
.map_err(|e| e.value(py).to_string());
|
||||
Ok(Self {
|
||||
file,
|
||||
path,
|
||||
cached_shape,
|
||||
cached_dtype,
|
||||
shape,
|
||||
datatype,
|
||||
conv,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
/// Map a `DType` to a numpy dtype string.
|
||||
fn dtype_to_numpy_str(dt: &DType) -> &'static str {
|
||||
match dt {
|
||||
DType::F64 => "float64",
|
||||
DType::F32 => "float32",
|
||||
DType::I64 => "int64",
|
||||
DType::I32 => "int32",
|
||||
DType::I16 => "int16",
|
||||
DType::I8 => "int8",
|
||||
DType::U64 => "uint64",
|
||||
DType::U32 => "uint32",
|
||||
DType::U16 => "uint16",
|
||||
DType::U8 => "uint8",
|
||||
DType::String | DType::VariableLengthString => "object",
|
||||
_ => "object",
|
||||
}
|
||||
}
|
||||
|
||||
#[pymethods]
|
||||
impl PyDataset {
|
||||
/// The shape of the dataset as a tuple.
|
||||
#[getter]
|
||||
fn shape(&self, py: Python<'_>) -> PyResult<Py<PyAny>> {
|
||||
let tuple = pyo3::types::PyTuple::new(py, self.cached_shape.iter().map(|&d| d as usize))?;
|
||||
Ok(tuple.into_any().unbind())
|
||||
fn converter(&self) -> PyResult<&Converter> {
|
||||
self.conv
|
||||
.as_ref()
|
||||
.map_err(|msg| PyTypeError::new_err(format!("{}: {msg}", node::name(&self.path))))
|
||||
}
|
||||
|
||||
/// The numpy dtype string of the dataset.
|
||||
#[getter]
|
||||
fn dtype(&self) -> &'static str {
|
||||
dtype_to_numpy_str(&self.cached_dtype)
|
||||
}
|
||||
/// Read the selection described by `plan` into a numpy array.
|
||||
fn read_plan<'py>(&self, py: Python<'py>, plan: &Plan) -> PyResult<Bound<'py, PyAny>> {
|
||||
let conv = self.converter()?;
|
||||
let dims = self.shape.as_deref().unwrap_or(&[]);
|
||||
let out_shape = plan.out_shape();
|
||||
|
||||
/// Attribute access (read-only).
|
||||
#[getter]
|
||||
fn attrs(&self) -> PyResult<PyAttrs> {
|
||||
let ds = self.file.dataset(&self.path).map_err(to_py_err)?;
|
||||
let map = ds.attrs().map_err(to_py_err)?;
|
||||
Ok(PyAttrs::from_read(map))
|
||||
}
|
||||
|
||||
/// Read data via indexing. Supports `ds[:]`, `ds[0]`, `ds[0:5]`, etc.
|
||||
///
|
||||
/// The full dataset is always read from the underlying file; the index
|
||||
/// is then applied on the resulting numpy array.
|
||||
fn __getitem__<'py>(&self, py: Python<'py>, key: &Bound<'py, PyAny>) -> PyResult<Py<PyAny>> {
|
||||
let arr = self.read_as_numpy(py)?;
|
||||
let indexed = arr.get_item(key)?;
|
||||
Ok(indexed.unbind())
|
||||
}
|
||||
|
||||
fn __repr__(&self) -> String {
|
||||
format!(
|
||||
"<HDF5 Dataset \"{}\": shape {:?}, dtype {}>",
|
||||
self.path,
|
||||
self.cached_shape,
|
||||
dtype_to_numpy_str(&self.cached_dtype),
|
||||
)
|
||||
}
|
||||
|
||||
fn __len__(&self) -> usize {
|
||||
self.cached_shape.first().copied().unwrap_or(0) as usize
|
||||
}
|
||||
}
|
||||
|
||||
impl PyDataset {
|
||||
/// Read the full dataset and return it as a numpy array (or list for strings).
|
||||
///
|
||||
/// For numeric types, the Rust I/O (file reading + decompression) is
|
||||
/// performed inside `py.detach()` so that the GIL is released
|
||||
/// during the potentially expensive operation. The numpy array
|
||||
/// construction still happens with the GIL held.
|
||||
fn read_as_numpy<'py>(&self, py: Python<'py>) -> PyResult<Bound<'py, PyAny>> {
|
||||
let file = &self.file;
|
||||
let path = &self.path;
|
||||
let shape: Vec<usize> = self.cached_shape.iter().map(|&d| d as usize).collect();
|
||||
|
||||
match &self.cached_dtype {
|
||||
DType::F64 => {
|
||||
let data = py
|
||||
.detach(|| file.dataset(path).and_then(|ds| ds.read_f64()))
|
||||
.map_err(to_py_err)?;
|
||||
let nd = ArrayD::from_shape_vec(IxDyn(&shape), data)
|
||||
.map_err(|e| PyErr::new::<pyo3::exceptions::PyValueError, _>(e.to_string()))?;
|
||||
let arr = PyArrayDyn::from_owned_array(py, nd);
|
||||
Ok(arr.into_any())
|
||||
}
|
||||
DType::F32 => {
|
||||
let data = py
|
||||
.detach(|| file.dataset(path).and_then(|ds| ds.read_f32()))
|
||||
.map_err(to_py_err)?;
|
||||
let nd = ArrayD::from_shape_vec(IxDyn(&shape), data)
|
||||
.map_err(|e| PyErr::new::<pyo3::exceptions::PyValueError, _>(e.to_string()))?;
|
||||
let arr = PyArrayDyn::from_owned_array(py, nd);
|
||||
Ok(arr.into_any())
|
||||
}
|
||||
DType::I32 => {
|
||||
let data = py
|
||||
.detach(|| file.dataset(path).and_then(|ds| ds.read_i32()))
|
||||
.map_err(to_py_err)?;
|
||||
let nd = ArrayD::from_shape_vec(IxDyn(&shape), data)
|
||||
.map_err(|e| PyErr::new::<pyo3::exceptions::PyValueError, _>(e.to_string()))?;
|
||||
let arr = PyArrayDyn::from_owned_array(py, nd);
|
||||
Ok(arr.into_any())
|
||||
}
|
||||
DType::I64 => {
|
||||
let data = py
|
||||
.detach(|| file.dataset(path).and_then(|ds| ds.read_i64()))
|
||||
.map_err(to_py_err)?;
|
||||
let nd = ArrayD::from_shape_vec(IxDyn(&shape), data)
|
||||
.map_err(|e| PyErr::new::<pyo3::exceptions::PyValueError, _>(e.to_string()))?;
|
||||
let arr = PyArrayDyn::from_owned_array(py, nd);
|
||||
Ok(arr.into_any())
|
||||
}
|
||||
DType::U8 => {
|
||||
// Try zero-copy first (contiguous layout), fall back to
|
||||
// read_u64 + cast for chunked/compact datasets.
|
||||
let data: Vec<u8> = py
|
||||
.detach(|| {
|
||||
let ds = file.dataset(path)?;
|
||||
match ds.read_u8_zerocopy() {
|
||||
Ok(slice) => Ok(slice.to_vec()),
|
||||
Err(_) => {
|
||||
let raw = ds.read_u64()?;
|
||||
Ok(raw.iter().map(|&v| v as u8).collect())
|
||||
let arr = if plan.is_empty() {
|
||||
conv.empty(py, &out_shape)?
|
||||
} else {
|
||||
let (reads, list_axis) = plan.reads(dims);
|
||||
let file = &*self.file;
|
||||
let path = self.path.as_str();
|
||||
let (vl, elem_size, unit) = (conv.is_vl(), conv.elem_size, conv.vl_unit);
|
||||
// Everything below touches only Rust data: release the GIL.
|
||||
let blocks: Vec<(Elements, Vec<usize>)> = py
|
||||
.detach(|| -> Result<_, ReadError> {
|
||||
let ds = file.dataset(path)?;
|
||||
let sb = file.superblock();
|
||||
let mut blocks = Vec::with_capacity(reads.len());
|
||||
for (sel, shape) in reads {
|
||||
let raw = ds.read_selection(&sel)?;
|
||||
let n: usize = shape.iter().product();
|
||||
let data = if vl {
|
||||
if raw.len() != n * elem_size {
|
||||
return Err(ReadError::Other(format!(
|
||||
"read {} bytes of variable-length references, expected {}",
|
||||
raw.len(),
|
||||
n * elem_size
|
||||
)));
|
||||
}
|
||||
}
|
||||
})
|
||||
.map_err(to_py_err)?;
|
||||
let nd = ArrayD::from_shape_vec(IxDyn(&shape), data)
|
||||
.map_err(|e| PyErr::new::<pyo3::exceptions::PyValueError, _>(e.to_string()))?;
|
||||
let arr = PyArrayDyn::from_owned_array(py, nd);
|
||||
Ok(arr.into_any())
|
||||
Elements::Vl(
|
||||
resolve_vl(
|
||||
file.as_bytes(),
|
||||
&raw,
|
||||
n,
|
||||
sb.offset_size,
|
||||
sb.length_size,
|
||||
unit,
|
||||
)
|
||||
.map_err(ReadError::Other)?,
|
||||
)
|
||||
} else {
|
||||
Elements::Bytes(raw)
|
||||
};
|
||||
blocks.push((data, shape));
|
||||
}
|
||||
Ok(blocks)
|
||||
})
|
||||
.map_err(|e| e.into_py(&self.path))?;
|
||||
|
||||
let mut arrays = Vec::with_capacity(blocks.len());
|
||||
for (data, shape) in blocks {
|
||||
arrays.push(conv.to_array(py, data, &shape, false)?);
|
||||
}
|
||||
DType::U64 => {
|
||||
let data = py
|
||||
.detach(|| file.dataset(path).and_then(|ds| ds.read_u64()))
|
||||
.map_err(to_py_err)?;
|
||||
let nd = ArrayD::from_shape_vec(IxDyn(&shape), data)
|
||||
.map_err(|e| PyErr::new::<pyo3::exceptions::PyValueError, _>(e.to_string()))?;
|
||||
let arr = PyArrayDyn::from_owned_array(py, nd);
|
||||
Ok(arr.into_any())
|
||||
let joined = if arrays.len() == 1 {
|
||||
arrays.pop().expect("one block")
|
||||
} else {
|
||||
let axis = list_axis.expect("several reads only for a list index");
|
||||
// Name the dtype: left to itself numpy canonicalises a
|
||||
// structured dtype here (drops padding, native byte order).
|
||||
let kwargs = pyo3::types::PyDict::new(py);
|
||||
kwargs.set_item("axis", axis)?;
|
||||
kwargs.set_item("dtype", arrays[0].getattr("dtype")?)?;
|
||||
kwargs.set_item("casting", "no")?;
|
||||
py.import("numpy")?.call_method(
|
||||
"concatenate",
|
||||
(PyList::new(py, arrays)?,),
|
||||
Some(&kwargs),
|
||||
)?
|
||||
};
|
||||
// Drop the axes indexed by an integer (length 1 in the blocks).
|
||||
let mut shape = out_shape.clone();
|
||||
if let crate::convert::Layout::Subarray(sub) = &conv.layout {
|
||||
shape.extend_from_slice(sub);
|
||||
}
|
||||
DType::String | DType::VariableLengthString => {
|
||||
// String reads need the GIL for PyList construction, but we
|
||||
// release it during the Rust I/O portion.
|
||||
let data = py
|
||||
.detach(|| file.dataset(path).and_then(|ds| ds.read_string()))
|
||||
.map_err(to_py_err)?;
|
||||
let list = PyList::new(py, &data)?;
|
||||
Ok(list.into_any())
|
||||
}
|
||||
other => Err(PyErr::new::<pyo3::exceptions::PyTypeError, _>(format!(
|
||||
"unsupported dataset dtype for reading: {other}"
|
||||
))),
|
||||
joined.call_method1("reshape", (PyTuple::new(py, shape)?,))?
|
||||
};
|
||||
|
||||
let arr = select_fields(py, arr, &plan.fields)?;
|
||||
if plan.scalar {
|
||||
return arr.get_item(PyTuple::empty(py));
|
||||
}
|
||||
Ok(arr)
|
||||
}
|
||||
}
|
||||
|
||||
/// An error from the read closure, turned into a Python error with the GIL.
|
||||
enum ReadError {
|
||||
Lib(clawhdf5_rs::Error),
|
||||
Other(String),
|
||||
}
|
||||
|
||||
impl From<clawhdf5_rs::Error> for ReadError {
|
||||
fn from(e: clawhdf5_rs::Error) -> Self {
|
||||
ReadError::Lib(e)
|
||||
}
|
||||
}
|
||||
|
||||
impl ReadError {
|
||||
fn into_py(self, path: &str) -> PyErr {
|
||||
match self {
|
||||
ReadError::Lib(e) => to_py_err(e),
|
||||
ReadError::Other(msg) => PyValueError::new_err(format!("{}: {msg}", node::name(path))),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
/// Keep only the named compound fields, as h5py's `ds['x']` / `ds['x', 'y']`.
|
||||
fn select_fields<'py>(
|
||||
py: Python<'py>,
|
||||
arr: Bound<'py, PyAny>,
|
||||
fields: &[String],
|
||||
) -> PyResult<Bound<'py, PyAny>> {
|
||||
if fields.is_empty() {
|
||||
return Ok(arr);
|
||||
}
|
||||
let names = arr.getattr("dtype")?.getattr("names")?;
|
||||
if names.is_none() {
|
||||
return Err(PyValueError::new_err(
|
||||
"Field names only allowed for compound types",
|
||||
));
|
||||
}
|
||||
let names: Vec<String> = names.extract()?;
|
||||
for f in fields {
|
||||
if !names.contains(f) {
|
||||
return Err(PyValueError::new_err(format!(
|
||||
"Field {f} does not appear in this type."
|
||||
)));
|
||||
}
|
||||
}
|
||||
let np = py.import("numpy")?;
|
||||
if let [one] = fields {
|
||||
return np.call_method1("ascontiguousarray", (arr.get_item(one)?,));
|
||||
}
|
||||
let picked = arr.get_item(PyList::new(py, fields)?)?;
|
||||
py.import("numpy.lib.recfunctions")?
|
||||
.call_method1("repack_fields", (picked,))
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn dtype_mapping() {
|
||||
assert_eq!(dtype_to_numpy_str(&DType::F64), "float64");
|
||||
assert_eq!(dtype_to_numpy_str(&DType::F32), "float32");
|
||||
assert_eq!(dtype_to_numpy_str(&DType::I32), "int32");
|
||||
assert_eq!(dtype_to_numpy_str(&DType::I64), "int64");
|
||||
assert_eq!(dtype_to_numpy_str(&DType::U8), "uint8");
|
||||
assert_eq!(dtype_to_numpy_str(&DType::String), "object");
|
||||
#[pymethods]
|
||||
impl PyDataset {
|
||||
/// The shape of the dataset (`None` for an empty/null dataspace).
|
||||
#[getter]
|
||||
fn shape<'py>(&self, py: Python<'py>) -> PyResult<Bound<'py, PyAny>> {
|
||||
match &self.shape {
|
||||
Some(s) => Ok(PyTuple::new(py, s)?.into_any()),
|
||||
None => Ok(py.None().into_bound(py)),
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn dataset_from_file() {
|
||||
let mut b = clawhdf5_rs::FileBuilder::new();
|
||||
b.create_dataset("vals").with_f64_data(&[1.0, 2.0, 3.0]);
|
||||
let bytes = b.finish().unwrap();
|
||||
let file = Arc::new(clawhdf5_rs::File::from_bytes(bytes).unwrap());
|
||||
let ds = PyDataset::new(file, "vals".into()).unwrap();
|
||||
assert_eq!(ds.cached_shape, vec![3]);
|
||||
assert_eq!(ds.cached_dtype, DType::F64);
|
||||
/// The maximum shape (`None` per unlimited dimension), like h5py.
|
||||
#[getter]
|
||||
fn maxshape<'py>(&self, py: Python<'py>) -> PyResult<Bound<'py, PyAny>> {
|
||||
let Some(shape) = &self.shape else {
|
||||
return Ok(py.None().into_bound(py));
|
||||
};
|
||||
let max = self
|
||||
.file
|
||||
.dataset(&self.path)
|
||||
.and_then(|ds| ds.max_dimensions())
|
||||
.map_err(to_py_err)?
|
||||
.unwrap_or_else(|| shape.clone());
|
||||
let items: Vec<Option<u64>> = max
|
||||
.into_iter()
|
||||
.map(|d| (d != u64::MAX).then_some(d))
|
||||
.collect();
|
||||
Ok(PyTuple::new(py, items)?.into_any())
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn dataset_len() {
|
||||
let mut b = clawhdf5_rs::FileBuilder::new();
|
||||
b.create_dataset("data")
|
||||
.with_i32_data(&[10, 20, 30, 40])
|
||||
.with_shape(&[2, 2]);
|
||||
let bytes = b.finish().unwrap();
|
||||
let file = Arc::new(clawhdf5_rs::File::from_bytes(bytes).unwrap());
|
||||
let ds = PyDataset::new(file, "data".into()).unwrap();
|
||||
assert_eq!(ds.__len__(), 2);
|
||||
/// The dataset's numpy dtype, as h5py reports it.
|
||||
#[getter]
|
||||
fn dtype<'py>(&self, py: Python<'py>) -> PyResult<Bound<'py, PyAny>> {
|
||||
Ok(self.converter()?.dtype.bind(py).clone())
|
||||
}
|
||||
|
||||
#[getter]
|
||||
fn ndim(&self) -> usize {
|
||||
self.shape.as_ref().map_or(0, Vec::len)
|
||||
}
|
||||
|
||||
/// Number of elements (`None` for an empty/null dataspace, as h5py).
|
||||
#[getter]
|
||||
fn size(&self) -> Option<u64> {
|
||||
self.shape.as_ref().map(|s| s.iter().product())
|
||||
}
|
||||
|
||||
/// The dataset's full name, e.g. `/group/data`.
|
||||
#[getter]
|
||||
fn name(&self) -> String {
|
||||
node::name(&self.path)
|
||||
}
|
||||
|
||||
/// The dataset's attributes (read-only, dict-like).
|
||||
#[getter]
|
||||
fn attrs(&self) -> PyResult<PyAttrs> {
|
||||
PyAttrs::read(Arc::clone(&self.file), &self.path)
|
||||
}
|
||||
|
||||
/// Read with h5py indexing: integers, slices with positive steps,
|
||||
/// `...`, one increasing list of integers, and compound field names.
|
||||
/// Only the selected elements are read from the file.
|
||||
fn __getitem__<'py>(
|
||||
&self,
|
||||
py: Python<'py>,
|
||||
key: &Bound<'py, PyAny>,
|
||||
) -> PyResult<Bound<'py, PyAny>> {
|
||||
let Some(dims) = &self.shape else {
|
||||
let is_empty_tuple = key.cast::<PyTuple>().is_ok_and(|t| t.is_empty());
|
||||
let is_ellipsis = key.is_instance_of::<pyo3::types::PyEllipsis>();
|
||||
if is_empty_tuple || is_ellipsis {
|
||||
let empty = PyEmpty::new(self.converter()?.dtype.clone_ref(py));
|
||||
return Ok(empty.into_pyobject(py)?.into_any());
|
||||
}
|
||||
return Err(PyValueError::new_err("Empty datasets cannot be sliced"));
|
||||
};
|
||||
let plan = select::parse(key, dims)?;
|
||||
self.read_plan(py, &plan)
|
||||
}
|
||||
|
||||
/// `numpy.asarray(ds)` reads the whole dataset.
|
||||
#[pyo3(signature = (dtype=None, copy=None))]
|
||||
fn __array__<'py>(
|
||||
&self,
|
||||
py: Python<'py>,
|
||||
dtype: Option<&Bound<'py, PyAny>>,
|
||||
copy: Option<bool>,
|
||||
) -> PyResult<Bound<'py, PyAny>> {
|
||||
let _ = copy; // every read is a fresh array
|
||||
let Some(dims) = &self.shape else {
|
||||
return Err(PyValueError::new_err("an empty dataset has no array value"));
|
||||
};
|
||||
let ellipsis = pyo3::types::PyEllipsis::get(py).to_owned().into_any();
|
||||
let plan = select::parse(&ellipsis, dims)?;
|
||||
let arr = self.read_plan(py, &plan)?;
|
||||
match dtype {
|
||||
Some(dt) => arr.call_method1("astype", (dt,)),
|
||||
None => Ok(arr),
|
||||
}
|
||||
}
|
||||
|
||||
fn __len__(&self) -> PyResult<usize> {
|
||||
match self.shape.as_deref() {
|
||||
Some([first, ..]) => Ok(*first as usize),
|
||||
_ => Err(PyTypeError::new_err(
|
||||
"Attempt to take len() of scalar dataset",
|
||||
)),
|
||||
}
|
||||
}
|
||||
|
||||
fn __repr__(&self, py: Python<'_>) -> String {
|
||||
let dtype = match &self.conv {
|
||||
Ok(c) => c
|
||||
.dtype
|
||||
.bind(py)
|
||||
.str()
|
||||
.map(|s| s.to_string())
|
||||
.unwrap_or_default(),
|
||||
Err(_) => format!("{:?}", self.datatype),
|
||||
};
|
||||
let shape = match &self.shape {
|
||||
Some(s) => format!("{s:?}"),
|
||||
None => "None".to_string(),
|
||||
};
|
||||
format!(
|
||||
"<HDF5 dataset \"{}\": shape {shape}, type \"{dtype}\">",
|
||||
node::name(&self.path)
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user