quantumclaw/clawhdf5
3
1
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 1 Wiki Activity

Fast contiguous and concurrent reads, VL data, nested groups and links, Python bindings #15

Merged
osobh merged 41 commits from feat/p2-perf-coverage into main 2026-09-26 14:57:01 +00:00
Conversation 0 Commits 41 Files Changed 65
+2305 -385
12 changed files with 2305 additions and 385 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Showing only changes of commit 2d4b211523 - Show all commits
CHANGELOG.md
+32
View File
@@ -9,6 +9,38 @@
tests imported `rustyhdf5`, so they failed at collection. Distribution, tests imported `rustyhdf5`, so they failed at collection. Distribution,
module and tests now all say `clawhdf5`, and the module has module and tests now all say `clawhdf5`, and the module has
`__version__`. `__version__`.
- **h5py-style reads that read only what is selected.** `ds[...]` used to
read the whole dataset and slice it in numpy, and knew six dtypes. Now
integers (negative from the end), slices with positive steps, `...`, one
increasing list of integers per key and compound field names map onto the
facade's hyperslab selection (a list becomes one hyperslab per run of
consecutive indices), with h5py's results (numpy scalar for an all-integer
key, 0-d array for `scalar[...]`) and h5py's errors for everything else
(negative steps, `None`, boolean masks, out-of-range indices).
`Dataset.dtype` is the numpy dtype h5py reports, for every integer and
IEEE float width (incl. `float16`) in either byte order, `bool`, enums
(base integer with `metadata['enum']`), complex (`r`/`i` compounds),
fixed strings (`S<n>`), variable-length strings (`object` of `bytes`, as
h5py), variable-length sequences (`object` of arrays), opaque (`V<n>`),
HDF5 array types and compounds (numpy structured, offsets and padding
kept, nested). The bytes the library returns become the numpy array's
buffer without a copy. Types the mapping cannot describe exactly
(references, bitfields, time, non-IEEE floats, integers with padding
bits, variable-length members inside compounds) raise `TypeError` rather
than return guessed data. Attributes come back as h5py returns them
(numpy scalars and arrays with the stored dtype, `str` for
variable-length strings, `numpy.bytes_` for fixed ones — **a change**:
string attributes written by this package are fixed-length and used to
come back as `str` — and `clawhdf5.Empty` for a null dataspace, which
datasets return too). `Group`/`File` gain `get`, `values`, `items`,
iteration, `len`, `name`, absolute and relative paths (`g['/a/b']`,
`g['c/d']`, `f['/']`); `Dataset` gains `ndim`, `size`, `maxshape`,
`name`, `len()` and `numpy.asarray(ds)`. File access and decoding run
with the GIL released, so Python threads read in parallel.
`crates/clawhdf5-py/tests/test_read_vs_h5py.py` compares every read with
h5py 3.16 (HDF5 2.0) on a file h5py writes. One difference is h5py's:
it returns variable-length sequences of big-endian floats unswapped; this
package returns the stored values.
### Plugin filters (2026-09-26) ### Plugin filters (2026-09-26)
- **LZF, bitshuffle, bzip2 and Blosc read and write, in pure Rust.** Files - **LZF, bitshuffle, bzip2 and Blosc read and write, in pure Rust.** Files
crates/clawhdf5-py/src/attrs.rs
+103 -45
View File
@@ -1,24 +1,31 @@
//! PyAttrs — dict-like access to HDF5 attributes. //! PyAttrs — dict-like access to HDF5 attributes.
use std::collections::HashMap;
use std::sync::{Arc, Mutex}; use std::sync::{Arc, Mutex};
use clawhdf5_format::attribute::AttributeMessage;
use pyo3::exceptions::{PyKeyError, PyTypeError, PyValueError};
use pyo3::prelude::*; use pyo3::prelude::*;
use pyo3::types::PyList; use pyo3::types::{PyList, PyTuple};
use crate::{OwnedAttrValue, attr_value_to_py, py_to_attr_value}; use crate::convert::{Converter, Elements, resolve_vl};
use crate::{OwnedAttrValue, PyEmpty, attr_value_to_py, node, py_to_attr_value};
/// Backing storage for attributes. /// Backing storage for attributes.
enum AttrsInner { enum AttrsInner {
/// Read-only attributes from an existing HDF5 object. /// Attributes of an object in a file opened for reading, sorted by name.
Read(HashMap<String, clawhdf5_rs::AttrValue>), Read {
file: Arc<clawhdf5_rs::File>,
attrs: Vec<AttributeMessage>,
},
/// Writable attribute list shared with a parent (PyFile or PyGroup). /// Writable attribute list shared with a parent (PyFile or PyGroup).
Write(Arc<Mutex<Vec<(String, OwnedAttrValue)>>>), Write(Arc<Mutex<Vec<(String, OwnedAttrValue)>>>),
} }
/// Dict-like access to HDF5 attributes. /// Dict-like access to HDF5 attributes.
/// ///
/// In read mode, provides immutable access to attribute key/value pairs. /// In read mode, values are what h5py returns: numpy scalars for scalar
/// attributes, numpy arrays otherwise, `str` for variable-length strings,
/// `numpy.bytes_` for fixed-length ones, and `Empty` for a null dataspace.
/// In write mode, attributes set here are accumulated and written when /// In write mode, attributes set here are accumulated and written when
/// the parent file is closed. /// the parent file is closed.
#[pyclass(name = "Attrs")] #[pyclass(name = "Attrs")]
@@ -27,11 +34,12 @@ pub struct PyAttrs {
} }
impl PyAttrs { impl PyAttrs {
/// Create a read-only attrs from an existing attribute map. /// The attributes of the object at `path` in a file opened for reading.
pub(crate) fn from_read(map: HashMap<String, clawhdf5_rs::AttrValue>) -> Self { pub(crate) fn read(file: Arc<clawhdf5_rs::File>, path: &str) -> PyResult<Self> {
Self { let attrs = node::attributes(&file, path)?;
inner: AttrsInner::Read(map), Ok(Self {
} inner: AttrsInner::Read { file, attrs },
})
} }
/// Create a writable attrs that shares storage with a parent object. /// Create a writable attrs that shares storage with a parent object.
@@ -46,11 +54,11 @@ impl PyAttrs {
impl PyAttrs { impl PyAttrs {
fn __getitem__(&self, py: Python<'_>, key: &str) -> PyResult<Py<PyAny>> { fn __getitem__(&self, py: Python<'_>, key: &str) -> PyResult<Py<PyAny>> {
match &self.inner { match &self.inner {
AttrsInner::Read(map) => match map.get(key) { AttrsInner::Read { file, attrs } => match attrs.iter().find(|a| a.name == key) {
Some(val) => Ok(attr_value_to_py(py, val)), Some(attr) => Ok(attr_to_py(py, file, attr)?.unbind()),
None => Err(PyErr::new::<pyo3::exceptions::PyKeyError, _>( None => Err(PyKeyError::new_err(format!(
key.to_string(), "Can't open attribute (can't locate attribute: '{key}')"
)), ))),
}, },
AttrsInner::Write(store) => { AttrsInner::Write(store) => {
let guard = store.lock().unwrap(); let guard = store.lock().unwrap();
@@ -60,16 +68,14 @@ impl PyAttrs {
return Ok(attr_value_to_py(py, &attr_val)); return Ok(attr_value_to_py(py, &attr_val));
} }
} }
Err(PyErr::new::<pyo3::exceptions::PyKeyError, _>( Err(PyKeyError::new_err(key.to_string()))
key.to_string(),
))
} }
} }
} }
fn __setitem__(&self, key: &str, value: &Bound<'_, PyAny>) -> PyResult<()> { fn __setitem__(&self, key: &str, value: &Bound<'_, PyAny>) -> PyResult<()> {
match &self.inner { match &self.inner {
AttrsInner::Read(_) => Err(PyErr::new::<pyo3::exceptions::PyIOError, _>( AttrsInner::Read { .. } => Err(PyErr::new::<pyo3::exceptions::PyIOError, _>(
"cannot set attributes on a read-only file", "cannot set attributes on a read-only file",
)), )),
AttrsInner::Write(store) => { AttrsInner::Write(store) => {
@@ -88,14 +94,14 @@ impl PyAttrs {
fn __len__(&self) -> usize { fn __len__(&self) -> usize {
match &self.inner { match &self.inner {
AttrsInner::Read(map) => map.len(), AttrsInner::Read { attrs, .. } => attrs.len(),
AttrsInner::Write(store) => store.lock().unwrap().len(), AttrsInner::Write(store) => store.lock().unwrap().len(),
} }
} }
fn __contains__(&self, key: &str) -> bool { fn __contains__(&self, key: &str) -> bool {
match &self.inner { match &self.inner {
AttrsInner::Read(map) => map.contains_key(key), AttrsInner::Read { attrs, .. } => attrs.iter().any(|a| a.name == key),
AttrsInner::Write(store) => store.lock().unwrap().iter().any(|(k, _)| k == key), AttrsInner::Write(store) => store.lock().unwrap().iter().any(|(k, _)| k == key),
} }
} }
@@ -111,10 +117,20 @@ impl PyAttrs {
format!("<HDF5 Attrs ({n} members)>") format!("<HDF5 Attrs ({n} members)>")
} }
/// The value of `key`, or `default` if there is no such attribute.
#[pyo3(signature = (key, default=None))]
fn get(&self, py: Python<'_>, key: &str, default: Option<Py<PyAny>>) -> PyResult<Py<PyAny>> {
if self.__contains__(key) {
self.__getitem__(py, key)
} else {
Ok(default.unwrap_or_else(|| py.None()))
}
}
/// Return attribute names as a list. /// Return attribute names as a list.
fn keys(&self, py: Python<'_>) -> PyResult<Py<PyAny>> { fn keys(&self, py: Python<'_>) -> PyResult<Py<PyAny>> {
let names: Vec<String> = match &self.inner { let names: Vec<String> = match &self.inner {
AttrsInner::Read(map) => map.keys().cloned().collect(), AttrsInner::Read { attrs, .. } => attrs.iter().map(|a| a.name.clone()).collect(),
AttrsInner::Write(store) => store AttrsInner::Write(store) => store
.lock() .lock()
.unwrap() .unwrap()
@@ -129,7 +145,10 @@ impl PyAttrs {
/// Return attribute values as a list. /// Return attribute values as a list.
fn values(&self, py: Python<'_>) -> PyResult<Py<PyAny>> { fn values(&self, py: Python<'_>) -> PyResult<Py<PyAny>> {
let vals: Vec<Py<PyAny>> = match &self.inner { let vals: Vec<Py<PyAny>> = match &self.inner {
AttrsInner::Read(map) => map.values().map(|v| attr_value_to_py(py, v)).collect(), AttrsInner::Read { file, attrs } => attrs
.iter()
.map(|a| attr_to_py(py, file, a).map(Bound::unbind))
.collect::<PyResult<_>>()?,
AttrsInner::Write(store) => store AttrsInner::Write(store) => store
.lock() .lock()
.unwrap() .unwrap()
@@ -147,10 +166,10 @@ impl PyAttrs {
/// Return attribute (key, value) pairs as a list of tuples. /// Return attribute (key, value) pairs as a list of tuples.
fn items(&self, py: Python<'_>) -> PyResult<Py<PyAny>> { fn items(&self, py: Python<'_>) -> PyResult<Py<PyAny>> {
let pairs: Vec<(String, Py<PyAny>)> = match &self.inner { let pairs: Vec<(String, Py<PyAny>)> = match &self.inner {
AttrsInner::Read(map) => map AttrsInner::Read { file, attrs } => attrs
.iter() .iter()
.map(|(k, v)| (k.clone(), attr_value_to_py(py, v))) .map(|a| Ok((a.name.clone(), attr_to_py(py, file, a)?.unbind())))
.collect(), .collect::<PyResult<_>>()?,
AttrsInner::Write(store) => store AttrsInner::Write(store) => store
.lock() .lock()
.unwrap() .unwrap()
@@ -166,28 +185,67 @@ impl PyAttrs {
} }
} }
/// An attribute's value as h5py returns it.
fn attr_to_py<'py>(
py: Python<'py>,
file: &clawhdf5_rs::File,
attr: &AttributeMessage,
) -> PyResult<Bound<'py, PyAny>> {
let sb = file.superblock();
let conv = Converter::new(py, &attr.datatype, sb.offset_size)
.map_err(|e| prefix_err(py, &attr.name, e))?;
if node::is_null(&attr.dataspace) {
return Ok(PyEmpty::new(conv.dtype).into_pyobject(py)?.into_any());
}
let shape: Vec<usize> = attr
.dataspace
.dimensions
.iter()
.map(|&d| d as usize)
.collect();
let n: usize = shape.iter().product();
let data = if conv.is_vl() {
let want = n * conv.elem_size;
if attr.raw_data.len() < want {
return Err(PyValueError::new_err(format!(
"attribute {}: {} bytes of variable-length references, expected {want}",
attr.name,
attr.raw_data.len(),
)));
}
let raw = &attr.raw_data[..want];
let file_data = file.as_bytes();
let (osz, lsz, unit) = (sb.offset_size, sb.length_size, conv.vl_unit);
Elements::Vl(
py.detach(|| resolve_vl(file_data, raw, n, osz, lsz, unit))
.map_err(|e| PyValueError::new_err(format!("attribute {}: {e}", attr.name)))?,
)
} else {
Elements::Bytes(attr.raw_data.clone())
};
let arr = conv
.to_array(py, data, &shape, true)
.map_err(|e| prefix_err(py, &attr.name, e))?;
if shape.is_empty() {
// A scalar dataspace: h5py returns the element itself.
return arr.get_item(PyTuple::empty(py));
}
Ok(arr)
}
fn prefix_err(py: Python<'_>, name: &str, e: PyErr) -> PyErr {
let msg = format!("attribute {name}: {}", e.value(py));
if e.is_instance_of::<PyTypeError>(py) {
PyTypeError::new_err(msg)
} else {
PyValueError::new_err(msg)
}
}
#[cfg(test)] #[cfg(test)]
mod tests { mod tests {
use super::*; use super::*;
#[test]
fn read_attrs_len() {
let mut map = HashMap::new();
map.insert("a".into(), clawhdf5_rs::AttrValue::I64(1));
map.insert("b".into(), clawhdf5_rs::AttrValue::F64(2.0));
let attrs = PyAttrs::from_read(map);
assert_eq!(attrs.__len__(), 2);
}
#[test]
fn read_attrs_contains() {
let mut map = HashMap::new();
map.insert("x".into(), clawhdf5_rs::AttrValue::String("hello".into()));
let attrs = PyAttrs::from_read(map);
assert!(attrs.__contains__("x"));
assert!(!attrs.__contains__("y"));
}
#[test] #[test]
fn write_attrs_len() { fn write_attrs_len() {
let store = Arc::new(Mutex::new(Vec::new())); let store = Arc::new(Mutex::new(Vec::new()));
crates/clawhdf5-py/src/convert.rs
+600
View File
@@ -0,0 +1,600 @@
//! HDF5 datatypes as numpy dtypes, and element bytes as numpy arrays.
//!
//! The dtype a file's datatype maps to is the one h5py reports for it
//! (byte order kept, compound offsets and padding kept, `r`/`i` compounds as
//! complex, the `FALSE`/`TRUE` enum as `bool`, fixed strings as `S<n>`,
//! variable-length data as `object`). For every fixed-size type that dtype
//! describes the file's element bytes exactly, so the bytes the library
//! returns become the array's buffer as they are: the `Vec<u8>` is handed to
//! numpy without a copy and viewed as the dtype.
//!
//! Anything this mapping cannot describe exactly — non-IEEE floats, integers
//! with padding bits, VAX byte order, references, bitfields, time, and
//! variable-length members inside compounds or arrays — is a `TypeError`,
//! never a best-effort guess.
use std::collections::HashMap;
use clawhdf5_format::datatype::{CharacterSet, Datatype, DatatypeByteOrder};
use clawhdf5_format::global_heap::GlobalHeapCollection;
use numpy::PyArray1;
use pyo3::exceptions::{PyTypeError, PyValueError};
use pyo3::prelude::*;
use pyo3::types::{PyBytes, PyDict, PyList, PyString, PyTuple};
/// How the elements of a datatype become Python values.
#[derive(Clone, Debug, PartialEq)]
pub(crate) enum Layout {
/// Fixed-size elements numpy reads as they are.
Fixed,
/// A top-level HDF5 array type: elements are viewed as the base dtype and
/// the array's dimensions are appended to the shape (as h5py does).
Subarray(Vec<usize>),
/// Variable-length string: a global heap reference per element.
VlString { utf8: bool },
/// Variable-length sequence of a fixed-size base type.
VlSequence,
}
/// Everything needed to turn a dataset's or attribute's bytes into numpy.
pub(crate) struct Converter {
/// The dtype reported to Python (`Dataset.dtype`).
pub dtype: Py<PyAny>,
/// The dtype the element bytes are viewed as: `dtype` itself, the base
/// of a subarray, or the base of a variable-length sequence.
pub view: Py<PyAny>,
pub layout: Layout,
/// Bytes per element in the raw buffer the library returns.
pub elem_size: usize,
/// For variable-length data, bytes per unit of an element's stored
/// length: 1 for strings, the base type's size for sequences.
pub vl_unit: usize,
}
fn unsupported(what: impl std::fmt::Display) -> PyErr {
PyTypeError::new_err(format!(
"clawhdf5 cannot read this datatype into numpy: {what}"
))
}
fn byte_order_char(order: &DatatypeByteOrder, size: u32) -> PyResult<&'static str> {
if size == 1 {
return Ok("|");
}
match order {
DatatypeByteOrder::LittleEndian => Ok("<"),
DatatypeByteOrder::BigEndian => Ok(">"),
DatatypeByteOrder::Vax => Err(unsupported("VAX byte order")),
}
}
/// The numpy format string of an integer type, if it is a plain one.
fn int_format(dt: &Datatype) -> PyResult<String> {
match dt {
Datatype::FixedPoint {
size,
byte_order,
signed,
bit_offset,
bit_precision,
} => {
if !matches!(size, 1 | 2 | 4 | 8) {
return Err(unsupported(format!("{size}-byte integer")));
}
if *bit_offset != 0 || u32::from(*bit_precision) != size * 8 {
return Err(unsupported(format!(
"integer with {bit_precision} significant bits at offset {bit_offset} in {size} bytes"
)));
}
let kind = if *signed { 'i' } else { 'u' };
Ok(format!(
"{}{kind}{size}",
byte_order_char(byte_order, *size)?
))
}
other => Err(unsupported(format!("{other:?} is not an integer"))),
}
}
/// The numpy format string of an IEEE 754 binary16/32/64 type.
fn float_format(dt: &Datatype) -> PyResult<String> {
let Datatype::FloatingPoint {
size,
byte_order,
bit_offset,
bit_precision,
exponent_location,
exponent_size,
mantissa_location,
mantissa_size,
exponent_bias,
} = dt
else {
return Err(unsupported(format!("{dt:?} is not a float")));
};
// (exponent location, exponent size, mantissa size, bias) of IEEE 754.
let ieee = match size {
2 => (10, 5, 10, 15),
4 => (23, 8, 23, 127),
8 => (52, 11, 52, 1023),
_ => return Err(unsupported(format!("{size}-byte float"))),
};
let layout = (
*exponent_location,
*exponent_size,
*mantissa_size,
*exponent_bias,
);
if *bit_offset != 0
|| u32::from(*bit_precision) != size * 8
|| *mantissa_location != 0
|| layout != ieee
{
return Err(unsupported(format!(
"non-IEEE {size}-byte float (exponent {exponent_size} bits at {exponent_location}, \
mantissa {mantissa_size} bits at {mantissa_location}, bias {exponent_bias})"
)));
}
Ok(format!("{}f{size}", byte_order_char(byte_order, *size)?))
}
/// `r`/`i` compounds of two identical IEEE floats are complex numbers in h5py.
fn complex_format(
size: u32,
members: &[clawhdf5_format::datatype::CompoundMember],
) -> Option<String> {
let [re, im] = members else { return None };
if re.name != "r" || im.name != "i" || re.datatype != im.datatype {
return None;
}
let Datatype::FloatingPoint {
size: fsize,
byte_order,
..
} = &re.datatype
else {
return None;
};
if !matches!(fsize, 4 | 8)
|| re.byte_offset != 0
|| im.byte_offset != u64::from(*fsize)
|| size != 2 * fsize
{
return None;
}
float_format(&re.datatype).ok()?;
let order = byte_order_char(byte_order, *fsize).ok()?;
Some(format!("{order}c{}", 2 * fsize))
}
/// The members of an enum as `{name: value}`.
fn enum_members<'py>(
py: Python<'py>,
base: &Datatype,
members: &[clawhdf5_format::datatype::EnumMember],
) -> PyResult<Bound<'py, PyDict>> {
let signed = matches!(base, Datatype::FixedPoint { signed: true, .. });
let dict = PyDict::new(py);
for m in members {
let value: Py<PyAny> = if signed {
let v = clawhdf5_format::data_read::read_as_i64(&m.value, base)
.map_err(|e| PyValueError::new_err(format!("enum member {}: {e}", m.name)))?;
let v = *v.first().ok_or_else(|| {
PyValueError::new_err(format!("enum member {} has no value", m.name))
})?;
v.into_pyobject(py)?.into_any().unbind()
} else {
let v = clawhdf5_format::data_read::read_as_u64(&m.value, base)
.map_err(|e| PyValueError::new_err(format!("enum member {}: {e}", m.name)))?;
let v = *v.first().ok_or_else(|| {
PyValueError::new_err(format!("enum member {} has no value", m.name))
})?;
v.into_pyobject(py)?.into_any().unbind()
};
dict.set_item(&m.name, value)?;
}
Ok(dict)
}
/// Whether an enum is h5py's boolean: a one-byte integer with exactly the
/// members `FALSE` = 0 and `TRUE` = 1.
fn is_h5py_bool(base: &Datatype, members: &[clawhdf5_format::datatype::EnumMember]) -> bool {
if base.type_size() != 1 || members.len() != 2 {
return false;
}
let value = |name: &str| {
members
.iter()
.find(|m| m.name == name)
.and_then(|m| m.value.first().copied())
};
value("FALSE") == Some(0) && value("TRUE") == Some(1)
}
fn np_dtype<'py>(py: Python<'py>, spec: impl IntoPyObject<'py>) -> PyResult<Bound<'py, PyAny>> {
py.import("numpy")?.getattr("dtype")?.call1((spec,))
}
fn np_dtype_with_metadata<'py>(
py: Python<'py>,
spec: impl IntoPyObject<'py>,
metadata: Bound<'py, PyDict>,
) -> PyResult<Bound<'py, PyAny>> {
let kwargs = PyDict::new(py);
kwargs.set_item("metadata", metadata)?;
py.import("numpy")?
.getattr("dtype")?
.call((spec,), Some(&kwargs))
}
/// The numpy dtype of a fixed-size datatype, whose element bytes numpy can
/// read as they are.
pub(crate) fn fixed_dtype<'py>(py: Python<'py>, dt: &Datatype) -> PyResult<Bound<'py, PyAny>> {
match dt {
Datatype::FixedPoint { .. } => np_dtype(py, int_format(dt)?),
Datatype::FloatingPoint { .. } => np_dtype(py, float_format(dt)?),
Datatype::String { size, charset, .. } => {
if *size == 0 {
return Err(unsupported("zero-length fixed string"));
}
let meta = PyDict::new(py);
let enc = match charset {
CharacterSet::Ascii => "ascii",
CharacterSet::Utf8 => "utf-8",
};
meta.set_item("h5py_encoding", enc)?;
np_dtype_with_metadata(py, format!("S{size}"), meta)
}
Datatype::Opaque { size, .. } => {
if *size == 0 {
return Err(unsupported("zero-length opaque type"));
}
np_dtype(py, format!("V{size}"))
}
Datatype::Enumeration {
base_type, members, ..
} => {
let base = int_format(base_type)?;
if is_h5py_bool(base_type, members) {
return np_dtype(py, "?");
}
let meta = PyDict::new(py);
meta.set_item("enum", enum_members(py, base_type, members)?)?;
np_dtype_with_metadata(py, base, meta)
}
Datatype::Compound { size, members } => {
if let Some(c) = complex_format(*size, members) {
return np_dtype(py, c);
}
let names = PyList::empty(py);
let formats = PyList::empty(py);
let offsets = PyList::empty(py);
for m in members {
let end = m.byte_offset.checked_add(u64::from(m.datatype.type_size()));
if end.is_none_or(|end| end > u64::from(*size)) {
return Err(PyValueError::new_err(format!(
"compound member {} lies outside the {size}-byte compound",
m.name
)));
}
names.append(&m.name)?;
formats.append(fixed_dtype(py, &m.datatype).map_err(|e| {
unsupported(format!("compound member {}: {}", m.name, e.value(py)))
})?)?;
offsets.append(m.byte_offset)?;
}
let spec = PyDict::new(py);
spec.set_item("names", names)?;
spec.set_item("formats", formats)?;
spec.set_item("offsets", offsets)?;
spec.set_item("itemsize", size)?;
np_dtype(py, spec)
}
Datatype::Array {
base_type,
dimensions,
} => {
let base = fixed_dtype(py, base_type)?;
let dims = PyTuple::new(py, dimensions)?;
np_dtype(py, (base, dims))
}
Datatype::VariableLength { is_string, .. } => Err(unsupported(if *is_string {
"variable-length string inside a compound or array type"
} else {
"variable-length sequence inside a compound or array type"
})),
Datatype::Reference { .. } => Err(unsupported("object/region references")),
Datatype::BitField { .. } => Err(unsupported("bitfield")),
Datatype::Time { .. } => Err(unsupported("time")),
}
}
impl Converter {
/// The converter for a dataset's or attribute's datatype.
pub(crate) fn new(py: Python<'_>, dt: &Datatype, offset_size: u8) -> PyResult<Self> {
match dt {
Datatype::VariableLength {
is_string: true,
charset,
..
} => {
let utf8 = matches!(charset, Some(CharacterSet::Utf8));
let meta = PyDict::new(py);
if utf8 {
meta.set_item("vlen", py.get_type::<PyString>())?;
} else {
meta.set_item("vlen", py.get_type::<PyBytes>())?;
}
let dtype = np_dtype_with_metadata(py, "O", meta)?;
Ok(Self {
view: dtype.clone().unbind(),
dtype: dtype.unbind(),
layout: Layout::VlString { utf8 },
elem_size: vl_ref_size(offset_size)?,
vl_unit: 1,
})
}
Datatype::VariableLength {
is_string: false,
base_type,
..
} => {
let base = fixed_dtype(py, base_type)?;
let meta = PyDict::new(py);
meta.set_item("vlen", &base)?;
let dtype = np_dtype_with_metadata(py, "O", meta)?;
Ok(Self {
dtype: dtype.unbind(),
view: base.unbind(),
layout: Layout::VlSequence,
elem_size: vl_ref_size(offset_size)?,
vl_unit: base_type.type_size() as usize,
})
}
Datatype::Array {
base_type,
dimensions,
} => {
let dtype = fixed_dtype(py, dt)?;
let base = fixed_dtype(py, base_type)?;
Ok(Self {
dtype: dtype.unbind(),
view: base.unbind(),
layout: Layout::Subarray(dimensions.iter().map(|&d| d as usize).collect()),
elem_size: dt.type_size() as usize,
vl_unit: 0,
})
}
_ => {
let dtype = fixed_dtype(py, dt)?;
Ok(Self {
view: dtype.clone().unbind(),
dtype: dtype.unbind(),
layout: Layout::Fixed,
elem_size: dt.type_size() as usize,
vl_unit: 0,
})
}
}
}
pub(crate) fn is_vl(&self) -> bool {
matches!(self.layout, Layout::VlString { .. } | Layout::VlSequence)
}
/// An empty array of `shape` (some dimension is zero).
pub(crate) fn empty<'py>(
&self,
py: Python<'py>,
shape: &[usize],
) -> PyResult<Bound<'py, PyAny>> {
let np = py.import("numpy")?;
match &self.layout {
Layout::Subarray(dims) => {
let mut full = shape.to_vec();
full.extend_from_slice(dims);
np.call_method1("empty", (PyTuple::new(py, full)?, self.view.bind(py)))
}
_ => np.call_method1("empty", (PyTuple::new(py, shape)?, self.dtype.bind(py))),
}
}
/// Turn decoded element data into a numpy array of `shape`.
///
/// `str_values` decodes variable-length strings to `str` (what h5py
/// does for attributes) instead of `bytes` (what it does for datasets).
pub(crate) fn to_array<'py>(
&self,
py: Python<'py>,
data: Elements,
shape: &[usize],
str_values: bool,
) -> PyResult<Bound<'py, PyAny>> {
let n: usize = shape.iter().product();
match (data, &self.layout) {
(Elements::Bytes(bytes), Layout::Fixed) => {
bytes_as_array(py, bytes, self.view.bind(py), shape)
}
(Elements::Bytes(bytes), Layout::Subarray(dims)) => {
let mut full = shape.to_vec();
full.extend_from_slice(dims);
bytes_as_array(py, bytes, self.view.bind(py), &full)
}
(Elements::Vl(items), Layout::VlString { .. }) => {
check_count(items.len(), n)?;
let mut objs: Vec<Py<PyAny>> = Vec::with_capacity(items.len());
for item in items {
let obj = if str_values {
PyBytes::new(py, &item)
.call_method1("decode", ("utf-8", "surrogateescape"))?
.unbind()
} else {
PyBytes::new(py, &item).into_any().unbind()
};
objs.push(obj);
}
object_array(py, objs, shape)
}
(Elements::Vl(items), Layout::VlSequence) => {
check_count(items.len(), n)?;
let base = self.view.bind(py);
let itemsize: usize = base.getattr("itemsize")?.extract()?;
let mut objs: Vec<Py<PyAny>> = Vec::with_capacity(items.len());
for item in items {
if item.len() % itemsize != 0 {
return Err(PyValueError::new_err(format!(
"variable-length element of {} bytes is not a whole number of {itemsize}-byte values",
item.len()
)));
}
let len = item.len() / itemsize;
objs.push(bytes_as_array(py, item, base, &[len])?.unbind());
}
object_array(py, objs, shape)
}
_ => Err(PyValueError::new_err(
"internal error: element data does not match the datatype",
)),
}
}
}
/// Element data as read, before it becomes numpy.
pub(crate) enum Elements {
/// The elements' bytes, back to back.
Bytes(Vec<u8>),
/// Each variable-length element's bytes, resolved from the global heap.
Vl(Vec<Vec<u8>>),
}
fn vl_ref_size(offset_size: u8) -> PyResult<usize> {
// The library sizes a variable-length element as 16 bytes (a length, an
// 8-byte heap address and an index) whatever the file's offset size.
// Refuse the other sizes rather than read misaligned references.
if offset_size != 8 {
return Err(unsupported(format!(
"variable-length data in a file with {offset_size}-byte offsets"
)));
}
Ok(4 + usize::from(offset_size) + 4)
}
fn check_count(got: usize, want: usize) -> PyResult<()> {
if got != want {
return Err(PyValueError::new_err(format!(
"read {got} elements, expected {want}"
)));
}
Ok(())
}
/// A numpy array over `bytes` without copying them: the `Vec` becomes the
/// array's buffer and is viewed as `dtype` with `shape`.
pub(crate) fn bytes_as_array<'py>(
py: Python<'py>,
bytes: Vec<u8>,
dtype: &Bound<'py, PyAny>,
shape: &[usize],
) -> PyResult<Bound<'py, PyAny>> {
let itemsize: usize = dtype.getattr("itemsize")?.extract()?;
let n: usize = shape.iter().product();
if n.checked_mul(itemsize) != Some(bytes.len()) {
return Err(PyValueError::new_err(format!(
"read {} bytes, expected {n} elements of {itemsize} bytes",
bytes.len()
)));
}
let shape = PyTuple::new(py, shape)?;
if n == 0 {
return py.import("numpy")?.call_method1("empty", (shape, dtype));
}
let raw = PyArray1::from_vec(py, bytes);
let arr = raw
.call_method1("view", (dtype,))?
.call_method1("reshape", (shape,))?;
// A `Vec<u8>` carries no alignment promise. numpy copes with unaligned
// arrays, but slowly and not in every routine, so hand out an aligned
// copy in the (allocator-dependent, rare) case the buffer is not.
if !arr
.getattr("flags")?
.getattr("aligned")?
.extract::<bool>()?
{
return arr.call_method0("copy");
}
Ok(arr)
}
fn object_array<'py>(
py: Python<'py>,
objs: Vec<Py<PyAny>>,
shape: &[usize],
) -> PyResult<Bound<'py, PyAny>> {
let arr = PyArray1::from_vec(py, objs);
arr.call_method1("reshape", (PyTuple::new(py, shape)?,))
}
/// Resolve variable-length elements (global heap references in `raw`) to
/// their bytes: each element's stored length times `unit` (1 for strings,
/// the base type's size for sequences). Pure Rust, so it runs without the
/// GIL.
pub(crate) fn resolve_vl(
file_data: &[u8],
raw: &[u8],
count: usize,
offset_size: u8,
length_size: u8,
unit: usize,
) -> Result<Vec<Vec<u8>>, String> {
let refs = clawhdf5_format::vl_data::parse_vl_references(raw, count as u64, offset_size)
.map_err(|e| e.to_string())?;
let undefined = match offset_size {
2 => 0xFFFF,
4 => 0xFFFF_FFFF,
_ => u64::MAX,
};
let mut collections: HashMap<u64, GlobalHeapCollection> = HashMap::new();
let mut out = Vec::with_capacity(refs.len());
for vl in &refs {
if vl.collection_address == 0 || vl.collection_address == undefined {
if vl.length != 0 {
return Err(format!(
"variable-length element of length {} has no heap address",
vl.length
));
}
out.push(Vec::new());
continue;
}
let coll = match collections.entry(vl.collection_address) {
std::collections::hash_map::Entry::Occupied(e) => e.into_mut(),
std::collections::hash_map::Entry::Vacant(e) => {
let addr = usize::try_from(vl.collection_address)
.map_err(|_| "global heap address out of range".to_string())?;
e.insert(
GlobalHeapCollection::parse(file_data, addr, length_size)
.map_err(|e| e.to_string())?,
)
}
};
let index = u16::try_from(vl.object_index)
.map_err(|_| format!("global heap object index {} out of range", vl.object_index))?;
let obj = coll.get_object(index).ok_or_else(|| {
format!(
"global heap object {index} not found in the collection at {}",
vl.collection_address
)
})?;
let need = (vl.length as usize)
.checked_mul(unit)
.ok_or("variable-length element too long")?;
if need > obj.data.len() {
return Err(format!(
"variable-length element of {need} bytes in a {}-byte heap object",
obj.data.len()
));
}
out.push(obj.data[..need].to_vec());
}
Ok(out)
}
crates/clawhdf5-py/src/dataset.rs
+295 -195
View File
@@ -1,241 +1,341 @@
//! PyDataset — read access to HDF5 datasets with numpy integration. //! PyDataset — h5py-style read access to HDF5 datasets.
//!
//! `ds[key]` parses the key into hyperslab selections (see `select`) and
//! reads only those elements through the facade's `read_selection`; the
//! bytes it returns become the numpy array's buffer without a copy (see
//! `convert`). All file access and decoding runs with the GIL released, so
//! Python threads reading the same or different datasets run in parallel.
use std::sync::Arc; use std::sync::Arc;
use numpy::PyArrayDyn; use clawhdf5_format::datatype::Datatype;
use numpy::ndarray::{ArrayD, IxDyn}; use pyo3::exceptions::{PyTypeError, PyValueError};
use pyo3::prelude::*; use pyo3::prelude::*;
use pyo3::types::PyList; use pyo3::types::{PyList, PyTuple};
use clawhdf5_rs::DType;
use crate::attrs::PyAttrs; use crate::attrs::PyAttrs;
use crate::to_py_err; use crate::convert::{Converter, Elements, resolve_vl};
use crate::select::{self, Plan};
use crate::{PyEmpty, node, to_py_err};
/// A handle to an HDF5 dataset (read mode). /// A dataset in a file opened for reading.
/// ///
/// Supports numpy-style indexing via `__getitem__`:
/// ```python /// ```python
/// ds = f['dataset_name'] /// ds = f['group/dataset']
/// data = ds[:] # read all data as numpy array /// ds.shape, ds.dtype, ds.attrs['units']
/// shape = ds.shape /// block = ds[10:20, ::2] # reads only the selected elements
/// dtype = ds.dtype
/// ``` /// ```
#[pyclass(name = "Dataset")] #[pyclass(name = "Dataset")]
pub struct PyDataset { pub struct PyDataset {
file: Arc<clawhdf5_rs::File>, file: Arc<clawhdf5_rs::File>,
path: String, path: String,
cached_shape: Vec<u64>, /// `None` for a dataset with a null dataspace (h5py's `Empty`).
cached_dtype: DType, shape: Option<Vec<u64>>,
datatype: Datatype,
/// Why the datatype cannot be read into numpy, if it cannot.
conv: Result<Converter, String>,
} }
impl PyDataset { impl PyDataset {
pub fn new(file: Arc<clawhdf5_rs::File>, path: String) -> PyResult<Self> { pub(crate) fn open(
py: Python<'_>,
file: Arc<clawhdf5_rs::File>,
path: String,
) -> PyResult<Self> {
let hdr = node::header(&file, &path)?;
let null = node::is_null(&node::dataspace(&file, &hdr)?);
let (shape, datatype) = {
let ds = file.dataset(&path).map_err(to_py_err)?; let ds = file.dataset(&path).map_err(to_py_err)?;
let cached_shape = ds.shape().map_err(to_py_err)?; let shape = if null {
let cached_dtype = ds.dtype().map_err(to_py_err)?; None
} else {
Some(ds.shape().map_err(to_py_err)?)
};
(shape, ds.raw_datatype().map_err(to_py_err)?)
};
let conv = Converter::new(py, &datatype, file.superblock().offset_size)
.map_err(|e| e.value(py).to_string());
Ok(Self { Ok(Self {
file, file,
path, path,
cached_shape, shape,
cached_dtype, datatype,
conv,
}) })
} }
fn converter(&self) -> PyResult<&Converter> {
self.conv
.as_ref()
.map_err(|msg| PyTypeError::new_err(format!("{}: {msg}", node::name(&self.path))))
}
/// Read the selection described by `plan` into a numpy array.
fn read_plan<'py>(&self, py: Python<'py>, plan: &Plan) -> PyResult<Bound<'py, PyAny>> {
let conv = self.converter()?;
let dims = self.shape.as_deref().unwrap_or(&[]);
let out_shape = plan.out_shape();
let arr = if plan.is_empty() {
conv.empty(py, &out_shape)?
} else {
let (reads, list_axis) = plan.reads(dims);
let file = &*self.file;
let path = self.path.as_str();
let (vl, elem_size, unit) = (conv.is_vl(), conv.elem_size, conv.vl_unit);
// Everything below touches only Rust data: release the GIL.
let blocks: Vec<(Elements, Vec<usize>)> = py
.detach(|| -> Result<_, ReadError> {
let ds = file.dataset(path)?;
let sb = file.superblock();
let mut blocks = Vec::with_capacity(reads.len());
for (sel, shape) in reads {
let raw = ds.read_selection(&sel)?;
let n: usize = shape.iter().product();
let data = if vl {
if raw.len() != n * elem_size {
return Err(ReadError::Other(format!(
"read {} bytes of variable-length references, expected {}",
raw.len(),
n * elem_size
)));
}
Elements::Vl(
resolve_vl(
file.as_bytes(),
&raw,
n,
sb.offset_size,
sb.length_size,
unit,
)
.map_err(ReadError::Other)?,
)
} else {
Elements::Bytes(raw)
};
blocks.push((data, shape));
}
Ok(blocks)
})
.map_err(|e| e.into_py(&self.path))?;
let mut arrays = Vec::with_capacity(blocks.len());
for (data, shape) in blocks {
arrays.push(conv.to_array(py, data, &shape, false)?);
}
let joined = if arrays.len() == 1 {
arrays.pop().expect("one block")
} else {
let axis = list_axis.expect("several reads only for a list index");
// Name the dtype: left to itself numpy canonicalises a
// structured dtype here (drops padding, native byte order).
let kwargs = pyo3::types::PyDict::new(py);
kwargs.set_item("axis", axis)?;
kwargs.set_item("dtype", arrays[0].getattr("dtype")?)?;
kwargs.set_item("casting", "no")?;
py.import("numpy")?.call_method(
"concatenate",
(PyList::new(py, arrays)?,),
Some(&kwargs),
)?
};
// Drop the axes indexed by an integer (length 1 in the blocks).
let mut shape = out_shape.clone();
if let crate::convert::Layout::Subarray(sub) = &conv.layout {
shape.extend_from_slice(sub);
}
joined.call_method1("reshape", (PyTuple::new(py, shape)?,))?
};
let arr = select_fields(py, arr, &plan.fields)?;
if plan.scalar {
return arr.get_item(PyTuple::empty(py));
}
Ok(arr)
}
} }
/// Map a `DType` to a numpy dtype string. /// An error from the read closure, turned into a Python error with the GIL.
fn dtype_to_numpy_str(dt: &DType) -> &'static str { enum ReadError {
match dt { Lib(clawhdf5_rs::Error),
DType::F64 => "float64", Other(String),
DType::F32 => "float32", }
DType::I64 => "int64",
DType::I32 => "int32", impl From<clawhdf5_rs::Error> for ReadError {
DType::I16 => "int16", fn from(e: clawhdf5_rs::Error) -> Self {
DType::I8 => "int8", ReadError::Lib(e)
DType::U64 => "uint64",
DType::U32 => "uint32",
DType::U16 => "uint16",
DType::U8 => "uint8",
DType::String | DType::VariableLengthString => "object",
_ => "object",
} }
} }
impl ReadError {
fn into_py(self, path: &str) -> PyErr {
match self {
ReadError::Lib(e) => to_py_err(e),
ReadError::Other(msg) => PyValueError::new_err(format!("{}: {msg}", node::name(path))),
}
}
}
/// Keep only the named compound fields, as h5py's `ds['x']` / `ds['x', 'y']`.
fn select_fields<'py>(
py: Python<'py>,
arr: Bound<'py, PyAny>,
fields: &[String],
) -> PyResult<Bound<'py, PyAny>> {
if fields.is_empty() {
return Ok(arr);
}
let names = arr.getattr("dtype")?.getattr("names")?;
if names.is_none() {
return Err(PyValueError::new_err(
"Field names only allowed for compound types",
));
}
let names: Vec<String> = names.extract()?;
for f in fields {
if !names.contains(f) {
return Err(PyValueError::new_err(format!(
"Field {f} does not appear in this type."
)));
}
}
let np = py.import("numpy")?;
if let [one] = fields {
return np.call_method1("ascontiguousarray", (arr.get_item(one)?,));
}
let picked = arr.get_item(PyList::new(py, fields)?)?;
py.import("numpy.lib.recfunctions")?
.call_method1("repack_fields", (picked,))
}
#[pymethods] #[pymethods]
impl PyDataset { impl PyDataset {
/// The shape of the dataset as a tuple. /// The shape of the dataset (`None` for an empty/null dataspace).
#[getter] #[getter]
fn shape(&self, py: Python<'_>) -> PyResult<Py<PyAny>> { fn shape<'py>(&self, py: Python<'py>) -> PyResult<Bound<'py, PyAny>> {
let tuple = pyo3::types::PyTuple::new(py, self.cached_shape.iter().map(|&d| d as usize))?; match &self.shape {
Ok(tuple.into_any().unbind()) Some(s) => Ok(PyTuple::new(py, s)?.into_any()),
None => Ok(py.None().into_bound(py)),
}
} }
/// The numpy dtype string of the dataset. /// The maximum shape (`None` per unlimited dimension), like h5py.
#[getter] #[getter]
fn dtype(&self) -> &'static str { fn maxshape<'py>(&self, py: Python<'py>) -> PyResult<Bound<'py, PyAny>> {
dtype_to_numpy_str(&self.cached_dtype) let Some(shape) = &self.shape else {
return Ok(py.None().into_bound(py));
};
let max = self
.file
.dataset(&self.path)
.and_then(|ds| ds.max_dimensions())
.map_err(to_py_err)?
.unwrap_or_else(|| shape.clone());
let items: Vec<Option<u64>> = max
.into_iter()
.map(|d| (d != u64::MAX).then_some(d))
.collect();
Ok(PyTuple::new(py, items)?.into_any())
} }
/// Attribute access (read-only). /// The dataset's numpy dtype, as h5py reports it.
#[getter]
fn dtype<'py>(&self, py: Python<'py>) -> PyResult<Bound<'py, PyAny>> {
Ok(self.converter()?.dtype.bind(py).clone())
}
#[getter]
fn ndim(&self) -> usize {
self.shape.as_ref().map_or(0, Vec::len)
}
/// Number of elements (`None` for an empty/null dataspace, as h5py).
#[getter]
fn size(&self) -> Option<u64> {
self.shape.as_ref().map(|s| s.iter().product())
}
/// The dataset's full name, e.g. `/group/data`.
#[getter]
fn name(&self) -> String {
node::name(&self.path)
}
/// The dataset's attributes (read-only, dict-like).
#[getter] #[getter]
fn attrs(&self) -> PyResult<PyAttrs> { fn attrs(&self) -> PyResult<PyAttrs> {
let ds = self.file.dataset(&self.path).map_err(to_py_err)?; PyAttrs::read(Arc::clone(&self.file), &self.path)
let map = ds.attrs().map_err(to_py_err)?;
Ok(PyAttrs::from_read(map))
} }
/// Read data via indexing. Supports `ds[:]`, `ds[0]`, `ds[0:5]`, etc. /// Read with h5py indexing: integers, slices with positive steps,
/// /// `...`, one increasing list of integers, and compound field names.
/// The full dataset is always read from the underlying file; the index /// Only the selected elements are read from the file.
/// is then applied on the resulting numpy array. fn __getitem__<'py>(
fn __getitem__<'py>(&self, py: Python<'py>, key: &Bound<'py, PyAny>) -> PyResult<Py<PyAny>> { &self,
let arr = self.read_as_numpy(py)?; py: Python<'py>,
let indexed = arr.get_item(key)?; key: &Bound<'py, PyAny>,
Ok(indexed.unbind()) ) -> PyResult<Bound<'py, PyAny>> {
let Some(dims) = &self.shape else {
let is_empty_tuple = key.cast::<PyTuple>().is_ok_and(|t| t.is_empty());
let is_ellipsis = key.is_instance_of::<pyo3::types::PyEllipsis>();
if is_empty_tuple || is_ellipsis {
let empty = PyEmpty::new(self.converter()?.dtype.clone_ref(py));
return Ok(empty.into_pyobject(py)?.into_any());
}
return Err(PyValueError::new_err("Empty datasets cannot be sliced"));
};
let plan = select::parse(key, dims)?;
self.read_plan(py, &plan)
} }
fn __repr__(&self) -> String { /// `numpy.asarray(ds)` reads the whole dataset.
#[pyo3(signature = (dtype=None, copy=None))]
fn __array__<'py>(
&self,
py: Python<'py>,
dtype: Option<&Bound<'py, PyAny>>,
copy: Option<bool>,
) -> PyResult<Bound<'py, PyAny>> {
let _ = copy; // every read is a fresh array
let Some(dims) = &self.shape else {
return Err(PyValueError::new_err("an empty dataset has no array value"));
};
let ellipsis = pyo3::types::PyEllipsis::get(py).to_owned().into_any();
let plan = select::parse(&ellipsis, dims)?;
let arr = self.read_plan(py, &plan)?;
match dtype {
Some(dt) => arr.call_method1("astype", (dt,)),
None => Ok(arr),
}
}
fn __len__(&self) -> PyResult<usize> {
match self.shape.as_deref() {
Some([first, ..]) => Ok(*first as usize),
_ => Err(PyTypeError::new_err(
"Attempt to take len() of scalar dataset",
)),
}
}
fn __repr__(&self, py: Python<'_>) -> String {
let dtype = match &self.conv {
Ok(c) => c
.dtype
.bind(py)
.str()
.map(|s| s.to_string())
.unwrap_or_default(),
Err(_) => format!("{:?}", self.datatype),
};
let shape = match &self.shape {
Some(s) => format!("{s:?}"),
None => "None".to_string(),
};
format!( format!(
"<HDF5 Dataset \"{}\": shape {:?}, dtype {}>", "<HDF5 dataset \"{}\": shape {shape}, type \"{dtype}\">",
self.path, node::name(&self.path)
self.cached_shape,
dtype_to_numpy_str(&self.cached_dtype),
) )
} }
fn __len__(&self) -> usize {
self.cached_shape.first().copied().unwrap_or(0) as usize
}
}
impl PyDataset {
/// Read the full dataset and return it as a numpy array (or list for strings).
///
/// For numeric types, the Rust I/O (file reading + decompression) is
/// performed inside `py.detach()` so that the GIL is released
/// during the potentially expensive operation. The numpy array
/// construction still happens with the GIL held.
fn read_as_numpy<'py>(&self, py: Python<'py>) -> PyResult<Bound<'py, PyAny>> {
let file = &self.file;
let path = &self.path;
let shape: Vec<usize> = self.cached_shape.iter().map(|&d| d as usize).collect();
match &self.cached_dtype {
DType::F64 => {
let data = py
.detach(|| file.dataset(path).and_then(|ds| ds.read_f64()))
.map_err(to_py_err)?;
let nd = ArrayD::from_shape_vec(IxDyn(&shape), data)
.map_err(|e| PyErr::new::<pyo3::exceptions::PyValueError, _>(e.to_string()))?;
let arr = PyArrayDyn::from_owned_array(py, nd);
Ok(arr.into_any())
}
DType::F32 => {
let data = py
.detach(|| file.dataset(path).and_then(|ds| ds.read_f32()))
.map_err(to_py_err)?;
let nd = ArrayD::from_shape_vec(IxDyn(&shape), data)
.map_err(|e| PyErr::new::<pyo3::exceptions::PyValueError, _>(e.to_string()))?;
let arr = PyArrayDyn::from_owned_array(py, nd);
Ok(arr.into_any())
}
DType::I32 => {
let data = py
.detach(|| file.dataset(path).and_then(|ds| ds.read_i32()))
.map_err(to_py_err)?;
let nd = ArrayD::from_shape_vec(IxDyn(&shape), data)
.map_err(|e| PyErr::new::<pyo3::exceptions::PyValueError, _>(e.to_string()))?;
let arr = PyArrayDyn::from_owned_array(py, nd);
Ok(arr.into_any())
}
DType::I64 => {
let data = py
.detach(|| file.dataset(path).and_then(|ds| ds.read_i64()))
.map_err(to_py_err)?;
let nd = ArrayD::from_shape_vec(IxDyn(&shape), data)
.map_err(|e| PyErr::new::<pyo3::exceptions::PyValueError, _>(e.to_string()))?;
let arr = PyArrayDyn::from_owned_array(py, nd);
Ok(arr.into_any())
}
DType::U8 => {
// Try zero-copy first (contiguous layout), fall back to
// read_u64 + cast for chunked/compact datasets.
let data: Vec<u8> = py
.detach(|| {
let ds = file.dataset(path)?;
match ds.read_u8_zerocopy() {
Ok(slice) => Ok(slice.to_vec()),
Err(_) => {
let raw = ds.read_u64()?;
Ok(raw.iter().map(|&v| v as u8).collect())
}
}
})
.map_err(to_py_err)?;
let nd = ArrayD::from_shape_vec(IxDyn(&shape), data)
.map_err(|e| PyErr::new::<pyo3::exceptions::PyValueError, _>(e.to_string()))?;
let arr = PyArrayDyn::from_owned_array(py, nd);
Ok(arr.into_any())
}
DType::U64 => {
let data = py
.detach(|| file.dataset(path).and_then(|ds| ds.read_u64()))
.map_err(to_py_err)?;
let nd = ArrayD::from_shape_vec(IxDyn(&shape), data)
.map_err(|e| PyErr::new::<pyo3::exceptions::PyValueError, _>(e.to_string()))?;
let arr = PyArrayDyn::from_owned_array(py, nd);
Ok(arr.into_any())
}
DType::String | DType::VariableLengthString => {
// String reads need the GIL for PyList construction, but we
// release it during the Rust I/O portion.
let data = py
.detach(|| file.dataset(path).and_then(|ds| ds.read_string()))
.map_err(to_py_err)?;
let list = PyList::new(py, &data)?;
Ok(list.into_any())
}
other => Err(PyErr::new::<pyo3::exceptions::PyTypeError, _>(format!(
"unsupported dataset dtype for reading: {other}"
))),
}
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn dtype_mapping() {
assert_eq!(dtype_to_numpy_str(&DType::F64), "float64");
assert_eq!(dtype_to_numpy_str(&DType::F32), "float32");
assert_eq!(dtype_to_numpy_str(&DType::I32), "int32");
assert_eq!(dtype_to_numpy_str(&DType::I64), "int64");
assert_eq!(dtype_to_numpy_str(&DType::U8), "uint8");
assert_eq!(dtype_to_numpy_str(&DType::String), "object");
}
#[test]
fn dataset_from_file() {
let mut b = clawhdf5_rs::FileBuilder::new();
b.create_dataset("vals").with_f64_data(&[1.0, 2.0, 3.0]);
let bytes = b.finish().unwrap();
let file = Arc::new(clawhdf5_rs::File::from_bytes(bytes).unwrap());
let ds = PyDataset::new(file, "vals".into()).unwrap();
assert_eq!(ds.cached_shape, vec![3]);
assert_eq!(ds.cached_dtype, DType::F64);
}
#[test]
fn dataset_len() {
let mut b = clawhdf5_rs::FileBuilder::new();
b.create_dataset("data")
.with_i32_data(&[10, 20, 30, 40])
.with_shape(&[2, 2]);
let bytes = b.finish().unwrap();
let file = Arc::new(clawhdf5_rs::File::from_bytes(bytes).unwrap());
let ds = PyDataset::new(file, "data".into()).unwrap();
assert_eq!(ds.__len__(), 2);
}
} }
crates/clawhdf5-py/src/file.rs
+55 -40
View File
@@ -3,11 +3,12 @@
use std::path::PathBuf; use std::path::PathBuf;
use std::sync::{Arc, Mutex}; use std::sync::{Arc, Mutex};
use pyo3::exceptions::PyKeyError;
use pyo3::prelude::*; use pyo3::prelude::*;
use pyo3::types::PyList;
use crate::attrs::PyAttrs; use crate::attrs::PyAttrs;
use crate::dataset::PyDataset; use crate::group::{self, PyGroup, WriteGroupState, finalize_write_group};
use crate::group::{PyGroup, WriteGroupState, finalize_write_group};
use crate::{DatasetSpec, OwnedAttrValue, apply_dataset_spec, extract_numpy_data, to_py_err}; use crate::{DatasetSpec, OwnedAttrValue, apply_dataset_spec, extract_numpy_data, to_py_err};
/// Internal state for write mode. /// Internal state for write mode.
@@ -35,6 +36,7 @@ struct WriteState {
#[pyclass(name = "File")] #[pyclass(name = "File")]
pub struct PyFile { pub struct PyFile {
inner: Option<FileInner>, inner: Option<FileInner>,
filename: String,
} }
enum FileInner { enum FileInner {
@@ -51,15 +53,20 @@ impl PyFile {
/// mode: 'r' for read (default), 'w' for write /// mode: 'r' for read (default), 'w' for write
#[new] #[new]
#[pyo3(signature = (path, mode="r"))] #[pyo3(signature = (path, mode="r"))]
fn new(path: &str, mode: &str) -> PyResult<Self> { fn new(py: Python<'_>, path: &str, mode: &str) -> PyResult<Self> {
let filename = path.to_string();
match mode { match mode {
"r" => { "r" => {
let file = clawhdf5_rs::File::open(path).map_err(to_py_err)?; let file = py
.detach(|| clawhdf5_rs::File::open(path))
.map_err(to_py_err)?;
Ok(Self { Ok(Self {
inner: Some(FileInner::Read(Arc::new(file))), inner: Some(FileInner::Read(Arc::new(file))),
filename,
}) })
} }
"w" => Ok(Self { "w" => Ok(Self {
filename,
inner: Some(FileInner::Write(WriteState { inner: Some(FileInner::Write(WriteState {
path: PathBuf::from(path), path: PathBuf::from(path),
root_datasets: Vec::new(), root_datasets: Vec::new(),
@@ -101,44 +108,56 @@ impl PyFile {
Ok(false) // don't suppress exceptions Ok(false) // don't suppress exceptions
} }
/// Get a child object (dataset or group) by path. /// Get a child object (dataset or group) by path; `f['/']` is the root.
fn __getitem__(&self, py: Python<'_>, key: &str) -> PyResult<Py<PyAny>> { fn __getitem__(&self, py: Python<'_>, key: &str) -> PyResult<Py<PyAny>> {
let file = self.read_file()?; group::get_item(py, self.read_file()?, "", key)
// Try dataset first
match file.dataset(key) {
Ok(_) => {
let ds = PyDataset::new(Arc::clone(file), key.to_string())?;
Ok(ds.into_pyobject(py)?.into_any().unbind())
}
Err(clawhdf5_rs::Error::NotADataset(_)) => {
let grp = PyGroup::from_read(Arc::clone(file), key.to_string());
Ok(grp.into_pyobject(py)?.into_any().unbind())
}
Err(_) => {
// Could be a group (no DataLayout message, no error)
match file.group(key) {
Ok(_) => {
let grp = PyGroup::from_read(Arc::clone(file), key.to_string());
Ok(grp.into_pyobject(py)?.into_any().unbind())
}
Err(e) => Err(PyErr::new::<pyo3::exceptions::PyKeyError, _>(format!(
"{key}: {e}"
))),
} }
/// `f.get(key, default=None)`.
#[pyo3(signature = (key, default=None))]
fn get(&self, py: Python<'_>, key: &str, default: Option<Py<PyAny>>) -> PyResult<Py<PyAny>> {
match group::get_item(py, self.read_file()?, "", key) {
Err(e) if e.is_instance_of::<PyKeyError>(py) => {
Ok(default.unwrap_or_else(|| py.None()))
} }
other => other,
} }
} }
/// List the names of all children in the root group. /// List the names of all children in the root group.
fn keys(&self, py: Python<'_>) -> PyResult<Py<PyAny>> { fn keys(&self, py: Python<'_>) -> PyResult<Py<PyAny>> {
let file = self.read_file()?; let names = group::member_names(self.read_file()?, "")?;
let root = file.root(); Ok(PyList::new(py, names)?.into_any().unbind())
let mut names = root.datasets().map_err(to_py_err)?; }
let groups = root.groups().map_err(to_py_err)?;
names.extend(groups); fn values(&self, py: Python<'_>) -> PyResult<Py<PyAny>> {
names.sort(); let vals = group::values(py, self.read_file()?, "")?;
let list = pyo3::types::PyList::new(py, &names)?; Ok(PyList::new(py, vals)?.into_any().unbind())
Ok(list.into_any().unbind()) }
fn items(&self, py: Python<'_>) -> PyResult<Py<PyAny>> {
let items = group::items(py, self.read_file()?, "")?;
Ok(PyList::new(py, items)?.into_any().unbind())
}
fn __iter__(&self, py: Python<'_>) -> PyResult<Py<PyAny>> {
self.keys(py)?.call_method0(py, "__iter__")
}
fn __len__(&self) -> PyResult<usize> {
Ok(group::member_names(self.read_file()?, "")?.len())
}
/// The root group's name, `/`.
#[getter]
fn name(&self) -> &'static str {
"/"
}
/// The path the file was opened with.
#[getter]
fn filename(&self) -> &str {
&self.filename
} }
/// Create a dataset in the root group (write mode only). /// Create a dataset in the root group (write mode only).
@@ -192,10 +211,7 @@ impl PyFile {
#[getter] #[getter]
fn attrs(&self) -> PyResult<PyAttrs> { fn attrs(&self) -> PyResult<PyAttrs> {
match self.inner.as_ref() { match self.inner.as_ref() {
Some(FileInner::Read(file)) => { Some(FileInner::Read(file)) => PyAttrs::read(Arc::clone(file), ""),
let map = file.root().attrs().map_err(to_py_err)?;
Ok(PyAttrs::from_read(map))
}
Some(FileInner::Write(state)) => Ok(PyAttrs::from_write(Arc::clone(&state.root_attrs))), Some(FileInner::Write(state)) => Ok(PyAttrs::from_write(Arc::clone(&state.root_attrs))),
None => Err(PyErr::new::<pyo3::exceptions::PyIOError, _>( None => Err(PyErr::new::<pyo3::exceptions::PyIOError, _>(
"file is closed", "file is closed",
@@ -216,8 +232,7 @@ impl PyFile {
} }
fn __contains__(&self, key: &str) -> PyResult<bool> { fn __contains__(&self, key: &str) -> PyResult<bool> {
let file = self.read_file()?; Ok(group::contains(self.read_file()?, "", key))
Ok(file.dataset(key).is_ok() || file.group(key).is_ok())
} }
} }
crates/clawhdf5-py/src/group.rs
+134 -92
View File
@@ -2,12 +2,12 @@
use std::sync::{Arc, Mutex}; use std::sync::{Arc, Mutex};
use pyo3::exceptions::{PyIOError, PyKeyError};
use pyo3::prelude::*; use pyo3::prelude::*;
use pyo3::types::PyList; use pyo3::types::PyList;
use crate::attrs::PyAttrs; use crate::attrs::PyAttrs;
use crate::dataset::PyDataset; use crate::{DatasetSpec, OwnedAttrValue, apply_dataset_spec, extract_numpy_data, node, to_py_err};
use crate::{DatasetSpec, OwnedAttrValue, apply_dataset_spec, extract_numpy_data, to_py_err};
/// Shared state for a group being written. /// Shared state for a group being written.
pub(crate) struct WriteGroupState { pub(crate) struct WriteGroupState {
@@ -18,15 +18,10 @@ pub(crate) struct WriteGroupState {
/// An HDF5 group. /// An HDF5 group.
/// ///
/// In read mode, provides `__getitem__` navigation and child listing. /// In read mode it behaves like an h5py group: `grp['name']`,
/// In write mode, supports `create_dataset` and `create_group` and /// `grp['sub/path']` and `grp['/absolute/path']`, `keys()`, `values()`,
/// attribute setting. /// `items()`, iteration, `len()`, `in`, `get()`, `name` and `attrs`.
/// /// In write mode, supports `create_dataset` and attribute setting.
/// ```python
/// grp = f['group_name']
/// grp.keys()
/// ds = grp['dataset']
/// ```
#[pyclass(name = "Group")] #[pyclass(name = "Group")]
pub struct PyGroup { pub struct PyGroup {
inner: GroupInner, inner: GroupInner,
@@ -52,46 +47,90 @@ impl PyGroup {
inner: GroupInner::Write(state), inner: GroupInner::Write(state),
} }
} }
fn read_parts(&self, what: &str) -> PyResult<(&Arc<clawhdf5_rs::File>, &str)> {
match &self.inner {
GroupInner::Read { file, path } => Ok((file, path)),
GroupInner::Write(_) => Err(PyIOError::new_err(format!(
"cannot {what} a group opened for writing"
))),
}
}
}
// Read-mode operations shared by `Group` and `File` (a file is its root
// group, as in h5py).
/// `group[key]`.
pub(crate) fn get_item(
py: Python<'_>,
file: &Arc<clawhdf5_rs::File>,
path: &str,
key: &str,
) -> PyResult<Py<PyAny>> {
node::open(py, file, node::join(path, key))
}
/// Names of the group's datasets and subgroups, sorted (h5py's order).
pub(crate) fn member_names(file: &clawhdf5_rs::File, path: &str) -> PyResult<Vec<String>> {
let group = if path.is_empty() {
file.root()
} else {
file.group(path).map_err(to_py_err)?
};
let mut names = group.datasets().map_err(to_py_err)?;
names.extend(group.groups().map_err(to_py_err)?);
names.sort_by(|a, b| a.as_bytes().cmp(b.as_bytes()));
names.dedup();
Ok(names)
}
pub(crate) fn contains(file: &clawhdf5_rs::File, path: &str, key: &str) -> bool {
node::exists(file, &node::join(path, key))
}
pub(crate) fn values(
py: Python<'_>,
file: &Arc<clawhdf5_rs::File>,
path: &str,
) -> PyResult<Vec<Py<PyAny>>> {
member_names(file, path)?
.iter()
.map(|n| get_item(py, file, path, n))
.collect()
}
pub(crate) fn items(
py: Python<'_>,
file: &Arc<clawhdf5_rs::File>,
path: &str,
) -> PyResult<Vec<(String, Py<PyAny>)>> {
member_names(file, path)?
.into_iter()
.map(|n| {
let v = get_item(py, file, path, &n)?;
Ok((n, v))
})
.collect()
} }
#[pymethods] #[pymethods]
impl PyGroup { impl PyGroup {
/// Get a child object (dataset or subgroup) by name or path. /// Get a child object (dataset or subgroup) by name or path.
fn __getitem__(&self, py: Python<'_>, key: &str) -> PyResult<Py<PyAny>> { fn __getitem__(&self, py: Python<'_>, key: &str) -> PyResult<Py<PyAny>> {
match &self.inner { let (file, path) = self.read_parts("read children from")?;
GroupInner::Read { file, path } => { get_item(py, file, path, key)
let full_path = if path.is_empty() {
key.to_string()
} else {
format!("{path}/{key}")
};
// Try dataset first
match file.dataset(&full_path) {
Ok(_) => {
let ds = PyDataset::new(Arc::clone(file), full_path)?;
Ok(ds.into_pyobject(py)?.into_any().unbind())
} }
Err(clawhdf5_rs::Error::NotADataset(_)) => {
let grp = PyGroup::from_read(Arc::clone(file), full_path); /// `group.get(key, default=None)`.
Ok(grp.into_pyobject(py)?.into_any().unbind()) #[pyo3(signature = (key, default=None))]
fn get(&self, py: Python<'_>, key: &str, default: Option<Py<PyAny>>) -> PyResult<Py<PyAny>> {
let (file, path) = self.read_parts("read children from")?;
match get_item(py, file, path, key) {
Err(e) if e.is_instance_of::<PyKeyError>(py) => {
Ok(default.unwrap_or_else(|| py.None()))
} }
Err(e) => { other => other,
// Could be a group without a DataLayout message
match file.group(&full_path) {
Ok(_) => {
let grp = PyGroup::from_read(Arc::clone(file), full_path);
Ok(grp.into_pyobject(py)?.into_any().unbind())
}
Err(_) => Err(PyErr::new::<pyo3::exceptions::PyKeyError, _>(format!(
"{key}: {e}"
))),
}
}
}
}
GroupInner::Write(_) => Err(PyErr::new::<pyo3::exceptions::PyIOError, _>(
"cannot read children from a group opened for writing",
)),
} }
} }
@@ -99,16 +138,7 @@ impl PyGroup {
fn keys(&self, py: Python<'_>) -> PyResult<Py<PyAny>> { fn keys(&self, py: Python<'_>) -> PyResult<Py<PyAny>> {
match &self.inner { match &self.inner {
GroupInner::Read { file, path } => { GroupInner::Read { file, path } => {
let group = if path.is_empty() { let list = PyList::new(py, member_names(file, path)?)?;
file.root()
} else {
file.group(path).map_err(to_py_err)?
};
let mut names = group.datasets().map_err(to_py_err)?;
let groups = group.groups().map_err(to_py_err)?;
names.extend(groups);
names.sort();
let list = PyList::new(py, &names)?;
Ok(list.into_any().unbind()) Ok(list.into_any().unbind())
} }
GroupInner::Write(state) => { GroupInner::Write(state) => {
@@ -120,6 +150,38 @@ impl PyGroup {
} }
} }
fn values(&self, py: Python<'_>) -> PyResult<Py<PyAny>> {
let (file, path) = self.read_parts("read children from")?;
Ok(PyList::new(py, values(py, file, path)?)?
.into_any()
.unbind())
}
fn items(&self, py: Python<'_>) -> PyResult<Py<PyAny>> {
let (file, path) = self.read_parts("read children from")?;
Ok(PyList::new(py, items(py, file, path)?)?.into_any().unbind())
}
fn __iter__(&self, py: Python<'_>) -> PyResult<Py<PyAny>> {
self.keys(py)?.call_method0(py, "__iter__")
}
fn __len__(&self) -> PyResult<usize> {
match &self.inner {
GroupInner::Read { file, path } => Ok(member_names(file, path)?.len()),
GroupInner::Write(state) => Ok(state.lock().unwrap().datasets.len()),
}
}
/// The group's full name, e.g. `/sensors`.
#[getter]
fn name(&self) -> String {
match &self.inner {
GroupInner::Read { path, .. } => node::name(path),
GroupInner::Write(state) => node::name(&state.lock().unwrap().name),
}
}
/// Create a dataset inside this group (write mode only). /// Create a dataset inside this group (write mode only).
/// ///
/// Parameters: /// Parameters:
@@ -161,7 +223,7 @@ impl PyGroup {
state.lock().unwrap().datasets.push(spec); state.lock().unwrap().datasets.push(spec);
Ok(()) Ok(())
} }
GroupInner::Read { .. } => Err(PyErr::new::<pyo3::exceptions::PyIOError, _>( GroupInner::Read { .. } => Err(PyIOError::new_err(
"cannot create datasets on a read-only group", "cannot create datasets on a read-only group",
)), )),
} }
@@ -171,15 +233,7 @@ impl PyGroup {
#[getter] #[getter]
fn attrs(&self) -> PyResult<PyAttrs> { fn attrs(&self) -> PyResult<PyAttrs> {
match &self.inner { match &self.inner {
GroupInner::Read { file, path } => { GroupInner::Read { file, path } => PyAttrs::read(Arc::clone(file), path),
let group = if path.is_empty() {
file.root()
} else {
file.group(path).map_err(to_py_err)?
};
let map = group.attrs().map_err(to_py_err)?;
Ok(PyAttrs::from_read(map))
}
GroupInner::Write(state) => { GroupInner::Write(state) => {
let store = Arc::clone(&state.lock().unwrap().attrs); let store = Arc::clone(&state.lock().unwrap().attrs);
Ok(PyAttrs::from_write(store)) Ok(PyAttrs::from_write(store))
@@ -189,12 +243,9 @@ impl PyGroup {
fn __repr__(&self) -> String { fn __repr__(&self) -> String {
match &self.inner { match &self.inner {
GroupInner::Read { path, .. } => { GroupInner::Read { file, path } => {
if path.is_empty() { let n = member_names(file, path).map_or(0, |m| m.len());
"<HDF5 Group \"/\" (root)>".to_string() format!("<HDF5 group \"{}\" ({n} members)>", node::name(path))
} else {
format!("<HDF5 Group \"/{path}\">")
}
} }
GroupInner::Write(state) => { GroupInner::Write(state) => {
let name = &state.lock().unwrap().name; let name = &state.lock().unwrap().name;
@@ -205,14 +256,7 @@ impl PyGroup {
fn __contains__(&self, key: &str) -> PyResult<bool> { fn __contains__(&self, key: &str) -> PyResult<bool> {
match &self.inner { match &self.inner {
GroupInner::Read { file, path } => { GroupInner::Read { file, path } => Ok(contains(file, path, key)),
let full_path = if path.is_empty() {
key.to_string()
} else {
format!("{path}/{key}")
};
Ok(file.dataset(&full_path).is_ok() || file.group(&full_path).is_ok())
}
GroupInner::Write(state) => { GroupInner::Write(state) => {
let guard = state.lock().unwrap(); let guard = state.lock().unwrap();
Ok(guard.datasets.iter().any(|d| d.name == key)) Ok(guard.datasets.iter().any(|d| d.name == key))
@@ -244,26 +288,24 @@ mod tests {
use super::*; use super::*;
#[test] #[test]
fn read_group_construction() { fn member_names_are_sorted() {
let mut b = clawhdf5_rs::FileBuilder::new(); let mut b = clawhdf5_rs::FileBuilder::new();
let mut g = b.create_group("grp"); b.create_dataset("zeta").with_f64_data(&[1.0]);
b.create_dataset("alpha").with_f64_data(&[1.0]);
let mut g = b.create_group("mid");
g.create_dataset("x").with_f64_data(&[1.0]); g.create_dataset("x").with_f64_data(&[1.0]);
let finished = g.finish(); let finished = g.finish();
b.add_group(finished); b.add_group(finished);
let bytes = b.finish().unwrap(); let bytes = b.finish().unwrap();
let file = Arc::new(clawhdf5_rs::File::from_bytes(bytes).unwrap()); let file = clawhdf5_rs::File::from_bytes(bytes).unwrap();
let _grp = PyGroup::from_read(file, "grp".into()); assert_eq!(
} member_names(&file, "").unwrap(),
vec!["alpha", "mid", "zeta"]
#[test] );
fn write_group_state() { assert_eq!(member_names(&file, "mid").unwrap(), vec!["x"]);
let state = WriteGroupState { assert!(contains(&file, "", "mid/x"));
name: "test".into(), assert!(contains(&file, "mid", "/alpha"));
datasets: vec![], assert!(!contains(&file, "", "nope"));
attrs: Arc::new(Mutex::new(vec![])),
};
let arc = Arc::new(Mutex::new(state));
let _grp = PyGroup::from_write(arc);
} }
#[test] #[test]
crates/clawhdf5-py/src/lib.rs
+54 -2
View File
@@ -10,9 +10,12 @@
//! ``` //! ```
mod attrs; mod attrs;
mod convert;
mod dataset; mod dataset;
mod file; mod file;
mod group; mod group;
mod node;
mod select;
use pyo3::prelude::*; use pyo3::prelude::*;
@@ -46,6 +49,54 @@ pub(crate) fn to_py_err(e: clawhdf5_rs::Error) -> PyErr {
} }
} }
/// The value of a dataset or attribute with a null dataspace: a type but no
/// data. Mirrors `h5py.Empty`.
#[pyclass(name = "Empty", frozen)]
pub struct PyEmpty {
dtype: Py<PyAny>,
}
impl PyEmpty {
pub(crate) fn new(dtype: Py<PyAny>) -> Self {
Self { dtype }
}
}
#[pymethods]
impl PyEmpty {
#[new]
fn py_new(py: Python<'_>, dtype: &Bound<'_, PyAny>) -> PyResult<Self> {
let dtype = py.import("numpy")?.getattr("dtype")?.call1((dtype,))?;
Ok(Self::new(dtype.unbind()))
}
#[getter]
fn dtype(&self, py: Python<'_>) -> Py<PyAny> {
self.dtype.clone_ref(py)
}
#[getter]
fn shape(&self, py: Python<'_>) -> Py<PyAny> {
py.None()
}
#[getter]
fn size(&self, py: Python<'_>) -> Py<PyAny> {
py.None()
}
fn __eq__(&self, py: Python<'_>, other: &Bound<'_, PyAny>) -> PyResult<bool> {
match other.cast::<PyEmpty>() {
Ok(o) => self.dtype.bind(py).eq(o.get().dtype.bind(py)),
Err(_) => Ok(false),
}
}
fn __repr__(&self, py: Python<'_>) -> PyResult<String> {
Ok(format!("Empty(dtype={})", self.dtype.bind(py).repr()?))
}
}
/// The data payload for a dataset being written. /// The data payload for a dataset being written.
#[derive(Clone)] #[derive(Clone)]
pub(crate) enum DatasetData { pub(crate) enum DatasetData {
@@ -224,6 +275,7 @@ fn clawhdf5(m: &Bound<'_, PyModule>) -> PyResult<()> {
m.add_class::<PyDataset>()?; m.add_class::<PyDataset>()?;
m.add_class::<PyGroup>()?; m.add_class::<PyGroup>()?;
m.add_class::<PyAttrs>()?; m.add_class::<PyAttrs>()?;
m.add_class::<PyEmpty>()?;
Ok(()) Ok(())
} }
@@ -233,9 +285,9 @@ mod tests {
#[test] #[test]
fn owned_attr_value_roundtrip() { fn owned_attr_value_roundtrip() {
let val = OwnedAttrValue::F64(3.14); let val = OwnedAttrValue::F64(2.5);
let attr: clawhdf5_rs::AttrValue = val.into(); let attr: clawhdf5_rs::AttrValue = val.into();
assert!(matches!(attr, clawhdf5_rs::AttrValue::F64(v) if (v - 3.14).abs() < 1e-10)); assert!(matches!(attr, clawhdf5_rs::AttrValue::F64(v) if (v - 2.5).abs() < 1e-10));
} }
#[test] #[test]
crates/clawhdf5-py/src/node.rs
+168
View File
@@ -0,0 +1,168 @@
//! Resolving paths to objects in a file opened for reading.
use std::sync::Arc;
use clawhdf5_format::attribute::AttributeMessage;
use clawhdf5_format::dataspace::{Dataspace, DataspaceType};
use clawhdf5_format::message_type::MessageType;
use clawhdf5_format::object_header::ObjectHeader;
use pyo3::exceptions::{PyKeyError, PyTypeError, PyValueError};
use pyo3::prelude::*;
use crate::dataset::PyDataset;
use crate::group::PyGroup;
/// Join `key` onto the group path `base` the way h5py does: an absolute key
/// starts from the root, a relative one from `base`. Paths are kept without
/// a leading `/`; the root is `""`.
pub(crate) fn join(base: &str, key: &str) -> String {
let parts = if key.starts_with('/') {
key.split('/').collect::<Vec<_>>()
} else {
base.split('/').chain(key.split('/')).collect()
};
parts
.into_iter()
.filter(|p| !p.is_empty() && *p != ".")
.collect::<Vec<_>>()
.join("/")
}
/// The HDF5 name (`/a/b`) of a path.
pub(crate) fn name(path: &str) -> String {
format!("/{path}")
}
/// The object header of the object at `path`.
pub(crate) fn header(file: &clawhdf5_rs::File, path: &str) -> PyResult<ObjectHeader> {
let sb = file.superblock();
let data = file.as_bytes();
let addr = if path.is_empty() {
sb.root_group_address
} else {
clawhdf5_format::group_v2::resolve_path_any(data, sb, path).map_err(|e| {
PyKeyError::new_err(format!(
"Unable to open object (object '{}' doesn't exist): {e}",
name(path)
))
})?
};
let addr = usize::try_from(addr)
.map_err(|_| PyValueError::new_err(format!("{}: address out of range", name(path))))?;
ObjectHeader::parse(data, addr, sb.offset_size, sb.length_size)
.map_err(|e| PyValueError::new_err(format!("{}: {e}", name(path))))
}
/// What an object header describes.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub(crate) enum Kind {
Dataset,
Group,
Datatype,
}
pub(crate) fn kind(hdr: &ObjectHeader) -> Option<Kind> {
let has = |t: MessageType| hdr.messages.iter().any(|m| m.msg_type == t);
if has(MessageType::DataLayout) {
Some(Kind::Dataset)
} else if has(MessageType::LinkInfo)
|| has(MessageType::Link)
|| has(MessageType::SymbolTable)
|| has(MessageType::GroupInfo)
{
Some(Kind::Group)
} else if has(MessageType::Datatype) {
Some(Kind::Datatype)
} else {
None
}
}
/// Open the object at `path` as a `Dataset` or `Group`.
pub(crate) fn open(
py: Python<'_>,
file: &Arc<clawhdf5_rs::File>,
path: String,
) -> PyResult<Py<PyAny>> {
let hdr = header(file, &path)?;
match kind(&hdr) {
Some(Kind::Dataset) => Ok(PyDataset::open(py, Arc::clone(file), path)?
.into_pyobject(py)?
.into_any()
.unbind()),
Some(Kind::Group) => Ok(PyGroup::from_read(Arc::clone(file), path)
.into_pyobject(py)?
.into_any()
.unbind()),
Some(Kind::Datatype) => Err(PyTypeError::new_err(format!(
"{}: committed (named) datatypes are not supported by clawhdf5",
name(&path)
))),
None => Err(PyValueError::new_err(format!(
"{}: not a dataset, group or datatype",
name(&path)
))),
}
}
/// Whether `path` names a dataset or group.
pub(crate) fn exists(file: &clawhdf5_rs::File, path: &str) -> bool {
header(file, path)
.ok()
.and_then(|h| kind(&h))
.is_some_and(|k| k != Kind::Datatype)
}
/// The dataspace message of an object header.
pub(crate) fn dataspace(file: &clawhdf5_rs::File, hdr: &ObjectHeader) -> PyResult<Dataspace> {
let sb = file.superblock();
let msg = hdr
.messages
.iter()
.find(|m| m.msg_type == MessageType::Dataspace)
.ok_or_else(|| PyValueError::new_err("object has no dataspace message"))?;
let data = clawhdf5_format::shared_message::message_data(
file.as_bytes(),
msg,
sb.offset_size,
sb.length_size,
)
.map_err(|e| PyValueError::new_err(e.to_string()))?;
Dataspace::parse(&data, sb.length_size).map_err(|e| PyValueError::new_err(e.to_string()))
}
pub(crate) fn is_null(space: &Dataspace) -> bool {
space.space_type == DataspaceType::Null
}
/// The attributes of the object at `path`, sorted by name (h5py's order).
/// Attributes whose messages cannot be parsed are left out, as the facade's
/// `attrs()` does.
pub(crate) fn attributes(file: &clawhdf5_rs::File, path: &str) -> PyResult<Vec<AttributeMessage>> {
let hdr = header(file, path)?;
let sb = file.superblock();
let (mut attrs, _errors) = clawhdf5_format::attribute::extract_attributes_tolerant(
file.as_bytes(),
&hdr,
sb.offset_size,
sb.length_size,
)
.map_err(|e| PyValueError::new_err(format!("{}: {e}", name(path))))?;
attrs.sort_by(|a, b| a.name.as_bytes().cmp(b.name.as_bytes()));
Ok(attrs)
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn join_paths() {
assert_eq!(join("", "a"), "a");
assert_eq!(join("a", "b/c"), "a/b/c");
assert_eq!(join("a/b", "/x"), "x");
assert_eq!(join("a", "/"), "");
assert_eq!(join("", "/a//b/"), "a/b");
assert_eq!(join("a", "./b"), "a/b");
}
}
crates/clawhdf5-py/src/select.rs
+366
View File
@@ -0,0 +1,366 @@
//! h5py-style indexing (`ds[1, 2:10:3, ...]`) mapped onto hyperslab
//! selections, so only the selected elements are read.
//!
//! The rules and error messages follow h5py's `selections.py`: integers
//! (negative from the end) drop their axis, slices must have a positive
//! step, one `Ellipsis` fills the unmentioned axes, a single increasing list
//! of integers may index one axis, and strings name compound fields.
//! Everything else (`None`/`np.newaxis`, boolean masks, several index lists)
//! is refused with the error h5py gives.
use clawhdf5_format::selection::Selection;
use pyo3::exceptions::{PyIndexError, PyTypeError, PyValueError};
use pyo3::prelude::*;
use pyo3::types::{PyEllipsis, PySlice, PyString, PyTuple};
/// The selection along one axis.
#[derive(Clone, Debug, PartialEq)]
pub(crate) enum Axis {
/// A single index: the axis is dropped from the result.
Index(u64),
/// `start, start + step, ...`, `count` of them.
Slice { start: u64, step: u64, count: u64 },
/// Increasing, distinct indices.
List(Vec<u64>),
}
impl Axis {
fn len(&self) -> u64 {
match self {
Axis::Index(_) => 1,
Axis::Slice { count, .. } => *count,
Axis::List(v) => v.len() as u64,
}
}
}
/// A parsed index expression.
#[derive(Clone, Debug, PartialEq)]
pub(crate) struct Plan {
/// One entry per dataset axis.
pub axes: Vec<Axis>,
/// Compound field names to keep (empty: all).
pub fields: Vec<String>,
/// For a scalar dataset: `ds[()]` gives a scalar, `ds[...]` a 0-d array.
/// For other datasets: every axis was an integer, so h5py gives a scalar.
pub scalar: bool,
}
impl Plan {
/// The shape of the result.
pub fn out_shape(&self) -> Vec<usize> {
self.axes
.iter()
.filter(|a| !matches!(a, Axis::Index(_)))
.map(|a| a.len() as usize)
.collect()
}
/// Whether the selection is empty.
pub fn is_empty(&self) -> bool {
self.axes.iter().any(|a| a.len() == 0)
}
/// The hyperslab reads that make up this selection, each with the shape
/// of its block (index axes kept at length 1). More than one only when an
/// axis is indexed by a list: one read per run of consecutive indices,
/// concatenated along `list_axis` afterwards.
pub fn reads(&self, dims: &[u64]) -> (Vec<(Selection, Vec<usize>)>, Option<usize>) {
let list_axis = self.axes.iter().position(|a| matches!(a, Axis::List(_)));
let runs: Vec<(u64, u64)> = match list_axis.map(|i| &self.axes[i]) {
Some(Axis::List(idx)) => consecutive_runs(idx),
_ => vec![(0, 0)],
};
let mut out = Vec::with_capacity(runs.len());
for (run_start, run_len) in runs {
let mut start = Vec::with_capacity(dims.len());
let mut stride = Vec::with_capacity(dims.len());
let mut count = Vec::with_capacity(dims.len());
for axis in &self.axes {
let (s, st, c) = match axis {
Axis::Index(i) => (*i, 1, 1),
Axis::Slice { start, step, count } => (*start, *step, *count),
Axis::List(_) => (run_start, 1, run_len),
};
start.push(s);
// A stride only matters between blocks; keep it >= 1.
stride.push(if c <= 1 { 1 } else { st });
count.push(c);
}
let block_shape: Vec<usize> = count.iter().map(|&c| c as usize).collect();
let whole = start.iter().all(|&s| s == 0)
&& stride.iter().all(|&s| s == 1)
&& count.as_slice() == dims;
let sel = if whole {
Selection::All
} else {
let block = vec![1; dims.len()];
Selection::Hyperslab {
start,
stride,
count,
block,
}
};
out.push((sel, block_shape));
}
(out, list_axis)
}
}
fn consecutive_runs(idx: &[u64]) -> Vec<(u64, u64)> {
let mut runs: Vec<(u64, u64)> = Vec::new();
for &i in idx {
match runs.last_mut() {
Some((s, n)) if *s + *n == i => *n += 1,
_ => runs.push((i, 1)),
}
}
runs
}
/// Parse `key` for a dataset of shape `dims`.
pub(crate) fn parse(key: &Bound<'_, PyAny>, dims: &[u64]) -> PyResult<Plan> {
let items: Vec<Bound<'_, PyAny>> = match key.cast::<PyTuple>() {
Ok(t) => t.iter().collect(),
Err(_) => vec![key.clone()],
};
let mut fields = Vec::new();
let mut args = Vec::new();
for item in items {
if let Ok(s) = item.cast::<PyString>() {
fields.push(s.to_str()?.to_owned());
} else {
args.push(item);
}
}
if args.iter().any(|a| a.is_none()) {
return Err(PyTypeError::new_err(
"Indexing with None (or np.newaxis) is not supported",
));
}
let rank = dims.len();
if rank == 0 {
return match args.as_slice() {
[] => Ok(Plan {
axes: vec![],
fields,
scalar: true,
}),
[a] if a.is_instance_of::<PyEllipsis>() => Ok(Plan {
axes: vec![],
fields,
scalar: false,
}),
_ => Err(PyValueError::new_err(
"Illegal slicing argument for scalar dataspace",
)),
};
}
// Expand the ellipsis (at most one) to full slices.
let n_ellipsis = args
.iter()
.filter(|a| a.is_instance_of::<PyEllipsis>())
.count();
if n_ellipsis > 1 {
return Err(PyValueError::new_err("Only one ellipsis may be used."));
}
let explicit = args.len() - n_ellipsis;
if explicit > rank {
return Err(PyValueError::new_err(format!(
"{explicit} indexing arguments for {rank} dimensions"
)));
}
let py = key.py();
let mut expanded: Vec<Option<Bound<'_, PyAny>>> = Vec::with_capacity(rank);
for a in args {
if a.is_instance_of::<PyEllipsis>() {
for _ in 0..(rank - explicit) {
expanded.push(None);
}
} else {
expanded.push(Some(a));
}
}
while expanded.len() < rank {
expanded.push(None);
}
let mut axes = Vec::with_capacity(rank);
for (arg, &n) in expanded.iter().zip(dims) {
axes.push(match arg {
None => Axis::Slice {
start: 0,
step: 1,
count: n,
},
Some(a) => parse_axis(py, a, n)?,
});
}
if axes.iter().filter(|a| matches!(a, Axis::List(_))).count() > 1 {
return Err(PyTypeError::new_err(
"Only one indexing vector or array is currently allowed for fancy indexing",
));
}
let scalar = axes.iter().all(|a| matches!(a, Axis::Index(_)));
Ok(Plan {
axes,
fields,
scalar,
})
}
fn parse_axis(py: Python<'_>, a: &Bound<'_, PyAny>, n: u64) -> PyResult<Axis> {
if a.is_none() {
return Err(PyTypeError::new_err(
"Indexing with None (or np.newaxis) is not supported",
));
}
if let Ok(s) = a.cast::<PySlice>() {
let n_isize = isize::try_from(n)
.map_err(|_| PyValueError::new_err("dimension too large to slice"))?;
let ind = s.indices(n_isize)?;
if ind.step < 1 {
return Err(PyValueError::new_err(format!(
"Step must be >= 1 (got {})",
ind.step
)));
}
// `slicelength` is the number of elements selected, >= 0.
let count = ind.slicelength as u64;
let start = if count == 0 { 0 } else { ind.start as u64 };
return Ok(Axis::Slice {
start,
step: ind.step as u64,
count,
});
}
let np = py.import("numpy")?;
let is_bool =
a.is_instance_of::<pyo3::types::PyBool>() || a.is_instance(&np.getattr("bool_")?)?;
let is_array_like = a.is_instance(&np.getattr("ndarray")?)?
|| a.is_instance_of::<pyo3::types::PyList>()
|| a.is_instance_of::<PyTuple>();
if !is_bool && !is_array_like && a.hasattr("__index__")? {
let i: i128 = a.call_method0("__index__")?.extract()?;
return Ok(Axis::Index(normalize(i, n)?));
}
if is_array_like {
let arr = np.call_method1("asarray", (a,))?;
let kind: String = arr.getattr("dtype")?.getattr("kind")?.extract()?;
if kind == "b" {
return Err(PyTypeError::new_err(
"Boolean mask indexing is not supported by clawhdf5",
));
}
let ndim: usize = arr.getattr("ndim")?.extract()?;
let size: usize = arr.getattr("size")?.extract()?;
if size > 0 && kind != "i" && kind != "u" {
return Err(PyTypeError::new_err(
"Indexing arrays must have integer dtypes",
));
}
if ndim > 1 {
return Err(PyTypeError::new_err(
"Only 1-D integer lists or arrays can be used for fancy indexing",
));
}
let vals: Vec<i128> = arr.call_method0("tolist")?.extract()?;
let mut idx = Vec::with_capacity(vals.len());
for v in vals {
idx.push(normalize(v, n)?);
}
if idx.windows(2).any(|w| w[0] >= w[1]) {
return Err(PyTypeError::new_err(
"Indexing elements must be in increasing order",
));
}
return Ok(Axis::List(idx));
}
Err(PyTypeError::new_err(format!(
"Illegal index type for clawhdf5 datasets: {}",
a.get_type().name()?
)))
}
fn normalize(i: i128, n: u64) -> PyResult<u64> {
let n_i = i128::from(n);
let j = if i < 0 { i + n_i } else { i };
if j < 0 || j >= n_i {
let hi = n_i - 1;
return Err(PyIndexError::new_err(format!(
"Index ({i}) out of range for (0-{hi})"
)));
}
Ok(j as u64)
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn runs_group_consecutive_indices() {
assert_eq!(
consecutive_runs(&[1, 2, 3, 7, 9, 10]),
vec![(1, 3), (7, 1), (9, 2)]
);
assert_eq!(consecutive_runs(&[]), vec![]);
}
#[test]
fn full_selection_reads_everything() {
let plan = Plan {
axes: vec![
Axis::Slice {
start: 0,
step: 1,
count: 4,
},
Axis::Slice {
start: 0,
step: 1,
count: 3,
},
],
fields: vec![],
scalar: false,
};
let (reads, list) = plan.reads(&[4, 3]);
assert_eq!(list, None);
assert_eq!(reads, vec![(Selection::All, vec![4, 3])]);
}
#[test]
fn index_and_step_map_to_a_hyperslab() {
let plan = Plan {
axes: vec![
Axis::Index(2),
Axis::Slice {
start: 1,
step: 3,
count: 2,
},
],
fields: vec![],
scalar: false,
};
let (reads, _) = plan.reads(&[4, 8]);
assert_eq!(
reads,
vec![(
Selection::Hyperslab {
start: vec![2, 1],
stride: vec![1, 3],
count: vec![1, 2],
block: vec![1, 1],
},
vec![1, 2]
)]
);
assert_eq!(plan.out_shape(), vec![2]);
}
}
crates/clawhdf5-py/tests/conftest.py
+18
View File
@@ -0,0 +1,18 @@
"""Shared fixtures for the clawhdf5 Python binding tests."""
import os
import pytest
@pytest.fixture(scope="session")
def h5py():
"""h5py, or a skip — unless CLAWHDF5_REQUIRE_INTEROP=1, which makes a
missing h5py a failure (as for the Rust interop suites)."""
try:
import h5py as mod
except ImportError:
if os.environ.get("CLAWHDF5_REQUIRE_INTEROP") == "1":
pytest.fail("h5py is required (CLAWHDF5_REQUIRE_INTEROP=1) but not importable")
pytest.skip("h5py not installed")
return mod
crates/clawhdf5-py/tests/test_read_vs_h5py.py
+469
View File
@@ -0,0 +1,469 @@
"""Every read through clawhdf5 compared against h5py (libhdf5) on files h5py
writes: dtypes, shapes, values and the type of what comes back (array,
numpy scalar, bytes, str, Empty), for every datatype the bindings map and a
spread of index expressions; plus the errors h5py gives for the same keys."""
import threading
from concurrent.futures import ThreadPoolExecutor
import numpy as np
import pytest
import clawhdf5
# ---------------------------------------------------------------------------
# The generated file
# ---------------------------------------------------------------------------
NUMERIC = [
"<i1", "<i2", "<i4", "<i8", "<u1", "<u2", "<u4", "<u8",
">i2", ">i4", ">i8", ">u2", ">u4", ">u8",
"<f2", "<f4", "<f8", ">f2", ">f4", ">f8",
]
def _values(dtype, shape, seed):
rng = np.random.default_rng(seed)
dt = np.dtype(dtype)
n = int(np.prod(shape))
if dt.kind == "f":
return rng.standard_normal(n).astype(dt).reshape(shape)
info = np.iinfo(dt)
return rng.integers(info.min, info.max, size=n, dtype=dt.newbyteorder("=")).astype(dt).reshape(shape)
def _compound_dtype():
return np.dtype(
{
"names": ["id", "pos", "label", "flag", "vec"],
"formats": ["<i4", ">f8", "S6", "u1", ("<f4", (3,))],
"offsets": [0, 8, 16, 22, 24],
"itemsize": 40,
}
)
def _nested_dtype():
inner = np.dtype([("a", "<i2"), ("b", "<f4")])
return np.dtype([("x", "<u8"), ("inner", inner), ("c", "<c8")])
def _write_fixture(h5py, path):
str_dt = h5py.string_dtype()
ascii_dt = h5py.string_dtype("ascii")
with h5py.File(path, "w") as f:
# Numeric types, both byte orders, 1-D contiguous and 2-D chunked+gzip.
for i, dt in enumerate(NUMERIC):
name = dt.replace("<", "le_").replace(">", "be_")
f.create_dataset(f"num/{name}_1d", data=_values(dt, (37,), i))
f.create_dataset(
f"num/{name}_2d_gzip",
data=_values(dt, (13, 11), 100 + i),
chunks=(4, 5),
compression="gzip",
)
f.create_dataset("num/f8_3d", data=_values("<f8", (6, 7, 5), 7), chunks=(2, 3, 5))
f.create_dataset(
"num/i4_3d_shuffle",
data=_values("<i4", (5, 9, 4), 8),
chunks=(3, 4, 2),
shuffle=True,
fletcher32=True,
compression="gzip",
)
f.create_dataset("num/scalar_f8", data=np.float64(3.25))
f.create_dataset("num/scalar_i2_be", data=np.array(-7, dtype=">i2"))
f.create_dataset("num/zero_size", shape=(0, 3), dtype="<f4")
f.create_dataset("num/empty", data=h5py.Empty("<f8"))
sparse = f.create_dataset("num/sparse_fill", shape=(40,), chunks=(8,), dtype="<i4", fillvalue=-3)
sparse[10:14] = [1, 2, 3, 4]
f.create_dataset("num/resizable", data=np.arange(12.0), maxshape=(None,), chunks=(5,))
# Compact layout (low level: h5py's create_dataset cannot ask for it).
dcpl = h5py.h5p.create(h5py.h5p.DATASET_CREATE)
dcpl.set_layout(h5py.h5d.COMPACT)
space = h5py.h5s.create_simple((9,))
dsid = h5py.h5d.create(f.id, b"num/compact_u2", h5py.h5t.py_create(np.dtype("<u2")), space, dcpl=dcpl)
dsid.write(h5py.h5s.ALL, h5py.h5s.ALL, np.arange(9, dtype="<u2") * 7)
# bool, enum, complex.
f.create_dataset("misc/bool", data=np.array([True, False, True, True, False]))
enum_dt = h5py.enum_dtype({"RED": 0, "GREEN": 1, "BLUE": 42}, basetype="i2")
f.create_dataset("misc/enum", data=np.array([0, 42, 1, 1, 0], dtype="<i2"), dtype=enum_dt)
uenum_dt = h5py.enum_dtype({"LO": 0, "HI": 2**40}, basetype="u8")
f.create_dataset("misc/enum_u8", data=np.array([0, 2**40], dtype="<u8"), dtype=uenum_dt)
f.create_dataset("misc/c8", data=(np.arange(6) + 1j * np.arange(6)).astype("<c8"))
f.create_dataset("misc/c16_2d", data=(np.arange(12) - 2j).reshape(3, 4).astype("<c16"))
f.create_dataset("misc/opaque", data=np.array([b"\x00\x01\x02", b"\xff\xfe\xfd"], dtype="V3"))
# Strings.
f.create_dataset("str/fixed", data=np.array([b"alpha", b"be", b"", b"gamma!"], dtype="S6"))
utf8_4 = h5py.string_dtype("utf-8", 4)
f.create_dataset(
"str/fixed_2d_utf8",
data=np.array([["é".encode(), b"b"], [b"c", b"dd"]], dtype=utf8_4),
)
f.create_dataset("str/vlen", data=["", "one", "twø", "a" * 300, "ünïcödé"], dtype=str_dt)
f.create_dataset("str/vlen_ascii", data=[b"x", b"yy", b"zzz"], dtype=ascii_dt)
f.create_dataset(
"str/vlen_2d_gzip",
data=np.array([[f"r{r}c{c}" * (r + c) for c in range(5)] for r in range(6)], dtype=object),
dtype=str_dt,
chunks=(2, 2),
compression="gzip",
)
f.create_dataset("str/vlen_scalar", data="just one", dtype=str_dt)
# Variable-length sequences.
vl_i = h5py.vlen_dtype(np.dtype("<i4"))
seqs = np.empty(4, dtype=object)
seqs[:] = [np.arange(3, dtype="<i4"), np.array([], dtype="<i4"), np.arange(10, dtype="<i4") * -1, np.array([7], dtype="<i4")]
f.create_dataset("vlen/i4", data=seqs, dtype=vl_i)
vl_f = h5py.vlen_dtype(np.dtype(">f8"))
fseqs = np.empty(3, dtype=object)
fseqs[:] = [np.linspace(0, 1, 5).astype(">f8"), np.array([2.5], dtype=">f8"), np.array([], dtype=">f8")]
f.create_dataset("vlen/f8_be", data=fseqs, dtype=vl_f)
# Compounds.
cdt = _compound_dtype()
rec = np.zeros(10, dtype=cdt)
rec["id"] = np.arange(10) * 3
rec["pos"] = np.linspace(-1, 1, 10)
rec["label"] = [f"n{i}".encode() for i in range(10)]
rec["flag"] = np.arange(10) % 2
rec["vec"] = np.arange(30, dtype="<f4").reshape(10, 3)
f.create_dataset("cmp/padded", data=rec)
f.create_dataset("cmp/padded_chunked", data=rec, chunks=(3,), compression="gzip")
ndt = _nested_dtype()
nrec = np.zeros((4, 3), dtype=ndt)
nrec["x"] = np.arange(12).reshape(4, 3)
nrec["inner"]["a"] = -np.arange(12).reshape(4, 3)
nrec["inner"]["b"] = np.arange(12).reshape(4, 3) / 4
nrec["c"] = np.arange(12).reshape(4, 3) * (1 + 1j)
f.create_dataset("cmp/nested_2d", data=nrec)
vdt = np.dtype([("n", "<i4"), ("s", h5py.string_dtype())])
vrec = np.array([(1, "a"), (2, "bb")], dtype=vdt)
f.create_dataset("cmp/with_vlen", data=vrec)
# A true HDF5 array datatype (h5py's high level would widen the shape).
tid = h5py.h5t.array_create(h5py.h5t.py_create(np.dtype("<i4")), (2, 3))
space = h5py.h5s.create_simple((4,))
dsid = h5py.h5d.create(f.id, b"cmp/array_type", tid, space)
dsid.write(h5py.h5s.ALL, h5py.h5s.ALL, np.arange(24, dtype="<i4").reshape(4, 2, 3), mtype=tid)
# Unsupported: references.
f.create_dataset("unsupported/refs", data=[f["num"].ref, f["misc"].ref], dtype=h5py.ref_dtype)
# Groups and attributes.
g = f.create_group("deep/er/est")
g.create_dataset("leaf", data=np.arange(5))
f.create_group("empty_group")
f.attrs["i4"] = np.int32(-5)
f.attrs["u8"] = np.uint64(2**63 + 1)
f.attrs["f2"] = np.float16(1.5)
f.attrs["f8_arr"] = np.array([1.0, 2.5, -3.0])
f.attrs["f8_one"] = np.array([4.0])
f.attrs["i2_2d_be"] = np.arange(6, dtype=">i2").reshape(2, 3)
f.attrs["bool"] = True
f.attrs["bool_arr"] = np.array([True, False])
f.attrs["vstr"] = "héllo"
f.attrs["vstr_arr"] = ["a", "bcd", ""]
f.attrs["fstr"] = np.bytes_(b"fixed")
f.attrs["fstr_arr"] = np.array([b"x", b"yz"], dtype="S2")
f.attrs["empty"] = h5py.Empty("<i4")
f.attrs["complex"] = np.complex128(1 - 2j)
f.attrs["compound"] = np.array((7, 2.5), dtype=[("a", "<i2"), ("b", "<f8")])
f.attrs["enum"] = np.array(42, dtype=enum_dt)
f["num/le_f8_1d"].attrs["units"] = "m/s"
f["num/le_f8_1d"].attrs["scale"] = np.float32(0.5)
g.attrs["depth"] = np.int8(3)
@pytest.fixture(scope="module")
def pair(h5py, tmp_path_factory):
path = str(tmp_path_factory.mktemp("h5") / "fixture.h5")
_write_fixture(h5py, path)
theirs = h5py.File(path, "r")
ours = clawhdf5.File(path, "r")
yield ours, theirs, path
theirs.close()
ours.close()
def _all_datasets(h5py, f):
names = []
f.visititems(lambda n, o: names.append(n) if isinstance(o, h5py.Dataset) else None)
return sorted(names)
# ---------------------------------------------------------------------------
# Comparison helpers
# ---------------------------------------------------------------------------
def assert_same(ours, theirs, what=""):
if type(theirs).__name__ == "Empty":
assert isinstance(ours, clawhdf5.Empty), what
assert ours.dtype == theirs.dtype, what
return
assert type(ours) is type(theirs), f"{what}: {type(ours)} vs {type(theirs)}"
if isinstance(theirs, np.ndarray):
assert ours.shape == theirs.shape, what
assert ours.dtype == theirs.dtype, f"{what}: {ours.dtype} vs {theirs.dtype}"
if theirs.dtype == object:
for a, b in zip(ours.ravel(), theirs.ravel()):
assert_same(a, b, what)
elif theirs.dtype.names is None and theirs.dtype.kind == "V":
assert ours.tobytes() == theirs.tobytes(), what
else:
np.testing.assert_array_equal(ours, theirs, err_msg=what)
elif isinstance(theirs, np.generic):
assert ours.dtype == theirs.dtype, what
if theirs.dtype.names is not None:
np.testing.assert_array_equal(np.asarray(ours), np.asarray(theirs), err_msg=what)
else:
assert ours == theirs or (ours != ours and theirs != theirs), what
else:
assert ours == theirs, what
def keys_for(shape):
if shape == ():
return [(), Ellipsis]
keys = [(), Ellipsis, 0, -1, slice(None), slice(None, None, 2), slice(1, None, 3), slice(0, 0), np.int64(0)]
n0 = shape[0]
if n0 == 0:
return [(), Ellipsis, slice(None), slice(None, None, 2), slice(0, 0)]
keys += [slice(n0 // 2, None), slice(-3, None), [0, n0 - 1] if n0 > 1 else [0], (Ellipsis,)]
if n0 > 5:
keys += [[1, 2, 3, 5], [0, 4, 5]]
if len(shape) >= 2:
n1 = shape[1]
keys += [
(0, 0),
(-1, -1),
(slice(1, 3), slice(None, None, 2)),
(Ellipsis, 1),
(1, Ellipsis),
(slice(None), [0, n1 - 1] if n1 > 1 else [0]),
(slice(None, None, 2), 1),
(slice(0, 2), slice(3, 1)),
]
if len(shape) >= 3:
keys += [(0, slice(None), -1), (slice(1, None, 2), 2, slice(None, None, 3)), (Ellipsis, 0, 0), (0, Ellipsis, 1)]
return keys
# h5py 3.16 (HDF5 2.0) returns the elements of a variable-length sequence of
# big-endian floats unswapped (0.25 comes back as 2.6e-319), so these are
# checked against the values written instead (test_vlen_big_endian).
H5PY_MISREADS = {"vlen/f8_be"}
ERROR_KEYS_1D = [slice(None, None, -1), 10**6, -(10**6), None, (0, 0, 0, 0, 0), [3, 1], (Ellipsis, Ellipsis), 1.5, "nope"]
# ---------------------------------------------------------------------------
# Tests
# ---------------------------------------------------------------------------
def test_every_dataset_matches_h5py(h5py, pair):
ours, theirs, _ = pair
checked = 0
for name in _all_datasets(h5py, theirs):
if name.startswith("unsupported/") or name in H5PY_MISREADS or name == "cmp/with_vlen":
continue
t = theirs[name]
o = ours[name]
assert o.shape == t.shape, name
assert o.dtype == t.dtype, f"{name}: {o.dtype} vs {t.dtype}"
assert dict(o.dtype.metadata or {}) == dict(t.dtype.metadata or {}), name
assert o.ndim == t.ndim and o.size == t.size, name
assert o.maxshape == t.maxshape, name
assert o.name == t.name, name
if t.shape is None:
assert_same(o[()], t[()], name)
continue
for key in keys_for(t.shape):
what = f"{name}[{key!r}]"
try:
expected = t[key]
except Exception as e: # noqa: BLE001 - h5py refuses: so must we
with pytest.raises(type(e)):
o[key]
continue
assert_same(o[key], expected, what)
checked += 1
assert checked > 500
def test_vlen_big_endian(pair):
ours, _, _ = pair
ds = ours["vlen/f8_be"]
assert ds.dtype.metadata["vlen"] == np.dtype(">f8")
got = ds[()]
expected = [np.linspace(0, 1, 5), np.array([2.5]), np.array([])]
assert got.shape == (3,)
for g, e in zip(got, expected):
assert g.dtype == np.dtype(">f8")
np.testing.assert_array_equal(g, e)
np.testing.assert_array_equal(ds[1], [2.5])
def test_errors_match_h5py(h5py, pair):
ours, theirs, _ = pair
for name in ["num/le_i4_1d", "num/be_f8_2d_gzip", "str/vlen", "num/scalar_f8"]:
for key in ERROR_KEYS_1D:
try:
theirs[name][key]
except Exception as e: # noqa: BLE001
with pytest.raises(type(e)):
ours[name][key]
else:
pass # valid for this shape; covered by the value test
def test_compound_fields_match_h5py(pair):
ours, theirs, _ = pair
for name in ["cmp/padded", "cmp/padded_chunked", "cmp/nested_2d"]:
t, o = theirs[name], ours[name]
for field in t.dtype.names:
assert_same(o[field], t[field], f"{name}[{field}]")
assert_same(o[field, 1:3], t[field, 1:3], f"{name}[{field}, 1:3]")
two = list(t.dtype.names[:2])
expected = t[tuple(two)]
got = o[tuple(two)]
assert got.dtype.names == expected.dtype.names
for field in two:
np.testing.assert_array_equal(got[field], expected[field])
with pytest.raises(ValueError):
ours["cmp/padded"]["no_such_field"]
with pytest.raises(ValueError):
ours["num/le_i4_1d"]["id"]
def test_numpy_asarray_and_len(pair):
ours, theirs, _ = pair
assert_same(np.asarray(ours["num/f8_3d"]), np.asarray(theirs["num/f8_3d"]))
assert len(ours["num/f8_3d"]) == len(theirs["num/f8_3d"])
with pytest.raises(TypeError):
len(ours["num/scalar_f8"])
def test_attributes_match_h5py(h5py, pair):
ours, theirs, _ = pair
for path in ["/", "num/le_f8_1d", "deep/er/est"]:
t = theirs[path].attrs
o = ours[path].attrs
assert list(o.keys()) == sorted(t.keys()), path
assert len(o) == len(t)
for k in t.keys():
assert k in o
assert_same(o[k], t[k], f"{path}.attrs[{k}]")
assert o.get("missing", 5) == 5
with pytest.raises(KeyError):
o["missing"]
assert [k for k, _ in o.items()] == list(o.keys())
def test_groups_match_h5py(h5py, pair):
ours, theirs, _ = pair
assert list(ours.keys()) == list(theirs.keys())
assert len(ours) == len(theirs)
for path in ["num", "deep", "deep/er", "deep/er/est", "empty_group"]:
o, t = ours[path], theirs[path]
assert isinstance(o, clawhdf5.Group)
assert list(o.keys()) == list(t.keys()), path
assert list(o) == list(t), path
assert len(o) == len(t), path
assert o.name == t.name
g = ours["deep/er"]
assert isinstance(g["est"], clawhdf5.Group)
assert isinstance(g["est/leaf"], clawhdf5.Dataset)
assert g["/deep/er/est/leaf"].name == "/deep/er/est/leaf"
assert "est/leaf" in g and "/num" in g and "nope" not in g
assert ours["/"].name == "/"
assert "deep/er/est/leaf" in ours
assert ours.get("nope") is None
with pytest.raises(KeyError):
ours["deep/nope"]
names = [k for k, v in ours["deep/er/est"].items()]
assert names == ["leaf"]
assert isinstance(ours["deep/er/est"].values()[0], clawhdf5.Dataset)
def test_unsupported_types_are_errors_not_data(pair):
ours, _, _ = pair
ds = ours["unsupported/refs"] # opening works
with pytest.raises(TypeError):
ds.dtype
with pytest.raises(TypeError):
ds[()]
with pytest.raises(TypeError):
ours["cmp/with_vlen"][()]
def test_boolean_masks_are_refused(pair):
ours, _, _ = pair
with pytest.raises(TypeError):
ours["num/le_i4_1d"][np.ones(37, dtype=bool)]
def test_reads_hand_numpy_the_rust_buffer(pair):
"""A fixed-size read is a view over the buffer the library filled, not a
copy of it."""
ours, _, _ = pair
arr = ours["num/le_f8_2d_gzip"][2:9, 1:4]
assert not arr.flags.owndata
assert arr.base is not None
assert arr.flags.aligned and arr.flags.c_contiguous
def test_only_the_selected_chunks_are_read(h5py, tmp_path):
"""Damage one chunk: a selection that avoids it still reads, one that
touches it fails. Reading everything and slicing afterwards (what the
bindings did before) failed both. (The library reads the whole dataset
anyway when a selection's bounding box covers more than half of it, so
the selections here stay below that.)"""
path = str(tmp_path / "damaged.h5")
data = np.arange(1000, dtype="<f8")
with h5py.File(path, "w") as f:
f.create_dataset("d", data=data, chunks=(100,), compression="gzip")
with h5py.File(path, "r") as f:
info = f["d"].id.get_chunk_info(9) # the last chunk
with open(path, "r+b") as fh:
fh.seek(info.byte_offset)
fh.write(b"\xff" * info.size)
with clawhdf5.File(path, "r") as f:
ds = f["d"]
np.testing.assert_array_equal(ds[0:400], data[0:400])
np.testing.assert_array_equal(ds[805:900], data[805:900])
np.testing.assert_array_equal(ds[5:450:7], data[5:450:7])
np.testing.assert_array_equal(ds[[3, 450, 899]], data[[3, 450, 899]])
with pytest.raises(Exception):
ds[950]
with pytest.raises(Exception):
ds[:]
def test_threads_read_the_same_file(pair):
"""Reads from many threads at once (the GIL is released during each
read) return exactly what h5py returns."""
ours, theirs, _ = pair
names = ["num/le_f8_2d_gzip", "num/i4_3d_shuffle", "str/vlen_2d_gzip", "cmp/padded_chunked", "num/be_i8_1d"]
expected = {n: theirs[n][()] for n in names}
errors = []
barrier = threading.Barrier(8)
def work(i):
barrier.wait()
for r in range(40):
n = names[(i + r) % len(names)]
got = ours[n][()]
try:
assert_same(got, expected[n], n)
except AssertionError as e:
errors.append(e)
with ThreadPoolExecutor(8) as pool:
list(pool.map(work, range(8)))
assert not errors, errors[:3]
crates/clawhdf5-py/tests/test_write_read.py
+3 -3
View File
@@ -87,7 +87,7 @@ def test_dataset_dtype(sample_read_file):
def test_read_root_attrs(sample_read_file): def test_read_root_attrs(sample_read_file):
with clawhdf5.File(sample_read_file, "r") as f: with clawhdf5.File(sample_read_file, "r") as f:
assert f.attrs["version"] == 1 assert f.attrs["version"] == 1
assert f.attrs["description"] == "test file" assert f.attrs["description"] == b"test file" # fixed-length string: numpy.bytes_, as in h5py
def test_attrs_len(sample_read_file): def test_attrs_len(sample_read_file):
@@ -132,7 +132,7 @@ def test_read_group_dataset(grouped_read_file):
def test_read_group_attrs(grouped_read_file): def test_read_group_attrs(grouped_read_file):
with clawhdf5.File(grouped_read_file, "r") as f: with clawhdf5.File(grouped_read_file, "r") as f:
grp = f["sensors"] grp = f["sensors"]
assert grp.attrs["location"] == "lab" assert grp.attrs["location"] == b"lab"
def test_nested_path_access(grouped_read_file): def test_nested_path_access(grouped_read_file):
@@ -176,7 +176,7 @@ def test_write_with_attrs(tmp_h5):
f.attrs["author"] = "test" f.attrs["author"] = "test"
f.attrs["count"] = 42 f.attrs["count"] = 42
with clawhdf5.File(tmp_h5, "r") as f: with clawhdf5.File(tmp_h5, "r") as f:
assert f.attrs["author"] == "test" assert f.attrs["author"] == b"test"
assert f.attrs["count"] == 42 assert f.attrs["count"] == 42