feat(py): h5py-style reads of only the selected elements, GIL released

ds[key] read the whole dataset and sliced it in numpy, and knew six
dtypes. Keys (ints, positive-step slices, Ellipsis, one increasing index
list, compound field names) now map onto hyperslab selections, and the
facade's read_selection bytes become the numpy buffer without a copy
(PyArray::from_vec viewed as the dtype). dtype mapping follows h5py for
all integer/IEEE float widths and byte orders, bool, enum, complex, fixed
and variable-length strings, vlen sequences, opaque, array types and
(nested, padded) compounds; anything it cannot describe exactly is a
TypeError. Attributes return what h5py returns; groups and files gain
the rest of the h5py mapping interface. Reads run under py.detach.

tests/test_read_vs_h5py.py compares >500 reads with h5py 3.16 on an
h5py-written file, checks errors match, that a damaged chunk outside the
selection is never touched, and 8 threads reading at once.

Co-Authored-By: Claude Opus 5.5 (1M context) <[email protected]>

crates/clawhdf5-py/src/node.rs

@@ -0,0 +1,168 @@
+//! Resolving paths to objects in a file opened for reading.
+
+use std::sync::Arc;
+
+use clawhdf5_format::attribute::AttributeMessage;
+use clawhdf5_format::dataspace::{Dataspace, DataspaceType};
+use clawhdf5_format::message_type::MessageType;
+use clawhdf5_format::object_header::ObjectHeader;
+use pyo3::exceptions::{PyKeyError, PyTypeError, PyValueError};
+use pyo3::prelude::*;
+
+use crate::dataset::PyDataset;
+use crate::group::PyGroup;
+
+/// Join `key` onto the group path `base` the way h5py does: an absolute key
+/// starts from the root, a relative one from `base`. Paths are kept without
+/// a leading `/`; the root is `""`.
+pub(crate) fn join(base: &str, key: &str) -> String {
+    let parts = if key.starts_with('/') {
+        key.split('/').collect::<Vec<_>>()
+    } else {
+        base.split('/').chain(key.split('/')).collect()
+    };
+    parts
+        .into_iter()
+        .filter(|p| !p.is_empty() && *p != ".")
+        .collect::<Vec<_>>()
+        .join("/")
+}
+
+/// The HDF5 name (`/a/b`) of a path.
+pub(crate) fn name(path: &str) -> String {
+    format!("/{path}")
+}
+
+/// The object header of the object at `path`.
+pub(crate) fn header(file: &clawhdf5_rs::File, path: &str) -> PyResult<ObjectHeader> {
+    let sb = file.superblock();
+    let data = file.as_bytes();
+    let addr = if path.is_empty() {
+        sb.root_group_address
+    } else {
+        clawhdf5_format::group_v2::resolve_path_any(data, sb, path).map_err(|e| {
+            PyKeyError::new_err(format!(
+                "Unable to open object (object '{}' doesn't exist): {e}",
+                name(path)
+            ))
+        })?
+    };
+    let addr = usize::try_from(addr)
+        .map_err(|_| PyValueError::new_err(format!("{}: address out of range", name(path))))?;
+    ObjectHeader::parse(data, addr, sb.offset_size, sb.length_size)
+        .map_err(|e| PyValueError::new_err(format!("{}: {e}", name(path))))
+}
+
+/// What an object header describes.
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub(crate) enum Kind {
+    Dataset,
+    Group,
+    Datatype,
+}
+
+pub(crate) fn kind(hdr: &ObjectHeader) -> Option<Kind> {
+    let has = |t: MessageType| hdr.messages.iter().any(|m| m.msg_type == t);
+    if has(MessageType::DataLayout) {
+        Some(Kind::Dataset)
+    } else if has(MessageType::LinkInfo)
+        || has(MessageType::Link)
+        || has(MessageType::SymbolTable)
+        || has(MessageType::GroupInfo)
+    {
+        Some(Kind::Group)
+    } else if has(MessageType::Datatype) {
+        Some(Kind::Datatype)
+    } else {
+        None
+    }
+}
+
+/// Open the object at `path` as a `Dataset` or `Group`.
+pub(crate) fn open(
+    py: Python<'_>,
+    file: &Arc<clawhdf5_rs::File>,
+    path: String,
+) -> PyResult<Py<PyAny>> {
+    let hdr = header(file, &path)?;
+    match kind(&hdr) {
+        Some(Kind::Dataset) => Ok(PyDataset::open(py, Arc::clone(file), path)?
+            .into_pyobject(py)?
+            .into_any()
+            .unbind()),
+        Some(Kind::Group) => Ok(PyGroup::from_read(Arc::clone(file), path)
+            .into_pyobject(py)?
+            .into_any()
+            .unbind()),
+        Some(Kind::Datatype) => Err(PyTypeError::new_err(format!(
+            "{}: committed (named) datatypes are not supported by clawhdf5",
+            name(&path)
+        ))),
+        None => Err(PyValueError::new_err(format!(
+            "{}: not a dataset, group or datatype",
+            name(&path)
+        ))),
+    }
+}
+
+/// Whether `path` names a dataset or group.
+pub(crate) fn exists(file: &clawhdf5_rs::File, path: &str) -> bool {
+    header(file, path)
+        .ok()
+        .and_then(|h| kind(&h))
+        .is_some_and(|k| k != Kind::Datatype)
+}
+
+/// The dataspace message of an object header.
+pub(crate) fn dataspace(file: &clawhdf5_rs::File, hdr: &ObjectHeader) -> PyResult<Dataspace> {
+    let sb = file.superblock();
+    let msg = hdr
+        .messages
+        .iter()
+        .find(|m| m.msg_type == MessageType::Dataspace)
+        .ok_or_else(|| PyValueError::new_err("object has no dataspace message"))?;
+    let data = clawhdf5_format::shared_message::message_data(
+        file.as_bytes(),
+        msg,
+        sb.offset_size,
+        sb.length_size,
+    )
+    .map_err(|e| PyValueError::new_err(e.to_string()))?;
+    Dataspace::parse(&data, sb.length_size).map_err(|e| PyValueError::new_err(e.to_string()))
+}
+
+pub(crate) fn is_null(space: &Dataspace) -> bool {
+    space.space_type == DataspaceType::Null
+}
+
+/// The attributes of the object at `path`, sorted by name (h5py's order).
+/// Attributes whose messages cannot be parsed are left out, as the facade's
+/// `attrs()` does.
+pub(crate) fn attributes(file: &clawhdf5_rs::File, path: &str) -> PyResult<Vec<AttributeMessage>> {
+    let hdr = header(file, path)?;
+    let sb = file.superblock();
+    let (mut attrs, _errors) = clawhdf5_format::attribute::extract_attributes_tolerant(
+        file.as_bytes(),
+        &hdr,
+        sb.offset_size,
+        sb.length_size,
+    )
+    .map_err(|e| PyValueError::new_err(format!("{}: {e}", name(path))))?;
+    attrs.sort_by(|a, b| a.name.as_bytes().cmp(b.name.as_bytes()));
+    Ok(attrs)
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn join_paths() {
+        assert_eq!(join("", "a"), "a");
+        assert_eq!(join("a", "b/c"), "a/b/c");
+        assert_eq!(join("a/b", "/x"), "x");
+        assert_eq!(join("a", "/"), "");
+        assert_eq!(join("", "/a//b/"), "a/b");
+        assert_eq!(join("a", "./b"), "a/b");
+    }
+}