feat(py): h5py-style reads of only the selected elements, GIL released

ds[key] read the whole dataset and sliced it in numpy, and knew six
dtypes. Keys (ints, positive-step slices, Ellipsis, one increasing index
list, compound field names) now map onto hyperslab selections, and the
facade's read_selection bytes become the numpy buffer without a copy
(PyArray::from_vec viewed as the dtype). dtype mapping follows h5py for
all integer/IEEE float widths and byte orders, bool, enum, complex, fixed
and variable-length strings, vlen sequences, opaque, array types and
(nested, padded) compounds; anything it cannot describe exactly is a
TypeError. Attributes return what h5py returns; groups and files gain
the rest of the h5py mapping interface. Reads run under py.detach.

tests/test_read_vs_h5py.py compares >500 reads with h5py 3.16 on an
h5py-written file, checks errors match, that a damaged chunk outside the
selection is never touched, and 8 threads reading at once.

Co-Authored-By: Claude Opus 5.5 (1M context) <[email protected]>

crates/clawhdf5-py/src/file.rs

@@ -3,11 +3,12 @@
 use std::path::PathBuf;
 use std::sync::{Arc, Mutex};
 
+use pyo3::exceptions::PyKeyError;
 use pyo3::prelude::*;
+use pyo3::types::PyList;
 
 use crate::attrs::PyAttrs;
-use crate::dataset::PyDataset;
-use crate::group::{PyGroup, WriteGroupState, finalize_write_group};
+use crate::group::{self, PyGroup, WriteGroupState, finalize_write_group};
 use crate::{DatasetSpec, OwnedAttrValue, apply_dataset_spec, extract_numpy_data, to_py_err};
 
 /// Internal state for write mode.
@@ -35,6 +36,7 @@ struct WriteState {
 #[pyclass(name = "File")]
 pub struct PyFile {
     inner: Option<FileInner>,
+    filename: String,
 }
 
 enum FileInner {
@@ -51,15 +53,20 @@ impl PyFile {
     ///   mode: 'r' for read (default), 'w' for write
     #[new]
     #[pyo3(signature = (path, mode="r"))]
-    fn new(path: &str, mode: &str) -> PyResult<Self> {
+    fn new(py: Python<'_>, path: &str, mode: &str) -> PyResult<Self> {
+        let filename = path.to_string();
         match mode {
             "r" => {
-                let file = clawhdf5_rs::File::open(path).map_err(to_py_err)?;
+                let file = py
+                    .detach(|| clawhdf5_rs::File::open(path))
+                    .map_err(to_py_err)?;
                 Ok(Self {
                     inner: Some(FileInner::Read(Arc::new(file))),
+                    filename,
                 })
             }
             "w" => Ok(Self {
+                filename,
                 inner: Some(FileInner::Write(WriteState {
                     path: PathBuf::from(path),
                     root_datasets: Vec::new(),
@@ -101,44 +108,56 @@ impl PyFile {
         Ok(false) // don't suppress exceptions
     }
 
-    /// Get a child object (dataset or group) by path.
+    /// Get a child object (dataset or group) by path; `f['/']` is the root.
     fn __getitem__(&self, py: Python<'_>, key: &str) -> PyResult<Py<PyAny>> {
-        let file = self.read_file()?;
-        // Try dataset first
-        match file.dataset(key) {
-            Ok(_) => {
-                let ds = PyDataset::new(Arc::clone(file), key.to_string())?;
-                Ok(ds.into_pyobject(py)?.into_any().unbind())
-            }
-            Err(clawhdf5_rs::Error::NotADataset(_)) => {
-                let grp = PyGroup::from_read(Arc::clone(file), key.to_string());
-                Ok(grp.into_pyobject(py)?.into_any().unbind())
-            }
-            Err(_) => {
-                // Could be a group (no DataLayout message, no error)
-                match file.group(key) {
-                    Ok(_) => {
-                        let grp = PyGroup::from_read(Arc::clone(file), key.to_string());
-                        Ok(grp.into_pyobject(py)?.into_any().unbind())
-                    }
-                    Err(e) => Err(PyErr::new::<pyo3::exceptions::PyKeyError, _>(format!(
-                        "{key}: {e}"
-                    ))),
-                }
+        group::get_item(py, self.read_file()?, "", key)
+    }
+
+    /// `f.get(key, default=None)`.
+    #[pyo3(signature = (key, default=None))]
+    fn get(&self, py: Python<'_>, key: &str, default: Option<Py<PyAny>>) -> PyResult<Py<PyAny>> {
+        match group::get_item(py, self.read_file()?, "", key) {
+            Err(e) if e.is_instance_of::<PyKeyError>(py) => {
+                Ok(default.unwrap_or_else(|| py.None()))
             }
+            other => other,
         }
     }
 
     /// List the names of all children in the root group.
     fn keys(&self, py: Python<'_>) -> PyResult<Py<PyAny>> {
-        let file = self.read_file()?;
-        let root = file.root();
-        let mut names = root.datasets().map_err(to_py_err)?;
-        let groups = root.groups().map_err(to_py_err)?;
-        names.extend(groups);
-        names.sort();
-        let list = pyo3::types::PyList::new(py, &names)?;
-        Ok(list.into_any().unbind())
+        let names = group::member_names(self.read_file()?, "")?;
+        Ok(PyList::new(py, names)?.into_any().unbind())
+    }
+
+    fn values(&self, py: Python<'_>) -> PyResult<Py<PyAny>> {
+        let vals = group::values(py, self.read_file()?, "")?;
+        Ok(PyList::new(py, vals)?.into_any().unbind())
+    }
+
+    fn items(&self, py: Python<'_>) -> PyResult<Py<PyAny>> {
+        let items = group::items(py, self.read_file()?, "")?;
+        Ok(PyList::new(py, items)?.into_any().unbind())
+    }
+
+    fn __iter__(&self, py: Python<'_>) -> PyResult<Py<PyAny>> {
+        self.keys(py)?.call_method0(py, "__iter__")
+    }
+
+    fn __len__(&self) -> PyResult<usize> {
+        Ok(group::member_names(self.read_file()?, "")?.len())
+    }
+
+    /// The root group's name, `/`.
+    #[getter]
+    fn name(&self) -> &'static str {
+        "/"
+    }
+
+    /// The path the file was opened with.
+    #[getter]
+    fn filename(&self) -> &str {
+        &self.filename
     }
 
     /// Create a dataset in the root group (write mode only).
@@ -192,10 +211,7 @@ impl PyFile {
     #[getter]
     fn attrs(&self) -> PyResult<PyAttrs> {
         match self.inner.as_ref() {
-            Some(FileInner::Read(file)) => {
-                let map = file.root().attrs().map_err(to_py_err)?;
-                Ok(PyAttrs::from_read(map))
-            }
+            Some(FileInner::Read(file)) => PyAttrs::read(Arc::clone(file), ""),
             Some(FileInner::Write(state)) => Ok(PyAttrs::from_write(Arc::clone(&state.root_attrs))),
             None => Err(PyErr::new::<pyo3::exceptions::PyIOError, _>(
                 "file is closed",
@@ -216,8 +232,7 @@ impl PyFile {
     }
 
     fn __contains__(&self, key: &str) -> PyResult<bool> {
-        let file = self.read_file()?;
-        Ok(file.dataset(key).is_ok() || file.group(key).is_ok())
+        Ok(group::contains(self.read_file()?, "", key))
     }
 }