quantumclaw/clawhdf5
3
1
Fork 0
Code Issues Pull Requests 3 Actions Packages Projects Releases 1 Wiki Activity

py: remote files (clawhdf5.File(url), File.open_url) through File::storage()

Browse Source 
The Python bindings could not open a remote file: they parsed through
File::as_bytes() in eight places (path lookups, object headers,
dataspaces, attributes, group listings, the global heap of
variable-length data), which a storage-backed file does not have.

- Every object of a File now shares one handle (src/handle.rs) that
  runs all file access, metadata included, with the GIL released and
  parses through File::storage() and the clawhdf5_format *_in functions.
  Local files take the same path (their storage is the mmap).
- clawhdf5.File(url) opens any scheme://... through
  clawhdf5_remote::storage_for_url (read-only; another mode is a
  ValueError). File.open_url(url, **options) takes the cache and HTTP
  options (block_size, cache_size, headers, retries, timeout,
  allow_full_download, max_full_download, require_validator,
  max_redirects, max_parallel); File.remote_stats gives the block
  cache's counters.
- Default build: plain HTTP only, no C. https (rustls/ring) and
  s3/gcs/azure (aws-lc-rs) are opt-in features of clawhdf5-py, and
  ci-test.sh's no-C check now covers the crate.
- A failed storage read (network error, file changed on the server) is an
  OSError, never KeyError/ValueError and never data; `key in group`
  raises it instead of answering False.

Tests: the read-vs-h5py suite runs locally and over HTTP (1 MiB and
1 KiB blocks) against a range-capable http.server in the test process
(conftest.RangeServer); test_remote.py covers request counts, cache
hits, a server without Range support, a changed file, a server that
hangs up, 16 threads, and a spinning thread that keeps running while a
read waits on 0.2 s requests.

Co-Authored-By: Claude Opus 5.5 (1M context) <[email protected]>
This commit is contained in:
osobh
2026-09-27 06:40:44 -05:00
co-authored by Claude Opus 5.5
parent a4c2aced55
commit 910d81904c
18 changed files with 1146 additions and 299 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/clawhdf5-py/src/dataset.rs
+87 -57
View File
@@ -6,21 +6,53 @@
//! whole dataset instead); the
//! bytes it returns become the numpy array's buffer without a copy (see
//! `convert`). All file access and decoding runs with the GIL released, so
//! Python threads reading the same or different datasets run in parallel.
//! Python threads reading the same or different datasets run in parallel,
//! and a remote file's network reads never hold the GIL.
use std::sync::Arc;
use clawhdf5_format::datatype::Datatype;
use clawhdf5_format::object_header::ObjectHeader;
use clawhdf5_rs::File;
use pyo3::exceptions::{PyTypeError, PyValueError};
use pyo3::prelude::*;
use pyo3::types::{PyList, PyTuple};
use crate::attrs::PyAttrs;
use crate::convert::{Converter, Elements, resolve_vl};
use crate::convert::{Converter, Elements, VlError, resolve_vl};
use crate::handle::Handle;
use crate::select::{self, Plan};
use crate::{PyEmpty, node, to_py_err};
/// What opening a dataset reads from the file (without the GIL).
pub(crate) struct DatasetMeta {
/// `None` for a dataset with a null dataspace (h5py's `Empty`).
shape: Option<Vec<u64>>,
chunks: Option<Vec<u64>>,
datatype: Datatype,
}
impl DatasetMeta {
pub(crate) fn load(f: &File, addr: u64, hdr: &ObjectHeader, path: &str) -> PyResult<Self> {
let null = node::is_null(&node::dataspace(f, hdr, path)?);
let ds = f.dataset_at(addr).map_err(to_py_err)?;
let shape = if null {
None
} else {
Some(ds.shape().map_err(to_py_err)?)
};
let datatype = ds.raw_datatype().map_err(to_py_err)?;
let chunks = shape
.as_ref()
.and_then(|s| node::chunk_shape(f, hdr, s.len()));
Ok(Self {
shape,
chunks,
datatype,
})
}
}
/// A dataset in a file opened for reading.
///
/// ```python
@@ -30,7 +62,7 @@ use crate::{PyEmpty, node, to_py_err};
/// ```
#[pyclass(name = "Dataset")]
pub struct PyDataset {
file: Arc<clawhdf5_rs::File>,
handle: Arc<Handle>,
path: String,
/// Where the dataset's object header is: reads open it from here rather
/// than resolve `path` again.
@@ -45,39 +77,24 @@ pub struct PyDataset {
}
impl PyDataset {
pub(crate) fn open(
pub(crate) fn new(
py: Python<'_>,
file: Arc<clawhdf5_rs::File>,
handle: Arc<Handle>,
path: String,
addr: u64,
hdr: &ObjectHeader,
) -> PyResult<Self> {
crate::no_panic(|| {
let null = node::is_null(&node::dataspace(&file, hdr)?);
let (shape, datatype) = {
let ds = file.dataset_at(addr).map_err(to_py_err)?;
let shape = if null {
None
} else {
Some(ds.shape().map_err(to_py_err)?)
};
(shape, ds.raw_datatype().map_err(to_py_err)?)
};
let conv = Converter::new(py, &datatype, file.superblock().offset_size)
.map_err(|e| e.value(py).to_string());
let chunks = shape
.as_ref()
.and_then(|s| node::chunk_shape(&file, hdr, s.len()));
Ok(Self {
file,
path,
addr,
shape,
chunks,
datatype,
conv,
})
})
meta: DatasetMeta,
) -> Self {
let conv = crate::no_panic(|| Converter::new(py, &meta.datatype, handle.offset_size))
.map_err(|e| e.value(py).to_string());
Self {
handle,
path,
addr,
shape: meta.shape,
chunks: meta.chunks,
datatype: meta.datatype,
conv,
}
}
fn converter(&self) -> PyResult<&Converter> {
@@ -103,10 +120,10 @@ impl PyDataset {
};
let (reads, list_axis) = plan.reads(dims, chunk_len, elem_size);
let read_shape = plan.read_shape();
let file = &*self.file;
let handle = &*self.handle;
let addr = self.addr;
// Everything below touches only Rust data: release the GIL.
let read = || -> Result<Elements, ReadError> {
let read = |file: &File| -> Result<Elements, ReadError> {
let ds = file.dataset_at(addr)?;
let mut blocks = Vec::with_capacity(reads.len());
for read in reads {
@@ -147,7 +164,7 @@ impl PyDataset {
let sb = file.superblock();
let n = read_shape.iter().product();
resolve_vl(
file.as_bytes(),
file.storage(),
&raw,
n,
sb.offset_size,
@@ -155,12 +172,20 @@ impl PyDataset {
unit,
)
.map(Elements::Vl)
.map_err(ReadError::Other)
.map_err(ReadError::Vl)
};
let data = py
.detach(|| {
std::panic::catch_unwind(std::panic::AssertUnwindSafe(read))
.unwrap_or_else(|p| Err(ReadError::Panic(crate::panic_text(&*p))))
handle
.with_detached(|f| {
Ok(
std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| read(f)))
.unwrap_or_else(|p| {
Err(ReadError::Panic(crate::panic_text(&*p)))
}),
)
})
.unwrap_or_else(|e| Err(ReadError::Py(e)))
})
.map_err(|e| e.into_py(&self.path))?;
let joined = conv.to_array(py, data, &read_shape, false)?;
@@ -183,6 +208,8 @@ impl PyDataset {
/// An error from the read closure, turned into a Python error with the GIL.
enum ReadError {
Lib(clawhdf5_rs::Error),
Vl(VlError),
Py(PyErr),
Other(String),
Panic(String),
}
@@ -197,6 +224,8 @@ impl ReadError {
fn into_py(self, path: &str) -> PyErr {
match self {
ReadError::Lib(e) => to_py_err(e),
ReadError::Vl(e) => e.into_py(&node::name(path)),
ReadError::Py(e) => e,
ReadError::Other(msg) => PyValueError::new_err(format!("{}: {msg}", node::name(path))),
ReadError::Panic(msg) => crate::InternalError::new_err(format!(
"{}: clawhdf5 internal error (please report it): {msg}",
@@ -252,22 +281,23 @@ impl PyDataset {
/// The maximum shape (`None` per unlimited dimension), like h5py.
#[getter]
fn maxshape<'py>(&self, py: Python<'py>) -> PyResult<Bound<'py, PyAny>> {
crate::no_panic(|| {
let Some(shape) = &self.shape else {
return Ok(py.None().into_bound(py));
};
let max = self
.file
.dataset_at(self.addr)
.and_then(|ds| ds.max_dimensions())
.map_err(to_py_err)?
.unwrap_or_else(|| shape.clone());
let items: Vec<Option<u64>> = max
.into_iter()
.map(|d| (d != u64::MAX).then_some(d))
.collect();
Ok(PyTuple::new(py, items)?.into_any())
})
let Some(shape) = &self.shape else {
return Ok(py.None().into_bound(py));
};
let addr = self.addr;
let max = self
.handle
.with(py, |f| {
f.dataset_at(addr)
.and_then(|ds| ds.max_dimensions())
.map_err(to_py_err)
})?
.unwrap_or_else(|| shape.clone());
let items: Vec<Option<u64>> = max
.into_iter()
.map(|d| (d != u64::MAX).then_some(d))
.collect();
Ok(PyTuple::new(py, items)?.into_any())
}
/// The dataset's numpy dtype, as h5py reports it.
@@ -295,8 +325,8 @@ impl PyDataset {
/// The dataset's attributes (read-only, dict-like).
#[getter]
fn attrs(&self) -> PyResult<PyAttrs> {
PyAttrs::read(Arc::clone(&self.file), self.addr, &self.path)
fn attrs(&self, py: Python<'_>) -> PyResult<PyAttrs> {
PyAttrs::read(py, Arc::clone(&self.handle), self.addr, &self.path)
}
/// Read with h5py indexing: integers, slices with positive steps,