quantumclaw/clawhdf5
3
1
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 1 Wiki Activity

py: remote files (clawhdf5.File(url), File.open_url) through File::storage()

Browse Source 
The Python bindings could not open a remote file: they parsed through
File::as_bytes() in eight places (path lookups, object headers,
dataspaces, attributes, group listings, the global heap of
variable-length data), which a storage-backed file does not have.

- Every object of a File now shares one handle (src/handle.rs) that
  runs all file access, metadata included, with the GIL released and
  parses through File::storage() and the clawhdf5_format *_in functions.
  Local files take the same path (their storage is the mmap).
- clawhdf5.File(url) opens any scheme://... through
  clawhdf5_remote::storage_for_url (read-only; another mode is a
  ValueError). File.open_url(url, **options) takes the cache and HTTP
  options (block_size, cache_size, headers, retries, timeout,
  allow_full_download, max_full_download, require_validator,
  max_redirects, max_parallel); File.remote_stats gives the block
  cache's counters.
- Default build: plain HTTP only, no C. https (rustls/ring) and
  s3/gcs/azure (aws-lc-rs) are opt-in features of clawhdf5-py, and
  ci-test.sh's no-C check now covers the crate.
- A failed storage read (network error, file changed on the server) is an
  OSError, never KeyError/ValueError and never data; `key in group`
  raises it instead of answering False.

Tests: the read-vs-h5py suite runs locally and over HTTP (1 MiB and
1 KiB blocks) against a range-capable http.server in the test process
(conftest.RangeServer); test_remote.py covers request counts, cache
hits, a server without Range support, a changed file, a server that
hangs up, 16 threads, and a spinning thread that keeps running while a
read waits on 0.2 s requests.

Co-Authored-By: Claude Opus 5.5 (1M context) <[email protected]>
This commit is contained in:
osobh
2026-09-27 06:40:44 -05:00
co-authored by Claude Opus 5.5
parent a4c2aced55
commit 910d81904c
18 changed files with 1146 additions and 299 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/clawhdf5-py/src/attrs.rs
+23 -19
View File
@@ -8,13 +8,14 @@ use pyo3::prelude::*;
use pyo3::types::{PyList, PyTuple};
use crate::convert::{Converter, Elements, resolve_vl};
use crate::handle::Handle;
use crate::{OwnedAttrValue, PyEmpty, attr_value_to_py, node, py_to_attr_value};
/// Backing storage for attributes.
enum AttrsInner {
/// Attributes of an object in a file opened for reading, sorted by name.
Read {
file: Arc<clawhdf5_rs::File>,
handle: Arc<Handle>,
attrs: Vec<AttributeMessage>,
},
/// Writable attribute list shared with a parent (PyFile or PyGroup).
@@ -36,10 +37,15 @@ pub struct PyAttrs {
impl PyAttrs {
/// The attributes of the object at `addr` (whose path is `path`) in a
/// file opened for reading.
pub(crate) fn read(file: Arc<clawhdf5_rs::File>, addr: u64, path: &str) -> PyResult<Self> {
let attrs = node::attributes(&file, addr, path)?;
pub(crate) fn read(
py: Python<'_>,
handle: Arc<Handle>,
addr: u64,
path: &str,
) -> PyResult<Self> {
let attrs = handle.with(py, |f| node::attributes(f, addr, path))?;
Ok(Self {
inner: AttrsInner::Read { file, attrs },
inner: AttrsInner::Read { handle, attrs },
})
}
@@ -55,8 +61,8 @@ impl PyAttrs {
impl PyAttrs {
fn __getitem__(&self, py: Python<'_>, key: &str) -> PyResult<Py<PyAny>> {
match &self.inner {
AttrsInner::Read { file, attrs } => match attrs.iter().find(|a| a.name == key) {
Some(attr) => Ok(attr_to_py(py, file, attr)?.unbind()),
AttrsInner::Read { handle, attrs } => match attrs.iter().find(|a| a.name == key) {
Some(attr) => Ok(attr_to_py(py, handle, attr)?.unbind()),
None => Err(PyKeyError::new_err(format!(
"Can't open attribute (can't locate attribute: '{key}')"
))),
@@ -146,9 +152,9 @@ impl PyAttrs {
/// Return attribute values as a list.
fn values(&self, py: Python<'_>) -> PyResult<Py<PyAny>> {
let vals: Vec<Py<PyAny>> = match &self.inner {
AttrsInner::Read { file, attrs } => attrs
AttrsInner::Read { handle, attrs } => attrs
.iter()
.map(|a| attr_to_py(py, file, a).map(Bound::unbind))
.map(|a| attr_to_py(py, handle, a).map(Bound::unbind))
.collect::<PyResult<_>>()?,
AttrsInner::Write(store) => store
.lock()
@@ -167,9 +173,9 @@ impl PyAttrs {
/// Return attribute (key, value) pairs as a list of tuples.
fn items(&self, py: Python<'_>) -> PyResult<Py<PyAny>> {
let pairs: Vec<(String, Py<PyAny>)> = match &self.inner {
AttrsInner::Read { file, attrs } => attrs
AttrsInner::Read { handle, attrs } => attrs
.iter()
.map(|a| Ok((a.name.clone(), attr_to_py(py, file, a)?.unbind())))
.map(|a| Ok((a.name.clone(), attr_to_py(py, handle, a)?.unbind())))
.collect::<PyResult<_>>()?,
AttrsInner::Write(store) => store
.lock()
@@ -189,12 +195,11 @@ impl PyAttrs {
/// An attribute's value as h5py returns it.
fn attr_to_py<'py>(
py: Python<'py>,
file: &clawhdf5_rs::File,
handle: &Handle,
attr: &AttributeMessage,
) -> PyResult<Bound<'py, PyAny>> {
crate::no_panic(|| {
let sb = file.superblock();
let conv = Converter::new(py, &attr.datatype, sb.offset_size)
let conv = Converter::new(py, &attr.datatype, handle.offset_size)
.map_err(|e| prefix_err(py, &attr.name, e))?;
if node::is_null(&attr.dataspace) {
return Ok(PyEmpty::new(conv.dtype).into_pyobject(py)?.into_any());
@@ -216,12 +221,11 @@ fn attr_to_py<'py>(
)));
}
let raw = &attr.raw_data[..want];
let file_data = file.as_bytes();
let (osz, lsz, unit) = (sb.offset_size, sb.length_size, conv.vl_unit);
Elements::Vl(
py.detach(|| resolve_vl(file_data, raw, n, osz, lsz, unit))
.map_err(|e| PyValueError::new_err(format!("attribute {}: {e}", attr.name)))?,
)
let (osz, lsz, unit) = (handle.offset_size, handle.length_size, conv.vl_unit);
let what = format!("attribute {}", attr.name);
Elements::Vl(handle.with(py, |f| {
resolve_vl(f.storage(), raw, n, osz, lsz, unit).map_err(|e| e.into_py(&what))
})?)
} else {
Elements::Bytes(attr.raw_data.clone())
};