py: remote files (clawhdf5.File(url), File.open_url) through File::storage()

The Python bindings could not open a remote file: they parsed through
File::as_bytes() in eight places (path lookups, object headers,
dataspaces, attributes, group listings, the global heap of
variable-length data), which a storage-backed file does not have.

- Every object of a File now shares one handle (src/handle.rs) that
  runs all file access, metadata included, with the GIL released and
  parses through File::storage() and the clawhdf5_format *_in functions.
  Local files take the same path (their storage is the mmap).
- clawhdf5.File(url) opens any scheme://... through
  clawhdf5_remote::storage_for_url (read-only; another mode is a
  ValueError). File.open_url(url, **options) takes the cache and HTTP
  options (block_size, cache_size, headers, retries, timeout,
  allow_full_download, max_full_download, require_validator,
  max_redirects, max_parallel); File.remote_stats gives the block
  cache's counters.
- Default build: plain HTTP only, no C. https (rustls/ring) and
  s3/gcs/azure (aws-lc-rs) are opt-in features of clawhdf5-py, and
  ci-test.sh's no-C check now covers the crate.
- A failed storage read (network error, file changed on the server) is an
  OSError, never KeyError/ValueError and never data; `key in group`
  raises it instead of answering False.

Tests: the read-vs-h5py suite runs locally and over HTTP (1 MiB and
1 KiB blocks) against a range-capable http.server in the test process
(conftest.RangeServer); test_remote.py covers request counts, cache
hits, a server without Range support, a changed file, a server that
hangs up, 16 threads, and a spinning thread that keeps running while a
read waits on 0.2 s requests.

Co-Authored-By: Claude Opus 5.5 (1M context) <[email protected]>

crates/clawhdf5-py/src/handle.rs

@@ -0,0 +1,130 @@
+//! The open file every object of a `File` shares.
+//!
+//! Every read goes through [`Handle::with`], which releases the GIL and
+//! parses through `File::storage()`, so the same code serves a local file
+//! (memory-mapped) and a remote one (`clawhdf5-remote`: range requests
+//! through a block cache, so a network read never holds the GIL).
+//!
+//! Lock discipline (no deadlock with the GIL): the file lock is only taken
+//! with the GIL released, and code that holds it never touches Python.
+
+use std::path::PathBuf;
+use std::sync::{Arc, PoisonError, RwLock};
+
+use clawhdf5_rs::File;
+use pyo3::exceptions::PyOSError;
+use pyo3::prelude::*;
+
+use crate::to_py_err;
+
+/// Where the file's bytes come from.
+pub(crate) enum Source {
+    /// A local path (memory-mapped).
+    Local(#[allow(dead_code)] PathBuf),
+    /// A URL, read through `clawhdf5-remote`'s block cache.
+    Remote {
+        url: String,
+        storage: Arc<clawhdf5_remote::RemoteStorage>,
+    },
+}
+
+pub(crate) struct Handle {
+    file: RwLock<Option<File>>,
+    source: Source,
+    pub offset_size: u8,
+    pub length_size: u8,
+    pub root: u64,
+}
+
+fn closed_after_failed_reopen() -> PyErr {
+    PyOSError::new_err("the file could not be reopened after an edit; open it again")
+}
+
+impl Handle {
+    fn new(file: File, source: Source) -> Arc<Self> {
+        let sb = file.superblock();
+        let (offset_size, length_size, root) =
+            (sb.offset_size, sb.length_size, sb.root_group_address);
+        Arc::new(Self {
+            file: RwLock::new(Some(file)),
+            source,
+            offset_size,
+            length_size,
+            root,
+        })
+    }
+
+    /// A local file, read-only.
+    pub(crate) fn open_local(py: Python<'_>, path: &str) -> PyResult<Arc<Self>> {
+        let file = py.detach(|| crate::no_panic(|| File::open(path).map_err(to_py_err)))?;
+        Ok(Self::new(file, Source::Local(PathBuf::from(path))))
+    }
+
+    /// A remote file (`http(s)://`, `s3://`, ...).
+    pub(crate) fn open_url(
+        py: Python<'_>,
+        url: &str,
+        options: &clawhdf5_remote::Options,
+    ) -> PyResult<Arc<Self>> {
+        let (file, storage) = py.detach(|| {
+            crate::no_panic(|| {
+                let storage = clawhdf5_remote::storage_for_url(url, options).map_err(remote_err)?;
+                let file = File::open_storage(storage.clone()).map_err(to_py_err)?;
+                Ok((file, storage))
+            })
+        })?;
+        Ok(Self::new(
+            file,
+            Source::Remote {
+                url: url.to_string(),
+                storage,
+            },
+        ))
+    }
+
+    /// Run `f` on the file with the GIL released (a remote read may wait
+    /// on the network; other Python threads run meanwhile). `f` must not
+    /// touch Python.
+    pub(crate) fn with<R: Send>(
+        &self,
+        py: Python<'_>,
+        f: impl FnOnce(&File) -> PyResult<R> + Send,
+    ) -> PyResult<R> {
+        py.detach(|| self.with_detached(f))
+    }
+
+    /// [`with`](Self::with) for code that already runs without the GIL.
+    pub(crate) fn with_detached<R>(&self, f: impl FnOnce(&File) -> PyResult<R>) -> PyResult<R> {
+        crate::no_panic(|| {
+            let guard = self.file.read().unwrap_or_else(PoisonError::into_inner);
+            let file = guard.as_ref().ok_or_else(closed_after_failed_reopen)?;
+            f(file)
+        })
+    }
+
+    /// The remote file's block cache.
+    pub(crate) fn remote_storage(&self) -> Option<&clawhdf5_remote::RemoteStorage> {
+        match &self.source {
+            Source::Remote { storage, .. } => Some(storage),
+            Source::Local(_) => None,
+        }
+    }
+
+    /// The URL of a remote file, credentials and query values redacted.
+    pub(crate) fn redacted_url(&self) -> Option<String> {
+        match &self.source {
+            Source::Remote { url, .. } => Some(clawhdf5_remote::redact_url(url)),
+            Source::Local(_) => None,
+        }
+    }
+}
+
+/// A `clawhdf5_remote::Error` as a Python exception: the network side
+/// (unreachable, a status, no range support, a changed file) is `OSError`,
+/// a file that is not HDF5 is what `to_py_err` makes of it.
+pub(crate) fn remote_err(e: clawhdf5_remote::Error) -> PyErr {
+    match e {
+        clawhdf5_remote::Error::Hdf5(e) => to_py_err(e),
+        other => PyOSError::new_err(other.to_string()),
+    }
+}