clawhdf5-remote: object stores through object_store (S3, GCS, Azure)

ObjectStoreStorage (feature `object-store`, pure Rust) reads one object
of any object_store store by ranged get_opts, pinned at open by a head
request: If-Match with its ETag (and the ETag and size of every response
compared), else its version or modification time. A change is
RemoteError::FileChanged. object_store is async and Storage is not, so
the storage owns a small multi-threaded tokio runtime (two workers) and
blocks the calling thread on it; the ranges of one read_ranges call are
fetched concurrently (up to 8). From inside another tokio runtime it
refuses with RemoteError::Usage instead of blocking a worker, and it
shuts its runtime down in the background on drop so dropping it in async
code does not panic.

open_object(store, path, options) opens a file through a block cache
(first block prefetched); open_url accepts s3://, gs:// and az:// with
the `s3`, `gcs` and `azure` features, configured from the environment by
object_store's from_env builders. Those pull object_store's cloud clients
and aws-lc-rs (C), so they are opt-in; without them the URL is a clean
UnsupportedScheme error naming the feature.

Tests against object_store's in-memory and local-file stores (no cloud):
every fixture's transcript equals File::open's, a multi-block object is
fetched in coalesced block runs, an object replaced while open is an
error, and a missing object or a read from inside a runtime is a clean
error. ci-test.sh lints all backends, runs these tests (with s3 for its
URL parsing test) and checks object-store for C in the no-C step.

Co-Authored-By: Claude Opus 5.5 (1M context) <[email protected]>

crates/clawhdf5-remote/src/lib.rs

@@ -12,12 +12,13 @@
 //! ```
 //!
 //! - [`open_url`] / [`open_url_with`]: `http://` (default feature `http`),
-//!   `https://` (feature `https`) → a [`clawhdf5::File`] with the whole
-//!   read API.
+//!   `https://` (feature `https`), `s3://`, `gs://`, `az://` (features `s3`,
+//!   `gcs`, `azure`) → a [`clawhdf5::File`] with the whole read API.
 //! - [`storage_for_url`] gives the cached storage itself, to open with
 //!   [`clawhdf5::File::open_storage`] and to read its [`CacheStats`].
 //! - [`HttpStorage`] (range `GET`s, pinned by ETag/Last-Modified, retried
-//!   with backoff), and [`BlockCache`] over any
+//!   with backoff), [`ObjectStoreStorage`] (any `object_store` store,
+//!   feature `object-store`), and [`BlockCache`] over any
 //!   [`Storage`](clawhdf5_format::storage::Storage).
 //!
 //! What costs what: opening costs one request (it also fetches the first
@@ -34,6 +35,8 @@ pub mod cache;
 pub mod error;
 #[cfg(feature = "http")]
 pub mod http;
+#[cfg(feature = "object-store")]
+pub mod object;
 
 use std::sync::Arc;
 
@@ -44,6 +47,10 @@ pub use cache::{BlockCache, CacheConfig, CacheStats};
 pub use error::{Error, RemoteError};
 #[cfg(feature = "http")]
 pub use http::{HttpOptions, HttpStats, HttpStorage};
+#[cfg(feature = "object-store")]
+pub use object::ObjectStoreStorage;
+#[cfg(feature = "object-store")]
+pub use object_store;
 
 /// A backend a [`BlockCache`] can read through.
 pub type Backend = Box<dyn Storage + Send + Sync>;
@@ -65,7 +72,9 @@ pub struct Options {
 /// Open the HDF5 file at `url` with default [`Options`].
 ///
 /// `http://…` needs the (default) `http` feature, `https://…` the `https`
-/// feature.
+/// feature, `s3://bucket/key`, `gs://bucket/key` and `az://container/key`
+/// the `s3`, `gcs` and `azure` features (credentials and region from the
+/// environment, as `object_store`'s `from_env` builders read them).
 pub fn open_url(url: &str) -> Result<File, Error> {
     open_url_with(url, &Options::default())
 }
@@ -111,8 +120,24 @@ fn http_storage(url: &str, _options: &Options) -> Result<Arc<RemoteStorage>, Err
     .into())
 }
 
+#[cfg(any(feature = "s3", feature = "gcs", feature = "azure"))]
+fn cloud_storage(url: &str, _scheme: &str, options: &Options) -> Result<Arc<RemoteStorage>, Error> {
+    let (store, path) = object::store_for_url(url)?;
+    let storage = ObjectStoreStorage::new(store, path)?;
+    Ok(Arc::new(cached(Box::new(storage), options)?))
+}
+
+#[cfg(not(any(feature = "s3", feature = "gcs", feature = "azure")))]
 fn cloud_storage(url: &str, scheme: &str, _options: &Options) -> Result<Arc<RemoteStorage>, Error> {
-    Err(RemoteError::UnsupportedScheme(format!("{url}: {scheme}:// is not supported yet")).into())
+    let feature = match scheme {
+        "s3" | "s3a" => "s3",
+        "gs" => "gcs",
+        _ => "azure",
+    };
+    Err(RemoteError::UnsupportedScheme(format!(
+        "{url}: {scheme}:// needs the `{feature}` feature of clawhdf5-remote"
+    ))
+    .into())
 }
 
 /// A [`BlockCache`] over `backend` with its first block fetched (readahead
@@ -125,3 +150,21 @@ pub fn cached(backend: Backend, options: &Options) -> Result<RemoteStorage, Erro
         .map_err(|e| Error::Hdf5(clawhdf5::Error::Format(e)))?;
     Ok(cache)
 }
+
+/// Open the object at `path` of any `object_store` store (in memory, local
+/// files, or a cloud store you configured) through a block cache.
+#[cfg(feature = "object-store")]
+pub fn open_object(
+    store: Arc<dyn object_store::ObjectStore>,
+    path: &str,
+    options: &Options,
+) -> Result<(File, Arc<RemoteStorage>), Error> {
+    let path = object_store::path::Path::parse(path)
+        .map_err(|e| RemoteError::InvalidUrl(format!("{path}: {e}")))?;
+    let storage = Arc::new(cached(
+        Box::new(ObjectStoreStorage::new(store, path)?),
+        options,
+    )?);
+    let file = File::open_storage(storage.clone())?;
+    Ok((file, storage))
+}