clawhdf5-remote: object stores through object_store (S3, GCS, Azure)

ObjectStoreStorage (feature `object-store`, pure Rust) reads one object
of any object_store store by ranged get_opts, pinned at open by a head
request: If-Match with its ETag (and the ETag and size of every response
compared), else its version or modification time. A change is
RemoteError::FileChanged. object_store is async and Storage is not, so
the storage owns a small multi-threaded tokio runtime (two workers) and
blocks the calling thread on it; the ranges of one read_ranges call are
fetched concurrently (up to 8). From inside another tokio runtime it
refuses with RemoteError::Usage instead of blocking a worker, and it
shuts its runtime down in the background on drop so dropping it in async
code does not panic.

open_object(store, path, options) opens a file through a block cache
(first block prefetched); open_url accepts s3://, gs:// and az:// with
the `s3`, `gcs` and `azure` features, configured from the environment by
object_store's from_env builders. Those pull object_store's cloud clients
and aws-lc-rs (C), so they are opt-in; without them the URL is a clean
UnsupportedScheme error naming the feature.

Tests against object_store's in-memory and local-file stores (no cloud):
every fixture's transcript equals File::open's, a multi-block object is
fetched in coalesced block runs, an object replaced while open is an
error, and a missing object or a read from inside a runtime is a clean
error. ci-test.sh lints all backends, runs these tests (with s3 for its
URL parsing test) and checks object-store for C in the no-C step.

Co-Authored-By: Claude Opus 5.5 (1M context) <[email protected]>

crates/clawhdf5-remote/tests/http.rs

@@ -474,10 +474,11 @@ fn bad_urls_and_statuses_are_clean_errors() {
         open_url("no-scheme"),
         Err(Error::Remote(RemoteError::InvalidUrl(_)))
     ));
-    assert!(matches!(
-        open_url("s3://bucket/key.h5"),
-        Err(Error::Remote(RemoteError::UnsupportedScheme(_)))
-    ));
+    #[cfg(not(feature = "s3"))]
+    {
+        let e = open_url("s3://bucket/key.h5").unwrap_err().to_string();
+        assert!(e.contains("`s3` feature"), "{e}");
+    }
     #[cfg(not(feature = "https"))]
     {
         let e = open_url("https://example.com/a.h5")

crates/clawhdf5-remote/tests/object_store.rs

@@ -0,0 +1,118 @@
+//! `ObjectStoreStorage` against object_store's in-memory and local-file
+//! backends (no cloud needed): values equal `File::open`'s, the block cache
+//! coalesces, and an object replaced while open is an error.
+
+#![cfg(feature = "object-store")]
+
+mod common;
+
+use std::sync::Arc;
+
+use clawhdf5::File;
+use clawhdf5_format::storage::Storage;
+use clawhdf5_remote::object_store::memory::InMemory;
+use clawhdf5_remote::object_store::path::Path as ObjectPath;
+use clawhdf5_remote::object_store::{ObjectStore, ObjectStoreExt, PutPayload};
+use clawhdf5_remote::{ObjectStoreStorage, Options, open_object};
+use common::{multi_block_file, transcript};
+
+fn put(store: &dyn ObjectStore, path: &str, bytes: Vec<u8>) {
+    let rt = tokio_rt();
+    rt.block_on(store.put(&ObjectPath::from(path), PutPayload::from(bytes)))
+        .unwrap();
+}
+
+fn tokio_rt() -> tokio::runtime::Runtime {
+    tokio::runtime::Builder::new_current_thread()
+        .enable_all()
+        .build()
+        .unwrap()
+}
+
+#[test]
+fn in_memory_store_reads_like_file_open() {
+    let store: Arc<dyn ObjectStore> = Arc::new(InMemory::new());
+    let mut compared = 0;
+    for (i, p) in common::fixtures().iter().enumerate() {
+        let bytes = std::fs::read(p).unwrap();
+        let key = format!("f{i}.h5");
+        put(store.as_ref(), &key, bytes);
+        let (Ok(local), Ok((mut remote, cache))) = (
+            File::open(p),
+            open_object(store.clone(), &key, &Options::default()),
+        ) else {
+            continue;
+        };
+        remote.set_vds_resolver(common::sibling_resolver(p.parent().unwrap().into()));
+        assert!(remote.contiguous_bytes().is_none());
+        assert_eq!(transcript(&remote), transcript(&local), "{}", p.display());
+        assert!(cache.stats().reads > 0);
+        compared += 1;
+    }
+    assert!(compared >= 40, "{compared}");
+}
+
+#[test]
+fn multi_block_object_is_fetched_in_coalesced_blocks() {
+    let store: Arc<dyn ObjectStore> = Arc::new(InMemory::new());
+    let bytes = multi_block_file();
+    put(store.as_ref(), "m.h5", bytes.clone());
+    let (remote, cache) = open_object(store, "m.h5", &Options::default()).unwrap();
+    let local = File::from_bytes(bytes.clone()).unwrap();
+    assert_eq!(
+        remote.dataset("big").unwrap().read_f64().unwrap(),
+        local.dataset("big").unwrap().read_f64().unwrap()
+    );
+    let s = cache.stats();
+    let blocks = (bytes.len() as u64).div_ceil(1 << 20);
+    assert!(s.bytes_fetched <= bytes.len() as u64);
+    assert!(s.requests <= blocks, "{s:?}");
+    assert_eq!(cache.inner().len(), bytes.len() as u64);
+}
+
+#[test]
+fn local_file_store_reads_like_file_open() {
+    let dir = tempfile::tempdir().unwrap();
+    let bytes = multi_block_file();
+    std::fs::write(dir.path().join("m.h5"), &bytes).unwrap();
+    let store: Arc<dyn ObjectStore> = Arc::new(
+        clawhdf5_remote::object_store::local::LocalFileSystem::new_with_prefix(dir.path()).unwrap(),
+    );
+    let (remote, _) = open_object(store, "m.h5", &Options::default()).unwrap();
+    let local = File::from_bytes(bytes).unwrap();
+    assert_eq!(transcript(&remote), transcript(&local));
+}
+
+#[test]
+fn an_object_replaced_while_open_is_an_error() {
+    let store: Arc<dyn ObjectStore> = Arc::new(InMemory::new());
+    let bytes = multi_block_file();
+    put(store.as_ref(), "m.h5", bytes.clone());
+    let (remote, _) = open_object(store.clone(), "m.h5", &Options::default()).unwrap();
+    assert_eq!(remote.root().groups().unwrap(), ["grp"]);
+    let mut other = bytes;
+    let n = other.len();
+    other[n / 2] ^= 0xff;
+    put(store.as_ref(), "m.h5", other);
+    let err = remote
+        .dataset("big")
+        .unwrap()
+        .read_f64()
+        .unwrap_err()
+        .to_string();
+    assert!(err.contains("changed while open"), "{err}");
+}
+
+#[test]
+fn missing_objects_and_async_callers_are_clean_errors() {
+    let store: Arc<dyn ObjectStore> = Arc::new(InMemory::new());
+    assert!(ObjectStoreStorage::new(store.clone(), ObjectPath::from("nope.h5")).is_err());
+    put(store.as_ref(), "m.h5", multi_block_file());
+    let storage = ObjectStoreStorage::new(store, ObjectPath::from("m.h5")).unwrap();
+    // From inside a runtime: refused, not a panic or a deadlock.
+    let rt = tokio_rt();
+    let r = rt.block_on(async { storage.read_at(0, 10).map(|b| b.len()) });
+    assert!(r.unwrap_err().to_string().contains("spawn_blocking"));
+    // Dropping the storage inside a runtime does not panic.
+    rt.block_on(async move { drop(storage) });
+}