Merge branch 'feat/p3-m3-remote' into feat/p3-remote-editor

# Conflicts:
#	crates/clawhdf5/tests/storage_equivalence.rs

crates/clawhdf5-format/src/vl_data.rs

@@ -305,6 +305,24 @@ impl<'a, S: crate::storage::Storage + ?Sized> VlResolver<'a, S> {
         Ok(Some(data))
     }
 
+    /// [`VlResolver::element`] over any storage: the element's bytes
+    /// (borrowed from the resolver's cache of heap collections, so they
+    /// live until the next call), or `None` for a null element.
+    pub fn element_in(
+        &mut self,
+        elem: &[u8],
+        base_size: usize,
+    ) -> Result<Option<&[u8]>, FormatError> {
+        let vl = parse_vl_references(elem, 1, self.offset_size)?;
+        self.resolve(&vl[0], base_size)
+    }
+
+    /// [`VlResolver::string_element`] over any storage (see
+    /// [`element_in`](Self::element_in)).
+    pub fn string_element_in(&mut self, elem: &[u8]) -> Result<Option<&[u8]>, FormatError> {
+        Ok(self.element_in(elem, 1)?.map(cut_at_nul))
+    }
+
     /// The strings of the variable-length string elements in `raw`, as
     /// bytes. A string ends at its first NUL, as libhdf5 returns it (it
     /// converts each to a C string); a null element is empty.
@@ -645,6 +663,32 @@ mod tests {
         }
     }
 
+    #[test]
+    fn element_in_over_a_storage_matches_element_over_a_slice() {
+        let mut file_data = vec![0u8; 512];
+        build_gcol_at(&mut file_data, 256, &[(1, b"Alice\0x"), (2, b"Bob")]);
+        let mut raw = build_vl_refs(&["Alice\0x", "Bob"], 256, 1, 8);
+        raw.extend(element(0, 0, 0, 8)); // null
+        raw.extend(element(9, 256, 1, 8)); // wrong length: an error
+        let storage = crate::storage::CountingStorage::new(file_data.clone());
+        let dynamic: &dyn crate::storage::Storage = &storage;
+        let mut slice = VlResolver::new(&file_data, 8, 8);
+        let mut any = VlResolver::new_in(dynamic, 8, 8);
+        for e in raw.chunks(16) {
+            let want = slice.element(e, 1).map(|o| o.map(<[u8]>::to_vec));
+            let got = any.element_in(e, 1).map(|o| o.map(<[u8]>::to_vec));
+            assert_eq!(format!("{want:?}"), format!("{got:?}"));
+            let want = slice.string_element(e).map(|o| o.map(<[u8]>::to_vec));
+            let got = any.string_element_in(e).map(|o| o.map(<[u8]>::to_vec));
+            assert_eq!(format!("{want:?}"), format!("{got:?}"));
+        }
+        assert_eq!(
+            any.string_element_in(&raw[..16]).unwrap(),
+            Some(&b"Alice"[..])
+        );
+        assert!(storage.reads() > 0);
+    }
+
     #[test]
     fn null_vl_element_zero_address() {
         let mut raw = Vec::new();

crates/clawhdf5-remote/Cargo.toml

@@ -0,0 +1,39 @@
+[package]
+name = "clawhdf5-remote"
+version = "2.7.0"
+edition = "2024"
+rust-version.workspace = true
+description = "Read HDF5 files over HTTP(S) range requests and object stores (S3, GCS, Azure) with clawhdf5, through a block cache"
+license = "MIT"
+repository = "https://git.redclaw.dev/quantumclaw/clawhdf5"
+readme = "README.md"
+keywords = ["hdf5", "http", "s3", "range-requests", "science"]
+categories = ["science", "network-programming"]
+
+[features]
+# Plain-HTTP range reads and the block cache: pure Rust, no TLS stack, no C.
+default = ["http"]
+http = ["dep:ureq"]
+# HTTPS through rustls (ring provider, Mozilla roots). ring compiles C and
+# assembly, so this is not part of the default build.
+https = ["http", "ureq/rustls"]
+# Any object_store backend (in-memory, local files, or one you configure),
+# driven by a small tokio runtime the storage owns. Pure Rust.
+object-store = ["dep:object_store", "dep:tokio", "dep:futures-util"]
+# s3:// gs:// az:// URLs in open_url, credentials from the environment.
+# object_store's cloud clients use aws-lc-rs (C), hence opt-in.
+s3 = ["object-store", "object_store/aws"]
+gcs = ["object-store", "object_store/gcp"]
+azure = ["object-store", "object_store/azure"]
+
+[dependencies]
+clawhdf5 = { path = "../clawhdf5", version = "2.7.0" }
+clawhdf5-format = { path = "../clawhdf5-format", version = "2.7.0" }
+ureq = { version = "3.4", optional = true, default-features = false }
+object_store = { version = "0.14", optional = true, default-features = false, features = ["fs"] }
+tokio = { version = "1", optional = true, default-features = false, features = ["rt-multi-thread"] }
+futures-util = { version = "0.3", optional = true, default-features = false, features = ["std"] }
+
+[dev-dependencies]
+tempfile = { workspace = true }
+tokio = { version = "1", default-features = false, features = ["rt"] }

crates/clawhdf5-remote/README.md

@@ -0,0 +1,116 @@
+# clawhdf5-remote
+
+Read HDF5 files where they live — on an HTTP(S) server or in an object
+store (S3, GCS, Azure) — with
+[clawhdf5](../../README.md), without downloading them first.
+
+```rust
+let file = clawhdf5_remote::open_url("http://127.0.0.1:8000/data.h5")?;
+let temps = file.dataset("/grid/temperature")?.read_f64()?;
+```
+
+The result is an ordinary `clawhdf5::File`: groups, datasets, attributes,
+selections, variable-length data. Only the bytes an operation needs are
+fetched, by `Range` requests, through a block cache.
+
+## How it reads
+
+- **Block cache** (`BlockCache`, mandatory for remote files; see
+  `docs/design/range-reads.md` §2): aligned blocks of 1 MiB by default, LRU
+  with a byte budget (64 MiB by default), the missing blocks of one read
+  fetched as runs of consecutive blocks (a gap of one block is fetched to
+  merge two runs), each request at most 8 MiB, the requests of one read in
+  parallel. A read that misses more than half the budget is not kept, so a
+  big dataset read does not evict the metadata. Readers on several threads
+  share one cache; a block is never fetched twice at once — a second reader
+  waits for the first one's request.
+- **Opening costs one request**: a `GET` of the first block, whose
+  `Content-Range` gives the file's length. HDF5 files keep the superblock
+  and usually the root group's metadata there, so listing a small file
+  often needs nothing more.
+- **Pinned to one version of the file**: a strong `ETag` is sent back as
+  `If-Match` (else `Last-Modified` as `If-Unmodified-Since`) and checked on
+  every response, as is the length. A file replaced while open is an error
+  (`RemoteError::FileChanged`), never a mix of old and new bytes. A server
+  with neither validator can only be checked by length;
+  `HttpOptions::require_validator` refuses it.
+- **Servers that ignore `Range`** (answer `200` with the whole file) are
+  refused (`RemoteError::RangeNotSupported`) without reading the body,
+  unless `HttpOptions::allow_full_download` is set; then the file is
+  downloaded once and read from memory.
+  A `200` whose body is no longer than the range asked for is the whole
+  (small) file, and is accepted.
+- **Retries**: connection failures, timeouts, `408`/`429`/`5xx` and bodies
+  that end early are retried with exponential backoff (3 retries, from
+  200 ms). Bodies are requested with `Accept-Encoding: identity`; an encoded
+  body is refused.
+- **Timeouts** scale with the request: `HttpOptions::timeout` (30 s) to
+  connect and to receive the headers, and for the body that plus its size
+  at `HttpOptions::min_speed` (16 KiB/s) — a slow link is not cut off
+  mid-block, a stalled connection still fails.
+- **Redirects** are followed up to `HttpOptions::max_redirects` (5; 0
+  refuses them), never from `https` to `http`. Once a redirect leaves the
+  URL's origin (scheme, host, port), `HttpOptions::headers` (API keys,
+  `Authorization`, cookies) are no longer sent.
+- **Credentials stay out of messages**: every error and `Debug` output
+  shows URLs through `redact_url` — no `user:password@`, query values
+  replaced by `REDACTED` (a presigned S3/GCS URL's signature lives there).
+- **Claimed lengths are not trusted**: nothing is allocated for the length
+  a server reports; a read spanning more than the cache budget is fetched
+  in pieces as data arrives, and `download(&storage, max_bytes)` reads a
+  whole file only up to a limit (`DEFAULT_MAX_DOWNLOAD`, 1 GiB).
+
+The zero-copy methods of `clawhdf5` (`read_raw_ref`, `read_*_zerocopy`,
+`File::as_bytes`) borrow the whole file from memory, so they are errors
+(`as_bytes` a panic; use `File::contiguous_bytes`) on a remote file.
+
+## Features
+
+| Feature | What | C code |
+|---|---|---|
+| `http` (default) | `http://` through `ureq`, no TLS | none |
+| `https` | `https://` through rustls, ring provider, Mozilla roots | ring (C and assembly) |
+| `object-store` | `ObjectStoreStorage` and `open_object` over any [`object_store`](https://docs.rs/object_store) store (in-memory, local files, or one you configure) | none |
+| `s3`, `gcs`, `azure` | `s3://bucket/key`, `gs://bucket/key`, `az://container/key` in `open_url`, configured from the environment (`AWS_*`, `GOOGLE_*`, `AZURE_*`) as object_store's `from_env` builders read it | aws-lc-rs (object_store's cloud clients) |
+
+The default build and `object-store` compile no C (`scripts/ci-test.sh`
+checks both).
+
+## Object stores
+
+`object_store` is async; `Storage` is synchronous (parsing is CPU work).
+`ObjectStoreStorage` owns a small tokio runtime (two worker threads): each
+read runs there while the calling thread waits, so it works from any
+thread, several at once — including `tokio::task::spawn_blocking` and code
+inside another runtime (where `spawn_blocking` is still the better place,
+since a read blocks the thread it is called on). The object is pinned by its ETag
+(`If-Match`, and compared on every response), else its version or
+modification time, and its size. The ranges of one read are fetched
+concurrently (up to 8).
+
+```rust
+use std::sync::Arc;
+use clawhdf5_remote::object_store::{memory::InMemory, ObjectStore};
+let store: Arc<dyn ObjectStore> = Arc::new(InMemory::new());
+// ... put a file at "data.h5" ...
+let (file, cache) = clawhdf5_remote::open_object(store, "data.h5", &Default::default())?;
+```
+
+The tests use object_store's in-memory and local-file stores; no cloud
+account is needed. The cloud schemes are only built (and unit-tested for
+URL parsing) in CI, not run against a real bucket.
+
+## Counting requests
+
+```rust
+use clawhdf5_remote::{storage_for_url, Options};
+let storage = storage_for_url(url, &Options::default())?;
+let file = clawhdf5::File::open_storage(storage.clone())?;
+// ... read ...
+let s = storage.stats(); // requests, bytes_fetched, hits, misses, cached_bytes, ...
+```
+
+`cargo run -p clawhdf5-remote --example range_server -- DIR` serves a
+directory with range support (the server the tests use), and
+`cargo run -p clawhdf5-remote --example read_url -- URL [DATASET]` lists a
+file and prints what it cost.

crates/clawhdf5-remote/examples/range_server.rs

@@ -0,0 +1,56 @@
+//! Serve the HDF5 files of a directory over HTTP with range support — the
+//! server the tests use — to try `clawhdf5_remote` and `h5rs` on URLs.
+//!
+//! ```text
+//! cargo run -p clawhdf5-remote --example range_server -- DIR [127.0.0.1:8000]
+//! ```
+//!
+//! Every file under `DIR` is served at its path relative to `DIR`. The
+//! files are read into memory at start. Requests are logged to stderr.
+
+#[path = "../tests/common/server.rs"]
+mod server;
+
+fn main() {
+    let mut args = std::env::args().skip(1);
+    let Some(dir) = args.next() else {
+        eprintln!("usage: range_server DIR [ADDR]");
+        std::process::exit(2);
+    };
+    let addr = args.next().unwrap_or_else(|| "127.0.0.1:8000".into());
+    let root = std::path::PathBuf::from(&dir);
+    let mut files = Vec::new();
+    let mut stack = vec![root.clone()];
+    while let Some(d) = stack.pop() {
+        let Ok(entries) = std::fs::read_dir(&d) else {
+            continue;
+        };
+        for e in entries.flatten() {
+            let p = e.path();
+            if p.is_dir() {
+                stack.push(p);
+            } else if let (Ok(rel), Ok(bytes)) = (p.strip_prefix(&root), std::fs::read(&p)) {
+                let url = format!("/{}", rel.to_string_lossy().replace('\\', "/"));
+                files.push((url, bytes));
+            }
+        }
+    }
+    files.sort();
+    let server = server::Server::bind(&addr, files.clone());
+    for (path, bytes) in &files {
+        eprintln!("{} ({} bytes)", server.url(path), bytes.len());
+    }
+    eprintln!("serving {} files on http://{}", files.len(), server.addr);
+    let mut logged = 0;
+    loop {
+        std::thread::sleep(std::time::Duration::from_millis(200));
+        let log = server.log();
+        for (path, range) in &log[logged.min(log.len())..] {
+            match range {
+                Some((a, b)) => eprintln!("GET {path} bytes={a}-{b}"),
+                None => eprintln!("GET {path} (whole file)"),
+            }
+        }
+        logged = log.len();
+    }
+}

crates/clawhdf5-remote/examples/read_url.rs

@@ -0,0 +1,45 @@
+//! List a remote HDF5 file and read one dataset, then print what it cost.
+//!
+//! ```text
+//! cargo run -p clawhdf5-remote --example read_url -- URL [DATASET]
+//! ```
+
+use clawhdf5::File;
+use clawhdf5_remote::{Options, storage_for_url};
+
+fn main() -> Result<(), Box<dyn std::error::Error>> {
+    let mut args = std::env::args().skip(1);
+    let url = args.next().ok_or("usage: read_url URL [DATASET]")?;
+    let dataset = args.next();
+
+    let storage = storage_for_url(&url, &Options::default())?;
+    let file = File::open_storage(storage.clone())?;
+
+    // Walk the tree.
+    let mut stack = vec![("/".to_string(), file.root())];
+    while let Some((path, group)) = stack.pop() {
+        for (name, addr) in group.entries()? {
+            let child = format!("{}/{name}", path.trim_end_matches('/'));
+            match file.dataset_at(addr) {
+                Ok(ds) => println!("{child}  dataset {:?} {:?}", ds.shape()?, ds.dtype()?),
+                Err(_) => {
+                    println!("{child}  group");
+                    stack.push((child, file.group_at(addr)));
+                }
+            }
+        }
+    }
+
+    if let Some(name) = dataset {
+        let values = file.dataset(&name)?.read_f64()?;
+        let shown: Vec<_> = values.iter().take(8).collect();
+        println!("{name}: {} values, first {shown:?}", values.len());
+    }
+
+    let s = storage.stats();
+    println!(
+        "{} range requests (the one at open included), {} bytes fetched, {} bytes cached",
+        s.requests, s.bytes_fetched, s.cached_bytes
+    );
+    Ok(())
+}

crates/clawhdf5-remote/src/error.rs

@@ -0,0 +1,297 @@
+//! Errors of the remote backends.
+
+use clawhdf5_format::error::FormatError;
+
+/// `url` as it may be shown in an error, a `Debug` output or a log: no
+/// userinfo (`user:password@`), no fragment, and the query string's values
+/// replaced by `REDACTED` (a presigned S3/GCS/Azure URL carries its
+/// signature or token there). Keys are kept when they look like plain
+/// names, so a message still says which kind of URL it was.
+///
+/// ```
+/// assert_eq!(
+///     clawhdf5_remote::redact_url("https://me:pw@host/f.h5?X-Amz-Signature=abc&a=1#x"),
+///     "https://host/f.h5?X-Amz-Signature=REDACTED&a=REDACTED"
+/// );
+/// ```
+pub fn redact_url(url: &str) -> String {
+    let (scheme, rest) = match url.split_once("://") {
+        Some((s, r)) => (Some(s), r),
+        None => (None, url),
+    };
+    let rest = rest.split('#').next().unwrap_or("");
+    let (before_query, query) = match rest.split_once('?') {
+        Some((a, q)) => (a, Some(q)),
+        None => (rest, None),
+    };
+    let mut out = String::with_capacity(url.len());
+    if let Some(s) = scheme {
+        out.push_str(s);
+        out.push_str("://");
+        let auth_end = before_query.find('/').unwrap_or(before_query.len());
+        let (authority, path) = before_query.split_at(auth_end);
+        out.push_str(
+            authority
+                .rsplit_once('@')
+                .map_or(authority, |(_, host)| host),
+        );
+        out.push_str(path);
+    } else {
+        out.push_str(before_query);
+    }
+    if let Some(q) = query {
+        out.push('?');
+        let plain = |k: &str| {
+            !k.is_empty()
+                && k.len() <= 64
+                && k.bytes()
+                    .all(|b| b.is_ascii_alphanumeric() || matches!(b, b'-' | b'_' | b'.'))
+        };
+        let parts: Vec<String> = q
+            .split('&')
+            .map(|kv| {
+                let k = kv.split('=').next().unwrap_or("");
+                if plain(k) {
+                    format!("{k}=REDACTED")
+                } else {
+                    "REDACTED".to_string()
+                }
+            })
+            .collect();
+        out.push_str(&parts.join("&"));
+    }
+    out
+}
+
+/// Replaces the secret parts of one URL (its userinfo and query string,
+/// and the URL itself) wherever they appear in a message — such as the
+/// text of an error from the HTTP client.
+#[cfg_attr(
+    not(any(feature = "http", feature = "s3", feature = "gcs", feature = "azure")),
+    allow(dead_code)
+)]
+#[derive(Debug, Clone)]
+pub(crate) struct Redactor {
+    shown: String,
+    secrets: Vec<(String, String)>,
+}
+
+#[cfg_attr(
+    not(any(feature = "http", feature = "s3", feature = "gcs", feature = "azure")),
+    allow(dead_code)
+)]
+impl Redactor {
+    pub(crate) fn new(url: &str) -> Redactor {
+        let shown = redact_url(url);
+        let mut secrets = vec![(url.to_string(), shown.clone())];
+        let rest = url.split_once("://").map_or(url, |(_, r)| r);
+        let authority = rest.split(['/', '?', '#']).next().unwrap_or("");
+        if let Some((userinfo, _)) = authority.rsplit_once('@')
+            && !userinfo.is_empty()
+        {
+            secrets.push((format!("{userinfo}@"), String::new()));
+            secrets.push((userinfo.to_string(), "REDACTED".into()));
+        }
+        if let Some((_, q)) = rest.split('#').next().unwrap_or("").split_once('?')
+            && !q.is_empty()
+        {
+            let shown_q = shown.split_once('?').map_or("", |(_, q)| q).to_string();
+            secrets.push((q.to_string(), shown_q));
+            for kv in q.split('&') {
+                if let Some((_, v)) = kv.split_once('=')
+                    && v.len() >= 4
+                {
+                    secrets.push((v.to_string(), "REDACTED".into()));
+                }
+            }
+        }
+        Redactor { shown, secrets }
+    }
+
+    /// The URL, redacted.
+    pub(crate) fn shown(&self) -> &str {
+        &self.shown
+    }
+
+    /// `msg` with every secret part of the URL replaced.
+    pub(crate) fn scrub(&self, msg: &str) -> String {
+        let mut m = msg.to_string();
+        for (secret, with) in &self.secrets {
+            if m.contains(secret.as_str()) {
+                m = m.replace(secret.as_str(), with);
+            }
+        }
+        m
+    }
+}
+
+/// Why a remote file could not be opened or read.
+///
+/// No message carries a URL's credentials: URLs appear as
+/// [`redact_url`] shows them.
+///
+/// Inside a [`clawhdf5::File`] read these arrive as
+/// `clawhdf5::Error::Format(FormatError::Storage(message))`, the message
+/// being this error's `Display`.
+#[derive(Debug, Clone, PartialEq, Eq)]
+#[non_exhaustive]
+pub enum RemoteError {
+    /// The URL is malformed.
+    InvalidUrl(String),
+    /// The URL's scheme is not supported by this build (for example
+    /// `s3://` without the `s3` feature, or `https://` without `https`).
+    UnsupportedScheme(String),
+    /// The server answered a range request with the whole file (status
+    /// 200), i.e. it does not support ranges, and a full download was not
+    /// allowed ([`HttpOptions::allow_full_download`](crate::HttpOptions)).
+    RangeNotSupported(String),
+    /// The file changed since it was opened (a different ETag,
+    /// Last-Modified or length, or a failed `If-Match` precondition).
+    /// Nothing read after the change is returned.
+    FileChanged(String),
+    /// The server answered with an unexpected status.
+    Status {
+        /// The HTTP status code.
+        code: u16,
+        /// What was being requested.
+        what: String,
+    },
+    /// A response did not carry what was asked for (a wrong
+    /// `Content-Range`, a body shorter or longer than announced), after
+    /// every retry.
+    BadResponse(String),
+    /// A network failure (connection, timeout, reset) after every retry.
+    Transport(String),
+    /// An error from the object store.
+    ObjectStore(String),
+    /// Called in a way the backend cannot serve.
+    Usage(String),
+    /// A redirect that is not followed: from `https` to `http`, to
+    /// another scheme, or beyond
+    /// [`HttpOptions::max_redirects`](crate::HttpOptions).
+    Redirect(String),
+    /// The file is larger than a download was allowed to be
+    /// ([`download`](crate::download)).
+    TooLarge {
+        /// The file's length, as the server reports it.
+        len: u64,
+        /// The limit.
+        limit: u64,
+    },
+    /// A read through a backend failed; its error, as text (the form a
+    /// [`clawhdf5::File`] read reports it in).
+    Backend(String),
+}
+
+impl RemoteError {
+    /// The error with every secret part of `r`'s URL scrubbed from its text.
+    #[cfg_attr(
+        not(any(feature = "http", feature = "s3", feature = "gcs", feature = "azure")),
+        allow(dead_code)
+    )]
+    pub(crate) fn scrubbed(self, r: &Redactor) -> RemoteError {
+        let f = |s: String| r.scrub(&s);
+        match self {
+            RemoteError::InvalidUrl(s) => RemoteError::InvalidUrl(f(s)),
+            RemoteError::UnsupportedScheme(s) => RemoteError::UnsupportedScheme(f(s)),
+            RemoteError::RangeNotSupported(s) => RemoteError::RangeNotSupported(f(s)),
+            RemoteError::FileChanged(s) => RemoteError::FileChanged(f(s)),
+            RemoteError::Status { code, what } => RemoteError::Status {
+                code,
+                what: f(what),
+            },
+            RemoteError::BadResponse(s) => RemoteError::BadResponse(f(s)),
+            RemoteError::Transport(s) => RemoteError::Transport(f(s)),
+            RemoteError::ObjectStore(s) => RemoteError::ObjectStore(f(s)),
+            RemoteError::Usage(s) => RemoteError::Usage(f(s)),
+            RemoteError::Redirect(s) => RemoteError::Redirect(f(s)),
+            e @ RemoteError::TooLarge { .. } => e,
+            RemoteError::Backend(s) => RemoteError::Backend(f(s)),
+        }
+    }
+
+    /// Whether retrying the same request may succeed.
+    #[cfg_attr(not(feature = "http"), allow(dead_code))]
+    pub(crate) fn is_transient(&self) -> bool {
+        match self {
+            RemoteError::Transport(_) | RemoteError::BadResponse(_) => true,
+            RemoteError::Status { code, .. } => {
+                matches!(code, 408 | 429 | 500 | 502 | 503 | 504)
+            }
+            _ => false,
+        }
+    }
+}
+
+impl std::fmt::Display for RemoteError {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        match self {
+            RemoteError::InvalidUrl(s) => write!(f, "invalid URL: {s}"),
+            RemoteError::UnsupportedScheme(s) => write!(f, "unsupported URL: {s}"),
+            RemoteError::RangeNotSupported(s) => {
+                write!(f, "the server does not support range requests: {s}")
+            }
+            RemoteError::FileChanged(s) => write!(f, "the remote file changed while open: {s}"),
+            RemoteError::Status { code, what } => write!(f, "HTTP status {code} for {what}"),
+            RemoteError::BadResponse(s) => write!(f, "bad response: {s}"),
+            RemoteError::Transport(s) => write!(f, "network error: {s}"),
+            RemoteError::ObjectStore(s) => write!(f, "object store: {s}"),
+            RemoteError::Usage(s) => write!(f, "{s}"),
+            RemoteError::Redirect(s) => write!(f, "redirect refused: {s}"),
+            RemoteError::TooLarge { len, limit } => write!(
+                f,
+                "the remote file is {len} bytes, more than the download limit of {limit} bytes"
+            ),
+            RemoteError::Backend(s) => write!(f, "{s}"),
+        }
+    }
+}
+
+impl std::error::Error for RemoteError {}
+
+impl From<RemoteError> for FormatError {
+    fn from(e: RemoteError) -> Self {
+        FormatError::Storage(e.to_string())
+    }
+}
+
+/// An error of [`open_url`](crate::open_url): the remote side, or the file
+/// itself.
+#[derive(Debug)]
+#[non_exhaustive]
+pub enum Error {
+    /// Reaching or reading the remote file failed.
+    Remote(RemoteError),
+    /// The bytes were read, but they are not an HDF5 file clawhdf5 can open.
+    Hdf5(clawhdf5::Error),
+}
+
+impl std::fmt::Display for Error {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        match self {
+            Error::Remote(e) => e.fmt(f),
+            Error::Hdf5(e) => e.fmt(f),
+        }
+    }
+}
+
+impl std::error::Error for Error {
+    fn source(&self) -> Option<&(dyn std::error::Error + 'static)> {
+        match self {
+            Error::Remote(e) => Some(e),
+            Error::Hdf5(e) => Some(e),
+        }
+    }
+}
+
+impl From<RemoteError> for Error {
+    fn from(e: RemoteError) -> Self {
+        Error::Remote(e)
+    }
+}
+
+impl From<clawhdf5::Error> for Error {
+    fn from(e: clawhdf5::Error) -> Self {
+        Error::Hdf5(e)
+    }
+}

crates/clawhdf5-remote/src/http.rs

@@ -0,0 +1,794 @@
+//! HTTP(S) range requests: [`HttpStorage`].
+//!
+//! Every read is a `GET` with a `Range: bytes=a-b` header, answered `206
+//! Partial Content`. The file is pinned when it is opened:
+//!
+//! - its length comes from the `Content-Range` of the first request (which
+//!   also fetches the first block, so opening costs one request);
+//! - a strong `ETag` is sent back as `If-Match` on every later request, and
+//!   compared with the `ETag` of every response; without one, `Last-Modified`
+//!   is sent as `If-Unmodified-Since` and compared; the length in every
+//!   `Content-Range` must stay the same. A file that changes while it is
+//!   open is [`RemoteError::FileChanged`], never a mix of old and new bytes.
+//!   (A server that sends neither validator cannot be checked beyond the
+//!   length; [`HttpOptions::require_validator`] refuses such servers.)
+//! - a `200` answer to the first request whose body is no longer than the
+//!   range asked for is the whole file (a server may answer so when the
+//!   range covers it): it is kept and read from memory.
+//! - a server that ignores `Range` and answers `200` with the whole file is
+//!   refused with [`RemoteError::RangeNotSupported`], unless
+//!   [`HttpOptions::allow_full_download`] is set: then the file is
+//!   downloaded once, at open, and read from memory.
+//!
+//! Redirects are followed (at most [`HttpOptions::max_redirects`]), but
+//! never from `https` to `http`, and the custom
+//! [`HttpOptions::headers`] are not sent to another origin.
+//!
+//! Timeouts scale with the request: [`HttpOptions::timeout`] to connect
+//! and to get the response headers, and for the body that plus its size at
+//! [`HttpOptions::min_speed`] — a slow link is not cut off mid-block, a
+//! stalled connection still is.
+//!
+//! Transient failures — connection errors, timeouts, `408`/`429`/`5xx`, and
+//! a body shorter or longer than its `Content-Range` — are retried with
+//! exponential backoff. Responses are requested with
+//! `Accept-Encoding: identity`, since a compressed body cannot be a byte
+//! range of the file.
+//!
+//! `HttpStorage` itself does not cache: each `read_at` is one request. Read
+//! it through [`BlockCache`](crate::BlockCache) (which [`open_url`](crate::open_url)
+//! does); its `read_ranges` fetches the ranges of one call in parallel.
+
+use std::borrow::Cow;
+use std::io::Read;
+use std::ops::Range;
+use std::sync::atomic::{AtomicU64, AtomicUsize, Ordering};
+use std::time::Duration;
+
+use clawhdf5_format::error::FormatError;
+use clawhdf5_format::storage::Storage;
+
+use crate::error::{Redactor, RemoteError, redact_url};
+
+/// Settings of an [`HttpStorage`].
+#[derive(Debug, Clone)]
+pub struct HttpOptions {
+    /// Retries of a request that failed transiently (so up to `retries + 1`
+    /// attempts).
+    pub retries: u32,
+    /// Delay before the first retry; doubled for each further one.
+    pub backoff: Duration,
+    /// Time allowed to connect, and then to receive the response headers.
+    /// The body gets this plus the time it takes at
+    /// [`min_speed`](Self::min_speed), so a request's budget grows with its
+    /// size: a slow but moving link is not cut off, a stalled one is.
+    pub timeout: Duration,
+    /// Slowest transfer rate tolerated, in bytes per second: receiving a
+    /// body of `n` bytes may take `timeout + n / min_speed` (16 KiB/s by
+    /// default: 94 s for a 1 MiB block, 9 min for an 8 MiB request).
+    pub min_speed: u64,
+    /// Requests of one `read_ranges` call in flight at once.
+    pub max_parallel: usize,
+    /// Bytes fetched by the first request, from offset 0 (the superblock and
+    /// usually the root group's metadata); at least 1.
+    pub first_request: u64,
+    /// When the server ignores `Range` (answers `200`), download the whole
+    /// file once and read it from memory, instead of failing.
+    pub allow_full_download: bool,
+    /// Largest file [`allow_full_download`](Self::allow_full_download) will
+    /// download.
+    pub max_full_download: u64,
+    /// Refuse a server that sends neither a strong `ETag` nor
+    /// `Last-Modified`, since a change of the file could then go unnoticed
+    /// (only its length is checked).
+    pub require_validator: bool,
+    /// Extra headers sent with every request to the URL's own origin (for
+    /// example `Authorization` or `X-Api-Key`). They are never sent to
+    /// another origin a redirect leads to.
+    pub headers: Vec<(String, String)>,
+    /// Redirects followed per request (0: none, a redirect is an error).
+    /// A redirect from `https` to plain `http` is always refused; once a
+    /// redirect leaves the URL's origin (scheme, host and port), the
+    /// [`headers`](Self::headers) are no longer sent. Every request of a
+    /// file follows the redirects again (the target is not remembered, as
+    /// a presigned target may expire).
+    pub max_redirects: u32,
+}
+
+impl Default for HttpOptions {
+    fn default() -> Self {
+        HttpOptions {
+            retries: 3,
+            backoff: Duration::from_millis(200),
+            timeout: Duration::from_secs(30),
+            min_speed: 16 << 10,
+            max_parallel: 8,
+            first_request: crate::cache::DEFAULT_BLOCK_SIZE,
+            allow_full_download: false,
+            max_full_download: 1 << 30,
+            require_validator: false,
+            headers: Vec::new(),
+            max_redirects: 5,
+        }
+    }
+}
+
+/// Requests and bytes an [`HttpStorage`] has used.
+#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
+pub struct HttpStats {
+    /// HTTP requests sent (retries included).
+    pub requests: u64,
+    /// Requests that were retries.
+    pub retries: u64,
+    /// Response body bytes received.
+    pub bytes: u64,
+}
+
+/// How the file is pinned.
+#[derive(Debug, Clone, PartialEq, Eq)]
+enum Validator {
+    ETag(String),
+    LastModified(String),
+    None,
+}
+
+/// An HTTP(S) file read by range requests. See the [module
+/// documentation](self).
+pub struct HttpStorage {
+    agent: ureq::Agent,
+    /// The URL as given, credentials and all: only ever sent to the server.
+    url: String,
+    /// Shows the URL without its credentials, in every message.
+    redactor: Redactor,
+    len: u64,
+    validator: Validator,
+    options: HttpOptions,
+    /// The whole file, when the server ignores ranges and a full download
+    /// was allowed.
+    full: Option<Vec<u8>>,
+    requests: AtomicU64,
+    retries: AtomicU64,
+    bytes: AtomicU64,
+}
+
+impl std::fmt::Debug for HttpStorage {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        f.debug_struct("HttpStorage")
+            .field("url", &self.redactor.shown())
+            .field("len", &self.len)
+            .field("validator", &self.validator)
+            .field("full_download", &self.full.is_some())
+            .finish()
+    }
+}
+
+/// A parsed `Content-Range: bytes a-b/total`.
+fn content_range(v: &str) -> Option<(u64, u64, Option<u64>)> {
+    let rest = v.trim().strip_prefix("bytes")?.trim_start();
+    let (span, total) = rest.split_once('/')?;
+    let (a, b) = span.trim().split_once('-')?;
+    let a: u64 = a.trim().parse().ok()?;
+    let b: u64 = b.trim().parse().ok()?;
+    if b < a {
+        return None;
+    }
+    let total = match total.trim() {
+        "*" => None,
+        t => Some(t.parse().ok()?),
+    };
+    Some((a, b, total))
+}
+
+fn header<'a>(resp: &'a ureq::http::Response<ureq::Body>, name: &str) -> Option<&'a str> {
+    resp.headers().get(name).and_then(|v| v.to_str().ok())
+}
+
+/// A body with a `Content-Encoding` is not a byte range of the file.
+fn check_identity(url: &str, resp: &ureq::http::Response<ureq::Body>) -> Result<(), RemoteError> {
+    match header(resp, "content-encoding") {
+        Some(enc) if !enc.trim().eq_ignore_ascii_case("identity") => {
+            Err(RemoteError::Usage(format!(
+                "{url}: the server sent a {enc}-encoded body despite Accept-Encoding: identity"
+            )))
+        }
+        _ => Ok(()),
+    }
+}
+
+/// Scheme, host (lowercase, no userinfo) and port of an absolute URL.
+fn origin(url: &str) -> Option<(String, String, u16)> {
+    let (scheme, rest) = url.split_once("://")?;
+    let scheme = scheme.to_ascii_lowercase();
+    let authority = rest.split(['/', '?', '#']).next().unwrap_or("");
+    let host_port = authority.rsplit_once('@').map_or(authority, |(_, h)| h);
+    let default = match scheme.as_str() {
+        "http" => 80,
+        "https" => 443,
+        _ => return None,
+    };
+    // "[v6]:port", "host:port", or either without a port.
+    let (host, port) = match host_port.rsplit_once(':') {
+        Some((h, p)) if !p.contains(']') => (h, p.parse().ok()?),
+        _ => (host_port, default),
+    };
+    Some((scheme, host.to_ascii_lowercase(), port))
+}
+
+fn same_origin(a: &str, b: &str) -> bool {
+    matches!((origin(a), origin(b)), (Some(x), Some(y)) if x == y)
+}
+
+/// The URL a redirect from `base` to `location` goes to, if it may be
+/// followed: `http`/`https` only, and never from `https` to `http`.
+fn redirect_target(base: &str, location: &str) -> Result<String, RemoteError> {
+    let location = location.trim();
+    let (scheme, rest) = base.split_once("://").unwrap_or(("http", base));
+    let authority_end = rest.find(['/', '?', '#']).unwrap_or(rest.len());
+    let (authority, path) = rest.split_at(authority_end);
+    let target = if location.contains("://") {
+        location.to_string()
+    } else if let Some(r) = location.strip_prefix("//") {
+        format!("{scheme}://{r}")
+    } else if location.starts_with('/') {
+        format!("{scheme}://{authority}{location}")
+    } else {
+        let path = path.split(['?', '#']).next().unwrap_or("");
+        let dir = path.rsplit_once('/').map_or("", |(d, _)| d);
+        format!("{scheme}://{authority}{dir}/{location}")
+    };
+    let Some((to_scheme, _, _)) = origin(&target) else {
+        return Err(RemoteError::Redirect(format!(
+            "{} redirects to {}, which is not an http(s) URL",
+            redact_url(base),
+            redact_url(&target)
+        )));
+    };
+    if scheme.eq_ignore_ascii_case("https") && to_scheme != "https" {
+        return Err(RemoteError::Redirect(format!(
+            "{} redirects to {}: a downgrade from https is refused",
+            redact_url(base),
+            redact_url(&target)
+        )));
+    }
+    Ok(target)
+}
+
+fn transport(e: ureq::Error) -> RemoteError {
+    match e {
+        ureq::Error::StatusCode(code) => RemoteError::Status {
+            code,
+            what: "request".into(),
+        },
+        ureq::Error::BadUri(s) => RemoteError::InvalidUrl(s),
+        other => RemoteError::Transport(other.to_string()),
+    }
+}
+
+impl HttpStorage {
+    /// Open `url` (`http://`, or `https://` with the `https` feature):
+    /// one ranged `GET` of the first [`HttpOptions::first_request`] bytes,
+    /// which gives the file's length and validators. Returns the storage
+    /// and the bytes that request fetched (the file's start), for a
+    /// [`BlockCache`](crate::BlockCache) to keep.
+    pub fn open(url: &str, options: HttpOptions) -> Result<(HttpStorage, Vec<u8>), RemoteError> {
+        let redactor = Redactor::new(url);
+        HttpStorage::open_inner(url, options, &redactor).map_err(|e| e.scrubbed(&redactor))
+    }
+
+    fn open_inner(
+        url: &str,
+        options: HttpOptions,
+        redactor: &Redactor,
+    ) -> Result<(HttpStorage, Vec<u8>), RemoteError> {
+        let shown = redactor.shown();
+        let lower = url.to_ascii_lowercase();
+        if lower.starts_with("https://") {
+            if !cfg!(feature = "https") {
+                return Err(RemoteError::UnsupportedScheme(format!(
+                    "{shown}: https:// needs the `https` feature of clawhdf5-remote"
+                )));
+            }
+        } else if !lower.starts_with("http://") {
+            return Err(RemoteError::UnsupportedScheme(shown.to_string()));
+        }
+        // Redirects are followed by `call`, which applies our rules.
+        let config = ureq::Agent::config_builder()
+            .http_status_as_error(false)
+            .max_redirects(0)
+            .timeout_connect(Some(options.timeout))
+            .timeout_recv_response(Some(options.timeout))
+            .build();
+        let mut storage = HttpStorage {
+            agent: ureq::Agent::new_with_config(config),
+            url: url.to_string(),
+            redactor: redactor.clone(),
+            len: 0,
+            validator: Validator::None,
+            options,
+            full: None,
+            requests: AtomicU64::new(0),
+            retries: AtomicU64::new(0),
+            bytes: AtomicU64::new(0),
+        };
+        let first = storage.with_retries(|| storage.probe())?;
+        let (len, validator, bytes, full) = first;
+        storage.len = len;
+        storage.validator = validator;
+        if full {
+            storage.full = Some(bytes);
+            return Ok((storage, Vec::new()));
+        }
+        if storage.options.require_validator && storage.validator == Validator::None {
+            return Err(RemoteError::Usage(format!(
+                "{shown}: the server sends neither a strong ETag nor Last-Modified, so a change \
+                 of the file could not be detected (HttpOptions::require_validator)"
+            )));
+        }
+        Ok((storage, bytes))
+    }
+
+    /// The URL as given — with any credentials it carries, so do not log
+    /// it; [`redact_url`](crate::redact_url) gives a form that can be.
+    pub fn url(&self) -> &str {
+        &self.url
+    }
+
+    /// Whether the file was downloaded whole, because the server does not
+    /// support ranges and [`HttpOptions::allow_full_download`] was set.
+    pub fn is_full_download(&self) -> bool {
+        self.full.is_some()
+    }
+
+    /// The `ETag` the file is pinned to, if the server sent a strong one.
+    pub fn etag(&self) -> Option<&str> {
+        match &self.validator {
+            Validator::ETag(e) => Some(e),
+            _ => None,
+        }
+    }
+
+    /// Requests and bytes so far.
+    pub fn stats(&self) -> HttpStats {
+        HttpStats {
+            requests: self.requests.load(Ordering::Relaxed),
+            retries: self.retries.load(Ordering::Relaxed),
+            bytes: self.bytes.load(Ordering::Relaxed),
+        }
+    }
+
+    fn with_retries<T>(
+        &self,
+        mut attempt: impl FnMut() -> Result<T, RemoteError>,
+    ) -> Result<T, RemoteError> {
+        let mut delay = self.options.backoff;
+        let mut n = 0;
+        loop {
+            match attempt() {
+                Ok(v) => return Ok(v),
+                Err(e) if e.is_transient() && n < self.options.retries => {
+                    n += 1;
+                    self.retries.fetch_add(1, Ordering::Relaxed);
+                    std::thread::sleep(delay);
+                    delay = delay.saturating_mul(2);
+                }
+                Err(e) => return Err(e),
+            }
+        }
+    }
+
+    /// One `GET` of `url`. The options' custom headers are sent only to
+    /// the URL's own origin (`trusted`).
+    fn request(
+        &self,
+        url: &str,
+        range: Option<(u64, u64)>,
+        trusted: bool,
+    ) -> ureq::RequestBuilder<ureq::typestate::WithoutBody> {
+        let mut req = self.agent.get(url).header("Accept-Encoding", "identity");
+        if let Some((a, b)) = range {
+            req = req.header("Range", format!("bytes={a}-{b}"));
+        }
+        // The body's budget scales with what it may carry: the range, or
+        // a whole file the server may send instead.
+        let mut body = range.map_or(0, |(a, b)| b.saturating_sub(a).saturating_add(1));
+        if self.options.allow_full_download {
+            body = body.max(self.options.max_full_download);
+        }
+        let secs = body as f64 / self.options.min_speed.max(1) as f64;
+        let body_timeout = self
+            .options
+            .timeout
+            .saturating_add(Duration::try_from_secs_f64(secs).unwrap_or(Duration::MAX));
+        let mut req = req.config().timeout_recv_body(Some(body_timeout)).build();
+        match &self.validator {
+            Validator::ETag(e) => req = req.header("If-Match", e),
+            Validator::LastModified(t) => req = req.header("If-Unmodified-Since", t),
+            Validator::None => {}
+        }
+        if trusted {
+            for (k, v) in &self.options.headers {
+                req = req.header(k, v);
+            }
+        }
+        req
+    }
+
+    /// Send a ranged `GET`, following redirects by the rules of
+    /// [`HttpOptions::max_redirects`]: at most that many, never from
+    /// `https` to anything else, and without the custom headers once the
+    /// chain has left the URL's origin. Each hop counts as a request.
+    fn call(
+        &self,
+        range: Option<(u64, u64)>,
+    ) -> Result<ureq::http::Response<ureq::Body>, RemoteError> {
+        let mut url = self.url.clone();
+        let mut trusted = true;
+        let mut hops = 0u32;
+        loop {
+            if hops > 0 {
+                self.requests.fetch_add(1, Ordering::Relaxed);
+            }
+            let resp = self
+                .request(&url, range, trusted)
+                .call()
+                .map_err(|e| transport(e).scrubbed(&Redactor::new(&url)))?;
+            if !matches!(resp.status().as_u16(), 301 | 302 | 303 | 307 | 308) {
+                return Ok(resp);
+            }
+            let status = resp.status().as_u16();
+            let Some(location) = header(&resp, "location") else {
+                return Err(RemoteError::BadResponse(format!(
+                    "{}: status {status} without a Location",
+                    redact_url(&url)
+                )));
+            };
+            let next = redirect_target(&url, location)?;
+            if hops >= self.options.max_redirects {
+                return Err(RemoteError::Redirect(format!(
+                    "{} redirects to {}: more than HttpOptions::max_redirects ({})",
+                    redact_url(&url),
+                    redact_url(&next),
+                    self.options.max_redirects
+                )));
+            }
+            if !same_origin(&next, &self.url) {
+                trusted = false;
+            }
+            url = next;
+            hops += 1;
+        }
+    }
+
+    /// Read a body of exactly `want` bytes (or up to `limit` when `want` is
+    /// unknown).
+    fn body(
+        &self,
+        resp: ureq::http::Response<ureq::Body>,
+        want: Option<u64>,
+        limit: u64,
+    ) -> Result<Vec<u8>, RemoteError> {
+        let cap = want.unwrap_or(limit);
+        let mut buf = Vec::with_capacity(usize::try_from(cap.min(64 << 20)).unwrap_or(0));
+        let reader = resp.into_body().into_reader();
+        let got = reader
+            .take(cap.saturating_add(1))
+            .read_to_end(&mut buf)
+            .map_err(|e| {
+                RemoteError::Transport(format!("{}: reading the body: {e}", self.redactor.shown()))
+            });
+        self.bytes.fetch_add(buf.len() as u64, Ordering::Relaxed);
+        got?;
+        match want {
+            Some(n) if buf.len() as u64 != n => Err(RemoteError::BadResponse(format!(
+                "{}: body of {} bytes, expected {n}",
+                self.redactor.shown(),
+                buf.len()
+            ))),
+            None if buf.len() as u64 > limit => Err(RemoteError::Usage(format!(
+                "{}: the file is larger than HttpOptions::max_full_download ({limit} bytes)",
+                self.redactor.shown()
+            ))),
+            _ => Ok(buf),
+        }
+    }
+
+    /// The first request: length, validators and the file's first bytes.
+    /// The last field is true when the server ignored the range and sent
+    /// the whole file (only kept when a full download is allowed).
+    fn probe(&self) -> Result<(u64, Validator, Vec<u8>, bool), RemoteError> {
+        let n = self.options.first_request.max(1);
+        self.requests.fetch_add(1, Ordering::Relaxed);
+        let resp = self.call(Some((0, n - 1)))?;
+        let status = resp.status().as_u16();
+        check_identity(self.redactor.shown(), &resp)?;
+        let validator = match (header(&resp, "etag"), header(&resp, "last-modified")) {
+            (Some(e), _) if !e.starts_with("W/") => Validator::ETag(e.to_string()),
+            (_, Some(t)) => Validator::LastModified(t.to_string()),
+            _ => Validator::None,
+        };
+        match status {
+            206 => {
+                let cr = header(&resp, "content-range").ok_or_else(|| {
+                    RemoteError::BadResponse(format!(
+                        "{}: 206 without Content-Range",
+                        self.redactor.shown()
+                    ))
+                })?;
+                let (a, b, total) = content_range(cr).ok_or_else(|| {
+                    RemoteError::BadResponse(format!(
+                        "{}: bad Content-Range {cr:?}",
+                        self.redactor.shown()
+                    ))
+                })?;
+                let total = total.ok_or_else(|| {
+                    RemoteError::BadResponse(format!(
+                        "{}: the server does not report the file's length (Content-Range {cr:?})",
+                        self.redactor.shown()
+                    ))
+                })?;
+                if a != 0 || b >= total || b > n - 1 {
+                    return Err(RemoteError::BadResponse(format!(
+                        "{}: asked for bytes 0-{}, got Content-Range {cr:?}",
+                        self.redactor.shown(),
+                        n - 1
+                    )));
+                }
+                let bytes = self.body(resp, Some(b - a + 1), 0)?;
+                Ok((total, validator, bytes, false))
+            }
+            200 => {
+                // RFC 9110 lets a server answer 200 when the range covers
+                // the whole file: a body no longer than the range asked
+                // for is the whole file, ranges supported or not.
+                let want: Option<u64> =
+                    header(&resp, "content-length").and_then(|v| v.trim().parse().ok());
+                let refused = || {
+                    RemoteError::RangeNotSupported(format!(
+                        "{} answered a range request with the whole file (status 200); set \
+                         HttpOptions::allow_full_download to download it",
+                        self.redactor.shown()
+                    ))
+                };
+                if !self.options.allow_full_download {
+                    return match want {
+                        Some(w) if w <= n => {
+                            let bytes = self.body(resp, Some(w), 0)?;
+                            Ok((bytes.len() as u64, validator, bytes, true))
+                        }
+                        Some(_) => Err(refused()),
+                        // No length: read at most the range asked for.
+                        None => match self.body(resp, None, n) {
+                            Ok(bytes) => Ok((bytes.len() as u64, validator, bytes, true)),
+                            Err(RemoteError::Usage(_)) => Err(refused()),
+                            Err(e) => Err(e),
+                        },
+                    };
+                }
+                if want.is_some_and(|w| w > self.options.max_full_download) {
+                    return Err(RemoteError::Usage(format!(
+                        "{}: the file is larger than HttpOptions::max_full_download ({} bytes)",
+                        self.redactor.shown(),
+                        self.options.max_full_download
+                    )));
+                }
+                let bytes = self.body(resp, want, self.options.max_full_download)?;
+                Ok((bytes.len() as u64, validator, bytes, true))
+            }
+            416 => Err(RemoteError::Usage(format!(
+                "{}: status 416 for the first bytes (an empty file?)",
+                self.redactor.shown()
+            ))),
+            code => Err(RemoteError::Status {
+                code,
+                what: self.redactor.shown().to_string(),
+            }),
+        }
+    }
+
+    /// One request for `[start, end)` (inside the file, non-empty).
+    fn fetch_once(&self, start: u64, end: u64) -> Result<Vec<u8>, RemoteError> {
+        self.requests.fetch_add(1, Ordering::Relaxed);
+        let resp = self.call(Some((start, end - 1)))?;
+        let changed =
+            |why: String| RemoteError::FileChanged(format!("{}: {why}", self.redactor.shown()));
+        check_identity(self.redactor.shown(), &resp)?;
+        match resp.status().as_u16() {
+            206 => {}
+            200 => {
+                return Err(RemoteError::RangeNotSupported(format!(
+                    "{} answered a range request with the whole file (status 200)",
+                    self.redactor.shown()
+                )));
+            }
+            412 => {
+                return Err(changed(
+                    "If-Match/If-Unmodified-Since failed (status 412)".into(),
+                ));
+            }
+            416 => return Err(changed("range no longer satisfiable (status 416)".into())),
+            code => {
+                return Err(RemoteError::Status {
+                    code,
+                    what: format!("{} bytes {start}-{}", self.redactor.shown(), end - 1),
+                });
+            }
+        }
+        match &self.validator {
+            Validator::ETag(e) => {
+                if let Some(got) = header(&resp, "etag")
+                    && got != e
+                {
+                    return Err(changed(format!("ETag {got} instead of {e}")));
+                }
+            }
+            Validator::LastModified(t) => {
+                if let Some(got) = header(&resp, "last-modified")
+                    && got != t
+                {
+                    return Err(changed(format!("Last-Modified {got} instead of {t}")));
+                }
+            }
+            Validator::None => {}
+        }
+        let cr = header(&resp, "content-range").ok_or_else(|| {
+            RemoteError::BadResponse(format!(
+                "{}: 206 without Content-Range",
+                self.redactor.shown()
+            ))
+        })?;
+        let (a, b, total) = content_range(cr).ok_or_else(|| {
+            RemoteError::BadResponse(format!(
+                "{}: bad Content-Range {cr:?}",
+                self.redactor.shown()
+            ))
+        })?;
+        if let Some(total) = total
+            && total != self.len
+        {
+            return Err(changed(format!("length {total} instead of {}", self.len)));
+        }
+        if a != start || b != end - 1 {
+            return Err(RemoteError::BadResponse(format!(
+                "{}: asked for bytes {start}-{}, got Content-Range {cr:?}",
+                self.redactor.shown(),
+                end - 1
+            )));
+        }
+        self.body(resp, Some(end - start), 0)
+    }
+
+    fn fetch(&self, start: u64, end: u64) -> Result<Vec<u8>, RemoteError> {
+        self.with_retries(|| self.fetch_once(start, end))
+            .map_err(|e| e.scrubbed(&self.redactor))
+    }
+
+    /// `[offset, offset + len)` clamped to the file, or `None` if empty.
+    fn clamp(&self, offset: u64, len: u64) -> Option<(u64, u64)> {
+        let end = offset.saturating_add(len).min(self.len);
+        (offset < end).then_some((offset, end))
+    }
+}
+
+impl Storage for HttpStorage {
+    fn read_at(&self, offset: u64, len: usize) -> Result<Cow<'_, [u8]>, FormatError> {
+        if let Some(all) = &self.full {
+            return all.as_slice().read_at(offset, len);
+        }
+        match self.clamp(offset, len as u64) {
+            None => Ok(Cow::Owned(Vec::new())),
+            Some((a, b)) => Ok(Cow::Owned(self.fetch(a, b)?)),
+        }
+    }
+
+    fn len(&self) -> u64 {
+        self.len
+    }
+
+    fn read_ranges(&self, ranges: &[Range<u64>]) -> Result<Vec<Cow<'_, [u8]>>, FormatError> {
+        if let Some(all) = &self.full {
+            return all.as_slice().read_ranges(ranges);
+        }
+        let parallel = self.options.max_parallel.clamp(1, ranges.len().max(1));
+        if parallel <= 1 {
+            return ranges
+                .iter()
+                .map(|r| self.read_at(r.start, (r.end.saturating_sub(r.start)) as usize))
+                .collect();
+        }
+        let next = AtomicUsize::new(0);
+        let failed = std::sync::atomic::AtomicBool::new(false);
+        type Slot = Option<Result<Vec<u8>, RemoteError>>;
+        let results: std::sync::Mutex<Vec<Slot>> =
+            std::sync::Mutex::new((0..ranges.len()).map(|_| None).collect());
+        std::thread::scope(|s| {
+            for _ in 0..parallel {
+                s.spawn(|| {
+                    loop {
+                        let i = next.fetch_add(1, Ordering::Relaxed);
+                        if i >= ranges.len() || failed.load(Ordering::Relaxed) {
+                            break;
+                        }
+                        let r = &ranges[i];
+                        let got = match self.clamp(r.start, r.end.saturating_sub(r.start)) {
+                            None => Ok(Vec::new()),
+                            Some((a, b)) => self.fetch(a, b),
+                        };
+                        if got.is_err() {
+                            failed.store(true, Ordering::Relaxed);
+                        }
+                        results
+                            .lock()
+                            .unwrap_or_else(std::sync::PoisonError::into_inner)[i] = Some(got);
+                    }
+                });
+            }
+        });
+        let results = results
+            .into_inner()
+            .unwrap_or_else(std::sync::PoisonError::into_inner);
+        let mut out = Vec::with_capacity(ranges.len());
+        for r in results {
+            match r {
+                Some(Ok(v)) => out.push(Cow::Owned(v)),
+                Some(Err(e)) => return Err(e.into()),
+                // Not fetched because another range failed first.
+                None => continue,
+            }
+        }
+        if out.len() != ranges.len() {
+            return Err(FormatError::Storage(format!(
+                "{}: a parallel range read failed",
+                self.redactor.shown()
+            )));
+        }
+        Ok(out)
+    }
+
+    fn as_contiguous(&self) -> Option<&[u8]> {
+        self.full.as_deref()
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::content_range;
+
+    #[test]
+    fn content_range_parses() {
+        assert_eq!(content_range("bytes 0-99/1000"), Some((0, 99, Some(1000))));
+        assert_eq!(content_range("bytes 5-5/*"), Some((5, 5, None)));
+        assert_eq!(content_range("bytes 9-5/10"), None);
+        assert_eq!(content_range("items 0-1/2"), None);
+        assert_eq!(content_range("bytes */1000"), None);
+    }
+
+    #[test]
+    fn redirect_targets_and_origins() {
+        use super::{redirect_target, same_origin};
+        let t = |b: &str, l: &str| redirect_target(b, l).map_err(|e| e.to_string());
+        assert_eq!(
+            t("http://a:8/d/f.h5?x=1", "g.h5").unwrap(),
+            "http://a:8/d/g.h5"
+        );
+        assert_eq!(t("http://a/d/f.h5", "/g.h5").unwrap(), "http://a/g.h5");
+        assert_eq!(t("https://a/d/f.h5", "//b/g.h5").unwrap(), "https://b/g.h5");
+        assert_eq!(t("http://a/f", "https://b/g").unwrap(), "https://b/g");
+        let e = t("https://a/f.h5", "http://a/f.h5").unwrap_err();
+        assert!(e.contains("downgrade"), "{e}");
+        assert!(t("HTTPS://a/f.h5", "http://b/f.h5").is_err());
+        assert!(t("https://a/f", "//b/g").is_ok());
+        assert!(t("http://a/f", "ftp://b/g").is_err());
+        let e = t("https://u:pw@a/f?sig=SECRET", "http://b/g?sig=OTHER").unwrap_err();
+        assert!(
+            !e.contains("SECRET") && !e.contains("OTHER") && !e.contains("pw"),
+            "{e}"
+        );
+        assert!(same_origin("http://a/x", "http://A:80/y"));
+        assert!(same_origin("https://u:p@a:443/x", "https://a/y"));
+        assert!(!same_origin("http://a/x", "https://a/x"));
+        assert!(!same_origin("http://a:1/x", "http://a:2/x"));
+        assert!(!same_origin("http://a/x", "http://b/x"));
+        assert!(same_origin("http://[::1]:8/x", "http://[::1]:8/y"));
+        assert!(!same_origin("http://[::1]/x", "http://[::1]:8/y"));
+    }
+}

crates/clawhdf5-remote/src/lib.rs

@@ -0,0 +1,228 @@
+//! Read HDF5 files where they are — on an HTTP(S) server or in an object
+//! store — without downloading them first.
+//!
+//! This is milestone M3 of `docs/design/range-reads.md`: remote backends for
+//! [`clawhdf5::File::open_storage`], each read through a mandatory
+//! [`BlockCache`].
+//!
+//! ```no_run
+//! let file = clawhdf5_remote::open_url("http://127.0.0.1:8000/data.h5")?;
+//! let temperature = file.dataset("/grid/temperature")?.read_f64()?;
+//! # Ok::<(), Box<dyn std::error::Error>>(())
+//! ```
+//!
+//! - [`open_url`] / [`open_url_with`]: `http://` (default feature `http`),
+//!   `https://` (feature `https`), `s3://`, `gs://`, `az://` (features `s3`,
+//!   `gcs`, `azure`) → a [`clawhdf5::File`] with the whole read API.
+//! - [`storage_for_url`] gives the cached storage itself, to open with
+//!   [`clawhdf5::File::open_storage`] and to read its [`CacheStats`].
+//! - [`download`] reads a whole remote file into memory, up to a limit.
+//! - [`HttpStorage`] (range `GET`s, pinned by ETag/Last-Modified, retried
+//!   with backoff), [`ObjectStoreStorage`] (any `object_store` store,
+//!   feature `object-store`), and [`BlockCache`] over any
+//!   [`Storage`](clawhdf5_format::storage::Storage).
+//!
+//! What costs what: opening costs one request (it also fetches the first
+//! block, 1 MiB by default); listing a file whose metadata sits in its
+//! first blocks costs nothing more; reading a chunked dataset costs one
+//! parallel batch of requests for the blocks holding its chunk index, then
+//! one for its chunks. The zero-copy methods of `clawhdf5`
+//! (`read_raw_ref`, `read_*_zerocopy`, `File::as_bytes`) need the file in
+//! memory and are errors (`as_bytes` a panic) on a remote file.
+
+#![warn(missing_docs)]
+
+pub mod cache;
+pub mod error;
+#[cfg(feature = "http")]
+pub mod http;
+#[cfg(feature = "object-store")]
+pub mod object;
+
+use std::sync::Arc;
+
+use clawhdf5::File;
+use clawhdf5_format::storage::Storage;
+
+pub use cache::{BlockCache, CacheConfig, CacheStats};
+pub use error::{Error, RemoteError, redact_url};
+#[cfg(feature = "http")]
+pub use http::{HttpOptions, HttpStats, HttpStorage};
+#[cfg(feature = "object-store")]
+pub use object::ObjectStoreStorage;
+#[cfg(feature = "object-store")]
+pub use object_store;
+
+/// A backend a [`BlockCache`] can read through.
+pub type Backend = Box<dyn Storage + Send + Sync>;
+
+/// The storage [`storage_for_url`] returns: a block cache over the URL's
+/// backend.
+pub type RemoteStorage = BlockCache<Backend>;
+
+/// Settings of [`open_url_with`] and [`storage_for_url`].
+#[derive(Debug, Clone, Default)]
+pub struct Options {
+    /// The block cache.
+    pub cache: CacheConfig,
+    /// HTTP(S) requests.
+    #[cfg(feature = "http")]
+    pub http: HttpOptions,
+}
+
+/// Open the HDF5 file at `url` with default [`Options`].
+///
+/// `http://…` needs the (default) `http` feature, `https://…` the `https`
+/// feature, `s3://bucket/key`, `gs://bucket/key` and `az://container/key`
+/// the `s3`, `gcs` and `azure` features (credentials and region from the
+/// environment, as `object_store`'s `from_env` builders read them).
+pub fn open_url(url: &str) -> Result<File, Error> {
+    open_url_with(url, &Options::default())
+}
+
+/// [`open_url`] with explicit [`Options`].
+pub fn open_url_with(url: &str, options: &Options) -> Result<File, Error> {
+    let storage = storage_for_url(url, options)?;
+    Ok(File::open_storage(storage)?)
+}
+
+/// The cached storage for `url`, with the first block already fetched:
+/// open it with [`clawhdf5::File::open_storage`] (a clone of the `Arc`),
+/// and read its [`BlockCache::stats`] as you go.
+pub fn storage_for_url(url: &str, options: &Options) -> Result<Arc<RemoteStorage>, Error> {
+    let scheme = url
+        .split_once("://")
+        .map(|(s, _)| s.to_ascii_lowercase())
+        .ok_or_else(|| RemoteError::InvalidUrl(format!("{}: no scheme", redact_url(url))))?;
+    match scheme.as_str() {
+        "http" | "https" => http_storage(url, options),
+        "s3" | "s3a" | "gs" | "az" | "azure" | "abfs" | "abfss" | "adl" => {
+            cloud_storage(url, &scheme, options)
+        }
+        _ => Err(RemoteError::UnsupportedScheme(redact_url(url)).into()),
+    }
+}
+
+#[cfg(feature = "http")]
+fn http_storage(url: &str, options: &Options) -> Result<Arc<RemoteStorage>, Error> {
+    let mut http = options.http.clone();
+    http.first_request = http.first_request.max(options.cache.block_size.max(1));
+    let (storage, first) = HttpStorage::open(url, http)?;
+    let cache = BlockCache::new(Box::new(storage) as Backend, options.cache.clone());
+    cache.insert(0, &first);
+    Ok(Arc::new(cache))
+}
+
+#[cfg(not(feature = "http"))]
+fn http_storage(url: &str, _options: &Options) -> Result<Arc<RemoteStorage>, Error> {
+    Err(RemoteError::UnsupportedScheme(format!(
+        "{}: http(s):// needs the `http` feature of clawhdf5-remote",
+        redact_url(url)
+    ))
+    .into())
+}
+
+#[cfg(any(feature = "s3", feature = "gcs", feature = "azure"))]
+fn cloud_storage(url: &str, _scheme: &str, options: &Options) -> Result<Arc<RemoteStorage>, Error> {
+    let (store, path) = object::store_for_url(url)?;
+    let storage = ObjectStoreStorage::new(store, path)?;
+    Ok(Arc::new(object_cached(storage, options)?))
+}
+
+#[cfg(not(any(feature = "s3", feature = "gcs", feature = "azure")))]
+fn cloud_storage(url: &str, scheme: &str, _options: &Options) -> Result<Arc<RemoteStorage>, Error> {
+    let feature = match scheme {
+        "s3" | "s3a" => "s3",
+        "gs" => "gcs",
+        _ => "azure",
+    };
+    Err(RemoteError::UnsupportedScheme(format!(
+        "{}: {scheme}:// needs the `{feature}` feature of clawhdf5-remote",
+        redact_url(url)
+    ))
+    .into())
+}
+
+/// A [`BlockCache`] over `backend` with its first block fetched (readahead
+/// of the superblock and the metadata usually written next to it). A
+/// failure of that fetch is [`Error::Remote`] ([`RemoteError::Backend`],
+/// with the backend's message).
+pub fn cached(backend: Backend, options: &Options) -> Result<RemoteStorage, Error> {
+    let cache = BlockCache::new(backend, options.cache.clone());
+    let first = cache.config().block_size;
+    cache
+        .prefetch(0, first)
+        .map_err(|e| RemoteError::Backend(e.to_string()))?;
+    Ok(cache)
+}
+
+/// [`cached`] for an [`ObjectStoreStorage`]: its first block is fetched
+/// directly, so a failure keeps its kind (`FileChanged`, `ObjectStore`).
+#[cfg(feature = "object-store")]
+fn object_cached(storage: ObjectStoreStorage, options: &Options) -> Result<RemoteStorage, Error> {
+    // The block size BlockCache::new will use.
+    let block = options.cache.block_size.max(512);
+    let first = storage.fetch_first(block)?;
+    let cache = BlockCache::new(Box::new(storage) as Backend, options.cache.clone());
+    cache.insert(0, &first);
+    Ok(cache)
+}
+
+/// Default limit of [`download`]: 1 GiB.
+pub const DEFAULT_MAX_DOWNLOAD: u64 = 1 << 30;
+
+/// The whole file behind `storage`, read into memory — at most
+/// `max_bytes` of it (for example [`DEFAULT_MAX_DOWNLOAD`]).
+///
+/// The length is only what the server claims, so it is never used to
+/// allocate: a file longer than `max_bytes` is refused with
+/// [`RemoteError::TooLarge`] before anything is read, and the buffer grows
+/// only as bytes arrive (64 MiB per step, fetched as parallel requests by a
+/// [`BlockCache`]). A read that comes back short is an error.
+pub fn download(storage: &dyn Storage, max_bytes: u64) -> Result<Vec<u8>, Error> {
+    let len = storage.len();
+    if len > max_bytes {
+        return Err(RemoteError::TooLarge {
+            len,
+            limit: max_bytes,
+        }
+        .into());
+    }
+    const STEP: u64 = 64 << 20;
+    let mut out = Vec::new();
+    let mut pos = 0u64;
+    while pos < len {
+        let want = (len - pos).min(STEP);
+        let got = storage
+            .read_at(pos, want as usize)
+            .map_err(|e| RemoteError::Backend(e.to_string()))?;
+        if got.len() as u64 != want {
+            return Err(RemoteError::BadResponse(format!(
+                "{} bytes at offset {pos} instead of {want}",
+                got.len()
+            ))
+            .into());
+        }
+        out.extend_from_slice(&got);
+        pos += want;
+    }
+    Ok(out)
+}
+
+/// Open the object at `path` of any `object_store` store (in memory, local
+/// files, or a cloud store you configured) through a block cache.
+#[cfg(feature = "object-store")]
+pub fn open_object(
+    store: Arc<dyn object_store::ObjectStore>,
+    path: &str,
+    options: &Options,
+) -> Result<(File, Arc<RemoteStorage>), Error> {
+    let path = object_store::path::Path::parse(path)
+        .map_err(|e| RemoteError::InvalidUrl(format!("{path}: {e}")))?;
+    let storage = Arc::new(object_cached(
+        ObjectStoreStorage::new(store, path)?,
+        options,
+    )?);
+    let file = File::open_storage(storage.clone())?;
+    Ok((file, storage))
+}

crates/clawhdf5-remote/src/object.rs

@@ -0,0 +1,297 @@
+//! Object stores (S3, GCS, Azure, local files, memory) through the
+//! [`object_store`] crate: [`ObjectStoreStorage`].
+//!
+//! `object_store` is async and [`Storage`] is synchronous (parsing is CPU
+//! work; `docs/design/range-reads.md` §3 (a)). The storage owns a small
+//! multi-threaded tokio runtime (two worker threads): each read is spawned
+//! on it and the calling thread waits for the result, so it can be used
+//! from any thread — several at once, and from async code too. From async
+//! code prefer `tokio::task::spawn_blocking` (a read blocks the thread it
+//! is called on, which inside a runtime is one of its workers).
+//!
+//! The object is pinned when the storage is made: its size, and its ETag
+//! (sent as `If-Match` with every read, and compared with every response)
+//! or, without one, its version or modification time. A change while it is
+//! open is [`RemoteError::FileChanged`]. The ranges of one `read_ranges`
+//! call are fetched concurrently (at most 8 at a time).
+
+use std::borrow::Cow;
+use std::ops::Range;
+use std::sync::Arc;
+use std::sync::atomic::{AtomicU64, Ordering};
+
+use clawhdf5_format::error::FormatError;
+use clawhdf5_format::storage::Storage;
+use futures_util::{StreamExt, TryStreamExt};
+use object_store::path::Path;
+use object_store::{GetOptions, GetRange, ObjectMeta, ObjectStore, ObjectStoreExt};
+
+use crate::error::RemoteError;
+
+/// Concurrent requests of one `read_ranges` call.
+const MAX_CONCURRENT: usize = 8;
+
+/// One object of an [`ObjectStore`], read by ranged `get`s. See the
+/// [module documentation](self).
+pub struct ObjectStoreStorage {
+    store: Arc<dyn ObjectStore>,
+    path: Path,
+    meta: ObjectMeta,
+    runtime: Option<tokio::runtime::Runtime>,
+    requests: AtomicU64,
+    bytes: AtomicU64,
+}
+
+impl std::fmt::Debug for ObjectStoreStorage {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        f.debug_struct("ObjectStoreStorage")
+            .field("store", &self.store.to_string())
+            .field("path", &self.path)
+            .field("size", &self.meta.size)
+            .field("e_tag", &self.meta.e_tag)
+            .finish()
+    }
+}
+
+fn os_error(e: object_store::Error) -> RemoteError {
+    match e {
+        object_store::Error::Precondition { .. } | object_store::Error::NotModified { .. } => {
+            RemoteError::FileChanged(e.to_string())
+        }
+        other => RemoteError::ObjectStore(other.to_string()),
+    }
+}
+
+impl ObjectStoreStorage {
+    /// Open the object at `path` of `store`: one `head` request for its
+    /// size and validators.
+    pub fn new(store: Arc<dyn ObjectStore>, path: Path) -> Result<Self, RemoteError> {
+        let runtime = tokio::runtime::Builder::new_multi_thread()
+            .worker_threads(2)
+            .thread_name("clawhdf5-remote")
+            .enable_all()
+            .build()
+            .map_err(|e| RemoteError::Usage(format!("cannot start a tokio runtime: {e}")))?;
+        let mut s = ObjectStoreStorage {
+            store,
+            path,
+            meta: ObjectMeta {
+                location: Path::default(),
+                last_modified: Default::default(),
+                size: 0,
+                e_tag: None,
+                version: None,
+            },
+            runtime: Some(runtime),
+            requests: AtomicU64::new(1),
+            bytes: AtomicU64::new(0),
+        };
+        let (store, path) = (s.store.clone(), s.path.clone());
+        s.meta = s.block_on(async move { store.head(&path).await.map_err(os_error) })?;
+        Ok(s)
+    }
+
+    /// The object's metadata as pinned at open.
+    pub fn meta(&self) -> &ObjectMeta {
+        &self.meta
+    }
+
+    /// Requests made (the `head` included) and bytes received.
+    pub fn stats(&self) -> (u64, u64) {
+        (
+            self.requests.load(Ordering::Relaxed),
+            self.bytes.load(Ordering::Relaxed),
+        )
+    }
+
+    /// Run `fut` on the storage's own runtime and wait for it. The future
+    /// never runs on the caller's thread, so the caller's context does not
+    /// matter: a plain thread, `spawn_blocking`, or even inside another
+    /// runtime (whose thread is then blocked for the duration of the read,
+    /// as by any blocking call, but nothing deadlocks or panics).
+    fn block_on<T: Send + 'static>(
+        &self,
+        fut: impl std::future::Future<Output = Result<T, RemoteError>> + Send + 'static,
+    ) -> Result<T, RemoteError> {
+        let rt = self.runtime.as_ref().expect("runtime lives until drop");
+        let (tx, rx) = std::sync::mpsc::sync_channel(1);
+        rt.spawn(async move {
+            let _ = tx.send(fut.await);
+        });
+        rx.recv().map_err(|_| {
+            RemoteError::ObjectStore("the object store task ended without a result".into())
+        })?
+    }
+
+    fn options(&self, range: Range<u64>) -> GetOptions {
+        let mut o = GetOptions {
+            range: Some(GetRange::Bounded(range)),
+            ..GetOptions::default()
+        };
+        if let Some(e) = &self.meta.e_tag {
+            o.if_match = Some(e.clone());
+        } else if let Some(v) = &self.meta.version {
+            o.version = Some(v.clone());
+        } else {
+            o.if_unmodified_since = Some(self.meta.last_modified);
+        }
+        o
+    }
+
+    /// The object's first `n` bytes (fewer if it is shorter).
+    pub(crate) fn fetch_first(&self, n: u64) -> Result<Vec<u8>, RemoteError> {
+        Ok(self
+            .fetch_all(std::slice::from_ref(&(0..n)))?
+            .pop()
+            .unwrap_or_default())
+    }
+
+    fn fetch_all(&self, ranges: &[Range<u64>]) -> Result<Vec<Vec<u8>>, RemoteError> {
+        let len = self.meta.size;
+        let jobs: Vec<(usize, Range<u64>)> = ranges
+            .iter()
+            .enumerate()
+            .filter_map(|(i, r)| {
+                let end = r.end.min(len);
+                (r.start < end).then_some((i, r.start..end))
+            })
+            .collect();
+        let store = self.store.clone();
+        let path = self.path.clone();
+        let pinned = self.meta.e_tag.clone();
+        let reqs: Vec<(usize, Range<u64>, GetOptions)> = jobs
+            .into_iter()
+            .map(|(i, r)| {
+                let o = self.options(r.clone());
+                (i, r, o)
+            })
+            .collect();
+        self.requests
+            .fetch_add(reqs.len() as u64, Ordering::Relaxed);
+        let fetched: Vec<(usize, Vec<u8>)> = self.block_on(async move {
+            futures_util::stream::iter(reqs)
+                .map(|(i, r, o)| {
+                    let (store, path, pinned) = (store.clone(), path.clone(), pinned.clone());
+                    async move {
+                        let got = store.get_opts(&path, o).await.map_err(os_error)?;
+                        if let (Some(want), Some(have)) = (&pinned, &got.meta.e_tag)
+                            && want != have
+                        {
+                            return Err(RemoteError::FileChanged(format!(
+                                "{path}: ETag {have} instead of {want}"
+                            )));
+                        }
+                        if got.meta.size != len {
+                            return Err(RemoteError::FileChanged(format!(
+                                "{path}: size {} instead of {len}",
+                                got.meta.size
+                            )));
+                        }
+                        let bytes = got.bytes().await.map_err(os_error)?;
+                        if bytes.len() as u64 != r.end - r.start {
+                            return Err(RemoteError::BadResponse(format!(
+                                "{path}: {} bytes for range {r:?}",
+                                bytes.len()
+                            )));
+                        }
+                        Ok::<_, RemoteError>((i, bytes.to_vec()))
+                    }
+                })
+                .buffer_unordered(MAX_CONCURRENT)
+                .try_collect()
+                .await
+        })?;
+        let mut out = vec![Vec::new(); ranges.len()];
+        for (i, b) in fetched {
+            self.bytes.fetch_add(b.len() as u64, Ordering::Relaxed);
+            out[i] = b;
+        }
+        Ok(out)
+    }
+}
+
+impl Drop for ObjectStoreStorage {
+    fn drop(&mut self) {
+        // Dropping a runtime blocks, which panics inside an async context.
+        if let Some(rt) = self.runtime.take() {
+            rt.shutdown_background();
+        }
+    }
+}
+
+impl Storage for ObjectStoreStorage {
+    fn read_at(&self, offset: u64, len: usize) -> Result<Cow<'_, [u8]>, FormatError> {
+        let range = offset..offset.saturating_add(len as u64);
+        let mut v = self.fetch_all(std::slice::from_ref(&range))?;
+        Ok(Cow::Owned(v.pop().unwrap_or_default()))
+    }
+
+    fn len(&self) -> u64 {
+        self.meta.size
+    }
+
+    fn read_ranges(&self, ranges: &[Range<u64>]) -> Result<Vec<Cow<'_, [u8]>>, FormatError> {
+        if ranges.iter().any(|r| r.end < r.start) {
+            return Err(FormatError::Storage(
+                "read range ends before it starts".into(),
+            ));
+        }
+        Ok(self
+            .fetch_all(ranges)?
+            .into_iter()
+            .map(Cow::Owned)
+            .collect())
+    }
+}
+
+/// The store and object path for a cloud URL (`s3://bucket/key`,
+/// `gs://bucket/key`, `az://container/key`, ...), configured from the
+/// environment as `object_store`'s `from_env` builders do (`AWS_*`,
+/// `GOOGLE_*`, `AZURE_*`).
+#[cfg(any(feature = "s3", feature = "gcs", feature = "azure"))]
+pub(crate) fn store_for_url(url: &str) -> Result<(Arc<dyn ObjectStore>, Path), RemoteError> {
+    let redactor = crate::error::Redactor::new(url);
+    let parsed = object_store::path::Path::parse(
+        url.split_once("://")
+            .and_then(|(_, rest)| rest.split_once('/'))
+            .map(|(_, key)| key)
+            .unwrap_or(""),
+    )
+    .map_err(|e| RemoteError::InvalidUrl(format!("{}: {e}", crate::redact_url(url))))?;
+    let scheme = url.split_once("://").map(|(s, _)| s.to_ascii_lowercase());
+    let store: Arc<dyn ObjectStore> = match scheme.as_deref() {
+        #[cfg(feature = "s3")]
+        Some("s3" | "s3a") => Arc::new(
+            object_store::aws::AmazonS3Builder::from_env()
+                .with_url(url)
+                .build()
+                .map_err(|e| os_error(e).scrubbed(&redactor))?,
+        ),
+        #[cfg(feature = "gcs")]
+        Some("gs") => Arc::new(
+            object_store::gcp::GoogleCloudStorageBuilder::from_env()
+                .with_url(url)
+                .build()
+                .map_err(|e| os_error(e).scrubbed(&redactor))?,
+        ),
+        #[cfg(feature = "azure")]
+        Some("az" | "azure" | "abfs" | "abfss" | "adl") => Arc::new(
+            object_store::azure::MicrosoftAzureBuilder::from_env()
+                .with_url(url)
+                .build()
+                .map_err(|e| os_error(e).scrubbed(&redactor))?,
+        ),
+        _ => return Err(RemoteError::UnsupportedScheme(crate::redact_url(url))),
+    };
+    Ok((store, parsed))
+}
+
+#[cfg(all(test, feature = "s3"))]
+mod tests {
+    #[test]
+    fn s3_urls_name_the_bucket_and_key() {
+        let (store, path) = super::store_for_url("s3://my-bucket/dir/file.h5").unwrap();
+        assert_eq!(path.as_ref(), "dir/file.h5");
+        assert!(store.to_string().contains("my-bucket"), "{store}");
+    }
+}

crates/clawhdf5-remote/tests/common/mod.rs

@@ -0,0 +1,318 @@
+//! Shared by the integration tests: the test server, a transcript of a
+//! file (tree, attributes, values) to compare two ways of reading it, and
+//! the test files.
+
+#![allow(dead_code)]
+
+pub mod server;
+
+use std::collections::{BTreeMap, HashSet, VecDeque};
+use std::fmt::Write as _;
+use std::path::{Path, PathBuf};
+use std::process::Command;
+use std::sync::Arc;
+
+use clawhdf5::{DType, File, Selection};
+use clawhdf5_format::error::FormatError;
+
+/// Objects visited per file.
+const MAX_OBJECTS: usize = 2000;
+/// Datasets with more bytes than this are not read (their metadata is).
+pub const MAX_DATA_BYTES: u64 = 64 << 20;
+
+/// A short, stable digest of a value's `Debug` form.
+fn digest<T: std::fmt::Debug>(v: &T) -> String {
+    let s = format!("{v:?}");
+    if s.len() <= 200 {
+        return s;
+    }
+    let mut h = 0xcbf2_9ce4_8422_2325u64;
+    for b in s.bytes() {
+        h = (h ^ u64::from(b)).wrapping_mul(0x100_0000_01b3);
+    }
+    format!("{}…[{} bytes, fnv {h:016x}]", &s[..80], s.len())
+}
+
+/// A data read's value, or `Err` (which chunk a damaged dataset reports
+/// can vary between two `File`s: the chunk cache lists in hash order).
+fn value<T: std::fmt::Debug, E>(r: &Result<T, E>) -> String {
+    match r {
+        Ok(v) => digest(v),
+        Err(_) => "Err".into(),
+    }
+}
+
+fn sorted<V: std::fmt::Debug>(m: std::collections::HashMap<String, V>) -> BTreeMap<String, V> {
+    m.into_iter().collect()
+}
+
+/// Everything a reader sees in `file`: every group's entries, every
+/// object's attributes, and every dataset's shape, types and values.
+pub fn transcript(file: &File) -> String {
+    let mut out = String::new();
+    let mut seen = HashSet::new();
+    let mut queue = VecDeque::from([(String::from("/"), file.superblock().root_group_address)]);
+    while let Some((path, addr)) = queue.pop_front() {
+        if seen.len() >= MAX_OBJECTS || !seen.insert(addr) {
+            continue;
+        }
+        let group = file.group_at(addr);
+        let entries = group.entries();
+        writeln!(out, "{path} @{addr} entries {}", digest(&entries)).unwrap();
+        if let Ok(ds) = file.dataset_at(addr) {
+            dataset(&mut out, &path, &ds);
+        }
+        let attrs = group.attrs_with_errors().map(|(a, e)| (sorted(a), e));
+        writeln!(out, "{path} attrs {}", digest(&attrs)).unwrap();
+        if let Ok(entries) = entries {
+            for (name, child) in entries {
+                queue.push_back((format!("{}/{name}", path.trim_end_matches('/')), child));
+            }
+        }
+    }
+    out
+}
+
+fn dataset(out: &mut String, path: &str, ds: &clawhdf5::Dataset<'_>) {
+    let shape = ds.shape();
+    let dtype = ds.dtype();
+    writeln!(
+        out,
+        "{path} shape {} dtype {} raw {}",
+        digest(&shape),
+        digest(&dtype),
+        digest(&ds.raw_datatype())
+    )
+    .unwrap();
+    let (Ok(shape), Ok(dtype), Ok(raw_dt)) = (shape, dtype, ds.raw_datatype()) else {
+        return;
+    };
+    let elements = shape.iter().try_fold(1u64, |a, &d| a.checked_mul(d));
+    let bytes = elements.and_then(|n| n.checked_mul(u64::from(raw_dt.type_size())));
+    if bytes.is_none_or(|b| b > MAX_DATA_BYTES) {
+        writeln!(out, "{path} too large to read").unwrap();
+        return;
+    }
+    writeln!(
+        out,
+        "{path} all {}",
+        value(&ds.read_selection(&Selection::All))
+    )
+    .unwrap();
+    if matches!(
+        dtype,
+        DType::F32
+            | DType::F64
+            | DType::I8
+            | DType::I16
+            | DType::I32
+            | DType::I64
+            | DType::U8
+            | DType::U16
+            | DType::U32
+            | DType::U64
+    ) {
+        writeln!(out, "{path} f64 {}", value(&ds.read_f64())).unwrap();
+        if let Some(&d0) = shape.first() {
+            let rank = shape.len();
+            let sel = Selection::Hyperslab {
+                start: std::iter::once(d0 / 3)
+                    .chain(std::iter::repeat_n(0, rank - 1))
+                    .collect(),
+                stride: vec![1; rank],
+                count: std::iter::once(d0.div_ceil(3))
+                    .chain(shape[1..].iter().copied())
+                    .collect(),
+                block: vec![1; rank],
+            };
+            writeln!(
+                out,
+                "{path} f64 third {}",
+                value(&ds.read_f64_selection(&sel))
+            )
+            .unwrap();
+        }
+    }
+    match &raw_dt {
+        clawhdf5_format::datatype::Datatype::String { .. }
+        | clawhdf5_format::datatype::Datatype::VariableLength {
+            is_string: true, ..
+        } => {
+            writeln!(out, "{path} strings {}", value(&ds.read_string_bytes())).unwrap();
+        }
+        clawhdf5_format::datatype::Datatype::VariableLength { .. } => {
+            writeln!(out, "{path} vlen {}", value(&ds.read_vlen::<f64>())).unwrap();
+        }
+        _ => {}
+    }
+}
+
+/// List the file as a tree view does — every group's entries, every
+/// dataset's shape and type — and return the largest dataset whose data
+/// is at most `MAX_DATA_BYTES` (address, bytes), the one a viewer would
+/// plot.
+pub fn list(file: &File) -> Option<(u64, u64)> {
+    let mut seen = HashSet::new();
+    let mut largest: Option<(u64, u64)> = None;
+    let mut queue = VecDeque::from([file.superblock().root_group_address]);
+    while let Some(addr) = queue.pop_front() {
+        if seen.len() >= MAX_OBJECTS || !seen.insert(addr) {
+            continue;
+        }
+        let group = file.group_at(addr);
+        if let Ok(ds) = file.dataset_at(addr) {
+            let _ = (ds.shape(), ds.dtype());
+            let bytes = ds.shape().ok().and_then(|s| {
+                let n = s.iter().try_fold(1u64, |a, &d| a.checked_mul(d))?;
+                let size = u64::from(ds.raw_datatype().ok()?.type_size());
+                n.checked_mul(size).filter(|&b| b <= MAX_DATA_BYTES)
+            });
+            if let Some(b) = bytes
+                && largest.is_none_or(|(_, l)| b > l)
+            {
+                largest = Some((addr, b));
+            }
+        }
+        if let Ok(entries) = group.entries() {
+            queue.extend(entries.into_iter().map(|(_, a)| a));
+        }
+    }
+    largest
+}
+
+/// Read the dataset at `addr` whole.
+pub fn read_one(file: &File, addr: u64) {
+    if let Ok(ds) = file.dataset_at(addr) {
+        let _ = ds.read_selection(&Selection::All);
+    }
+}
+
+/// [`list`], then [`read_one`] of the dataset it picks.
+pub fn list_and_read_one(file: &File) {
+    if let Some((addr, _)) = list(file) {
+        read_one(file, addr);
+    }
+}
+
+/// External virtual-dataset sources read from `dir`, as `File::open` finds
+/// them next to the file.
+pub fn sibling_resolver(dir: PathBuf) -> clawhdf5::VdsResolver {
+    Arc::new(move |name: &str| {
+        let p = Path::new(name);
+        if name.is_empty()
+            || !p
+                .components()
+                .all(|c| matches!(c, std::path::Component::Normal(_)))
+        {
+            return Err(FormatError::Storage(format!("{name:?} not followed")));
+        }
+        match std::fs::read(dir.join(p)) {
+            Ok(bytes) => Ok(Some(bytes)),
+            Err(e) if e.kind() == std::io::ErrorKind::NotFound => Ok(None),
+            Err(e) => Err(FormatError::Storage(e.to_string())),
+        }
+    })
+}
+
+/// HDF5 files under `dir`, recursively.
+pub fn hdf5_files(dir: &Path, out: &mut Vec<PathBuf>) {
+    let Ok(entries) = std::fs::read_dir(dir) else {
+        return;
+    };
+    for e in entries.flatten() {
+        let p = e.path();
+        if p.is_dir() {
+            hdf5_files(&p, out);
+        } else if p
+            .extension()
+            .and_then(|x| x.to_str())
+            .is_some_and(|x| matches!(x, "h5" | "hdf5" | "he5" | "nc" | "h5ad" | "hdf"))
+        {
+            out.push(p);
+        }
+    }
+}
+
+/// The repository's HDF5 test fixtures.
+pub fn fixtures() -> Vec<PathBuf> {
+    let root = Path::new(env!("CARGO_MANIFEST_DIR"));
+    let mut files = Vec::new();
+    hdf5_files(&root.join("../clawhdf5/tests/fixtures"), &mut files);
+    hdf5_files(&root.join("../clawhdf5-format/tests/fixtures"), &mut files);
+    files.sort();
+    files
+}
+
+/// Files of `CLAWHDF5_REMOTE_CORPUS` (directories separated like `PATH`).
+pub fn corpus() -> Option<Vec<PathBuf>> {
+    let dirs = std::env::var("CLAWHDF5_REMOTE_CORPUS").ok()?;
+    let mut files = Vec::new();
+    for d in std::env::split_paths(&dirs) {
+        hdf5_files(&d, &mut files);
+    }
+    files.sort();
+    Some(files)
+}
+
+pub fn python() -> String {
+    std::env::var("CLAWHDF5_PYTHON").unwrap_or_else(|_| "python3".to_string())
+}
+
+pub fn interop_required() -> bool {
+    std::env::var("CLAWHDF5_REQUIRE_INTEROP").is_ok_and(|v| v == "1")
+}
+
+/// Whether python3 with h5py and numpy runs; panics when interop is
+/// required and it does not.
+pub fn have_h5py() -> bool {
+    let ok = Command::new(python())
+        .args(["-c", "import h5py, numpy"])
+        .output()
+        .map(|o| o.status.success())
+        .unwrap_or(false);
+    assert!(
+        ok || !interop_required(),
+        "CLAWHDF5_REQUIRE_INTEROP=1 but python3 with h5py is not available"
+    );
+    if !ok {
+        eprintln!("SKIP: python3 with h5py not available");
+    }
+    ok
+}
+
+/// Run a Python script; its stdout.
+pub fn run_python(script: &str, args: &[&str]) -> String {
+    let out = Command::new(python())
+        .arg("-c")
+        .arg(script)
+        .args(args)
+        .output()
+        .expect("failed to run python");
+    assert!(
+        out.status.success(),
+        "python failed:\n{}",
+        String::from_utf8_lossy(&out.stderr)
+    );
+    String::from_utf8(out.stdout).unwrap()
+}
+
+/// A clawhdf5-written file with a multi-block chunked dataset (`/big`,
+/// 1 000 000 f64 in chunks of 10 000, deflated), a contiguous one and a
+/// group — several blocks of 1 MiB, with no Python needed.
+pub fn multi_block_file() -> Vec<u8> {
+    let mut b = clawhdf5::FileBuilder::new();
+    b.set_attr("title", clawhdf5::AttrValue::String("remote test".into()));
+    let big: Vec<f64> = (0..1_000_000u64)
+        .map(|i| ((i * 2_654_435_761) % 1_000_003) as f64 * 0.5)
+        .collect();
+    b.create_dataset("big")
+        .with_f64_data(&big)
+        .with_chunks(&[10_000])
+        .with_deflate(1);
+    let flat: Vec<f64> = (0..300_000u64).map(|i| i as f64).collect();
+    b.create_dataset("flat").with_f64_data(&flat);
+    let mut g = b.create_group("grp");
+    g.create_dataset("small").with_f64_data(&[1.0, 2.0, 3.0]);
+    b.add_group(g.finish());
+    b.finish().unwrap()
+}

crates/clawhdf5-remote/tests/common/server.rs

@@ -0,0 +1,438 @@
+//! A small HTTP/1.1 file server on 127.0.0.1 for tests and the example:
+//! `Range: bytes=a-b` requests (206, 416), `HEAD`, keep-alive, strong ETags
+//! and Last-Modified with `If-Match` / `If-Unmodified-Since` (412), and
+//! switches to misbehave — ignore ranges (200 with the whole file), cut
+//! bodies short, answer 503, respond slowly, send no validators. It counts
+//! requests and body bytes, and logs every range asked for — only for the
+//! paths it serves: a request for any other path (a local port scanner's
+//! `GET /`, say) is answered 404 and not counted, so request budgets in
+//! tests stay exact.
+
+#![allow(dead_code)]
+
+use std::collections::HashMap;
+use std::io::{BufRead, BufReader, Write};
+use std::net::{TcpListener, TcpStream};
+use std::sync::atomic::{AtomicBool, AtomicU32, AtomicU64, Ordering};
+use std::sync::{Arc, Mutex, RwLock};
+use std::time::Duration;
+
+/// A logged GET: the path and the range asked for (`None`: whole file).
+pub type LogEntry = (String, Option<(u64, u64)>);
+
+struct Resource {
+    data: Arc<Vec<u8>>,
+    etag: String,
+    last_modified: String,
+}
+
+/// Switches and counters shared with the connection threads.
+#[derive(Default)]
+pub struct Shared {
+    files: RwLock<HashMap<String, Resource>>,
+    version: AtomicU64,
+    /// Answer every request with 200 and the whole file.
+    pub ignore_range: AtomicBool,
+    /// Send no ETag and no Last-Modified.
+    pub no_validators: AtomicBool,
+    /// Send a weak ETag only (no Last-Modified).
+    pub weak_etag: AtomicBool,
+    /// Send Last-Modified but no ETag.
+    pub no_etag: AtomicBool,
+    /// Label bodies `Content-Encoding: gzip` (they are not).
+    pub gzip_label: AtomicBool,
+    /// Cut the body of the next N ranged responses in half (then close the
+    /// connection).
+    pub truncate_next: AtomicU32,
+    /// Answer the next N requests with 503.
+    pub fail_next: AtomicU32,
+    /// Sleep this long before answering each request.
+    pub delay_ms: AtomicU64,
+    /// When non-zero, claim the file is this long (in `Content-Range` and
+    /// when checking ranges) and serve zeros past its real end: a hostile
+    /// server lying about the length.
+    pub fake_total: AtomicU64,
+    /// When non-zero, answer every request for a served file with this
+    /// status (and an empty body).
+    pub force_status: AtomicU32,
+    /// When non-zero, send bodies at about this many bytes per second.
+    pub throttle_bps: AtomicU64,
+    /// When non-zero, stop this many milliseconds after the headers and
+    /// half the body (a stalled connection).
+    pub stall_ms: AtomicU64,
+    /// Answer ranges with a `Content-Range` one byte off.
+    pub wrong_range: AtomicBool,
+    /// Path → `Location`: answered `302 Found` (counted as a request).
+    pub redirects: RwLock<HashMap<String, String>>,
+    /// (path, headers) of every counted request, header names lowercase.
+    pub seen: Mutex<Vec<(String, HashMap<String, String>)>>,
+    /// Requests for a served path (every status); requests for other
+    /// paths are not counted.
+    pub requests: AtomicU64,
+    /// Body bytes sent.
+    pub bytes: AtomicU64,
+    /// (path, range) of every GET.
+    pub log: Mutex<Vec<LogEntry>>,
+    stop: AtomicBool,
+}
+
+/// A running server; stops accepting when dropped.
+pub struct Server {
+    pub addr: std::net::SocketAddr,
+    pub shared: Arc<Shared>,
+}
+
+impl Server {
+    /// Serve `files` (URL path such as `/a.h5` → bytes) on `127.0.0.1`, on
+    /// a free port.
+    pub fn start(files: Vec<(String, Vec<u8>)>) -> Server {
+        Server::bind("127.0.0.1:0", files)
+    }
+
+    /// Serve `files` on `addr`.
+    pub fn bind(addr: &str, files: Vec<(String, Vec<u8>)>) -> Server {
+        let listener = TcpListener::bind(addr).expect("bind");
+        let addr = listener.local_addr().unwrap();
+        let shared = Arc::new(Shared::default());
+        for (path, data) in files {
+            shared.put(&path, data);
+        }
+        let s = shared.clone();
+        std::thread::spawn(move || {
+            for conn in listener.incoming() {
+                if s.stop.load(Ordering::SeqCst) {
+                    break;
+                }
+                let Ok(conn) = conn else { continue };
+                let s = s.clone();
+                std::thread::spawn(move || {
+                    let _ = serve(conn, &s);
+                });
+            }
+        });
+        Server { addr, shared }
+    }
+
+    /// The URL of `path` (which starts with `/`).
+    pub fn url(&self, path: &str) -> String {
+        format!("http://{}{path}", self.addr)
+    }
+
+    /// Requests served so far.
+    pub fn requests(&self) -> u64 {
+        self.shared.requests.load(Ordering::SeqCst)
+    }
+
+    /// Body bytes sent so far.
+    pub fn bytes(&self) -> u64 {
+        self.shared.bytes.load(Ordering::SeqCst)
+    }
+
+    /// Zero the counters and the logs.
+    pub fn reset(&self) {
+        self.shared.requests.store(0, Ordering::SeqCst);
+        self.shared.bytes.store(0, Ordering::SeqCst);
+        self.shared.log.lock().unwrap().clear();
+        self.shared.seen.lock().unwrap().clear();
+    }
+
+    /// Answer `path` with a redirect to `location`.
+    pub fn redirect(&self, path: &str, location: &str) {
+        self.shared
+            .redirects
+            .write()
+            .unwrap()
+            .insert(path.to_string(), location.to_string());
+    }
+
+    /// Whether any request the server counted carried header `name`.
+    pub fn saw_header(&self, name: &str) -> bool {
+        self.shared
+            .seen
+            .lock()
+            .unwrap()
+            .iter()
+            .any(|(_, h)| h.contains_key(name))
+    }
+
+    /// The ranges asked for so far.
+    pub fn log(&self) -> Vec<LogEntry> {
+        self.shared.log.lock().unwrap().clone()
+    }
+}
+
+impl Drop for Server {
+    fn drop(&mut self) {
+        self.shared.stop.store(true, Ordering::SeqCst);
+        let _ = TcpStream::connect(self.addr);
+    }
+}
+
+impl Shared {
+    /// Add or replace a file (a replacement gets a new ETag and
+    /// Last-Modified).
+    pub fn put(&self, path: &str, data: Vec<u8>) {
+        let v = self.version.fetch_add(1, Ordering::SeqCst) + 1;
+        let secs = 1_700_000_000 + v;
+        self.files.write().unwrap().insert(
+            path.to_string(),
+            Resource {
+                data: Arc::new(data),
+                etag: format!("\"v{v}-{path}\""),
+                last_modified: http_date(secs),
+            },
+        );
+    }
+}
+
+/// An IMF-fixdate for `secs` since the epoch (enough for distinct values).
+fn http_date(secs: u64) -> String {
+    const DAYS: [&str; 7] = ["Thu", "Fri", "Sat", "Sun", "Mon", "Tue", "Wed"];
+    const MONTHS: [&str; 12] = [
+        "Jan", "Feb", "Mar", "Apr", "May", "Jun", "Jul", "Aug", "Sep", "Oct", "Nov", "Dec",
+    ];
+    let days = secs / 86_400;
+    let rem = secs % 86_400;
+    // Civil-from-days (Howard Hinnant).
+    let z = days as i64 + 719_468;
+    let era = z.div_euclid(146_097);
+    let doe = z - era * 146_097;
+    let yoe = (doe - doe / 1460 + doe / 36_524 - doe / 146_096) / 365;
+    let y = yoe + era * 400;
+    let doy = doe - (365 * yoe + yoe / 4 - yoe / 100);
+    let mp = (5 * doy + 2) / 153;
+    let d = doy - (153 * mp + 2) / 5 + 1;
+    let m = if mp < 10 { mp + 3 } else { mp - 9 };
+    let y = if m <= 2 { y + 1 } else { y };
+    format!(
+        "{}, {:02} {} {} {:02}:{:02}:{:02} GMT",
+        DAYS[(days % 7) as usize],
+        d,
+        MONTHS[(m - 1) as usize],
+        y,
+        rem / 3600,
+        rem % 3600 / 60,
+        rem % 60
+    )
+}
+
+fn parse_range(v: &str, len: u64) -> Option<Result<(u64, u64), ()>> {
+    let spec = v.trim().strip_prefix("bytes=")?;
+    if spec.contains(',') {
+        return None; // multiple ranges: serve the whole file
+    }
+    let (a, b) = spec.split_once('-')?;
+    let a: u64 = a.trim().parse().ok()?;
+    let b: u64 = match b.trim() {
+        "" => u64::MAX,
+        b => b.parse().ok()?,
+    };
+    if b < a {
+        return None;
+    }
+    if a >= len {
+        return Some(Err(()));
+    }
+    Some(Ok((a, b.min(len - 1))))
+}
+
+fn serve(conn: TcpStream, s: &Shared) -> std::io::Result<()> {
+    conn.set_read_timeout(Some(Duration::from_secs(30)))?;
+    // One write per response and no Nagle delay: otherwise every request
+    // waits for a delayed ACK (~40 ms).
+    conn.set_nodelay(true)?;
+    let mut reader = BufReader::new(conn.try_clone()?);
+    let mut out = conn;
+    loop {
+        let mut line = String::new();
+        if reader.read_line(&mut line)? == 0 {
+            return Ok(());
+        }
+        let mut parts = line.split_whitespace();
+        let method = parts.next().unwrap_or("").to_string();
+        let path = parts.next().unwrap_or("").to_string();
+        let mut headers = HashMap::new();
+        loop {
+            let mut h = String::new();
+            if reader.read_line(&mut h)? == 0 {
+                return Ok(());
+            }
+            let h = h.trim_end();
+            if h.is_empty() {
+                break;
+            }
+            if let Some((k, v)) = h.split_once(':') {
+                headers.insert(k.trim().to_ascii_lowercase(), v.trim().to_string());
+            }
+        }
+        // The query string (a presigned URL's signature, say) is not part
+        // of the file's name.
+        let path = path.split('?').next().unwrap_or("").to_string();
+        let close = headers
+            .get("connection")
+            .is_some_and(|v| v.eq_ignore_ascii_case("close"));
+        let redirect = s.redirects.read().unwrap().get(&path).cloned();
+        if let Some(location) = redirect {
+            s.requests.fetch_add(1, Ordering::SeqCst);
+            s.seen.lock().unwrap().push((path.clone(), headers.clone()));
+            write!(
+                out,
+                "HTTP/1.1 302 Found\r\nLocation: {location}\r\nContent-Length: 0\r\n\r\n"
+            )?;
+            continue;
+        }
+        let res = {
+            let files = s.files.read().unwrap();
+            files
+                .get(&path)
+                .map(|r| (r.data.clone(), r.etag.clone(), r.last_modified.clone()))
+        };
+        let Some((data, etag, lm)) = res else {
+            // Not ours: not counted, not delayed, no failure injected.
+            write!(out, "HTTP/1.1 404 Not Found\r\nContent-Length: 0\r\n\r\n")?;
+            if close {
+                return Ok(());
+            }
+            continue;
+        };
+        s.requests.fetch_add(1, Ordering::SeqCst);
+        s.seen.lock().unwrap().push((path.clone(), headers.clone()));
+        let delay = s.delay_ms.load(Ordering::SeqCst);
+        if delay > 0 {
+            std::thread::sleep(Duration::from_millis(delay));
+        }
+        let forced = s.force_status.load(Ordering::SeqCst);
+        if forced != 0 {
+            write!(out, "HTTP/1.1 {forced} Forced\r\nContent-Length: 0\r\n\r\n")?;
+            continue;
+        }
+        if s.fail_next
+            .fetch_update(Ordering::SeqCst, Ordering::SeqCst, |n| n.checked_sub(1))
+            .is_ok()
+        {
+            write!(
+                out,
+                "HTTP/1.1 503 Service Unavailable\r\nContent-Length: 0\r\n\r\n"
+            )?;
+            continue;
+        }
+        let fake = s.fake_total.load(Ordering::SeqCst);
+        let len = if fake > 0 { fake } else { data.len() as u64 };
+        let mut validators = String::new();
+        if s.weak_etag.load(Ordering::SeqCst) {
+            validators = format!("ETag: W/{etag}\r\n");
+        } else if s.no_etag.load(Ordering::SeqCst) {
+            validators = format!("Last-Modified: {lm}\r\n");
+        } else if !s.no_validators.load(Ordering::SeqCst) {
+            validators = format!("ETag: {etag}\r\nLast-Modified: {lm}\r\n");
+        }
+        let precondition_failed = headers.get("if-match").is_some_and(|v| v != &etag)
+            || headers.get("if-unmodified-since").is_some_and(|v| v != &lm);
+        if precondition_failed {
+            write!(
+                out,
+                "HTTP/1.1 412 Precondition Failed\r\n{validators}Content-Length: 0\r\n\r\n"
+            )?;
+            continue;
+        }
+        let range = if s.ignore_range.load(Ordering::SeqCst) {
+            None
+        } else {
+            headers.get("range").and_then(|v| parse_range(v, len))
+        };
+        if method == "GET" {
+            s.log
+                .lock()
+                .unwrap()
+                .push((path.clone(), range.and_then(Result::ok)));
+        }
+        let mut padded = Vec::new();
+        let (status, body, extra) = match range {
+            Some(Err(())) => {
+                write!(
+                    out,
+                    "HTTP/1.1 416 Range Not Satisfiable\r\nContent-Range: bytes */{len}\r\n\
+                     {validators}Content-Length: 0\r\n\r\n"
+                )?;
+                continue;
+            }
+            Some(Ok((a, b))) => (
+                "206 Partial Content",
+                slice_or_zeros(&data, a, b, &mut padded),
+                if s.wrong_range.load(Ordering::SeqCst) {
+                    format!("Content-Range: bytes {}-{}/{len}\r\n", a + 1, b + 1)
+                } else {
+                    format!("Content-Range: bytes {a}-{b}/{len}\r\n")
+                },
+            ),
+            None => ("200 OK", &data[..], String::new()),
+        };
+        let extra = if s.gzip_label.load(Ordering::SeqCst) {
+            format!("{extra}Content-Encoding: gzip\r\n")
+        } else {
+            extra
+        };
+        let head = format!(
+            "HTTP/1.1 {status}\r\nContent-Length: {}\r\nAccept-Ranges: bytes\r\n{extra}{validators}\r\n",
+            body.len()
+        );
+        if method == "HEAD" {
+            out.write_all(head.as_bytes())?;
+            continue;
+        }
+        let truncate = status.starts_with("206")
+            && s.truncate_next
+                .fetch_update(Ordering::SeqCst, Ordering::SeqCst, |n| n.checked_sub(1))
+                .is_ok();
+        let body = if truncate {
+            &body[..body.len() / 2]
+        } else {
+            body
+        };
+        // Counted before the client can have the bytes, so a test that
+        // resets the counters after a read never sees them arrive late.
+        s.bytes.fetch_add(body.len() as u64, Ordering::SeqCst);
+        let bps = s.throttle_bps.load(Ordering::SeqCst);
+        let stall = s.stall_ms.load(Ordering::SeqCst);
+        if bps > 0 || stall > 0 {
+            out.write_all(head.as_bytes())?;
+            let (first, rest) = body.split_at(if stall > 0 { body.len() / 2 } else { 0 });
+            out.write_all(first)?;
+            out.flush()?;
+            if stall > 0 {
+                std::thread::sleep(Duration::from_millis(stall));
+            }
+            for piece in rest.chunks(4096) {
+                out.write_all(piece)?;
+                out.flush()?;
+                if let Some(us) = (4096 * 1_000_000u64).checked_div(bps) {
+                    std::thread::sleep(Duration::from_micros(us));
+                }
+            }
+        } else {
+            let mut response = head.into_bytes();
+            response.extend_from_slice(body);
+            out.write_all(&response)?;
+            out.flush()?;
+        }
+        if truncate || close {
+            let _ = out.shutdown(std::net::Shutdown::Both);
+            return Ok(());
+        }
+    }
+}
+
+/// `data[a..=b]`, padded with zeros past its end (into `padded`).
+fn slice_or_zeros<'a>(data: &'a [u8], a: u64, b: u64, padded: &'a mut Vec<u8>) -> &'a [u8] {
+    let real = data.len() as u64;
+    if b < real {
+        return &data[a as usize..=b as usize];
+    }
+    let n = usize::try_from(b - a + 1).expect("range fits in memory");
+    padded.resize(n, 0);
+    if a < real {
+        let have = (real - a) as usize;
+        padded[..have].copy_from_slice(&data[a as usize..]);
+    }
+    padded
+}

crates/clawhdf5-remote/tests/http.rs

@@ -0,0 +1,860 @@
+//! HTTP range reads against a local server (no internet): every value read
+//! over HTTP equals `File::open`'s, and the server's misbehaviour — no
+//! range support, a file replaced mid-read, cut-off bodies, errors, slow
+//! answers under concurrent readers — is an error or the right data, never
+//! wrong data.
+//!
+//! `CLAWHDF5_REMOTE_CORPUS=dir[:dir...]` adds every HDF5 file under those
+//! directories (the conformance corpus is `conformance/.cache/corpus`), and
+//! `CLAWHDF5_REMOTE_REPORT=1` prints the per-file request counts.
+
+#![cfg(feature = "http")]
+
+mod common;
+
+use std::sync::Arc;
+use std::sync::atomic::Ordering;
+use std::time::Duration;
+
+use clawhdf5::File;
+use clawhdf5_remote::{
+    CacheConfig, Error, HttpOptions, HttpStorage, Options, RemoteError, open_url, open_url_with,
+    storage_for_url,
+};
+use common::server::Server;
+use common::{list_and_read_one, multi_block_file, transcript};
+
+/// Options for tests: fast retries.
+fn quick() -> Options {
+    Options {
+        http: HttpOptions {
+            backoff: Duration::from_millis(5),
+            ..HttpOptions::default()
+        },
+        ..Options::default()
+    }
+}
+
+fn changed(e: &str) -> bool {
+    e.contains("changed while open")
+}
+
+/// Serve `files`, open each by URL and through `File::open`, and require
+/// identical transcripts. Returns (files compared, files both refused).
+fn compare(files: &[std::path::PathBuf], report: bool) -> (usize, usize) {
+    let served: Vec<(String, Vec<u8>)> = files
+        .iter()
+        .enumerate()
+        .filter_map(|(i, p)| {
+            let bytes = std::fs::read(p).ok()?;
+            let name = p.file_name()?.to_str()?.replace(' ', "_");
+            Some((format!("/f{i}/{name}"), bytes))
+        })
+        .collect();
+    let server = Server::start(served.clone());
+    let (mut same, mut refused) = (0, 0);
+    let mut totals = [0u64; 7];
+    for (i, p) in files.iter().enumerate() {
+        let Some((url_path, bytes)) = served
+            .iter()
+            .find(|(u, _)| u.starts_with(&format!("/f{i}/")))
+        else {
+            continue;
+        };
+        let url = server.url(url_path);
+        let local = File::open(p);
+        let remote = storage_for_url(&url, &quick())
+            .map_err(|e| e.to_string())
+            .and_then(|s| {
+                File::open_storage(s.clone())
+                    .map(|mut f| {
+                        f.set_vds_resolver(common::sibling_resolver(p.parent().unwrap().into()));
+                        (f, s)
+                    })
+                    .map_err(|e| e.to_string())
+            });
+        let (local, (remote, storage)) = match (local, remote) {
+            (Ok(l), Ok(r)) => (l, r),
+            (Err(l), Err(r)) => {
+                assert!(
+                    !r.contains("HTTP") && !r.contains("network"),
+                    "{url}: {r} ({l})"
+                );
+                refused += 1;
+                continue;
+            }
+            (l, r) => panic!(
+                "{}: File::open {:?}, open_url {:?}",
+                p.display(),
+                l.map(|_| ()),
+                r.map(|_| ())
+            ),
+        };
+        let want = transcript(&local);
+        let got = transcript(&remote);
+        if want != got {
+            let first = want
+                .lines()
+                .zip(got.lines())
+                .find(|(w, g)| w != g)
+                .map(|(w, g)| format!("\n local:  {w}\n remote: {g}"))
+                .unwrap_or_default();
+            panic!("{}: open_url differs from File::open{first}", p.display());
+        }
+        assert!(storage.stats().reads > 0, "read through the cache");
+        same += 1;
+
+        // Cost of "open + list" (a tree view) and of then reading the
+        // largest dataset (a plot): with the block cache, and with none
+        // (every read a request). Requests and bytes as the server saw
+        // them, the open included.
+        server.reset();
+        let with = storage_for_url(&url, &quick()).unwrap();
+        let f = File::open_storage(with.clone()).unwrap();
+        let pick = common::list(&f);
+        let (list_requests, list_bytes) = (server.requests(), server.bytes());
+        if let Some((addr, _)) = pick {
+            common::read_one(&f, addr);
+        }
+        let (cached_requests, cached_bytes) = (server.requests(), server.bytes());
+        server.reset();
+        let (bare, _) = HttpStorage::open(&url, quick().http).unwrap();
+        if let Ok(f) = File::open_storage(Arc::new(bare)) {
+            list_and_read_one(&f);
+        }
+        let uncached_requests = server.requests();
+        for (t, v) in totals.iter_mut().zip([
+            list_requests,
+            list_bytes,
+            cached_requests,
+            cached_bytes,
+            uncached_requests,
+            bytes.len() as u64,
+            1,
+        ]) {
+            *t += v;
+        }
+        if report {
+            eprintln!(
+                "open+list {list_requests:>4} req {list_bytes:>10} B | +read {:>5} req \
+                 {cached_bytes:>10} B | uncached {uncached_requests:>7} req | file {:>10} B  {}",
+                cached_requests,
+                bytes.len(),
+                p.display()
+            );
+        }
+        // Files within one block: opening fetched everything.
+        if bytes.len() as u64 <= with.config().block_size {
+            assert_eq!(cached_requests, 1, "{}", p.display());
+        }
+        // The budget of docs/design/range-reads.md (Testing): listing the
+        // IMERG file (file A of section 2) takes at most 3 requests.
+        if p.ends_with("xarray-data/imerghh_730.hdf5") {
+            assert!(list_requests <= 3, "{}: {list_requests}", p.display());
+        }
+    }
+    eprintln!(
+        "{} files ({} bytes): open + list {} requests, {} bytes; + read the largest \
+         dataset {} requests, {} bytes (1 MiB block cache); without a cache {} requests",
+        totals[6], totals[5], totals[0], totals[1], totals[2], totals[3], totals[4]
+    );
+    (same, refused)
+}
+
+#[test]
+fn fixtures_read_identically_over_http() {
+    let files = common::fixtures();
+    assert!(files.len() >= 45, "{} fixtures", files.len());
+    let report = std::env::var("CLAWHDF5_REMOTE_REPORT").is_ok_and(|v| v == "1");
+    let (same, _) = compare(&files, report);
+    assert!(same >= 40, "{same}");
+}
+
+#[test]
+fn corpus_reads_identically_over_http() {
+    let Some(files) = common::corpus() else {
+        eprintln!("CLAWHDF5_REMOTE_CORPUS not set; skipping the corpus");
+        return;
+    };
+    let report = std::env::var("CLAWHDF5_REMOTE_REPORT").is_ok_and(|v| v == "1");
+    let (same, refused) = compare(&files, report);
+    eprintln!("corpus: {same} files identical, {refused} refused by both");
+    assert!(same > 0);
+}
+
+#[test]
+fn multi_block_file_values_and_request_budget() {
+    let bytes = multi_block_file();
+    assert!(bytes.len() > 3 << 20, "{}", bytes.len());
+    let server = Server::start(vec![("/m.h5".into(), bytes.clone())]);
+    let url = server.url("/m.h5");
+    let local = File::from_bytes(bytes.clone()).unwrap();
+    let storage = storage_for_url(&url, &quick()).unwrap();
+    assert_eq!(server.requests(), 1, "opening is one request");
+    let remote = File::open_storage(storage.clone()).unwrap();
+    assert!(remote.contiguous_bytes().is_none());
+    let mut names = remote.root().datasets().unwrap();
+    names.sort();
+    assert_eq!(names, ["big", "flat"]);
+    assert_eq!(remote.root().groups().unwrap(), ["grp"]);
+    assert_eq!(
+        remote.dataset("grp/small").unwrap().read_f64().unwrap(),
+        [1.0, 2.0, 3.0]
+    );
+    let big = remote.dataset("big").unwrap().read_f64().unwrap();
+    assert_eq!(big, local.dataset("big").unwrap().read_f64().unwrap());
+    let flat = remote.dataset("flat").unwrap().read_f64().unwrap();
+    assert_eq!(flat.len(), 300_000);
+    assert_eq!(flat[299_999], 299_999.0);
+    // Every byte was fetched at most once, in whole 1 MiB blocks.
+    let log = server.log();
+    let mut blocks: Vec<u64> = Vec::new();
+    for (_, r) in &log {
+        let (a, b) = r.expect("every request is ranged");
+        assert_eq!(a % (1 << 20), 0, "block-aligned");
+        blocks.extend(a >> 20..=b >> 20);
+    }
+    let n = blocks.len();
+    blocks.sort_unstable();
+    blocks.dedup();
+    assert_eq!(n, blocks.len(), "a block was fetched twice: {log:?}");
+    assert!(
+        server.requests() <= (bytes.len() as u64 >> 20) + 2,
+        "{} requests for a {} byte file",
+        server.requests(),
+        bytes.len()
+    );
+    // Reading again costs nothing.
+    let before = server.requests();
+    assert_eq!(transcript(&remote), transcript(&local));
+    assert_eq!(server.requests(), before);
+}
+
+#[test]
+fn h5py_written_file_over_http() {
+    if !common::have_h5py() {
+        return;
+    }
+    let dir = tempfile::tempdir().unwrap();
+    let path = dir.path().join("py.h5");
+    common::run_python(
+        r#"
+import sys, h5py, numpy as np
+with h5py.File(sys.argv[1], "w", libver="latest") as f:
+    f.attrs["title"] = "h5py remote test"
+    g = f.create_group("sensors")
+    for i in range(40):
+        d = g.create_dataset(f"s{i:02d}", data=np.arange(1000, dtype="<i4") * (i + 1))
+        d.attrs["index"] = i
+    t = f.create_dataset("temps", data=np.sin(np.arange(2_000_000) / 1000.0),
+                         chunks=(50_000,), compression="gzip", shuffle=True)
+    t.attrs["units"] = "K"
+    f.create_dataset("names", data=[b"alpha", b"beta", b"gamma"])
+    f.create_dataset("vl", data=["one", "two", "three"], dtype=h5py.string_dtype())
+    f["link"] = h5py.SoftLink("/sensors/s03")
+"#,
+        &[path.to_str().unwrap()],
+    );
+    // What libhdf5 reads, for the values the test checks.
+    let sums = common::run_python(
+        r#"
+import sys, h5py, numpy as np
+with h5py.File(sys.argv[1], "r") as f:
+    print(repr(float(f["temps"][:].sum())), int(f["sensors/s39"][:].sum()), f["vl"].asstr()[1])
+"#,
+        &[path.to_str().unwrap()],
+    );
+    let bytes = std::fs::read(&path).unwrap();
+    let server = Server::start(vec![("/py.h5".into(), bytes.clone())]);
+    let remote = open_url_with(&server.url("/py.h5"), &quick()).unwrap();
+    let local = File::open(&path).unwrap();
+    assert_eq!(transcript(&remote), transcript(&local));
+    let temps = remote.dataset("temps").unwrap().read_f64().unwrap();
+    let s39: i64 = remote
+        .dataset("sensors/s39")
+        .unwrap()
+        .read_i64()
+        .unwrap()
+        .iter()
+        .sum();
+    let vl = remote.dataset("vl").unwrap().read_string().unwrap();
+    let mut it = sums.split_whitespace();
+    let want_sum: f64 = it.next().unwrap().parse().unwrap();
+    let got_sum: f64 = temps.iter().sum();
+    assert!((got_sum - want_sum).abs() <= 1e-6 * want_sum.abs().max(1.0));
+    assert_eq!(s39, it.next().unwrap().parse::<i64>().unwrap());
+    assert_eq!(vl[1], it.next().unwrap());
+    assert_eq!(
+        remote.dataset("link").unwrap().read_i32().unwrap(),
+        local.dataset("sensors/s03").unwrap().read_i32().unwrap()
+    );
+}
+
+#[test]
+fn a_server_that_ignores_range_is_refused_or_downloaded_when_allowed() {
+    let bytes = multi_block_file();
+    let server = Server::start(vec![("/m.h5".into(), bytes.clone())]);
+    server.shared.ignore_range.store(true, Ordering::SeqCst);
+    let url = server.url("/m.h5");
+    let err = open_url_with(&url, &quick()).unwrap_err();
+    assert!(
+        matches!(err, Error::Remote(RemoteError::RangeNotSupported(_))),
+        "{err}"
+    );
+    assert_eq!(server.requests(), 1, "refused at the first response");
+    let mut opts = quick();
+    opts.http.allow_full_download = true;
+    let storage = storage_for_url(&url, &opts).unwrap();
+    let f = File::open_storage(storage.clone()).unwrap();
+    assert_eq!(f.contiguous_bytes(), Some(&bytes[..]), "read from memory");
+    assert_eq!(server.requests(), 2);
+    let local = File::from_bytes(bytes).unwrap();
+    assert_eq!(transcript(&f), transcript(&local));
+    assert_eq!(server.requests(), 2, "no further requests");
+    // Too large for the download limit.
+    opts.http.max_full_download = 1000;
+    assert!(open_url_with(&url, &opts).is_err());
+}
+
+#[test]
+fn a_file_replaced_mid_read_is_an_error_not_mixed_data() {
+    for mode in ["etag", "last-modified", "none"] {
+        let bytes = multi_block_file();
+        let server = Server::start(vec![("/m.h5".into(), bytes.clone())]);
+        match mode {
+            "last-modified" => server.shared.no_etag.store(true, Ordering::SeqCst),
+            "none" => server.shared.no_validators.store(true, Ordering::SeqCst),
+            _ => {}
+        }
+        let url = server.url("/m.h5");
+        let f = open_url_with(&url, &quick()).unwrap();
+        assert_eq!(f.root().groups().unwrap(), ["grp"], "{mode}");
+        // Replace the file: same layout, other values (and, for "none",
+        // one byte longer, which is all that can be checked).
+        let mut other = bytes.clone();
+        let n = other.len();
+        for b in &mut other[n / 2..n / 2 + 1000] {
+            *b ^= 0xff;
+        }
+        if mode == "none" {
+            other.push(0);
+        }
+        server.shared.put("/m.h5", other);
+        let err = f
+            .dataset("big")
+            .unwrap()
+            .read_f64()
+            .unwrap_err()
+            .to_string();
+        assert!(changed(&err), "{mode}: {err}");
+    }
+}
+
+#[test]
+fn a_weak_etag_or_no_validator_is_refused_when_required() {
+    let server = Server::start(vec![("/m.h5".into(), multi_block_file())]);
+    server.shared.weak_etag.store(true, Ordering::SeqCst);
+    let mut opts = quick();
+    opts.http.require_validator = true;
+    assert!(open_url_with(&server.url("/m.h5"), &opts).is_err());
+    opts.http.require_validator = false;
+    assert!(open_url_with(&server.url("/m.h5"), &opts).is_ok());
+}
+
+#[test]
+fn truncated_bodies_are_retried_then_an_error() {
+    let bytes = multi_block_file();
+    let server = Server::start(vec![("/m.h5".into(), bytes.clone())]);
+    let url = server.url("/m.h5");
+    let local = File::from_bytes(bytes).unwrap();
+    let want = local.dataset("big").unwrap().read_f64().unwrap();
+
+    // One cut-off body: retried, right values.
+    let (http, first) = HttpStorage::open(&url, quick().http).unwrap();
+    assert_eq!(first.len(), 1 << 20);
+    let storage = Arc::new(clawhdf5_remote::BlockCache::new(
+        http,
+        CacheConfig::default(),
+    ));
+    let f = File::open_storage(storage.clone()).unwrap();
+    server.shared.truncate_next.store(1, Ordering::SeqCst);
+    assert_eq!(f.dataset("big").unwrap().read_f64().unwrap(), want);
+    assert_eq!(storage.inner().stats().retries, 1);
+
+    // Every body cut off: an error, never partial data.
+    let f = open_url_with(&url, &quick()).unwrap();
+    server.shared.truncate_next.store(1000, Ordering::SeqCst);
+    let err = f
+        .dataset("big")
+        .unwrap()
+        .read_f64()
+        .unwrap_err()
+        .to_string();
+    assert!(
+        err.contains("network error") || err.contains("bad response"),
+        "{err}"
+    );
+    server.shared.truncate_next.store(0, Ordering::SeqCst);
+    // The failure was not cached: the next read succeeds.
+    assert_eq!(f.dataset("big").unwrap().read_f64().unwrap(), want);
+}
+
+#[test]
+fn server_errors_are_retried_with_backoff() {
+    let bytes = multi_block_file();
+    let server = Server::start(vec![("/m.h5".into(), bytes.clone())]);
+    server.shared.fail_next.store(2, Ordering::SeqCst);
+    let url = server.url("/m.h5");
+    let f = open_url_with(&url, &quick()).unwrap();
+    let local = File::from_bytes(bytes).unwrap();
+    server.shared.fail_next.store(3, Ordering::SeqCst);
+    assert_eq!(
+        f.dataset("flat").unwrap().read_f64().unwrap(),
+        local.dataset("flat").unwrap().read_f64().unwrap()
+    );
+    // More failures than retries: an error.
+    let f = open_url_with(&url, &quick()).unwrap();
+    server.shared.fail_next.store(100, Ordering::SeqCst);
+    let err = f
+        .dataset("big")
+        .unwrap()
+        .read_f64()
+        .unwrap_err()
+        .to_string();
+    assert!(err.contains("503"), "{err}");
+}
+
+#[test]
+fn slow_server_concurrent_readers_fetch_each_block_once() {
+    let bytes = multi_block_file();
+    let server = Server::start(vec![("/m.h5".into(), bytes.clone())]);
+    let url = server.url("/m.h5");
+    let opts = Options {
+        cache: CacheConfig {
+            block_size: 256 << 10,
+            max_request: 256 << 10,
+            ..CacheConfig::default()
+        },
+        ..quick()
+    };
+    let storage = storage_for_url(&url, &opts).unwrap();
+    let file = File::open_storage(storage.clone()).unwrap();
+    let local = File::from_bytes(bytes).unwrap();
+    let want_big = local.dataset("big").unwrap().read_f64().unwrap();
+    let want_flat = local.dataset("flat").unwrap().read_f64().unwrap();
+    server.shared.delay_ms.store(40, Ordering::SeqCst);
+    server.reset();
+    std::thread::scope(|s| {
+        for t in 0..8 {
+            let (file, want_big, want_flat) = (&file, &want_big, &want_flat);
+            s.spawn(move || {
+                if t % 2 == 0 {
+                    assert_eq!(&file.dataset("big").unwrap().read_f64().unwrap(), want_big);
+                } else {
+                    assert_eq!(
+                        &file.dataset("flat").unwrap().read_f64().unwrap(),
+                        want_flat
+                    );
+                }
+            });
+        }
+    });
+    let log = server.log();
+    let mut starts: Vec<u64> = log.iter().map(|(_, r)| r.unwrap().0).collect();
+    let n = starts.len();
+    starts.sort_unstable();
+    starts.dedup();
+    assert_eq!(n, starts.len(), "a block was fetched twice");
+    assert!(storage.stats().waits > 0, "readers shared fetches");
+}
+
+#[test]
+fn bad_urls_and_statuses_are_clean_errors() {
+    let server = Server::start(vec![("/m.h5".into(), multi_block_file())]);
+    let err = open_url_with(&server.url("/missing.h5"), &quick()).unwrap_err();
+    assert!(
+        matches!(err, Error::Remote(RemoteError::Status { code: 404, .. })),
+        "{err}"
+    );
+    assert!(matches!(
+        open_url("ftp://example.com/a.h5"),
+        Err(Error::Remote(RemoteError::UnsupportedScheme(_)))
+    ));
+    assert!(matches!(
+        open_url("no-scheme"),
+        Err(Error::Remote(RemoteError::InvalidUrl(_)))
+    ));
+    #[cfg(not(feature = "s3"))]
+    {
+        let e = open_url("s3://bucket/key.h5").unwrap_err().to_string();
+        assert!(e.contains("`s3` feature"), "{e}");
+    }
+    #[cfg(not(feature = "https"))]
+    {
+        let e = open_url("https://example.com/a.h5")
+            .unwrap_err()
+            .to_string();
+        assert!(e.contains("`https` feature"), "{e}");
+    }
+    // A content-encoded body is not a byte range.
+    let server = Server::start(vec![("/m.h5".into(), multi_block_file())]);
+    server.shared.gzip_label.store(true, Ordering::SeqCst);
+    let e = open_url(&server.url("/m.h5")).unwrap_err().to_string();
+    assert!(e.contains("gzip-encoded"), "{e}");
+    // Not HDF5.
+    let server = Server::start(vec![("/x.h5".into(), vec![7u8; 5000])]);
+    assert!(matches!(
+        open_url(&server.url("/x.h5")),
+        Err(Error::Hdf5(_))
+    ));
+    // A server that closes every connection unanswered: a network error
+    // after the retries. (Not a closed port: another test's server could
+    // take it meanwhile.)
+    let port = {
+        let l = std::net::TcpListener::bind("127.0.0.1:0").unwrap();
+        let port = l.local_addr().unwrap().port();
+        std::thread::spawn(move || {
+            for c in l.incoming() {
+                drop(c);
+            }
+        });
+        port
+    };
+    let err = open_url_with(&format!("http://127.0.0.1:{port}/a.h5"), &quick()).unwrap_err();
+    assert!(
+        matches!(err, Error::Remote(RemoteError::Transport(_))),
+        "{err}"
+    );
+}
+
+/// A request for a path the server does not serve (a local port scanner's
+/// `GET /`) does not count against a test's request budget.
+#[test]
+fn requests_for_other_paths_are_not_counted() {
+    use std::io::{Read, Write};
+    let server = Server::start(vec![("/m.h5".into(), multi_block_file())]);
+    let mut probe = std::net::TcpStream::connect(server.addr).unwrap();
+    probe
+        .write_all(b"GET / HTTP/1.1\r\nHost: x\r\nConnection: close\r\n\r\n")
+        .unwrap();
+    let mut answer = String::new();
+    probe.read_to_string(&mut answer).unwrap();
+    assert!(answer.starts_with("HTTP/1.1 404"), "{answer}");
+    storage_for_url(&server.url("/m.h5"), &quick()).unwrap();
+    assert_eq!(server.requests(), 1, "only the open counts");
+    assert_eq!(server.log().len(), 1);
+}
+
+/// A server claiming a length near `u64::MAX` (and serving zeros past the
+/// real data), with a file whose addresses point at the end of that range:
+/// clean errors or zeros, never an arithmetic overflow (a panic in debug).
+#[test]
+fn a_server_claiming_a_huge_length_does_not_overflow() {
+    let root = std::path::Path::new(env!("CARGO_MANIFEST_DIR"));
+    let mut bytes =
+        std::fs::read(root.join("../clawhdf5-format/tests/fixtures/legacy/h5ex_g_iterate.h5"))
+            .unwrap();
+    // Superblock v0: end-of-file address, then the root symbol table
+    // entry's object header, B-tree and heap addresses.
+    bytes[0x28..0x30].copy_from_slice(&(u64::MAX - 1).to_le_bytes());
+    for at in [0x40, 0x50, 0x58] {
+        bytes[at..at + 8].copy_from_slice(&0xFFFF_FFFF_FFFF_F000u64.to_le_bytes());
+    }
+    let server = Server::start(vec![("/h.h5".into(), bytes)]);
+    for total in [u64::MAX, u64::MAX - 1, 1 << 62] {
+        server.shared.fake_total.store(total, Ordering::SeqCst);
+        let url = server.url("/h.h5");
+        let storage = storage_for_url(&url, &quick()).unwrap();
+        assert_eq!(clawhdf5_format::storage::Storage::len(&*storage), total);
+        for (off, n) in [(total - 100, 50), (total - 10, 100), (total - 1, 1)] {
+            let got = clawhdf5_format::storage::Storage::read_at(&*storage, off, n).unwrap();
+            assert_eq!(got.len() as u64, (total - off).min(n as u64));
+            assert!(got.iter().all(|&b| b == 0));
+        }
+        if let Ok(f) = File::open_storage(storage) {
+            let _ = transcript(&f);
+        }
+        let _ = open_url_with(&url, &quick()).map(|f| transcript(&f));
+    }
+}
+
+/// `download` reads a whole file in bounded steps, and refuses a claimed
+/// length beyond its limit before reading anything.
+#[test]
+fn download_is_bounded_by_its_limit_not_the_claimed_length() {
+    let bytes = multi_block_file();
+    let server = Server::start(vec![("/m.h5".into(), bytes.clone())]);
+    let url = server.url("/m.h5");
+    let storage = storage_for_url(&url, &quick()).unwrap();
+    let got = clawhdf5_remote::download(&*storage, clawhdf5_remote::DEFAULT_MAX_DOWNLOAD).unwrap();
+    assert_eq!(got, bytes);
+    let e = clawhdf5_remote::download(&*storage, 1000).unwrap_err();
+    assert!(
+        matches!(e, Error::Remote(RemoteError::TooLarge { limit: 1000, .. })),
+        "{e}"
+    );
+    server.shared.fake_total.store(1 << 62, Ordering::SeqCst);
+    let storage = storage_for_url(&url, &quick()).unwrap();
+    server.reset();
+    let e =
+        clawhdf5_remote::download(&*storage, clawhdf5_remote::DEFAULT_MAX_DOWNLOAD).unwrap_err();
+    assert!(
+        matches!(e, Error::Remote(RemoteError::TooLarge { len, .. }) if len == 1 << 62),
+        "{e}"
+    );
+    assert_eq!(server.requests(), 0, "refused before reading");
+}
+
+/// A URL's credentials — userinfo, and the query string of a presigned
+/// URL — never appear in an error message or a `Debug` output, whatever
+/// failed.
+#[test]
+fn credentials_never_appear_in_errors_or_debug() {
+    const SECRETS: [&str; 4] = ["hunter2", "SECRETSIG", "AKIDSECRET", "user:"];
+    fn clean(what: &str, text: &str) {
+        for s in SECRETS {
+            assert!(!text.contains(s), "{what}: {s} leaked in {text}");
+        }
+    }
+    fn check_err<T>(what: &str, r: Result<T, Error>) {
+        let Err(e) = r else {
+            panic!("{what}: expected an error")
+        };
+        clean(what, &format!("{e}"));
+        clean(what, &format!("{e:?}"));
+    }
+    let bytes = multi_block_file();
+    let server = Server::start(vec![("/m.h5".into(), bytes.clone())]);
+    let secret_url = |path: &str| {
+        format!(
+            "http://user:hunter2@{}{path}?X-Amz-Credential=AKIDSECRET&X-Amz-Signature=SECRETSIG",
+            server.addr
+        )
+    };
+    let url = secret_url("/m.h5");
+    let mut opts = quick();
+    opts.http.retries = 0;
+
+    // Opening works through such a URL, and Debug shows it redacted.
+    let (http, _) = HttpStorage::open(&url, opts.http.clone()).unwrap();
+    let debug = format!("{http:?}");
+    clean("Debug", &debug);
+    assert!(debug.contains("X-Amz-Signature=REDACTED"), "{debug}");
+    assert_eq!(http.url(), url, "the URL itself is kept for requests");
+
+    check_err("404", open_url_with(&secret_url("/missing.h5"), &opts));
+    server.shared.force_status.store(403, Ordering::SeqCst);
+    check_err("403", open_url_with(&url, &opts));
+    server.shared.force_status.store(0, Ordering::SeqCst);
+    server.shared.wrong_range.store(true, Ordering::SeqCst);
+    check_err("bad range at open", open_url_with(&url, &opts));
+    server.shared.wrong_range.store(false, Ordering::SeqCst);
+    server.shared.ignore_range.store(true, Ordering::SeqCst);
+    check_err("no range support", open_url_with(&url, &opts));
+    server.shared.ignore_range.store(false, Ordering::SeqCst);
+    server.shared.gzip_label.store(true, Ordering::SeqCst);
+    check_err("encoded body", open_url_with(&url, &opts));
+    server.shared.gzip_label.store(false, Ordering::SeqCst);
+
+    // Errors of reads after the open, through a File.
+    let f = open_url_with(&url, &opts).unwrap();
+    server.shared.wrong_range.store(true, Ordering::SeqCst);
+    check_err(
+        "bad range",
+        f.dataset("big")
+            .map(|d| d.read_f64())
+            .and_then(|r| r)
+            .map_err(Error::Hdf5),
+    );
+    server.shared.wrong_range.store(false, Ordering::SeqCst);
+    let f = open_url_with(&url, &opts).unwrap();
+    server.shared.force_status.store(403, Ordering::SeqCst);
+    check_err(
+        "403 on a read",
+        f.dataset("big")
+            .map(|d| d.read_f64())
+            .and_then(|r| r)
+            .map_err(Error::Hdf5),
+    );
+    server.shared.force_status.store(0, Ordering::SeqCst);
+    let f = open_url_with(&url, &opts).unwrap();
+    let mut other = bytes.clone();
+    let n = other.len();
+    other[n / 2] ^= 0xff;
+    server.shared.put("/m.h5", other);
+    check_err(
+        "ETag change",
+        f.dataset("big")
+            .map(|d| d.read_f64())
+            .and_then(|r| r)
+            .map_err(Error::Hdf5),
+    );
+    // A timeout.
+    let mut slow = opts.clone();
+    slow.http.timeout = Duration::from_millis(100);
+    server.shared.delay_ms.store(1000, Ordering::SeqCst);
+    check_err("timeout", open_url_with(&url, &slow));
+    server.shared.delay_ms.store(0, Ordering::SeqCst);
+    // A connection closed unanswered, a bad scheme.
+    let port = {
+        let l = std::net::TcpListener::bind("127.0.0.1:0").unwrap();
+        let port = l.local_addr().unwrap().port();
+        std::thread::spawn(move || {
+            for c in l.incoming() {
+                drop(c);
+            }
+        });
+        port
+    };
+    check_err(
+        "connection closed",
+        open_url_with(
+            &format!("http://user:[email protected]:{port}/a.h5?X-Amz-Signature=SECRETSIG"),
+            &opts,
+        ),
+    );
+    check_err(
+        "scheme",
+        open_url("ftp://user:[email protected]/a.h5?X-Amz-Signature=SECRETSIG"),
+    );
+    assert_eq!(
+        clawhdf5_remote::redact_url("https://me:pw@host:8/d/f.h5?X-Amz-Signature=abc&a=1#frag"),
+        "https://host:8/d/f.h5?X-Amz-Signature=REDACTED&a=REDACTED"
+    );
+}
+
+/// Redirects are followed within limits: custom credential headers reach
+/// only the URL's own origin (not another port a redirect leads to), a
+/// same-origin redirect keeps them, loops end at `max_redirects`, and
+/// `max_redirects = 0` refuses any redirect. (The https→http downgrade
+/// refusal is a unit test of `redirect_target`: no TLS server here.)
+#[test]
+fn redirects_are_followed_safely() {
+    let bytes = multi_block_file();
+    let local = File::from_bytes(bytes.clone()).unwrap();
+    let want = local.dataset("big").unwrap().read_f64().unwrap();
+    let target = Server::start(vec![("/m.h5".into(), bytes.clone())]);
+    let front = Server::start(vec![("/m.h5".into(), bytes.clone())]);
+    front.redirect("/r.h5", &target.url("/m.h5"));
+    front.redirect("/s.h5", "/m.h5");
+    front.redirect("/loop.h5", "/loop.h5");
+    let mut opts = quick();
+    opts.http.headers = vec![
+        ("X-Api-Key".into(), "sekrit".into()),
+        ("Authorization".into(), "Bearer tok".into()),
+    ];
+
+    // Cross-origin (another port): followed, the headers stay behind.
+    let f = open_url_with(&front.url("/r.h5"), &opts).unwrap();
+    assert_eq!(f.dataset("big").unwrap().read_f64().unwrap(), want);
+    assert!(front.saw_header("x-api-key"), "sent to the URL's origin");
+    assert!(target.requests() > 1);
+    assert!(
+        !target.saw_header("x-api-key") && !target.saw_header("authorization"),
+        "credential headers forwarded across origins: {:?}",
+        target.shared.seen.lock().unwrap()
+    );
+
+    // Same origin: followed with the headers.
+    front.reset();
+    let f = open_url_with(&front.url("/s.h5"), &opts).unwrap();
+    assert_eq!(f.dataset("big").unwrap().read_f64().unwrap(), want);
+    let seen = front.shared.seen.lock().unwrap().clone();
+    assert!(
+        seen.iter()
+            .filter(|(p, _)| p == "/m.h5")
+            .all(|(_, h)| h.get("x-api-key").map(String::as_str) == Some("sekrit"))
+    );
+
+    // A loop ends after max_redirects (5 by default): 6 requests.
+    front.reset();
+    let e = open_url_with(&front.url("/loop.h5"), &opts).unwrap_err();
+    assert!(matches!(e, Error::Remote(RemoteError::Redirect(_))), "{e}");
+    assert_eq!(front.requests(), 6);
+
+    // No redirects allowed.
+    opts.http.max_redirects = 0;
+    let e = open_url_with(&front.url("/s.h5"), &opts).unwrap_err();
+    assert!(matches!(e, Error::Remote(RemoteError::Redirect(_))), "{e}");
+}
+
+/// A server may answer the first request (bytes 0 to 1 MiB - 1) with 200
+/// when that covers the whole file: a body no longer than the range asked
+/// for is accepted as the whole file, in one request.
+#[test]
+fn a_200_covering_the_requested_range_is_the_whole_file() {
+    let root = std::path::Path::new(env!("CARGO_MANIFEST_DIR"));
+    let tall = root.join("../clawhdf5/tests/fixtures/tall.h5");
+    let bytes = std::fs::read(&tall).unwrap();
+    assert!(bytes.len() < 1 << 20);
+    let server = Server::start(vec![("/t.h5".into(), bytes.clone())]);
+    server.shared.ignore_range.store(true, Ordering::SeqCst);
+    let storage = storage_for_url(&server.url("/t.h5"), &quick()).unwrap();
+    let f = File::open_storage(storage).unwrap();
+    assert_eq!(f.contiguous_bytes(), Some(&bytes[..]));
+    assert_eq!(transcript(&f), transcript(&File::open(&tall).unwrap()));
+    assert_eq!(server.requests(), 1);
+    // Larger than the range asked for: still refused.
+    let mut opts = quick().http;
+    opts.first_request = 4096;
+    let e = HttpStorage::open(&server.url("/t.h5"), opts).unwrap_err();
+    assert!(matches!(e, RemoteError::RangeNotSupported(_)), "{e}");
+}
+
+/// A slow link is not cut off: the body's time budget grows with its size
+/// (`timeout` + size at `min_speed`), so a 256 KiB block at 256 KiB/s
+/// (1 s) reads with a 300 ms `timeout`. A stalled body still fails, soon.
+#[test]
+fn slow_links_read_and_stalled_ones_fail() {
+    let bytes = multi_block_file();
+    let server = Server::start(vec![("/m.h5".into(), bytes.clone())]);
+    let url = server.url("/m.h5");
+    let mut opts = quick();
+    opts.cache.block_size = 256 << 10;
+    opts.cache.max_request = 256 << 10;
+    opts.http.timeout = Duration::from_millis(300);
+    server
+        .shared
+        .throttle_bps
+        .store(256 << 10, Ordering::SeqCst);
+    let f = open_url_with(&url, &opts).unwrap();
+    assert_eq!(f.root().groups().unwrap(), ["grp"]);
+    assert_eq!(
+        f.dataset("grp/small").unwrap().read_f64().unwrap(),
+        [1.0, 2.0, 3.0]
+    );
+    server.shared.throttle_bps.store(0, Ordering::SeqCst);
+
+    // Stalled mid-body for 20 s: a timeout well before that.
+    server.shared.stall_ms.store(20_000, Ordering::SeqCst);
+    opts.http.retries = 0;
+    opts.http.min_speed = 64 << 20;
+    let t = std::time::Instant::now();
+    let e = open_url_with(&url, &opts).unwrap_err();
+    assert!(matches!(e, Error::Remote(RemoteError::Transport(_))), "{e}");
+    assert!(t.elapsed() < Duration::from_secs(5), "{:?}", t.elapsed());
+}
+
+/// `cached` reports a failed first fetch as a remote error, not as an
+/// HDF5 format error.
+#[test]
+fn cached_reports_a_failed_prefetch_as_remote() {
+    use clawhdf5_format::error::FormatError;
+    use std::borrow::Cow;
+    struct Down;
+    impl clawhdf5_format::storage::Storage for Down {
+        fn read_at(&self, _: u64, _: usize) -> Result<Cow<'_, [u8]>, FormatError> {
+            Err(RemoteError::Transport("connection refused".into()).into())
+        }
+        fn len(&self) -> u64 {
+            1 << 20
+        }
+    }
+    let e = clawhdf5_remote::cached(Box::new(Down), &Options::default())
+        .map(|_| ())
+        .unwrap_err();
+    assert!(
+        matches!(&e, Error::Remote(RemoteError::Backend(m)) if m.contains("connection refused")),
+        "{e:?}"
+    );
+}

crates/clawhdf5-remote/tests/object_store.rs

@@ -0,0 +1,159 @@
+//! `ObjectStoreStorage` against object_store's in-memory and local-file
+//! backends (no cloud needed): values equal `File::open`'s, the block cache
+//! coalesces, and an object replaced while open is an error.
+
+#![cfg(feature = "object-store")]
+
+mod common;
+
+use std::sync::Arc;
+
+use clawhdf5::File;
+use clawhdf5_format::storage::Storage;
+use clawhdf5_remote::object_store::memory::InMemory;
+use clawhdf5_remote::object_store::path::Path as ObjectPath;
+use clawhdf5_remote::object_store::{ObjectStore, ObjectStoreExt, PutPayload};
+use clawhdf5_remote::{ObjectStoreStorage, Options, open_object};
+use common::{multi_block_file, transcript};
+
+fn put(store: &dyn ObjectStore, path: &str, bytes: Vec<u8>) {
+    let rt = tokio_rt();
+    rt.block_on(store.put(&ObjectPath::from(path), PutPayload::from(bytes)))
+        .unwrap();
+}
+
+fn tokio_rt() -> tokio::runtime::Runtime {
+    tokio::runtime::Builder::new_current_thread()
+        .enable_all()
+        .build()
+        .unwrap()
+}
+
+#[test]
+fn in_memory_store_reads_like_file_open() {
+    let store: Arc<dyn ObjectStore> = Arc::new(InMemory::new());
+    let mut compared = 0;
+    for (i, p) in common::fixtures().iter().enumerate() {
+        let bytes = std::fs::read(p).unwrap();
+        let key = format!("f{i}.h5");
+        put(store.as_ref(), &key, bytes);
+        let (Ok(local), Ok((mut remote, cache))) = (
+            File::open(p),
+            open_object(store.clone(), &key, &Options::default()),
+        ) else {
+            continue;
+        };
+        remote.set_vds_resolver(common::sibling_resolver(p.parent().unwrap().into()));
+        assert!(remote.contiguous_bytes().is_none());
+        assert_eq!(transcript(&remote), transcript(&local), "{}", p.display());
+        assert!(cache.stats().reads > 0);
+        compared += 1;
+    }
+    assert!(compared >= 40, "{compared}");
+}
+
+#[test]
+fn multi_block_object_is_fetched_in_coalesced_blocks() {
+    let store: Arc<dyn ObjectStore> = Arc::new(InMemory::new());
+    let bytes = multi_block_file();
+    put(store.as_ref(), "m.h5", bytes.clone());
+    let (remote, cache) = open_object(store, "m.h5", &Options::default()).unwrap();
+    let local = File::from_bytes(bytes.clone()).unwrap();
+    assert_eq!(
+        remote.dataset("big").unwrap().read_f64().unwrap(),
+        local.dataset("big").unwrap().read_f64().unwrap()
+    );
+    let s = cache.stats();
+    let blocks = (bytes.len() as u64).div_ceil(1 << 20);
+    assert!(s.bytes_fetched <= bytes.len() as u64);
+    assert!(s.requests <= blocks, "{s:?}");
+    assert_eq!(cache.inner().len(), bytes.len() as u64);
+}
+
+#[test]
+fn local_file_store_reads_like_file_open() {
+    let dir = tempfile::tempdir().unwrap();
+    let bytes = multi_block_file();
+    std::fs::write(dir.path().join("m.h5"), &bytes).unwrap();
+    let store: Arc<dyn ObjectStore> = Arc::new(
+        clawhdf5_remote::object_store::local::LocalFileSystem::new_with_prefix(dir.path()).unwrap(),
+    );
+    let (remote, _) = open_object(store, "m.h5", &Options::default()).unwrap();
+    let local = File::from_bytes(bytes).unwrap();
+    assert_eq!(transcript(&remote), transcript(&local));
+}
+
+#[test]
+fn an_object_replaced_while_open_is_an_error() {
+    let store: Arc<dyn ObjectStore> = Arc::new(InMemory::new());
+    let bytes = multi_block_file();
+    put(store.as_ref(), "m.h5", bytes.clone());
+    let (remote, _) = open_object(store.clone(), "m.h5", &Options::default()).unwrap();
+    assert_eq!(remote.root().groups().unwrap(), ["grp"]);
+    let mut other = bytes;
+    let n = other.len();
+    other[n / 2] ^= 0xff;
+    put(store.as_ref(), "m.h5", other);
+    let err = remote
+        .dataset("big")
+        .unwrap()
+        .read_f64()
+        .unwrap_err()
+        .to_string();
+    assert!(err.contains("changed while open"), "{err}");
+}
+
+#[test]
+fn missing_objects_and_async_callers_are_clean_errors() {
+    let store: Arc<dyn ObjectStore> = Arc::new(InMemory::new());
+    assert!(ObjectStoreStorage::new(store.clone(), ObjectPath::from("nope.h5")).is_err());
+    put(store.as_ref(), "m.h5", multi_block_file());
+    let storage = ObjectStoreStorage::new(store, ObjectPath::from("m.h5")).unwrap();
+    // From inside a current-thread runtime: works (the read runs on the
+    // storage's own runtime), no panic, no deadlock.
+    let rt = tokio_rt();
+    let r = rt.block_on(async { storage.read_at(0, 10).map(|b| b.len()) });
+    assert_eq!(r.unwrap(), 10);
+    // Dropping the storage inside a runtime does not panic.
+    rt.block_on(async move { drop(storage) });
+}
+
+/// The advice for async callers works: a read in `spawn_blocking` of a
+/// multi-threaded runtime (where `Handle::try_current` is Ok), and a read
+/// straight inside a current-thread runtime's task.
+#[test]
+fn object_stores_read_from_spawn_blocking_and_inside_runtimes() {
+    let store: Arc<dyn ObjectStore> = Arc::new(InMemory::new());
+    let bytes = multi_block_file();
+    put(store.as_ref(), "m.h5", bytes.clone());
+    let want = File::from_bytes(bytes)
+        .unwrap()
+        .dataset("big")
+        .unwrap()
+        .read_f64()
+        .unwrap();
+    let rt = tokio::runtime::Builder::new_multi_thread()
+        .worker_threads(2)
+        .build()
+        .unwrap();
+    let (s, w) = (store.clone(), want.clone());
+    let got = rt
+        .block_on(async move {
+            tokio::task::spawn_blocking(move || {
+                assert!(tokio::runtime::Handle::try_current().is_ok());
+                let (f, _) =
+                    open_object(s, "m.h5", &Options::default()).map_err(|e| e.to_string())?;
+                let v = f.dataset("big").unwrap().read_f64().unwrap();
+                Ok::<_, String>(v == w)
+            })
+            .await
+        })
+        .unwrap();
+    assert_eq!(got, Ok(true));
+    let rt = tokio_rt();
+    let got = rt.block_on(async {
+        let (f, _) = open_object(store, "m.h5", &Options::default()).unwrap();
+        f.dataset("big").unwrap().read_f64().unwrap()
+    });
+    assert_eq!(got, want);
+}

crates/clawhdf5-tools/Cargo.toml

@@ -14,9 +14,16 @@ readme = "README.md"
 name = "h5rs"
 path = "src/main.rs"
 
+[features]
+# FILE arguments may be URLs: http:// with `remote` (no C), https:// with
+# `remote-https` (rustls + ring, which compiles C).
+remote = ["dep:clawhdf5-remote"]
+remote-https = ["remote", "clawhdf5-remote/https"]
+
 [dependencies]
 clawhdf5 = { path = "../clawhdf5", version = "2.7.0" }
 clawhdf5-format = { path = "../clawhdf5-format", version = "2.7.0" }
+clawhdf5-remote = { path = "../clawhdf5-remote", version = "2.7.0", optional = true }
 serde_json = "1"
 
 [dev-dependencies]

crates/clawhdf5-tools/README.md

@@ -24,6 +24,29 @@ Every command takes `--max-bytes N` where it reads values (default 1 GiB): a
 dataset whose dataspace claims more than that is reported instead of read, so
 a corrupt size cannot exhaust memory.
 
+### Remote files
+
+Built with the `remote` feature, every FILE argument may be an `http://`
+URL (`remote-https` adds `https://`, through rustls and ring, which compiles
+C; the default build has neither). The file is read by range requests
+through [clawhdf5-remote](../clawhdf5-remote/README.md)'s block cache, so
+`ls` of a large file fetches its metadata blocks, not the file:
+
+```bash
+cargo install --path crates/clawhdf5-tools --features remote
+h5rs ls -r -v http://127.0.0.1:8000/file.h5
+h5rs dump http://127.0.0.1:8000/file.h5
+h5rs diff local.h5 http://127.0.0.1:8000/file.h5
+```
+
+A URL names the whole file (`FILE/OBJECT` suffixes are for local paths).
+`check` validates every byte, so it downloads a remote file whole first —
+up to `--max-download N` bytes (default 1 GiB), refusing a longer file
+before reading any of it. URLs are printed without their credentials
+(userinfo, query string values).
+The output is the local file's (`tests/remote.rs` compares every
+subcommand).
+
 ## `h5rs ls`
 
 ```console

crates/clawhdf5-tools/src/check.rs

@@ -28,7 +28,7 @@ use crate::h5::{Error, ErrorKind, H5, Kind};
 use crate::info::{self, DsInfo};
 
 pub const USAGE: &str = "\
-usage: h5rs check [--data] [-q] [--max-bytes N] FILE
+usage: h5rs check [--data] [-q] [--max-bytes N] [--max-download N] FILE
 
 Validate FILE's structure: walk every object from the root group, parse
 every header message, verify the checksums of version 2+ structures
@@ -45,6 +45,9 @@ problem is printed with the address of the structure involved.
                   datasets and attributes into its global heap collection
   -q, --quiet     print only the problems, not the summary
   --max-bytes N   largest dataset read by --data (default 1 GiB)
+  --max-download N
+                  largest remote (URL) file downloaded to check it
+                  (default 1 GiB)
 
 Exit status: 0 no problems, 1 problems found, 2 usage error or file not
 found, 3 internal error.";
@@ -124,6 +127,7 @@ pub fn run(args: &mut Args, out: &mut Out) -> std::io::Result<i32> {
     let mut read_data = false;
     let mut quiet = false;
     let mut max_bytes = None;
+    let mut max_download = 1u64 << 30;
     let mut file = None;
     while let Some(a) = args.next() {
         match a.as_str() {
@@ -133,6 +137,10 @@ pub fn run(args: &mut Args, out: &mut Out) -> std::io::Result<i32> {
                 Some(n) => max_bytes = Some(n),
                 None => return args.usage_error(out, "--max-bytes needs a number", USAGE),
             },
+            "--max-download" => match args.number() {
+                Some(n) => max_download = n,
+                None => return args.usage_error(out, "--max-download needs a number", USAGE),
+            },
             "-h" | "--help" => {
                 writeln!(out.o, "{USAGE}")?;
                 return Ok(0);
@@ -148,13 +156,24 @@ pub fn run(args: &mut Args, out: &mut Out) -> std::io::Result<i32> {
         return args.usage_error(out, "missing FILE", USAGE);
     };
     let path = std::path::Path::new(&file);
-    if !path.is_file() {
-        writeln!(out.e, "h5rs check: {file}: no such file")?;
-        return Ok(2);
-    }
-    let mut h5 = match H5::open(path) {
-        Ok(h) => h,
-        Err(_) => return unopenable(path, out),
+    let mut h5 = if crate::h5::is_url(&file) {
+        // check validates every byte, so a remote file is downloaded whole.
+        match H5::open_arg_whole(&file, max_download) {
+            Ok(h) => h,
+            Err(e) => {
+                writeln!(out.e, "h5rs check: {e}")?;
+                return Ok(2);
+            }
+        }
+    } else {
+        if !path.is_file() {
+            writeln!(out.e, "h5rs check: {file}: no such file")?;
+            return Ok(2);
+        }
+        match H5::open(path) {
+            Ok(h) => h,
+            Err(_) => return unopenable(path, out),
+        }
     };
     if let Some(m) = max_bytes {
         h5.max_bytes = m;
@@ -185,7 +204,7 @@ pub fn run(args: &mut Args, out: &mut Out) -> std::io::Result<i32> {
         }
     }
     if !quiet {
-        c.summary(&file, out)?;
+        c.summary(&crate::h5::shown(&file), out)?;
     }
     Ok(if c.panicked {
         3

crates/clawhdf5-tools/src/diff.rs

@@ -174,7 +174,7 @@ pub fn run(args: &mut Args, out: &mut Out) -> std::io::Result<i32> {
     }
     let mut files = Vec::new();
     for f in &pos[..2] {
-        match H5::open(std::path::Path::new(f)) {
+        match H5::open_arg(f) {
             Ok(mut h) => {
                 if let Some(m) = max_bytes {
                     h.max_bytes = m;
@@ -196,7 +196,8 @@ pub fn run(args: &mut Args, out: &mut Out) -> std::io::Result<i32> {
             Err(_) => {
                 writeln!(
                     out.e,
-                    "h5rs diff: object <{obj}> could not be found in <{f}>"
+                    "h5rs diff: object <{obj}> could not be found in <{}>",
+                    crate::h5::shown(f)
                 )?;
                 return Ok(2);
             }
@@ -214,7 +215,7 @@ pub fn run(args: &mut Args, out: &mut Out) -> std::io::Result<i32> {
         let entries = match collected {
             Ok(e) => e,
             Err(e) => {
-                writeln!(out.e, "h5rs diff: {f}: {e}")?;
+                writeln!(out.e, "h5rs diff: {}: {e}", crate::h5::shown(f))?;
                 return Ok(2);
             }
         };

crates/clawhdf5-tools/src/dump.rs

@@ -82,7 +82,7 @@ pub fn run(args: &mut Args, out: &mut Out) -> std::io::Result<i32> {
     let Some(file) = file else {
         return args.usage_error(out, "missing FILE", USAGE);
     };
-    let mut h5 = match H5::open(std::path::Path::new(&file)) {
+    let mut h5 = match H5::open_arg(&file) {
         Ok(h) => h,
         Err(e) => {
             writeln!(out.e, "h5rs dump: {e}")?;
@@ -98,6 +98,7 @@ pub fn run(args: &mut Args, out: &mut Out) -> std::io::Result<i32> {
         problems: 0,
         paths: OnceCell::new(),
     };
+    let file = crate::h5::shown(&file);
     let fname = std::path::Path::new(&file)
         .file_name()
         .map(|s| s.to_string_lossy().into_owned())

crates/clawhdf5-tools/src/h5.rs

@@ -10,9 +10,9 @@ use std::collections::HashMap;
 use std::path::{Path, PathBuf};
 
 use clawhdf5::File;
-use clawhdf5_format::attribute::{AttributeMessage, extract_attributes_tolerant};
+use clawhdf5_format::attribute::{AttributeMessage, extract_attributes_tolerant_in};
 use clawhdf5_format::attribute_info::AttributeInfoMessage;
-use clawhdf5_format::btree_v2::{BTreeV2Header, collect_btree_v2_records};
+use clawhdf5_format::btree_v2::{BTreeV2Header, collect_btree_v2_records_in};
 use clawhdf5_format::data_layout::DataLayout;
 use clawhdf5_format::dataspace::{Dataspace, DataspaceType};
 use clawhdf5_format::datatype::Datatype;
@@ -24,6 +24,7 @@ use clawhdf5_format::link_info::LinkInfoMessage;
 use clawhdf5_format::link_message::{LinkMessage, LinkTarget};
 use clawhdf5_format::message_type::MessageType;
 use clawhdf5_format::object_header::ObjectHeader;
+use clawhdf5_format::storage::Storage;
 use clawhdf5_format::superblock::Superblock;
 use clawhdf5_format::symbol_table::SymbolTableMessage;
 
@@ -186,8 +187,11 @@ pub struct Link {
 
 /// An open file.
 pub struct H5 {
+    /// The file's path, or its URL for a remote file.
     pub path: PathBuf,
     pub file: File,
+    /// Size of the whole file in bytes (user block included).
+    pub size: u64,
     pub max_bytes: u64,
     /// Fractal heaps whose blocks were verified: `None` = sound.
     verified_heaps: RefCell<HashMap<u64, Option<Error>>>,
@@ -204,19 +208,96 @@ impl H5 {
                 path.display()
             ))
         })?;
-        Ok(H5 {
-            path: path.to_path_buf(),
+        let size = std::fs::metadata(path).map(|m| m.len()).unwrap_or(0);
+        Ok(H5::new(path.to_path_buf(), file, size))
+    }
+
+    fn new(path: PathBuf, file: File, size: u64) -> H5 {
+        H5 {
+            path,
             file,
+            size,
             max_bytes: DEFAULT_MAX_BYTES,
             verified_heaps: RefCell::new(HashMap::new()),
-        })
+        }
+    }
+
+    /// Open a command-line FILE argument: a path, or with the `remote`
+    /// feature an `http(s)://` (or `s3://`, `gs://`, `az://`) URL, read by
+    /// range requests through a block cache.
+    pub fn open_arg(arg: &str) -> Result<H5> {
+        if !is_url(arg) {
+            return H5::open(Path::new(arg));
+        }
+        let name = shown(arg);
+        #[cfg(feature = "remote")]
+        {
+            let storage =
+                clawhdf5_remote::storage_for_url(arg, &clawhdf5_remote::Options::default())
+                    .map_err(|e| Error::new(format!("{name}: {e}")))?;
+            let size = storage.len();
+            let file = File::open_storage(storage).map_err(|e| {
+                Error::new(format!("{name}: not an HDF5 file this tool can open: {e}"))
+            })?;
+            Ok(H5::new(PathBuf::from(&name), file, size))
+        }
+        #[cfg(not(feature = "remote"))]
+        Err(Error::new(format!(
+            "{name}: URLs need h5rs built with the `remote` feature"
+        )))
+    }
+
+    /// [`H5::open_arg`], with a remote file downloaded whole into memory
+    /// first — for `check`, which validates every byte of the file anyway
+    /// and parses it as one slice ([`H5::data`]). A remote file longer
+    /// than `max_download` bytes is refused before anything is read: its
+    /// length is only what the server claims.
+    pub fn open_arg_whole(arg: &str, max_download: u64) -> Result<H5> {
+        if !is_url(arg) {
+            return H5::open_arg(arg);
+        }
+        let name = shown(arg);
+        #[cfg(feature = "remote")]
+        {
+            // One open (one probe of the server); the download then reads
+            // through the same cache, the first block already in it.
+            let storage =
+                clawhdf5_remote::storage_for_url(arg, &clawhdf5_remote::Options::default())
+                    .map_err(|e| Error::new(format!("{name}: {e}")))?;
+            let bytes = clawhdf5_remote::download(&*storage, max_download)
+                .map_err(|e| Error::new(format!("{name}: {e}")))?;
+            let size = bytes.len() as u64;
+            let file = File::from_bytes(bytes).map_err(|e| {
+                Error::new(format!("{name}: not an HDF5 file this tool can open: {e}"))
+            })?;
+            Ok(H5::new(PathBuf::from(&name), file, size))
+        }
+        #[cfg(not(feature = "remote"))]
+        {
+            let _ = max_download;
+            Err(Error::new(format!(
+                "{name}: URLs need h5rs built with the `remote` feature"
+            )))
+        }
     }
 
     /// The file's bytes from the superblock on: what every address indexes.
+    ///
+    /// # Panics
+    ///
+    /// For a remote file opened by [`H5::open_arg`]; read it through
+    /// [`H5::store`], or open it with [`H5::open_arg_whole`].
     pub fn data(&self) -> &[u8] {
         self.file.as_bytes()
     }
 
+    /// The same bytes as a [`Storage`], for local and remote files alike:
+    /// in memory a read is a slice of [`H5::data`], remote it is served by
+    /// the block cache.
+    pub fn store(&self) -> &dyn Storage {
+        self.file.storage()
+    }
+
     pub fn sb(&self) -> &Superblock {
         self.file.superblock()
     }
@@ -239,8 +320,8 @@ impl H5 {
         if let Some(e) = self.file.cache_image_error() {
             return Err(Error::at(addr, format!("metadata cache image: {e}")));
         }
-        let off = usize::try_from(addr).map_err(|_| Error::at(addr, "address out of range"))?;
-        ObjectHeader::parse(self.data(), off, self.os(), self.ls())
+        to_usize(addr)?;
+        ObjectHeader::parse_in(self.store(), addr, self.os(), self.ls())
             .map_err(|e| Error::at(addr, format!("object header: {e}")))
     }
 
@@ -249,11 +330,14 @@ impl H5 {
     pub fn payload(&self, h: &ObjectHeader, t: MessageType) -> Result<Option<Vec<u8>>> {
         match h.messages.iter().find(|m| m.msg_type == t) {
             None => Ok(None),
-            Some(m) => {
-                clawhdf5_format::shared_message::message_data(self.data(), m, self.os(), self.ls())
-                    .map(|c| Some(c.into_owned()))
-                    .map_err(|e| Error::new(format!("{t:?} message: {e}")))
-            }
+            Some(m) => clawhdf5_format::shared_message::message_data_in(
+                self.store(),
+                m,
+                self.os(),
+                self.ls(),
+            )
+            .map(|c| Some(c.into_owned()))
+            .map_err(|e| Error::new(format!("{t:?} message: {e}"))),
         }
     }
 
@@ -305,8 +389,9 @@ impl H5 {
             self.verified_heap(fh)
                 .map_err(|e| e.context("dense attribute storage"))?;
         }
-        let (mut attrs, errs) = extract_attributes_tolerant(self.data(), h, self.os(), self.ls())
-            .map_err(|e| Error::new(format!("attributes: {e}")))?;
+        let (mut attrs, errs) =
+            extract_attributes_tolerant_in(self.store(), h, self.os(), self.ls())
+                .map_err(|e| Error::new(format!("attributes: {e}")))?;
         attrs.sort_by(|a, b| a.name.cmp(&b.name));
         Ok((attrs, errs.iter().map(|e| e.to_string()).collect()))
     }
@@ -316,7 +401,7 @@ impl H5 {
     pub fn links(&self, h: &ObjectHeader) -> Result<Vec<Link>> {
         let os = self.os();
         let ls = self.ls();
-        let data = self.data();
+        let data = self.store();
         let mut out = Vec::new();
         if let Some(m) = h
             .messages
@@ -325,7 +410,7 @@ impl H5 {
         {
             let stm = SymbolTableMessage::parse(&m.data, os)
                 .map_err(|e| Error::new(format!("symbol table message: {e}")))?;
-            let entries = group_v1::resolve_v1_group_entries(data, &stm, os, ls)
+            let entries = group_v1::resolve_v1_group_entries_in(data, &stm, os, ls)
                 .map_err(|e| Error::at(stm.btree_address, format!("symbol table: {e}")))?;
             let has_soft = entries.iter().any(group_v1::is_v1_soft_link);
             for e in entries {
@@ -337,7 +422,7 @@ impl H5 {
                 }
             }
             if has_soft {
-                let soft = group_v1::v1_soft_links(data, &stm, os, ls)
+                let soft = group_v1::v1_soft_links_in(data, &stm, os, ls)
                     .map_err(|e| Error::at(stm.btree_address, format!("soft links: {e}")))?;
                 for (name, target) in soft {
                     out.push(Link {
@@ -387,15 +472,15 @@ impl H5 {
         name_type: u8,
     ) -> Result<Vec<Vec<u8>>> {
         self.verified_heap(heap)?;
-        let data = self.data();
+        let data = self.store();
         let os = self.os();
         let ls = self.ls();
-        let fh = FractalHeapHeader::parse(data, to_usize(heap)?, os, ls)
+        let fh = FractalHeapHeader::parse_in(data, to_usize(heap).map(|_| heap)?, os, ls)
             .map_err(|e| Error::at(heap, format!("fractal heap header: {e}")))?;
         let bt = btree.ok_or_else(|| Error::at(heap, "dense storage without a name index"))?;
-        let hdr = BTreeV2Header::parse(data, to_usize(bt)?, os, ls)
+        let hdr = BTreeV2Header::parse_in(data, to_usize(bt).map(|_| bt)?, os, ls)
             .map_err(|e| Error::at(bt, format!("v2 B-tree header: {e}")))?;
-        let recs = collect_btree_v2_records(data, &hdr, os, ls)
+        let recs = collect_btree_v2_records_in(data, &hdr, os, ls)
             .map_err(|e| Error::at(bt, format!("v2 B-tree: {e}")))?;
         // Name-index records: hash(4) + heap ID; creation-order ones: order(8) + heap ID.
         let skip = if hdr.tree_type == name_type { 4 } else { 8 };
@@ -407,7 +492,7 @@ impl H5 {
                 .get(skip..skip + idlen)
                 .ok_or_else(|| Error::at(bt, "v2 B-tree record shorter than a heap ID"))?;
             let obj = fh
-                .read_managed_object(data, id, os)
+                .read_managed_object_in(data, id, os)
                 .map_err(|e| Error::at(heap, format!("fractal heap object: {e}")))?;
             out.push(obj);
         }
@@ -561,7 +646,7 @@ impl H5 {
         if p.is_empty() {
             return Ok(self.root());
         }
-        clawhdf5_format::group_v2::resolve_path_any(self.data(), self.sb(), p)
+        clawhdf5_format::group_v2::resolve_path_any_in(self.store(), self.sb(), p)
             .map_err(|e| Error::new(format!("{path}: {e}")))
     }
 }
@@ -681,7 +766,9 @@ pub fn byte_len(ds: &Dataspace, dt: &Datatype) -> Result<u64> {
 /// Split a `FILE[/object/path]` argument the way h5ls does: the longest
 /// prefix that is an existing file is the file.
 pub fn split_file_arg(arg: &str) -> (String, Option<String>) {
-    if Path::new(arg).is_file() {
+    // A URL names the file only (its path cannot be split against the
+    // local file system).
+    if is_url(arg) || Path::new(arg).is_file() {
         return (arg.to_string(), None);
     }
     let mut idx: Vec<usize> = arg.match_indices('/').map(|(i, _)| i).collect();
@@ -694,3 +781,35 @@ pub fn split_file_arg(arg: &str) -> (String, Option<String>) {
     }
     (arg.to_string(), None)
 }
+
+/// A FILE argument as it may be printed: a URL without its credentials
+/// (userinfo, query string values — a presigned URL's signature), a path
+/// as given.
+pub fn shown(arg: &str) -> String {
+    if !is_url(arg) {
+        return arg.to_string();
+    }
+    #[cfg(feature = "remote")]
+    {
+        clawhdf5_remote::redact_url(arg)
+    }
+    #[cfg(not(feature = "remote"))]
+    {
+        let (scheme, rest) = arg.split_once("://").unwrap_or(("", arg));
+        let rest = rest.split(['?', '#']).next().unwrap_or("");
+        let host_end = rest.find('/').unwrap_or(rest.len());
+        let (authority, path) = rest.split_at(host_end);
+        let host = authority.rsplit_once('@').map_or(authority, |(_, h)| h);
+        format!("{scheme}://{host}{path}")
+    }
+}
+
+/// Whether a FILE argument is a URL (`scheme://...`) rather than a path.
+pub fn is_url(arg: &str) -> bool {
+    arg.split_once("://").is_some_and(|(scheme, _)| {
+        !scheme.is_empty()
+            && scheme
+                .chars()
+                .all(|c| c.is_ascii_alphanumeric() || matches!(c, '+' | '-' | '.'))
+    })
+}

crates/clawhdf5-tools/src/heap_blocks.rs

@@ -4,10 +4,12 @@
 //! heap header's flag says so). The library reads only the blocks an object
 //! lives in and does not verify block checksums, so `check` does it here.
 
+use std::borrow::Cow;
 use std::collections::HashSet;
 
 use clawhdf5_format::checksum::jenkins_lookup3;
 use clawhdf5_format::fractal_heap::FractalHeapHeader;
+use clawhdf5_format::storage::{Storage, read_exact_at};
 
 use crate::h5::{Error, H5};
 
@@ -26,7 +28,7 @@ pub struct HeapReport {
 }
 
 struct Walk<'a> {
-    data: &'a [u8],
+    data: &'a dyn Storage,
     heap: u64,
     fh: FractalHeapHeader,
     checksum_dblocks: bool,
@@ -60,14 +62,14 @@ fn log2(v: u64) -> u32 {
 /// parsed (and its checksum verified) by the library; an error there is
 /// returned as the only problem.
 pub fn verify(h5: &H5, heap: u64) -> HeapReport {
-    let data = h5.data();
+    let data = h5.store();
     let Ok(off) = usize::try_from(heap) else {
         return HeapReport {
             problems: vec![Error::at(heap, "fractal heap address out of range")],
             ..Default::default()
         };
     };
-    let fh = match FractalHeapHeader::parse(data, off, h5.os(), h5.ls()) {
+    let fh = match FractalHeapHeader::parse_in(data, heap, h5.os(), h5.ls()) {
         Ok(f) => f,
         Err(e) => {
             return HeapReport {
@@ -77,7 +79,15 @@ pub fn verify(h5: &H5, heap: u64) -> HeapReport {
         }
     };
     // Flags: signature(4) version(1) heap ID length(2) filter length(2) flags(1).
-    let flags = data.get(off + 9).copied().unwrap_or(0);
+    let flags = match data.read_at(off as u64 + 9, 1) {
+        Ok(b) => b.first().copied().unwrap_or(0),
+        Err(e) => {
+            return HeapReport {
+                problems: vec![Error::at(heap, format!("fractal heap header: {e}"))],
+                ..Default::default()
+            };
+        }
+    };
     let mut w = Walk {
         data,
         heap,
@@ -107,11 +117,42 @@ pub fn verify(h5: &H5, heap: u64) -> HeapReport {
     w.r
 }
 
-impl Walk<'_> {
+impl<'a> Walk<'a> {
     fn problem(&mut self, addr: u64, msg: impl Into<String>) {
         self.r.problems.push(Error::at(addr, msg));
     }
 
+    /// Bytes `[start, end)` of the file: `Ok(None)` when they run past its
+    /// end (what a slice `get` of the whole file answered), `Err` when the
+    /// storage fails to read them (a remote file).
+    fn get(&self, start: usize, end: usize) -> Result<Option<Cow<'a, [u8]>>, String> {
+        let Some(len) = end.checked_sub(start) else {
+            return Ok(None);
+        };
+        if end as u64 > self.data.len() {
+            return Ok(None);
+        }
+        read_exact_at(self.data, start as u64, len)
+            .map(Some)
+            .map_err(|e| e.to_string())
+    }
+
+    /// [`Walk::get`], recording a read failure as a problem at `addr`.
+    fn get_or_note(
+        &mut self,
+        addr: u64,
+        start: usize,
+        end: usize,
+    ) -> Option<Option<Cow<'a, [u8]>>> {
+        match self.get(start, end) {
+            Ok(b) => Some(b),
+            Err(e) => {
+                self.problem(addr, format!("fractal heap block: {e}"));
+                None
+            }
+        }
+    }
+
     fn row_size(&self, row: usize) -> Option<u64> {
         let s = self.fh.starting_block_size;
         if row <= 1 {
@@ -155,10 +196,11 @@ impl Walk<'_> {
             return None;
         };
         let hdr_len = 5 + self.os + self.boff_bytes;
-        let Some(b) = start
-            .checked_add(hdr_len)
-            .and_then(|e| self.data.get(start..e))
-        else {
+        let b = match start.checked_add(hdr_len) {
+            Some(e) => self.get_or_note(addr, start, e)?,
+            None => None,
+        };
+        let Some(b) = b else {
             self.problem(
                 addr,
                 format!("fractal heap {what} block lies past the end of the file"),
@@ -209,7 +251,10 @@ impl Walk<'_> {
             self.problem(addr, "fractal heap direct block size out of range");
             return;
         };
-        let Some(block) = self.data.get(start..end) else {
+        let Some(block) = self.get_or_note(addr, start, end) else {
+            return;
+        };
+        let Some(block) = block else {
             self.problem(
                 addr,
                 "fractal heap direct block extends past the end of the file",
@@ -259,14 +304,17 @@ impl Walk<'_> {
             };
             let direct = row < direct_rows;
             for _ in 0..width {
-                let Some(b) = self.data.get(pos..pos + self.os) else {
+                let Some(b) = self.get_or_note(addr, pos, pos + self.os) else {
+                    return;
+                };
+                let Some(b) = b else {
                     self.problem(
                         addr,
                         "fractal heap indirect block extends past the end of the file",
                     );
                     return;
                 };
-                let child = le(b);
+                let child = le(&b);
                 pos += self.os;
                 if direct && filtered {
                     pos += self.ls + 4;
@@ -277,7 +325,10 @@ impl Walk<'_> {
                 off = off.saturating_add(rs);
             }
         }
-        let Some(stored) = self.data.get(pos..pos + 4) else {
+        let Some(stored) = self.get_or_note(addr, pos, pos + 4) else {
+            return;
+        };
+        let Some(stored) = stored else {
             self.problem(
                 addr,
                 "fractal heap indirect block extends past the end of the file",
@@ -285,7 +336,10 @@ impl Walk<'_> {
             return;
         };
         let stored = u32::from_le_bytes([stored[0], stored[1], stored[2], stored[3]]);
-        let computed = jenkins_lookup3(&self.data[start..pos]);
+        let Some(Some(body)) = self.get_or_note(addr, start, pos) else {
+            return;
+        };
+        let computed = jenkins_lookup3(&body);
         self.r.checksums += 1;
         if computed != stored {
             self.problem(

crates/clawhdf5-tools/src/info.rs

@@ -1,7 +1,7 @@
 //! Dataset facts shared by `ls`, `dump`, `stat` and `check`: shape text,
 //! layout, filters and storage.
 
-use clawhdf5_format::chunked_read::{ChunkInfo, list_chunks};
+use clawhdf5_format::chunked_read::{ChunkInfo, list_chunks_in};
 use clawhdf5_format::data_layout::DataLayout;
 use clawhdf5_format::dataspace::{Dataspace, DataspaceType};
 use clawhdf5_format::datatype::Datatype;
@@ -189,8 +189,8 @@ pub fn chunks(
     let Some(addr) = *btree_address else {
         return Ok(Vec::new());
     };
-    list_chunks(
-        h5.data(),
+    list_chunks_in(
+        h5.store(),
         layout,
         ds,
         dt.type_size() as usize,

crates/clawhdf5-tools/src/ls.rs

@@ -63,7 +63,7 @@ pub fn run(args: &mut Args, out: &mut Out) -> std::io::Result<i32> {
         return args.usage_error(out, "missing FILE", USAGE);
     };
     let (file, obj) = split_file_arg(&target);
-    let mut h5 = match H5::open(std::path::Path::new(&file)) {
+    let mut h5 = match H5::open_arg(&file) {
         Ok(h) => h,
         Err(e) => {
             writeln!(out.e, "h5rs ls: {e}")?;

crates/clawhdf5-tools/src/stat.rs

@@ -86,7 +86,7 @@ pub fn run(args: &mut Args, out: &mut Out) -> std::io::Result<i32> {
     let Some(file) = file else {
         return args.usage_error(out, "missing FILE", USAGE);
     };
-    let h5 = match H5::open(std::path::Path::new(&file)) {
+    let h5 = match H5::open_arg(&file) {
         Ok(h) => h,
         Err(e) => {
             writeln!(out.e, "h5rs stat: {e}")?;
@@ -206,7 +206,7 @@ pub fn run(args: &mut Args, out: &mut Out) -> std::io::Result<i32> {
     if let Err(e) = walk {
         errors.push(e.to_string());
     }
-    report(&h5, &file, &s, out)?;
+    report(&h5, &crate::h5::shown(&file), &s, out)?;
     for e in &errors {
         writeln!(out.e, "h5rs stat: {e}")?;
     }
@@ -291,7 +291,7 @@ fn report(h5: &H5, file: &str, s: &Stats, out: &mut Out) -> std::io::Result<()>
         s.attr_objects
     )?;
     writeln!(o, "\tMax. # of attributes to objects: {}", s.max_attrs)?;
-    let total = std::fs::metadata(&h5.path).map(|m| m.len()).unwrap_or(0);
+    let total = h5.size;
     let ub = h5.file.user_block_size();
     writeln!(o, "Summary of file space information:")?;
     writeln!(o, "  User block: {ub} bytes")?;

crates/clawhdf5-tools/src/value.rs

@@ -151,14 +151,14 @@ pub struct Decoder<'a> {
     pub h5: &'a H5,
     /// Variable-length elements are resolved as the library resolves them
     /// (so as libhdf5 does), not by a decoder of our own.
-    vl: RefCell<VlResolver<'a>>,
+    vl: RefCell<VlResolver<'a, dyn clawhdf5_format::storage::Storage + 'a>>,
 }
 
 impl<'a> Decoder<'a> {
     pub fn new(h5: &'a H5) -> Self {
         Self {
             h5,
-            vl: RefCell::new(VlResolver::new(h5.data(), h5.os(), h5.ls())),
+            vl: RefCell::new(VlResolver::new_in(h5.store(), h5.os(), h5.ls())),
         }
     }
 
@@ -268,7 +268,7 @@ impl<'a> Decoder<'a> {
     /// has it.
     fn decode_vlen(&self, is_string: bool, base: &Datatype, b: &[u8], depth: u32) -> Value {
         if is_string {
-            return match self.vl.borrow_mut().string_element(b) {
+            return match self.vl.borrow_mut().string_element_in(b) {
                 Ok(Some(s)) => Value::Str(String::from_utf8_lossy(s).into_owned()),
                 Ok(None) => Value::NullStr,
                 Err(e) => Value::Error(e.to_string()),
@@ -278,8 +278,9 @@ impl<'a> Decoder<'a> {
         if bs == 0 {
             return Value::Error("VL base type of size 0".into());
         }
-        let obj = match self.vl.borrow_mut().element(b, bs) {
-            Ok(o) => o.unwrap_or(&[]),
+        // Copied out: decoding an element may resolve nested ones.
+        let obj = match self.vl.borrow_mut().element_in(b, bs) {
+            Ok(o) => o.unwrap_or(&[]).to_vec(),
             Err(e) => return Value::Error(e.to_string()),
         };
         Value::Seq(

crates/clawhdf5-tools/tests/remote.rs

@@ -0,0 +1,171 @@
+//! `h5rs` on URLs (feature `remote`): every subcommand prints for
+//! `http://…/file.h5` what it prints for the local file (the name aside).
+//! The files are served by the range-request test server of
+//! clawhdf5-remote on 127.0.0.1.
+
+#![cfg(feature = "remote")]
+
+#[path = "../../clawhdf5-remote/tests/common/server.rs"]
+mod server;
+
+use std::path::{Path, PathBuf};
+use std::process::Command;
+
+fn h5rs(args: &[&str]) -> (String, i32) {
+    let out = Command::new(env!("CARGO_BIN_EXE_h5rs"))
+        .args(args)
+        .output()
+        .expect("run h5rs");
+    let text = format!(
+        "{}{}",
+        String::from_utf8_lossy(&out.stdout),
+        String::from_utf8_lossy(&out.stderr)
+    );
+    (text, out.status.code().unwrap_or(-1))
+}
+
+fn fixtures() -> Vec<PathBuf> {
+    let root = Path::new(env!("CARGO_MANIFEST_DIR"));
+    [
+        "../clawhdf5/tests/fixtures/tall.h5",
+        "../clawhdf5/tests/fixtures/written_by_v2_7_0.h5",
+        "../clawhdf5/tests/fixtures/written_by_v2_7_0_paged.h5",
+        "../clawhdf5/tests/fixtures/h5clear_mdc_image.h5",
+        "../clawhdf5-format/tests/fixtures/fractal_heap_multiblock.h5",
+        "../clawhdf5-format/tests/fixtures/legacy/tcompound.h5",
+        "../clawhdf5-format/tests/fixtures/legacy/h5ex_g_iterate.h5",
+    ]
+    .iter()
+    .map(|p| root.join(p))
+    .collect()
+}
+
+#[test]
+fn every_subcommand_reads_a_url_like_the_local_file() {
+    let files = fixtures();
+    let served: Vec<(String, Vec<u8>)> = files
+        .iter()
+        .enumerate()
+        .map(|(i, p)| {
+            let name = p.file_name().unwrap().to_str().unwrap();
+            (format!("/{i}/{name}"), std::fs::read(p).unwrap())
+        })
+        .collect();
+    let server = server::Server::start(served.clone());
+    for (p, (url_path, _)) in files.iter().zip(&served) {
+        let url = server.url(url_path);
+        let local = p.to_str().unwrap();
+        for cmd in [
+            &["ls", "-r", "-v"][..],
+            &["dump"],
+            &["dump", "--json"],
+            &["stat"],
+            &["check", "--data"],
+        ] {
+            fn args<'a>(cmd: &[&'a str], f: &'a str) -> Vec<&'a str> {
+                cmd.iter().copied().chain([f]).collect()
+            }
+            let (want, want_rc) = h5rs(&args(cmd, local));
+            let (got, got_rc) = h5rs(&args(cmd, &url));
+            assert_eq!(
+                got.replace(&url, local),
+                want,
+                "h5rs {} {url}",
+                cmd.join(" ")
+            );
+            assert_eq!(got_rc, want_rc, "h5rs {} {url}", cmd.join(" "));
+        }
+        let (a, rc) = h5rs(&["diff", local, &url]);
+        assert_eq!(rc, 0, "h5rs diff {local} {url}: {a}");
+    }
+}
+
+#[test]
+fn url_errors_are_clean() {
+    let server = server::Server::start(vec![("/x.h5".into(), vec![1u8; 100])]);
+    let (out, rc) = h5rs(&["ls", &server.url("/missing.h5")]);
+    assert_eq!(rc, 2, "{out}");
+    assert!(out.contains("404"), "{out}");
+    let (out, rc) = h5rs(&["ls", &server.url("/x.h5")]);
+    assert_eq!(rc, 2, "{out}");
+    assert!(out.contains("not an HDF5 file"), "{out}");
+    let (out, rc) = h5rs(&["check", &server.url("/missing.h5")]);
+    assert_eq!(rc, 2, "{out}");
+    #[cfg(not(feature = "remote-https"))]
+    {
+        let (out, rc) = h5rs(&["ls", "https://example.com/a.h5"]);
+        assert_eq!(rc, 2, "{out}");
+        assert!(out.contains("`https` feature"), "{out}");
+    }
+}
+
+/// `check` downloads a remote file whole, but never trusts the length the
+/// server claims: a server claiming 2^62 bytes for a small file is refused
+/// before anything is allocated (it used to abort the process), and
+/// `--max-download` caps real files too.
+#[test]
+fn check_refuses_a_remote_file_beyond_the_download_limit() {
+    use std::sync::atomic::Ordering;
+    let tall = Path::new(env!("CARGO_MANIFEST_DIR")).join("../clawhdf5/tests/fixtures/tall.h5");
+    let server = server::Server::start(vec![("/t.h5".into(), std::fs::read(&tall).unwrap())]);
+    let url = server.url("/t.h5");
+    let (out, rc) = h5rs(&["check", &url]);
+    assert_eq!(rc, 0, "{out}");
+    let (out, rc) = h5rs(&["check", "--max-download", "1000", &url]);
+    assert_eq!(rc, 2, "{out}");
+    assert!(out.contains("download limit of 1000 bytes"), "{out}");
+    server.shared.fake_total.store(1 << 62, Ordering::SeqCst);
+    let (out, rc) = h5rs(&["check", &url]);
+    assert_eq!(rc, 2, "{out}");
+    assert!(out.contains("more than the download limit"), "{out}");
+}
+
+/// A URL's credentials (userinfo, a presigned URL's query string) are not
+/// printed: not in errors, not in the file name of the output.
+#[test]
+fn credentials_in_urls_are_not_printed() {
+    let tall = Path::new(env!("CARGO_MANIFEST_DIR")).join("../clawhdf5/tests/fixtures/tall.h5");
+    let server = server::Server::start(vec![("/t.h5".into(), std::fs::read(&tall).unwrap())]);
+    let url = |path: &str| {
+        format!(
+            "http://user:hunter2@{}{path}?X-Amz-Signature=SECRETSIG",
+            server.addr
+        )
+    };
+    for args in [
+        vec!["ls", "-r"],
+        vec!["dump"],
+        vec!["stat"],
+        vec!["check"],
+        vec!["check", "--max-download", "10"],
+    ] {
+        for path in ["/t.h5", "/missing.h5"] {
+            let u = url(path);
+            let mut a = args.clone();
+            a.push(&u);
+            let (out, _) = h5rs(&a);
+            assert!(
+                !out.contains("hunter2") && !out.contains("SECRETSIG"),
+                "h5rs {}: {out}",
+                a.join(" ")
+            );
+        }
+    }
+    let (out, rc) = h5rs(&["diff", tall.to_str().unwrap(), &url("/t.h5"), "/nope"]);
+    assert_eq!(rc, 2, "{out}");
+    assert!(
+        !out.contains("hunter2") && !out.contains("SECRETSIG"),
+        "{out}"
+    );
+}
+
+/// `check URL` opens the file once: for a file within the first block,
+/// one request in all (it probed the server twice before).
+#[test]
+fn check_url_probes_the_server_once() {
+    let tall = Path::new(env!("CARGO_MANIFEST_DIR")).join("../clawhdf5/tests/fixtures/tall.h5");
+    let server = server::Server::start(vec![("/t.h5".into(), std::fs::read(&tall).unwrap())]);
+    let (out, rc) = h5rs(&["check", "--data", &server.url("/t.h5")]);
+    assert_eq!(rc, 0, "{out}");
+    assert_eq!(server.requests(), 1, "{:?}", server.log());
+}

crates/clawhdf5/src/reader.rs

@@ -619,6 +619,17 @@ impl File {
         self.data.contiguous()
     }
 
+    /// The bytes [`as_bytes`](Self::as_bytes) returns, as a [`Storage`],
+    /// for every backend: from the superblock on, bounded by the recorded
+    /// end of file, with a metadata cache image laid over them. Code that
+    /// parses the file itself with the `clawhdf5_format` `*_in` functions
+    /// reads through this, so it works on remote files too; for a file in
+    /// memory its [`Storage::as_contiguous`] is [`as_bytes`](Self::as_bytes)
+    /// (and every read a slice of it).
+    pub fn storage(&self) -> &(dyn Storage + Send + Sync) {
+        &self.data
+    }
+
     /// The error of a metadata cache image libhdf5 cannot load, when the
     /// file has one. Such a file opens, as in libhdf5, and every object
     /// lookup fails with this error; code that parses [`Self::as_bytes`]

crates/clawhdf5/tests/storage_equivalence.rs

@@ -577,3 +577,33 @@ fn harness_compares_errors_not_just_failures() {
     // A value against an error is never let through.
     assert!(unexplained_difference(&path, &want, &g.join("\n")).is_some());
 }
+
+/// `File::storage` is the view `as_bytes` gives, for every backend: the
+/// user block skipped, bounded by the end of file, a cache image laid over.
+#[test]
+fn file_storage_is_the_as_bytes_view_for_every_backend() {
+    let root = Path::new(env!("CARGO_MANIFEST_DIR"));
+    let mut files = Vec::new();
+    hdf5_files(&root.join("tests/fixtures"), &mut files);
+    hdf5_files(&root.join("../clawhdf5-format/tests/fixtures"), &mut files);
+    let mut compared = 0;
+    for p in files {
+        let Ok(local) = File::open(&p) else { continue };
+        let bytes = std::fs::read(&p).unwrap();
+        let remote = File::open_storage(Arc::new(CountingStorage::new(bytes))).unwrap();
+        let want = local.as_bytes();
+        let view = local.storage();
+        assert_eq!(view.as_contiguous(), Some(want), "{}", p.display());
+        let got = remote.storage();
+        assert!(got.as_contiguous().is_none());
+        assert_eq!(got.len(), want.len() as u64, "{}", p.display());
+        assert_eq!(
+            &*got.read_at(0, want.len()).unwrap(),
+            want,
+            "{}",
+            p.display()
+        );
+        compared += 1;
+    }
+    assert!(compared >= 40, "{compared}");
+}