From db2554dd8119da91828b6ec8813c2c47dbc1e720 Mon Sep 17 00:00:00 2001 From: osobh Date: Sat, 26 Sep 2026 17:13:28 -0500 Subject: [PATCH 01/20] clawhdf5-remote: block cache and HTTP range reads (open_url) Range-read milestone M3, first half: a new crate with the block cache the design makes mandatory for remote files and an HTTP backend, so open_url("http://...") gives a clawhdf5::File over File::open_storage. BlockCache wraps any Storage: aligned blocks (1 MiB by default, the size docs/design/range-reads.md section 2 measured), LRU with a byte budget, the missing blocks of one read_at/read_ranges fetched with one backend read_ranges call as runs of consecutive blocks (a one-block gap filled to merge runs, each request at most 8 MiB), and reads that miss more than half the budget not kept. Thread-safe without holding the lock across a fetch: a block being fetched is in flight, a second reader waits for it instead of fetching it again, and a failed fetch fails its waiters and is not cached. A backend holding the file in memory passes through. HttpStorage (ureq, no TLS by default; `https` adds rustls with ring): opening is one ranged GET of the first block, whose Content-Range gives the length (the cache keeps the bytes). The file is pinned by a strong ETag (If-Match), else Last-Modified (If-Unmodified-Since), and its length, checked on every response: a change is RemoteError::FileChanged, never mixed data. A server that ignores Range is refused without reading the body unless a full download is allowed. Connection errors, timeouts, 408/429/5xx and short bodies are retried with exponential backoff; Accept-Encoding: identity, and an encoded body is refused. read_ranges fetches its ranges in parallel. Tests (a std-only HTTP/1.1 server in tests/common/server.rs, also the range_server example): every fixture read over HTTP gives File::open's transcript (CLAWHDF5_REMOTE_CORPUS adds the conformance corpus), with request counts per file with and without the cache; an h5py-written file against libhdf5's values; a multi-block file fetched in whole blocks, each once; a server ignoring Range; a file replaced mid-read (ETag, Last-Modified, length only); truncated bodies and 503s (retried, then an error, never cached); a slow server with 8 concurrent readers (no block fetched twice); bad URLs, 404, encoded bodies, non-HDF5 data. The cache has unit tests for coalescing, splitting, LRU order, large reads, failures and concurrent in-flight dedup. ci-test.sh: clawhdf5-remote joins the no-C default-build check, and its https feature is linted. Co-Authored-By: Claude Opus 5.5 (1M context) --- CLAUDE.md | 15 +- Cargo.toml | 1 + crates/clawhdf5-remote/Cargo.toml | 27 + crates/clawhdf5-remote/README.md | 71 ++ .../clawhdf5-remote/examples/range_server.rs | 56 ++ crates/clawhdf5-remote/examples/read_url.rs | 45 + crates/clawhdf5-remote/src/cache.rs | 825 ++++++++++++++++++ crates/clawhdf5-remote/src/error.rs | 125 +++ crates/clawhdf5-remote/src/http.rs | 565 ++++++++++++ crates/clawhdf5-remote/src/lib.rs | 127 +++ crates/clawhdf5-remote/tests/common/mod.rs | 302 +++++++ crates/clawhdf5-remote/tests/common/server.rs | 331 +++++++ crates/clawhdf5-remote/tests/http.rs | 517 +++++++++++ scripts/ci-test.sh | 23 +- 14 files changed, 3024 insertions(+), 6 deletions(-) create mode 100644 crates/clawhdf5-remote/Cargo.toml create mode 100644 crates/clawhdf5-remote/README.md create mode 100644 crates/clawhdf5-remote/examples/range_server.rs create mode 100644 crates/clawhdf5-remote/examples/read_url.rs create mode 100644 crates/clawhdf5-remote/src/cache.rs create mode 100644 crates/clawhdf5-remote/src/error.rs create mode 100644 crates/clawhdf5-remote/src/http.rs create mode 100644 crates/clawhdf5-remote/src/lib.rs create mode 100644 crates/clawhdf5-remote/tests/common/mod.rs create mode 100644 crates/clawhdf5-remote/tests/common/server.rs create mode 100644 crates/clawhdf5-remote/tests/http.rs diff --git a/CLAUDE.md b/CLAUDE.md index f493066..57a5fa4 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -5,7 +5,7 @@ Pure-Rust HDF5 format implementation with HNSW vector search, WAL-backed persist ## Architecture -Cargo workspace with 18 crates under `crates/` (plus `libaec-sys`, an internal FFI bindings crate for the optional `szip` feature): +Cargo workspace with 19 crates under `crates/` (plus `libaec-sys`, an internal FFI bindings crate for the optional `szip` feature): | Crate | Role | |-------|------| @@ -26,6 +26,7 @@ Cargo workspace with 18 crates under `crates/` (plus `libaec-sys`, an internal F | `clawhdf5-napi` | Node.js native addon bindings | | `clawhdf5-py` | PyO3 Python bindings | | `clawhdf5-wasm` | WebAssembly (wasm-bindgen) reader for the browser; demo in `examples/wasm-viewer/` | +| `clawhdf5-remote` | Remote files: `open_url` over HTTP(S) range requests through a mandatory block cache (`BlockCache`) | | `clawhdf5-bench` | Benchmark suite | ## Key Features @@ -157,6 +158,18 @@ Cargo workspace with 18 crates under `crates/` (plus `libaec-sys`, an internal F `docs/known-issues.md`). Test changes with `cargo test -p clawhdf5-tools --test edit_interop` (h5py, h5dump, `h5rs check`). +- Remote files (`clawhdf5-remote`, range-read milestone M3 of + `docs/design/range-reads.md`): `open_url("http://…")` gives a + `clawhdf5::File` over `File::open_storage`, read through `BlockCache` + (1 MiB blocks, LRU byte budget, per-block in-flight dedup across threads, + runs coalesced into parallel requests). `HttpStorage` pins the file by + ETag/Last-Modified and length (a change is `RemoteError::FileChanged`), + refuses servers that ignore `Range` unless a full download is allowed, + and retries transient failures. Default build is plain HTTP with no C; + `https` (rustls + ring) is opt-in. Tests run a std-only HTTP server + (`tests/common/server.rs`, also the `range_server` example); + `CLAWHDF5_REMOTE_CORPUS=conformance/.cache/corpus` compares every corpus + file over HTTP with `File::open`. - GPU-accelerated vector distance computation (`clawhdf5-gpu`, wgpu); HDF5 I/O itself is CPU-only - Browser: `clawhdf5-wasm` (wasm-bindgen, read-only, file held in memory; no Zstd/SZIP since they link C) and the `examples/wasm-viewer/` page. diff --git a/Cargo.toml b/Cargo.toml index 0801b3d..dbb2f5c 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -18,6 +18,7 @@ members = [ "crates/clawhdf5-bench", "crates/clawhdf5-tools", "crates/clawhdf5-wasm", + "crates/clawhdf5-remote", "crates/libaec-sys", ] resolver = "2" diff --git a/crates/clawhdf5-remote/Cargo.toml b/crates/clawhdf5-remote/Cargo.toml new file mode 100644 index 0000000..6e68a73 --- /dev/null +++ b/crates/clawhdf5-remote/Cargo.toml @@ -0,0 +1,27 @@ +[package] +name = "clawhdf5-remote" +version = "2.7.0" +edition = "2024" +rust-version.workspace = true +description = "Read HDF5 files over HTTP(S) range requests with clawhdf5, through a block cache" +license = "MIT" +repository = "https://git.redclaw.dev/quantumclaw/clawhdf5" +readme = "README.md" +keywords = ["hdf5", "http", "s3", "range-requests", "science"] +categories = ["science", "network-programming"] + +[features] +# Plain-HTTP range reads and the block cache: pure Rust, no TLS stack, no C. +default = ["http"] +http = ["dep:ureq"] +# HTTPS through rustls (ring provider, Mozilla roots). ring compiles C and +# assembly, so this is not part of the default build. +https = ["http", "ureq/rustls"] + +[dependencies] +clawhdf5 = { path = "../clawhdf5", version = "2.7.0" } +clawhdf5-format = { path = "../clawhdf5-format", version = "2.7.0" } +ureq = { version = "3.4", optional = true, default-features = false } + +[dev-dependencies] +tempfile = { workspace = true } diff --git a/crates/clawhdf5-remote/README.md b/crates/clawhdf5-remote/README.md new file mode 100644 index 0000000..bb20f71 --- /dev/null +++ b/crates/clawhdf5-remote/README.md @@ -0,0 +1,71 @@ +# clawhdf5-remote + +Read HDF5 files where they live — on an HTTP(S) server — with +[clawhdf5](../../README.md), without downloading them first. + +```rust +let file = clawhdf5_remote::open_url("http://127.0.0.1:8000/data.h5")?; +let temps = file.dataset("/grid/temperature")?.read_f64()?; +``` + +The result is an ordinary `clawhdf5::File`: groups, datasets, attributes, +selections, variable-length data. Only the bytes an operation needs are +fetched, by `Range` requests, through a block cache. + +## How it reads + +- **Block cache** (`BlockCache`, mandatory for remote files; see + `docs/design/range-reads.md` §2): aligned blocks of 1 MiB by default, LRU + with a byte budget (64 MiB by default), the missing blocks of one read + fetched as runs of consecutive blocks (a gap of one block is fetched to + merge two runs), each request at most 8 MiB, the requests of one read in + parallel. A read that misses more than half the budget is not kept, so a + big dataset read does not evict the metadata. Readers on several threads + share one cache; a block is never fetched twice at once — a second reader + waits for the first one's request. +- **Opening costs one request**: a `GET` of the first block, whose + `Content-Range` gives the file's length. HDF5 files keep the superblock + and usually the root group's metadata there, so listing a small file + often needs nothing more. +- **Pinned to one version of the file**: a strong `ETag` is sent back as + `If-Match` (else `Last-Modified` as `If-Unmodified-Since`) and checked on + every response, as is the length. A file replaced while open is an error + (`RemoteError::FileChanged`), never a mix of old and new bytes. A server + with neither validator can only be checked by length; + `HttpOptions::require_validator` refuses it. +- **Servers that ignore `Range`** (answer `200` with the whole file) are + refused (`RemoteError::RangeNotSupported`) without reading the body, + unless `HttpOptions::allow_full_download` is set; then the file is + downloaded once and read from memory. +- **Retries**: connection failures, timeouts, `408`/`429`/`5xx` and bodies + that end early are retried with exponential backoff (3 retries, from + 200 ms). Bodies are requested with `Accept-Encoding: identity`; an encoded + body is refused. + +The zero-copy methods of `clawhdf5` (`read_raw_ref`, `read_*_zerocopy`, +`File::as_bytes`) borrow the whole file from memory, so they are errors +(`as_bytes` a panic; use `File::contiguous_bytes`) on a remote file. + +## Features + +| Feature | What | C code | +|---|---|---| +| `http` (default) | `http://` through `ureq`, no TLS | none | +| `https` | `https://` through rustls, ring provider, Mozilla roots | ring (C and assembly) | + +The default build compiles no C (`scripts/ci-test.sh` checks it). + +## Counting requests + +```rust +use clawhdf5_remote::{storage_for_url, Options}; +let storage = storage_for_url(url, &Options::default())?; +let file = clawhdf5::File::open_storage(storage.clone())?; +// ... read ... +let s = storage.stats(); // requests, bytes_fetched, hits, misses, cached_bytes, ... +``` + +`cargo run -p clawhdf5-remote --example range_server -- DIR` serves a +directory with range support (the server the tests use), and +`cargo run -p clawhdf5-remote --example read_url -- URL [DATASET]` lists a +file and prints what it cost. diff --git a/crates/clawhdf5-remote/examples/range_server.rs b/crates/clawhdf5-remote/examples/range_server.rs new file mode 100644 index 0000000..609693f --- /dev/null +++ b/crates/clawhdf5-remote/examples/range_server.rs @@ -0,0 +1,56 @@ +//! Serve the HDF5 files of a directory over HTTP with range support — the +//! server the tests use — to try `clawhdf5_remote` and `h5rs` on URLs. +//! +//! ```text +//! cargo run -p clawhdf5-remote --example range_server -- DIR [127.0.0.1:8000] +//! ``` +//! +//! Every file under `DIR` is served at its path relative to `DIR`. The +//! files are read into memory at start. Requests are logged to stderr. + +#[path = "../tests/common/server.rs"] +mod server; + +fn main() { + let mut args = std::env::args().skip(1); + let Some(dir) = args.next() else { + eprintln!("usage: range_server DIR [ADDR]"); + std::process::exit(2); + }; + let addr = args.next().unwrap_or_else(|| "127.0.0.1:8000".into()); + let root = std::path::PathBuf::from(&dir); + let mut files = Vec::new(); + let mut stack = vec![root.clone()]; + while let Some(d) = stack.pop() { + let Ok(entries) = std::fs::read_dir(&d) else { + continue; + }; + for e in entries.flatten() { + let p = e.path(); + if p.is_dir() { + stack.push(p); + } else if let (Ok(rel), Ok(bytes)) = (p.strip_prefix(&root), std::fs::read(&p)) { + let url = format!("/{}", rel.to_string_lossy().replace('\\', "/")); + files.push((url, bytes)); + } + } + } + files.sort(); + let server = server::Server::bind(&addr, files.clone()); + for (path, bytes) in &files { + eprintln!("{} ({} bytes)", server.url(path), bytes.len()); + } + eprintln!("serving {} files on http://{}", files.len(), server.addr); + let mut logged = 0; + loop { + std::thread::sleep(std::time::Duration::from_millis(200)); + let log = server.log(); + for (path, range) in &log[logged.min(log.len())..] { + match range { + Some((a, b)) => eprintln!("GET {path} bytes={a}-{b}"), + None => eprintln!("GET {path} (whole file)"), + } + } + logged = log.len(); + } +} diff --git a/crates/clawhdf5-remote/examples/read_url.rs b/crates/clawhdf5-remote/examples/read_url.rs new file mode 100644 index 0000000..145b4ce --- /dev/null +++ b/crates/clawhdf5-remote/examples/read_url.rs @@ -0,0 +1,45 @@ +//! List a remote HDF5 file and read one dataset, then print what it cost. +//! +//! ```text +//! cargo run -p clawhdf5-remote --example read_url -- URL [DATASET] +//! ``` + +use clawhdf5::File; +use clawhdf5_remote::{Options, storage_for_url}; + +fn main() -> Result<(), Box> { + let mut args = std::env::args().skip(1); + let url = args.next().ok_or("usage: read_url URL [DATASET]")?; + let dataset = args.next(); + + let storage = storage_for_url(&url, &Options::default())?; + let file = File::open_storage(storage.clone())?; + + // Walk the tree. + let mut stack = vec![("/".to_string(), file.root())]; + while let Some((path, group)) = stack.pop() { + for (name, addr) in group.entries()? { + let child = format!("{}/{name}", path.trim_end_matches('/')); + match file.dataset_at(addr) { + Ok(ds) => println!("{child} dataset {:?} {:?}", ds.shape()?, ds.dtype()?), + Err(_) => { + println!("{child} group"); + stack.push((child, file.group_at(addr))); + } + } + } + } + + if let Some(name) = dataset { + let values = file.dataset(&name)?.read_f64()?; + let shown: Vec<_> = values.iter().take(8).collect(); + println!("{name}: {} values, first {shown:?}", values.len()); + } + + let s = storage.stats(); + println!( + "{} range requests (the one at open included), {} bytes fetched, {} bytes cached", + s.requests, s.bytes_fetched, s.cached_bytes + ); + Ok(()) +} diff --git a/crates/clawhdf5-remote/src/cache.rs b/crates/clawhdf5-remote/src/cache.rs new file mode 100644 index 0000000..233938c --- /dev/null +++ b/crates/clawhdf5-remote/src/cache.rs @@ -0,0 +1,825 @@ +//! The block cache every remote backend is read through. +//! +//! `docs/design/range-reads.md` §2 measured why it is required: the parsers +//! revisit the same structures many times (410 reads to list the 21 objects +//! of a 7.7 MB file), so mapping reads one-to-one onto requests is hopeless, +//! while a cache of 1 MiB blocks lists the same file in 2 requests. +//! +//! [`BlockCache`] wraps any [`Storage`] (the backend that does the fetching) +//! and serves reads from fixed-size, aligned blocks: +//! +//! - **LRU with a byte budget.** Blocks are evicted least-recently-used +//! first once the cached bytes exceed [`CacheConfig::capacity`]. Reads +//! hold their blocks by reference count, so eviction never invalidates +//! data a read is still using. +//! - **Coalescing.** All the blocks one `read_at`/`read_ranges` call misses +//! are fetched with one `read_ranges` call on the backend, as runs of +//! consecutive blocks (a gap of up to [`CacheConfig::coalesce_gap`] bytes +//! of uncached blocks is fetched too, to merge two runs), each run at +//! most [`CacheConfig::max_request`] bytes. A backend fetches the runs of +//! one call in parallel. +//! - **Concurrency.** The cache's lock is held only to look blocks up and +//! to insert them, never across a fetch. A block being fetched is marked +//! in flight: a second reader that needs it waits for that fetch instead +//! of issuing its own, so concurrent readers never fetch a block twice. +//! A failed fetch fails every reader waiting for it, and is not cached. +//! - **Large reads do not flush the cache.** A call whose missing blocks +//! add up to more than half the budget is served without keeping them +//! (a big chunked read would otherwise evict all the metadata). +//! - **Local storages pass through.** A backend that holds the whole file +//! in memory ([`Storage::as_contiguous`]) is read directly. + +use std::borrow::Cow; +use std::collections::{BTreeMap, BTreeSet, HashMap}; +use std::ops::Range; +use std::sync::atomic::{AtomicU64, Ordering}; +use std::sync::{Arc, Condvar, Mutex, MutexGuard}; + +use clawhdf5_format::error::FormatError; +use clawhdf5_format::storage::Storage; + +/// Default block size: 1 MiB, the size `docs/design/range-reads.md` §2 +/// measured (metadata of the test files in 1–15 blocks; the chunk index of a +/// netCDF file spread over 14 blocks of a 48 MB file). +pub const DEFAULT_BLOCK_SIZE: u64 = 1 << 20; + +/// Settings of a [`BlockCache`]. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct CacheConfig { + /// Size of a block, in bytes; every fetch is whole, aligned blocks + /// (the last block of the file is shorter). + pub block_size: u64, + /// Byte budget of cached blocks. + pub capacity: u64, + /// Two runs of missing blocks separated by at most this many bytes of + /// uncached blocks are fetched as one request (the gap included). + pub coalesce_gap: u64, + /// Largest single request, in bytes (rounded down to whole blocks, at + /// least one block). Longer runs are split, so a backend can fetch the + /// pieces in parallel. + pub max_request: u64, +} + +impl Default for CacheConfig { + fn default() -> Self { + CacheConfig { + block_size: DEFAULT_BLOCK_SIZE, + capacity: 64 << 20, + coalesce_gap: DEFAULT_BLOCK_SIZE, + max_request: 8 << 20, + } + } +} + +/// What a [`BlockCache`] has done so far. +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] +pub struct CacheStats { + /// Reads served (`read_at` calls plus ranges of `read_ranges` calls). + pub reads: u64, + /// Block lookups that found the block cached. + pub hits: u64, + /// Block lookups that had to fetch the block (or wait for it). + pub misses: u64, + /// Block lookups that waited for another reader's fetch of the block. + pub waits: u64, + /// Ranges requested from the backend: for HTTP, one request each. + pub requests: u64, + /// `read_ranges` calls made on the backend. + pub fetch_calls: u64, + /// Bytes fetched from the backend. + pub bytes_fetched: u64, + /// Blocks evicted to stay within the budget. + pub evictions: u64, + /// Bytes currently cached. + pub cached_bytes: u64, +} + +type Block = Arc<[u8]>; + +/// A fetch in progress: the readers waiting for a block wait on this. +struct Flight { + result: Mutex>>, + done: Condvar, +} + +impl Flight { + fn new() -> Arc { + Arc::new(Flight { + result: Mutex::new(None), + done: Condvar::new(), + }) + } + + fn finish(&self, r: Result) { + let mut slot = lock(&self.result); + if slot.is_none() { + *slot = Some(r); + } + self.done.notify_all(); + } + + fn wait(&self) -> Result { + let mut slot = lock(&self.result); + loop { + if let Some(r) = slot.as_ref() { + return r.clone(); + } + slot = self + .done + .wait(slot) + .unwrap_or_else(std::sync::PoisonError::into_inner); + } + } +} + +enum Slot { + Ready { data: Block, tick: u64 }, + Pending(Arc), +} + +#[derive(Default)] +struct State { + blocks: HashMap, + /// tick -> block index, oldest first. + lru: BTreeMap, + tick: u64, + bytes: u64, +} + +fn lock(m: &Mutex) -> MutexGuard<'_, T> { + m.lock().unwrap_or_else(std::sync::PoisonError::into_inner) +} + +#[derive(Default)] +struct Counters { + reads: AtomicU64, + hits: AtomicU64, + misses: AtomicU64, + waits: AtomicU64, + requests: AtomicU64, + fetch_calls: AtomicU64, + bytes_fetched: AtomicU64, + evictions: AtomicU64, +} + +/// A [`Storage`] that serves reads of `inner` from an LRU cache of +/// fixed-size blocks. See the [module documentation](self). +pub struct BlockCache { + inner: S, + config: CacheConfig, + len: u64, + state: Mutex, + counters: Counters, +} + +/// Fails every flight a fetch claimed and did not complete (an error or a +/// panic in the backend), so no reader waits forever. +struct FlightGuard<'a, S> { + cache: &'a BlockCache, + flights: Vec<(u64, Arc)>, +} + +impl Drop for FlightGuard<'_, S> { + fn drop(&mut self) { + if self.flights.is_empty() { + return; + } + let mut st = lock(&self.cache.state); + for (i, f) in &self.flights { + if matches!(st.blocks.get(i), Some(Slot::Pending(p)) if Arc::ptr_eq(p, f)) { + st.blocks.remove(i); + } + } + drop(st); + for (_, f) in &self.flights { + f.finish(Err("the fetch of this block failed".into())); + } + } +} + +impl BlockCache { + /// Cache `inner` with the given settings. The length of the file is + /// taken from `inner` once, here: a remote file is pinned at open. + pub fn new(inner: S, mut config: CacheConfig) -> Self { + config.block_size = config.block_size.max(512); + config.capacity = config.capacity.max(config.block_size); + config.max_request = (config.max_request / config.block_size).max(1) * config.block_size; + let len = inner.len(); + BlockCache { + inner, + config, + len, + state: Mutex::new(State::default()), + counters: Counters::default(), + } + } + + /// The backend. + pub fn inner(&self) -> &S { + &self.inner + } + + /// The settings in use (after rounding). + pub fn config(&self) -> &CacheConfig { + &self.config + } + + /// Counters since the cache was made (or last [`reset_stats`](Self::reset_stats)). + pub fn stats(&self) -> CacheStats { + let c = &self.counters; + CacheStats { + reads: c.reads.load(Ordering::Relaxed), + hits: c.hits.load(Ordering::Relaxed), + misses: c.misses.load(Ordering::Relaxed), + waits: c.waits.load(Ordering::Relaxed), + requests: c.requests.load(Ordering::Relaxed), + fetch_calls: c.fetch_calls.load(Ordering::Relaxed), + bytes_fetched: c.bytes_fetched.load(Ordering::Relaxed), + evictions: c.evictions.load(Ordering::Relaxed), + cached_bytes: lock(&self.state).bytes, + } + } + + /// Zero the counters (the cached blocks stay). + pub fn reset_stats(&self) { + let c = &self.counters; + for a in [ + &c.reads, + &c.hits, + &c.misses, + &c.waits, + &c.requests, + &c.fetch_calls, + &c.bytes_fetched, + &c.evictions, + ] { + a.store(0, Ordering::Relaxed); + } + } + + /// Drop every cached block. + pub fn clear(&self) { + let mut st = lock(&self.state); + st.blocks.retain(|_, s| matches!(s, Slot::Pending(_))); + st.lru.clear(); + st.bytes = 0; + } + + /// Fetch the blocks covering `[offset, offset + len)` now (readahead), + /// keeping them cached. + pub fn prefetch(&self, offset: u64, len: u64) -> Result<(), FormatError> { + if self.inner.as_contiguous().is_some() { + return Ok(()); + } + let Some(blocks) = self.block_span(offset, len) else { + return Ok(()); + }; + self.blocks(&blocks.collect::>(), true).map(|_| ()) + } + + /// Put bytes already fetched (such as the first block, which a backend + /// may get while it probes the file's length) into the cache: `bytes` + /// are the file's bytes at `offset`. Only whole blocks (or the file's + /// last, short block) are kept; a block already cached is left alone. + /// The bytes count as one request in [`CacheStats`]. + pub fn insert(&self, offset: u64, bytes: &[u8]) { + self.counters.requests.fetch_add(1, Ordering::Relaxed); + self.counters + .bytes_fetched + .fetch_add(bytes.len() as u64, Ordering::Relaxed); + let bs = self.config.block_size; + let end = offset.saturating_add(bytes.len() as u64).min(self.len); + let mut i = offset.div_ceil(bs); + let mut st = lock(&self.state); + while i * bs < end { + let start = i * bs; + let block_end = (start + bs).min(self.len); + if block_end > end { + break; + } + if !st.blocks.contains_key(&i) { + let rel = (start - offset) as usize..(block_end - offset) as usize; + self.keep(&mut st, i, Arc::from(&bytes[rel])); + } + i += 1; + } + self.evict(&mut st); + } + + /// Block indices covering `[offset, offset + len)`, clamped to the file. + fn block_span(&self, offset: u64, len: u64) -> Option> { + let end = offset.saturating_add(len).min(self.len); + if offset >= end { + return None; + } + let bs = self.config.block_size; + Some(offset / bs..(end - 1) / bs + 1) + } + + fn block_len(&self, i: u64) -> u64 { + let start = i * self.config.block_size; + (start + self.config.block_size).min(self.len) - start + } + + fn keep(&self, st: &mut State, i: u64, data: Block) { + st.tick += 1; + let tick = st.tick; + st.bytes += <[u8]>::len(&data) as u64; + st.lru.insert(tick, i); + st.blocks.insert(i, Slot::Ready { data, tick }); + } + + fn evict(&self, st: &mut State) { + while st.bytes > self.config.capacity { + let Some((_, i)) = st.lru.pop_first() else { + break; + }; + if let Some(Slot::Ready { data, .. }) = st.blocks.remove(&i) { + st.bytes -= <[u8]>::len(&data) as u64; + self.counters.evictions.fetch_add(1, Ordering::Relaxed); + } + } + } + + /// The blocks `wanted` (sorted, distinct), fetching the missing ones. + fn blocks(&self, wanted: &[u64], may_keep: bool) -> Result, FormatError> { + let mut have = HashMap::with_capacity(wanted.len()); + let mut waits = Vec::new(); + let mut guard = FlightGuard { + cache: self, + flights: Vec::new(), + }; + { + let mut st = lock(&self.state); + for &i in wanted { + match st.blocks.get(&i) { + Some(Slot::Ready { data, tick }) => { + let (data, old) = (data.clone(), *tick); + st.tick += 1; + let tick = st.tick; + st.lru.remove(&old); + st.lru.insert(tick, i); + if let Some(Slot::Ready { tick: t, .. }) = st.blocks.get_mut(&i) { + *t = tick; + } + self.counters.hits.fetch_add(1, Ordering::Relaxed); + have.insert(i, data); + } + Some(Slot::Pending(f)) => { + self.counters.misses.fetch_add(1, Ordering::Relaxed); + self.counters.waits.fetch_add(1, Ordering::Relaxed); + waits.push((i, f.clone())); + } + None => { + self.counters.misses.fetch_add(1, Ordering::Relaxed); + let f = Flight::new(); + st.blocks.insert(i, Slot::Pending(f.clone())); + guard.flights.push((i, f)); + } + } + } + // Fill small gaps between runs with blocks nobody has, so the + // runs merge into one request. + if self.config.coalesce_gap > 0 && guard.flights.len() > 1 { + let gap_blocks = self.config.coalesce_gap / self.config.block_size; + let mut extra = Vec::new(); + for w in guard.flights.windows(2) { + let (a, b) = (w[0].0, w[1].0); + let gap = b - a - 1; + if gap == 0 || gap > gap_blocks { + continue; + } + if (a + 1..b).all(|j| !st.blocks.contains_key(&j)) { + extra.extend(a + 1..b); + } + } + for j in extra { + let f = Flight::new(); + st.blocks.insert(j, Slot::Pending(f.clone())); + guard.flights.push((j, f)); + } + guard.flights.sort_by_key(|(i, _)| *i); + } + } + + if !guard.flights.is_empty() { + let runs = self.runs(&guard.flights); + let missing_bytes: u64 = runs.iter().map(|r| r.end - r.start).sum(); + let keep = may_keep && missing_bytes <= self.config.capacity / 2; + self.counters.fetch_calls.fetch_add(1, Ordering::Relaxed); + self.counters + .requests + .fetch_add(runs.len() as u64, Ordering::Relaxed); + let fetched = self.inner.read_ranges(&runs)?; + if fetched.len() != runs.len() { + return Err(FormatError::Storage(format!( + "backend returned {} ranges for {} requested", + fetched.len(), + runs.len() + ))); + } + let mut got: Vec<(u64, Block)> = Vec::with_capacity(guard.flights.len()); + for (run, bytes) in runs.iter().zip(&fetched) { + self.counters + .bytes_fetched + .fetch_add(bytes.len() as u64, Ordering::Relaxed); + if bytes.len() as u64 != run.end - run.start { + return Err(FormatError::Storage(format!( + "short read from the backend: {} of {} bytes at offset {}", + bytes.len(), + run.end - run.start, + run.start + ))); + } + let bs = self.config.block_size; + let mut i = run.start / bs; + let mut pos = 0usize; + while pos < bytes.len() { + let n = self.block_len(i) as usize; + got.push((i, Arc::from(&bytes[pos..pos + n]))); + pos += n; + i += 1; + } + } + drop(fetched); + let flights = std::mem::take(&mut guard.flights); + let mut st = lock(&self.state); + let mut by_index: HashMap = got.into_iter().collect(); + for (i, f) in &flights { + let Some(data) = by_index.remove(i) else { + continue; + }; + let ours = matches!(st.blocks.get(i), Some(Slot::Pending(p)) if Arc::ptr_eq(p, f)); + if ours { + if keep { + self.keep(&mut st, *i, data.clone()); + } else { + st.blocks.remove(i); + } + } + f.finish(Ok(data.clone())); + have.insert(*i, data); + } + self.evict(&mut st); + drop(st); + // Any flight without data (cannot happen: every run is split + // into its blocks) is failed rather than left waiting. + guard.flights = flights + .into_iter() + .filter(|(i, _)| !have.contains_key(i)) + .collect(); + if !guard.flights.is_empty() { + return Err(FormatError::Storage( + "backend did not return every block".into(), + )); + } + } + + for (i, f) in waits { + match f.wait() { + Ok(data) => { + have.insert(i, data); + } + Err(e) => return Err(FormatError::Storage(e)), + } + } + Ok(have) + } + + /// Byte ranges of the runs of consecutive blocks in `flights` (sorted), + /// each at most `max_request` long. + fn runs(&self, flights: &[(u64, Arc)]) -> Vec> { + let bs = self.config.block_size; + let per_request = self.config.max_request / bs; + let mut runs: Vec<(u64, u64)> = Vec::new(); + for &(i, _) in flights { + match runs.last_mut() { + Some((first, last)) if *last + 1 == i && i - *first < per_request => *last = i, + _ => runs.push((i, i)), + } + } + runs.into_iter() + .map(|(a, b)| a * bs..(b * bs + self.block_len(b))) + .collect() + } + + /// Copy `[offset, end)` out of `blocks`. + fn assemble(&self, offset: u64, end: u64, blocks: &HashMap) -> Vec { + let bs = self.config.block_size; + let mut out = Vec::with_capacity((end - offset) as usize); + let mut pos = offset; + while pos < end { + let i = pos / bs; + let block = &blocks[&i]; + let from = (pos - i * bs) as usize; + let to = ((end - i * bs) as usize).min(<[u8]>::len(block)); + out.extend_from_slice(&block[from..to]); + pos = i * bs + to as u64; + } + out + } +} + +impl Storage for BlockCache { + fn read_at(&self, offset: u64, len: usize) -> Result, FormatError> { + if let Some(all) = self.inner.as_contiguous() { + return all.read_at(offset, len); + } + self.counters.reads.fetch_add(1, Ordering::Relaxed); + let Some(span) = self.block_span(offset, len as u64) else { + return Ok(Cow::Owned(Vec::new())); + }; + let end = offset.saturating_add(len as u64).min(self.len); + let blocks = self.blocks(&span.collect::>(), true)?; + Ok(Cow::Owned(self.assemble(offset, end, &blocks))) + } + + fn len(&self) -> u64 { + self.len + } + + fn read_ranges(&self, ranges: &[Range]) -> Result>, FormatError> { + if let Some(all) = self.inner.as_contiguous() { + return all.read_ranges(ranges); + } + self.counters + .reads + .fetch_add(ranges.len() as u64, Ordering::Relaxed); + let mut wanted = BTreeSet::new(); + for r in ranges { + if r.end < r.start { + return Err(FormatError::Storage( + "read range ends before it starts".into(), + )); + } + if let Some(span) = self.block_span(r.start, r.end - r.start) { + wanted.extend(span); + } + } + let wanted: Vec = wanted.into_iter().collect(); + let blocks = self.blocks(&wanted, true)?; + Ok(ranges + .iter() + .map(|r| { + let end = r.end.min(self.len); + if r.start >= end { + Cow::Owned(Vec::new()) + } else { + Cow::Owned(self.assemble(r.start, end, &blocks)) + } + }) + .collect()) + } + + fn as_contiguous(&self) -> Option<&[u8]> { + self.inner.as_contiguous() + } +} + +#[cfg(test)] +mod tests { + use super::*; + use clawhdf5_format::storage::CountingStorage; + + fn data(n: usize) -> Vec { + (0..n).map(|i| (i * 7 + i / 251) as u8).collect() + } + + fn cache(n: usize, config: CacheConfig) -> (Vec, BlockCache) { + let d = data(n); + (d.clone(), BlockCache::new(CountingStorage::new(d), config)) + } + + fn small() -> CacheConfig { + CacheConfig { + block_size: 1024, + capacity: 8 * 1024, + coalesce_gap: 0, + max_request: 4 * 1024, + } + } + + #[test] + fn reads_match_the_data_and_hit_the_cache() { + let mut cfg = small(); + cfg.capacity = 64 * 1024; + let (d, c) = cache(10_000, cfg); + for (off, len) in [ + (0, 10), + (1000, 100), + (1020, 10), + (9990, 100), + (20_000, 5), + (0, 10_000), + ] { + let got = c.read_at(off, len).unwrap(); + let s = (off as usize).min(d.len()); + let e = (off as usize + len).min(d.len()); + assert_eq!(&*got, &d[s..e], "{off} {len}"); + } + let before = c.inner().reads(); + assert_eq!(&*c.read_at(5000, 3000).unwrap(), &d[5000..8000]); + assert_eq!(c.inner().reads(), before, "all cached"); + assert!(c.stats().hits > 0); + } + + #[test] + fn read_ranges_coalesces_consecutive_missing_blocks() { + let (d, c) = cache(20_000, small()); + let ranges = [100..200, 1500..1600, 2100..3000, 9000..9100]; + let got = c.read_ranges(&ranges).unwrap(); + for (r, g) in ranges.iter().zip(&got) { + assert_eq!(&**g, &d[r.start as usize..r.end as usize]); + } + // Blocks 0-2 are one run, block 8 another: one backend call, two ranges. + assert_eq!(c.stats().fetch_calls, 1); + assert_eq!(c.stats().requests, 2); + assert_eq!(c.inner().reads(), 2); + } + + #[test] + fn a_small_gap_is_fetched_to_merge_runs() { + let mut cfg = small(); + cfg.coalesce_gap = 1024; + let (d, c) = cache(20_000, cfg); + let ranges = [0..10, 2048..2058, 5000..5010]; + let got = c.read_ranges(&ranges).unwrap(); + for (r, g) in ranges.iter().zip(&got) { + assert_eq!(&**g, &d[r.start as usize..r.end as usize]); + } + // Blocks 0 and 2 merge over block 1; block 4 is two blocks away. + assert_eq!(c.stats().requests, 2); + } + + #[test] + fn long_runs_are_split_at_max_request() { + let (d, c) = cache(20_000, small()); + assert_eq!(&*c.read_at(0, 10_000).unwrap(), &d[..10_000]); + // 10 blocks, 4 per request: 3 requests in one call. + assert_eq!(c.stats().requests, 3); + assert_eq!(c.stats().fetch_calls, 1); + } + + #[test] + fn lru_stays_within_budget_and_evicts_oldest() { + let (d, c) = cache(40_000, small()); + for i in 0..8u64 { + c.read_at(i * 1024, 1).unwrap(); + } + c.read_at(0, 1).unwrap(); // block 0 is now the newest + c.read_at(8 * 1024, 1).unwrap(); // evicts block 1 + let s = c.stats(); + assert!(s.cached_bytes <= 8 * 1024); + assert_eq!(s.evictions, 1); + let before = c.inner().reads(); + c.read_at(0, 1).unwrap(); + assert_eq!(c.inner().reads(), before, "block 0 kept"); + assert_eq!(&*c.read_at(1024, 5).unwrap(), &d[1024..1029]); + assert_eq!(c.inner().reads(), before + 1, "block 1 refetched"); + } + + #[test] + fn large_reads_do_not_flush_the_cache() { + let (d, c) = cache(40_000, small()); + c.read_at(0, 1).unwrap(); + assert_eq!(&*c.read_at(10_000, 20_000).unwrap(), &d[10_000..30_000]); + assert_eq!(c.stats().cached_bytes, 1024, "only block 0 kept"); + } + + #[test] + fn insert_keeps_whole_blocks_only() { + let (d, c) = cache(5000, small()); + c.insert(0, &d[..2500]); + assert_eq!(c.stats().cached_bytes, 2048); + c.insert(4096, &d[4096..]); + assert_eq!(c.stats().cached_bytes, 2048 + 904); + assert_eq!(&*c.read_at(0, 2048).unwrap(), &d[..2048]); + assert_eq!(&*c.read_at(4500, 500).unwrap(), &d[4500..]); + assert_eq!(c.inner().reads(), 0); + } + + /// A backend that fails its first `n` fetches. + struct Flaky { + data: Vec, + fail: AtomicU64, + } + + impl Storage for Flaky { + fn read_at(&self, offset: u64, len: usize) -> Result, FormatError> { + if self + .fail + .fetch_update(Ordering::SeqCst, Ordering::SeqCst, |n| n.checked_sub(1)) + .is_ok() + { + return Err(FormatError::Storage("boom".into())); + } + self.data + .as_slice() + .read_at(offset, len) + .map(|c| Cow::Owned(c.into_owned())) + } + fn len(&self) -> u64 { + self.data.len() as u64 + } + } + + #[test] + fn a_failed_fetch_is_an_error_and_is_not_cached() { + let d = data(5000); + let c = BlockCache::new( + Flaky { + data: d.clone(), + fail: AtomicU64::new(1), + }, + small(), + ); + assert!(c.read_at(0, 10).is_err()); + assert_eq!(&*c.read_at(0, 10).unwrap(), &d[..10]); + } + + /// A backend that returns fewer bytes than asked inside the file. + struct Short(Vec); + + impl Storage for Short { + fn read_at(&self, offset: u64, len: usize) -> Result, FormatError> { + let got = self.0.as_slice().read_at(offset, len)?; + Ok(Cow::Owned(got[..got.len() / 2].to_vec())) + } + fn len(&self) -> u64 { + self.0.len() as u64 + } + } + + #[test] + fn a_short_backend_read_is_an_error() { + let c = BlockCache::new(Short(data(5000)), small()); + assert!(c.read_at(0, 10).is_err()); + assert!(c.read_at(0, 10).is_err(), "not cached"); + } + + #[test] + fn contiguous_backends_pass_through() { + let d = data(5000); + let c = BlockCache::new(d.clone(), small()); + assert_eq!(c.as_contiguous(), Some(&d[..])); + assert!(matches!(c.read_at(0, 10).unwrap(), Cow::Borrowed(_))); + assert_eq!(c.stats().requests, 0); + } + + /// A slow backend: concurrent readers of the same blocks share fetches. + struct Slow { + data: Vec, + fetched: Mutex>>, + } + + impl Storage for Slow { + fn read_at(&self, offset: u64, len: usize) -> Result, FormatError> { + std::thread::sleep(std::time::Duration::from_millis(30)); + lock(&self.fetched).push(offset..offset + len as u64); + self.data + .as_slice() + .read_at(offset, len) + .map(|c| Cow::Owned(c.into_owned())) + } + fn len(&self) -> u64 { + self.data.len() as u64 + } + } + + #[test] + fn concurrent_readers_never_fetch_a_block_twice() { + let d = data(64 * 1024); + let cfg = CacheConfig { + block_size: 1024, + capacity: 1 << 20, + coalesce_gap: 0, + max_request: 1024, + }; + let c = BlockCache::new( + Slow { + data: d.clone(), + fetched: Mutex::new(Vec::new()), + }, + cfg, + ); + std::thread::scope(|s| { + for t in 0..8u64 { + let (c, d) = (&c, &d); + s.spawn(move || { + for k in 0..16u64 { + let off = ((k * 3 + t) % 32) * 1024 + 100; + let got = c.read_at(off, 2000).unwrap(); + assert_eq!(&*got, &d[off as usize..off as usize + 2000]); + } + }); + } + }); + let fetched = lock(&c.inner().fetched).clone(); + let mut blocks: Vec = fetched.iter().map(|r| r.start / 1024).collect(); + let n = blocks.len(); + blocks.sort_unstable(); + blocks.dedup(); + assert_eq!(n, blocks.len(), "a block was fetched twice: {fetched:?}"); + assert!(c.stats().waits > 0, "readers should have shared fetches"); + } +} diff --git a/crates/clawhdf5-remote/src/error.rs b/crates/clawhdf5-remote/src/error.rs new file mode 100644 index 0000000..d5d1feb --- /dev/null +++ b/crates/clawhdf5-remote/src/error.rs @@ -0,0 +1,125 @@ +//! Errors of the remote backends. + +use clawhdf5_format::error::FormatError; + +/// Why a remote file could not be opened or read. +/// +/// Inside a [`clawhdf5::File`] read these arrive as +/// `clawhdf5::Error::Format(FormatError::Storage(message))`, the message +/// being this error's `Display`. +#[derive(Debug, Clone, PartialEq, Eq)] +#[non_exhaustive] +pub enum RemoteError { + /// The URL is malformed. + InvalidUrl(String), + /// The URL's scheme is not supported by this build (for example + /// `s3://` without the `s3` feature, or `https://` without `https`). + UnsupportedScheme(String), + /// The server answered a range request with the whole file (status + /// 200), i.e. it does not support ranges, and a full download was not + /// allowed ([`HttpOptions::allow_full_download`](crate::HttpOptions)). + RangeNotSupported(String), + /// The file changed since it was opened (a different ETag, + /// Last-Modified or length, or a failed `If-Match` precondition). + /// Nothing read after the change is returned. + FileChanged(String), + /// The server answered with an unexpected status. + Status { + /// The HTTP status code. + code: u16, + /// What was being requested. + what: String, + }, + /// A response did not carry what was asked for (a wrong + /// `Content-Range`, a body shorter or longer than announced), after + /// every retry. + BadResponse(String), + /// A network failure (connection, timeout, reset) after every retry. + Transport(String), + /// An error from the object store. + ObjectStore(String), + /// Called in a way the backend cannot serve (for example a blocking + /// read from inside an async runtime). + Usage(String), +} + +impl RemoteError { + /// Whether retrying the same request may succeed. + #[cfg_attr(not(feature = "http"), allow(dead_code))] + pub(crate) fn is_transient(&self) -> bool { + match self { + RemoteError::Transport(_) | RemoteError::BadResponse(_) => true, + RemoteError::Status { code, .. } => { + matches!(code, 408 | 429 | 500 | 502 | 503 | 504) + } + _ => false, + } + } +} + +impl std::fmt::Display for RemoteError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + RemoteError::InvalidUrl(s) => write!(f, "invalid URL: {s}"), + RemoteError::UnsupportedScheme(s) => write!(f, "unsupported URL: {s}"), + RemoteError::RangeNotSupported(s) => { + write!(f, "the server does not support range requests: {s}") + } + RemoteError::FileChanged(s) => write!(f, "the remote file changed while open: {s}"), + RemoteError::Status { code, what } => write!(f, "HTTP status {code} for {what}"), + RemoteError::BadResponse(s) => write!(f, "bad response: {s}"), + RemoteError::Transport(s) => write!(f, "network error: {s}"), + RemoteError::ObjectStore(s) => write!(f, "object store: {s}"), + RemoteError::Usage(s) => write!(f, "{s}"), + } + } +} + +impl std::error::Error for RemoteError {} + +impl From for FormatError { + fn from(e: RemoteError) -> Self { + FormatError::Storage(e.to_string()) + } +} + +/// An error of [`open_url`](crate::open_url): the remote side, or the file +/// itself. +#[derive(Debug)] +#[non_exhaustive] +pub enum Error { + /// Reaching or reading the remote file failed. + Remote(RemoteError), + /// The bytes were read, but they are not an HDF5 file clawhdf5 can open. + Hdf5(clawhdf5::Error), +} + +impl std::fmt::Display for Error { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Error::Remote(e) => e.fmt(f), + Error::Hdf5(e) => e.fmt(f), + } + } +} + +impl std::error::Error for Error { + fn source(&self) -> Option<&(dyn std::error::Error + 'static)> { + match self { + Error::Remote(e) => Some(e), + Error::Hdf5(e) => Some(e), + } + } +} + +impl From for Error { + fn from(e: RemoteError) -> Self { + Error::Remote(e) + } +} + +impl From for Error { + fn from(e: clawhdf5::Error) -> Self { + Error::Hdf5(e) + } +} diff --git a/crates/clawhdf5-remote/src/http.rs b/crates/clawhdf5-remote/src/http.rs new file mode 100644 index 0000000..ce3d68f --- /dev/null +++ b/crates/clawhdf5-remote/src/http.rs @@ -0,0 +1,565 @@ +//! HTTP(S) range requests: [`HttpStorage`]. +//! +//! Every read is a `GET` with a `Range: bytes=a-b` header, answered `206 +//! Partial Content`. The file is pinned when it is opened: +//! +//! - its length comes from the `Content-Range` of the first request (which +//! also fetches the first block, so opening costs one request); +//! - a strong `ETag` is sent back as `If-Match` on every later request, and +//! compared with the `ETag` of every response; without one, `Last-Modified` +//! is sent as `If-Unmodified-Since` and compared; the length in every +//! `Content-Range` must stay the same. A file that changes while it is +//! open is [`RemoteError::FileChanged`], never a mix of old and new bytes. +//! (A server that sends neither validator cannot be checked beyond the +//! length; [`HttpOptions::require_validator`] refuses such servers.) +//! - a server that ignores `Range` and answers `200` with the whole file is +//! refused with [`RemoteError::RangeNotSupported`], unless +//! [`HttpOptions::allow_full_download`] is set: then the file is +//! downloaded once, at open, and read from memory. +//! +//! Transient failures — connection errors, timeouts, `408`/`429`/`5xx`, and +//! a body shorter or longer than its `Content-Range` — are retried with +//! exponential backoff. Responses are requested with +//! `Accept-Encoding: identity`, since a compressed body cannot be a byte +//! range of the file. +//! +//! `HttpStorage` itself does not cache: each `read_at` is one request. Read +//! it through [`BlockCache`](crate::BlockCache) (which [`open_url`](crate::open_url) +//! does); its `read_ranges` fetches the ranges of one call in parallel. + +use std::borrow::Cow; +use std::io::Read; +use std::ops::Range; +use std::sync::atomic::{AtomicU64, AtomicUsize, Ordering}; +use std::time::Duration; + +use clawhdf5_format::error::FormatError; +use clawhdf5_format::storage::Storage; + +use crate::error::RemoteError; + +/// Settings of an [`HttpStorage`]. +#[derive(Debug, Clone)] +pub struct HttpOptions { + /// Retries of a request that failed transiently (so up to `retries + 1` + /// attempts). + pub retries: u32, + /// Delay before the first retry; doubled for each further one. + pub backoff: Duration, + /// Timeout of one request, from connecting to the end of the body. + pub timeout: Duration, + /// Requests of one `read_ranges` call in flight at once. + pub max_parallel: usize, + /// Bytes fetched by the first request, from offset 0 (the superblock and + /// usually the root group's metadata); at least 1. + pub first_request: u64, + /// When the server ignores `Range` (answers `200`), download the whole + /// file once and read it from memory, instead of failing. + pub allow_full_download: bool, + /// Largest file [`allow_full_download`](Self::allow_full_download) will + /// download. + pub max_full_download: u64, + /// Refuse a server that sends neither a strong `ETag` nor + /// `Last-Modified`, since a change of the file could then go unnoticed + /// (only its length is checked). + pub require_validator: bool, + /// Extra headers sent with every request (for example + /// `Authorization`). + pub headers: Vec<(String, String)>, +} + +impl Default for HttpOptions { + fn default() -> Self { + HttpOptions { + retries: 3, + backoff: Duration::from_millis(200), + timeout: Duration::from_secs(60), + max_parallel: 8, + first_request: crate::cache::DEFAULT_BLOCK_SIZE, + allow_full_download: false, + max_full_download: 1 << 30, + require_validator: false, + headers: Vec::new(), + } + } +} + +/// Requests and bytes an [`HttpStorage`] has used. +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] +pub struct HttpStats { + /// HTTP requests sent (retries included). + pub requests: u64, + /// Requests that were retries. + pub retries: u64, + /// Response body bytes received. + pub bytes: u64, +} + +/// How the file is pinned. +#[derive(Debug, Clone, PartialEq, Eq)] +enum Validator { + ETag(String), + LastModified(String), + None, +} + +/// An HTTP(S) file read by range requests. See the [module +/// documentation](self). +pub struct HttpStorage { + agent: ureq::Agent, + url: String, + len: u64, + validator: Validator, + options: HttpOptions, + /// The whole file, when the server ignores ranges and a full download + /// was allowed. + full: Option>, + requests: AtomicU64, + retries: AtomicU64, + bytes: AtomicU64, +} + +impl std::fmt::Debug for HttpStorage { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("HttpStorage") + .field("url", &self.url) + .field("len", &self.len) + .field("validator", &self.validator) + .field("full_download", &self.full.is_some()) + .finish() + } +} + +/// A parsed `Content-Range: bytes a-b/total`. +fn content_range(v: &str) -> Option<(u64, u64, Option)> { + let rest = v.trim().strip_prefix("bytes")?.trim_start(); + let (span, total) = rest.split_once('/')?; + let (a, b) = span.trim().split_once('-')?; + let a: u64 = a.trim().parse().ok()?; + let b: u64 = b.trim().parse().ok()?; + if b < a { + return None; + } + let total = match total.trim() { + "*" => None, + t => Some(t.parse().ok()?), + }; + Some((a, b, total)) +} + +fn header<'a>(resp: &'a ureq::http::Response, name: &str) -> Option<&'a str> { + resp.headers().get(name).and_then(|v| v.to_str().ok()) +} + +/// A body with a `Content-Encoding` is not a byte range of the file. +fn check_identity(url: &str, resp: &ureq::http::Response) -> Result<(), RemoteError> { + match header(resp, "content-encoding") { + Some(enc) if !enc.trim().eq_ignore_ascii_case("identity") => { + Err(RemoteError::Usage(format!( + "{url}: the server sent a {enc}-encoded body despite Accept-Encoding: identity" + ))) + } + _ => Ok(()), + } +} + +fn transport(e: ureq::Error) -> RemoteError { + match e { + ureq::Error::StatusCode(code) => RemoteError::Status { + code, + what: "request".into(), + }, + ureq::Error::BadUri(s) => RemoteError::InvalidUrl(s), + other => RemoteError::Transport(other.to_string()), + } +} + +impl HttpStorage { + /// Open `url` (`http://`, or `https://` with the `https` feature): + /// one ranged `GET` of the first [`HttpOptions::first_request`] bytes, + /// which gives the file's length and validators. Returns the storage + /// and the bytes that request fetched (the file's start), for a + /// [`BlockCache`](crate::BlockCache) to keep. + pub fn open(url: &str, options: HttpOptions) -> Result<(HttpStorage, Vec), RemoteError> { + let lower = url.to_ascii_lowercase(); + if lower.starts_with("https://") { + if !cfg!(feature = "https") { + return Err(RemoteError::UnsupportedScheme(format!( + "{url}: https:// needs the `https` feature of clawhdf5-remote" + ))); + } + } else if !lower.starts_with("http://") { + return Err(RemoteError::UnsupportedScheme(url.to_string())); + } + let config = ureq::Agent::config_builder() + .http_status_as_error(false) + .timeout_global(Some(options.timeout)) + .build(); + let mut storage = HttpStorage { + agent: ureq::Agent::new_with_config(config), + url: url.to_string(), + len: 0, + validator: Validator::None, + options, + full: None, + requests: AtomicU64::new(0), + retries: AtomicU64::new(0), + bytes: AtomicU64::new(0), + }; + let first = storage.with_retries(|| storage.probe())?; + let (len, validator, bytes, full) = first; + storage.len = len; + storage.validator = validator; + if full { + storage.full = Some(bytes); + return Ok((storage, Vec::new())); + } + if storage.options.require_validator && storage.validator == Validator::None { + return Err(RemoteError::Usage(format!( + "{url}: the server sends neither a strong ETag nor Last-Modified, so a change \ + of the file could not be detected (HttpOptions::require_validator)" + ))); + } + Ok((storage, bytes)) + } + + /// The URL. + pub fn url(&self) -> &str { + &self.url + } + + /// Whether the file was downloaded whole, because the server does not + /// support ranges and [`HttpOptions::allow_full_download`] was set. + pub fn is_full_download(&self) -> bool { + self.full.is_some() + } + + /// The `ETag` the file is pinned to, if the server sent a strong one. + pub fn etag(&self) -> Option<&str> { + match &self.validator { + Validator::ETag(e) => Some(e), + _ => None, + } + } + + /// Requests and bytes so far. + pub fn stats(&self) -> HttpStats { + HttpStats { + requests: self.requests.load(Ordering::Relaxed), + retries: self.retries.load(Ordering::Relaxed), + bytes: self.bytes.load(Ordering::Relaxed), + } + } + + fn with_retries( + &self, + mut attempt: impl FnMut() -> Result, + ) -> Result { + let mut delay = self.options.backoff; + let mut n = 0; + loop { + match attempt() { + Ok(v) => return Ok(v), + Err(e) if e.is_transient() && n < self.options.retries => { + n += 1; + self.retries.fetch_add(1, Ordering::Relaxed); + std::thread::sleep(delay); + delay = delay.saturating_mul(2); + } + Err(e) => return Err(e), + } + } + } + + fn request( + &self, + range: Option<(u64, u64)>, + ) -> ureq::RequestBuilder { + let mut req = self + .agent + .get(&self.url) + .header("Accept-Encoding", "identity"); + if let Some((a, b)) = range { + req = req.header("Range", format!("bytes={a}-{b}")); + } + match &self.validator { + Validator::ETag(e) => req = req.header("If-Match", e), + Validator::LastModified(t) => req = req.header("If-Unmodified-Since", t), + Validator::None => {} + } + for (k, v) in &self.options.headers { + req = req.header(k, v); + } + req + } + + /// Read a body of exactly `want` bytes (or up to `limit` when `want` is + /// unknown). + fn body( + &self, + resp: ureq::http::Response, + want: Option, + limit: u64, + ) -> Result, RemoteError> { + let cap = want.unwrap_or(limit); + let mut buf = Vec::with_capacity(usize::try_from(cap.min(64 << 20)).unwrap_or(0)); + let reader = resp.into_body().into_reader(); + let got = reader + .take(cap.saturating_add(1)) + .read_to_end(&mut buf) + .map_err(|e| RemoteError::Transport(format!("{}: reading the body: {e}", self.url))); + self.bytes.fetch_add(buf.len() as u64, Ordering::Relaxed); + got?; + match want { + Some(n) if buf.len() as u64 != n => Err(RemoteError::BadResponse(format!( + "{}: body of {} bytes, expected {n}", + self.url, + buf.len() + ))), + None if buf.len() as u64 > limit => Err(RemoteError::Usage(format!( + "{}: the file is larger than HttpOptions::max_full_download ({limit} bytes)", + self.url + ))), + _ => Ok(buf), + } + } + + /// The first request: length, validators and the file's first bytes. + /// The last field is true when the server ignored the range and sent + /// the whole file (only kept when a full download is allowed). + fn probe(&self) -> Result<(u64, Validator, Vec, bool), RemoteError> { + let n = self.options.first_request.max(1); + self.requests.fetch_add(1, Ordering::Relaxed); + let resp = self.request(Some((0, n - 1))).call().map_err(transport)?; + let status = resp.status().as_u16(); + check_identity(&self.url, &resp)?; + let validator = match (header(&resp, "etag"), header(&resp, "last-modified")) { + (Some(e), _) if !e.starts_with("W/") => Validator::ETag(e.to_string()), + (_, Some(t)) => Validator::LastModified(t.to_string()), + _ => Validator::None, + }; + match status { + 206 => { + let cr = header(&resp, "content-range").ok_or_else(|| { + RemoteError::BadResponse(format!("{}: 206 without Content-Range", self.url)) + })?; + let (a, b, total) = content_range(cr).ok_or_else(|| { + RemoteError::BadResponse(format!("{}: bad Content-Range {cr:?}", self.url)) + })?; + let total = total.ok_or_else(|| { + RemoteError::BadResponse(format!( + "{}: the server does not report the file's length (Content-Range {cr:?})", + self.url + )) + })?; + if a != 0 || b >= total || b > n - 1 { + return Err(RemoteError::BadResponse(format!( + "{}: asked for bytes 0-{}, got Content-Range {cr:?}", + self.url, + n - 1 + ))); + } + let bytes = self.body(resp, Some(b - a + 1), 0)?; + Ok((total, validator, bytes, false)) + } + 200 => { + if !self.options.allow_full_download { + return Err(RemoteError::RangeNotSupported(format!( + "{} answered a range request with the whole file (status 200); set \ + HttpOptions::allow_full_download to download it", + self.url + ))); + } + let want = header(&resp, "content-length").and_then(|v| v.trim().parse().ok()); + if want.is_some_and(|w: u64| w > self.options.max_full_download) { + return Err(RemoteError::Usage(format!( + "{}: the file is larger than HttpOptions::max_full_download ({} bytes)", + self.url, self.options.max_full_download + ))); + } + let bytes = self.body(resp, want, self.options.max_full_download)?; + Ok((bytes.len() as u64, validator, bytes, true)) + } + 416 => Err(RemoteError::Usage(format!( + "{}: status 416 for the first bytes (an empty file?)", + self.url + ))), + code => Err(RemoteError::Status { + code, + what: self.url.clone(), + }), + } + } + + /// One request for `[start, end)` (inside the file, non-empty). + fn fetch_once(&self, start: u64, end: u64) -> Result, RemoteError> { + self.requests.fetch_add(1, Ordering::Relaxed); + let resp = self + .request(Some((start, end - 1))) + .call() + .map_err(transport)?; + let changed = |why: String| RemoteError::FileChanged(format!("{}: {why}", self.url)); + check_identity(&self.url, &resp)?; + match resp.status().as_u16() { + 206 => {} + 200 => { + return Err(RemoteError::RangeNotSupported(format!( + "{} answered a range request with the whole file (status 200)", + self.url + ))); + } + 412 => { + return Err(changed( + "If-Match/If-Unmodified-Since failed (status 412)".into(), + )); + } + 416 => return Err(changed("range no longer satisfiable (status 416)".into())), + code => { + return Err(RemoteError::Status { + code, + what: format!("{} bytes {start}-{}", self.url, end - 1), + }); + } + } + match &self.validator { + Validator::ETag(e) => { + if let Some(got) = header(&resp, "etag") + && got != e + { + return Err(changed(format!("ETag {got} instead of {e}"))); + } + } + Validator::LastModified(t) => { + if let Some(got) = header(&resp, "last-modified") + && got != t + { + return Err(changed(format!("Last-Modified {got} instead of {t}"))); + } + } + Validator::None => {} + } + let cr = header(&resp, "content-range").ok_or_else(|| { + RemoteError::BadResponse(format!("{}: 206 without Content-Range", self.url)) + })?; + let (a, b, total) = content_range(cr).ok_or_else(|| { + RemoteError::BadResponse(format!("{}: bad Content-Range {cr:?}", self.url)) + })?; + if let Some(total) = total + && total != self.len + { + return Err(changed(format!("length {total} instead of {}", self.len))); + } + if a != start || b != end - 1 { + return Err(RemoteError::BadResponse(format!( + "{}: asked for bytes {start}-{}, got Content-Range {cr:?}", + self.url, + end - 1 + ))); + } + self.body(resp, Some(end - start), 0) + } + + fn fetch(&self, start: u64, end: u64) -> Result, RemoteError> { + self.with_retries(|| self.fetch_once(start, end)) + } + + /// `[offset, offset + len)` clamped to the file, or `None` if empty. + fn clamp(&self, offset: u64, len: u64) -> Option<(u64, u64)> { + let end = offset.saturating_add(len).min(self.len); + (offset < end).then_some((offset, end)) + } +} + +impl Storage for HttpStorage { + fn read_at(&self, offset: u64, len: usize) -> Result, FormatError> { + if let Some(all) = &self.full { + return all.as_slice().read_at(offset, len); + } + match self.clamp(offset, len as u64) { + None => Ok(Cow::Owned(Vec::new())), + Some((a, b)) => Ok(Cow::Owned(self.fetch(a, b)?)), + } + } + + fn len(&self) -> u64 { + self.len + } + + fn read_ranges(&self, ranges: &[Range]) -> Result>, FormatError> { + if let Some(all) = &self.full { + return all.as_slice().read_ranges(ranges); + } + let parallel = self.options.max_parallel.clamp(1, ranges.len().max(1)); + if parallel <= 1 { + return ranges + .iter() + .map(|r| self.read_at(r.start, (r.end.saturating_sub(r.start)) as usize)) + .collect(); + } + let next = AtomicUsize::new(0); + let failed = std::sync::atomic::AtomicBool::new(false); + type Slot = Option, RemoteError>>; + let results: std::sync::Mutex> = + std::sync::Mutex::new((0..ranges.len()).map(|_| None).collect()); + std::thread::scope(|s| { + for _ in 0..parallel { + s.spawn(|| { + loop { + let i = next.fetch_add(1, Ordering::Relaxed); + if i >= ranges.len() || failed.load(Ordering::Relaxed) { + break; + } + let r = &ranges[i]; + let got = match self.clamp(r.start, r.end.saturating_sub(r.start)) { + None => Ok(Vec::new()), + Some((a, b)) => self.fetch(a, b), + }; + if got.is_err() { + failed.store(true, Ordering::Relaxed); + } + results + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner)[i] = Some(got); + } + }); + } + }); + let results = results + .into_inner() + .unwrap_or_else(std::sync::PoisonError::into_inner); + let mut out = Vec::with_capacity(ranges.len()); + for r in results { + match r { + Some(Ok(v)) => out.push(Cow::Owned(v)), + Some(Err(e)) => return Err(e.into()), + // Not fetched because another range failed first. + None => continue, + } + } + if out.len() != ranges.len() { + return Err(FormatError::Storage(format!( + "{}: a parallel range read failed", + self.url + ))); + } + Ok(out) + } + + fn as_contiguous(&self) -> Option<&[u8]> { + self.full.as_deref() + } +} + +#[cfg(test)] +mod tests { + use super::content_range; + + #[test] + fn content_range_parses() { + assert_eq!(content_range("bytes 0-99/1000"), Some((0, 99, Some(1000)))); + assert_eq!(content_range("bytes 5-5/*"), Some((5, 5, None))); + assert_eq!(content_range("bytes 9-5/10"), None); + assert_eq!(content_range("items 0-1/2"), None); + assert_eq!(content_range("bytes */1000"), None); + } +} diff --git a/crates/clawhdf5-remote/src/lib.rs b/crates/clawhdf5-remote/src/lib.rs new file mode 100644 index 0000000..597fb63 --- /dev/null +++ b/crates/clawhdf5-remote/src/lib.rs @@ -0,0 +1,127 @@ +//! Read HDF5 files where they are — on an HTTP(S) server or in an object +//! store — without downloading them first. +//! +//! This is milestone M3 of `docs/design/range-reads.md`: remote backends for +//! [`clawhdf5::File::open_storage`], each read through a mandatory +//! [`BlockCache`]. +//! +//! ```no_run +//! let file = clawhdf5_remote::open_url("http://127.0.0.1:8000/data.h5")?; +//! let temperature = file.dataset("/grid/temperature")?.read_f64()?; +//! # Ok::<(), Box>(()) +//! ``` +//! +//! - [`open_url`] / [`open_url_with`]: `http://` (default feature `http`), +//! `https://` (feature `https`) → a [`clawhdf5::File`] with the whole +//! read API. +//! - [`storage_for_url`] gives the cached storage itself, to open with +//! [`clawhdf5::File::open_storage`] and to read its [`CacheStats`]. +//! - [`HttpStorage`] (range `GET`s, pinned by ETag/Last-Modified, retried +//! with backoff), and [`BlockCache`] over any +//! [`Storage`](clawhdf5_format::storage::Storage). +//! +//! What costs what: opening costs one request (it also fetches the first +//! block, 1 MiB by default); listing a file whose metadata sits in its +//! first blocks costs nothing more; reading a chunked dataset costs one +//! parallel batch of requests for the blocks holding its chunk index, then +//! one for its chunks. The zero-copy methods of `clawhdf5` +//! (`read_raw_ref`, `read_*_zerocopy`, `File::as_bytes`) need the file in +//! memory and are errors (`as_bytes` a panic) on a remote file. + +#![warn(missing_docs)] + +pub mod cache; +pub mod error; +#[cfg(feature = "http")] +pub mod http; + +use std::sync::Arc; + +use clawhdf5::File; +use clawhdf5_format::storage::Storage; + +pub use cache::{BlockCache, CacheConfig, CacheStats}; +pub use error::{Error, RemoteError}; +#[cfg(feature = "http")] +pub use http::{HttpOptions, HttpStats, HttpStorage}; + +/// A backend a [`BlockCache`] can read through. +pub type Backend = Box; + +/// The storage [`storage_for_url`] returns: a block cache over the URL's +/// backend. +pub type RemoteStorage = BlockCache; + +/// Settings of [`open_url_with`] and [`storage_for_url`]. +#[derive(Debug, Clone, Default)] +pub struct Options { + /// The block cache. + pub cache: CacheConfig, + /// HTTP(S) requests. + #[cfg(feature = "http")] + pub http: HttpOptions, +} + +/// Open the HDF5 file at `url` with default [`Options`]. +/// +/// `http://…` needs the (default) `http` feature, `https://…` the `https` +/// feature. +pub fn open_url(url: &str) -> Result { + open_url_with(url, &Options::default()) +} + +/// [`open_url`] with explicit [`Options`]. +pub fn open_url_with(url: &str, options: &Options) -> Result { + let storage = storage_for_url(url, options)?; + Ok(File::open_storage(storage)?) +} + +/// The cached storage for `url`, with the first block already fetched: +/// open it with [`clawhdf5::File::open_storage`] (a clone of the `Arc`), +/// and read its [`BlockCache::stats`] as you go. +pub fn storage_for_url(url: &str, options: &Options) -> Result, Error> { + let scheme = url + .split_once("://") + .map(|(s, _)| s.to_ascii_lowercase()) + .ok_or_else(|| RemoteError::InvalidUrl(format!("{url}: no scheme")))?; + match scheme.as_str() { + "http" | "https" => http_storage(url, options), + "s3" | "s3a" | "gs" | "az" | "azure" | "abfs" | "abfss" | "adl" => { + cloud_storage(url, &scheme, options) + } + _ => Err(RemoteError::UnsupportedScheme(url.to_string()).into()), + } +} + +#[cfg(feature = "http")] +fn http_storage(url: &str, options: &Options) -> Result, Error> { + let mut http = options.http.clone(); + http.first_request = http.first_request.max(options.cache.block_size.max(1)); + let (storage, first) = HttpStorage::open(url, http)?; + let cache = BlockCache::new(Box::new(storage) as Backend, options.cache.clone()); + cache.insert(0, &first); + Ok(Arc::new(cache)) +} + +#[cfg(not(feature = "http"))] +fn http_storage(url: &str, _options: &Options) -> Result, Error> { + Err(RemoteError::UnsupportedScheme(format!( + "{url}: http(s):// needs the `http` feature of clawhdf5-remote" + )) + .into()) +} + +fn cloud_storage(url: &str, scheme: &str, _options: &Options) -> Result, Error> { + Err(RemoteError::UnsupportedScheme(format!("{url}: {scheme}:// is not supported yet")).into()) +} + +/// A [`BlockCache`] over `backend` with its first block fetched (readahead +/// of the superblock and the metadata usually written next to it). +pub fn cached(backend: Backend, options: &Options) -> Result { + let cache = BlockCache::new(backend, options.cache.clone()); + let first = cache.config().block_size; + cache + .prefetch(0, first) + .map_err(|e| Error::Hdf5(clawhdf5::Error::Format(e)))?; + Ok(cache) +} diff --git a/crates/clawhdf5-remote/tests/common/mod.rs b/crates/clawhdf5-remote/tests/common/mod.rs new file mode 100644 index 0000000..498b603 --- /dev/null +++ b/crates/clawhdf5-remote/tests/common/mod.rs @@ -0,0 +1,302 @@ +//! Shared by the integration tests: the test server, a transcript of a +//! file (tree, attributes, values) to compare two ways of reading it, and +//! the test files. + +#![allow(dead_code)] + +pub mod server; + +use std::collections::{BTreeMap, HashSet, VecDeque}; +use std::fmt::Write as _; +use std::path::{Path, PathBuf}; +use std::process::Command; +use std::sync::Arc; + +use clawhdf5::{DType, File, Selection}; +use clawhdf5_format::error::FormatError; + +/// Objects visited per file. +const MAX_OBJECTS: usize = 2000; +/// Datasets with more bytes than this are not read (their metadata is). +pub const MAX_DATA_BYTES: u64 = 64 << 20; + +/// A short, stable digest of a value's `Debug` form. +fn digest(v: &T) -> String { + let s = format!("{v:?}"); + if s.len() <= 200 { + return s; + } + let mut h = 0xcbf2_9ce4_8422_2325u64; + for b in s.bytes() { + h = (h ^ u64::from(b)).wrapping_mul(0x100_0000_01b3); + } + format!("{}…[{} bytes, fnv {h:016x}]", &s[..80], s.len()) +} + +/// A data read's value, or `Err` (which chunk a damaged dataset reports +/// can vary between two `File`s: the chunk cache lists in hash order). +fn value(r: &Result) -> String { + match r { + Ok(v) => digest(v), + Err(_) => "Err".into(), + } +} + +fn sorted(m: std::collections::HashMap) -> BTreeMap { + m.into_iter().collect() +} + +/// Everything a reader sees in `file`: every group's entries, every +/// object's attributes, and every dataset's shape, types and values. +pub fn transcript(file: &File) -> String { + let mut out = String::new(); + let mut seen = HashSet::new(); + let mut queue = VecDeque::from([(String::from("/"), file.superblock().root_group_address)]); + while let Some((path, addr)) = queue.pop_front() { + if seen.len() >= MAX_OBJECTS || !seen.insert(addr) { + continue; + } + let group = file.group_at(addr); + let entries = group.entries(); + writeln!(out, "{path} @{addr} entries {}", digest(&entries)).unwrap(); + if let Ok(ds) = file.dataset_at(addr) { + dataset(&mut out, &path, &ds); + } + let attrs = group.attrs_with_errors().map(|(a, e)| (sorted(a), e)); + writeln!(out, "{path} attrs {}", digest(&attrs)).unwrap(); + if let Ok(entries) = entries { + for (name, child) in entries { + queue.push_back((format!("{}/{name}", path.trim_end_matches('/')), child)); + } + } + } + out +} + +fn dataset(out: &mut String, path: &str, ds: &clawhdf5::Dataset<'_>) { + let shape = ds.shape(); + let dtype = ds.dtype(); + writeln!( + out, + "{path} shape {} dtype {} raw {}", + digest(&shape), + digest(&dtype), + digest(&ds.raw_datatype()) + ) + .unwrap(); + let (Ok(shape), Ok(dtype), Ok(raw_dt)) = (shape, dtype, ds.raw_datatype()) else { + return; + }; + let elements = shape.iter().try_fold(1u64, |a, &d| a.checked_mul(d)); + let bytes = elements.and_then(|n| n.checked_mul(u64::from(raw_dt.type_size()))); + if bytes.is_none_or(|b| b > MAX_DATA_BYTES) { + writeln!(out, "{path} too large to read").unwrap(); + return; + } + writeln!( + out, + "{path} all {}", + value(&ds.read_selection(&Selection::All)) + ) + .unwrap(); + if matches!( + dtype, + DType::F32 + | DType::F64 + | DType::I8 + | DType::I16 + | DType::I32 + | DType::I64 + | DType::U8 + | DType::U16 + | DType::U32 + | DType::U64 + ) { + writeln!(out, "{path} f64 {}", value(&ds.read_f64())).unwrap(); + if let Some(&d0) = shape.first() { + let rank = shape.len(); + let sel = Selection::Hyperslab { + start: std::iter::once(d0 / 3) + .chain(std::iter::repeat_n(0, rank - 1)) + .collect(), + stride: vec![1; rank], + count: std::iter::once(d0.div_ceil(3)) + .chain(shape[1..].iter().copied()) + .collect(), + block: vec![1; rank], + }; + writeln!( + out, + "{path} f64 third {}", + value(&ds.read_f64_selection(&sel)) + ) + .unwrap(); + } + } + match &raw_dt { + clawhdf5_format::datatype::Datatype::String { .. } + | clawhdf5_format::datatype::Datatype::VariableLength { + is_string: true, .. + } => { + writeln!(out, "{path} strings {}", value(&ds.read_string_bytes())).unwrap(); + } + clawhdf5_format::datatype::Datatype::VariableLength { .. } => { + writeln!(out, "{path} vlen {}", value(&ds.read_vlen::())).unwrap(); + } + _ => {} + } +} + +/// Open, list every group, and read the first dataset found whose data is +/// at most `MAX_DATA_BYTES` (a tree view plus one plot). +pub fn list_and_read_one(file: &File) { + let mut seen = HashSet::new(); + let mut read_one = false; + let mut queue = VecDeque::from([file.superblock().root_group_address]); + while let Some(addr) = queue.pop_front() { + if seen.len() >= MAX_OBJECTS || !seen.insert(addr) { + continue; + } + let group = file.group_at(addr); + if let Ok(ds) = file.dataset_at(addr) { + let _ = (ds.shape(), ds.dtype()); + if !read_one { + let small = ds.shape().ok().and_then(|s| { + let n = s.iter().try_fold(1u64, |a, &d| a.checked_mul(d))?; + let size = u64::from(ds.raw_datatype().ok()?.type_size()); + n.checked_mul(size).filter(|&b| b <= MAX_DATA_BYTES) + }); + if small.is_some() { + let _ = ds.read_selection(&Selection::All); + read_one = true; + } + } + } + if let Ok(entries) = group.entries() { + queue.extend(entries.into_iter().map(|(_, a)| a)); + } + } +} + +/// External virtual-dataset sources read from `dir`, as `File::open` finds +/// them next to the file. +pub fn sibling_resolver(dir: PathBuf) -> clawhdf5::VdsResolver { + Arc::new(move |name: &str| { + let p = Path::new(name); + if name.is_empty() + || !p + .components() + .all(|c| matches!(c, std::path::Component::Normal(_))) + { + return Err(FormatError::Storage(format!("{name:?} not followed"))); + } + match std::fs::read(dir.join(p)) { + Ok(bytes) => Ok(Some(bytes)), + Err(e) if e.kind() == std::io::ErrorKind::NotFound => Ok(None), + Err(e) => Err(FormatError::Storage(e.to_string())), + } + }) +} + +/// HDF5 files under `dir`, recursively. +pub fn hdf5_files(dir: &Path, out: &mut Vec) { + let Ok(entries) = std::fs::read_dir(dir) else { + return; + }; + for e in entries.flatten() { + let p = e.path(); + if p.is_dir() { + hdf5_files(&p, out); + } else if p + .extension() + .and_then(|x| x.to_str()) + .is_some_and(|x| matches!(x, "h5" | "hdf5" | "he5" | "nc" | "h5ad" | "hdf")) + { + out.push(p); + } + } +} + +/// The repository's HDF5 test fixtures. +pub fn fixtures() -> Vec { + let root = Path::new(env!("CARGO_MANIFEST_DIR")); + let mut files = Vec::new(); + hdf5_files(&root.join("../clawhdf5/tests/fixtures"), &mut files); + hdf5_files(&root.join("../clawhdf5-format/tests/fixtures"), &mut files); + files.sort(); + files +} + +/// Files of `CLAWHDF5_REMOTE_CORPUS` (directories separated like `PATH`). +pub fn corpus() -> Option> { + let dirs = std::env::var("CLAWHDF5_REMOTE_CORPUS").ok()?; + let mut files = Vec::new(); + for d in std::env::split_paths(&dirs) { + hdf5_files(&d, &mut files); + } + files.sort(); + Some(files) +} + +pub fn python() -> String { + std::env::var("CLAWHDF5_PYTHON").unwrap_or_else(|_| "python3".to_string()) +} + +pub fn interop_required() -> bool { + std::env::var("CLAWHDF5_REQUIRE_INTEROP").is_ok_and(|v| v == "1") +} + +/// Whether python3 with h5py and numpy runs; panics when interop is +/// required and it does not. +pub fn have_h5py() -> bool { + let ok = Command::new(python()) + .args(["-c", "import h5py, numpy"]) + .output() + .map(|o| o.status.success()) + .unwrap_or(false); + assert!( + ok || !interop_required(), + "CLAWHDF5_REQUIRE_INTEROP=1 but python3 with h5py is not available" + ); + if !ok { + eprintln!("SKIP: python3 with h5py not available"); + } + ok +} + +/// Run a Python script; its stdout. +pub fn run_python(script: &str, args: &[&str]) -> String { + let out = Command::new(python()) + .arg("-c") + .arg(script) + .args(args) + .output() + .expect("failed to run python"); + assert!( + out.status.success(), + "python failed:\n{}", + String::from_utf8_lossy(&out.stderr) + ); + String::from_utf8(out.stdout).unwrap() +} + +/// A clawhdf5-written file with a multi-block chunked dataset (`/big`, +/// 1 000 000 f64 in chunks of 10 000, deflated), a contiguous one and a +/// group — several blocks of 1 MiB, with no Python needed. +pub fn multi_block_file() -> Vec { + let mut b = clawhdf5::FileBuilder::new(); + b.set_attr("title", clawhdf5::AttrValue::String("remote test".into())); + let big: Vec = (0..1_000_000u64) + .map(|i| ((i * 2_654_435_761) % 1_000_003) as f64 * 0.5) + .collect(); + b.create_dataset("big") + .with_f64_data(&big) + .with_chunks(&[10_000]) + .with_deflate(1); + let flat: Vec = (0..300_000u64).map(|i| i as f64).collect(); + b.create_dataset("flat").with_f64_data(&flat); + let mut g = b.create_group("grp"); + g.create_dataset("small").with_f64_data(&[1.0, 2.0, 3.0]); + b.add_group(g.finish()); + b.finish().unwrap() +} diff --git a/crates/clawhdf5-remote/tests/common/server.rs b/crates/clawhdf5-remote/tests/common/server.rs new file mode 100644 index 0000000..e556d7f --- /dev/null +++ b/crates/clawhdf5-remote/tests/common/server.rs @@ -0,0 +1,331 @@ +//! A small HTTP/1.1 file server on 127.0.0.1 for tests and the example: +//! `Range: bytes=a-b` requests (206, 416), `HEAD`, keep-alive, strong ETags +//! and Last-Modified with `If-Match` / `If-Unmodified-Since` (412), and +//! switches to misbehave — ignore ranges (200 with the whole file), cut +//! bodies short, answer 503, respond slowly, send no validators. It counts +//! requests and body bytes, and logs every range asked for. + +#![allow(dead_code)] + +use std::collections::HashMap; +use std::io::{BufRead, BufReader, Write}; +use std::net::{TcpListener, TcpStream}; +use std::sync::atomic::{AtomicBool, AtomicU32, AtomicU64, Ordering}; +use std::sync::{Arc, Mutex, RwLock}; +use std::time::Duration; + +/// A logged GET: the path and the range asked for (`None`: whole file). +pub type LogEntry = (String, Option<(u64, u64)>); + +struct Resource { + data: Arc>, + etag: String, + last_modified: String, +} + +/// Switches and counters shared with the connection threads. +#[derive(Default)] +pub struct Shared { + files: RwLock>, + version: AtomicU64, + /// Answer every request with 200 and the whole file. + pub ignore_range: AtomicBool, + /// Send no ETag and no Last-Modified. + pub no_validators: AtomicBool, + /// Send a weak ETag only (no Last-Modified). + pub weak_etag: AtomicBool, + /// Send Last-Modified but no ETag. + pub no_etag: AtomicBool, + /// Label bodies `Content-Encoding: gzip` (they are not). + pub gzip_label: AtomicBool, + /// Cut the body of the next N ranged responses in half (then close the + /// connection). + pub truncate_next: AtomicU32, + /// Answer the next N requests with 503. + pub fail_next: AtomicU32, + /// Sleep this long before answering each request. + pub delay_ms: AtomicU64, + /// Requests served (every status). + pub requests: AtomicU64, + /// Body bytes sent. + pub bytes: AtomicU64, + /// (path, range) of every GET. + pub log: Mutex>, + stop: AtomicBool, +} + +/// A running server; stops accepting when dropped. +pub struct Server { + pub addr: std::net::SocketAddr, + pub shared: Arc, +} + +impl Server { + /// Serve `files` (URL path such as `/a.h5` → bytes) on `127.0.0.1`, on + /// a free port. + pub fn start(files: Vec<(String, Vec)>) -> Server { + Server::bind("127.0.0.1:0", files) + } + + /// Serve `files` on `addr`. + pub fn bind(addr: &str, files: Vec<(String, Vec)>) -> Server { + let listener = TcpListener::bind(addr).expect("bind"); + let addr = listener.local_addr().unwrap(); + let shared = Arc::new(Shared::default()); + for (path, data) in files { + shared.put(&path, data); + } + let s = shared.clone(); + std::thread::spawn(move || { + for conn in listener.incoming() { + if s.stop.load(Ordering::SeqCst) { + break; + } + let Ok(conn) = conn else { continue }; + let s = s.clone(); + std::thread::spawn(move || { + let _ = serve(conn, &s); + }); + } + }); + Server { addr, shared } + } + + /// The URL of `path` (which starts with `/`). + pub fn url(&self, path: &str) -> String { + format!("http://{}{path}", self.addr) + } + + /// Requests served so far. + pub fn requests(&self) -> u64 { + self.shared.requests.load(Ordering::SeqCst) + } + + /// Body bytes sent so far. + pub fn bytes(&self) -> u64 { + self.shared.bytes.load(Ordering::SeqCst) + } + + /// Zero the counters and the log. + pub fn reset(&self) { + self.shared.requests.store(0, Ordering::SeqCst); + self.shared.bytes.store(0, Ordering::SeqCst); + self.shared.log.lock().unwrap().clear(); + } + + /// The ranges asked for so far. + pub fn log(&self) -> Vec { + self.shared.log.lock().unwrap().clone() + } +} + +impl Drop for Server { + fn drop(&mut self) { + self.shared.stop.store(true, Ordering::SeqCst); + let _ = TcpStream::connect(self.addr); + } +} + +impl Shared { + /// Add or replace a file (a replacement gets a new ETag and + /// Last-Modified). + pub fn put(&self, path: &str, data: Vec) { + let v = self.version.fetch_add(1, Ordering::SeqCst) + 1; + let secs = 1_700_000_000 + v; + self.files.write().unwrap().insert( + path.to_string(), + Resource { + data: Arc::new(data), + etag: format!("\"v{v}-{path}\""), + last_modified: http_date(secs), + }, + ); + } +} + +/// An IMF-fixdate for `secs` since the epoch (enough for distinct values). +fn http_date(secs: u64) -> String { + const DAYS: [&str; 7] = ["Thu", "Fri", "Sat", "Sun", "Mon", "Tue", "Wed"]; + const MONTHS: [&str; 12] = [ + "Jan", "Feb", "Mar", "Apr", "May", "Jun", "Jul", "Aug", "Sep", "Oct", "Nov", "Dec", + ]; + let days = secs / 86_400; + let rem = secs % 86_400; + // Civil-from-days (Howard Hinnant). + let z = days as i64 + 719_468; + let era = z.div_euclid(146_097); + let doe = z - era * 146_097; + let yoe = (doe - doe / 1460 + doe / 36_524 - doe / 146_096) / 365; + let y = yoe + era * 400; + let doy = doe - (365 * yoe + yoe / 4 - yoe / 100); + let mp = (5 * doy + 2) / 153; + let d = doy - (153 * mp + 2) / 5 + 1; + let m = if mp < 10 { mp + 3 } else { mp - 9 }; + let y = if m <= 2 { y + 1 } else { y }; + format!( + "{}, {:02} {} {} {:02}:{:02}:{:02} GMT", + DAYS[(days % 7) as usize], + d, + MONTHS[(m - 1) as usize], + y, + rem / 3600, + rem % 3600 / 60, + rem % 60 + ) +} + +fn parse_range(v: &str, len: u64) -> Option> { + let spec = v.trim().strip_prefix("bytes=")?; + if spec.contains(',') { + return None; // multiple ranges: serve the whole file + } + let (a, b) = spec.split_once('-')?; + let a: u64 = a.trim().parse().ok()?; + let b: u64 = match b.trim() { + "" => u64::MAX, + b => b.parse().ok()?, + }; + if b < a { + return None; + } + if a >= len { + return Some(Err(())); + } + Some(Ok((a, b.min(len - 1)))) +} + +fn serve(conn: TcpStream, s: &Shared) -> std::io::Result<()> { + conn.set_read_timeout(Some(Duration::from_secs(30)))?; + // One write per response and no Nagle delay: otherwise every request + // waits for a delayed ACK (~40 ms). + conn.set_nodelay(true)?; + let mut reader = BufReader::new(conn.try_clone()?); + let mut out = conn; + loop { + let mut line = String::new(); + if reader.read_line(&mut line)? == 0 { + return Ok(()); + } + let mut parts = line.split_whitespace(); + let method = parts.next().unwrap_or("").to_string(); + let path = parts.next().unwrap_or("").to_string(); + let mut headers = HashMap::new(); + loop { + let mut h = String::new(); + if reader.read_line(&mut h)? == 0 { + return Ok(()); + } + let h = h.trim_end(); + if h.is_empty() { + break; + } + if let Some((k, v)) = h.split_once(':') { + headers.insert(k.trim().to_ascii_lowercase(), v.trim().to_string()); + } + } + s.requests.fetch_add(1, Ordering::SeqCst); + let delay = s.delay_ms.load(Ordering::SeqCst); + if delay > 0 { + std::thread::sleep(Duration::from_millis(delay)); + } + let close = headers + .get("connection") + .is_some_and(|v| v.eq_ignore_ascii_case("close")); + if s.fail_next + .fetch_update(Ordering::SeqCst, Ordering::SeqCst, |n| n.checked_sub(1)) + .is_ok() + { + write!( + out, + "HTTP/1.1 503 Service Unavailable\r\nContent-Length: 0\r\n\r\n" + )?; + continue; + } + let res = { + let files = s.files.read().unwrap(); + files + .get(&path) + .map(|r| (r.data.clone(), r.etag.clone(), r.last_modified.clone())) + }; + let Some((data, etag, lm)) = res else { + write!(out, "HTTP/1.1 404 Not Found\r\nContent-Length: 0\r\n\r\n")?; + continue; + }; + let len = data.len() as u64; + let mut validators = String::new(); + if s.weak_etag.load(Ordering::SeqCst) { + validators = format!("ETag: W/{etag}\r\n"); + } else if s.no_etag.load(Ordering::SeqCst) { + validators = format!("Last-Modified: {lm}\r\n"); + } else if !s.no_validators.load(Ordering::SeqCst) { + validators = format!("ETag: {etag}\r\nLast-Modified: {lm}\r\n"); + } + let precondition_failed = headers.get("if-match").is_some_and(|v| v != &etag) + || headers.get("if-unmodified-since").is_some_and(|v| v != &lm); + if precondition_failed { + write!( + out, + "HTTP/1.1 412 Precondition Failed\r\n{validators}Content-Length: 0\r\n\r\n" + )?; + continue; + } + let range = if s.ignore_range.load(Ordering::SeqCst) { + None + } else { + headers.get("range").and_then(|v| parse_range(v, len)) + }; + if method == "GET" { + s.log + .lock() + .unwrap() + .push((path.clone(), range.and_then(Result::ok))); + } + let (status, body, extra) = match range { + Some(Err(())) => { + write!( + out, + "HTTP/1.1 416 Range Not Satisfiable\r\nContent-Range: bytes */{len}\r\n\ + {validators}Content-Length: 0\r\n\r\n" + )?; + continue; + } + Some(Ok((a, b))) => ( + "206 Partial Content", + &data[a as usize..=b as usize], + format!("Content-Range: bytes {a}-{b}/{len}\r\n"), + ), + None => ("200 OK", &data[..], String::new()), + }; + let extra = if s.gzip_label.load(Ordering::SeqCst) { + format!("{extra}Content-Encoding: gzip\r\n") + } else { + extra + }; + let head = format!( + "HTTP/1.1 {status}\r\nContent-Length: {}\r\nAccept-Ranges: bytes\r\n{extra}{validators}\r\n", + body.len() + ); + if method == "HEAD" { + out.write_all(head.as_bytes())?; + continue; + } + let truncate = status.starts_with("206") + && s.truncate_next + .fetch_update(Ordering::SeqCst, Ordering::SeqCst, |n| n.checked_sub(1)) + .is_ok(); + let body = if truncate { + &body[..body.len() / 2] + } else { + body + }; + let mut response = head.into_bytes(); + response.extend_from_slice(body); + out.write_all(&response)?; + out.flush()?; + s.bytes.fetch_add(body.len() as u64, Ordering::SeqCst); + if truncate || close { + let _ = out.shutdown(std::net::Shutdown::Both); + return Ok(()); + } + } +} diff --git a/crates/clawhdf5-remote/tests/http.rs b/crates/clawhdf5-remote/tests/http.rs new file mode 100644 index 0000000..e7faed8 --- /dev/null +++ b/crates/clawhdf5-remote/tests/http.rs @@ -0,0 +1,517 @@ +//! HTTP range reads against a local server (no internet): every value read +//! over HTTP equals `File::open`'s, and the server's misbehaviour — no +//! range support, a file replaced mid-read, cut-off bodies, errors, slow +//! answers under concurrent readers — is an error or the right data, never +//! wrong data. +//! +//! `CLAWHDF5_REMOTE_CORPUS=dir[:dir...]` adds every HDF5 file under those +//! directories (the conformance corpus is `conformance/.cache/corpus`), and +//! `CLAWHDF5_REMOTE_REPORT=1` prints the per-file request counts. + +#![cfg(feature = "http")] + +mod common; + +use std::sync::Arc; +use std::sync::atomic::Ordering; +use std::time::Duration; + +use clawhdf5::File; +use clawhdf5_remote::{ + CacheConfig, Error, HttpOptions, HttpStorage, Options, RemoteError, open_url, open_url_with, + storage_for_url, +}; +use common::server::Server; +use common::{list_and_read_one, multi_block_file, transcript}; + +/// Options for tests: fast retries. +fn quick() -> Options { + Options { + http: HttpOptions { + backoff: Duration::from_millis(5), + ..HttpOptions::default() + }, + ..Options::default() + } +} + +fn changed(e: &str) -> bool { + e.contains("changed while open") +} + +/// Serve `files`, open each by URL and through `File::open`, and require +/// identical transcripts. Returns (files compared, files both refused). +fn compare(files: &[std::path::PathBuf], report: bool) -> (usize, usize) { + let served: Vec<(String, Vec)> = files + .iter() + .enumerate() + .filter_map(|(i, p)| { + let bytes = std::fs::read(p).ok()?; + let name = p.file_name()?.to_str()?.replace(' ', "_"); + Some((format!("/f{i}/{name}"), bytes)) + }) + .collect(); + let server = Server::start(served.clone()); + let (mut same, mut refused) = (0, 0); + let mut totals = [0u64; 5]; + for (i, p) in files.iter().enumerate() { + let Some((url_path, bytes)) = served + .iter() + .find(|(u, _)| u.starts_with(&format!("/f{i}/"))) + else { + continue; + }; + let url = server.url(url_path); + let local = File::open(p); + let remote = storage_for_url(&url, &quick()) + .map_err(|e| e.to_string()) + .and_then(|s| { + File::open_storage(s.clone()) + .map(|mut f| { + f.set_vds_resolver(common::sibling_resolver(p.parent().unwrap().into())); + (f, s) + }) + .map_err(|e| e.to_string()) + }); + let (local, (remote, storage)) = match (local, remote) { + (Ok(l), Ok(r)) => (l, r), + (Err(l), Err(r)) => { + assert!( + !r.contains("HTTP") && !r.contains("network"), + "{url}: {r} ({l})" + ); + refused += 1; + continue; + } + (l, r) => panic!( + "{}: File::open {:?}, open_url {:?}", + p.display(), + l.map(|_| ()), + r.map(|_| ()) + ), + }; + let want = transcript(&local); + let got = transcript(&remote); + if want != got { + let first = want + .lines() + .zip(got.lines()) + .find(|(w, g)| w != g) + .map(|(w, g)| format!("\n local: {w}\n remote: {g}")) + .unwrap_or_default(); + panic!("{}: open_url differs from File::open{first}", p.display()); + } + assert!(storage.stats().reads > 0, "read through the cache"); + same += 1; + + // Cost of "list + read one dataset": with the block cache, and + // with none (every read a request). + let with = storage_for_url(&url, &quick()).unwrap(); + server.reset(); + if let Ok(f) = File::open_storage(with.clone()) { + list_and_read_one(&f); + } + let (cached_requests, cached_bytes) = (server.requests(), server.bytes()); + let (bare, _) = HttpStorage::open(&url, quick().http).unwrap(); + let bare = Arc::new(bare); + server.reset(); + if let Ok(f) = File::open_storage(bare.clone()) { + list_and_read_one(&f); + } + let uncached_requests = server.requests(); + totals[0] += 1 + cached_requests; // + the open probe + totals[1] += cached_bytes + with.config().block_size.min(bytes.len() as u64); + totals[2] += 1 + uncached_requests; + totals[3] += bytes.len() as u64; + totals[4] += 1; + if report { + eprintln!( + "requests cached {:>6} uncached {:>8} bytes {:>12} of {:>12} {}", + 1 + cached_requests, + 1 + uncached_requests, + cached_bytes + with.config().block_size.min(bytes.len() as u64), + bytes.len(), + p.display() + ); + } + // Files within one block: opening fetched everything. + if bytes.len() as u64 <= with.config().block_size { + assert_eq!(cached_requests, 0, "{}", p.display()); + } + } + eprintln!( + "list + read one dataset over {} files: {} requests with the 1 MiB block cache \ + ({} bytes transferred, files {} bytes), {} requests without a cache", + totals[4], totals[0], totals[1], totals[3], totals[2] + ); + (same, refused) +} + +#[test] +fn fixtures_read_identically_over_http() { + let files = common::fixtures(); + assert!(files.len() >= 45, "{} fixtures", files.len()); + let report = std::env::var("CLAWHDF5_REMOTE_REPORT").is_ok_and(|v| v == "1"); + let (same, _) = compare(&files, report); + assert!(same >= 40, "{same}"); +} + +#[test] +fn corpus_reads_identically_over_http() { + let Some(files) = common::corpus() else { + eprintln!("CLAWHDF5_REMOTE_CORPUS not set; skipping the corpus"); + return; + }; + let report = std::env::var("CLAWHDF5_REMOTE_REPORT").is_ok_and(|v| v == "1"); + let (same, refused) = compare(&files, report); + eprintln!("corpus: {same} files identical, {refused} refused by both"); + assert!(same > 0); +} + +#[test] +fn multi_block_file_values_and_request_budget() { + let bytes = multi_block_file(); + assert!(bytes.len() > 3 << 20, "{}", bytes.len()); + let server = Server::start(vec![("/m.h5".into(), bytes.clone())]); + let url = server.url("/m.h5"); + let local = File::from_bytes(bytes.clone()).unwrap(); + let storage = storage_for_url(&url, &quick()).unwrap(); + assert_eq!(server.requests(), 1, "opening is one request"); + let remote = File::open_storage(storage.clone()).unwrap(); + assert!(remote.contiguous_bytes().is_none()); + let mut names = remote.root().datasets().unwrap(); + names.sort(); + assert_eq!(names, ["big", "flat"]); + assert_eq!(remote.root().groups().unwrap(), ["grp"]); + assert_eq!( + remote.dataset("grp/small").unwrap().read_f64().unwrap(), + [1.0, 2.0, 3.0] + ); + let big = remote.dataset("big").unwrap().read_f64().unwrap(); + assert_eq!(big, local.dataset("big").unwrap().read_f64().unwrap()); + let flat = remote.dataset("flat").unwrap().read_f64().unwrap(); + assert_eq!(flat.len(), 300_000); + assert_eq!(flat[299_999], 299_999.0); + // Every byte was fetched at most once, in whole 1 MiB blocks. + let log = server.log(); + let mut blocks: Vec = Vec::new(); + for (_, r) in &log { + let (a, b) = r.expect("every request is ranged"); + assert_eq!(a % (1 << 20), 0, "block-aligned"); + blocks.extend(a >> 20..=b >> 20); + } + let n = blocks.len(); + blocks.sort_unstable(); + blocks.dedup(); + assert_eq!(n, blocks.len(), "a block was fetched twice: {log:?}"); + assert!( + server.requests() <= (bytes.len() as u64 >> 20) + 2, + "{} requests for a {} byte file", + server.requests(), + bytes.len() + ); + // Reading again costs nothing. + let before = server.requests(); + assert_eq!(transcript(&remote), transcript(&local)); + assert_eq!(server.requests(), before); +} + +#[test] +fn h5py_written_file_over_http() { + if !common::have_h5py() { + return; + } + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("py.h5"); + common::run_python( + r#" +import sys, h5py, numpy as np +with h5py.File(sys.argv[1], "w", libver="latest") as f: + f.attrs["title"] = "h5py remote test" + g = f.create_group("sensors") + for i in range(40): + d = g.create_dataset(f"s{i:02d}", data=np.arange(1000, dtype="().unwrap()); + assert_eq!(vl[1], it.next().unwrap()); + assert_eq!( + remote.dataset("link").unwrap().read_i32().unwrap(), + local.dataset("sensors/s03").unwrap().read_i32().unwrap() + ); +} + +#[test] +fn a_server_that_ignores_range_is_refused_or_downloaded_when_allowed() { + let bytes = multi_block_file(); + let server = Server::start(vec![("/m.h5".into(), bytes.clone())]); + server.shared.ignore_range.store(true, Ordering::SeqCst); + let url = server.url("/m.h5"); + let err = open_url_with(&url, &quick()).unwrap_err(); + assert!( + matches!(err, Error::Remote(RemoteError::RangeNotSupported(_))), + "{err}" + ); + assert!( + server.bytes() < bytes.len() as u64, + "refusing must not download the file ({} bytes read)", + server.bytes() + ); + let mut opts = quick(); + opts.http.allow_full_download = true; + let storage = storage_for_url(&url, &opts).unwrap(); + let f = File::open_storage(storage.clone()).unwrap(); + assert_eq!(f.contiguous_bytes(), Some(&bytes[..]), "read from memory"); + assert_eq!(server.requests(), 2); + let local = File::from_bytes(bytes).unwrap(); + assert_eq!(transcript(&f), transcript(&local)); + assert_eq!(server.requests(), 2, "no further requests"); + // Too large for the download limit. + opts.http.max_full_download = 1000; + assert!(open_url_with(&url, &opts).is_err()); +} + +#[test] +fn a_file_replaced_mid_read_is_an_error_not_mixed_data() { + for mode in ["etag", "last-modified", "none"] { + let bytes = multi_block_file(); + let server = Server::start(vec![("/m.h5".into(), bytes.clone())]); + match mode { + "last-modified" => server.shared.no_etag.store(true, Ordering::SeqCst), + "none" => server.shared.no_validators.store(true, Ordering::SeqCst), + _ => {} + } + let url = server.url("/m.h5"); + let f = open_url_with(&url, &quick()).unwrap(); + assert_eq!(f.root().groups().unwrap(), ["grp"], "{mode}"); + // Replace the file: same layout, other values (and, for "none", + // one byte longer, which is all that can be checked). + let mut other = bytes.clone(); + let n = other.len(); + for b in &mut other[n / 2..n / 2 + 1000] { + *b ^= 0xff; + } + if mode == "none" { + other.push(0); + } + server.shared.put("/m.h5", other); + let err = f + .dataset("big") + .unwrap() + .read_f64() + .unwrap_err() + .to_string(); + assert!(changed(&err), "{mode}: {err}"); + } +} + +#[test] +fn a_weak_etag_or_no_validator_is_refused_when_required() { + let server = Server::start(vec![("/m.h5".into(), multi_block_file())]); + server.shared.weak_etag.store(true, Ordering::SeqCst); + let mut opts = quick(); + opts.http.require_validator = true; + assert!(open_url_with(&server.url("/m.h5"), &opts).is_err()); + opts.http.require_validator = false; + assert!(open_url_with(&server.url("/m.h5"), &opts).is_ok()); +} + +#[test] +fn truncated_bodies_are_retried_then_an_error() { + let bytes = multi_block_file(); + let server = Server::start(vec![("/m.h5".into(), bytes.clone())]); + let url = server.url("/m.h5"); + let local = File::from_bytes(bytes).unwrap(); + let want = local.dataset("big").unwrap().read_f64().unwrap(); + + // One cut-off body: retried, right values. + let (http, first) = HttpStorage::open(&url, quick().http).unwrap(); + assert_eq!(first.len(), 1 << 20); + let storage = Arc::new(clawhdf5_remote::BlockCache::new( + http, + CacheConfig::default(), + )); + let f = File::open_storage(storage.clone()).unwrap(); + server.shared.truncate_next.store(1, Ordering::SeqCst); + assert_eq!(f.dataset("big").unwrap().read_f64().unwrap(), want); + assert_eq!(storage.inner().stats().retries, 1); + + // Every body cut off: an error, never partial data. + let f = open_url_with(&url, &quick()).unwrap(); + server.shared.truncate_next.store(1000, Ordering::SeqCst); + let err = f + .dataset("big") + .unwrap() + .read_f64() + .unwrap_err() + .to_string(); + assert!( + err.contains("network error") || err.contains("bad response"), + "{err}" + ); + server.shared.truncate_next.store(0, Ordering::SeqCst); + // The failure was not cached: the next read succeeds. + assert_eq!(f.dataset("big").unwrap().read_f64().unwrap(), want); +} + +#[test] +fn server_errors_are_retried_with_backoff() { + let bytes = multi_block_file(); + let server = Server::start(vec![("/m.h5".into(), bytes.clone())]); + server.shared.fail_next.store(2, Ordering::SeqCst); + let url = server.url("/m.h5"); + let f = open_url_with(&url, &quick()).unwrap(); + let local = File::from_bytes(bytes).unwrap(); + server.shared.fail_next.store(3, Ordering::SeqCst); + assert_eq!( + f.dataset("flat").unwrap().read_f64().unwrap(), + local.dataset("flat").unwrap().read_f64().unwrap() + ); + // More failures than retries: an error. + let f = open_url_with(&url, &quick()).unwrap(); + server.shared.fail_next.store(100, Ordering::SeqCst); + let err = f + .dataset("big") + .unwrap() + .read_f64() + .unwrap_err() + .to_string(); + assert!(err.contains("503"), "{err}"); +} + +#[test] +fn slow_server_concurrent_readers_fetch_each_block_once() { + let bytes = multi_block_file(); + let server = Server::start(vec![("/m.h5".into(), bytes.clone())]); + let url = server.url("/m.h5"); + let opts = Options { + cache: CacheConfig { + block_size: 256 << 10, + max_request: 256 << 10, + ..CacheConfig::default() + }, + ..quick() + }; + let storage = storage_for_url(&url, &opts).unwrap(); + let file = File::open_storage(storage.clone()).unwrap(); + let local = File::from_bytes(bytes).unwrap(); + let want_big = local.dataset("big").unwrap().read_f64().unwrap(); + let want_flat = local.dataset("flat").unwrap().read_f64().unwrap(); + server.shared.delay_ms.store(40, Ordering::SeqCst); + server.reset(); + std::thread::scope(|s| { + for t in 0..8 { + let (file, want_big, want_flat) = (&file, &want_big, &want_flat); + s.spawn(move || { + if t % 2 == 0 { + assert_eq!(&file.dataset("big").unwrap().read_f64().unwrap(), want_big); + } else { + assert_eq!( + &file.dataset("flat").unwrap().read_f64().unwrap(), + want_flat + ); + } + }); + } + }); + let log = server.log(); + let mut starts: Vec = log.iter().map(|(_, r)| r.unwrap().0).collect(); + let n = starts.len(); + starts.sort_unstable(); + starts.dedup(); + assert_eq!(n, starts.len(), "a block was fetched twice"); + assert!(storage.stats().waits > 0, "readers shared fetches"); +} + +#[test] +fn bad_urls_and_statuses_are_clean_errors() { + let server = Server::start(vec![("/m.h5".into(), multi_block_file())]); + let err = open_url_with(&server.url("/missing.h5"), &quick()).unwrap_err(); + assert!( + matches!(err, Error::Remote(RemoteError::Status { code: 404, .. })), + "{err}" + ); + assert!(matches!( + open_url("ftp://example.com/a.h5"), + Err(Error::Remote(RemoteError::UnsupportedScheme(_))) + )); + assert!(matches!( + open_url("no-scheme"), + Err(Error::Remote(RemoteError::InvalidUrl(_))) + )); + assert!(matches!( + open_url("s3://bucket/key.h5"), + Err(Error::Remote(RemoteError::UnsupportedScheme(_))) + )); + #[cfg(not(feature = "https"))] + { + let e = open_url("https://example.com/a.h5") + .unwrap_err() + .to_string(); + assert!(e.contains("`https` feature"), "{e}"); + } + // A content-encoded body is not a byte range. + let server = Server::start(vec![("/m.h5".into(), multi_block_file())]); + server.shared.gzip_label.store(true, Ordering::SeqCst); + let e = open_url(&server.url("/m.h5")).unwrap_err().to_string(); + assert!(e.contains("gzip-encoded"), "{e}"); + // Not HDF5. + let server = Server::start(vec![("/x.h5".into(), vec![7u8; 5000])]); + assert!(matches!( + open_url(&server.url("/x.h5")), + Err(Error::Hdf5(_)) + )); + // A server that closes every connection unanswered: a network error + // after the retries. (Not a closed port: another test's server could + // take it meanwhile.) + let port = { + let l = std::net::TcpListener::bind("127.0.0.1:0").unwrap(); + let port = l.local_addr().unwrap().port(); + std::thread::spawn(move || { + for c in l.incoming() { + drop(c); + } + }); + port + }; + let err = open_url_with(&format!("http://127.0.0.1:{port}/a.h5"), &quick()).unwrap_err(); + assert!( + matches!(err, Error::Remote(RemoteError::Transport(_))), + "{err}" + ); +} diff --git a/scripts/ci-test.sh b/scripts/ci-test.sh index ad9af8b..2fce9c1 100755 --- a/scripts/ci-test.sh +++ b/scripts/ci-test.sh @@ -109,23 +109,36 @@ run_step "cargo clippy (fast-deflate / zlib-ng)" cargo clippy \ --features clawhdf5-format/fast-deflate,clawhdf5-filters/fast-deflate \ -- -D warnings +# clawhdf5-remote's HTTPS backend (rustls). +run_step "cargo clippy (remote, https)" cargo clippy \ + -p clawhdf5-remote \ + --all-targets \ + --features https \ + -- -D warnings + # The README promises that the core crates build no C by default. Hold it to # that: fail if a crate that compiles C (a *-sys crate, cc or cmake) enters the # default dependency tree of any of them. clawhdf5-migrate (bundled SQLite), # clawhdf5-napi (Node) and clawhdf5-gpu (graphics drivers) are exempt. # js-sys (clawhdf5-wasm's bindings to JavaScript) builds no C. +# clawhdf5-remote is checked by default (plain HTTP); its https feature +# (ring) builds C and is opt-in. no_c_in_default_build() { - local crate found=0 - for crate in clawhdf5-format clawhdf5-io clawhdf5-filters clawhdf5 \ + local entry crate features found=0 + for entry in clawhdf5-format clawhdf5-io clawhdf5-filters clawhdf5 \ clawhdf5-agent clawhdf5-ann clawhdf5-accel clawhdf5-netcdf4 clawhdf5-cli \ clawhdf5-tools \ - clawhdf5-wasm; do + clawhdf5-wasm \ + clawhdf5-remote; do + crate=${entry%%:*} + features=() + [ "$entry" != "$crate" ] && features=(--features "${entry#*:}") local c_deps - c_deps=$(cargo tree -q -p "$crate" -e normal,build --prefix none \ + c_deps=$(cargo tree -q -p "$crate" "${features[@]}" -e normal,build --prefix none \ | grep -E '^([a-z0-9_-]+-sys|cc|cmake) v' \ | grep -v '^js-sys v' | sort -u) if [ -n "$c_deps" ]; then - echo "$crate pulls in C by default:" + echo "$entry pulls in C by default:" echo "$c_deps" | sed 's/^/ /' found=1 fi From 4ff3e40fea420eee174c8b4201445fcc27180969 Mon Sep 17 00:00:00 2001 From: osobh Date: Sat, 26 Sep 2026 17:15:06 -0500 Subject: [PATCH 02/20] clawhdf5-remote: object stores through object_store (S3, GCS, Azure) ObjectStoreStorage (feature `object-store`, pure Rust) reads one object of any object_store store by ranged get_opts, pinned at open by a head request: If-Match with its ETag (and the ETag and size of every response compared), else its version or modification time. A change is RemoteError::FileChanged. object_store is async and Storage is not, so the storage owns a small multi-threaded tokio runtime (two workers) and blocks the calling thread on it; the ranges of one read_ranges call are fetched concurrently (up to 8). From inside another tokio runtime it refuses with RemoteError::Usage instead of blocking a worker, and it shuts its runtime down in the background on drop so dropping it in async code does not panic. open_object(store, path, options) opens a file through a block cache (first block prefetched); open_url accepts s3://, gs:// and az:// with the `s3`, `gcs` and `azure` features, configured from the environment by object_store's from_env builders. Those pull object_store's cloud clients and aws-lc-rs (C), so they are opt-in; without them the URL is a clean UnsupportedScheme error naming the feature. Tests against object_store's in-memory and local-file stores (no cloud): every fixture's transcript equals File::open's, a multi-block object is fetched in coalesced block runs, an object replaced while open is an error, and a missing object or a read from inside a runtime is a clean error. ci-test.sh lints all backends, runs these tests (with s3 for its URL parsing test) and checks object-store for C in the no-C step. Co-Authored-By: Claude Opus 5.5 (1M context) --- CLAUDE.md | 9 +- crates/clawhdf5-remote/Cargo.toml | 14 +- crates/clawhdf5-remote/README.md | 32 ++- crates/clawhdf5-remote/src/lib.rs | 53 +++- crates/clawhdf5-remote/src/object.rs | 285 +++++++++++++++++++ crates/clawhdf5-remote/tests/http.rs | 9 +- crates/clawhdf5-remote/tests/object_store.rs | 118 ++++++++ scripts/ci-test.sh | 18 +- 8 files changed, 517 insertions(+), 21 deletions(-) create mode 100644 crates/clawhdf5-remote/src/object.rs create mode 100644 crates/clawhdf5-remote/tests/object_store.rs diff --git a/CLAUDE.md b/CLAUDE.md index 57a5fa4..9a4f5b1 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -26,7 +26,7 @@ Cargo workspace with 19 crates under `crates/` (plus `libaec-sys`, an internal F | `clawhdf5-napi` | Node.js native addon bindings | | `clawhdf5-py` | PyO3 Python bindings | | `clawhdf5-wasm` | WebAssembly (wasm-bindgen) reader for the browser; demo in `examples/wasm-viewer/` | -| `clawhdf5-remote` | Remote files: `open_url` over HTTP(S) range requests through a mandatory block cache (`BlockCache`) | +| `clawhdf5-remote` | Remote files: `open_url` over HTTP(S) range requests and object stores (`object_store`: S3, GCS, Azure) through a mandatory block cache (`BlockCache`) | | `clawhdf5-bench` | Benchmark suite | ## Key Features @@ -165,8 +165,11 @@ Cargo workspace with 19 crates under `crates/` (plus `libaec-sys`, an internal F runs coalesced into parallel requests). `HttpStorage` pins the file by ETag/Last-Modified and length (a change is `RemoteError::FileChanged`), refuses servers that ignore `Range` unless a full download is allowed, - and retries transient failures. Default build is plain HTTP with no C; - `https` (rustls + ring) is opt-in. Tests run a std-only HTTP server + and retries transient failures. `ObjectStoreStorage` (feature + `object-store`, pure Rust) blocks on a small owned tokio runtime and + refuses to run inside another runtime. Default build is plain HTTP with + no C; `https` (rustls + ring) and `s3`/`gcs`/`azure` (aws-lc-rs) are + opt-in. Tests run a std-only HTTP server (`tests/common/server.rs`, also the `range_server` example); `CLAWHDF5_REMOTE_CORPUS=conformance/.cache/corpus` compares every corpus file over HTTP with `File::open`. diff --git a/crates/clawhdf5-remote/Cargo.toml b/crates/clawhdf5-remote/Cargo.toml index 6e68a73..ab64c2c 100644 --- a/crates/clawhdf5-remote/Cargo.toml +++ b/crates/clawhdf5-remote/Cargo.toml @@ -3,7 +3,7 @@ name = "clawhdf5-remote" version = "2.7.0" edition = "2024" rust-version.workspace = true -description = "Read HDF5 files over HTTP(S) range requests with clawhdf5, through a block cache" +description = "Read HDF5 files over HTTP(S) range requests and object stores (S3, GCS, Azure) with clawhdf5, through a block cache" license = "MIT" repository = "https://git.redclaw.dev/quantumclaw/clawhdf5" readme = "README.md" @@ -17,11 +17,23 @@ http = ["dep:ureq"] # HTTPS through rustls (ring provider, Mozilla roots). ring compiles C and # assembly, so this is not part of the default build. https = ["http", "ureq/rustls"] +# Any object_store backend (in-memory, local files, or one you configure), +# driven by a small tokio runtime the storage owns. Pure Rust. +object-store = ["dep:object_store", "dep:tokio", "dep:futures-util"] +# s3:// gs:// az:// URLs in open_url, credentials from the environment. +# object_store's cloud clients use aws-lc-rs (C), hence opt-in. +s3 = ["object-store", "object_store/aws"] +gcs = ["object-store", "object_store/gcp"] +azure = ["object-store", "object_store/azure"] [dependencies] clawhdf5 = { path = "../clawhdf5", version = "2.7.0" } clawhdf5-format = { path = "../clawhdf5-format", version = "2.7.0" } ureq = { version = "3.4", optional = true, default-features = false } +object_store = { version = "0.14", optional = true, default-features = false, features = ["fs"] } +tokio = { version = "1", optional = true, default-features = false, features = ["rt-multi-thread"] } +futures-util = { version = "0.3", optional = true, default-features = false, features = ["std"] } [dev-dependencies] tempfile = { workspace = true } +tokio = { version = "1", default-features = false, features = ["rt"] } diff --git a/crates/clawhdf5-remote/README.md b/crates/clawhdf5-remote/README.md index bb20f71..b4a487d 100644 --- a/crates/clawhdf5-remote/README.md +++ b/crates/clawhdf5-remote/README.md @@ -1,6 +1,7 @@ # clawhdf5-remote -Read HDF5 files where they live — on an HTTP(S) server — with +Read HDF5 files where they live — on an HTTP(S) server or in an object +store (S3, GCS, Azure) — with [clawhdf5](../../README.md), without downloading them first. ```rust @@ -52,8 +53,35 @@ The zero-copy methods of `clawhdf5` (`read_raw_ref`, `read_*_zerocopy`, |---|---|---| | `http` (default) | `http://` through `ureq`, no TLS | none | | `https` | `https://` through rustls, ring provider, Mozilla roots | ring (C and assembly) | +| `object-store` | `ObjectStoreStorage` and `open_object` over any [`object_store`](https://docs.rs/object_store) store (in-memory, local files, or one you configure) | none | +| `s3`, `gcs`, `azure` | `s3://bucket/key`, `gs://bucket/key`, `az://container/key` in `open_url`, configured from the environment (`AWS_*`, `GOOGLE_*`, `AZURE_*`) as object_store's `from_env` builders read it | aws-lc-rs (object_store's cloud clients) | -The default build compiles no C (`scripts/ci-test.sh` checks it). +The default build and `object-store` compile no C (`scripts/ci-test.sh` +checks both). + +## Object stores + +`object_store` is async; `Storage` is synchronous (parsing is CPU work). +`ObjectStoreStorage` owns a small tokio runtime (two worker threads) and +blocks the calling thread on it for each read, so it is read from ordinary +threads, several at once. From inside an async runtime it refuses +(`RemoteError::Usage`) rather than block a worker: read in +`tokio::task::spawn_blocking`. The object is pinned by its ETag +(`If-Match`, and compared on every response), else its version or +modification time, and its size. The ranges of one read are fetched +concurrently (up to 8). + +```rust +use std::sync::Arc; +use clawhdf5_remote::object_store::{memory::InMemory, ObjectStore}; +let store: Arc = Arc::new(InMemory::new()); +// ... put a file at "data.h5" ... +let (file, cache) = clawhdf5_remote::open_object(store, "data.h5", &Default::default())?; +``` + +The tests use object_store's in-memory and local-file stores; no cloud +account is needed. The cloud schemes are only built (and unit-tested for +URL parsing) in CI, not run against a real bucket. ## Counting requests diff --git a/crates/clawhdf5-remote/src/lib.rs b/crates/clawhdf5-remote/src/lib.rs index 597fb63..f233954 100644 --- a/crates/clawhdf5-remote/src/lib.rs +++ b/crates/clawhdf5-remote/src/lib.rs @@ -12,12 +12,13 @@ //! ``` //! //! - [`open_url`] / [`open_url_with`]: `http://` (default feature `http`), -//! `https://` (feature `https`) → a [`clawhdf5::File`] with the whole -//! read API. +//! `https://` (feature `https`), `s3://`, `gs://`, `az://` (features `s3`, +//! `gcs`, `azure`) → a [`clawhdf5::File`] with the whole read API. //! - [`storage_for_url`] gives the cached storage itself, to open with //! [`clawhdf5::File::open_storage`] and to read its [`CacheStats`]. //! - [`HttpStorage`] (range `GET`s, pinned by ETag/Last-Modified, retried -//! with backoff), and [`BlockCache`] over any +//! with backoff), [`ObjectStoreStorage`] (any `object_store` store, +//! feature `object-store`), and [`BlockCache`] over any //! [`Storage`](clawhdf5_format::storage::Storage). //! //! What costs what: opening costs one request (it also fetches the first @@ -34,6 +35,8 @@ pub mod cache; pub mod error; #[cfg(feature = "http")] pub mod http; +#[cfg(feature = "object-store")] +pub mod object; use std::sync::Arc; @@ -44,6 +47,10 @@ pub use cache::{BlockCache, CacheConfig, CacheStats}; pub use error::{Error, RemoteError}; #[cfg(feature = "http")] pub use http::{HttpOptions, HttpStats, HttpStorage}; +#[cfg(feature = "object-store")] +pub use object::ObjectStoreStorage; +#[cfg(feature = "object-store")] +pub use object_store; /// A backend a [`BlockCache`] can read through. pub type Backend = Box; @@ -65,7 +72,9 @@ pub struct Options { /// Open the HDF5 file at `url` with default [`Options`]. /// /// `http://…` needs the (default) `http` feature, `https://…` the `https` -/// feature. +/// feature, `s3://bucket/key`, `gs://bucket/key` and `az://container/key` +/// the `s3`, `gcs` and `azure` features (credentials and region from the +/// environment, as `object_store`'s `from_env` builders read them). pub fn open_url(url: &str) -> Result { open_url_with(url, &Options::default()) } @@ -111,8 +120,24 @@ fn http_storage(url: &str, _options: &Options) -> Result, Err .into()) } +#[cfg(any(feature = "s3", feature = "gcs", feature = "azure"))] +fn cloud_storage(url: &str, _scheme: &str, options: &Options) -> Result, Error> { + let (store, path) = object::store_for_url(url)?; + let storage = ObjectStoreStorage::new(store, path)?; + Ok(Arc::new(cached(Box::new(storage), options)?)) +} + +#[cfg(not(any(feature = "s3", feature = "gcs", feature = "azure")))] fn cloud_storage(url: &str, scheme: &str, _options: &Options) -> Result, Error> { - Err(RemoteError::UnsupportedScheme(format!("{url}: {scheme}:// is not supported yet")).into()) + let feature = match scheme { + "s3" | "s3a" => "s3", + "gs" => "gcs", + _ => "azure", + }; + Err(RemoteError::UnsupportedScheme(format!( + "{url}: {scheme}:// needs the `{feature}` feature of clawhdf5-remote" + )) + .into()) } /// A [`BlockCache`] over `backend` with its first block fetched (readahead @@ -125,3 +150,21 @@ pub fn cached(backend: Backend, options: &Options) -> Result, + path: &str, + options: &Options, +) -> Result<(File, Arc), Error> { + let path = object_store::path::Path::parse(path) + .map_err(|e| RemoteError::InvalidUrl(format!("{path}: {e}")))?; + let storage = Arc::new(cached( + Box::new(ObjectStoreStorage::new(store, path)?), + options, + )?); + let file = File::open_storage(storage.clone())?; + Ok((file, storage)) +} diff --git a/crates/clawhdf5-remote/src/object.rs b/crates/clawhdf5-remote/src/object.rs new file mode 100644 index 0000000..6bafe8d --- /dev/null +++ b/crates/clawhdf5-remote/src/object.rs @@ -0,0 +1,285 @@ +//! Object stores (S3, GCS, Azure, local files, memory) through the +//! [`object_store`] crate: [`ObjectStoreStorage`]. +//! +//! `object_store` is async and [`Storage`] is synchronous (parsing is CPU +//! work; `docs/design/range-reads.md` §3 (a)). The storage owns a small +//! multi-threaded tokio runtime (two worker threads) and blocks the calling +//! thread on it for each read, so it can be used from ordinary threads — +//! several at once. Calling it from inside another tokio runtime would +//! block that runtime's worker, so it refuses with +//! [`RemoteError::Usage`]: from async code, read in +//! `tokio::task::spawn_blocking`. +//! +//! The object is pinned when the storage is made: its size, and its ETag +//! (sent as `If-Match` with every read, and compared with every response) +//! or, without one, its version or modification time. A change while it is +//! open is [`RemoteError::FileChanged`]. The ranges of one `read_ranges` +//! call are fetched concurrently (at most 8 at a time). + +use std::borrow::Cow; +use std::ops::Range; +use std::sync::Arc; +use std::sync::atomic::{AtomicU64, Ordering}; + +use clawhdf5_format::error::FormatError; +use clawhdf5_format::storage::Storage; +use futures_util::{StreamExt, TryStreamExt}; +use object_store::path::Path; +use object_store::{GetOptions, GetRange, ObjectMeta, ObjectStore, ObjectStoreExt}; + +use crate::error::RemoteError; + +/// Concurrent requests of one `read_ranges` call. +const MAX_CONCURRENT: usize = 8; + +/// One object of an [`ObjectStore`], read by ranged `get`s. See the +/// [module documentation](self). +pub struct ObjectStoreStorage { + store: Arc, + path: Path, + meta: ObjectMeta, + runtime: Option, + requests: AtomicU64, + bytes: AtomicU64, +} + +impl std::fmt::Debug for ObjectStoreStorage { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("ObjectStoreStorage") + .field("store", &self.store.to_string()) + .field("path", &self.path) + .field("size", &self.meta.size) + .field("e_tag", &self.meta.e_tag) + .finish() + } +} + +fn os_error(e: object_store::Error) -> RemoteError { + match e { + object_store::Error::Precondition { .. } | object_store::Error::NotModified { .. } => { + RemoteError::FileChanged(e.to_string()) + } + other => RemoteError::ObjectStore(other.to_string()), + } +} + +impl ObjectStoreStorage { + /// Open the object at `path` of `store`: one `head` request for its + /// size and validators. + pub fn new(store: Arc, path: Path) -> Result { + let runtime = tokio::runtime::Builder::new_multi_thread() + .worker_threads(2) + .thread_name("clawhdf5-remote") + .enable_all() + .build() + .map_err(|e| RemoteError::Usage(format!("cannot start a tokio runtime: {e}")))?; + let mut s = ObjectStoreStorage { + store, + path, + meta: ObjectMeta { + location: Path::default(), + last_modified: Default::default(), + size: 0, + e_tag: None, + version: None, + }, + runtime: Some(runtime), + requests: AtomicU64::new(1), + bytes: AtomicU64::new(0), + }; + let (store, path) = (s.store.clone(), s.path.clone()); + s.meta = s.block_on(async move { store.head(&path).await.map_err(os_error) })?; + Ok(s) + } + + /// The object's metadata as pinned at open. + pub fn meta(&self) -> &ObjectMeta { + &self.meta + } + + /// Requests made (the `head` included) and bytes received. + pub fn stats(&self) -> (u64, u64) { + ( + self.requests.load(Ordering::Relaxed), + self.bytes.load(Ordering::Relaxed), + ) + } + + fn block_on( + &self, + fut: impl std::future::Future>, + ) -> Result { + if tokio::runtime::Handle::try_current().is_ok() { + return Err(RemoteError::Usage( + "ObjectStoreStorage blocks on its own runtime and cannot be read from inside \ + an async runtime; read in tokio::task::spawn_blocking" + .into(), + )); + } + let rt = self.runtime.as_ref().expect("runtime lives until drop"); + rt.block_on(fut) + } + + fn options(&self, range: Range) -> GetOptions { + let mut o = GetOptions { + range: Some(GetRange::Bounded(range)), + ..GetOptions::default() + }; + if let Some(e) = &self.meta.e_tag { + o.if_match = Some(e.clone()); + } else if let Some(v) = &self.meta.version { + o.version = Some(v.clone()); + } else { + o.if_unmodified_since = Some(self.meta.last_modified); + } + o + } + + fn fetch_all(&self, ranges: &[Range]) -> Result>, RemoteError> { + let len = self.meta.size; + let jobs: Vec<(usize, Range)> = ranges + .iter() + .enumerate() + .filter_map(|(i, r)| { + let end = r.end.min(len); + (r.start < end).then_some((i, r.start..end)) + }) + .collect(); + let store = self.store.clone(); + let path = self.path.clone(); + let pinned = self.meta.e_tag.clone(); + let reqs: Vec<(usize, Range, GetOptions)> = jobs + .into_iter() + .map(|(i, r)| { + let o = self.options(r.clone()); + (i, r, o) + }) + .collect(); + self.requests + .fetch_add(reqs.len() as u64, Ordering::Relaxed); + let fetched: Vec<(usize, Vec)> = self.block_on(async move { + futures_util::stream::iter(reqs) + .map(|(i, r, o)| { + let (store, path, pinned) = (store.clone(), path.clone(), pinned.clone()); + async move { + let got = store.get_opts(&path, o).await.map_err(os_error)?; + if let (Some(want), Some(have)) = (&pinned, &got.meta.e_tag) + && want != have + { + return Err(RemoteError::FileChanged(format!( + "{path}: ETag {have} instead of {want}" + ))); + } + if got.meta.size != len { + return Err(RemoteError::FileChanged(format!( + "{path}: size {} instead of {len}", + got.meta.size + ))); + } + let bytes = got.bytes().await.map_err(os_error)?; + if bytes.len() as u64 != r.end - r.start { + return Err(RemoteError::BadResponse(format!( + "{path}: {} bytes for range {r:?}", + bytes.len() + ))); + } + Ok::<_, RemoteError>((i, bytes.to_vec())) + } + }) + .buffer_unordered(MAX_CONCURRENT) + .try_collect() + .await + })?; + let mut out = vec![Vec::new(); ranges.len()]; + for (i, b) in fetched { + self.bytes.fetch_add(b.len() as u64, Ordering::Relaxed); + out[i] = b; + } + Ok(out) + } +} + +impl Drop for ObjectStoreStorage { + fn drop(&mut self) { + // Dropping a runtime blocks, which panics inside an async context. + if let Some(rt) = self.runtime.take() { + rt.shutdown_background(); + } + } +} + +impl Storage for ObjectStoreStorage { + fn read_at(&self, offset: u64, len: usize) -> Result, FormatError> { + let range = offset..offset.saturating_add(len as u64); + let mut v = self.fetch_all(std::slice::from_ref(&range))?; + Ok(Cow::Owned(v.pop().unwrap_or_default())) + } + + fn len(&self) -> u64 { + self.meta.size + } + + fn read_ranges(&self, ranges: &[Range]) -> Result>, FormatError> { + if ranges.iter().any(|r| r.end < r.start) { + return Err(FormatError::Storage( + "read range ends before it starts".into(), + )); + } + Ok(self + .fetch_all(ranges)? + .into_iter() + .map(Cow::Owned) + .collect()) + } +} + +/// The store and object path for a cloud URL (`s3://bucket/key`, +/// `gs://bucket/key`, `az://container/key`, ...), configured from the +/// environment as `object_store`'s `from_env` builders do (`AWS_*`, +/// `GOOGLE_*`, `AZURE_*`). +#[cfg(any(feature = "s3", feature = "gcs", feature = "azure"))] +pub(crate) fn store_for_url(url: &str) -> Result<(Arc, Path), RemoteError> { + let parsed = object_store::path::Path::parse( + url.split_once("://") + .and_then(|(_, rest)| rest.split_once('/')) + .map(|(_, key)| key) + .unwrap_or(""), + ) + .map_err(|e| RemoteError::InvalidUrl(format!("{url}: {e}")))?; + let scheme = url.split_once("://").map(|(s, _)| s.to_ascii_lowercase()); + let store: Arc = match scheme.as_deref() { + #[cfg(feature = "s3")] + Some("s3" | "s3a") => Arc::new( + object_store::aws::AmazonS3Builder::from_env() + .with_url(url) + .build() + .map_err(os_error)?, + ), + #[cfg(feature = "gcs")] + Some("gs") => Arc::new( + object_store::gcp::GoogleCloudStorageBuilder::from_env() + .with_url(url) + .build() + .map_err(os_error)?, + ), + #[cfg(feature = "azure")] + Some("az" | "azure" | "abfs" | "abfss" | "adl") => Arc::new( + object_store::azure::MicrosoftAzureBuilder::from_env() + .with_url(url) + .build() + .map_err(os_error)?, + ), + _ => return Err(RemoteError::UnsupportedScheme(url.to_string())), + }; + Ok((store, parsed)) +} + +#[cfg(all(test, feature = "s3"))] +mod tests { + #[test] + fn s3_urls_name_the_bucket_and_key() { + let (store, path) = super::store_for_url("s3://my-bucket/dir/file.h5").unwrap(); + assert_eq!(path.as_ref(), "dir/file.h5"); + assert!(store.to_string().contains("my-bucket"), "{store}"); + } +} diff --git a/crates/clawhdf5-remote/tests/http.rs b/crates/clawhdf5-remote/tests/http.rs index e7faed8..197e268 100644 --- a/crates/clawhdf5-remote/tests/http.rs +++ b/crates/clawhdf5-remote/tests/http.rs @@ -474,10 +474,11 @@ fn bad_urls_and_statuses_are_clean_errors() { open_url("no-scheme"), Err(Error::Remote(RemoteError::InvalidUrl(_))) )); - assert!(matches!( - open_url("s3://bucket/key.h5"), - Err(Error::Remote(RemoteError::UnsupportedScheme(_))) - )); + #[cfg(not(feature = "s3"))] + { + let e = open_url("s3://bucket/key.h5").unwrap_err().to_string(); + assert!(e.contains("`s3` feature"), "{e}"); + } #[cfg(not(feature = "https"))] { let e = open_url("https://example.com/a.h5") diff --git a/crates/clawhdf5-remote/tests/object_store.rs b/crates/clawhdf5-remote/tests/object_store.rs new file mode 100644 index 0000000..0d2f5fa --- /dev/null +++ b/crates/clawhdf5-remote/tests/object_store.rs @@ -0,0 +1,118 @@ +//! `ObjectStoreStorage` against object_store's in-memory and local-file +//! backends (no cloud needed): values equal `File::open`'s, the block cache +//! coalesces, and an object replaced while open is an error. + +#![cfg(feature = "object-store")] + +mod common; + +use std::sync::Arc; + +use clawhdf5::File; +use clawhdf5_format::storage::Storage; +use clawhdf5_remote::object_store::memory::InMemory; +use clawhdf5_remote::object_store::path::Path as ObjectPath; +use clawhdf5_remote::object_store::{ObjectStore, ObjectStoreExt, PutPayload}; +use clawhdf5_remote::{ObjectStoreStorage, Options, open_object}; +use common::{multi_block_file, transcript}; + +fn put(store: &dyn ObjectStore, path: &str, bytes: Vec) { + let rt = tokio_rt(); + rt.block_on(store.put(&ObjectPath::from(path), PutPayload::from(bytes))) + .unwrap(); +} + +fn tokio_rt() -> tokio::runtime::Runtime { + tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .unwrap() +} + +#[test] +fn in_memory_store_reads_like_file_open() { + let store: Arc = Arc::new(InMemory::new()); + let mut compared = 0; + for (i, p) in common::fixtures().iter().enumerate() { + let bytes = std::fs::read(p).unwrap(); + let key = format!("f{i}.h5"); + put(store.as_ref(), &key, bytes); + let (Ok(local), Ok((mut remote, cache))) = ( + File::open(p), + open_object(store.clone(), &key, &Options::default()), + ) else { + continue; + }; + remote.set_vds_resolver(common::sibling_resolver(p.parent().unwrap().into())); + assert!(remote.contiguous_bytes().is_none()); + assert_eq!(transcript(&remote), transcript(&local), "{}", p.display()); + assert!(cache.stats().reads > 0); + compared += 1; + } + assert!(compared >= 40, "{compared}"); +} + +#[test] +fn multi_block_object_is_fetched_in_coalesced_blocks() { + let store: Arc = Arc::new(InMemory::new()); + let bytes = multi_block_file(); + put(store.as_ref(), "m.h5", bytes.clone()); + let (remote, cache) = open_object(store, "m.h5", &Options::default()).unwrap(); + let local = File::from_bytes(bytes.clone()).unwrap(); + assert_eq!( + remote.dataset("big").unwrap().read_f64().unwrap(), + local.dataset("big").unwrap().read_f64().unwrap() + ); + let s = cache.stats(); + let blocks = (bytes.len() as u64).div_ceil(1 << 20); + assert!(s.bytes_fetched <= bytes.len() as u64); + assert!(s.requests <= blocks, "{s:?}"); + assert_eq!(cache.inner().len(), bytes.len() as u64); +} + +#[test] +fn local_file_store_reads_like_file_open() { + let dir = tempfile::tempdir().unwrap(); + let bytes = multi_block_file(); + std::fs::write(dir.path().join("m.h5"), &bytes).unwrap(); + let store: Arc = Arc::new( + clawhdf5_remote::object_store::local::LocalFileSystem::new_with_prefix(dir.path()).unwrap(), + ); + let (remote, _) = open_object(store, "m.h5", &Options::default()).unwrap(); + let local = File::from_bytes(bytes).unwrap(); + assert_eq!(transcript(&remote), transcript(&local)); +} + +#[test] +fn an_object_replaced_while_open_is_an_error() { + let store: Arc = Arc::new(InMemory::new()); + let bytes = multi_block_file(); + put(store.as_ref(), "m.h5", bytes.clone()); + let (remote, _) = open_object(store.clone(), "m.h5", &Options::default()).unwrap(); + assert_eq!(remote.root().groups().unwrap(), ["grp"]); + let mut other = bytes; + let n = other.len(); + other[n / 2] ^= 0xff; + put(store.as_ref(), "m.h5", other); + let err = remote + .dataset("big") + .unwrap() + .read_f64() + .unwrap_err() + .to_string(); + assert!(err.contains("changed while open"), "{err}"); +} + +#[test] +fn missing_objects_and_async_callers_are_clean_errors() { + let store: Arc = Arc::new(InMemory::new()); + assert!(ObjectStoreStorage::new(store.clone(), ObjectPath::from("nope.h5")).is_err()); + put(store.as_ref(), "m.h5", multi_block_file()); + let storage = ObjectStoreStorage::new(store, ObjectPath::from("m.h5")).unwrap(); + // From inside a runtime: refused, not a panic or a deadlock. + let rt = tokio_rt(); + let r = rt.block_on(async { storage.read_at(0, 10).map(|b| b.len()) }); + assert!(r.unwrap_err().to_string().contains("spawn_blocking")); + // Dropping the storage inside a runtime does not panic. + rt.block_on(async move { drop(storage) }); +} diff --git a/scripts/ci-test.sh b/scripts/ci-test.sh index 2fce9c1..fe7242c 100755 --- a/scripts/ci-test.sh +++ b/scripts/ci-test.sh @@ -109,11 +109,12 @@ run_step "cargo clippy (fast-deflate / zlib-ng)" cargo clippy \ --features clawhdf5-format/fast-deflate,clawhdf5-filters/fast-deflate \ -- -D warnings -# clawhdf5-remote's HTTPS backend (rustls). -run_step "cargo clippy (remote, https)" cargo clippy \ +# clawhdf5-remote's optional backends: object_store (in-memory and local +# stores in the tests), HTTPS through rustls, and the cloud stores. +run_step "cargo clippy (remote, all backends)" cargo clippy \ -p clawhdf5-remote \ --all-targets \ - --features https \ + --features object-store,https,s3,gcs,azure \ -- -D warnings # The README promises that the core crates build no C by default. Hold it to @@ -121,15 +122,16 @@ run_step "cargo clippy (remote, https)" cargo clippy \ # default dependency tree of any of them. clawhdf5-migrate (bundled SQLite), # clawhdf5-napi (Node) and clawhdf5-gpu (graphics drivers) are exempt. # js-sys (clawhdf5-wasm's bindings to JavaScript) builds no C. -# clawhdf5-remote is checked by default (plain HTTP); its https feature -# (ring) builds C and is opt-in. +# clawhdf5-remote is checked by default (plain HTTP) and with its +# object-store feature; its https (ring) and s3/gcs/azure (aws-lc-rs) +# features build C and are opt-in. no_c_in_default_build() { local entry crate features found=0 for entry in clawhdf5-format clawhdf5-io clawhdf5-filters clawhdf5 \ clawhdf5-agent clawhdf5-ann clawhdf5-accel clawhdf5-netcdf4 clawhdf5-cli \ clawhdf5-tools \ clawhdf5-wasm \ - clawhdf5-remote; do + clawhdf5-remote clawhdf5-remote:object-store; do crate=${entry%%:*} features=() [ "$entry" != "$crate" ] && features=(--features "${entry#*:}") @@ -204,6 +206,10 @@ run_step "cargo test (facade parallel)" cargo test \ -p clawhdf5 \ --features parallel +run_step "cargo test (remote, object_store backend, s3 URLs)" cargo test \ + -p clawhdf5-remote \ + --features object-store,s3 + run_step "cargo test (ann parallel)" cargo test \ -p clawhdf5-ann \ --features parallel From c54c64cc9bf3638f347df247055f6a22c6a6f87d Mon Sep 17 00:00:00 2001 From: osobh Date: Sat, 26 Sep 2026 17:16:35 -0500 Subject: [PATCH 03/20] clawhdf5: File::storage gives the file's view as a Storage MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The bytes File::as_bytes returns (from the superblock on, bounded by the recorded end of file, a metadata cache image laid over), as a &(dyn Storage + Send + Sync) for every backend. Code that parses the file itself with the clawhdf5_format *_in functions — h5rs does — can then read a file opened with File::open_storage (a remote file) as well as a local one; in memory its as_contiguous() is as_bytes(), so local reads stay slices. Test: for every fixture, File::open's storage() is as_bytes() as its contiguous view, and File::open_storage over a read_at-only storage gives the same bytes through storage().read_at. Co-Authored-By: Claude Opus 5.5 (1M context) --- crates/clawhdf5/src/reader.rs | 11 +++++++ crates/clawhdf5/tests/storage_equivalence.rs | 30 ++++++++++++++++++++ 2 files changed, 41 insertions(+) diff --git a/crates/clawhdf5/src/reader.rs b/crates/clawhdf5/src/reader.rs index 6bd7b35..2a9140c 100644 --- a/crates/clawhdf5/src/reader.rs +++ b/crates/clawhdf5/src/reader.rs @@ -523,6 +523,17 @@ impl File { self.data.contiguous() } + /// The bytes [`as_bytes`](Self::as_bytes) returns, as a [`Storage`], + /// for every backend: from the superblock on, bounded by the recorded + /// end of file, with a metadata cache image laid over them. Code that + /// parses the file itself with the `clawhdf5_format` `*_in` functions + /// reads through this, so it works on remote files too; for a file in + /// memory its [`Storage::as_contiguous`] is [`as_bytes`](Self::as_bytes) + /// (and every read a slice of it). + pub fn storage(&self) -> &(dyn Storage + Send + Sync) { + &self.data + } + /// The error of a metadata cache image libhdf5 cannot load, when the /// file has one. Such a file opens, as in libhdf5, and every object /// lookup fails with this error; code that parses [`Self::as_bytes`] diff --git a/crates/clawhdf5/tests/storage_equivalence.rs b/crates/clawhdf5/tests/storage_equivalence.rs index 7a2749b..6793d47 100644 --- a/crates/clawhdf5/tests/storage_equivalence.rs +++ b/crates/clawhdf5/tests/storage_equivalence.rs @@ -432,3 +432,33 @@ fn storage_backed_files_keep_their_zero_copy_views_only_in_memory() { "as_bytes over a range storage must not answer" ); } + +/// `File::storage` is the view `as_bytes` gives, for every backend: the +/// user block skipped, bounded by the end of file, a cache image laid over. +#[test] +fn file_storage_is_the_as_bytes_view_for_every_backend() { + let root = Path::new(env!("CARGO_MANIFEST_DIR")); + let mut files = Vec::new(); + hdf5_files(&root.join("tests/fixtures"), &mut files); + hdf5_files(&root.join("../clawhdf5-format/tests/fixtures"), &mut files); + let mut compared = 0; + for p in files { + let Ok(local) = File::open(&p) else { continue }; + let bytes = std::fs::read(&p).unwrap(); + let remote = File::open_storage(Arc::new(CountingStorage::new(bytes))).unwrap(); + let want = local.as_bytes(); + let view = local.storage(); + assert_eq!(view.as_contiguous(), Some(want), "{}", p.display()); + let got = remote.storage(); + assert!(got.as_contiguous().is_none()); + assert_eq!(got.len(), want.len() as u64, "{}", p.display()); + assert_eq!( + &*got.read_at(0, want.len()).unwrap(), + want, + "{}", + p.display() + ); + compared += 1; + } + assert!(compared >= 40, "{compared}"); +} From a4f586e6574791b49180a2e35e6cca12368b85ea Mon Sep 17 00:00:00 2001 From: osobh Date: Sat, 26 Sep 2026 17:20:50 -0500 Subject: [PATCH 04/20] format: VlResolver::element_in and string_element_in over any Storage VlResolver::element and string_element return slices of the whole file, so they exist only for a resolver over &[u8]. Their *_in forms work for any Storage (a remote file): the element's bytes borrowed from the resolver's cache of heap collections, with the same null-element, NUL and size checks. h5rs decodes variable-length values with them. Test: over a read_at-only storage they give what element/string_element give over the slice, for a string with an embedded NUL, a null element and an element whose heap object has the wrong size. Co-Authored-By: Claude Opus 5.5 (1M context) --- crates/clawhdf5-format/src/vl_data.rs | 44 +++++++++++++++++++++++++++ 1 file changed, 44 insertions(+) diff --git a/crates/clawhdf5-format/src/vl_data.rs b/crates/clawhdf5-format/src/vl_data.rs index f37638b..378ada4 100644 --- a/crates/clawhdf5-format/src/vl_data.rs +++ b/crates/clawhdf5-format/src/vl_data.rs @@ -305,6 +305,24 @@ impl<'a, S: crate::storage::Storage + ?Sized> VlResolver<'a, S> { Ok(Some(data)) } + /// [`VlResolver::element`] over any storage: the element's bytes + /// (borrowed from the resolver's cache of heap collections, so they + /// live until the next call), or `None` for a null element. + pub fn element_in( + &mut self, + elem: &[u8], + base_size: usize, + ) -> Result, FormatError> { + let vl = parse_vl_references(elem, 1, self.offset_size)?; + self.resolve(&vl[0], base_size) + } + + /// [`VlResolver::string_element`] over any storage (see + /// [`element_in`](Self::element_in)). + pub fn string_element_in(&mut self, elem: &[u8]) -> Result, FormatError> { + Ok(self.element_in(elem, 1)?.map(cut_at_nul)) + } + /// The strings of the variable-length string elements in `raw`, as /// bytes. A string ends at its first NUL, as libhdf5 returns it (it /// converts each to a C string); a null element is empty. @@ -645,6 +663,32 @@ mod tests { } } + #[test] + fn element_in_over_a_storage_matches_element_over_a_slice() { + let mut file_data = vec![0u8; 512]; + build_gcol_at(&mut file_data, 256, &[(1, b"Alice\0x"), (2, b"Bob")]); + let mut raw = build_vl_refs(&["Alice\0x", "Bob"], 256, 1, 8); + raw.extend(element(0, 0, 0, 8)); // null + raw.extend(element(9, 256, 1, 8)); // wrong length: an error + let storage = crate::storage::CountingStorage::new(file_data.clone()); + let dynamic: &dyn crate::storage::Storage = &storage; + let mut slice = VlResolver::new(&file_data, 8, 8); + let mut any = VlResolver::new_in(dynamic, 8, 8); + for e in raw.chunks(16) { + let want = slice.element(e, 1).map(|o| o.map(<[u8]>::to_vec)); + let got = any.element_in(e, 1).map(|o| o.map(<[u8]>::to_vec)); + assert_eq!(format!("{want:?}"), format!("{got:?}")); + let want = slice.string_element(e).map(|o| o.map(<[u8]>::to_vec)); + let got = any.string_element_in(e).map(|o| o.map(<[u8]>::to_vec)); + assert_eq!(format!("{want:?}"), format!("{got:?}")); + } + assert_eq!( + any.string_element_in(&raw[..16]).unwrap(), + Some(&b"Alice"[..]) + ); + assert!(storage.reads() > 0); + } + #[test] fn null_vl_element_zero_address() { let mut raw = Vec::new(); From c513f7e6d787db81ce3329e57b59b3daaddde291 Mon Sep 17 00:00:00 2001 From: osobh Date: Sat, 26 Sep 2026 17:23:47 -0500 Subject: [PATCH 05/20] h5rs: URLs as FILE arguments (feature `remote`) With the `remote` feature (`remote-https` for https://), ls, dump, stat and diff take an http(s):// (or s3://, gs://, az:// with those clawhdf5-remote features) URL wherever they take a file, and read it by range requests through clawhdf5-remote's block cache. check validates every byte, so it downloads a remote file whole and checks it as before. Without the feature a URL is a clean error naming it. The tools read the file through File::storage instead of as_bytes: object headers, shared messages, attributes, v1 and v2 group links, dense storage (fractal heaps and v2 B-trees), path resolution, chunk listings and variable-length values go through the format crate's *_in functions, and the fractal-heap block verifier reads each block through the storage (a read failure of a remote file is reported as a problem, not as "past the end of the file"). A local file's storage is its mapped bytes, so its reads are still slices. stat's file size comes from the opened file, so it is right for a URL. Tests: tests/remote.rs serves fixtures (old and new formats, a paged file, a metadata cache image, a multi-block fractal heap, compounds, v1 groups) with the clawhdf5-remote test server and requires every subcommand's output and exit status for the URL to equal the local file's, and diff of the two to be clean; 404s, non-HDF5 bodies and https without its feature are clean errors. Local output is unchanged: the old and new h5rs print the same for ls -r -v, dump, stat and check --data on the 747 conformance and CVE corpus files (tank, 2026-09-26; the dumps of h5diff_hyper1/2.h5 were too large for the comparison script, their ls, stat and check agree), except cve-2025-2310.h5, whose dump error messages differ between runs of the old binary too (which failing chunk is reported first). ci-test.sh lints h5rs with remote-https, runs the URL tests and checks h5rs with remote for C. Co-Authored-By: Claude Opus 5.5 (1M context) --- crates/clawhdf5-tools/Cargo.toml | 7 ++ crates/clawhdf5-tools/README.md | 20 ++++ crates/clawhdf5-tools/src/check.rs | 25 ++-- crates/clawhdf5-tools/src/diff.rs | 2 +- crates/clawhdf5-tools/src/dump.rs | 2 +- crates/clawhdf5-tools/src/h5.rs | 139 +++++++++++++++++++---- crates/clawhdf5-tools/src/heap_blocks.rs | 82 ++++++++++--- crates/clawhdf5-tools/src/info.rs | 6 +- crates/clawhdf5-tools/src/ls.rs | 2 +- crates/clawhdf5-tools/src/stat.rs | 4 +- crates/clawhdf5-tools/src/value.rs | 11 +- crates/clawhdf5-tools/tests/remote.rs | 100 ++++++++++++++++ scripts/ci-test.sh | 18 ++- 13 files changed, 357 insertions(+), 61 deletions(-) create mode 100644 crates/clawhdf5-tools/tests/remote.rs diff --git a/crates/clawhdf5-tools/Cargo.toml b/crates/clawhdf5-tools/Cargo.toml index d54f39d..d6cf7d5 100644 --- a/crates/clawhdf5-tools/Cargo.toml +++ b/crates/clawhdf5-tools/Cargo.toml @@ -14,9 +14,16 @@ readme = "README.md" name = "h5rs" path = "src/main.rs" +[features] +# FILE arguments may be URLs: http:// with `remote` (no C), https:// with +# `remote-https` (rustls + ring, which compiles C). +remote = ["dep:clawhdf5-remote"] +remote-https = ["remote", "clawhdf5-remote/https"] + [dependencies] clawhdf5 = { path = "../clawhdf5", version = "2.7.0" } clawhdf5-format = { path = "../clawhdf5-format", version = "2.7.0" } +clawhdf5-remote = { path = "../clawhdf5-remote", version = "2.7.0", optional = true } serde_json = "1" [dev-dependencies] diff --git a/crates/clawhdf5-tools/README.md b/crates/clawhdf5-tools/README.md index 8f089c3..42d829d 100644 --- a/crates/clawhdf5-tools/README.md +++ b/crates/clawhdf5-tools/README.md @@ -24,6 +24,26 @@ Every command takes `--max-bytes N` where it reads values (default 1 GiB): a dataset whose dataspace claims more than that is reported instead of read, so a corrupt size cannot exhaust memory. +### Remote files + +Built with the `remote` feature, every FILE argument may be an `http://` +URL (`remote-https` adds `https://`, through rustls and ring, which compiles +C; the default build has neither). The file is read by range requests +through [clawhdf5-remote](../clawhdf5-remote/README.md)'s block cache, so +`ls` of a large file fetches its metadata blocks, not the file: + +```bash +cargo install --path crates/clawhdf5-tools --features remote +h5rs ls -r -v http://127.0.0.1:8000/file.h5 +h5rs dump http://127.0.0.1:8000/file.h5 +h5rs diff local.h5 http://127.0.0.1:8000/file.h5 +``` + +A URL names the whole file (`FILE/OBJECT` suffixes are for local paths). +`check` validates every byte, so it downloads a remote file whole first. +The output is the local file's (`tests/remote.rs` compares every +subcommand). + ## `h5rs ls` ```console diff --git a/crates/clawhdf5-tools/src/check.rs b/crates/clawhdf5-tools/src/check.rs index da90247..e7d97e5 100644 --- a/crates/clawhdf5-tools/src/check.rs +++ b/crates/clawhdf5-tools/src/check.rs @@ -148,13 +148,24 @@ pub fn run(args: &mut Args, out: &mut Out) -> std::io::Result { return args.usage_error(out, "missing FILE", USAGE); }; let path = std::path::Path::new(&file); - if !path.is_file() { - writeln!(out.e, "h5rs check: {file}: no such file")?; - return Ok(2); - } - let mut h5 = match H5::open(path) { - Ok(h) => h, - Err(_) => return unopenable(path, out), + let mut h5 = if crate::h5::is_url(&file) { + // check validates every byte, so a remote file is downloaded whole. + match H5::open_arg_whole(&file) { + Ok(h) => h, + Err(e) => { + writeln!(out.e, "h5rs check: {e}")?; + return Ok(2); + } + } + } else { + if !path.is_file() { + writeln!(out.e, "h5rs check: {file}: no such file")?; + return Ok(2); + } + match H5::open(path) { + Ok(h) => h, + Err(_) => return unopenable(path, out), + } }; if let Some(m) = max_bytes { h5.max_bytes = m; diff --git a/crates/clawhdf5-tools/src/diff.rs b/crates/clawhdf5-tools/src/diff.rs index 98d4ba5..af684c5 100644 --- a/crates/clawhdf5-tools/src/diff.rs +++ b/crates/clawhdf5-tools/src/diff.rs @@ -174,7 +174,7 @@ pub fn run(args: &mut Args, out: &mut Out) -> std::io::Result { } let mut files = Vec::new(); for f in &pos[..2] { - match H5::open(std::path::Path::new(f)) { + match H5::open_arg(f) { Ok(mut h) => { if let Some(m) = max_bytes { h.max_bytes = m; diff --git a/crates/clawhdf5-tools/src/dump.rs b/crates/clawhdf5-tools/src/dump.rs index 7b1d8e7..4b59f00 100644 --- a/crates/clawhdf5-tools/src/dump.rs +++ b/crates/clawhdf5-tools/src/dump.rs @@ -82,7 +82,7 @@ pub fn run(args: &mut Args, out: &mut Out) -> std::io::Result { let Some(file) = file else { return args.usage_error(out, "missing FILE", USAGE); }; - let mut h5 = match H5::open(std::path::Path::new(&file)) { + let mut h5 = match H5::open_arg(&file) { Ok(h) => h, Err(e) => { writeln!(out.e, "h5rs dump: {e}")?; diff --git a/crates/clawhdf5-tools/src/h5.rs b/crates/clawhdf5-tools/src/h5.rs index 5f71a14..c63ccb3 100644 --- a/crates/clawhdf5-tools/src/h5.rs +++ b/crates/clawhdf5-tools/src/h5.rs @@ -10,9 +10,9 @@ use std::collections::HashMap; use std::path::{Path, PathBuf}; use clawhdf5::File; -use clawhdf5_format::attribute::{AttributeMessage, extract_attributes_tolerant}; +use clawhdf5_format::attribute::{AttributeMessage, extract_attributes_tolerant_in}; use clawhdf5_format::attribute_info::AttributeInfoMessage; -use clawhdf5_format::btree_v2::{BTreeV2Header, collect_btree_v2_records}; +use clawhdf5_format::btree_v2::{BTreeV2Header, collect_btree_v2_records_in}; use clawhdf5_format::data_layout::DataLayout; use clawhdf5_format::dataspace::{Dataspace, DataspaceType}; use clawhdf5_format::datatype::Datatype; @@ -24,6 +24,7 @@ use clawhdf5_format::link_info::LinkInfoMessage; use clawhdf5_format::link_message::{LinkMessage, LinkTarget}; use clawhdf5_format::message_type::MessageType; use clawhdf5_format::object_header::ObjectHeader; +use clawhdf5_format::storage::Storage; use clawhdf5_format::superblock::Superblock; use clawhdf5_format::symbol_table::SymbolTableMessage; @@ -186,8 +187,11 @@ pub struct Link { /// An open file. pub struct H5 { + /// The file's path, or its URL for a remote file. pub path: PathBuf, pub file: File, + /// Size of the whole file in bytes (user block included). + pub size: u64, pub max_bytes: u64, /// Fractal heaps whose blocks were verified: `None` = sound. verified_heaps: RefCell>>, @@ -204,19 +208,90 @@ impl H5 { path.display() )) })?; - Ok(H5 { - path: path.to_path_buf(), + let size = std::fs::metadata(path).map(|m| m.len()).unwrap_or(0); + Ok(H5::new(path.to_path_buf(), file, size)) + } + + fn new(path: PathBuf, file: File, size: u64) -> H5 { + H5 { + path, file, + size, max_bytes: DEFAULT_MAX_BYTES, verified_heaps: RefCell::new(HashMap::new()), - }) + } + } + + /// Open a command-line FILE argument: a path, or with the `remote` + /// feature an `http(s)://` (or `s3://`, `gs://`, `az://`) URL, read by + /// range requests through a block cache. + pub fn open_arg(arg: &str) -> Result
{ + if !is_url(arg) { + return H5::open(Path::new(arg)); + } + #[cfg(feature = "remote")] + { + let storage = + clawhdf5_remote::storage_for_url(arg, &clawhdf5_remote::Options::default()) + .map_err(|e| Error::new(format!("{arg}: {e}")))?; + let size = storage.len(); + let file = File::open_storage(storage).map_err(|e| { + Error::new(format!("{arg}: not an HDF5 file this tool can open: {e}")) + })?; + Ok(H5::new(PathBuf::from(arg), file, size)) + } + #[cfg(not(feature = "remote"))] + Err(Error::new(format!( + "{arg}: URLs need h5rs built with the `remote` feature" + ))) + } + + /// [`H5::open_arg`], with a remote file downloaded whole into memory + /// first — for `check`, which validates every byte of the file anyway + /// and parses it as one slice ([`H5::data`]). + pub fn open_arg_whole(arg: &str) -> Result
{ + let h5 = H5::open_arg(arg)?; + if h5.file.contiguous_bytes().is_some() { + return Ok(h5); + } + #[cfg(feature = "remote")] + { + let storage = + clawhdf5_remote::storage_for_url(arg, &clawhdf5_remote::Options::default()) + .map_err(|e| Error::new(format!("{arg}: {e}")))?; + let len = usize::try_from(storage.len()) + .map_err(|_| Error::new(format!("{arg}: too large to download")))?; + let bytes = storage + .read_at(0, len) + .map_err(|e| Error::new(format!("{arg}: {e}")))? + .into_owned(); + let size = bytes.len() as u64; + let file = File::from_bytes(bytes).map_err(|e| { + Error::new(format!("{arg}: not an HDF5 file this tool can open: {e}")) + })?; + Ok(H5::new(PathBuf::from(arg), file, size)) + } + #[cfg(not(feature = "remote"))] + unreachable!("open_arg refuses URLs without the remote feature") } /// The file's bytes from the superblock on: what every address indexes. + /// + /// # Panics + /// + /// For a remote file opened by [`H5::open_arg`]; read it through + /// [`H5::store`], or open it with [`H5::open_arg_whole`]. pub fn data(&self) -> &[u8] { self.file.as_bytes() } + /// The same bytes as a [`Storage`], for local and remote files alike: + /// in memory a read is a slice of [`H5::data`], remote it is served by + /// the block cache. + pub fn store(&self) -> &dyn Storage { + self.file.storage() + } + pub fn sb(&self) -> &Superblock { self.file.superblock() } @@ -239,8 +314,8 @@ impl H5 { if let Some(e) = self.file.cache_image_error() { return Err(Error::at(addr, format!("metadata cache image: {e}"))); } - let off = usize::try_from(addr).map_err(|_| Error::at(addr, "address out of range"))?; - ObjectHeader::parse(self.data(), off, self.os(), self.ls()) + to_usize(addr)?; + ObjectHeader::parse_in(self.store(), addr, self.os(), self.ls()) .map_err(|e| Error::at(addr, format!("object header: {e}"))) } @@ -249,11 +324,14 @@ impl H5 { pub fn payload(&self, h: &ObjectHeader, t: MessageType) -> Result>> { match h.messages.iter().find(|m| m.msg_type == t) { None => Ok(None), - Some(m) => { - clawhdf5_format::shared_message::message_data(self.data(), m, self.os(), self.ls()) - .map(|c| Some(c.into_owned())) - .map_err(|e| Error::new(format!("{t:?} message: {e}"))) - } + Some(m) => clawhdf5_format::shared_message::message_data_in( + self.store(), + m, + self.os(), + self.ls(), + ) + .map(|c| Some(c.into_owned())) + .map_err(|e| Error::new(format!("{t:?} message: {e}"))), } } @@ -305,8 +383,9 @@ impl H5 { self.verified_heap(fh) .map_err(|e| e.context("dense attribute storage"))?; } - let (mut attrs, errs) = extract_attributes_tolerant(self.data(), h, self.os(), self.ls()) - .map_err(|e| Error::new(format!("attributes: {e}")))?; + let (mut attrs, errs) = + extract_attributes_tolerant_in(self.store(), h, self.os(), self.ls()) + .map_err(|e| Error::new(format!("attributes: {e}")))?; attrs.sort_by(|a, b| a.name.cmp(&b.name)); Ok((attrs, errs.iter().map(|e| e.to_string()).collect())) } @@ -316,7 +395,7 @@ impl H5 { pub fn links(&self, h: &ObjectHeader) -> Result> { let os = self.os(); let ls = self.ls(); - let data = self.data(); + let data = self.store(); let mut out = Vec::new(); if let Some(m) = h .messages @@ -325,7 +404,7 @@ impl H5 { { let stm = SymbolTableMessage::parse(&m.data, os) .map_err(|e| Error::new(format!("symbol table message: {e}")))?; - let entries = group_v1::resolve_v1_group_entries(data, &stm, os, ls) + let entries = group_v1::resolve_v1_group_entries_in(data, &stm, os, ls) .map_err(|e| Error::at(stm.btree_address, format!("symbol table: {e}")))?; let has_soft = entries.iter().any(group_v1::is_v1_soft_link); for e in entries { @@ -337,7 +416,7 @@ impl H5 { } } if has_soft { - let soft = group_v1::v1_soft_links(data, &stm, os, ls) + let soft = group_v1::v1_soft_links_in(data, &stm, os, ls) .map_err(|e| Error::at(stm.btree_address, format!("soft links: {e}")))?; for (name, target) in soft { out.push(Link { @@ -387,15 +466,15 @@ impl H5 { name_type: u8, ) -> Result>> { self.verified_heap(heap)?; - let data = self.data(); + let data = self.store(); let os = self.os(); let ls = self.ls(); - let fh = FractalHeapHeader::parse(data, to_usize(heap)?, os, ls) + let fh = FractalHeapHeader::parse_in(data, to_usize(heap).map(|_| heap)?, os, ls) .map_err(|e| Error::at(heap, format!("fractal heap header: {e}")))?; let bt = btree.ok_or_else(|| Error::at(heap, "dense storage without a name index"))?; - let hdr = BTreeV2Header::parse(data, to_usize(bt)?, os, ls) + let hdr = BTreeV2Header::parse_in(data, to_usize(bt).map(|_| bt)?, os, ls) .map_err(|e| Error::at(bt, format!("v2 B-tree header: {e}")))?; - let recs = collect_btree_v2_records(data, &hdr, os, ls) + let recs = collect_btree_v2_records_in(data, &hdr, os, ls) .map_err(|e| Error::at(bt, format!("v2 B-tree: {e}")))?; // Name-index records: hash(4) + heap ID; creation-order ones: order(8) + heap ID. let skip = if hdr.tree_type == name_type { 4 } else { 8 }; @@ -407,7 +486,7 @@ impl H5 { .get(skip..skip + idlen) .ok_or_else(|| Error::at(bt, "v2 B-tree record shorter than a heap ID"))?; let obj = fh - .read_managed_object(data, id, os) + .read_managed_object_in(data, id, os) .map_err(|e| Error::at(heap, format!("fractal heap object: {e}")))?; out.push(obj); } @@ -561,7 +640,7 @@ impl H5 { if p.is_empty() { return Ok(self.root()); } - clawhdf5_format::group_v2::resolve_path_any(self.data(), self.sb(), p) + clawhdf5_format::group_v2::resolve_path_any_in(self.store(), self.sb(), p) .map_err(|e| Error::new(format!("{path}: {e}"))) } } @@ -681,7 +760,9 @@ pub fn byte_len(ds: &Dataspace, dt: &Datatype) -> Result { /// Split a `FILE[/object/path]` argument the way h5ls does: the longest /// prefix that is an existing file is the file. pub fn split_file_arg(arg: &str) -> (String, Option) { - if Path::new(arg).is_file() { + // A URL names the file only (its path cannot be split against the + // local file system). + if is_url(arg) || Path::new(arg).is_file() { return (arg.to_string(), None); } let mut idx: Vec = arg.match_indices('/').map(|(i, _)| i).collect(); @@ -694,3 +775,13 @@ pub fn split_file_arg(arg: &str) -> (String, Option) { } (arg.to_string(), None) } + +/// Whether a FILE argument is a URL (`scheme://...`) rather than a path. +pub fn is_url(arg: &str) -> bool { + arg.split_once("://").is_some_and(|(scheme, _)| { + !scheme.is_empty() + && scheme + .chars() + .all(|c| c.is_ascii_alphanumeric() || matches!(c, '+' | '-' | '.')) + }) +} diff --git a/crates/clawhdf5-tools/src/heap_blocks.rs b/crates/clawhdf5-tools/src/heap_blocks.rs index a1c40cd..59cc6d3 100644 --- a/crates/clawhdf5-tools/src/heap_blocks.rs +++ b/crates/clawhdf5-tools/src/heap_blocks.rs @@ -4,10 +4,12 @@ //! heap header's flag says so). The library reads only the blocks an object //! lives in and does not verify block checksums, so `check` does it here. +use std::borrow::Cow; use std::collections::HashSet; use clawhdf5_format::checksum::jenkins_lookup3; use clawhdf5_format::fractal_heap::FractalHeapHeader; +use clawhdf5_format::storage::{Storage, read_exact_at}; use crate::h5::{Error, H5}; @@ -26,7 +28,7 @@ pub struct HeapReport { } struct Walk<'a> { - data: &'a [u8], + data: &'a dyn Storage, heap: u64, fh: FractalHeapHeader, checksum_dblocks: bool, @@ -60,14 +62,14 @@ fn log2(v: u64) -> u32 { /// parsed (and its checksum verified) by the library; an error there is /// returned as the only problem. pub fn verify(h5: &H5, heap: u64) -> HeapReport { - let data = h5.data(); + let data = h5.store(); let Ok(off) = usize::try_from(heap) else { return HeapReport { problems: vec![Error::at(heap, "fractal heap address out of range")], ..Default::default() }; }; - let fh = match FractalHeapHeader::parse(data, off, h5.os(), h5.ls()) { + let fh = match FractalHeapHeader::parse_in(data, heap, h5.os(), h5.ls()) { Ok(f) => f, Err(e) => { return HeapReport { @@ -77,7 +79,15 @@ pub fn verify(h5: &H5, heap: u64) -> HeapReport { } }; // Flags: signature(4) version(1) heap ID length(2) filter length(2) flags(1). - let flags = data.get(off + 9).copied().unwrap_or(0); + let flags = match data.read_at(off as u64 + 9, 1) { + Ok(b) => b.first().copied().unwrap_or(0), + Err(e) => { + return HeapReport { + problems: vec![Error::at(heap, format!("fractal heap header: {e}"))], + ..Default::default() + }; + } + }; let mut w = Walk { data, heap, @@ -107,11 +117,42 @@ pub fn verify(h5: &H5, heap: u64) -> HeapReport { w.r } -impl Walk<'_> { +impl<'a> Walk<'a> { fn problem(&mut self, addr: u64, msg: impl Into) { self.r.problems.push(Error::at(addr, msg)); } + /// Bytes `[start, end)` of the file: `Ok(None)` when they run past its + /// end (what a slice `get` of the whole file answered), `Err` when the + /// storage fails to read them (a remote file). + fn get(&self, start: usize, end: usize) -> Result>, String> { + let Some(len) = end.checked_sub(start) else { + return Ok(None); + }; + if end as u64 > self.data.len() { + return Ok(None); + } + read_exact_at(self.data, start as u64, len) + .map(Some) + .map_err(|e| e.to_string()) + } + + /// [`Walk::get`], recording a read failure as a problem at `addr`. + fn get_or_note( + &mut self, + addr: u64, + start: usize, + end: usize, + ) -> Option>> { + match self.get(start, end) { + Ok(b) => Some(b), + Err(e) => { + self.problem(addr, format!("fractal heap block: {e}")); + None + } + } + } + fn row_size(&self, row: usize) -> Option { let s = self.fh.starting_block_size; if row <= 1 { @@ -155,10 +196,11 @@ impl Walk<'_> { return None; }; let hdr_len = 5 + self.os + self.boff_bytes; - let Some(b) = start - .checked_add(hdr_len) - .and_then(|e| self.data.get(start..e)) - else { + let b = match start.checked_add(hdr_len) { + Some(e) => self.get_or_note(addr, start, e)?, + None => None, + }; + let Some(b) = b else { self.problem( addr, format!("fractal heap {what} block lies past the end of the file"), @@ -209,7 +251,10 @@ impl Walk<'_> { self.problem(addr, "fractal heap direct block size out of range"); return; }; - let Some(block) = self.data.get(start..end) else { + let Some(block) = self.get_or_note(addr, start, end) else { + return; + }; + let Some(block) = block else { self.problem( addr, "fractal heap direct block extends past the end of the file", @@ -259,14 +304,17 @@ impl Walk<'_> { }; let direct = row < direct_rows; for _ in 0..width { - let Some(b) = self.data.get(pos..pos + self.os) else { + let Some(b) = self.get_or_note(addr, pos, pos + self.os) else { + return; + }; + let Some(b) = b else { self.problem( addr, "fractal heap indirect block extends past the end of the file", ); return; }; - let child = le(b); + let child = le(&b); pos += self.os; if direct && filtered { pos += self.ls + 4; @@ -277,7 +325,10 @@ impl Walk<'_> { off = off.saturating_add(rs); } } - let Some(stored) = self.data.get(pos..pos + 4) else { + let Some(stored) = self.get_or_note(addr, pos, pos + 4) else { + return; + }; + let Some(stored) = stored else { self.problem( addr, "fractal heap indirect block extends past the end of the file", @@ -285,7 +336,10 @@ impl Walk<'_> { return; }; let stored = u32::from_le_bytes([stored[0], stored[1], stored[2], stored[3]]); - let computed = jenkins_lookup3(&self.data[start..pos]); + let Some(Some(body)) = self.get_or_note(addr, start, pos) else { + return; + }; + let computed = jenkins_lookup3(&body); self.r.checksums += 1; if computed != stored { self.problem( diff --git a/crates/clawhdf5-tools/src/info.rs b/crates/clawhdf5-tools/src/info.rs index 606457b..d169f73 100644 --- a/crates/clawhdf5-tools/src/info.rs +++ b/crates/clawhdf5-tools/src/info.rs @@ -1,7 +1,7 @@ //! Dataset facts shared by `ls`, `dump`, `stat` and `check`: shape text, //! layout, filters and storage. -use clawhdf5_format::chunked_read::{ChunkInfo, list_chunks}; +use clawhdf5_format::chunked_read::{ChunkInfo, list_chunks_in}; use clawhdf5_format::data_layout::DataLayout; use clawhdf5_format::dataspace::{Dataspace, DataspaceType}; use clawhdf5_format::datatype::Datatype; @@ -189,8 +189,8 @@ pub fn chunks( let Some(addr) = *btree_address else { return Ok(Vec::new()); }; - list_chunks( - h5.data(), + list_chunks_in( + h5.store(), layout, ds, dt.type_size() as usize, diff --git a/crates/clawhdf5-tools/src/ls.rs b/crates/clawhdf5-tools/src/ls.rs index ef966f6..203cce3 100644 --- a/crates/clawhdf5-tools/src/ls.rs +++ b/crates/clawhdf5-tools/src/ls.rs @@ -63,7 +63,7 @@ pub fn run(args: &mut Args, out: &mut Out) -> std::io::Result { return args.usage_error(out, "missing FILE", USAGE); }; let (file, obj) = split_file_arg(&target); - let mut h5 = match H5::open(std::path::Path::new(&file)) { + let mut h5 = match H5::open_arg(&file) { Ok(h) => h, Err(e) => { writeln!(out.e, "h5rs ls: {e}")?; diff --git a/crates/clawhdf5-tools/src/stat.rs b/crates/clawhdf5-tools/src/stat.rs index c7e44da..32e91fe 100644 --- a/crates/clawhdf5-tools/src/stat.rs +++ b/crates/clawhdf5-tools/src/stat.rs @@ -86,7 +86,7 @@ pub fn run(args: &mut Args, out: &mut Out) -> std::io::Result { let Some(file) = file else { return args.usage_error(out, "missing FILE", USAGE); }; - let h5 = match H5::open(std::path::Path::new(&file)) { + let h5 = match H5::open_arg(&file) { Ok(h) => h, Err(e) => { writeln!(out.e, "h5rs stat: {e}")?; @@ -291,7 +291,7 @@ fn report(h5: &H5, file: &str, s: &Stats, out: &mut Out) -> std::io::Result<()> s.attr_objects )?; writeln!(o, "\tMax. # of attributes to objects: {}", s.max_attrs)?; - let total = std::fs::metadata(&h5.path).map(|m| m.len()).unwrap_or(0); + let total = h5.size; let ub = h5.file.user_block_size(); writeln!(o, "Summary of file space information:")?; writeln!(o, " User block: {ub} bytes")?; diff --git a/crates/clawhdf5-tools/src/value.rs b/crates/clawhdf5-tools/src/value.rs index d9255f1..c4f4af6 100644 --- a/crates/clawhdf5-tools/src/value.rs +++ b/crates/clawhdf5-tools/src/value.rs @@ -151,14 +151,14 @@ pub struct Decoder<'a> { pub h5: &'a H5, /// Variable-length elements are resolved as the library resolves them /// (so as libhdf5 does), not by a decoder of our own. - vl: RefCell>, + vl: RefCell>, } impl<'a> Decoder<'a> { pub fn new(h5: &'a H5) -> Self { Self { h5, - vl: RefCell::new(VlResolver::new(h5.data(), h5.os(), h5.ls())), + vl: RefCell::new(VlResolver::new_in(h5.store(), h5.os(), h5.ls())), } } @@ -268,7 +268,7 @@ impl<'a> Decoder<'a> { /// has it. fn decode_vlen(&self, is_string: bool, base: &Datatype, b: &[u8], depth: u32) -> Value { if is_string { - return match self.vl.borrow_mut().string_element(b) { + return match self.vl.borrow_mut().string_element_in(b) { Ok(Some(s)) => Value::Str(String::from_utf8_lossy(s).into_owned()), Ok(None) => Value::NullStr, Err(e) => Value::Error(e.to_string()), @@ -278,8 +278,9 @@ impl<'a> Decoder<'a> { if bs == 0 { return Value::Error("VL base type of size 0".into()); } - let obj = match self.vl.borrow_mut().element(b, bs) { - Ok(o) => o.unwrap_or(&[]), + // Copied out: decoding an element may resolve nested ones. + let obj = match self.vl.borrow_mut().element_in(b, bs) { + Ok(o) => o.unwrap_or(&[]).to_vec(), Err(e) => return Value::Error(e.to_string()), }; Value::Seq( diff --git a/crates/clawhdf5-tools/tests/remote.rs b/crates/clawhdf5-tools/tests/remote.rs new file mode 100644 index 0000000..d14bd61 --- /dev/null +++ b/crates/clawhdf5-tools/tests/remote.rs @@ -0,0 +1,100 @@ +//! `h5rs` on URLs (feature `remote`): every subcommand prints for +//! `http://…/file.h5` what it prints for the local file (the name aside). +//! The files are served by the range-request test server of +//! clawhdf5-remote on 127.0.0.1. + +#![cfg(feature = "remote")] + +#[path = "../../clawhdf5-remote/tests/common/server.rs"] +mod server; + +use std::path::{Path, PathBuf}; +use std::process::Command; + +fn h5rs(args: &[&str]) -> (String, i32) { + let out = Command::new(env!("CARGO_BIN_EXE_h5rs")) + .args(args) + .output() + .expect("run h5rs"); + let text = format!( + "{}{}", + String::from_utf8_lossy(&out.stdout), + String::from_utf8_lossy(&out.stderr) + ); + (text, out.status.code().unwrap_or(-1)) +} + +fn fixtures() -> Vec { + let root = Path::new(env!("CARGO_MANIFEST_DIR")); + [ + "../clawhdf5/tests/fixtures/tall.h5", + "../clawhdf5/tests/fixtures/written_by_v2_7_0.h5", + "../clawhdf5/tests/fixtures/written_by_v2_7_0_paged.h5", + "../clawhdf5/tests/fixtures/h5clear_mdc_image.h5", + "../clawhdf5-format/tests/fixtures/fractal_heap_multiblock.h5", + "../clawhdf5-format/tests/fixtures/legacy/tcompound.h5", + "../clawhdf5-format/tests/fixtures/legacy/h5ex_g_iterate.h5", + ] + .iter() + .map(|p| root.join(p)) + .collect() +} + +#[test] +fn every_subcommand_reads_a_url_like_the_local_file() { + let files = fixtures(); + let served: Vec<(String, Vec)> = files + .iter() + .enumerate() + .map(|(i, p)| { + let name = p.file_name().unwrap().to_str().unwrap(); + (format!("/{i}/{name}"), std::fs::read(p).unwrap()) + }) + .collect(); + let server = server::Server::start(served.clone()); + for (p, (url_path, _)) in files.iter().zip(&served) { + let url = server.url(url_path); + let local = p.to_str().unwrap(); + for cmd in [ + &["ls", "-r", "-v"][..], + &["dump"], + &["dump", "--json"], + &["stat"], + &["check", "--data"], + ] { + fn args<'a>(cmd: &[&'a str], f: &'a str) -> Vec<&'a str> { + cmd.iter().copied().chain([f]).collect() + } + let (want, want_rc) = h5rs(&args(cmd, local)); + let (got, got_rc) = h5rs(&args(cmd, &url)); + assert_eq!( + got.replace(&url, local), + want, + "h5rs {} {url}", + cmd.join(" ") + ); + assert_eq!(got_rc, want_rc, "h5rs {} {url}", cmd.join(" ")); + } + let (a, rc) = h5rs(&["diff", local, &url]); + assert_eq!(rc, 0, "h5rs diff {local} {url}: {a}"); + } +} + +#[test] +fn url_errors_are_clean() { + let server = server::Server::start(vec![("/x.h5".into(), vec![1u8; 100])]); + let (out, rc) = h5rs(&["ls", &server.url("/missing.h5")]); + assert_eq!(rc, 2, "{out}"); + assert!(out.contains("404"), "{out}"); + let (out, rc) = h5rs(&["ls", &server.url("/x.h5")]); + assert_eq!(rc, 2, "{out}"); + assert!(out.contains("not an HDF5 file"), "{out}"); + let (out, rc) = h5rs(&["check", &server.url("/missing.h5")]); + assert_eq!(rc, 2, "{out}"); + #[cfg(not(feature = "remote-https"))] + { + let (out, rc) = h5rs(&["ls", "https://example.com/a.h5"]); + assert_eq!(rc, 2, "{out}"); + assert!(out.contains("`https` feature"), "{out}"); + } +} diff --git a/scripts/ci-test.sh b/scripts/ci-test.sh index fe7242c..4f678ad 100755 --- a/scripts/ci-test.sh +++ b/scripts/ci-test.sh @@ -117,21 +117,28 @@ run_step "cargo clippy (remote, all backends)" cargo clippy \ --features object-store,https,s3,gcs,azure \ -- -D warnings +# h5rs with URL arguments. +run_step "cargo clippy (h5rs remote)" cargo clippy \ + -p clawhdf5-tools \ + --all-targets \ + --features remote-https \ + -- -D warnings + # The README promises that the core crates build no C by default. Hold it to # that: fail if a crate that compiles C (a *-sys crate, cc or cmake) enters the # default dependency tree of any of them. clawhdf5-migrate (bundled SQLite), # clawhdf5-napi (Node) and clawhdf5-gpu (graphics drivers) are exempt. # js-sys (clawhdf5-wasm's bindings to JavaScript) builds no C. # clawhdf5-remote is checked by default (plain HTTP) and with its -# object-store feature; its https (ring) and s3/gcs/azure (aws-lc-rs) -# features build C and are opt-in. +# object-store feature, and h5rs with URL support (remote); the https +# (ring) and s3/gcs/azure (aws-lc-rs) features build C and are opt-in. no_c_in_default_build() { local entry crate features found=0 for entry in clawhdf5-format clawhdf5-io clawhdf5-filters clawhdf5 \ clawhdf5-agent clawhdf5-ann clawhdf5-accel clawhdf5-netcdf4 clawhdf5-cli \ clawhdf5-tools \ clawhdf5-wasm \ - clawhdf5-remote clawhdf5-remote:object-store; do + clawhdf5-remote clawhdf5-remote:object-store clawhdf5-tools:remote; do crate=${entry%%:*} features=() [ "$entry" != "$crate" ] && features=(--features "${entry#*:}") @@ -210,6 +217,11 @@ run_step "cargo test (remote, object_store backend, s3 URLs)" cargo test \ -p clawhdf5-remote \ --features object-store,s3 +run_step "cargo test (h5rs on URLs)" cargo test \ + -p clawhdf5-tools \ + --features remote \ + --test remote + run_step "cargo test (ann parallel)" cargo test \ -p clawhdf5-ann \ --features parallel From ebe51f8e9798e539c26ced33df63c972d1e70654 Mon Sep 17 00:00:00 2001 From: osobh Date: Sat, 26 Sep 2026 17:26:57 -0500 Subject: [PATCH 06/20] clawhdf5-remote tests: open + list and a dataset read counted apart The per-file report now separates a tree view (open, every group's entries, every dataset's shape and type) from reading the largest dataset under 64 MiB, and checks the budget the design's testing section asks for: listing the IMERG file (file A of docs/design/range-reads.md section 2) takes at most 3 requests when CLAWHDF5_REMOTE_CORPUS includes it. The test server now counts a response's bytes before sending it: a client could read a body and reset the counters before the server thread had added it, so the counts of the next file were occasionally too high. Co-Authored-By: Claude Opus 5.5 (1M context) --- crates/clawhdf5-remote/tests/common/mod.rs | 44 +++++++++----- crates/clawhdf5-remote/tests/common/server.rs | 4 +- crates/clawhdf5-remote/tests/http.rs | 58 ++++++++++++------- 3 files changed, 69 insertions(+), 37 deletions(-) diff --git a/crates/clawhdf5-remote/tests/common/mod.rs b/crates/clawhdf5-remote/tests/common/mod.rs index 498b603..9aeb1bc 100644 --- a/crates/clawhdf5-remote/tests/common/mod.rs +++ b/crates/clawhdf5-remote/tests/common/mod.rs @@ -147,11 +147,13 @@ fn dataset(out: &mut String, path: &str, ds: &clawhdf5::Dataset<'_>) { } } -/// Open, list every group, and read the first dataset found whose data is -/// at most `MAX_DATA_BYTES` (a tree view plus one plot). -pub fn list_and_read_one(file: &File) { +/// List the file as a tree view does — every group's entries, every +/// dataset's shape and type — and return the largest dataset whose data +/// is at most `MAX_DATA_BYTES` (address, bytes), the one a viewer would +/// plot. +pub fn list(file: &File) -> Option<(u64, u64)> { let mut seen = HashSet::new(); - let mut read_one = false; + let mut largest: Option<(u64, u64)> = None; let mut queue = VecDeque::from([file.superblock().root_group_address]); while let Some(addr) = queue.pop_front() { if seen.len() >= MAX_OBJECTS || !seen.insert(addr) { @@ -160,22 +162,36 @@ pub fn list_and_read_one(file: &File) { let group = file.group_at(addr); if let Ok(ds) = file.dataset_at(addr) { let _ = (ds.shape(), ds.dtype()); - if !read_one { - let small = ds.shape().ok().and_then(|s| { - let n = s.iter().try_fold(1u64, |a, &d| a.checked_mul(d))?; - let size = u64::from(ds.raw_datatype().ok()?.type_size()); - n.checked_mul(size).filter(|&b| b <= MAX_DATA_BYTES) - }); - if small.is_some() { - let _ = ds.read_selection(&Selection::All); - read_one = true; - } + let bytes = ds.shape().ok().and_then(|s| { + let n = s.iter().try_fold(1u64, |a, &d| a.checked_mul(d))?; + let size = u64::from(ds.raw_datatype().ok()?.type_size()); + n.checked_mul(size).filter(|&b| b <= MAX_DATA_BYTES) + }); + if let Some(b) = bytes + && largest.is_none_or(|(_, l)| b > l) + { + largest = Some((addr, b)); } } if let Ok(entries) = group.entries() { queue.extend(entries.into_iter().map(|(_, a)| a)); } } + largest +} + +/// Read the dataset at `addr` whole. +pub fn read_one(file: &File, addr: u64) { + if let Ok(ds) = file.dataset_at(addr) { + let _ = ds.read_selection(&Selection::All); + } +} + +/// [`list`], then [`read_one`] of the dataset it picks. +pub fn list_and_read_one(file: &File) { + if let Some((addr, _)) = list(file) { + read_one(file, addr); + } } /// External virtual-dataset sources read from `dir`, as `File::open` finds diff --git a/crates/clawhdf5-remote/tests/common/server.rs b/crates/clawhdf5-remote/tests/common/server.rs index e556d7f..4749f50 100644 --- a/crates/clawhdf5-remote/tests/common/server.rs +++ b/crates/clawhdf5-remote/tests/common/server.rs @@ -320,9 +320,11 @@ fn serve(conn: TcpStream, s: &Shared) -> std::io::Result<()> { }; let mut response = head.into_bytes(); response.extend_from_slice(body); + // Counted before the client can have the bytes, so a test that + // resets the counters after a read never sees them arrive late. + s.bytes.fetch_add(body.len() as u64, Ordering::SeqCst); out.write_all(&response)?; out.flush()?; - s.bytes.fetch_add(body.len() as u64, Ordering::SeqCst); if truncate || close { let _ = out.shutdown(std::net::Shutdown::Both); return Ok(()); diff --git a/crates/clawhdf5-remote/tests/http.rs b/crates/clawhdf5-remote/tests/http.rs index 197e268..b0a0e34 100644 --- a/crates/clawhdf5-remote/tests/http.rs +++ b/crates/clawhdf5-remote/tests/http.rs @@ -53,7 +53,7 @@ fn compare(files: &[std::path::PathBuf], report: bool) -> (usize, usize) { .collect(); let server = Server::start(served.clone()); let (mut same, mut refused) = (0, 0); - let mut totals = [0u64; 5]; + let mut totals = [0u64; 7]; for (i, p) in files.iter().enumerate() { let Some((url_path, bytes)) = served .iter() @@ -104,45 +104,59 @@ fn compare(files: &[std::path::PathBuf], report: bool) -> (usize, usize) { assert!(storage.stats().reads > 0, "read through the cache"); same += 1; - // Cost of "list + read one dataset": with the block cache, and - // with none (every read a request). - let with = storage_for_url(&url, &quick()).unwrap(); + // Cost of "open + list" (a tree view) and of then reading the + // largest dataset (a plot): with the block cache, and with none + // (every read a request). Requests and bytes as the server saw + // them, the open included. server.reset(); - if let Ok(f) = File::open_storage(with.clone()) { - list_and_read_one(&f); + let with = storage_for_url(&url, &quick()).unwrap(); + let f = File::open_storage(with.clone()).unwrap(); + let pick = common::list(&f); + let (list_requests, list_bytes) = (server.requests(), server.bytes()); + if let Some((addr, _)) = pick { + common::read_one(&f, addr); } let (cached_requests, cached_bytes) = (server.requests(), server.bytes()); - let (bare, _) = HttpStorage::open(&url, quick().http).unwrap(); - let bare = Arc::new(bare); server.reset(); - if let Ok(f) = File::open_storage(bare.clone()) { + let (bare, _) = HttpStorage::open(&url, quick().http).unwrap(); + if let Ok(f) = File::open_storage(Arc::new(bare)) { list_and_read_one(&f); } let uncached_requests = server.requests(); - totals[0] += 1 + cached_requests; // + the open probe - totals[1] += cached_bytes + with.config().block_size.min(bytes.len() as u64); - totals[2] += 1 + uncached_requests; - totals[3] += bytes.len() as u64; - totals[4] += 1; + for (t, v) in totals.iter_mut().zip([ + list_requests, + list_bytes, + cached_requests, + cached_bytes, + uncached_requests, + bytes.len() as u64, + 1, + ]) { + *t += v; + } if report { eprintln!( - "requests cached {:>6} uncached {:>8} bytes {:>12} of {:>12} {}", - 1 + cached_requests, - 1 + uncached_requests, - cached_bytes + with.config().block_size.min(bytes.len() as u64), + "open+list {list_requests:>4} req {list_bytes:>10} B | +read {:>5} req \ + {cached_bytes:>10} B | uncached {uncached_requests:>7} req | file {:>10} B {}", + cached_requests, bytes.len(), p.display() ); } // Files within one block: opening fetched everything. if bytes.len() as u64 <= with.config().block_size { - assert_eq!(cached_requests, 0, "{}", p.display()); + assert_eq!(cached_requests, 1, "{}", p.display()); + } + // The budget of docs/design/range-reads.md (Testing): listing the + // IMERG file (file A of section 2) takes at most 3 requests. + if p.ends_with("xarray-data/imerghh_730.hdf5") { + assert!(list_requests <= 3, "{}: {list_requests}", p.display()); } } eprintln!( - "list + read one dataset over {} files: {} requests with the 1 MiB block cache \ - ({} bytes transferred, files {} bytes), {} requests without a cache", - totals[4], totals[0], totals[1], totals[3], totals[2] + "{} files ({} bytes): open + list {} requests, {} bytes; + read the largest \ + dataset {} requests, {} bytes (1 MiB block cache); without a cache {} requests", + totals[6], totals[5], totals[0], totals[1], totals[2], totals[3], totals[4] ); (same, refused) } From 955dd1c691fa127237358bb8fba50119f9fcad66 Mon Sep 17 00:00:00 2001 From: osobh Date: Sat, 26 Sep 2026 17:29:09 -0500 Subject: [PATCH 07/20] =?UTF-8?q?docs:=20range-read=20milestone=20M3=20?= =?UTF-8?q?=E2=80=94=20remote=20files?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit README: "Reading remote files" (open_url, the range_server and read_url examples with their real output against the fixtures, h5rs on URLs), the crate in the crate map and the unreleased highlights. CHANGELOG: the clawhdf5-remote crate, h5rs URLs, File::storage and VlResolver::element_in, with the request counts over the conformance corpus (tank, 2026-09-26, the command given). known-issues: the M2 range-read entry updated (the cache now exists; h5rs reads through storage) and a new entry for the remote backends' limits (no Python or browser URLs yet, fixed block size, cloud stores not run against a real bucket, validators, credentials). Design doc: M3 status with the choices that differ from the plan (a crate rather than a clawhdf5-io feature, ureq for HTTP so the default build has no C) and the corpus counts. Co-Authored-By: Claude Opus 5.5 (1M context) --- CHANGELOG.md | 56 +++++++++++++++++++++++++++++++ README.md | 56 +++++++++++++++++++++++++++++-- docs/design/range-reads.md | 43 ++++++++++++++++++++++-- docs/known-issues.md | 68 +++++++++++++++++++++++++++++--------- 4 files changed, 203 insertions(+), 20 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 661a2eb..69941c3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,62 @@ ## Unreleased +### Range reads, milestone M3: remote files (2026-09-26) +- **New crate `clawhdf5-remote`.** `open_url("http://host/file.h5")` gives + a `clawhdf5::File` (through `File::open_storage`) that reads the file by + HTTP `Range` requests; `storage_for_url` returns the cached storage to + read its statistics. Features: `http` (default; `ureq` without TLS, no + C), `https` (rustls with ring, which compiles C), `object-store` (any + `object_store` store, pure Rust), `s3`/`gcs`/`azure` (`s3://`, `gs://`, + `az://` URLs, configured from the environment; object_store's cloud + clients use aws-lc-rs, C). + - **`BlockCache`** wraps any `Storage`: aligned 1 MiB blocks (the size + `docs/design/range-reads.md` §2 measured), LRU with a byte budget + (64 MiB), the blocks one read misses fetched with one backend + `read_ranges` call as runs of consecutive blocks (a one-block gap is + fetched to merge two runs; at most 8 MiB per request). Readers on + several threads share it without holding its lock across a request, and + a block being fetched is waited for, not fetched again. A read that + misses more than half the budget is not kept (a large dataset does not + evict the metadata). A failed fetch is an error for every reader waiting + on it and is not cached. + - **`HttpStorage`**: opening is one ranged `GET` of the first block, + whose `Content-Range` gives the length. The file is pinned by its strong + `ETag` (`If-Match`) or else `Last-Modified` (`If-Unmodified-Since`), and + its length, checked on every response: a file changed while open is + `RemoteError::FileChanged`, never mixed data. A server that ignores + `Range` is refused without reading the body unless + `HttpOptions::allow_full_download`. Connection failures, timeouts, + `408`/`429`/`5xx` and cut-off bodies are retried with exponential + backoff; bodies are asked for with `Accept-Encoding: identity` and an + encoded one is refused. The ranges of one call are fetched in parallel. + - **`ObjectStoreStorage`** reads one object of any `object_store` store, + pinned by ETag (else version or modification time) and size. It blocks + on a small tokio runtime it owns; called from inside another runtime it + refuses (`RemoteError::Usage`) instead of blocking a worker. + `open_object(store, path, options)` opens a file through a block cache. + - Counted on the conformance corpus (tank, 2026-09-26, + `CLAWHDF5_REMOTE_CORPUS=conformance/.cache/corpus CLAWHDF5_REMOTE_REPORT=1 + cargo test --release -p clawhdf5-remote --test http -- --nocapture corpus`): + the 621 files that open (254 MB) read over HTTP exactly as through + `File::open`; opening and listing them all (every group's entries, every + dataset's shape and type) took 640 requests and 55.5 MB, and then + reading each file's largest dataset under 64 MiB 96 more requests + (171 MB in all). Without the cache the same work is 141 936 requests. + Listing the 7.7 MB IMERG file (file A of the design's §2) takes 2 + requests; the tests hold it to at most 3. +- **`h5rs` takes URLs** with the new `remote` feature (`remote-https` for + `https://`): `ls`, `dump`, `stat` and `diff` read by range requests; + `check` downloads the file whole, since it validates every byte. The + tools now read through `File::storage` and the format crate's `*_in` + functions; local output is unchanged. +- **`File::storage()`** (facade) returns the file's bytes from the + superblock on (the `as_bytes` view, cache image laid over) as a + `&(dyn Storage + Send + Sync)` for every backend, so code that parses a + file itself works on remote files too. +- **`VlResolver::element_in` / `string_element_in`** (format): the + `element`/`string_element` lookups over any `Storage`. + ### Range reads, milestone M2: raw data and `File::open_storage` (2026-09-26) - **`clawhdf5::File::open_storage(Arc)`** opens a file served by any `clawhdf5_format::storage::Storage` and gives the diff --git a/README.md b/README.md index 88ad61b..cfa82d4 100644 --- a/README.md +++ b/README.md @@ -96,6 +96,12 @@ breaking change, are in [CHANGELOG.md](CHANGELOG.md). filtered top-k, never slower than unfiltered), and opt-in re-ranking and confidence rejection, which used to be reachable only through `ClawhdfBackend`. +**Remote files (unreleased)** +- New crate `clawhdf5-remote`: `open_url("http://…")` reads a file on an + HTTP server (or in S3/GCS/Azure, opt-in) by range requests through a + block cache, without downloading it; `h5rs` takes URLs with its `remote` + feature. See [Reading remote files](#reading-remote-files). + **Tooling** - CI now runs the h5py/netCDF4 interop suites for real (they had been skipping silently) and runs an aarch64 job for the NEON kernels. @@ -450,6 +456,50 @@ Each call changes the file in place (no rewrite) and syncs it. What it cannot change safely is refused before anything is written; see [known issues](docs/known-issues.md) for the limits. +### Reading remote files + +[`clawhdf5-remote`](crates/clawhdf5-remote/README.md) opens a file on an +HTTP server (or, with its `s3`/`gcs`/`azure` features, in an object store) +without downloading it: the read API is the same `clawhdf5::File`, and +only the bytes an operation needs are fetched, by `Range` requests through +a block cache (1 MiB blocks; opening fetches the first one). A file that +changes on the server while it is open is an error, never a mix of old and +new bytes. + +```rust +let file = clawhdf5_remote::open_url("http://127.0.0.1:8000/tall.h5")?; +let values = file.dataset("/g2/dset2.1")?.read_f64()?; +``` + +To try it without a server of your own, the crate's test server serves a +directory with range support: + +```bash +cargo run -p clawhdf5-remote --example range_server -- crates/clawhdf5/tests/fixtures 127.0.0.1:8000 +# in another shell: list the file, read one dataset, print what it cost +cargo run -p clawhdf5-remote --example read_url -- http://127.0.0.1:8000/tall.h5 /g2/dset2.1 +``` + +```text +/g1 group +/g2 group +/g2/dset2.1 dataset [10] F32 +/g2/dset2.2 dataset [3, 5] F32 +/g1/g1.1 group +/g1/g1.2 group +/g1/g1.2/g1.2.1 group +/g1/g1.1/dset1.1.1 dataset [10, 10] I32 +/g1/g1.1/dset1.1.2 dataset [20] I32 +/g2/dset2.1: 10 values, first [1.0, 1.100000023841858, 1.2000000476837158, ...] +1 range requests (the one at open included), 9968 bytes fetched, 9968 bytes cached +``` + +(`tall.h5` is 9 968 bytes, so the first block holds all of it.) `h5rs` +built with `--features remote` takes the same URLs: +`h5rs ls -r http://127.0.0.1:8000/tall.h5`. Plain HTTP builds no C; +`https://` is the `https` feature (rustls with ring, which compiles C). +Limits are in [known issues](docs/known-issues.md). + ### Python `crates/clawhdf5-py` is a Python package (PyO3 + numpy) that reads HDF5 with @@ -678,7 +728,7 @@ let exported = backend.export_markdown("MEMORY.md")?; ## Crate Map ``` -clawhdf5 workspace (17 crates, ~86K lines of Rust in src/, ~104K with tests +clawhdf5 workspace (19 crates, ~86K lines of Rust in src/, ~104K with tests and benches; plus libaec-sys, an internal FFI bindings crate for the optional szip feature) │ @@ -690,7 +740,8 @@ clawhdf5 workspace (17 crates, ~86K lines of Rust in src/, ~104K with tests │ ├── clawhdf5 — High-level API │ ├── clawhdf5-netcdf4 — NetCDF-4 support │ ├── clawhdf5-accel — SIMD (AVX2, NEON incl. SDOT int8; AVX-512 behind `avx512`) -│ └── clawhdf5-gpu — GPU compute (wgpu, hand-written WGSL compute shaders) +│ ├── clawhdf5-gpu — GPU compute (wgpu, hand-written WGSL compute shaders) +│ └── clawhdf5-remote — Remote files: HTTP(S) range requests, object stores, block cache │ ├── Agent Memory │ ├── clawhdf5-agent — Memory engine (24.7K lines, 32 modules; chained-CRC WAL) @@ -705,6 +756,7 @@ clawhdf5 workspace (17 crates, ~86K lines of Rust in src/, ~104K with tests │ └── clawhdf5-wasm — Browser (WebAssembly, wasm-bindgen; read-only) │ └── Tooling + ├── clawhdf5-tools — h5rs: ls, dump, stat, diff, check └── clawhdf5-bench — Benchmark suite ``` diff --git a/docs/design/range-reads.md b/docs/design/range-reads.md index 1098922..2c7d1a4 100644 --- a/docs/design/range-reads.md +++ b/docs/design/range-reads.md @@ -5,8 +5,10 @@ change. Progress: M0 and M1 are done, and so is M2 (branch `feat/p3-m2-raw-data`): every read path of the format crate works through `Storage`, v2 B-trees, dense groups and raw data included, and `File::open_storage` gives the facade's read API over any `Storage` (see -`CHANGELOG.md`, "Range reads, milestone M2"). M3 (a remote backend with -its block cache) is next. Every count in §1–§2 was +`CHANGELOG.md`, "Range reads, milestone M2"). M3 is done on branch +`feat/p3-m3-remote`: the `clawhdf5-remote` crate (block cache, HTTP(S), +object stores) and URLs in `h5rs` (see the M3 status below). M4 (wasm) is +next. Every count in §1–§2 was taken on `tank` on 2026-09-26 at commit `de2a53f`, with the commands given next to it. No timing numbers appear here on purpose: the machine was shared with other build jobs when this was written. @@ -458,6 +460,43 @@ fast path within benchmark noise. §2; the page size for paged files; the first block prefetched on open) and a request counter exposed for tests and users. - Python bindings: `clawhdf5.File("s3://…")` / `https://` through it. +- *Status 2026-09-26:* done on branch `feat/p3-m3-remote`, except the + Python bindings, with these choices: + - A new crate, `clawhdf5-remote`, instead of a `remote` feature of + `clawhdf5-io`: `open_url` returns a `clawhdf5::File`, and `clawhdf5-io` + sits below the facade. + - HTTP(S) through `ureq` (`HttpStorage`), not object_store's HTTP store: + that one pulls reqwest with aws-lc-rs (C), while plain HTTP through + ureq builds no C, so it is the default feature; `https` adds rustls + with ring. S3/GCS/Azure go through `object_store` (`ObjectStoreStorage`, + features `s3`/`gcs`/`azure`, opt-in because of aws-lc-rs); the + `object-store` feature alone (in-memory, local files, a store you + build) is pure Rust. object_store is async: the storage blocks on a + two-thread tokio runtime of its own, and refuses to run inside another + runtime. + - `BlockCache` (any `Storage`): 1 MiB blocks and a 64 MiB LRU budget by + default, the first block fetched at open (for HTTP by the request + that learns the length), the missing blocks of one read fetched as + runs of consecutive blocks in one parallel batch, per-block in-flight + deduplication across threads, and reads that miss more than half the + budget not kept. The page size of paged files is not used as the block + size yet. `CacheStats` and `HttpStats` count requests and bytes. + - The file is pinned at open by ETag (else Last-Modified, or the object's + version) and length; a change is an error, not mixed data. + - `h5rs` (feature `remote`) reads through the new `File::storage()`, the + file's view as a `Storage`, so its parsing works on remote files; its + `check` downloads the file whole. + - Measured with `crates/clawhdf5-remote/tests/http.rs` (tank, 2026-09-26, + `CLAWHDF5_REMOTE_CORPUS=conformance/.cache/corpus + CLAWHDF5_REMOTE_REPORT=1 cargo test --release -p clawhdf5-remote --test + http -- --nocapture corpus`), requests as the test server counted + them: the 621 corpus files that open read over HTTP exactly as through + `File::open`. Open + list (every group's entries, every dataset's shape + and type) of all of them: 640 requests, 55.5 MB of 254 MB; then reading + each file's largest dataset under 64 MiB: 96 more (171 MB in all). The + same work without a cache: 141 936 requests. Per file: A lists in 2 + requests (§2 predicted 2 blocks of 1 MiB), B in 1, C in 7 (its whole + 6.4 MB: 35 001 object headers spread over the file). **M4 — wasm lazy loading (1–2 weeks).** - `clawhdf5-wasm`: `openUrl(url) -> Promise` backed by `fetch` with a diff --git a/docs/known-issues.md b/docs/known-issues.md index 313bc8e..860e1eb 100644 --- a/docs/known-issues.md +++ b/docs/known-issues.md @@ -766,20 +766,22 @@ which is what libhdf5 itself writes. ## Range reads (`File::open_storage`) limits **Status:** open (added 2026-09-26, milestone M2 of -`docs/design/range-reads.md`). `File::open_storage` reads any -`clawhdf5_format::storage::Storage` through the whole read API, and every -format-crate read path works through `Storage::read_at`/`read_ranges`, but: +`docs/design/range-reads.md`; remote backends added by M3). `File::open_storage` +reads any `clawhdf5_format::storage::Storage` through the whole read API, +every format-crate read path works through `Storage::read_at`/`read_ranges`, +and `clawhdf5-remote` serves HTTP(S) and object-store files through a block +cache, but: -- **No remote backend and no block cache yet** (milestone M3). A `Storage` - is asked for each structure as the parsers need it, several times over - for some (an object header is re-read by each lookup through it): one - pass over the conformance corpus — open, list, every attribute, every - dataset once — is 176 092 `read_at` calls for 621 files, 92 489 of them - for the 35 001-group `h5stat_newgrat.h5` (2026-09-26, tank, - `crates/clawhdf5/tests/storage_equivalence.rs` with - `CLAWHDF5_STORAGE_CORPUS`). A backend over a network needs a cache in - front of it. `Storage::read_ranges` defaults to one `read_at` per range; - coalescing is the backend's job. +- **A `Storage` without a cache is asked for each structure as the parsers + need it**, several times over for some (an object header is re-read by + each lookup through it): one pass over the conformance corpus — open, + list, every attribute, every dataset once — is 176 092 `read_at` calls + for 621 files, 92 489 of them for the 35 001-group `h5stat_newgrat.h5` + (2026-09-26, tank, `crates/clawhdf5/tests/storage_equivalence.rs` with + `CLAWHDF5_STORAGE_CORPUS`). A backend of your own over a network needs a + cache in front of it: wrap it in `clawhdf5_remote::BlockCache`, as + `open_url` does. `Storage::read_ranges` defaults to one `read_at` per + range; coalescing is the backend's (or the cache's) job. - A group lookup by name in a version-1 (symbol-table) group lists the whole group (dense groups use their name index). Over a range backend that is one read per symbol-table node and name, per lookup. @@ -790,10 +792,12 @@ format-crate read path works through `Storage::read_at`/`read_ranges`, but: `read_*_zerocopy`) need the file in memory and answer `FormatError::ContiguousStorageRequired` otherwise; `File::as_bytes()` panics for such a file (`File::contiguous_bytes()` is the fallible form). - `LazyFile`, `MmapFile`, the Python and wasm bindings and `h5rs` still read - a whole file. + `LazyFile`, `MmapFile` and the Python and wasm bindings still read a + whole file (`h5rs` reads through `File::storage`, and takes URLs with its + `remote` feature). - The file's length is read once, at open: a growing file (SWMR) is not - followed (milestone M5). + followed (milestone M5). A remote file is pinned at open, so one that + grows is `RemoteError::FileChanged`. - Not new, but visible through the equivalence tests: a full read through the file's chunk cache (`read_raw_data_cached`, `read_raw_data_indexed`, and so `Dataset::read_*`) lists a damaged dataset's chunks in hash-map @@ -801,6 +805,38 @@ format-crate read path works through `Storage::read_at`/`read_ranges`, but: the next (`cve-2025-2310.h5`); the values of a dataset that reads are not affected. +## Remote files (`clawhdf5-remote`) limits + +**Status:** open (added 2026-09-26, milestone M3 of +`docs/design/range-reads.md`). + +- **Python and the browser cannot open URLs yet.** `clawhdf5.File` (PyO3) + parses through `File::as_bytes`, which a remote file does not have; the + wasm reader's `openUrl` is milestone M4. +- **The block size is fixed** (1 MiB unless `CacheConfig` says otherwise). + The design's policy of using a paged file's page size as the block size + is not implemented, and only the first block is read ahead. +- **Checked against local servers and one public one.** The tests use an + in-process HTTP/1.1 server and object_store's in-memory and local-file + stores. HTTPS was checked by hand against `raw.githubusercontent.com` + (2026-09-26, tank: `h5rs dump` of h5py's `vlen_string_dset.h5` by URL + equals the downloaded file's). The `s3`, `gcs` and `azure` backends are + built and their URL parsing tested, but they have not been run against a + real bucket. +- **A server with neither a strong ETag nor Last-Modified** can only be + checked by length, so a same-length replacement mid-read would go + unnoticed; `HttpOptions::require_validator` refuses such servers. A weak + ETag (`W/"…"`) cannot be sent as `If-Match`, so it counts as none. +- **Credentials:** HTTP takes extra headers (`HttpOptions::headers`, e.g. + `Authorization`); `h5rs` has no option for them. The cloud stores read + credentials from the environment only. +- Each `ObjectStoreStorage` owns a tokio runtime with two worker threads. +- `h5rs check` downloads a remote file whole (it validates every byte), and + a URL cannot carry a `FILE/OBJECT` suffix; `h5rs` uses the default cache + settings. +- The zero-copy methods and `File::as_bytes` are unavailable on a remote + file (see the range-read limits above). + ## `clawhdf5-wasm` (browser) limits **Status:** open (by design for now; added 2026-09-26). From 4f5697fdd9ff99e2d9b4c00648b6361d166a2592 Mon Sep 17 00:00:00 2001 From: osobh Date: Sat, 26 Sep 2026 17:31:26 -0500 Subject: [PATCH 08/20] clawhdf5-remote: readers waiting on a failed fetch get its error A reader that waited for another reader's fetch of a block got "the fetch of this block failed" when that fetch failed, not why: a file replaced on the server while open was reported as FileChanged to one thread and as an anonymous failure to the others. The fetch's error is now handed to every reader waiting on it. Regression test: four threads read the same block from a slow backend whose fetches fail with a "changed while open" error; each gets that error (it failed for the waiters before this change). Also fixes the ignore-Range test, broken by the previous commit: the test server now counts a body before sending it, so "the refused body was not read" is checked as "refused at the first response". Co-Authored-By: Claude Opus 5.5 (1M context) --- crates/clawhdf5-remote/src/cache.rs | 73 ++++++++++++++++++++++------ crates/clawhdf5-remote/tests/http.rs | 6 +-- 2 files changed, 58 insertions(+), 21 deletions(-) diff --git a/crates/clawhdf5-remote/src/cache.rs b/crates/clawhdf5-remote/src/cache.rs index 233938c..a434814 100644 --- a/crates/clawhdf5-remote/src/cache.rs +++ b/crates/clawhdf5-remote/src/cache.rs @@ -98,7 +98,7 @@ type Block = Arc<[u8]>; /// A fetch in progress: the readers waiting for a block wait on this. struct Flight { - result: Mutex>>, + result: Mutex>>, done: Condvar, } @@ -110,7 +110,7 @@ impl Flight { }) } - fn finish(&self, r: Result) { + fn finish(&self, r: Result) { let mut slot = lock(&self.result); if slot.is_none() { *slot = Some(r); @@ -118,7 +118,7 @@ impl Flight { self.done.notify_all(); } - fn wait(&self) -> Result { + fn wait(&self) -> Result { let mut slot = lock(&self.result); loop { if let Some(r) = slot.as_ref() { @@ -173,10 +173,20 @@ pub struct BlockCache { } /// Fails every flight a fetch claimed and did not complete (an error or a -/// panic in the backend), so no reader waits forever. +/// panic in the backend), so no reader waits forever. The waiting readers +/// get the fetch's error. struct FlightGuard<'a, S> { cache: &'a BlockCache, flights: Vec<(u64, Arc)>, + error: Option, +} + +impl FlightGuard<'_, S> { + /// Record `e` for the readers waiting on this fetch; returns it. + fn fail(&mut self, e: FormatError) -> FormatError { + self.error = Some(e.clone()); + e + } } impl Drop for FlightGuard<'_, S> { @@ -191,8 +201,12 @@ impl Drop for FlightGuard<'_, S> { } } drop(st); + let error = self + .error + .take() + .unwrap_or_else(|| FormatError::Storage("the fetch of this block failed".into())); for (_, f) in &self.flights { - f.finish(Err("the fetch of this block failed".into())); + f.finish(Err(error.clone())); } } } @@ -348,6 +362,7 @@ impl BlockCache { let mut guard = FlightGuard { cache: self, flights: Vec::new(), + error: None, }; { let mut st = lock(&self.state); @@ -410,13 +425,16 @@ impl BlockCache { self.counters .requests .fetch_add(runs.len() as u64, Ordering::Relaxed); - let fetched = self.inner.read_ranges(&runs)?; + let fetched = match self.inner.read_ranges(&runs) { + Ok(f) => f, + Err(e) => return Err(guard.fail(e)), + }; if fetched.len() != runs.len() { - return Err(FormatError::Storage(format!( + return Err(guard.fail(FormatError::Storage(format!( "backend returned {} ranges for {} requested", fetched.len(), runs.len() - ))); + )))); } let mut got: Vec<(u64, Block)> = Vec::with_capacity(guard.flights.len()); for (run, bytes) in runs.iter().zip(&fetched) { @@ -424,12 +442,12 @@ impl BlockCache { .bytes_fetched .fetch_add(bytes.len() as u64, Ordering::Relaxed); if bytes.len() as u64 != run.end - run.start { - return Err(FormatError::Storage(format!( + return Err(guard.fail(FormatError::Storage(format!( "short read from the backend: {} of {} bytes at offset {}", bytes.len(), run.end - run.start, run.start - ))); + )))); } let bs = self.config.block_size; let mut i = run.start / bs; @@ -476,12 +494,7 @@ impl BlockCache { } for (i, f) in waits { - match f.wait() { - Ok(data) => { - have.insert(i, data); - } - Err(e) => return Err(FormatError::Storage(e)), - } + have.insert(i, f.wait()?); } Ok(have) } @@ -786,6 +799,34 @@ mod tests { } } + /// A slow backend whose fetches all fail with a telling error. + struct SlowFailing; + + impl Storage for SlowFailing { + fn read_at(&self, _: u64, _: usize) -> Result, FormatError> { + std::thread::sleep(std::time::Duration::from_millis(50)); + Err(FormatError::Storage("the file changed while open".into())) + } + fn len(&self) -> u64 { + 1 << 20 + } + } + + #[test] + fn readers_waiting_on_a_failed_fetch_get_its_error() { + let c = BlockCache::new(SlowFailing, small()); + std::thread::scope(|s| { + let hs: Vec<_> = (0..4) + .map(|_| s.spawn(|| c.read_at(0, 10).map(|b| b.len()))) + .collect(); + for h in hs { + let e = h.join().unwrap().unwrap_err().to_string(); + assert!(e.contains("changed while open"), "{e}"); + } + }); + assert!(c.stats().waits > 0); + } + #[test] fn concurrent_readers_never_fetch_a_block_twice() { let d = data(64 * 1024); diff --git a/crates/clawhdf5-remote/tests/http.rs b/crates/clawhdf5-remote/tests/http.rs index b0a0e34..5a68072 100644 --- a/crates/clawhdf5-remote/tests/http.rs +++ b/crates/clawhdf5-remote/tests/http.rs @@ -301,11 +301,7 @@ fn a_server_that_ignores_range_is_refused_or_downloaded_when_allowed() { matches!(err, Error::Remote(RemoteError::RangeNotSupported(_))), "{err}" ); - assert!( - server.bytes() < bytes.len() as u64, - "refusing must not download the file ({} bytes read)", - server.bytes() - ); + assert_eq!(server.requests(), 1, "refused at the first response"); let mut opts = quick(); opts.http.allow_full_download = true; let storage = storage_for_url(&url, &opts).unwrap(); From 5062b907bd37e9578699dd9b0041149fdd655b09 Mon Sep 17 00:00:00 2001 From: osobh Date: Sat, 26 Sep 2026 18:23:27 -0500 Subject: [PATCH 09/20] clawhdf5-remote tests: the server counts only requests for its files A local port scanner's GET / reached the test listeners and was counted, failing the exact request budgets (and consuming injected 503s). Requests for paths the server does not serve are now answered 404 without being counted, delayed or failed; the query string is not part of the path. Co-Authored-By: Claude Opus 5.5 (1M context) --- crates/clawhdf5-remote/tests/common/server.rs | 41 ++++++++++++------- crates/clawhdf5-remote/tests/http.rs | 18 ++++++++ 2 files changed, 44 insertions(+), 15 deletions(-) diff --git a/crates/clawhdf5-remote/tests/common/server.rs b/crates/clawhdf5-remote/tests/common/server.rs index 4749f50..a060a9d 100644 --- a/crates/clawhdf5-remote/tests/common/server.rs +++ b/crates/clawhdf5-remote/tests/common/server.rs @@ -3,7 +3,10 @@ //! and Last-Modified with `If-Match` / `If-Unmodified-Since` (412), and //! switches to misbehave — ignore ranges (200 with the whole file), cut //! bodies short, answer 503, respond slowly, send no validators. It counts -//! requests and body bytes, and logs every range asked for. +//! requests and body bytes, and logs every range asked for — only for the +//! paths it serves: a request for any other path (a local port scanner's +//! `GET /`, say) is answered 404 and not counted, so request budgets in +//! tests stay exact. #![allow(dead_code)] @@ -45,7 +48,8 @@ pub struct Shared { pub fail_next: AtomicU32, /// Sleep this long before answering each request. pub delay_ms: AtomicU64, - /// Requests served (every status). + /// Requests for a served path (every status); requests for other + /// paths are not counted. pub requests: AtomicU64, /// Body bytes sent. pub bytes: AtomicU64, @@ -223,14 +227,31 @@ fn serve(conn: TcpStream, s: &Shared) -> std::io::Result<()> { headers.insert(k.trim().to_ascii_lowercase(), v.trim().to_string()); } } + // The query string (a presigned URL's signature, say) is not part + // of the file's name. + let path = path.split('?').next().unwrap_or("").to_string(); + let close = headers + .get("connection") + .is_some_and(|v| v.eq_ignore_ascii_case("close")); + let res = { + let files = s.files.read().unwrap(); + files + .get(&path) + .map(|r| (r.data.clone(), r.etag.clone(), r.last_modified.clone())) + }; + let Some((data, etag, lm)) = res else { + // Not ours: not counted, not delayed, no failure injected. + write!(out, "HTTP/1.1 404 Not Found\r\nContent-Length: 0\r\n\r\n")?; + if close { + return Ok(()); + } + continue; + }; s.requests.fetch_add(1, Ordering::SeqCst); let delay = s.delay_ms.load(Ordering::SeqCst); if delay > 0 { std::thread::sleep(Duration::from_millis(delay)); } - let close = headers - .get("connection") - .is_some_and(|v| v.eq_ignore_ascii_case("close")); if s.fail_next .fetch_update(Ordering::SeqCst, Ordering::SeqCst, |n| n.checked_sub(1)) .is_ok() @@ -241,16 +262,6 @@ fn serve(conn: TcpStream, s: &Shared) -> std::io::Result<()> { )?; continue; } - let res = { - let files = s.files.read().unwrap(); - files - .get(&path) - .map(|r| (r.data.clone(), r.etag.clone(), r.last_modified.clone())) - }; - let Some((data, etag, lm)) = res else { - write!(out, "HTTP/1.1 404 Not Found\r\nContent-Length: 0\r\n\r\n")?; - continue; - }; let len = data.len() as u64; let mut validators = String::new(); if s.weak_etag.load(Ordering::SeqCst) { diff --git a/crates/clawhdf5-remote/tests/http.rs b/crates/clawhdf5-remote/tests/http.rs index 5a68072..2c95cd1 100644 --- a/crates/clawhdf5-remote/tests/http.rs +++ b/crates/clawhdf5-remote/tests/http.rs @@ -526,3 +526,21 @@ fn bad_urls_and_statuses_are_clean_errors() { "{err}" ); } + +/// A request for a path the server does not serve (a local port scanner's +/// `GET /`) does not count against a test's request budget. +#[test] +fn requests_for_other_paths_are_not_counted() { + use std::io::{Read, Write}; + let server = Server::start(vec![("/m.h5".into(), multi_block_file())]); + let mut probe = std::net::TcpStream::connect(server.addr).unwrap(); + probe + .write_all(b"GET / HTTP/1.1\r\nHost: x\r\nConnection: close\r\n\r\n") + .unwrap(); + let mut answer = String::new(); + probe.read_to_string(&mut answer).unwrap(); + assert!(answer.starts_with("HTTP/1.1 404"), "{answer}"); + storage_for_url(&server.url("/m.h5"), &quick()).unwrap(); + assert_eq!(server.requests(), 1, "only the open counts"); + assert_eq!(server.log().len(), 1); +} From e8aaf050be83990b6d9031a83e241b245feb8e0d Mon Sep 17 00:00:00 2001 From: osobh Date: Sat, 26 Sep 2026 18:24:48 -0500 Subject: [PATCH 10/20] clawhdf5-remote: no overflow on lengths near u64::MAX A server can claim any length in Content-Range. block_len computed start + block_size, which overflowed in the last blocks of a file claimed to be near u64::MAX (a panic in debug builds, a wrapped value in release); insert() multiplied block indices unchecked. The cache's block arithmetic is now saturating/checked, and a run that does not split into whole blocks is an error instead of an endless loop or a slice panic. The test server gains fake_total (claim a length, serve zeros past the data); a test reads the last bytes of such files and opens a file whose superblock EOF and root addresses sit near u64::MAX. Co-Authored-By: Claude Opus 5.5 (1M context) --- crates/clawhdf5-remote/src/cache.rs | 70 ++++++++++++++++--- crates/clawhdf5-remote/tests/common/server.rs | 25 ++++++- crates/clawhdf5-remote/tests/http.rs | 33 +++++++++ 3 files changed, 118 insertions(+), 10 deletions(-) diff --git a/crates/clawhdf5-remote/src/cache.rs b/crates/clawhdf5-remote/src/cache.rs index a434814..151cc84 100644 --- a/crates/clawhdf5-remote/src/cache.rs +++ b/crates/clawhdf5-remote/src/cache.rs @@ -305,9 +305,9 @@ impl BlockCache { let end = offset.saturating_add(bytes.len() as u64).min(self.len); let mut i = offset.div_ceil(bs); let mut st = lock(&self.state); - while i * bs < end { - let start = i * bs; - let block_end = (start + bs).min(self.len); + // Checked: a hostile server can claim a length near u64::MAX. + while let Some(start) = i.checked_mul(bs).filter(|&s| s < end) { + let block_end = start.saturating_add(bs).min(self.len); if block_end > end { break; } @@ -330,9 +330,14 @@ impl BlockCache { Some(offset / bs..(end - 1) / bs + 1) } + /// Length of block `i` (0 past the end of the file). Saturating: the + /// length may be anything a server claimed, up to `u64::MAX`. fn block_len(&self, i: u64) -> u64 { - let start = i * self.config.block_size; - (start + self.config.block_size).min(self.len) - start + let start = i.saturating_mul(self.config.block_size); + start + .saturating_add(self.config.block_size) + .min(self.len) + .saturating_sub(start) } fn keep(&self, st: &mut State, i: u64, data: Block) { @@ -453,8 +458,17 @@ impl BlockCache { let mut i = run.start / bs; let mut pos = 0usize; while pos < bytes.len() { - let n = self.block_len(i) as usize; - got.push((i, Arc::from(&bytes[pos..pos + n]))); + let n = usize::try_from(self.block_len(i)).unwrap_or(usize::MAX); + let Some(block) = pos.checked_add(n).and_then(|e| bytes.get(pos..e)) else { + return Err(guard.fail(FormatError::Storage(format!( + "a run at offset {} does not split into whole blocks", + run.start + )))); + }; + if n == 0 { + break; + } + got.push((i, Arc::from(block))); pos += n; i += 1; } @@ -512,7 +526,9 @@ impl BlockCache { } } runs.into_iter() - .map(|(a, b)| a * bs..(b * bs + self.block_len(b))) + .map(|(a, b)| { + a.saturating_mul(bs)..b.saturating_mul(bs).saturating_add(self.block_len(b)) + }) .collect() } @@ -863,4 +879,42 @@ mod tests { assert_eq!(n, blocks.len(), "a block was fetched twice: {fetched:?}"); assert!(c.stats().waits > 0, "readers should have shared fetches"); } + + /// A backend claiming any length (a hostile server's `Content-Range`) + /// and serving zeros. + struct Zeros(u64); + + impl Storage for Zeros { + fn read_at(&self, offset: u64, len: usize) -> Result, FormatError> { + let end = offset.saturating_add(len as u64).min(self.0); + Ok(Cow::Owned(vec![0; end.saturating_sub(offset) as usize])) + } + fn len(&self) -> u64 { + self.0 + } + } + + #[test] + fn lengths_near_u64_max_do_not_overflow() { + for len in [u64::MAX, u64::MAX - 1, u64::MAX - 1000, 1 << 63] { + let c = BlockCache::new(Zeros(len), small()); + for (off, n) in [ + (len - 100, 50), + (len - 10, 100), + (len - 1, 1), + (len - 3000, 3000), + (u64::MAX, 10), + ] { + let got = c.read_at(off, n).unwrap(); + assert_eq!(got.len() as u64, len.saturating_sub(off).min(n as u64)); + } + let got = c + .read_ranges(&[len - 5000..len, u64::MAX - 1..u64::MAX]) + .unwrap(); + assert_eq!(got[0].len(), 5000); + c.insert(len - 5, &[0; 5]); + c.insert(u64::MAX - 5, &[0; 5]); + c.prefetch(len - 1, 10).unwrap(); + } + } } diff --git a/crates/clawhdf5-remote/tests/common/server.rs b/crates/clawhdf5-remote/tests/common/server.rs index a060a9d..13f279a 100644 --- a/crates/clawhdf5-remote/tests/common/server.rs +++ b/crates/clawhdf5-remote/tests/common/server.rs @@ -48,6 +48,10 @@ pub struct Shared { pub fail_next: AtomicU32, /// Sleep this long before answering each request. pub delay_ms: AtomicU64, + /// When non-zero, claim the file is this long (in `Content-Range` and + /// when checking ranges) and serve zeros past its real end: a hostile + /// server lying about the length. + pub fake_total: AtomicU64, /// Requests for a served path (every status); requests for other /// paths are not counted. pub requests: AtomicU64, @@ -262,7 +266,8 @@ fn serve(conn: TcpStream, s: &Shared) -> std::io::Result<()> { )?; continue; } - let len = data.len() as u64; + let fake = s.fake_total.load(Ordering::SeqCst); + let len = if fake > 0 { fake } else { data.len() as u64 }; let mut validators = String::new(); if s.weak_etag.load(Ordering::SeqCst) { validators = format!("ETag: W/{etag}\r\n"); @@ -291,6 +296,7 @@ fn serve(conn: TcpStream, s: &Shared) -> std::io::Result<()> { .unwrap() .push((path.clone(), range.and_then(Result::ok))); } + let mut padded = Vec::new(); let (status, body, extra) = match range { Some(Err(())) => { write!( @@ -302,7 +308,7 @@ fn serve(conn: TcpStream, s: &Shared) -> std::io::Result<()> { } Some(Ok((a, b))) => ( "206 Partial Content", - &data[a as usize..=b as usize], + slice_or_zeros(&data, a, b, &mut padded), format!("Content-Range: bytes {a}-{b}/{len}\r\n"), ), None => ("200 OK", &data[..], String::new()), @@ -342,3 +348,18 @@ fn serve(conn: TcpStream, s: &Shared) -> std::io::Result<()> { } } } + +/// `data[a..=b]`, padded with zeros past its end (into `padded`). +fn slice_or_zeros<'a>(data: &'a [u8], a: u64, b: u64, padded: &'a mut Vec) -> &'a [u8] { + let real = data.len() as u64; + if b < real { + return &data[a as usize..=b as usize]; + } + let n = usize::try_from(b - a + 1).expect("range fits in memory"); + padded.resize(n, 0); + if a < real { + let have = (real - a) as usize; + padded[..have].copy_from_slice(&data[a as usize..]); + } + padded +} diff --git a/crates/clawhdf5-remote/tests/http.rs b/crates/clawhdf5-remote/tests/http.rs index 2c95cd1..2955f54 100644 --- a/crates/clawhdf5-remote/tests/http.rs +++ b/crates/clawhdf5-remote/tests/http.rs @@ -544,3 +544,36 @@ fn requests_for_other_paths_are_not_counted() { assert_eq!(server.requests(), 1, "only the open counts"); assert_eq!(server.log().len(), 1); } + +/// A server claiming a length near `u64::MAX` (and serving zeros past the +/// real data), with a file whose addresses point at the end of that range: +/// clean errors or zeros, never an arithmetic overflow (a panic in debug). +#[test] +fn a_server_claiming_a_huge_length_does_not_overflow() { + let root = std::path::Path::new(env!("CARGO_MANIFEST_DIR")); + let mut bytes = + std::fs::read(root.join("../clawhdf5-format/tests/fixtures/legacy/h5ex_g_iterate.h5")) + .unwrap(); + // Superblock v0: end-of-file address, then the root symbol table + // entry's object header, B-tree and heap addresses. + bytes[0x28..0x30].copy_from_slice(&(u64::MAX - 1).to_le_bytes()); + for at in [0x40, 0x50, 0x58] { + bytes[at..at + 8].copy_from_slice(&0xFFFF_FFFF_FFFF_F000u64.to_le_bytes()); + } + let server = Server::start(vec![("/h.h5".into(), bytes)]); + for total in [u64::MAX, u64::MAX - 1, 1 << 62] { + server.shared.fake_total.store(total, Ordering::SeqCst); + let url = server.url("/h.h5"); + let storage = storage_for_url(&url, &quick()).unwrap(); + assert_eq!(clawhdf5_format::storage::Storage::len(&*storage), total); + for (off, n) in [(total - 100, 50), (total - 10, 100), (total - 1, 1)] { + let got = clawhdf5_format::storage::Storage::read_at(&*storage, off, n).unwrap(); + assert_eq!(got.len() as u64, (total - off).min(n as u64)); + assert!(got.iter().all(|&b| b == 0)); + } + if let Ok(f) = File::open_storage(storage) { + let _ = transcript(&f); + } + let _ = open_url_with(&url, &quick()).map(|f| transcript(&f)); + } +} From 8df5b209a78f87cdf324baee12d2d3e086527ae7 Mon Sep 17 00:00:00 2001 From: osobh Date: Sat, 26 Sep 2026 18:27:14 -0500 Subject: [PATCH 11/20] clawhdf5-remote, h5rs: never allocate a length the server only claims h5rs check URL read the whole file with one read_at(0, len), len being whatever Content-Range said. BlockCache listed every block index of the span and preallocated len bytes: a server claiming 2^62 bytes for a 10 KB file made h5rs abort (memory allocation of 35184372088832 bytes failed). - BlockCache: a read spanning more than the budget (or eight max_requests) is fetched piece by piece and not kept, its output growing only as data arrives; read_ranges falls back to that per range; prefetch is clamped to the budget. - New clawhdf5_remote::download(storage, max_bytes): refuses a claimed length above the limit (RemoteError::TooLarge) before any request, then reads in 64 MiB steps. New RemoteError::Backend for read errors. - h5rs check downloads through it, with --max-download N (default 1 GiB). Co-Authored-By: Claude Opus 5.5 (1M context) --- crates/clawhdf5-remote/src/cache.rs | 115 ++++++++++++++++++++++++-- crates/clawhdf5-remote/src/error.rs | 19 ++++- crates/clawhdf5-remote/src/lib.rs | 42 ++++++++++ crates/clawhdf5-remote/tests/http.rs | 27 ++++++ crates/clawhdf5-tools/README.md | 4 +- crates/clawhdf5-tools/src/check.rs | 12 ++- crates/clawhdf5-tools/src/h5.rs | 19 +++-- crates/clawhdf5-tools/tests/remote.rs | 21 +++++ 8 files changed, 239 insertions(+), 20 deletions(-) diff --git a/crates/clawhdf5-remote/src/cache.rs b/crates/clawhdf5-remote/src/cache.rs index 151cc84..2d4fa8e 100644 --- a/crates/clawhdf5-remote/src/cache.rs +++ b/crates/clawhdf5-remote/src/cache.rs @@ -25,7 +25,10 @@ //! A failed fetch fails every reader waiting for it, and is not cached. //! - **Large reads do not flush the cache.** A call whose missing blocks //! add up to more than half the budget is served without keeping them -//! (a big chunked read would otherwise evict all the metadata). +//! (a big chunked read would otherwise evict all the metadata). A read +//! spanning more than the budget (or eight `max_request`s, if more) is +//! fetched piece by piece, its output growing only as data arrives: +//! nothing is allocated for a length a server merely claims. //! - **Local storages pass through.** A backend that holds the whole file //! in memory ([`Storage::as_contiguous`]) is read directly. @@ -285,12 +288,46 @@ impl BlockCache { if self.inner.as_contiguous().is_some() { return Ok(()); } - let Some(blocks) = self.block_span(offset, len) else { + // Readahead beyond the budget would only evict itself. + let Some(blocks) = self.block_span(offset, len.min(self.config.capacity)) else { return Ok(()); }; self.blocks(&blocks.collect::>(), true).map(|_| ()) } + /// Blocks fetched per step of a read spanning more than this: the + /// larger of the budget and a full parallel batch of requests (eight + /// `max_request`s), 64 MiB by default. + fn piece_blocks(&self) -> u64 { + let bytes = self + .config + .capacity + .max(self.config.max_request.saturating_mul(8)); + (bytes / self.config.block_size).max(1) + } + + /// A read of more blocks than [`piece_blocks`](Self::piece_blocks): + /// fetched and copied one piece at a time, nothing kept. The block list + /// is never materialised and the output grows only as data arrives, so + /// a length a server merely claims (up to `u64::MAX`) costs nothing + /// until bytes actually come back. + fn read_large(&self, offset: u64, end: u64, span: Range) -> Result, FormatError> { + let bs = self.config.block_size; + let piece = self.piece_blocks(); + let mut out = Vec::new(); + let mut first = span.start; + while first < span.end { + let last = first.saturating_add(piece).min(span.end); + let wanted: Vec = (first..last).collect(); + let blocks = self.blocks(&wanted, false)?; + let a = offset.max(first.saturating_mul(bs)); + let b = end.min(last.saturating_mul(bs)); + self.assemble_into(&mut out, a, b, &blocks); + first = last; + } + Ok(out) + } + /// Put bytes already fetched (such as the first block, which a backend /// may get while it probes the file's length) into the cache: `bytes` /// are the file's bytes at `offset`. Only whole blocks (or the file's @@ -534,8 +571,20 @@ impl BlockCache { /// Copy `[offset, end)` out of `blocks`. fn assemble(&self, offset: u64, end: u64, blocks: &HashMap) -> Vec { + let mut out = Vec::with_capacity(usize::try_from(end - offset).unwrap_or(0)); + self.assemble_into(&mut out, offset, end, blocks); + out + } + + /// Append `[offset, end)` out of `blocks` to `out`. + fn assemble_into( + &self, + out: &mut Vec, + offset: u64, + end: u64, + blocks: &HashMap, + ) { let bs = self.config.block_size; - let mut out = Vec::with_capacity((end - offset) as usize); let mut pos = offset; while pos < end { let i = pos / bs; @@ -545,7 +594,6 @@ impl BlockCache { out.extend_from_slice(&block[from..to]); pos = i * bs + to as u64; } - out } } @@ -559,6 +607,9 @@ impl Storage for BlockCache { return Ok(Cow::Owned(Vec::new())); }; let end = offset.saturating_add(len as u64).min(self.len); + if span.end - span.start > self.piece_blocks() { + return Ok(Cow::Owned(self.read_large(offset, end, span)?)); + } let blocks = self.blocks(&span.collect::>(), true)?; Ok(Cow::Owned(self.assemble(offset, end, &blocks))) } @@ -574,7 +625,8 @@ impl Storage for BlockCache { self.counters .reads .fetch_add(ranges.len() as u64, Ordering::Relaxed); - let mut wanted = BTreeSet::new(); + let mut spans = Vec::with_capacity(ranges.len()); + let mut total = 0u64; for r in ranges { if r.end < r.start { return Err(FormatError::Storage( @@ -582,9 +634,30 @@ impl Storage for BlockCache { )); } if let Some(span) = self.block_span(r.start, r.end - r.start) { - wanted.extend(span); + total = total.saturating_add(span.end - span.start); + spans.push(span); } } + if total > self.piece_blocks() { + // Too many blocks for one batch: range by range, each in pieces. + return ranges + .iter() + .map(|r| { + let end = r.end.min(self.len); + match self.block_span(r.start, r.end - r.start) { + None => Ok(Cow::Owned(Vec::new())), + Some(span) if span.end - span.start > self.piece_blocks() => { + Ok(Cow::Owned(self.read_large(r.start, end, span)?)) + } + Some(span) => { + let blocks = self.blocks(&span.collect::>(), true)?; + Ok(Cow::Owned(self.assemble(r.start, end, &blocks))) + } + } + }) + .collect(); + } + let wanted: BTreeSet = spans.into_iter().flatten().collect(); let wanted: Vec = wanted.into_iter().collect(); let blocks = self.blocks(&wanted, true)?; Ok(ranges @@ -917,4 +990,34 @@ mod tests { c.prefetch(len - 1, 10).unwrap(); } } + + /// A backend claiming 2^62 bytes that holds only a few: a read of + /// "the whole file" fails at the first short piece instead of + /// allocating (or listing the blocks of) the claimed length. + struct Claims(Vec); + + impl Storage for Claims { + fn read_at(&self, offset: u64, len: usize) -> Result, FormatError> { + self.0 + .as_slice() + .read_at(offset, len) + .map(|c| Cow::Owned(c.into_owned())) + } + fn len(&self) -> u64 { + 1 << 62 + } + } + + #[test] + fn a_huge_claimed_length_is_not_allocated() { + let c = BlockCache::new(Claims(data(5000)), CacheConfig::default()); + let e = c.read_at(0, usize::MAX).unwrap_err().to_string(); + assert!(e.contains("short read"), "{e}"); + let e = c + .read_ranges(&[0..u64::MAX, 10..20]) + .unwrap_err() + .to_string(); + assert!(e.contains("short read"), "{e}"); + c.prefetch(0, u64::MAX).unwrap_err(); + } } diff --git a/crates/clawhdf5-remote/src/error.rs b/crates/clawhdf5-remote/src/error.rs index d5d1feb..b2546b6 100644 --- a/crates/clawhdf5-remote/src/error.rs +++ b/crates/clawhdf5-remote/src/error.rs @@ -38,9 +38,19 @@ pub enum RemoteError { Transport(String), /// An error from the object store. ObjectStore(String), - /// Called in a way the backend cannot serve (for example a blocking - /// read from inside an async runtime). + /// Called in a way the backend cannot serve. Usage(String), + /// The file is larger than a download was allowed to be + /// ([`download`](crate::download)). + TooLarge { + /// The file's length, as the server reports it. + len: u64, + /// The limit. + limit: u64, + }, + /// A read through a backend failed; its error, as text (the form a + /// [`clawhdf5::File`] read reports it in). + Backend(String), } impl RemoteError { @@ -71,6 +81,11 @@ impl std::fmt::Display for RemoteError { RemoteError::Transport(s) => write!(f, "network error: {s}"), RemoteError::ObjectStore(s) => write!(f, "object store: {s}"), RemoteError::Usage(s) => write!(f, "{s}"), + RemoteError::TooLarge { len, limit } => write!( + f, + "the remote file is {len} bytes, more than the download limit of {limit} bytes" + ), + RemoteError::Backend(s) => write!(f, "{s}"), } } } diff --git a/crates/clawhdf5-remote/src/lib.rs b/crates/clawhdf5-remote/src/lib.rs index f233954..a441bab 100644 --- a/crates/clawhdf5-remote/src/lib.rs +++ b/crates/clawhdf5-remote/src/lib.rs @@ -16,6 +16,7 @@ //! `gcs`, `azure`) → a [`clawhdf5::File`] with the whole read API. //! - [`storage_for_url`] gives the cached storage itself, to open with //! [`clawhdf5::File::open_storage`] and to read its [`CacheStats`]. +//! - [`download`] reads a whole remote file into memory, up to a limit. //! - [`HttpStorage`] (range `GET`s, pinned by ETag/Last-Modified, retried //! with backoff), [`ObjectStoreStorage`] (any `object_store` store, //! feature `object-store`), and [`BlockCache`] over any @@ -151,6 +152,47 @@ pub fn cached(backend: Backend, options: &Options) -> Result Result, Error> { + let len = storage.len(); + if len > max_bytes { + return Err(RemoteError::TooLarge { + len, + limit: max_bytes, + } + .into()); + } + const STEP: u64 = 64 << 20; + let mut out = Vec::new(); + let mut pos = 0u64; + while pos < len { + let want = (len - pos).min(STEP); + let got = storage + .read_at(pos, want as usize) + .map_err(|e| RemoteError::Backend(e.to_string()))?; + if got.len() as u64 != want { + return Err(RemoteError::BadResponse(format!( + "{} bytes at offset {pos} instead of {want}", + got.len() + )) + .into()); + } + out.extend_from_slice(&got); + pos += want; + } + Ok(out) +} + /// Open the object at `path` of any `object_store` store (in memory, local /// files, or a cloud store you configured) through a block cache. #[cfg(feature = "object-store")] diff --git a/crates/clawhdf5-remote/tests/http.rs b/crates/clawhdf5-remote/tests/http.rs index 2955f54..b4411b6 100644 --- a/crates/clawhdf5-remote/tests/http.rs +++ b/crates/clawhdf5-remote/tests/http.rs @@ -577,3 +577,30 @@ fn a_server_claiming_a_huge_length_does_not_overflow() { let _ = open_url_with(&url, &quick()).map(|f| transcript(&f)); } } + +/// `download` reads a whole file in bounded steps, and refuses a claimed +/// length beyond its limit before reading anything. +#[test] +fn download_is_bounded_by_its_limit_not_the_claimed_length() { + let bytes = multi_block_file(); + let server = Server::start(vec![("/m.h5".into(), bytes.clone())]); + let url = server.url("/m.h5"); + let storage = storage_for_url(&url, &quick()).unwrap(); + let got = clawhdf5_remote::download(&*storage, clawhdf5_remote::DEFAULT_MAX_DOWNLOAD).unwrap(); + assert_eq!(got, bytes); + let e = clawhdf5_remote::download(&*storage, 1000).unwrap_err(); + assert!( + matches!(e, Error::Remote(RemoteError::TooLarge { limit: 1000, .. })), + "{e}" + ); + server.shared.fake_total.store(1 << 62, Ordering::SeqCst); + let storage = storage_for_url(&url, &quick()).unwrap(); + server.reset(); + let e = + clawhdf5_remote::download(&*storage, clawhdf5_remote::DEFAULT_MAX_DOWNLOAD).unwrap_err(); + assert!( + matches!(e, Error::Remote(RemoteError::TooLarge { len, .. }) if len == 1 << 62), + "{e}" + ); + assert_eq!(server.requests(), 0, "refused before reading"); +} diff --git a/crates/clawhdf5-tools/README.md b/crates/clawhdf5-tools/README.md index 42d829d..fc120ad 100644 --- a/crates/clawhdf5-tools/README.md +++ b/crates/clawhdf5-tools/README.md @@ -40,7 +40,9 @@ h5rs diff local.h5 http://127.0.0.1:8000/file.h5 ``` A URL names the whole file (`FILE/OBJECT` suffixes are for local paths). -`check` validates every byte, so it downloads a remote file whole first. +`check` validates every byte, so it downloads a remote file whole first — +up to `--max-download N` bytes (default 1 GiB), refusing a longer file +before reading any of it. The output is the local file's (`tests/remote.rs` compares every subcommand). diff --git a/crates/clawhdf5-tools/src/check.rs b/crates/clawhdf5-tools/src/check.rs index e7d97e5..ea8e09c 100644 --- a/crates/clawhdf5-tools/src/check.rs +++ b/crates/clawhdf5-tools/src/check.rs @@ -28,7 +28,7 @@ use crate::h5::{Error, ErrorKind, H5, Kind}; use crate::info::{self, DsInfo}; pub const USAGE: &str = "\ -usage: h5rs check [--data] [-q] [--max-bytes N] FILE +usage: h5rs check [--data] [-q] [--max-bytes N] [--max-download N] FILE Validate FILE's structure: walk every object from the root group, parse every header message, verify the checksums of version 2+ structures @@ -45,6 +45,9 @@ problem is printed with the address of the structure involved. datasets and attributes into its global heap collection -q, --quiet print only the problems, not the summary --max-bytes N largest dataset read by --data (default 1 GiB) + --max-download N + largest remote (URL) file downloaded to check it + (default 1 GiB) Exit status: 0 no problems, 1 problems found, 2 usage error or file not found, 3 internal error."; @@ -124,6 +127,7 @@ pub fn run(args: &mut Args, out: &mut Out) -> std::io::Result { let mut read_data = false; let mut quiet = false; let mut max_bytes = None; + let mut max_download = 1u64 << 30; let mut file = None; while let Some(a) = args.next() { match a.as_str() { @@ -133,6 +137,10 @@ pub fn run(args: &mut Args, out: &mut Out) -> std::io::Result { Some(n) => max_bytes = Some(n), None => return args.usage_error(out, "--max-bytes needs a number", USAGE), }, + "--max-download" => match args.number() { + Some(n) => max_download = n, + None => return args.usage_error(out, "--max-download needs a number", USAGE), + }, "-h" | "--help" => { writeln!(out.o, "{USAGE}")?; return Ok(0); @@ -150,7 +158,7 @@ pub fn run(args: &mut Args, out: &mut Out) -> std::io::Result { let path = std::path::Path::new(&file); let mut h5 = if crate::h5::is_url(&file) { // check validates every byte, so a remote file is downloaded whole. - match H5::open_arg_whole(&file) { + match H5::open_arg_whole(&file, max_download) { Ok(h) => h, Err(e) => { writeln!(out.e, "h5rs check: {e}")?; diff --git a/crates/clawhdf5-tools/src/h5.rs b/crates/clawhdf5-tools/src/h5.rs index c63ccb3..d840074 100644 --- a/crates/clawhdf5-tools/src/h5.rs +++ b/crates/clawhdf5-tools/src/h5.rs @@ -248,8 +248,10 @@ impl H5 { /// [`H5::open_arg`], with a remote file downloaded whole into memory /// first — for `check`, which validates every byte of the file anyway - /// and parses it as one slice ([`H5::data`]). - pub fn open_arg_whole(arg: &str) -> Result
{ + /// and parses it as one slice ([`H5::data`]). A remote file longer + /// than `max_download` bytes is refused before anything is read: its + /// length is only what the server claims. + pub fn open_arg_whole(arg: &str, max_download: u64) -> Result
{ let h5 = H5::open_arg(arg)?; if h5.file.contiguous_bytes().is_some() { return Ok(h5); @@ -259,12 +261,8 @@ impl H5 { let storage = clawhdf5_remote::storage_for_url(arg, &clawhdf5_remote::Options::default()) .map_err(|e| Error::new(format!("{arg}: {e}")))?; - let len = usize::try_from(storage.len()) - .map_err(|_| Error::new(format!("{arg}: too large to download")))?; - let bytes = storage - .read_at(0, len) - .map_err(|e| Error::new(format!("{arg}: {e}")))? - .into_owned(); + let bytes = clawhdf5_remote::download(&*storage, max_download) + .map_err(|e| Error::new(format!("{arg}: {e}")))?; let size = bytes.len() as u64; let file = File::from_bytes(bytes).map_err(|e| { Error::new(format!("{arg}: not an HDF5 file this tool can open: {e}")) @@ -272,7 +270,10 @@ impl H5 { Ok(H5::new(PathBuf::from(arg), file, size)) } #[cfg(not(feature = "remote"))] - unreachable!("open_arg refuses URLs without the remote feature") + { + let _ = max_download; + unreachable!("open_arg refuses URLs without the remote feature") + } } /// The file's bytes from the superblock on: what every address indexes. diff --git a/crates/clawhdf5-tools/tests/remote.rs b/crates/clawhdf5-tools/tests/remote.rs index d14bd61..426d1f7 100644 --- a/crates/clawhdf5-tools/tests/remote.rs +++ b/crates/clawhdf5-tools/tests/remote.rs @@ -98,3 +98,24 @@ fn url_errors_are_clean() { assert!(out.contains("`https` feature"), "{out}"); } } + +/// `check` downloads a remote file whole, but never trusts the length the +/// server claims: a server claiming 2^62 bytes for a small file is refused +/// before anything is allocated (it used to abort the process), and +/// `--max-download` caps real files too. +#[test] +fn check_refuses_a_remote_file_beyond_the_download_limit() { + use std::sync::atomic::Ordering; + let tall = Path::new(env!("CARGO_MANIFEST_DIR")).join("../clawhdf5/tests/fixtures/tall.h5"); + let server = server::Server::start(vec![("/t.h5".into(), std::fs::read(&tall).unwrap())]); + let url = server.url("/t.h5"); + let (out, rc) = h5rs(&["check", &url]); + assert_eq!(rc, 0, "{out}"); + let (out, rc) = h5rs(&["check", "--max-download", "1000", &url]); + assert_eq!(rc, 2, "{out}"); + assert!(out.contains("download limit of 1000 bytes"), "{out}"); + server.shared.fake_total.store(1 << 62, Ordering::SeqCst); + let (out, rc) = h5rs(&["check", &url]); + assert_eq!(rc, 2, "{out}"); + assert!(out.contains("more than the download limit"), "{out}"); +} From c04e34620eda88821ccb052d9c4bbb230817bb81 Mon Sep 17 00:00:00 2001 From: osobh Date: Sat, 26 Sep 2026 18:30:02 -0500 Subject: [PATCH 12/20] clawhdf5-remote, h5rs: URLs' credentials are never shown MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Every RemoteError message and HttpStorage's Debug output held the URL as given, with any user:password@ and the query string — for a presigned S3/GCS/Azure URL, its signature or token. An application logging the error leaked the credential. - New clawhdf5_remote::redact_url: no userinfo, no fragment, query values replaced by REDACTED (plain key names kept). - HttpStorage formats every message with the redacted URL, and scrubs the URL's secret parts from errors of the HTTP client (whose texts can echo the URI); Debug shows the redacted URL. storage_for_url's and the object store URL errors are redacted too. HttpStorage::url() still returns the URL as given, documented as not for logging. - h5rs prints FILE arguments that are URLs redacted: in errors and in dump/stat/check/diff output. - The test server can force a status and send a wrong Content-Range. Tests: 404, 403 (at open and on a read), wrong Content-Range (at open and on a read), no range support, encoded body, ETag change, timeout, connection closed and bad scheme errors, Display and Debug, contain none of the secrets; h5rs likewise for every subcommand. Co-Authored-By: Claude Opus 5.5 (1M context) --- crates/clawhdf5-remote/src/error.rs | 140 ++++++++++++++++++ crates/clawhdf5-remote/src/http.rs | 84 +++++++---- crates/clawhdf5-remote/src/lib.rs | 12 +- crates/clawhdf5-remote/src/object.rs | 11 +- crates/clawhdf5-remote/tests/common/server.rs | 16 +- crates/clawhdf5-remote/tests/http.rs | 118 +++++++++++++++ crates/clawhdf5-tools/src/check.rs | 2 +- crates/clawhdf5-tools/src/diff.rs | 5 +- crates/clawhdf5-tools/src/dump.rs | 1 + crates/clawhdf5-tools/src/h5.rs | 40 ++++- crates/clawhdf5-tools/src/stat.rs | 2 +- crates/clawhdf5-tools/tests/remote.rs | 39 +++++ 12 files changed, 421 insertions(+), 49 deletions(-) diff --git a/crates/clawhdf5-remote/src/error.rs b/crates/clawhdf5-remote/src/error.rs index b2546b6..20494cd 100644 --- a/crates/clawhdf5-remote/src/error.rs +++ b/crates/clawhdf5-remote/src/error.rs @@ -2,8 +2,126 @@ use clawhdf5_format::error::FormatError; +/// `url` as it may be shown in an error, a `Debug` output or a log: no +/// userinfo (`user:password@`), no fragment, and the query string's values +/// replaced by `REDACTED` (a presigned S3/GCS/Azure URL carries its +/// signature or token there). Keys are kept when they look like plain +/// names, so a message still says which kind of URL it was. +/// +/// ``` +/// assert_eq!( +/// clawhdf5_remote::redact_url("https://me:pw@host/f.h5?X-Amz-Signature=abc&a=1#x"), +/// "https://host/f.h5?X-Amz-Signature=REDACTED&a=REDACTED" +/// ); +/// ``` +pub fn redact_url(url: &str) -> String { + let (scheme, rest) = match url.split_once("://") { + Some((s, r)) => (Some(s), r), + None => (None, url), + }; + let rest = rest.split('#').next().unwrap_or(""); + let (before_query, query) = match rest.split_once('?') { + Some((a, q)) => (a, Some(q)), + None => (rest, None), + }; + let mut out = String::with_capacity(url.len()); + if let Some(s) = scheme { + out.push_str(s); + out.push_str("://"); + let auth_end = before_query.find('/').unwrap_or(before_query.len()); + let (authority, path) = before_query.split_at(auth_end); + out.push_str( + authority + .rsplit_once('@') + .map_or(authority, |(_, host)| host), + ); + out.push_str(path); + } else { + out.push_str(before_query); + } + if let Some(q) = query { + out.push('?'); + let plain = |k: &str| { + !k.is_empty() + && k.len() <= 64 + && k.bytes() + .all(|b| b.is_ascii_alphanumeric() || matches!(b, b'-' | b'_' | b'.')) + }; + let parts: Vec = q + .split('&') + .map(|kv| { + let k = kv.split('=').next().unwrap_or(""); + if plain(k) { + format!("{k}=REDACTED") + } else { + "REDACTED".to_string() + } + }) + .collect(); + out.push_str(&parts.join("&")); + } + out +} + +/// Replaces the secret parts of one URL (its userinfo and query string, +/// and the URL itself) wherever they appear in a message — such as the +/// text of an error from the HTTP client. +#[derive(Debug, Clone)] +pub(crate) struct Redactor { + shown: String, + secrets: Vec<(String, String)>, +} + +impl Redactor { + pub(crate) fn new(url: &str) -> Redactor { + let shown = redact_url(url); + let mut secrets = vec![(url.to_string(), shown.clone())]; + let rest = url.split_once("://").map_or(url, |(_, r)| r); + let authority = rest.split(['/', '?', '#']).next().unwrap_or(""); + if let Some((userinfo, _)) = authority.rsplit_once('@') + && !userinfo.is_empty() + { + secrets.push((format!("{userinfo}@"), String::new())); + secrets.push((userinfo.to_string(), "REDACTED".into())); + } + if let Some((_, q)) = rest.split('#').next().unwrap_or("").split_once('?') + && !q.is_empty() + { + let shown_q = shown.split_once('?').map_or("", |(_, q)| q).to_string(); + secrets.push((q.to_string(), shown_q)); + for kv in q.split('&') { + if let Some((_, v)) = kv.split_once('=') + && v.len() >= 4 + { + secrets.push((v.to_string(), "REDACTED".into())); + } + } + } + Redactor { shown, secrets } + } + + /// The URL, redacted. + pub(crate) fn shown(&self) -> &str { + &self.shown + } + + /// `msg` with every secret part of the URL replaced. + pub(crate) fn scrub(&self, msg: &str) -> String { + let mut m = msg.to_string(); + for (secret, with) in &self.secrets { + if m.contains(secret.as_str()) { + m = m.replace(secret.as_str(), with); + } + } + m + } +} + /// Why a remote file could not be opened or read. /// +/// No message carries a URL's credentials: URLs appear as +/// [`redact_url`] shows them. +/// /// Inside a [`clawhdf5::File`] read these arrive as /// `clawhdf5::Error::Format(FormatError::Storage(message))`, the message /// being this error's `Display`. @@ -54,6 +172,28 @@ pub enum RemoteError { } impl RemoteError { + /// The error with every secret part of `r`'s URL scrubbed from its text. + #[cfg_attr(not(any(feature = "http", feature = "object-store")), allow(dead_code))] + pub(crate) fn scrubbed(self, r: &Redactor) -> RemoteError { + let f = |s: String| r.scrub(&s); + match self { + RemoteError::InvalidUrl(s) => RemoteError::InvalidUrl(f(s)), + RemoteError::UnsupportedScheme(s) => RemoteError::UnsupportedScheme(f(s)), + RemoteError::RangeNotSupported(s) => RemoteError::RangeNotSupported(f(s)), + RemoteError::FileChanged(s) => RemoteError::FileChanged(f(s)), + RemoteError::Status { code, what } => RemoteError::Status { + code, + what: f(what), + }, + RemoteError::BadResponse(s) => RemoteError::BadResponse(f(s)), + RemoteError::Transport(s) => RemoteError::Transport(f(s)), + RemoteError::ObjectStore(s) => RemoteError::ObjectStore(f(s)), + RemoteError::Usage(s) => RemoteError::Usage(f(s)), + e @ RemoteError::TooLarge { .. } => e, + RemoteError::Backend(s) => RemoteError::Backend(f(s)), + } + } + /// Whether retrying the same request may succeed. #[cfg_attr(not(feature = "http"), allow(dead_code))] pub(crate) fn is_transient(&self) -> bool { diff --git a/crates/clawhdf5-remote/src/http.rs b/crates/clawhdf5-remote/src/http.rs index ce3d68f..14f221e 100644 --- a/crates/clawhdf5-remote/src/http.rs +++ b/crates/clawhdf5-remote/src/http.rs @@ -36,7 +36,7 @@ use std::time::Duration; use clawhdf5_format::error::FormatError; use clawhdf5_format::storage::Storage; -use crate::error::RemoteError; +use crate::error::{Redactor, RemoteError}; /// Settings of an [`HttpStorage`]. #[derive(Debug, Clone)] @@ -107,7 +107,10 @@ enum Validator { /// documentation](self). pub struct HttpStorage { agent: ureq::Agent, + /// The URL as given, credentials and all: only ever sent to the server. url: String, + /// Shows the URL without its credentials, in every message. + redactor: Redactor, len: u64, validator: Validator, options: HttpOptions, @@ -122,7 +125,7 @@ pub struct HttpStorage { impl std::fmt::Debug for HttpStorage { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { f.debug_struct("HttpStorage") - .field("url", &self.url) + .field("url", &self.redactor.shown()) .field("len", &self.len) .field("validator", &self.validator) .field("full_download", &self.full.is_some()) @@ -181,15 +184,25 @@ impl HttpStorage { /// and the bytes that request fetched (the file's start), for a /// [`BlockCache`](crate::BlockCache) to keep. pub fn open(url: &str, options: HttpOptions) -> Result<(HttpStorage, Vec), RemoteError> { + let redactor = Redactor::new(url); + HttpStorage::open_inner(url, options, &redactor).map_err(|e| e.scrubbed(&redactor)) + } + + fn open_inner( + url: &str, + options: HttpOptions, + redactor: &Redactor, + ) -> Result<(HttpStorage, Vec), RemoteError> { + let shown = redactor.shown(); let lower = url.to_ascii_lowercase(); if lower.starts_with("https://") { if !cfg!(feature = "https") { return Err(RemoteError::UnsupportedScheme(format!( - "{url}: https:// needs the `https` feature of clawhdf5-remote" + "{shown}: https:// needs the `https` feature of clawhdf5-remote" ))); } } else if !lower.starts_with("http://") { - return Err(RemoteError::UnsupportedScheme(url.to_string())); + return Err(RemoteError::UnsupportedScheme(shown.to_string())); } let config = ureq::Agent::config_builder() .http_status_as_error(false) @@ -198,6 +211,7 @@ impl HttpStorage { let mut storage = HttpStorage { agent: ureq::Agent::new_with_config(config), url: url.to_string(), + redactor: redactor.clone(), len: 0, validator: Validator::None, options, @@ -216,14 +230,15 @@ impl HttpStorage { } if storage.options.require_validator && storage.validator == Validator::None { return Err(RemoteError::Usage(format!( - "{url}: the server sends neither a strong ETag nor Last-Modified, so a change \ + "{shown}: the server sends neither a strong ETag nor Last-Modified, so a change \ of the file could not be detected (HttpOptions::require_validator)" ))); } Ok((storage, bytes)) } - /// The URL. + /// The URL as given — with any credentials it carries, so do not log + /// it; [`redact_url`](crate::redact_url) gives a form that can be. pub fn url(&self) -> &str { &self.url } @@ -307,18 +322,20 @@ impl HttpStorage { let got = reader .take(cap.saturating_add(1)) .read_to_end(&mut buf) - .map_err(|e| RemoteError::Transport(format!("{}: reading the body: {e}", self.url))); + .map_err(|e| { + RemoteError::Transport(format!("{}: reading the body: {e}", self.redactor.shown())) + }); self.bytes.fetch_add(buf.len() as u64, Ordering::Relaxed); got?; match want { Some(n) if buf.len() as u64 != n => Err(RemoteError::BadResponse(format!( "{}: body of {} bytes, expected {n}", - self.url, + self.redactor.shown(), buf.len() ))), None if buf.len() as u64 > limit => Err(RemoteError::Usage(format!( "{}: the file is larger than HttpOptions::max_full_download ({limit} bytes)", - self.url + self.redactor.shown() ))), _ => Ok(buf), } @@ -332,7 +349,7 @@ impl HttpStorage { self.requests.fetch_add(1, Ordering::Relaxed); let resp = self.request(Some((0, n - 1))).call().map_err(transport)?; let status = resp.status().as_u16(); - check_identity(&self.url, &resp)?; + check_identity(self.redactor.shown(), &resp)?; let validator = match (header(&resp, "etag"), header(&resp, "last-modified")) { (Some(e), _) if !e.starts_with("W/") => Validator::ETag(e.to_string()), (_, Some(t)) => Validator::LastModified(t.to_string()), @@ -341,21 +358,27 @@ impl HttpStorage { match status { 206 => { let cr = header(&resp, "content-range").ok_or_else(|| { - RemoteError::BadResponse(format!("{}: 206 without Content-Range", self.url)) + RemoteError::BadResponse(format!( + "{}: 206 without Content-Range", + self.redactor.shown() + )) })?; let (a, b, total) = content_range(cr).ok_or_else(|| { - RemoteError::BadResponse(format!("{}: bad Content-Range {cr:?}", self.url)) + RemoteError::BadResponse(format!( + "{}: bad Content-Range {cr:?}", + self.redactor.shown() + )) })?; let total = total.ok_or_else(|| { RemoteError::BadResponse(format!( "{}: the server does not report the file's length (Content-Range {cr:?})", - self.url + self.redactor.shown() )) })?; if a != 0 || b >= total || b > n - 1 { return Err(RemoteError::BadResponse(format!( "{}: asked for bytes 0-{}, got Content-Range {cr:?}", - self.url, + self.redactor.shown(), n - 1 ))); } @@ -367,14 +390,15 @@ impl HttpStorage { return Err(RemoteError::RangeNotSupported(format!( "{} answered a range request with the whole file (status 200); set \ HttpOptions::allow_full_download to download it", - self.url + self.redactor.shown() ))); } let want = header(&resp, "content-length").and_then(|v| v.trim().parse().ok()); if want.is_some_and(|w: u64| w > self.options.max_full_download) { return Err(RemoteError::Usage(format!( "{}: the file is larger than HttpOptions::max_full_download ({} bytes)", - self.url, self.options.max_full_download + self.redactor.shown(), + self.options.max_full_download ))); } let bytes = self.body(resp, want, self.options.max_full_download)?; @@ -382,11 +406,11 @@ impl HttpStorage { } 416 => Err(RemoteError::Usage(format!( "{}: status 416 for the first bytes (an empty file?)", - self.url + self.redactor.shown() ))), code => Err(RemoteError::Status { code, - what: self.url.clone(), + what: self.redactor.shown().to_string(), }), } } @@ -398,14 +422,15 @@ impl HttpStorage { .request(Some((start, end - 1))) .call() .map_err(transport)?; - let changed = |why: String| RemoteError::FileChanged(format!("{}: {why}", self.url)); - check_identity(&self.url, &resp)?; + let changed = + |why: String| RemoteError::FileChanged(format!("{}: {why}", self.redactor.shown())); + check_identity(self.redactor.shown(), &resp)?; match resp.status().as_u16() { 206 => {} 200 => { return Err(RemoteError::RangeNotSupported(format!( "{} answered a range request with the whole file (status 200)", - self.url + self.redactor.shown() ))); } 412 => { @@ -417,7 +442,7 @@ impl HttpStorage { code => { return Err(RemoteError::Status { code, - what: format!("{} bytes {start}-{}", self.url, end - 1), + what: format!("{} bytes {start}-{}", self.redactor.shown(), end - 1), }); } } @@ -439,10 +464,16 @@ impl HttpStorage { Validator::None => {} } let cr = header(&resp, "content-range").ok_or_else(|| { - RemoteError::BadResponse(format!("{}: 206 without Content-Range", self.url)) + RemoteError::BadResponse(format!( + "{}: 206 without Content-Range", + self.redactor.shown() + )) })?; let (a, b, total) = content_range(cr).ok_or_else(|| { - RemoteError::BadResponse(format!("{}: bad Content-Range {cr:?}", self.url)) + RemoteError::BadResponse(format!( + "{}: bad Content-Range {cr:?}", + self.redactor.shown() + )) })?; if let Some(total) = total && total != self.len @@ -452,7 +483,7 @@ impl HttpStorage { if a != start || b != end - 1 { return Err(RemoteError::BadResponse(format!( "{}: asked for bytes {start}-{}, got Content-Range {cr:?}", - self.url, + self.redactor.shown(), end - 1 ))); } @@ -461,6 +492,7 @@ impl HttpStorage { fn fetch(&self, start: u64, end: u64) -> Result, RemoteError> { self.with_retries(|| self.fetch_once(start, end)) + .map_err(|e| e.scrubbed(&self.redactor)) } /// `[offset, offset + len)` clamped to the file, or `None` if empty. @@ -539,7 +571,7 @@ impl Storage for HttpStorage { if out.len() != ranges.len() { return Err(FormatError::Storage(format!( "{}: a parallel range read failed", - self.url + self.redactor.shown() ))); } Ok(out) diff --git a/crates/clawhdf5-remote/src/lib.rs b/crates/clawhdf5-remote/src/lib.rs index a441bab..cedbfe9 100644 --- a/crates/clawhdf5-remote/src/lib.rs +++ b/crates/clawhdf5-remote/src/lib.rs @@ -45,7 +45,7 @@ use clawhdf5::File; use clawhdf5_format::storage::Storage; pub use cache::{BlockCache, CacheConfig, CacheStats}; -pub use error::{Error, RemoteError}; +pub use error::{Error, RemoteError, redact_url}; #[cfg(feature = "http")] pub use http::{HttpOptions, HttpStats, HttpStorage}; #[cfg(feature = "object-store")] @@ -93,13 +93,13 @@ pub fn storage_for_url(url: &str, options: &Options) -> Result http_storage(url, options), "s3" | "s3a" | "gs" | "az" | "azure" | "abfs" | "abfss" | "adl" => { cloud_storage(url, &scheme, options) } - _ => Err(RemoteError::UnsupportedScheme(url.to_string()).into()), + _ => Err(RemoteError::UnsupportedScheme(redact_url(url)).into()), } } @@ -116,7 +116,8 @@ fn http_storage(url: &str, options: &Options) -> Result, Erro #[cfg(not(feature = "http"))] fn http_storage(url: &str, _options: &Options) -> Result, Error> { Err(RemoteError::UnsupportedScheme(format!( - "{url}: http(s):// needs the `http` feature of clawhdf5-remote" + "{}: http(s):// needs the `http` feature of clawhdf5-remote", + redact_url(url) )) .into()) } @@ -136,7 +137,8 @@ fn cloud_storage(url: &str, scheme: &str, _options: &Options) -> Result "azure", }; Err(RemoteError::UnsupportedScheme(format!( - "{url}: {scheme}:// needs the `{feature}` feature of clawhdf5-remote" + "{}: {scheme}:// needs the `{feature}` feature of clawhdf5-remote", + redact_url(url) )) .into()) } diff --git a/crates/clawhdf5-remote/src/object.rs b/crates/clawhdf5-remote/src/object.rs index 6bafe8d..c8c5664 100644 --- a/crates/clawhdf5-remote/src/object.rs +++ b/crates/clawhdf5-remote/src/object.rs @@ -239,13 +239,14 @@ impl Storage for ObjectStoreStorage { /// `GOOGLE_*`, `AZURE_*`). #[cfg(any(feature = "s3", feature = "gcs", feature = "azure"))] pub(crate) fn store_for_url(url: &str) -> Result<(Arc, Path), RemoteError> { + let redactor = crate::error::Redactor::new(url); let parsed = object_store::path::Path::parse( url.split_once("://") .and_then(|(_, rest)| rest.split_once('/')) .map(|(_, key)| key) .unwrap_or(""), ) - .map_err(|e| RemoteError::InvalidUrl(format!("{url}: {e}")))?; + .map_err(|e| RemoteError::InvalidUrl(format!("{}: {e}", crate::redact_url(url))))?; let scheme = url.split_once("://").map(|(s, _)| s.to_ascii_lowercase()); let store: Arc = match scheme.as_deref() { #[cfg(feature = "s3")] @@ -253,23 +254,23 @@ pub(crate) fn store_for_url(url: &str) -> Result<(Arc, Path), R object_store::aws::AmazonS3Builder::from_env() .with_url(url) .build() - .map_err(os_error)?, + .map_err(|e| os_error(e).scrubbed(&redactor))?, ), #[cfg(feature = "gcs")] Some("gs") => Arc::new( object_store::gcp::GoogleCloudStorageBuilder::from_env() .with_url(url) .build() - .map_err(os_error)?, + .map_err(|e| os_error(e).scrubbed(&redactor))?, ), #[cfg(feature = "azure")] Some("az" | "azure" | "abfs" | "abfss" | "adl") => Arc::new( object_store::azure::MicrosoftAzureBuilder::from_env() .with_url(url) .build() - .map_err(os_error)?, + .map_err(|e| os_error(e).scrubbed(&redactor))?, ), - _ => return Err(RemoteError::UnsupportedScheme(url.to_string())), + _ => return Err(RemoteError::UnsupportedScheme(crate::redact_url(url))), }; Ok((store, parsed)) } diff --git a/crates/clawhdf5-remote/tests/common/server.rs b/crates/clawhdf5-remote/tests/common/server.rs index 13f279a..2cdd7c1 100644 --- a/crates/clawhdf5-remote/tests/common/server.rs +++ b/crates/clawhdf5-remote/tests/common/server.rs @@ -52,6 +52,11 @@ pub struct Shared { /// when checking ranges) and serve zeros past its real end: a hostile /// server lying about the length. pub fake_total: AtomicU64, + /// When non-zero, answer every request for a served file with this + /// status (and an empty body). + pub force_status: AtomicU32, + /// Answer ranges with a `Content-Range` one byte off. + pub wrong_range: AtomicBool, /// Requests for a served path (every status); requests for other /// paths are not counted. pub requests: AtomicU64, @@ -256,6 +261,11 @@ fn serve(conn: TcpStream, s: &Shared) -> std::io::Result<()> { if delay > 0 { std::thread::sleep(Duration::from_millis(delay)); } + let forced = s.force_status.load(Ordering::SeqCst); + if forced != 0 { + write!(out, "HTTP/1.1 {forced} Forced\r\nContent-Length: 0\r\n\r\n")?; + continue; + } if s.fail_next .fetch_update(Ordering::SeqCst, Ordering::SeqCst, |n| n.checked_sub(1)) .is_ok() @@ -309,7 +319,11 @@ fn serve(conn: TcpStream, s: &Shared) -> std::io::Result<()> { Some(Ok((a, b))) => ( "206 Partial Content", slice_or_zeros(&data, a, b, &mut padded), - format!("Content-Range: bytes {a}-{b}/{len}\r\n"), + if s.wrong_range.load(Ordering::SeqCst) { + format!("Content-Range: bytes {}-{}/{len}\r\n", a + 1, b + 1) + } else { + format!("Content-Range: bytes {a}-{b}/{len}\r\n") + }, ), None => ("200 OK", &data[..], String::new()), }; diff --git a/crates/clawhdf5-remote/tests/http.rs b/crates/clawhdf5-remote/tests/http.rs index b4411b6..c61ee6e 100644 --- a/crates/clawhdf5-remote/tests/http.rs +++ b/crates/clawhdf5-remote/tests/http.rs @@ -604,3 +604,121 @@ fn download_is_bounded_by_its_limit_not_the_claimed_length() { ); assert_eq!(server.requests(), 0, "refused before reading"); } + +/// A URL's credentials — userinfo, and the query string of a presigned +/// URL — never appear in an error message or a `Debug` output, whatever +/// failed. +#[test] +fn credentials_never_appear_in_errors_or_debug() { + const SECRETS: [&str; 4] = ["hunter2", "SECRETSIG", "AKIDSECRET", "user:"]; + fn clean(what: &str, text: &str) { + for s in SECRETS { + assert!(!text.contains(s), "{what}: {s} leaked in {text}"); + } + } + fn check_err(what: &str, r: Result) { + let Err(e) = r else { + panic!("{what}: expected an error") + }; + clean(what, &format!("{e}")); + clean(what, &format!("{e:?}")); + } + let bytes = multi_block_file(); + let server = Server::start(vec![("/m.h5".into(), bytes.clone())]); + let secret_url = |path: &str| { + format!( + "http://user:hunter2@{}{path}?X-Amz-Credential=AKIDSECRET&X-Amz-Signature=SECRETSIG", + server.addr + ) + }; + let url = secret_url("/m.h5"); + let mut opts = quick(); + opts.http.retries = 0; + + // Opening works through such a URL, and Debug shows it redacted. + let (http, _) = HttpStorage::open(&url, opts.http.clone()).unwrap(); + let debug = format!("{http:?}"); + clean("Debug", &debug); + assert!(debug.contains("X-Amz-Signature=REDACTED"), "{debug}"); + assert_eq!(http.url(), url, "the URL itself is kept for requests"); + + check_err("404", open_url_with(&secret_url("/missing.h5"), &opts)); + server.shared.force_status.store(403, Ordering::SeqCst); + check_err("403", open_url_with(&url, &opts)); + server.shared.force_status.store(0, Ordering::SeqCst); + server.shared.wrong_range.store(true, Ordering::SeqCst); + check_err("bad range at open", open_url_with(&url, &opts)); + server.shared.wrong_range.store(false, Ordering::SeqCst); + server.shared.ignore_range.store(true, Ordering::SeqCst); + check_err("no range support", open_url_with(&url, &opts)); + server.shared.ignore_range.store(false, Ordering::SeqCst); + server.shared.gzip_label.store(true, Ordering::SeqCst); + check_err("encoded body", open_url_with(&url, &opts)); + server.shared.gzip_label.store(false, Ordering::SeqCst); + + // Errors of reads after the open, through a File. + let f = open_url_with(&url, &opts).unwrap(); + server.shared.wrong_range.store(true, Ordering::SeqCst); + check_err( + "bad range", + f.dataset("big") + .map(|d| d.read_f64()) + .and_then(|r| r) + .map_err(Error::Hdf5), + ); + server.shared.wrong_range.store(false, Ordering::SeqCst); + let f = open_url_with(&url, &opts).unwrap(); + server.shared.force_status.store(403, Ordering::SeqCst); + check_err( + "403 on a read", + f.dataset("big") + .map(|d| d.read_f64()) + .and_then(|r| r) + .map_err(Error::Hdf5), + ); + server.shared.force_status.store(0, Ordering::SeqCst); + let f = open_url_with(&url, &opts).unwrap(); + let mut other = bytes.clone(); + let n = other.len(); + other[n / 2] ^= 0xff; + server.shared.put("/m.h5", other); + check_err( + "ETag change", + f.dataset("big") + .map(|d| d.read_f64()) + .and_then(|r| r) + .map_err(Error::Hdf5), + ); + // A timeout. + let mut slow = opts.clone(); + slow.http.timeout = Duration::from_millis(100); + server.shared.delay_ms.store(1000, Ordering::SeqCst); + check_err("timeout", open_url_with(&url, &slow)); + server.shared.delay_ms.store(0, Ordering::SeqCst); + // A connection closed unanswered, a bad scheme. + let port = { + let l = std::net::TcpListener::bind("127.0.0.1:0").unwrap(); + let port = l.local_addr().unwrap().port(); + std::thread::spawn(move || { + for c in l.incoming() { + drop(c); + } + }); + port + }; + check_err( + "connection closed", + open_url_with( + &format!("http://user:hunter2@127.0.0.1:{port}/a.h5?X-Amz-Signature=SECRETSIG"), + &opts, + ), + ); + check_err( + "scheme", + open_url("ftp://user:hunter2@example.com/a.h5?X-Amz-Signature=SECRETSIG"), + ); + assert_eq!( + clawhdf5_remote::redact_url("https://me:pw@host:8/d/f.h5?X-Amz-Signature=abc&a=1#frag"), + "https://host:8/d/f.h5?X-Amz-Signature=REDACTED&a=REDACTED" + ); +} diff --git a/crates/clawhdf5-tools/src/check.rs b/crates/clawhdf5-tools/src/check.rs index ea8e09c..4a36554 100644 --- a/crates/clawhdf5-tools/src/check.rs +++ b/crates/clawhdf5-tools/src/check.rs @@ -204,7 +204,7 @@ pub fn run(args: &mut Args, out: &mut Out) -> std::io::Result { } } if !quiet { - c.summary(&file, out)?; + c.summary(&crate::h5::shown(&file), out)?; } Ok(if c.panicked { 3 diff --git a/crates/clawhdf5-tools/src/diff.rs b/crates/clawhdf5-tools/src/diff.rs index af684c5..d02bb8c 100644 --- a/crates/clawhdf5-tools/src/diff.rs +++ b/crates/clawhdf5-tools/src/diff.rs @@ -196,7 +196,8 @@ pub fn run(args: &mut Args, out: &mut Out) -> std::io::Result { Err(_) => { writeln!( out.e, - "h5rs diff: object <{obj}> could not be found in <{f}>" + "h5rs diff: object <{obj}> could not be found in <{}>", + crate::h5::shown(f) )?; return Ok(2); } @@ -214,7 +215,7 @@ pub fn run(args: &mut Args, out: &mut Out) -> std::io::Result { let entries = match collected { Ok(e) => e, Err(e) => { - writeln!(out.e, "h5rs diff: {f}: {e}")?; + writeln!(out.e, "h5rs diff: {}: {e}", crate::h5::shown(f))?; return Ok(2); } }; diff --git a/crates/clawhdf5-tools/src/dump.rs b/crates/clawhdf5-tools/src/dump.rs index 4b59f00..d7076e0 100644 --- a/crates/clawhdf5-tools/src/dump.rs +++ b/crates/clawhdf5-tools/src/dump.rs @@ -98,6 +98,7 @@ pub fn run(args: &mut Args, out: &mut Out) -> std::io::Result { problems: 0, paths: OnceCell::new(), }; + let file = crate::h5::shown(&file); let fname = std::path::Path::new(&file) .file_name() .map(|s| s.to_string_lossy().into_owned()) diff --git a/crates/clawhdf5-tools/src/h5.rs b/crates/clawhdf5-tools/src/h5.rs index d840074..c130f1f 100644 --- a/crates/clawhdf5-tools/src/h5.rs +++ b/crates/clawhdf5-tools/src/h5.rs @@ -229,20 +229,21 @@ impl H5 { if !is_url(arg) { return H5::open(Path::new(arg)); } + let name = shown(arg); #[cfg(feature = "remote")] { let storage = clawhdf5_remote::storage_for_url(arg, &clawhdf5_remote::Options::default()) - .map_err(|e| Error::new(format!("{arg}: {e}")))?; + .map_err(|e| Error::new(format!("{name}: {e}")))?; let size = storage.len(); let file = File::open_storage(storage).map_err(|e| { - Error::new(format!("{arg}: not an HDF5 file this tool can open: {e}")) + Error::new(format!("{name}: not an HDF5 file this tool can open: {e}")) })?; - Ok(H5::new(PathBuf::from(arg), file, size)) + Ok(H5::new(PathBuf::from(&name), file, size)) } #[cfg(not(feature = "remote"))] Err(Error::new(format!( - "{arg}: URLs need h5rs built with the `remote` feature" + "{name}: URLs need h5rs built with the `remote` feature" ))) } @@ -256,18 +257,19 @@ impl H5 { if h5.file.contiguous_bytes().is_some() { return Ok(h5); } + let name = shown(arg); #[cfg(feature = "remote")] { let storage = clawhdf5_remote::storage_for_url(arg, &clawhdf5_remote::Options::default()) - .map_err(|e| Error::new(format!("{arg}: {e}")))?; + .map_err(|e| Error::new(format!("{name}: {e}")))?; let bytes = clawhdf5_remote::download(&*storage, max_download) - .map_err(|e| Error::new(format!("{arg}: {e}")))?; + .map_err(|e| Error::new(format!("{name}: {e}")))?; let size = bytes.len() as u64; let file = File::from_bytes(bytes).map_err(|e| { - Error::new(format!("{arg}: not an HDF5 file this tool can open: {e}")) + Error::new(format!("{name}: not an HDF5 file this tool can open: {e}")) })?; - Ok(H5::new(PathBuf::from(arg), file, size)) + Ok(H5::new(PathBuf::from(&name), file, size)) } #[cfg(not(feature = "remote"))] { @@ -777,6 +779,28 @@ pub fn split_file_arg(arg: &str) -> (String, Option) { (arg.to_string(), None) } +/// A FILE argument as it may be printed: a URL without its credentials +/// (userinfo, query string values — a presigned URL's signature), a path +/// as given. +pub fn shown(arg: &str) -> String { + if !is_url(arg) { + return arg.to_string(); + } + #[cfg(feature = "remote")] + { + clawhdf5_remote::redact_url(arg) + } + #[cfg(not(feature = "remote"))] + { + let (scheme, rest) = arg.split_once("://").unwrap_or(("", arg)); + let rest = rest.split(['?', '#']).next().unwrap_or(""); + let host_end = rest.find('/').unwrap_or(rest.len()); + let (authority, path) = rest.split_at(host_end); + let host = authority.rsplit_once('@').map_or(authority, |(_, h)| h); + format!("{scheme}://{host}{path}") + } +} + /// Whether a FILE argument is a URL (`scheme://...`) rather than a path. pub fn is_url(arg: &str) -> bool { arg.split_once("://").is_some_and(|(scheme, _)| { diff --git a/crates/clawhdf5-tools/src/stat.rs b/crates/clawhdf5-tools/src/stat.rs index 32e91fe..611fdf2 100644 --- a/crates/clawhdf5-tools/src/stat.rs +++ b/crates/clawhdf5-tools/src/stat.rs @@ -206,7 +206,7 @@ pub fn run(args: &mut Args, out: &mut Out) -> std::io::Result { if let Err(e) = walk { errors.push(e.to_string()); } - report(&h5, &file, &s, out)?; + report(&h5, &crate::h5::shown(&file), &s, out)?; for e in &errors { writeln!(out.e, "h5rs stat: {e}")?; } diff --git a/crates/clawhdf5-tools/tests/remote.rs b/crates/clawhdf5-tools/tests/remote.rs index 426d1f7..0ae0b40 100644 --- a/crates/clawhdf5-tools/tests/remote.rs +++ b/crates/clawhdf5-tools/tests/remote.rs @@ -119,3 +119,42 @@ fn check_refuses_a_remote_file_beyond_the_download_limit() { assert_eq!(rc, 2, "{out}"); assert!(out.contains("more than the download limit"), "{out}"); } + +/// A URL's credentials (userinfo, a presigned URL's query string) are not +/// printed: not in errors, not in the file name of the output. +#[test] +fn credentials_in_urls_are_not_printed() { + let tall = Path::new(env!("CARGO_MANIFEST_DIR")).join("../clawhdf5/tests/fixtures/tall.h5"); + let server = server::Server::start(vec![("/t.h5".into(), std::fs::read(&tall).unwrap())]); + let url = |path: &str| { + format!( + "http://user:hunter2@{}{path}?X-Amz-Signature=SECRETSIG", + server.addr + ) + }; + for args in [ + vec!["ls", "-r"], + vec!["dump"], + vec!["stat"], + vec!["check"], + vec!["check", "--max-download", "10"], + ] { + for path in ["/t.h5", "/missing.h5"] { + let u = url(path); + let mut a = args.clone(); + a.push(&u); + let (out, _) = h5rs(&a); + assert!( + !out.contains("hunter2") && !out.contains("SECRETSIG"), + "h5rs {}: {out}", + a.join(" ") + ); + } + } + let (out, rc) = h5rs(&["diff", tall.to_str().unwrap(), &url("/t.h5"), "/nope"]); + assert_eq!(rc, 2, "{out}"); + assert!( + !out.contains("hunter2") && !out.contains("SECRETSIG"), + "{out}" + ); +} From 680c90b3a8f19662b54b0ba235e05944a94c7a62 Mon Sep 17 00:00:00 2001 From: osobh Date: Sat, 26 Sep 2026 18:31:39 -0500 Subject: [PATCH 13/20] clawhdf5-remote: redirects are followed safely MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ureq's defaults followed up to 10 redirects, including from https to plain http, and forwarded the custom HttpOptions::headers (X-Api-Key, Cookie, ...) to whatever host a redirect named — only Authorization was stripped. HttpStorage now follows redirects itself (ureq's max_redirects is 0): - at most HttpOptions::max_redirects per request (default 5; 0 refuses any redirect), then RemoteError::Redirect; - never from https to another scheme, nor to a non-http(s) URL; - once a redirect leaves the URL's origin (scheme, host, port), none of the custom headers is sent any more (Authorization included); - each hop counts as a request; errors show the target redacted. Tests: a redirect to another local port reads the right data and the target never sees X-Api-Key or Authorization (it did before); a same-origin redirect keeps them; a loop stops after 6 requests; 0 refuses; unit tests for target resolution, the https downgrade and origins. Co-Authored-By: Claude Opus 5.5 (1M context) --- crates/clawhdf5-remote/src/error.rs | 6 + crates/clawhdf5-remote/src/http.rs | 177 ++++++++++++++++-- crates/clawhdf5-remote/tests/common/server.rs | 37 +++- crates/clawhdf5-remote/tests/http.rs | 55 ++++++ 4 files changed, 260 insertions(+), 15 deletions(-) diff --git a/crates/clawhdf5-remote/src/error.rs b/crates/clawhdf5-remote/src/error.rs index 20494cd..6e3c551 100644 --- a/crates/clawhdf5-remote/src/error.rs +++ b/crates/clawhdf5-remote/src/error.rs @@ -158,6 +158,10 @@ pub enum RemoteError { ObjectStore(String), /// Called in a way the backend cannot serve. Usage(String), + /// A redirect that is not followed: from `https` to `http`, to + /// another scheme, or beyond + /// [`HttpOptions::max_redirects`](crate::HttpOptions). + Redirect(String), /// The file is larger than a download was allowed to be /// ([`download`](crate::download)). TooLarge { @@ -189,6 +193,7 @@ impl RemoteError { RemoteError::Transport(s) => RemoteError::Transport(f(s)), RemoteError::ObjectStore(s) => RemoteError::ObjectStore(f(s)), RemoteError::Usage(s) => RemoteError::Usage(f(s)), + RemoteError::Redirect(s) => RemoteError::Redirect(f(s)), e @ RemoteError::TooLarge { .. } => e, RemoteError::Backend(s) => RemoteError::Backend(f(s)), } @@ -221,6 +226,7 @@ impl std::fmt::Display for RemoteError { RemoteError::Transport(s) => write!(f, "network error: {s}"), RemoteError::ObjectStore(s) => write!(f, "object store: {s}"), RemoteError::Usage(s) => write!(f, "{s}"), + RemoteError::Redirect(s) => write!(f, "redirect refused: {s}"), RemoteError::TooLarge { len, limit } => write!( f, "the remote file is {len} bytes, more than the download limit of {limit} bytes" diff --git a/crates/clawhdf5-remote/src/http.rs b/crates/clawhdf5-remote/src/http.rs index 14f221e..9fd10c6 100644 --- a/crates/clawhdf5-remote/src/http.rs +++ b/crates/clawhdf5-remote/src/http.rs @@ -17,6 +17,10 @@ //! [`HttpOptions::allow_full_download`] is set: then the file is //! downloaded once, at open, and read from memory. //! +//! Redirects are followed (at most [`HttpOptions::max_redirects`]), but +//! never from `https` to `http`, and the custom +//! [`HttpOptions::headers`] are not sent to another origin. +//! //! Transient failures — connection errors, timeouts, `408`/`429`/`5xx`, and //! a body shorter or longer than its `Content-Range` — are retried with //! exponential backoff. Responses are requested with @@ -36,7 +40,7 @@ use std::time::Duration; use clawhdf5_format::error::FormatError; use clawhdf5_format::storage::Storage; -use crate::error::{Redactor, RemoteError}; +use crate::error::{Redactor, RemoteError, redact_url}; /// Settings of an [`HttpStorage`]. #[derive(Debug, Clone)] @@ -63,9 +67,17 @@ pub struct HttpOptions { /// `Last-Modified`, since a change of the file could then go unnoticed /// (only its length is checked). pub require_validator: bool, - /// Extra headers sent with every request (for example - /// `Authorization`). + /// Extra headers sent with every request to the URL's own origin (for + /// example `Authorization` or `X-Api-Key`). They are never sent to + /// another origin a redirect leads to. pub headers: Vec<(String, String)>, + /// Redirects followed per request (0: none, a redirect is an error). + /// A redirect from `https` to plain `http` is always refused; once a + /// redirect leaves the URL's origin (scheme, host and port), the + /// [`headers`](Self::headers) are no longer sent. Every request of a + /// file follows the redirects again (the target is not remembered, as + /// a presigned target may expire). + pub max_redirects: u32, } impl Default for HttpOptions { @@ -80,6 +92,7 @@ impl Default for HttpOptions { max_full_download: 1 << 30, require_validator: false, headers: Vec::new(), + max_redirects: 5, } } } @@ -166,6 +179,64 @@ fn check_identity(url: &str, resp: &ureq::http::Response) -> Result< } } +/// Scheme, host (lowercase, no userinfo) and port of an absolute URL. +fn origin(url: &str) -> Option<(String, String, u16)> { + let (scheme, rest) = url.split_once("://")?; + let scheme = scheme.to_ascii_lowercase(); + let authority = rest.split(['/', '?', '#']).next().unwrap_or(""); + let host_port = authority.rsplit_once('@').map_or(authority, |(_, h)| h); + let default = match scheme.as_str() { + "http" => 80, + "https" => 443, + _ => return None, + }; + // "[v6]:port", "host:port", or either without a port. + let (host, port) = match host_port.rsplit_once(':') { + Some((h, p)) if !p.contains(']') => (h, p.parse().ok()?), + _ => (host_port, default), + }; + Some((scheme, host.to_ascii_lowercase(), port)) +} + +fn same_origin(a: &str, b: &str) -> bool { + matches!((origin(a), origin(b)), (Some(x), Some(y)) if x == y) +} + +/// The URL a redirect from `base` to `location` goes to, if it may be +/// followed: `http`/`https` only, and never from `https` to `http`. +fn redirect_target(base: &str, location: &str) -> Result { + let location = location.trim(); + let (scheme, rest) = base.split_once("://").unwrap_or(("http", base)); + let authority_end = rest.find(['/', '?', '#']).unwrap_or(rest.len()); + let (authority, path) = rest.split_at(authority_end); + let target = if location.contains("://") { + location.to_string() + } else if let Some(r) = location.strip_prefix("//") { + format!("{scheme}://{r}") + } else if location.starts_with('/') { + format!("{scheme}://{authority}{location}") + } else { + let path = path.split(['?', '#']).next().unwrap_or(""); + let dir = path.rsplit_once('/').map_or("", |(d, _)| d); + format!("{scheme}://{authority}{dir}/{location}") + }; + let Some((to_scheme, _, _)) = origin(&target) else { + return Err(RemoteError::Redirect(format!( + "{} redirects to {}, which is not an http(s) URL", + redact_url(base), + redact_url(&target) + ))); + }; + if scheme.eq_ignore_ascii_case("https") && to_scheme != "https" { + return Err(RemoteError::Redirect(format!( + "{} redirects to {}: a downgrade from https is refused", + redact_url(base), + redact_url(&target) + ))); + } + Ok(target) +} + fn transport(e: ureq::Error) -> RemoteError { match e { ureq::Error::StatusCode(code) => RemoteError::Status { @@ -204,8 +275,10 @@ impl HttpStorage { } else if !lower.starts_with("http://") { return Err(RemoteError::UnsupportedScheme(shown.to_string())); } + // Redirects are followed by `call`, which applies our rules. let config = ureq::Agent::config_builder() .http_status_as_error(false) + .max_redirects(0) .timeout_global(Some(options.timeout)) .build(); let mut storage = HttpStorage { @@ -286,14 +359,15 @@ impl HttpStorage { } } + /// One `GET` of `url`. The options' custom headers are sent only to + /// the URL's own origin (`trusted`). fn request( &self, + url: &str, range: Option<(u64, u64)>, + trusted: bool, ) -> ureq::RequestBuilder { - let mut req = self - .agent - .get(&self.url) - .header("Accept-Encoding", "identity"); + let mut req = self.agent.get(url).header("Accept-Encoding", "identity"); if let Some((a, b)) = range { req = req.header("Range", format!("bytes={a}-{b}")); } @@ -302,12 +376,60 @@ impl HttpStorage { Validator::LastModified(t) => req = req.header("If-Unmodified-Since", t), Validator::None => {} } - for (k, v) in &self.options.headers { - req = req.header(k, v); + if trusted { + for (k, v) in &self.options.headers { + req = req.header(k, v); + } } req } + /// Send a ranged `GET`, following redirects by the rules of + /// [`HttpOptions::max_redirects`]: at most that many, never from + /// `https` to anything else, and without the custom headers once the + /// chain has left the URL's origin. Each hop counts as a request. + fn call( + &self, + range: Option<(u64, u64)>, + ) -> Result, RemoteError> { + let mut url = self.url.clone(); + let mut trusted = true; + let mut hops = 0u32; + loop { + if hops > 0 { + self.requests.fetch_add(1, Ordering::Relaxed); + } + let resp = self + .request(&url, range, trusted) + .call() + .map_err(|e| transport(e).scrubbed(&Redactor::new(&url)))?; + if !matches!(resp.status().as_u16(), 301 | 302 | 303 | 307 | 308) { + return Ok(resp); + } + let status = resp.status().as_u16(); + let Some(location) = header(&resp, "location") else { + return Err(RemoteError::BadResponse(format!( + "{}: status {status} without a Location", + redact_url(&url) + ))); + }; + let next = redirect_target(&url, location)?; + if hops >= self.options.max_redirects { + return Err(RemoteError::Redirect(format!( + "{} redirects to {}: more than HttpOptions::max_redirects ({})", + redact_url(&url), + redact_url(&next), + self.options.max_redirects + ))); + } + if !same_origin(&next, &self.url) { + trusted = false; + } + url = next; + hops += 1; + } + } + /// Read a body of exactly `want` bytes (or up to `limit` when `want` is /// unknown). fn body( @@ -347,7 +469,7 @@ impl HttpStorage { fn probe(&self) -> Result<(u64, Validator, Vec, bool), RemoteError> { let n = self.options.first_request.max(1); self.requests.fetch_add(1, Ordering::Relaxed); - let resp = self.request(Some((0, n - 1))).call().map_err(transport)?; + let resp = self.call(Some((0, n - 1)))?; let status = resp.status().as_u16(); check_identity(self.redactor.shown(), &resp)?; let validator = match (header(&resp, "etag"), header(&resp, "last-modified")) { @@ -418,10 +540,7 @@ impl HttpStorage { /// One request for `[start, end)` (inside the file, non-empty). fn fetch_once(&self, start: u64, end: u64) -> Result, RemoteError> { self.requests.fetch_add(1, Ordering::Relaxed); - let resp = self - .request(Some((start, end - 1))) - .call() - .map_err(transport)?; + let resp = self.call(Some((start, end - 1)))?; let changed = |why: String| RemoteError::FileChanged(format!("{}: {why}", self.redactor.shown())); check_identity(self.redactor.shown(), &resp)?; @@ -594,4 +713,34 @@ mod tests { assert_eq!(content_range("items 0-1/2"), None); assert_eq!(content_range("bytes */1000"), None); } + + #[test] + fn redirect_targets_and_origins() { + use super::{redirect_target, same_origin}; + let t = |b: &str, l: &str| redirect_target(b, l).map_err(|e| e.to_string()); + assert_eq!( + t("http://a:8/d/f.h5?x=1", "g.h5").unwrap(), + "http://a:8/d/g.h5" + ); + assert_eq!(t("http://a/d/f.h5", "/g.h5").unwrap(), "http://a/g.h5"); + assert_eq!(t("https://a/d/f.h5", "//b/g.h5").unwrap(), "https://b/g.h5"); + assert_eq!(t("http://a/f", "https://b/g").unwrap(), "https://b/g"); + let e = t("https://a/f.h5", "http://a/f.h5").unwrap_err(); + assert!(e.contains("downgrade"), "{e}"); + assert!(t("HTTPS://a/f.h5", "http://b/f.h5").is_err()); + assert!(t("https://a/f", "//b/g").is_ok()); + assert!(t("http://a/f", "ftp://b/g").is_err()); + let e = t("https://u:pw@a/f?sig=SECRET", "http://b/g?sig=OTHER").unwrap_err(); + assert!( + !e.contains("SECRET") && !e.contains("OTHER") && !e.contains("pw"), + "{e}" + ); + assert!(same_origin("http://a/x", "http://A:80/y")); + assert!(same_origin("https://u:p@a:443/x", "https://a/y")); + assert!(!same_origin("http://a/x", "https://a/x")); + assert!(!same_origin("http://a:1/x", "http://a:2/x")); + assert!(!same_origin("http://a/x", "http://b/x")); + assert!(same_origin("http://[::1]:8/x", "http://[::1]:8/y")); + assert!(!same_origin("http://[::1]/x", "http://[::1]:8/y")); + } } diff --git a/crates/clawhdf5-remote/tests/common/server.rs b/crates/clawhdf5-remote/tests/common/server.rs index 2cdd7c1..1708084 100644 --- a/crates/clawhdf5-remote/tests/common/server.rs +++ b/crates/clawhdf5-remote/tests/common/server.rs @@ -57,6 +57,10 @@ pub struct Shared { pub force_status: AtomicU32, /// Answer ranges with a `Content-Range` one byte off. pub wrong_range: AtomicBool, + /// Path → `Location`: answered `302 Found` (counted as a request). + pub redirects: RwLock>, + /// (path, headers) of every counted request, header names lowercase. + pub seen: Mutex)>>, /// Requests for a served path (every status); requests for other /// paths are not counted. pub requests: AtomicU64, @@ -119,11 +123,31 @@ impl Server { self.shared.bytes.load(Ordering::SeqCst) } - /// Zero the counters and the log. + /// Zero the counters and the logs. pub fn reset(&self) { self.shared.requests.store(0, Ordering::SeqCst); self.shared.bytes.store(0, Ordering::SeqCst); self.shared.log.lock().unwrap().clear(); + self.shared.seen.lock().unwrap().clear(); + } + + /// Answer `path` with a redirect to `location`. + pub fn redirect(&self, path: &str, location: &str) { + self.shared + .redirects + .write() + .unwrap() + .insert(path.to_string(), location.to_string()); + } + + /// Whether any request the server counted carried header `name`. + pub fn saw_header(&self, name: &str) -> bool { + self.shared + .seen + .lock() + .unwrap() + .iter() + .any(|(_, h)| h.contains_key(name)) } /// The ranges asked for so far. @@ -242,6 +266,16 @@ fn serve(conn: TcpStream, s: &Shared) -> std::io::Result<()> { let close = headers .get("connection") .is_some_and(|v| v.eq_ignore_ascii_case("close")); + let redirect = s.redirects.read().unwrap().get(&path).cloned(); + if let Some(location) = redirect { + s.requests.fetch_add(1, Ordering::SeqCst); + s.seen.lock().unwrap().push((path.clone(), headers.clone())); + write!( + out, + "HTTP/1.1 302 Found\r\nLocation: {location}\r\nContent-Length: 0\r\n\r\n" + )?; + continue; + } let res = { let files = s.files.read().unwrap(); files @@ -257,6 +291,7 @@ fn serve(conn: TcpStream, s: &Shared) -> std::io::Result<()> { continue; }; s.requests.fetch_add(1, Ordering::SeqCst); + s.seen.lock().unwrap().push((path.clone(), headers.clone())); let delay = s.delay_ms.load(Ordering::SeqCst); if delay > 0 { std::thread::sleep(Duration::from_millis(delay)); diff --git a/crates/clawhdf5-remote/tests/http.rs b/crates/clawhdf5-remote/tests/http.rs index c61ee6e..08119f3 100644 --- a/crates/clawhdf5-remote/tests/http.rs +++ b/crates/clawhdf5-remote/tests/http.rs @@ -722,3 +722,58 @@ fn credentials_never_appear_in_errors_or_debug() { "https://host:8/d/f.h5?X-Amz-Signature=REDACTED&a=REDACTED" ); } + +/// Redirects are followed within limits: custom credential headers reach +/// only the URL's own origin (not another port a redirect leads to), a +/// same-origin redirect keeps them, loops end at `max_redirects`, and +/// `max_redirects = 0` refuses any redirect. (The https→http downgrade +/// refusal is a unit test of `redirect_target`: no TLS server here.) +#[test] +fn redirects_are_followed_safely() { + let bytes = multi_block_file(); + let local = File::from_bytes(bytes.clone()).unwrap(); + let want = local.dataset("big").unwrap().read_f64().unwrap(); + let target = Server::start(vec![("/m.h5".into(), bytes.clone())]); + let front = Server::start(vec![("/m.h5".into(), bytes.clone())]); + front.redirect("/r.h5", &target.url("/m.h5")); + front.redirect("/s.h5", "/m.h5"); + front.redirect("/loop.h5", "/loop.h5"); + let mut opts = quick(); + opts.http.headers = vec![ + ("X-Api-Key".into(), "sekrit".into()), + ("Authorization".into(), "Bearer tok".into()), + ]; + + // Cross-origin (another port): followed, the headers stay behind. + let f = open_url_with(&front.url("/r.h5"), &opts).unwrap(); + assert_eq!(f.dataset("big").unwrap().read_f64().unwrap(), want); + assert!(front.saw_header("x-api-key"), "sent to the URL's origin"); + assert!(target.requests() > 1); + assert!( + !target.saw_header("x-api-key") && !target.saw_header("authorization"), + "credential headers forwarded across origins: {:?}", + target.shared.seen.lock().unwrap() + ); + + // Same origin: followed with the headers. + front.reset(); + let f = open_url_with(&front.url("/s.h5"), &opts).unwrap(); + assert_eq!(f.dataset("big").unwrap().read_f64().unwrap(), want); + let seen = front.shared.seen.lock().unwrap().clone(); + assert!( + seen.iter() + .filter(|(p, _)| p == "/m.h5") + .all(|(_, h)| h.get("x-api-key").map(String::as_str) == Some("sekrit")) + ); + + // A loop ends after max_redirects (5 by default): 6 requests. + front.reset(); + let e = open_url_with(&front.url("/loop.h5"), &opts).unwrap_err(); + assert!(matches!(e, Error::Remote(RemoteError::Redirect(_))), "{e}"); + assert_eq!(front.requests(), 6); + + // No redirects allowed. + opts.http.max_redirects = 0; + let e = open_url_with(&front.url("/s.h5"), &opts).unwrap_err(); + assert!(matches!(e, Error::Remote(RemoteError::Redirect(_))), "{e}"); +} From 61e34927dce32be401a6a563608c1875b4b1582c Mon Sep 17 00:00:00 2001 From: osobh Date: Sat, 26 Sep 2026 18:32:15 -0500 Subject: [PATCH 14/20] clawhdf5-remote: a 200 covering the requested range is the whole file The first request asks for bytes=0-1048575. RFC 9110 lets a server answer 200 when the range covers the whole representation, so a file under 1 MiB on a server that does support ranges could be refused as 'does not support range requests'. A 200 whose Content-Length (or, without one, its body, read at most that far) is within the range asked for is now kept as the whole file and read from memory; a longer one is still refused unless allow_full_download is set. Test: a 9968-byte file served with 200 opens in one request with the transcript of File::open (it was refused before); with a 4096-byte first request it is still refused. Co-Authored-By: Claude Opus 5.5 (1M context) --- crates/clawhdf5-remote/src/http.rs | 32 +++++++++++++++++++++++----- crates/clawhdf5-remote/tests/http.rs | 23 ++++++++++++++++++++ 2 files changed, 50 insertions(+), 5 deletions(-) diff --git a/crates/clawhdf5-remote/src/http.rs b/crates/clawhdf5-remote/src/http.rs index 9fd10c6..67b5612 100644 --- a/crates/clawhdf5-remote/src/http.rs +++ b/crates/clawhdf5-remote/src/http.rs @@ -12,6 +12,9 @@ //! open is [`RemoteError::FileChanged`], never a mix of old and new bytes. //! (A server that sends neither validator cannot be checked beyond the //! length; [`HttpOptions::require_validator`] refuses such servers.) +//! - a `200` answer to the first request whose body is no longer than the +//! range asked for is the whole file (a server may answer so when the +//! range covers it): it is kept and read from memory. //! - a server that ignores `Range` and answers `200` with the whole file is //! refused with [`RemoteError::RangeNotSupported`], unless //! [`HttpOptions::allow_full_download`] is set: then the file is @@ -508,15 +511,34 @@ impl HttpStorage { Ok((total, validator, bytes, false)) } 200 => { - if !self.options.allow_full_download { - return Err(RemoteError::RangeNotSupported(format!( + // RFC 9110 lets a server answer 200 when the range covers + // the whole file: a body no longer than the range asked + // for is the whole file, ranges supported or not. + let want: Option = + header(&resp, "content-length").and_then(|v| v.trim().parse().ok()); + let refused = || { + RemoteError::RangeNotSupported(format!( "{} answered a range request with the whole file (status 200); set \ HttpOptions::allow_full_download to download it", self.redactor.shown() - ))); + )) + }; + if !self.options.allow_full_download { + return match want { + Some(w) if w <= n => { + let bytes = self.body(resp, Some(w), 0)?; + Ok((bytes.len() as u64, validator, bytes, true)) + } + Some(_) => Err(refused()), + // No length: read at most the range asked for. + None => match self.body(resp, None, n) { + Ok(bytes) => Ok((bytes.len() as u64, validator, bytes, true)), + Err(RemoteError::Usage(_)) => Err(refused()), + Err(e) => Err(e), + }, + }; } - let want = header(&resp, "content-length").and_then(|v| v.trim().parse().ok()); - if want.is_some_and(|w: u64| w > self.options.max_full_download) { + if want.is_some_and(|w| w > self.options.max_full_download) { return Err(RemoteError::Usage(format!( "{}: the file is larger than HttpOptions::max_full_download ({} bytes)", self.redactor.shown(), diff --git a/crates/clawhdf5-remote/tests/http.rs b/crates/clawhdf5-remote/tests/http.rs index 08119f3..679f977 100644 --- a/crates/clawhdf5-remote/tests/http.rs +++ b/crates/clawhdf5-remote/tests/http.rs @@ -777,3 +777,26 @@ fn redirects_are_followed_safely() { let e = open_url_with(&front.url("/s.h5"), &opts).unwrap_err(); assert!(matches!(e, Error::Remote(RemoteError::Redirect(_))), "{e}"); } + +/// A server may answer the first request (bytes 0 to 1 MiB - 1) with 200 +/// when that covers the whole file: a body no longer than the range asked +/// for is accepted as the whole file, in one request. +#[test] +fn a_200_covering_the_requested_range_is_the_whole_file() { + let root = std::path::Path::new(env!("CARGO_MANIFEST_DIR")); + let tall = root.join("../clawhdf5/tests/fixtures/tall.h5"); + let bytes = std::fs::read(&tall).unwrap(); + assert!(bytes.len() < 1 << 20); + let server = Server::start(vec![("/t.h5".into(), bytes.clone())]); + server.shared.ignore_range.store(true, Ordering::SeqCst); + let storage = storage_for_url(&server.url("/t.h5"), &quick()).unwrap(); + let f = File::open_storage(storage).unwrap(); + assert_eq!(f.contiguous_bytes(), Some(&bytes[..])); + assert_eq!(transcript(&f), transcript(&File::open(&tall).unwrap())); + assert_eq!(server.requests(), 1); + // Larger than the range asked for: still refused. + let mut opts = quick().http; + opts.first_request = 4096; + let e = HttpStorage::open(&server.url("/t.h5"), opts).unwrap_err(); + assert!(matches!(e, RemoteError::RangeNotSupported(_)), "{e}"); +} From 30a1ed6b9c3be83d2e3ae982885452be0c0203d5 Mon Sep 17 00:00:00 2001 From: osobh Date: Sat, 26 Sep 2026 18:34:31 -0500 Subject: [PATCH 15/20] clawhdf5-remote: request timeouts scale with the body timeout_global (60 s) covered a whole request, and a request can carry 8 MiB (max_request): below about 140 KB/s every block run timed out, was retried from scratch and failed, so a slow link could not read remote files at all. HttpOptions::timeout (now 30 s) bounds connecting and receiving the response headers; the body gets timeout + its size at the new HttpOptions::min_speed (16 KiB/s by default: 94 s for a 1 MiB block). A slow but moving link is not cut off; a stalled one still fails. (ureq has no idle timeout; its body timeout is a total budget.) The test server can throttle bodies and stall mid-body. Test: a 256 KiB block at 256 KiB/s reads with a 300 ms timeout (it failed before), and a body stalled for 20 s fails in under 5 s. Co-Authored-By: Claude Opus 5.5 (1M context) --- crates/clawhdf5-remote/src/http.rs | 32 +++++++++++++++-- crates/clawhdf5-remote/tests/common/server.rs | 32 ++++++++++++++--- crates/clawhdf5-remote/tests/http.rs | 34 +++++++++++++++++++ 3 files changed, 91 insertions(+), 7 deletions(-) diff --git a/crates/clawhdf5-remote/src/http.rs b/crates/clawhdf5-remote/src/http.rs index 67b5612..221ca6c 100644 --- a/crates/clawhdf5-remote/src/http.rs +++ b/crates/clawhdf5-remote/src/http.rs @@ -24,6 +24,11 @@ //! never from `https` to `http`, and the custom //! [`HttpOptions::headers`] are not sent to another origin. //! +//! Timeouts scale with the request: [`HttpOptions::timeout`] to connect +//! and to get the response headers, and for the body that plus its size at +//! [`HttpOptions::min_speed`] — a slow link is not cut off mid-block, a +//! stalled connection still is. +//! //! Transient failures — connection errors, timeouts, `408`/`429`/`5xx`, and //! a body shorter or longer than its `Content-Range` — are retried with //! exponential backoff. Responses are requested with @@ -53,8 +58,15 @@ pub struct HttpOptions { pub retries: u32, /// Delay before the first retry; doubled for each further one. pub backoff: Duration, - /// Timeout of one request, from connecting to the end of the body. + /// Time allowed to connect, and then to receive the response headers. + /// The body gets this plus the time it takes at + /// [`min_speed`](Self::min_speed), so a request's budget grows with its + /// size: a slow but moving link is not cut off, a stalled one is. pub timeout: Duration, + /// Slowest transfer rate tolerated, in bytes per second: receiving a + /// body of `n` bytes may take `timeout + n / min_speed` (16 KiB/s by + /// default: 94 s for a 1 MiB block, 9 min for an 8 MiB request). + pub min_speed: u64, /// Requests of one `read_ranges` call in flight at once. pub max_parallel: usize, /// Bytes fetched by the first request, from offset 0 (the superblock and @@ -88,7 +100,8 @@ impl Default for HttpOptions { HttpOptions { retries: 3, backoff: Duration::from_millis(200), - timeout: Duration::from_secs(60), + timeout: Duration::from_secs(30), + min_speed: 16 << 10, max_parallel: 8, first_request: crate::cache::DEFAULT_BLOCK_SIZE, allow_full_download: false, @@ -282,7 +295,8 @@ impl HttpStorage { let config = ureq::Agent::config_builder() .http_status_as_error(false) .max_redirects(0) - .timeout_global(Some(options.timeout)) + .timeout_connect(Some(options.timeout)) + .timeout_recv_response(Some(options.timeout)) .build(); let mut storage = HttpStorage { agent: ureq::Agent::new_with_config(config), @@ -374,6 +388,18 @@ impl HttpStorage { if let Some((a, b)) = range { req = req.header("Range", format!("bytes={a}-{b}")); } + // The body's budget scales with what it may carry: the range, or + // a whole file the server may send instead. + let mut body = range.map_or(0, |(a, b)| b.saturating_sub(a).saturating_add(1)); + if self.options.allow_full_download { + body = body.max(self.options.max_full_download); + } + let secs = body as f64 / self.options.min_speed.max(1) as f64; + let body_timeout = self + .options + .timeout + .saturating_add(Duration::try_from_secs_f64(secs).unwrap_or(Duration::MAX)); + let mut req = req.config().timeout_recv_body(Some(body_timeout)).build(); match &self.validator { Validator::ETag(e) => req = req.header("If-Match", e), Validator::LastModified(t) => req = req.header("If-Unmodified-Since", t), diff --git a/crates/clawhdf5-remote/tests/common/server.rs b/crates/clawhdf5-remote/tests/common/server.rs index 1708084..ec7c8ae 100644 --- a/crates/clawhdf5-remote/tests/common/server.rs +++ b/crates/clawhdf5-remote/tests/common/server.rs @@ -55,6 +55,11 @@ pub struct Shared { /// When non-zero, answer every request for a served file with this /// status (and an empty body). pub force_status: AtomicU32, + /// When non-zero, send bodies at about this many bytes per second. + pub throttle_bps: AtomicU64, + /// When non-zero, stop this many milliseconds after the headers and + /// half the body (a stalled connection). + pub stall_ms: AtomicU64, /// Answer ranges with a `Content-Range` one byte off. pub wrong_range: AtomicBool, /// Path → `Location`: answered `302 Found` (counted as a request). @@ -384,13 +389,32 @@ fn serve(conn: TcpStream, s: &Shared) -> std::io::Result<()> { } else { body }; - let mut response = head.into_bytes(); - response.extend_from_slice(body); // Counted before the client can have the bytes, so a test that // resets the counters after a read never sees them arrive late. s.bytes.fetch_add(body.len() as u64, Ordering::SeqCst); - out.write_all(&response)?; - out.flush()?; + let bps = s.throttle_bps.load(Ordering::SeqCst); + let stall = s.stall_ms.load(Ordering::SeqCst); + if bps > 0 || stall > 0 { + out.write_all(head.as_bytes())?; + let (first, rest) = body.split_at(if stall > 0 { body.len() / 2 } else { 0 }); + out.write_all(first)?; + out.flush()?; + if stall > 0 { + std::thread::sleep(Duration::from_millis(stall)); + } + for piece in rest.chunks(4096) { + out.write_all(piece)?; + out.flush()?; + if bps > 0 { + std::thread::sleep(Duration::from_micros(4096 * 1_000_000 / bps)); + } + } + } else { + let mut response = head.into_bytes(); + response.extend_from_slice(body); + out.write_all(&response)?; + out.flush()?; + } if truncate || close { let _ = out.shutdown(std::net::Shutdown::Both); return Ok(()); diff --git a/crates/clawhdf5-remote/tests/http.rs b/crates/clawhdf5-remote/tests/http.rs index 679f977..2bf6da1 100644 --- a/crates/clawhdf5-remote/tests/http.rs +++ b/crates/clawhdf5-remote/tests/http.rs @@ -800,3 +800,37 @@ fn a_200_covering_the_requested_range_is_the_whole_file() { let e = HttpStorage::open(&server.url("/t.h5"), opts).unwrap_err(); assert!(matches!(e, RemoteError::RangeNotSupported(_)), "{e}"); } + +/// A slow link is not cut off: the body's time budget grows with its size +/// (`timeout` + size at `min_speed`), so a 256 KiB block at 256 KiB/s +/// (1 s) reads with a 300 ms `timeout`. A stalled body still fails, soon. +#[test] +fn slow_links_read_and_stalled_ones_fail() { + let bytes = multi_block_file(); + let server = Server::start(vec![("/m.h5".into(), bytes.clone())]); + let url = server.url("/m.h5"); + let mut opts = quick(); + opts.cache.block_size = 256 << 10; + opts.cache.max_request = 256 << 10; + opts.http.timeout = Duration::from_millis(300); + server + .shared + .throttle_bps + .store(256 << 10, Ordering::SeqCst); + let f = open_url_with(&url, &opts).unwrap(); + assert_eq!(f.root().groups().unwrap(), ["grp"]); + assert_eq!( + f.dataset("grp/small").unwrap().read_f64().unwrap(), + [1.0, 2.0, 3.0] + ); + server.shared.throttle_bps.store(0, Ordering::SeqCst); + + // Stalled mid-body for 20 s: a timeout well before that. + server.shared.stall_ms.store(20_000, Ordering::SeqCst); + opts.http.retries = 0; + opts.http.min_speed = 64 << 20; + let t = std::time::Instant::now(); + let e = open_url_with(&url, &opts).unwrap_err(); + assert!(matches!(e, Error::Remote(RemoteError::Transport(_))), "{e}"); + assert!(t.elapsed() < Duration::from_secs(5), "{:?}", t.elapsed()); +} From c5b2afbc35598a3cbc9ddf8744ce8bfe37ffaf13 Mon Sep 17 00:00:00 2001 From: osobh Date: Sat, 26 Sep 2026 18:35:36 -0500 Subject: [PATCH 16/20] clawhdf5-remote: ObjectStoreStorage works from any thread MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit It refused whenever Handle::try_current() was Ok, which is also the case inside spawn_blocking threads — so the workaround its own error message recommended failed the same way, and the backend could only be used from a bare std::thread in a tokio application. Reads are now spawned on the storage's own runtime and the caller waits on a channel: the future never runs on the caller's thread, so neither a spawn_blocking thread nor a current-thread runtime can deadlock or panic (a read inside a runtime blocks that thread, like any blocking call; the docs still recommend spawn_blocking there). Tests: a read in spawn_blocking of a multi-thread runtime and a read inside a current-thread runtime's task give File::open's values (both errors before). Co-Authored-By: Claude Opus 5.5 (1M context) --- CHANGELOG.md | 7 +-- crates/clawhdf5-remote/README.md | 10 ++--- crates/clawhdf5-remote/src/object.rs | 35 ++++++++------- crates/clawhdf5-remote/tests/object_store.rs | 45 +++++++++++++++++++- docs/design/range-reads.md | 7 +-- 5 files changed, 75 insertions(+), 29 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 69941c3..344afbf 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -32,9 +32,10 @@ backoff; bodies are asked for with `Accept-Encoding: identity` and an encoded one is refused. The ranges of one call are fetched in parallel. - **`ObjectStoreStorage`** reads one object of any `object_store` store, - pinned by ETag (else version or modification time) and size. It blocks - on a small tokio runtime it owns; called from inside another runtime it - refuses (`RemoteError::Usage`) instead of blocking a worker. + pinned by ETag (else version or modification time) and size. Each read + runs on a small tokio runtime the storage owns while the caller waits, + so it works from any thread, `spawn_blocking` and other runtimes + included. `open_object(store, path, options)` opens a file through a block cache. - Counted on the conformance corpus (tank, 2026-09-26, `CLAWHDF5_REMOTE_CORPUS=conformance/.cache/corpus CLAWHDF5_REMOTE_REPORT=1 diff --git a/crates/clawhdf5-remote/README.md b/crates/clawhdf5-remote/README.md index b4a487d..c64c7c6 100644 --- a/crates/clawhdf5-remote/README.md +++ b/crates/clawhdf5-remote/README.md @@ -62,11 +62,11 @@ checks both). ## Object stores `object_store` is async; `Storage` is synchronous (parsing is CPU work). -`ObjectStoreStorage` owns a small tokio runtime (two worker threads) and -blocks the calling thread on it for each read, so it is read from ordinary -threads, several at once. From inside an async runtime it refuses -(`RemoteError::Usage`) rather than block a worker: read in -`tokio::task::spawn_blocking`. The object is pinned by its ETag +`ObjectStoreStorage` owns a small tokio runtime (two worker threads): each +read runs there while the calling thread waits, so it works from any +thread, several at once — including `tokio::task::spawn_blocking` and code +inside another runtime (where `spawn_blocking` is still the better place, +since a read blocks the thread it is called on). The object is pinned by its ETag (`If-Match`, and compared on every response), else its version or modification time, and its size. The ranges of one read are fetched concurrently (up to 8). diff --git a/crates/clawhdf5-remote/src/object.rs b/crates/clawhdf5-remote/src/object.rs index c8c5664..ecfb7ac 100644 --- a/crates/clawhdf5-remote/src/object.rs +++ b/crates/clawhdf5-remote/src/object.rs @@ -3,12 +3,11 @@ //! //! `object_store` is async and [`Storage`] is synchronous (parsing is CPU //! work; `docs/design/range-reads.md` §3 (a)). The storage owns a small -//! multi-threaded tokio runtime (two worker threads) and blocks the calling -//! thread on it for each read, so it can be used from ordinary threads — -//! several at once. Calling it from inside another tokio runtime would -//! block that runtime's worker, so it refuses with -//! [`RemoteError::Usage`]: from async code, read in -//! `tokio::task::spawn_blocking`. +//! multi-threaded tokio runtime (two worker threads): each read is spawned +//! on it and the calling thread waits for the result, so it can be used +//! from any thread — several at once, and from async code too. From async +//! code prefer `tokio::task::spawn_blocking` (a read blocks the thread it +//! is called on, which inside a runtime is one of its workers). //! //! The object is pinned when the storage is made: its size, and its ETag //! (sent as `If-Match` with every read, and compared with every response) @@ -105,19 +104,23 @@ impl ObjectStoreStorage { ) } - fn block_on( + /// Run `fut` on the storage's own runtime and wait for it. The future + /// never runs on the caller's thread, so the caller's context does not + /// matter: a plain thread, `spawn_blocking`, or even inside another + /// runtime (whose thread is then blocked for the duration of the read, + /// as by any blocking call, but nothing deadlocks or panics). + fn block_on( &self, - fut: impl std::future::Future>, + fut: impl std::future::Future> + Send + 'static, ) -> Result { - if tokio::runtime::Handle::try_current().is_ok() { - return Err(RemoteError::Usage( - "ObjectStoreStorage blocks on its own runtime and cannot be read from inside \ - an async runtime; read in tokio::task::spawn_blocking" - .into(), - )); - } let rt = self.runtime.as_ref().expect("runtime lives until drop"); - rt.block_on(fut) + let (tx, rx) = std::sync::mpsc::sync_channel(1); + rt.spawn(async move { + let _ = tx.send(fut.await); + }); + rx.recv().map_err(|_| { + RemoteError::ObjectStore("the object store task ended without a result".into()) + })? } fn options(&self, range: Range) -> GetOptions { diff --git a/crates/clawhdf5-remote/tests/object_store.rs b/crates/clawhdf5-remote/tests/object_store.rs index 0d2f5fa..a3208ba 100644 --- a/crates/clawhdf5-remote/tests/object_store.rs +++ b/crates/clawhdf5-remote/tests/object_store.rs @@ -109,10 +109,51 @@ fn missing_objects_and_async_callers_are_clean_errors() { assert!(ObjectStoreStorage::new(store.clone(), ObjectPath::from("nope.h5")).is_err()); put(store.as_ref(), "m.h5", multi_block_file()); let storage = ObjectStoreStorage::new(store, ObjectPath::from("m.h5")).unwrap(); - // From inside a runtime: refused, not a panic or a deadlock. + // From inside a current-thread runtime: works (the read runs on the + // storage's own runtime), no panic, no deadlock. let rt = tokio_rt(); let r = rt.block_on(async { storage.read_at(0, 10).map(|b| b.len()) }); - assert!(r.unwrap_err().to_string().contains("spawn_blocking")); + assert_eq!(r.unwrap(), 10); // Dropping the storage inside a runtime does not panic. rt.block_on(async move { drop(storage) }); } + +/// The advice for async callers works: a read in `spawn_blocking` of a +/// multi-threaded runtime (where `Handle::try_current` is Ok), and a read +/// straight inside a current-thread runtime's task. +#[test] +fn object_stores_read_from_spawn_blocking_and_inside_runtimes() { + let store: Arc = Arc::new(InMemory::new()); + let bytes = multi_block_file(); + put(store.as_ref(), "m.h5", bytes.clone()); + let want = File::from_bytes(bytes) + .unwrap() + .dataset("big") + .unwrap() + .read_f64() + .unwrap(); + let rt = tokio::runtime::Builder::new_multi_thread() + .worker_threads(2) + .build() + .unwrap(); + let (s, w) = (store.clone(), want.clone()); + let got = rt + .block_on(async move { + tokio::task::spawn_blocking(move || { + assert!(tokio::runtime::Handle::try_current().is_ok()); + let (f, _) = + open_object(s, "m.h5", &Options::default()).map_err(|e| e.to_string())?; + let v = f.dataset("big").unwrap().read_f64().unwrap(); + Ok::<_, String>(v == w) + }) + .await + }) + .unwrap(); + assert_eq!(got, Ok(true)); + let rt = tokio_rt(); + let got = rt.block_on(async { + let (f, _) = open_object(store, "m.h5", &Options::default()).unwrap(); + f.dataset("big").unwrap().read_f64().unwrap() + }); + assert_eq!(got, want); +} diff --git a/docs/design/range-reads.md b/docs/design/range-reads.md index 2c7d1a4..123dc3b 100644 --- a/docs/design/range-reads.md +++ b/docs/design/range-reads.md @@ -471,9 +471,10 @@ fast path within benchmark noise. with ring. S3/GCS/Azure go through `object_store` (`ObjectStoreStorage`, features `s3`/`gcs`/`azure`, opt-in because of aws-lc-rs); the `object-store` feature alone (in-memory, local files, a store you - build) is pure Rust. object_store is async: the storage blocks on a - two-thread tokio runtime of its own, and refuses to run inside another - runtime. + build) is pure Rust. object_store is async: each read runs on a + two-thread tokio runtime of the storage's own while the caller waits, + so the caller's context (a plain thread, `spawn_blocking`, another + runtime) does not matter. - `BlockCache` (any `Storage`): 1 MiB blocks and a 64 MiB LRU budget by default, the first block fetched at open (for HTTP by the request that learns the length), the missing blocks of one read fetched as From ef480746daa4e6876caf8e0f2113d0123439dc43 Mon Sep 17 00:00:00 2001 From: osobh Date: Sat, 26 Sep 2026 18:36:35 -0500 Subject: [PATCH 17/20] clawhdf5-remote: a failed first fetch is Error::Remote cached() mapped an error of its open-time prefetch (a network error, a changed file) to Error::Hdf5(Format(Storage)), misclassifying it for callers that match on the variant. It is now Error::Remote (RemoteError::Backend with the backend's message). open_object and the s3/gs/az URLs fetch the first block of an ObjectStoreStorage directly, so their errors keep their kind (FileChanged, ObjectStore). Test: cached() over a backend whose reads fail gives Error::Remote(Backend) (Error::Hdf5 before). Co-Authored-By: Claude Opus 5.5 (1M context) --- crates/clawhdf5-remote/src/lib.rs | 24 +++++++++++++++++++----- crates/clawhdf5-remote/src/object.rs | 5 +++++ crates/clawhdf5-remote/tests/http.rs | 24 ++++++++++++++++++++++++ 3 files changed, 48 insertions(+), 5 deletions(-) diff --git a/crates/clawhdf5-remote/src/lib.rs b/crates/clawhdf5-remote/src/lib.rs index cedbfe9..2835440 100644 --- a/crates/clawhdf5-remote/src/lib.rs +++ b/crates/clawhdf5-remote/src/lib.rs @@ -126,7 +126,7 @@ fn http_storage(url: &str, _options: &Options) -> Result, Err fn cloud_storage(url: &str, _scheme: &str, options: &Options) -> Result, Error> { let (store, path) = object::store_for_url(url)?; let storage = ObjectStoreStorage::new(store, path)?; - Ok(Arc::new(cached(Box::new(storage), options)?)) + Ok(Arc::new(object_cached(storage, options)?)) } #[cfg(not(any(feature = "s3", feature = "gcs", feature = "azure")))] @@ -144,13 +144,27 @@ fn cloud_storage(url: &str, scheme: &str, _options: &Options) -> Result Result { let cache = BlockCache::new(backend, options.cache.clone()); let first = cache.config().block_size; cache .prefetch(0, first) - .map_err(|e| Error::Hdf5(clawhdf5::Error::Format(e)))?; + .map_err(|e| RemoteError::Backend(e.to_string()))?; + Ok(cache) +} + +/// [`cached`] for an [`ObjectStoreStorage`]: its first block is fetched +/// directly, so a failure keeps its kind (`FileChanged`, `ObjectStore`). +#[cfg(feature = "object-store")] +fn object_cached(storage: ObjectStoreStorage, options: &Options) -> Result { + // The block size BlockCache::new will use. + let block = options.cache.block_size.max(512); + let first = storage.fetch_first(block)?; + let cache = BlockCache::new(Box::new(storage) as Backend, options.cache.clone()); + cache.insert(0, &first); Ok(cache) } @@ -205,8 +219,8 @@ pub fn open_object( ) -> Result<(File, Arc), Error> { let path = object_store::path::Path::parse(path) .map_err(|e| RemoteError::InvalidUrl(format!("{path}: {e}")))?; - let storage = Arc::new(cached( - Box::new(ObjectStoreStorage::new(store, path)?), + let storage = Arc::new(object_cached( + ObjectStoreStorage::new(store, path)?, options, )?); let file = File::open_storage(storage.clone())?; diff --git a/crates/clawhdf5-remote/src/object.rs b/crates/clawhdf5-remote/src/object.rs index ecfb7ac..c0b7748 100644 --- a/crates/clawhdf5-remote/src/object.rs +++ b/crates/clawhdf5-remote/src/object.rs @@ -138,6 +138,11 @@ impl ObjectStoreStorage { o } + /// The object's first `n` bytes (fewer if it is shorter). + pub(crate) fn fetch_first(&self, n: u64) -> Result, RemoteError> { + Ok(self.fetch_all(&[0..n])?.pop().unwrap_or_default()) + } + fn fetch_all(&self, ranges: &[Range]) -> Result>, RemoteError> { let len = self.meta.size; let jobs: Vec<(usize, Range)> = ranges diff --git a/crates/clawhdf5-remote/tests/http.rs b/crates/clawhdf5-remote/tests/http.rs index 2bf6da1..a63fb72 100644 --- a/crates/clawhdf5-remote/tests/http.rs +++ b/crates/clawhdf5-remote/tests/http.rs @@ -834,3 +834,27 @@ fn slow_links_read_and_stalled_ones_fail() { assert!(matches!(e, Error::Remote(RemoteError::Transport(_))), "{e}"); assert!(t.elapsed() < Duration::from_secs(5), "{:?}", t.elapsed()); } + +/// `cached` reports a failed first fetch as a remote error, not as an +/// HDF5 format error. +#[test] +fn cached_reports_a_failed_prefetch_as_remote() { + use clawhdf5_format::error::FormatError; + use std::borrow::Cow; + struct Down; + impl clawhdf5_format::storage::Storage for Down { + fn read_at(&self, _: u64, _: usize) -> Result, FormatError> { + Err(RemoteError::Transport("connection refused".into()).into()) + } + fn len(&self) -> u64 { + 1 << 20 + } + } + let e = clawhdf5_remote::cached(Box::new(Down), &Options::default()) + .map(|_| ()) + .unwrap_err(); + assert!( + matches!(&e, Error::Remote(RemoteError::Backend(m)) if m.contains("connection refused")), + "{e:?}" + ); +} From efb88f94e31198c7359d37097b2e90e6c5772188 Mon Sep 17 00:00:00 2001 From: osobh Date: Sat, 26 Sep 2026 18:36:35 -0500 Subject: [PATCH 18/20] h5rs: check URL opens the remote file once open_arg_whole opened and parsed the remote file through open_arg, then opened it again to download it, so every `h5rs check URL` probed the server twice. It now opens the storage once and downloads through the same block cache (whose first block the probe already filled). Test: check --data of a file within one block costs exactly one request (two before). Co-Authored-By: Claude Opus 5.5 (1M context) --- crates/clawhdf5-tools/src/h5.rs | 11 +++++++---- crates/clawhdf5-tools/tests/remote.rs | 11 +++++++++++ 2 files changed, 18 insertions(+), 4 deletions(-) diff --git a/crates/clawhdf5-tools/src/h5.rs b/crates/clawhdf5-tools/src/h5.rs index c130f1f..7cba7c3 100644 --- a/crates/clawhdf5-tools/src/h5.rs +++ b/crates/clawhdf5-tools/src/h5.rs @@ -253,13 +253,14 @@ impl H5 { /// than `max_download` bytes is refused before anything is read: its /// length is only what the server claims. pub fn open_arg_whole(arg: &str, max_download: u64) -> Result
{ - let h5 = H5::open_arg(arg)?; - if h5.file.contiguous_bytes().is_some() { - return Ok(h5); + if !is_url(arg) { + return H5::open_arg(arg); } let name = shown(arg); #[cfg(feature = "remote")] { + // One open (one probe of the server); the download then reads + // through the same cache, the first block already in it. let storage = clawhdf5_remote::storage_for_url(arg, &clawhdf5_remote::Options::default()) .map_err(|e| Error::new(format!("{name}: {e}")))?; @@ -274,7 +275,9 @@ impl H5 { #[cfg(not(feature = "remote"))] { let _ = max_download; - unreachable!("open_arg refuses URLs without the remote feature") + Err(Error::new(format!( + "{name}: URLs need h5rs built with the `remote` feature" + ))) } } diff --git a/crates/clawhdf5-tools/tests/remote.rs b/crates/clawhdf5-tools/tests/remote.rs index 0ae0b40..de39814 100644 --- a/crates/clawhdf5-tools/tests/remote.rs +++ b/crates/clawhdf5-tools/tests/remote.rs @@ -158,3 +158,14 @@ fn credentials_in_urls_are_not_printed() { "{out}" ); } + +/// `check URL` opens the file once: for a file within the first block, +/// one request in all (it probed the server twice before). +#[test] +fn check_url_probes_the_server_once() { + let tall = Path::new(env!("CARGO_MANIFEST_DIR")).join("../clawhdf5/tests/fixtures/tall.h5"); + let server = server::Server::start(vec![("/t.h5".into(), std::fs::read(&tall).unwrap())]); + let (out, rc) = h5rs(&["check", "--data", &server.url("/t.h5")]); + assert_eq!(rc, 0, "{out}"); + assert_eq!(server.requests(), 1, "{:?}", server.log()); +} From 0e98ffc498e702afc34636b3f43df75a5d8b9c35 Mon Sep 17 00:00:00 2001 From: osobh Date: Sat, 26 Sep 2026 18:37:05 -0500 Subject: [PATCH 19/20] docs: remote files after the adversarial review CHANGELOG, the clawhdf5-remote and h5rs READMEs and the remote-files known issues: redirect rules, scaled timeouts (min_speed), URL redaction, claimed lengths never allocated (download, --max-download), a 200 for a small file accepted, and ObjectStoreStorage from any thread. Co-Authored-By: Claude Opus 5.5 (1M context) --- CHANGELOG.md | 19 ++++++++++++++++++- crates/clawhdf5-remote/README.md | 17 +++++++++++++++++ crates/clawhdf5-tools/README.md | 3 ++- docs/known-issues.md | 20 ++++++++++++++++---- 4 files changed, 53 insertions(+), 6 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 344afbf..a281969 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -31,6 +31,21 @@ `408`/`429`/`5xx` and cut-off bodies are retried with exponential backoff; bodies are asked for with `Accept-Encoding: identity` and an encoded one is refused. The ranges of one call are fetched in parallel. + A `200` answer to the first request whose body fits the range asked + for is taken as the whole file (a server may answer so for a small + file). Timeouts scale with the request: `HttpOptions::timeout` (30 s) + to connect and to get the headers, plus the body's size at + `HttpOptions::min_speed` (16 KiB/s), so a slow link is not cut off. + Redirects: at most `HttpOptions::max_redirects` (5), never from `https` + to `http`, and `HttpOptions::headers` are not sent to another origin. + No error or `Debug` output contains a URL's userinfo or query values + (`redact_url`; presigned URLs carry their signature there). + - **Hostile lengths**: the length a server claims is never used to + allocate. The cache's arithmetic is checked (a length near `u64::MAX` + used to overflow), a read spanning more than the budget is fetched + piece by piece with its output growing as data arrives, and + `download(storage, max_bytes)` reads a whole file only up to a limit + (`RemoteError::TooLarge` before any request otherwise). - **`ObjectStoreStorage`** reads one object of any `object_store` store, pinned by ETag (else version or modification time) and size. Each read runs on a small tokio runtime the storage owns while the caller waits, @@ -49,7 +64,9 @@ requests; the tests hold it to at most 3. - **`h5rs` takes URLs** with the new `remote` feature (`remote-https` for `https://`): `ls`, `dump`, `stat` and `diff` read by range requests; - `check` downloads the file whole, since it validates every byte. The + `check` downloads the file whole, since it validates every byte, up to + `--max-download N` (1 GiB by default). URLs are printed without their + credentials. The tools now read through `File::storage` and the format crate's `*_in` functions; local output is unchanged. - **`File::storage()`** (facade) returns the file's bytes from the diff --git a/crates/clawhdf5-remote/README.md b/crates/clawhdf5-remote/README.md index c64c7c6..cd663b4 100644 --- a/crates/clawhdf5-remote/README.md +++ b/crates/clawhdf5-remote/README.md @@ -38,10 +38,27 @@ fetched, by `Range` requests, through a block cache. refused (`RemoteError::RangeNotSupported`) without reading the body, unless `HttpOptions::allow_full_download` is set; then the file is downloaded once and read from memory. + A `200` whose body is no longer than the range asked for is the whole + (small) file, and is accepted. - **Retries**: connection failures, timeouts, `408`/`429`/`5xx` and bodies that end early are retried with exponential backoff (3 retries, from 200 ms). Bodies are requested with `Accept-Encoding: identity`; an encoded body is refused. +- **Timeouts** scale with the request: `HttpOptions::timeout` (30 s) to + connect and to receive the headers, and for the body that plus its size + at `HttpOptions::min_speed` (16 KiB/s) — a slow link is not cut off + mid-block, a stalled connection still fails. +- **Redirects** are followed up to `HttpOptions::max_redirects` (5; 0 + refuses them), never from `https` to `http`. Once a redirect leaves the + URL's origin (scheme, host, port), `HttpOptions::headers` (API keys, + `Authorization`, cookies) are no longer sent. +- **Credentials stay out of messages**: every error and `Debug` output + shows URLs through `redact_url` — no `user:password@`, query values + replaced by `REDACTED` (a presigned S3/GCS URL's signature lives there). +- **Claimed lengths are not trusted**: nothing is allocated for the length + a server reports; a read spanning more than the cache budget is fetched + in pieces as data arrives, and `download(&storage, max_bytes)` reads a + whole file only up to a limit (`DEFAULT_MAX_DOWNLOAD`, 1 GiB). The zero-copy methods of `clawhdf5` (`read_raw_ref`, `read_*_zerocopy`, `File::as_bytes`) borrow the whole file from memory, so they are errors diff --git a/crates/clawhdf5-tools/README.md b/crates/clawhdf5-tools/README.md index fc120ad..24ecb9c 100644 --- a/crates/clawhdf5-tools/README.md +++ b/crates/clawhdf5-tools/README.md @@ -42,7 +42,8 @@ h5rs diff local.h5 http://127.0.0.1:8000/file.h5 A URL names the whole file (`FILE/OBJECT` suffixes are for local paths). `check` validates every byte, so it downloads a remote file whole first — up to `--max-download N` bytes (default 1 GiB), refusing a longer file -before reading any of it. +before reading any of it. URLs are printed without their credentials +(userinfo, query string values). The output is the local file's (`tests/remote.rs` compares every subcommand). diff --git a/docs/known-issues.md b/docs/known-issues.md index 860e1eb..7275020 100644 --- a/docs/known-issues.md +++ b/docs/known-issues.md @@ -829,11 +829,23 @@ cache, but: ETag (`W/"…"`) cannot be sent as `If-Match`, so it counts as none. - **Credentials:** HTTP takes extra headers (`HttpOptions::headers`, e.g. `Authorization`); `h5rs` has no option for them. The cloud stores read - credentials from the environment only. + credentials from the environment only. Messages and `Debug` output show + URLs through `redact_url` (no userinfo, query values `REDACTED`); + `HttpStorage::url()` returns the URL as given and must not be logged. +- **Redirects:** at most `HttpOptions::max_redirects` (5) per request, + never from `https` to `http`, and the custom headers are not sent once a + redirect leaves the URL's origin. The redirect target is not remembered: + every request of a redirected file costs its hops again. +- **Timeouts:** ureq has no idle timeout, only a total one for the body, + so the body's budget is `timeout + size / min_speed` (30 s + 16 KiB/s + by default). A connection that stalls mid-body is detected only when + that budget runs out (94 s for a 1 MiB block, 9 min for an 8 MiB run). - Each `ObjectStoreStorage` owns a tokio runtime with two worker threads. -- `h5rs check` downloads a remote file whole (it validates every byte), and - a URL cannot carry a `FILE/OBJECT` suffix; `h5rs` uses the default cache - settings. + A read called from inside another runtime blocks that runtime's thread + for its duration (it works, but `spawn_blocking` is the better place). +- `h5rs check` downloads a remote file whole (it validates every byte), up + to `--max-download` (1 GiB by default), and a URL cannot carry a + `FILE/OBJECT` suffix; `h5rs` uses the default cache settings. - The zero-copy methods and `File::as_bytes` are unavailable on a remote file (see the range-read limits above). From 67e72b30d772bb57832186336e6587e782947a4c Mon Sep 17 00:00:00 2001 From: osobh Date: Sat, 26 Sep 2026 18:37:47 -0500 Subject: [PATCH 20/20] clawhdf5-remote: clippy clean with every feature set checked_div in the test server's throttle, a slice for the single range of fetch_first, and dead-code allowances for the redaction helpers in a build with neither http nor a cloud store. Co-Authored-By: Claude Opus 5.5 (1M context) --- crates/clawhdf5-remote/src/error.rs | 13 ++++++++++++- crates/clawhdf5-remote/src/object.rs | 5 ++++- crates/clawhdf5-remote/tests/common/server.rs | 4 ++-- 3 files changed, 18 insertions(+), 4 deletions(-) diff --git a/crates/clawhdf5-remote/src/error.rs b/crates/clawhdf5-remote/src/error.rs index 6e3c551..2e56bef 100644 --- a/crates/clawhdf5-remote/src/error.rs +++ b/crates/clawhdf5-remote/src/error.rs @@ -66,12 +66,20 @@ pub fn redact_url(url: &str) -> String { /// Replaces the secret parts of one URL (its userinfo and query string, /// and the URL itself) wherever they appear in a message — such as the /// text of an error from the HTTP client. +#[cfg_attr( + not(any(feature = "http", feature = "s3", feature = "gcs", feature = "azure")), + allow(dead_code) +)] #[derive(Debug, Clone)] pub(crate) struct Redactor { shown: String, secrets: Vec<(String, String)>, } +#[cfg_attr( + not(any(feature = "http", feature = "s3", feature = "gcs", feature = "azure")), + allow(dead_code) +)] impl Redactor { pub(crate) fn new(url: &str) -> Redactor { let shown = redact_url(url); @@ -177,7 +185,10 @@ pub enum RemoteError { impl RemoteError { /// The error with every secret part of `r`'s URL scrubbed from its text. - #[cfg_attr(not(any(feature = "http", feature = "object-store")), allow(dead_code))] + #[cfg_attr( + not(any(feature = "http", feature = "s3", feature = "gcs", feature = "azure")), + allow(dead_code) + )] pub(crate) fn scrubbed(self, r: &Redactor) -> RemoteError { let f = |s: String| r.scrub(&s); match self { diff --git a/crates/clawhdf5-remote/src/object.rs b/crates/clawhdf5-remote/src/object.rs index c0b7748..3ac9955 100644 --- a/crates/clawhdf5-remote/src/object.rs +++ b/crates/clawhdf5-remote/src/object.rs @@ -140,7 +140,10 @@ impl ObjectStoreStorage { /// The object's first `n` bytes (fewer if it is shorter). pub(crate) fn fetch_first(&self, n: u64) -> Result, RemoteError> { - Ok(self.fetch_all(&[0..n])?.pop().unwrap_or_default()) + Ok(self + .fetch_all(std::slice::from_ref(&(0..n)))? + .pop() + .unwrap_or_default()) } fn fetch_all(&self, ranges: &[Range]) -> Result>, RemoteError> { diff --git a/crates/clawhdf5-remote/tests/common/server.rs b/crates/clawhdf5-remote/tests/common/server.rs index ec7c8ae..cc27e6f 100644 --- a/crates/clawhdf5-remote/tests/common/server.rs +++ b/crates/clawhdf5-remote/tests/common/server.rs @@ -405,8 +405,8 @@ fn serve(conn: TcpStream, s: &Shared) -> std::io::Result<()> { for piece in rest.chunks(4096) { out.write_all(piece)?; out.flush()?; - if bps > 0 { - std::thread::sleep(Duration::from_micros(4096 * 1_000_000 / bps)); + if let Some(us) = (4096 * 1_000_000u64).checked_div(bps) { + std::thread::sleep(Duration::from_micros(us)); } } } else {