osobhandClaude Opus 5.5
db2554dd81
Range-read milestone M3, first half: a new crate with the block cache the
design makes mandatory for remote files and an HTTP backend, so
open_url("http://...") gives a clawhdf5::File over File::open_storage.
BlockCache wraps any Storage: aligned blocks (1 MiB by default, the size
docs/design/range-reads.md section 2 measured), LRU with a byte budget,
the missing blocks of one read_at/read_ranges fetched with one backend
read_ranges call as runs of consecutive blocks (a one-block gap filled to
merge runs, each request at most 8 MiB), and reads that miss more than
half the budget not kept. Thread-safe without holding the lock across a
fetch: a block being fetched is in flight, a second reader waits for it
instead of fetching it again, and a failed fetch fails its waiters and is
not cached. A backend holding the file in memory passes through.
HttpStorage (ureq, no TLS by default; `https` adds rustls with ring):
opening is one ranged GET of the first block, whose Content-Range gives
the length (the cache keeps the bytes). The file is pinned by a strong
ETag (If-Match), else Last-Modified (If-Unmodified-Since), and its length,
checked on every response: a change is RemoteError::FileChanged, never
mixed data. A server that ignores Range is refused without reading the
body unless a full download is allowed. Connection errors, timeouts,
408/429/5xx and short bodies are retried with exponential backoff;
Accept-Encoding: identity, and an encoded body is refused. read_ranges
fetches its ranges in parallel.
Tests (a std-only HTTP/1.1 server in tests/common/server.rs, also the
range_server example): every fixture read over HTTP gives File::open's
transcript (CLAWHDF5_REMOTE_CORPUS adds the conformance corpus), with
request counts per file with and without the cache; an h5py-written file
against libhdf5's values; a multi-block file fetched in whole blocks, each
once; a server ignoring Range; a file replaced mid-read (ETag,
Last-Modified, length only); truncated bodies and 503s (retried, then an
error, never cached); a slow server with 8 concurrent readers (no block
fetched twice); bad URLs, 404, encoded bodies, non-HDF5 data. The cache
has unit tests for coalescing, splitting, LRU order, large reads,
failures and concurrent in-flight dedup.
ci-test.sh: clawhdf5-remote joins the no-C default-build check, and its
https feature is linted.
Co-Authored-By: Claude Opus 5.5 (1M context) <[email protected]>
49 lines
1.3 KiB
TOML
49 lines
1.3 KiB
TOML
[workspace]
|
|
members = [
|
|
"crates/clawhdf5-format",
|
|
"crates/clawhdf5-io",
|
|
"crates/clawhdf5-filters",
|
|
"crates/clawhdf5-derive",
|
|
"crates/clawhdf5",
|
|
"crates/clawhdf5-netcdf4",
|
|
"crates/clawhdf5-ann",
|
|
"crates/clawhdf5-py",
|
|
"crates/clawhdf5-gpu",
|
|
"crates/clawhdf5-accel",
|
|
"crates/clawhdf5-agent",
|
|
"crates/clawhdf5-migrate",
|
|
"crates/clawhdf5-android",
|
|
"crates/clawhdf5-cli",
|
|
"crates/clawhdf5-napi",
|
|
"crates/clawhdf5-bench",
|
|
"crates/clawhdf5-tools",
|
|
"crates/clawhdf5-wasm",
|
|
"crates/clawhdf5-remote",
|
|
"crates/libaec-sys",
|
|
]
|
|
resolver = "2"
|
|
|
|
[workspace.package]
|
|
version = "2.7.0"
|
|
edition = "2024"
|
|
# Oldest toolchain that builds the whole workspace; CI checks it. wgpu (in
|
|
# clawhdf5-gpu) requires 1.92.
|
|
rust-version = "1.92"
|
|
license = "MIT"
|
|
repository = "https://git.redclaw.dev/quantumclaw/clawhdf5"
|
|
|
|
[workspace.dependencies]
|
|
tempfile = "3"
|
|
criterion = { version = "0.5", features = ["html_reports"] }
|
|
half = "2.7"
|
|
serde = { version = "1", features = ["derive"] }
|
|
|
|
# The browser build of clawhdf5-wasm (examples/wasm-viewer/build.sh): size
|
|
# over speed, whole-program optimisation. Native profiles are unaffected.
|
|
[profile.wasm-release]
|
|
inherits = "release"
|
|
opt-level = "s"
|
|
lto = true
|
|
codegen-units = 1
|
|
panic = "abort"
|