osobhandClaude Opus 5.5
4ff3e40fea
ObjectStoreStorage (feature `object-store`, pure Rust) reads one object of any object_store store by ranged get_opts, pinned at open by a head request: If-Match with its ETag (and the ETag and size of every response compared), else its version or modification time. A change is RemoteError::FileChanged. object_store is async and Storage is not, so the storage owns a small multi-threaded tokio runtime (two workers) and blocks the calling thread on it; the ranges of one read_ranges call are fetched concurrently (up to 8). From inside another tokio runtime it refuses with RemoteError::Usage instead of blocking a worker, and it shuts its runtime down in the background on drop so dropping it in async code does not panic. open_object(store, path, options) opens a file through a block cache (first block prefetched); open_url accepts s3://, gs:// and az:// with the `s3`, `gcs` and `azure` features, configured from the environment by object_store's from_env builders. Those pull object_store's cloud clients and aws-lc-rs (C), so they are opt-in; without them the URL is a clean UnsupportedScheme error naming the feature. Tests against object_store's in-memory and local-file stores (no cloud): every fixture's transcript equals File::open's, a multi-block object is fetched in coalesced block runs, an object replaced while open is an error, and a missing object or a read from inside a runtime is a clean error. ci-test.sh lints all backends, runs these tests (with s3 for its URL parsing test) and checks object-store for C in the no-C step. Co-Authored-By: Claude Opus 5.5 (1M context) <[email protected]>
40 lines
1.8 KiB
TOML
40 lines
1.8 KiB
TOML
[package]
|
|
name = "clawhdf5-remote"
|
|
version = "2.7.0"
|
|
edition = "2024"
|
|
rust-version.workspace = true
|
|
description = "Read HDF5 files over HTTP(S) range requests and object stores (S3, GCS, Azure) with clawhdf5, through a block cache"
|
|
license = "MIT"
|
|
repository = "https://git.redclaw.dev/quantumclaw/clawhdf5"
|
|
readme = "README.md"
|
|
keywords = ["hdf5", "http", "s3", "range-requests", "science"]
|
|
categories = ["science", "network-programming"]
|
|
|
|
[features]
|
|
# Plain-HTTP range reads and the block cache: pure Rust, no TLS stack, no C.
|
|
default = ["http"]
|
|
http = ["dep:ureq"]
|
|
# HTTPS through rustls (ring provider, Mozilla roots). ring compiles C and
|
|
# assembly, so this is not part of the default build.
|
|
https = ["http", "ureq/rustls"]
|
|
# Any object_store backend (in-memory, local files, or one you configure),
|
|
# driven by a small tokio runtime the storage owns. Pure Rust.
|
|
object-store = ["dep:object_store", "dep:tokio", "dep:futures-util"]
|
|
# s3:// gs:// az:// URLs in open_url, credentials from the environment.
|
|
# object_store's cloud clients use aws-lc-rs (C), hence opt-in.
|
|
s3 = ["object-store", "object_store/aws"]
|
|
gcs = ["object-store", "object_store/gcp"]
|
|
azure = ["object-store", "object_store/azure"]
|
|
|
|
[dependencies]
|
|
clawhdf5 = { path = "../clawhdf5", version = "2.7.0" }
|
|
clawhdf5-format = { path = "../clawhdf5-format", version = "2.7.0" }
|
|
ureq = { version = "3.4", optional = true, default-features = false }
|
|
object_store = { version = "0.14", optional = true, default-features = false, features = ["fs"] }
|
|
tokio = { version = "1", optional = true, default-features = false, features = ["rt-multi-thread"] }
|
|
futures-util = { version = "0.3", optional = true, default-features = false, features = ["std"] }
|
|
|
|
[dev-dependencies]
|
|
tempfile = { workspace = true }
|
|
tokio = { version = "1", default-features = false, features = ["rt"] }
|