clawhdf5-remote: block cache and HTTP range reads (open_url)
Range-read milestone M3, first half: a new crate with the block cache the
design makes mandatory for remote files and an HTTP backend, so
open_url("http://...") gives a clawhdf5::File over File::open_storage.
BlockCache wraps any Storage: aligned blocks (1 MiB by default, the size
docs/design/range-reads.md section 2 measured), LRU with a byte budget,
the missing blocks of one read_at/read_ranges fetched with one backend
read_ranges call as runs of consecutive blocks (a one-block gap filled to
merge runs, each request at most 8 MiB), and reads that miss more than
half the budget not kept. Thread-safe without holding the lock across a
fetch: a block being fetched is in flight, a second reader waits for it
instead of fetching it again, and a failed fetch fails its waiters and is
not cached. A backend holding the file in memory passes through.
HttpStorage (ureq, no TLS by default; `https` adds rustls with ring):
opening is one ranged GET of the first block, whose Content-Range gives
the length (the cache keeps the bytes). The file is pinned by a strong
ETag (If-Match), else Last-Modified (If-Unmodified-Since), and its length,
checked on every response: a change is RemoteError::FileChanged, never
mixed data. A server that ignores Range is refused without reading the
body unless a full download is allowed. Connection errors, timeouts,
408/429/5xx and short bodies are retried with exponential backoff;
Accept-Encoding: identity, and an encoded body is refused. read_ranges
fetches its ranges in parallel.
Tests (a std-only HTTP/1.1 server in tests/common/server.rs, also the
range_server example): every fixture read over HTTP gives File::open's
transcript (CLAWHDF5_REMOTE_CORPUS adds the conformance corpus), with
request counts per file with and without the cache; an h5py-written file
against libhdf5's values; a multi-block file fetched in whole blocks, each
once; a server ignoring Range; a file replaced mid-read (ETag,
Last-Modified, length only); truncated bodies and 503s (retried, then an
error, never cached); a slow server with 8 concurrent readers (no block
fetched twice); bad URLs, 404, encoded bodies, non-HDF5 data. The cache
has unit tests for coalescing, splitting, LRU order, large reads,
failures and concurrent in-flight dedup.
ci-test.sh: clawhdf5-remote joins the no-C default-build check, and its
https feature is linted.
Co-Authored-By: Claude Opus 5.5 (1M context) <[email protected]>
This commit is contained in:
osobh
@@ -0,0 +1,302 @@
|
||||
//! Shared by the integration tests: the test server, a transcript of a
|
||||
//! file (tree, attributes, values) to compare two ways of reading it, and
|
||||
//! the test files.
|
||||
|
||||
#![allow(dead_code)]
|
||||
|
||||
pub mod server;
|
||||
|
||||
use std::collections::{BTreeMap, HashSet, VecDeque};
|
||||
use std::fmt::Write as _;
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::process::Command;
|
||||
use std::sync::Arc;
|
||||
|
||||
use clawhdf5::{DType, File, Selection};
|
||||
use clawhdf5_format::error::FormatError;
|
||||
|
||||
/// Objects visited per file.
|
||||
const MAX_OBJECTS: usize = 2000;
|
||||
/// Datasets with more bytes than this are not read (their metadata is).
|
||||
pub const MAX_DATA_BYTES: u64 = 64 << 20;
|
||||
|
||||
/// A short, stable digest of a value's `Debug` form.
|
||||
fn digest<T: std::fmt::Debug>(v: &T) -> String {
|
||||
let s = format!("{v:?}");
|
||||
if s.len() <= 200 {
|
||||
return s;
|
||||
}
|
||||
let mut h = 0xcbf2_9ce4_8422_2325u64;
|
||||
for b in s.bytes() {
|
||||
h = (h ^ u64::from(b)).wrapping_mul(0x100_0000_01b3);
|
||||
}
|
||||
format!("{}…[{} bytes, fnv {h:016x}]", &s[..80], s.len())
|
||||
}
|
||||
|
||||
/// A data read's value, or `Err` (which chunk a damaged dataset reports
|
||||
/// can vary between two `File`s: the chunk cache lists in hash order).
|
||||
fn value<T: std::fmt::Debug, E>(r: &Result<T, E>) -> String {
|
||||
match r {
|
||||
Ok(v) => digest(v),
|
||||
Err(_) => "Err".into(),
|
||||
}
|
||||
}
|
||||
|
||||
fn sorted<V: std::fmt::Debug>(m: std::collections::HashMap<String, V>) -> BTreeMap<String, V> {
|
||||
m.into_iter().collect()
|
||||
}
|
||||
|
||||
/// Everything a reader sees in `file`: every group's entries, every
|
||||
/// object's attributes, and every dataset's shape, types and values.
|
||||
pub fn transcript(file: &File) -> String {
|
||||
let mut out = String::new();
|
||||
let mut seen = HashSet::new();
|
||||
let mut queue = VecDeque::from([(String::from("/"), file.superblock().root_group_address)]);
|
||||
while let Some((path, addr)) = queue.pop_front() {
|
||||
if seen.len() >= MAX_OBJECTS || !seen.insert(addr) {
|
||||
continue;
|
||||
}
|
||||
let group = file.group_at(addr);
|
||||
let entries = group.entries();
|
||||
writeln!(out, "{path} @{addr} entries {}", digest(&entries)).unwrap();
|
||||
if let Ok(ds) = file.dataset_at(addr) {
|
||||
dataset(&mut out, &path, &ds);
|
||||
}
|
||||
let attrs = group.attrs_with_errors().map(|(a, e)| (sorted(a), e));
|
||||
writeln!(out, "{path} attrs {}", digest(&attrs)).unwrap();
|
||||
if let Ok(entries) = entries {
|
||||
for (name, child) in entries {
|
||||
queue.push_back((format!("{}/{name}", path.trim_end_matches('/')), child));
|
||||
}
|
||||
}
|
||||
}
|
||||
out
|
||||
}
|
||||
|
||||
fn dataset(out: &mut String, path: &str, ds: &clawhdf5::Dataset<'_>) {
|
||||
let shape = ds.shape();
|
||||
let dtype = ds.dtype();
|
||||
writeln!(
|
||||
out,
|
||||
"{path} shape {} dtype {} raw {}",
|
||||
digest(&shape),
|
||||
digest(&dtype),
|
||||
digest(&ds.raw_datatype())
|
||||
)
|
||||
.unwrap();
|
||||
let (Ok(shape), Ok(dtype), Ok(raw_dt)) = (shape, dtype, ds.raw_datatype()) else {
|
||||
return;
|
||||
};
|
||||
let elements = shape.iter().try_fold(1u64, |a, &d| a.checked_mul(d));
|
||||
let bytes = elements.and_then(|n| n.checked_mul(u64::from(raw_dt.type_size())));
|
||||
if bytes.is_none_or(|b| b > MAX_DATA_BYTES) {
|
||||
writeln!(out, "{path} too large to read").unwrap();
|
||||
return;
|
||||
}
|
||||
writeln!(
|
||||
out,
|
||||
"{path} all {}",
|
||||
value(&ds.read_selection(&Selection::All))
|
||||
)
|
||||
.unwrap();
|
||||
if matches!(
|
||||
dtype,
|
||||
DType::F32
|
||||
| DType::F64
|
||||
| DType::I8
|
||||
| DType::I16
|
||||
| DType::I32
|
||||
| DType::I64
|
||||
| DType::U8
|
||||
| DType::U16
|
||||
| DType::U32
|
||||
| DType::U64
|
||||
) {
|
||||
writeln!(out, "{path} f64 {}", value(&ds.read_f64())).unwrap();
|
||||
if let Some(&d0) = shape.first() {
|
||||
let rank = shape.len();
|
||||
let sel = Selection::Hyperslab {
|
||||
start: std::iter::once(d0 / 3)
|
||||
.chain(std::iter::repeat_n(0, rank - 1))
|
||||
.collect(),
|
||||
stride: vec![1; rank],
|
||||
count: std::iter::once(d0.div_ceil(3))
|
||||
.chain(shape[1..].iter().copied())
|
||||
.collect(),
|
||||
block: vec![1; rank],
|
||||
};
|
||||
writeln!(
|
||||
out,
|
||||
"{path} f64 third {}",
|
||||
value(&ds.read_f64_selection(&sel))
|
||||
)
|
||||
.unwrap();
|
||||
}
|
||||
}
|
||||
match &raw_dt {
|
||||
clawhdf5_format::datatype::Datatype::String { .. }
|
||||
| clawhdf5_format::datatype::Datatype::VariableLength {
|
||||
is_string: true, ..
|
||||
} => {
|
||||
writeln!(out, "{path} strings {}", value(&ds.read_string_bytes())).unwrap();
|
||||
}
|
||||
clawhdf5_format::datatype::Datatype::VariableLength { .. } => {
|
||||
writeln!(out, "{path} vlen {}", value(&ds.read_vlen::<f64>())).unwrap();
|
||||
}
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
|
||||
/// Open, list every group, and read the first dataset found whose data is
|
||||
/// at most `MAX_DATA_BYTES` (a tree view plus one plot).
|
||||
pub fn list_and_read_one(file: &File) {
|
||||
let mut seen = HashSet::new();
|
||||
let mut read_one = false;
|
||||
let mut queue = VecDeque::from([file.superblock().root_group_address]);
|
||||
while let Some(addr) = queue.pop_front() {
|
||||
if seen.len() >= MAX_OBJECTS || !seen.insert(addr) {
|
||||
continue;
|
||||
}
|
||||
let group = file.group_at(addr);
|
||||
if let Ok(ds) = file.dataset_at(addr) {
|
||||
let _ = (ds.shape(), ds.dtype());
|
||||
if !read_one {
|
||||
let small = ds.shape().ok().and_then(|s| {
|
||||
let n = s.iter().try_fold(1u64, |a, &d| a.checked_mul(d))?;
|
||||
let size = u64::from(ds.raw_datatype().ok()?.type_size());
|
||||
n.checked_mul(size).filter(|&b| b <= MAX_DATA_BYTES)
|
||||
});
|
||||
if small.is_some() {
|
||||
let _ = ds.read_selection(&Selection::All);
|
||||
read_one = true;
|
||||
}
|
||||
}
|
||||
}
|
||||
if let Ok(entries) = group.entries() {
|
||||
queue.extend(entries.into_iter().map(|(_, a)| a));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// External virtual-dataset sources read from `dir`, as `File::open` finds
|
||||
/// them next to the file.
|
||||
pub fn sibling_resolver(dir: PathBuf) -> clawhdf5::VdsResolver {
|
||||
Arc::new(move |name: &str| {
|
||||
let p = Path::new(name);
|
||||
if name.is_empty()
|
||||
|| !p
|
||||
.components()
|
||||
.all(|c| matches!(c, std::path::Component::Normal(_)))
|
||||
{
|
||||
return Err(FormatError::Storage(format!("{name:?} not followed")));
|
||||
}
|
||||
match std::fs::read(dir.join(p)) {
|
||||
Ok(bytes) => Ok(Some(bytes)),
|
||||
Err(e) if e.kind() == std::io::ErrorKind::NotFound => Ok(None),
|
||||
Err(e) => Err(FormatError::Storage(e.to_string())),
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
/// HDF5 files under `dir`, recursively.
|
||||
pub fn hdf5_files(dir: &Path, out: &mut Vec<PathBuf>) {
|
||||
let Ok(entries) = std::fs::read_dir(dir) else {
|
||||
return;
|
||||
};
|
||||
for e in entries.flatten() {
|
||||
let p = e.path();
|
||||
if p.is_dir() {
|
||||
hdf5_files(&p, out);
|
||||
} else if p
|
||||
.extension()
|
||||
.and_then(|x| x.to_str())
|
||||
.is_some_and(|x| matches!(x, "h5" | "hdf5" | "he5" | "nc" | "h5ad" | "hdf"))
|
||||
{
|
||||
out.push(p);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// The repository's HDF5 test fixtures.
|
||||
pub fn fixtures() -> Vec<PathBuf> {
|
||||
let root = Path::new(env!("CARGO_MANIFEST_DIR"));
|
||||
let mut files = Vec::new();
|
||||
hdf5_files(&root.join("../clawhdf5/tests/fixtures"), &mut files);
|
||||
hdf5_files(&root.join("../clawhdf5-format/tests/fixtures"), &mut files);
|
||||
files.sort();
|
||||
files
|
||||
}
|
||||
|
||||
/// Files of `CLAWHDF5_REMOTE_CORPUS` (directories separated like `PATH`).
|
||||
pub fn corpus() -> Option<Vec<PathBuf>> {
|
||||
let dirs = std::env::var("CLAWHDF5_REMOTE_CORPUS").ok()?;
|
||||
let mut files = Vec::new();
|
||||
for d in std::env::split_paths(&dirs) {
|
||||
hdf5_files(&d, &mut files);
|
||||
}
|
||||
files.sort();
|
||||
Some(files)
|
||||
}
|
||||
|
||||
pub fn python() -> String {
|
||||
std::env::var("CLAWHDF5_PYTHON").unwrap_or_else(|_| "python3".to_string())
|
||||
}
|
||||
|
||||
pub fn interop_required() -> bool {
|
||||
std::env::var("CLAWHDF5_REQUIRE_INTEROP").is_ok_and(|v| v == "1")
|
||||
}
|
||||
|
||||
/// Whether python3 with h5py and numpy runs; panics when interop is
|
||||
/// required and it does not.
|
||||
pub fn have_h5py() -> bool {
|
||||
let ok = Command::new(python())
|
||||
.args(["-c", "import h5py, numpy"])
|
||||
.output()
|
||||
.map(|o| o.status.success())
|
||||
.unwrap_or(false);
|
||||
assert!(
|
||||
ok || !interop_required(),
|
||||
"CLAWHDF5_REQUIRE_INTEROP=1 but python3 with h5py is not available"
|
||||
);
|
||||
if !ok {
|
||||
eprintln!("SKIP: python3 with h5py not available");
|
||||
}
|
||||
ok
|
||||
}
|
||||
|
||||
/// Run a Python script; its stdout.
|
||||
pub fn run_python(script: &str, args: &[&str]) -> String {
|
||||
let out = Command::new(python())
|
||||
.arg("-c")
|
||||
.arg(script)
|
||||
.args(args)
|
||||
.output()
|
||||
.expect("failed to run python");
|
||||
assert!(
|
||||
out.status.success(),
|
||||
"python failed:\n{}",
|
||||
String::from_utf8_lossy(&out.stderr)
|
||||
);
|
||||
String::from_utf8(out.stdout).unwrap()
|
||||
}
|
||||
|
||||
/// A clawhdf5-written file with a multi-block chunked dataset (`/big`,
|
||||
/// 1 000 000 f64 in chunks of 10 000, deflated), a contiguous one and a
|
||||
/// group — several blocks of 1 MiB, with no Python needed.
|
||||
pub fn multi_block_file() -> Vec<u8> {
|
||||
let mut b = clawhdf5::FileBuilder::new();
|
||||
b.set_attr("title", clawhdf5::AttrValue::String("remote test".into()));
|
||||
let big: Vec<f64> = (0..1_000_000u64)
|
||||
.map(|i| ((i * 2_654_435_761) % 1_000_003) as f64 * 0.5)
|
||||
.collect();
|
||||
b.create_dataset("big")
|
||||
.with_f64_data(&big)
|
||||
.with_chunks(&[10_000])
|
||||
.with_deflate(1);
|
||||
let flat: Vec<f64> = (0..300_000u64).map(|i| i as f64).collect();
|
||||
b.create_dataset("flat").with_f64_data(&flat);
|
||||
let mut g = b.create_group("grp");
|
||||
g.create_dataset("small").with_f64_data(&[1.0, 2.0, 3.0]);
|
||||
b.add_group(g.finish());
|
||||
b.finish().unwrap()
|
||||
}
|
||||
@@ -0,0 +1,331 @@
|
||||
//! A small HTTP/1.1 file server on 127.0.0.1 for tests and the example:
|
||||
//! `Range: bytes=a-b` requests (206, 416), `HEAD`, keep-alive, strong ETags
|
||||
//! and Last-Modified with `If-Match` / `If-Unmodified-Since` (412), and
|
||||
//! switches to misbehave — ignore ranges (200 with the whole file), cut
|
||||
//! bodies short, answer 503, respond slowly, send no validators. It counts
|
||||
//! requests and body bytes, and logs every range asked for.
|
||||
|
||||
#![allow(dead_code)]
|
||||
|
||||
use std::collections::HashMap;
|
||||
use std::io::{BufRead, BufReader, Write};
|
||||
use std::net::{TcpListener, TcpStream};
|
||||
use std::sync::atomic::{AtomicBool, AtomicU32, AtomicU64, Ordering};
|
||||
use std::sync::{Arc, Mutex, RwLock};
|
||||
use std::time::Duration;
|
||||
|
||||
/// A logged GET: the path and the range asked for (`None`: whole file).
|
||||
pub type LogEntry = (String, Option<(u64, u64)>);
|
||||
|
||||
struct Resource {
|
||||
data: Arc<Vec<u8>>,
|
||||
etag: String,
|
||||
last_modified: String,
|
||||
}
|
||||
|
||||
/// Switches and counters shared with the connection threads.
|
||||
#[derive(Default)]
|
||||
pub struct Shared {
|
||||
files: RwLock<HashMap<String, Resource>>,
|
||||
version: AtomicU64,
|
||||
/// Answer every request with 200 and the whole file.
|
||||
pub ignore_range: AtomicBool,
|
||||
/// Send no ETag and no Last-Modified.
|
||||
pub no_validators: AtomicBool,
|
||||
/// Send a weak ETag only (no Last-Modified).
|
||||
pub weak_etag: AtomicBool,
|
||||
/// Send Last-Modified but no ETag.
|
||||
pub no_etag: AtomicBool,
|
||||
/// Label bodies `Content-Encoding: gzip` (they are not).
|
||||
pub gzip_label: AtomicBool,
|
||||
/// Cut the body of the next N ranged responses in half (then close the
|
||||
/// connection).
|
||||
pub truncate_next: AtomicU32,
|
||||
/// Answer the next N requests with 503.
|
||||
pub fail_next: AtomicU32,
|
||||
/// Sleep this long before answering each request.
|
||||
pub delay_ms: AtomicU64,
|
||||
/// Requests served (every status).
|
||||
pub requests: AtomicU64,
|
||||
/// Body bytes sent.
|
||||
pub bytes: AtomicU64,
|
||||
/// (path, range) of every GET.
|
||||
pub log: Mutex<Vec<LogEntry>>,
|
||||
stop: AtomicBool,
|
||||
}
|
||||
|
||||
/// A running server; stops accepting when dropped.
|
||||
pub struct Server {
|
||||
pub addr: std::net::SocketAddr,
|
||||
pub shared: Arc<Shared>,
|
||||
}
|
||||
|
||||
impl Server {
|
||||
/// Serve `files` (URL path such as `/a.h5` → bytes) on `127.0.0.1`, on
|
||||
/// a free port.
|
||||
pub fn start(files: Vec<(String, Vec<u8>)>) -> Server {
|
||||
Server::bind("127.0.0.1:0", files)
|
||||
}
|
||||
|
||||
/// Serve `files` on `addr`.
|
||||
pub fn bind(addr: &str, files: Vec<(String, Vec<u8>)>) -> Server {
|
||||
let listener = TcpListener::bind(addr).expect("bind");
|
||||
let addr = listener.local_addr().unwrap();
|
||||
let shared = Arc::new(Shared::default());
|
||||
for (path, data) in files {
|
||||
shared.put(&path, data);
|
||||
}
|
||||
let s = shared.clone();
|
||||
std::thread::spawn(move || {
|
||||
for conn in listener.incoming() {
|
||||
if s.stop.load(Ordering::SeqCst) {
|
||||
break;
|
||||
}
|
||||
let Ok(conn) = conn else { continue };
|
||||
let s = s.clone();
|
||||
std::thread::spawn(move || {
|
||||
let _ = serve(conn, &s);
|
||||
});
|
||||
}
|
||||
});
|
||||
Server { addr, shared }
|
||||
}
|
||||
|
||||
/// The URL of `path` (which starts with `/`).
|
||||
pub fn url(&self, path: &str) -> String {
|
||||
format!("http://{}{path}", self.addr)
|
||||
}
|
||||
|
||||
/// Requests served so far.
|
||||
pub fn requests(&self) -> u64 {
|
||||
self.shared.requests.load(Ordering::SeqCst)
|
||||
}
|
||||
|
||||
/// Body bytes sent so far.
|
||||
pub fn bytes(&self) -> u64 {
|
||||
self.shared.bytes.load(Ordering::SeqCst)
|
||||
}
|
||||
|
||||
/// Zero the counters and the log.
|
||||
pub fn reset(&self) {
|
||||
self.shared.requests.store(0, Ordering::SeqCst);
|
||||
self.shared.bytes.store(0, Ordering::SeqCst);
|
||||
self.shared.log.lock().unwrap().clear();
|
||||
}
|
||||
|
||||
/// The ranges asked for so far.
|
||||
pub fn log(&self) -> Vec<LogEntry> {
|
||||
self.shared.log.lock().unwrap().clone()
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for Server {
|
||||
fn drop(&mut self) {
|
||||
self.shared.stop.store(true, Ordering::SeqCst);
|
||||
let _ = TcpStream::connect(self.addr);
|
||||
}
|
||||
}
|
||||
|
||||
impl Shared {
|
||||
/// Add or replace a file (a replacement gets a new ETag and
|
||||
/// Last-Modified).
|
||||
pub fn put(&self, path: &str, data: Vec<u8>) {
|
||||
let v = self.version.fetch_add(1, Ordering::SeqCst) + 1;
|
||||
let secs = 1_700_000_000 + v;
|
||||
self.files.write().unwrap().insert(
|
||||
path.to_string(),
|
||||
Resource {
|
||||
data: Arc::new(data),
|
||||
etag: format!("\"v{v}-{path}\""),
|
||||
last_modified: http_date(secs),
|
||||
},
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
/// An IMF-fixdate for `secs` since the epoch (enough for distinct values).
|
||||
fn http_date(secs: u64) -> String {
|
||||
const DAYS: [&str; 7] = ["Thu", "Fri", "Sat", "Sun", "Mon", "Tue", "Wed"];
|
||||
const MONTHS: [&str; 12] = [
|
||||
"Jan", "Feb", "Mar", "Apr", "May", "Jun", "Jul", "Aug", "Sep", "Oct", "Nov", "Dec",
|
||||
];
|
||||
let days = secs / 86_400;
|
||||
let rem = secs % 86_400;
|
||||
// Civil-from-days (Howard Hinnant).
|
||||
let z = days as i64 + 719_468;
|
||||
let era = z.div_euclid(146_097);
|
||||
let doe = z - era * 146_097;
|
||||
let yoe = (doe - doe / 1460 + doe / 36_524 - doe / 146_096) / 365;
|
||||
let y = yoe + era * 400;
|
||||
let doy = doe - (365 * yoe + yoe / 4 - yoe / 100);
|
||||
let mp = (5 * doy + 2) / 153;
|
||||
let d = doy - (153 * mp + 2) / 5 + 1;
|
||||
let m = if mp < 10 { mp + 3 } else { mp - 9 };
|
||||
let y = if m <= 2 { y + 1 } else { y };
|
||||
format!(
|
||||
"{}, {:02} {} {} {:02}:{:02}:{:02} GMT",
|
||||
DAYS[(days % 7) as usize],
|
||||
d,
|
||||
MONTHS[(m - 1) as usize],
|
||||
y,
|
||||
rem / 3600,
|
||||
rem % 3600 / 60,
|
||||
rem % 60
|
||||
)
|
||||
}
|
||||
|
||||
fn parse_range(v: &str, len: u64) -> Option<Result<(u64, u64), ()>> {
|
||||
let spec = v.trim().strip_prefix("bytes=")?;
|
||||
if spec.contains(',') {
|
||||
return None; // multiple ranges: serve the whole file
|
||||
}
|
||||
let (a, b) = spec.split_once('-')?;
|
||||
let a: u64 = a.trim().parse().ok()?;
|
||||
let b: u64 = match b.trim() {
|
||||
"" => u64::MAX,
|
||||
b => b.parse().ok()?,
|
||||
};
|
||||
if b < a {
|
||||
return None;
|
||||
}
|
||||
if a >= len {
|
||||
return Some(Err(()));
|
||||
}
|
||||
Some(Ok((a, b.min(len - 1))))
|
||||
}
|
||||
|
||||
fn serve(conn: TcpStream, s: &Shared) -> std::io::Result<()> {
|
||||
conn.set_read_timeout(Some(Duration::from_secs(30)))?;
|
||||
// One write per response and no Nagle delay: otherwise every request
|
||||
// waits for a delayed ACK (~40 ms).
|
||||
conn.set_nodelay(true)?;
|
||||
let mut reader = BufReader::new(conn.try_clone()?);
|
||||
let mut out = conn;
|
||||
loop {
|
||||
let mut line = String::new();
|
||||
if reader.read_line(&mut line)? == 0 {
|
||||
return Ok(());
|
||||
}
|
||||
let mut parts = line.split_whitespace();
|
||||
let method = parts.next().unwrap_or("").to_string();
|
||||
let path = parts.next().unwrap_or("").to_string();
|
||||
let mut headers = HashMap::new();
|
||||
loop {
|
||||
let mut h = String::new();
|
||||
if reader.read_line(&mut h)? == 0 {
|
||||
return Ok(());
|
||||
}
|
||||
let h = h.trim_end();
|
||||
if h.is_empty() {
|
||||
break;
|
||||
}
|
||||
if let Some((k, v)) = h.split_once(':') {
|
||||
headers.insert(k.trim().to_ascii_lowercase(), v.trim().to_string());
|
||||
}
|
||||
}
|
||||
s.requests.fetch_add(1, Ordering::SeqCst);
|
||||
let delay = s.delay_ms.load(Ordering::SeqCst);
|
||||
if delay > 0 {
|
||||
std::thread::sleep(Duration::from_millis(delay));
|
||||
}
|
||||
let close = headers
|
||||
.get("connection")
|
||||
.is_some_and(|v| v.eq_ignore_ascii_case("close"));
|
||||
if s.fail_next
|
||||
.fetch_update(Ordering::SeqCst, Ordering::SeqCst, |n| n.checked_sub(1))
|
||||
.is_ok()
|
||||
{
|
||||
write!(
|
||||
out,
|
||||
"HTTP/1.1 503 Service Unavailable\r\nContent-Length: 0\r\n\r\n"
|
||||
)?;
|
||||
continue;
|
||||
}
|
||||
let res = {
|
||||
let files = s.files.read().unwrap();
|
||||
files
|
||||
.get(&path)
|
||||
.map(|r| (r.data.clone(), r.etag.clone(), r.last_modified.clone()))
|
||||
};
|
||||
let Some((data, etag, lm)) = res else {
|
||||
write!(out, "HTTP/1.1 404 Not Found\r\nContent-Length: 0\r\n\r\n")?;
|
||||
continue;
|
||||
};
|
||||
let len = data.len() as u64;
|
||||
let mut validators = String::new();
|
||||
if s.weak_etag.load(Ordering::SeqCst) {
|
||||
validators = format!("ETag: W/{etag}\r\n");
|
||||
} else if s.no_etag.load(Ordering::SeqCst) {
|
||||
validators = format!("Last-Modified: {lm}\r\n");
|
||||
} else if !s.no_validators.load(Ordering::SeqCst) {
|
||||
validators = format!("ETag: {etag}\r\nLast-Modified: {lm}\r\n");
|
||||
}
|
||||
let precondition_failed = headers.get("if-match").is_some_and(|v| v != &etag)
|
||||
|| headers.get("if-unmodified-since").is_some_and(|v| v != &lm);
|
||||
if precondition_failed {
|
||||
write!(
|
||||
out,
|
||||
"HTTP/1.1 412 Precondition Failed\r\n{validators}Content-Length: 0\r\n\r\n"
|
||||
)?;
|
||||
continue;
|
||||
}
|
||||
let range = if s.ignore_range.load(Ordering::SeqCst) {
|
||||
None
|
||||
} else {
|
||||
headers.get("range").and_then(|v| parse_range(v, len))
|
||||
};
|
||||
if method == "GET" {
|
||||
s.log
|
||||
.lock()
|
||||
.unwrap()
|
||||
.push((path.clone(), range.and_then(Result::ok)));
|
||||
}
|
||||
let (status, body, extra) = match range {
|
||||
Some(Err(())) => {
|
||||
write!(
|
||||
out,
|
||||
"HTTP/1.1 416 Range Not Satisfiable\r\nContent-Range: bytes */{len}\r\n\
|
||||
{validators}Content-Length: 0\r\n\r\n"
|
||||
)?;
|
||||
continue;
|
||||
}
|
||||
Some(Ok((a, b))) => (
|
||||
"206 Partial Content",
|
||||
&data[a as usize..=b as usize],
|
||||
format!("Content-Range: bytes {a}-{b}/{len}\r\n"),
|
||||
),
|
||||
None => ("200 OK", &data[..], String::new()),
|
||||
};
|
||||
let extra = if s.gzip_label.load(Ordering::SeqCst) {
|
||||
format!("{extra}Content-Encoding: gzip\r\n")
|
||||
} else {
|
||||
extra
|
||||
};
|
||||
let head = format!(
|
||||
"HTTP/1.1 {status}\r\nContent-Length: {}\r\nAccept-Ranges: bytes\r\n{extra}{validators}\r\n",
|
||||
body.len()
|
||||
);
|
||||
if method == "HEAD" {
|
||||
out.write_all(head.as_bytes())?;
|
||||
continue;
|
||||
}
|
||||
let truncate = status.starts_with("206")
|
||||
&& s.truncate_next
|
||||
.fetch_update(Ordering::SeqCst, Ordering::SeqCst, |n| n.checked_sub(1))
|
||||
.is_ok();
|
||||
let body = if truncate {
|
||||
&body[..body.len() / 2]
|
||||
} else {
|
||||
body
|
||||
};
|
||||
let mut response = head.into_bytes();
|
||||
response.extend_from_slice(body);
|
||||
out.write_all(&response)?;
|
||||
out.flush()?;
|
||||
s.bytes.fetch_add(body.len() as u64, Ordering::SeqCst);
|
||||
if truncate || close {
|
||||
let _ = out.shutdown(std::net::Shutdown::Both);
|
||||
return Ok(());
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,517 @@
|
||||
//! HTTP range reads against a local server (no internet): every value read
|
||||
//! over HTTP equals `File::open`'s, and the server's misbehaviour — no
|
||||
//! range support, a file replaced mid-read, cut-off bodies, errors, slow
|
||||
//! answers under concurrent readers — is an error or the right data, never
|
||||
//! wrong data.
|
||||
//!
|
||||
//! `CLAWHDF5_REMOTE_CORPUS=dir[:dir...]` adds every HDF5 file under those
|
||||
//! directories (the conformance corpus is `conformance/.cache/corpus`), and
|
||||
//! `CLAWHDF5_REMOTE_REPORT=1` prints the per-file request counts.
|
||||
|
||||
#![cfg(feature = "http")]
|
||||
|
||||
mod common;
|
||||
|
||||
use std::sync::Arc;
|
||||
use std::sync::atomic::Ordering;
|
||||
use std::time::Duration;
|
||||
|
||||
use clawhdf5::File;
|
||||
use clawhdf5_remote::{
|
||||
CacheConfig, Error, HttpOptions, HttpStorage, Options, RemoteError, open_url, open_url_with,
|
||||
storage_for_url,
|
||||
};
|
||||
use common::server::Server;
|
||||
use common::{list_and_read_one, multi_block_file, transcript};
|
||||
|
||||
/// Options for tests: fast retries.
|
||||
fn quick() -> Options {
|
||||
Options {
|
||||
http: HttpOptions {
|
||||
backoff: Duration::from_millis(5),
|
||||
..HttpOptions::default()
|
||||
},
|
||||
..Options::default()
|
||||
}
|
||||
}
|
||||
|
||||
fn changed(e: &str) -> bool {
|
||||
e.contains("changed while open")
|
||||
}
|
||||
|
||||
/// Serve `files`, open each by URL and through `File::open`, and require
|
||||
/// identical transcripts. Returns (files compared, files both refused).
|
||||
fn compare(files: &[std::path::PathBuf], report: bool) -> (usize, usize) {
|
||||
let served: Vec<(String, Vec<u8>)> = files
|
||||
.iter()
|
||||
.enumerate()
|
||||
.filter_map(|(i, p)| {
|
||||
let bytes = std::fs::read(p).ok()?;
|
||||
let name = p.file_name()?.to_str()?.replace(' ', "_");
|
||||
Some((format!("/f{i}/{name}"), bytes))
|
||||
})
|
||||
.collect();
|
||||
let server = Server::start(served.clone());
|
||||
let (mut same, mut refused) = (0, 0);
|
||||
let mut totals = [0u64; 5];
|
||||
for (i, p) in files.iter().enumerate() {
|
||||
let Some((url_path, bytes)) = served
|
||||
.iter()
|
||||
.find(|(u, _)| u.starts_with(&format!("/f{i}/")))
|
||||
else {
|
||||
continue;
|
||||
};
|
||||
let url = server.url(url_path);
|
||||
let local = File::open(p);
|
||||
let remote = storage_for_url(&url, &quick())
|
||||
.map_err(|e| e.to_string())
|
||||
.and_then(|s| {
|
||||
File::open_storage(s.clone())
|
||||
.map(|mut f| {
|
||||
f.set_vds_resolver(common::sibling_resolver(p.parent().unwrap().into()));
|
||||
(f, s)
|
||||
})
|
||||
.map_err(|e| e.to_string())
|
||||
});
|
||||
let (local, (remote, storage)) = match (local, remote) {
|
||||
(Ok(l), Ok(r)) => (l, r),
|
||||
(Err(l), Err(r)) => {
|
||||
assert!(
|
||||
!r.contains("HTTP") && !r.contains("network"),
|
||||
"{url}: {r} ({l})"
|
||||
);
|
||||
refused += 1;
|
||||
continue;
|
||||
}
|
||||
(l, r) => panic!(
|
||||
"{}: File::open {:?}, open_url {:?}",
|
||||
p.display(),
|
||||
l.map(|_| ()),
|
||||
r.map(|_| ())
|
||||
),
|
||||
};
|
||||
let want = transcript(&local);
|
||||
let got = transcript(&remote);
|
||||
if want != got {
|
||||
let first = want
|
||||
.lines()
|
||||
.zip(got.lines())
|
||||
.find(|(w, g)| w != g)
|
||||
.map(|(w, g)| format!("\n local: {w}\n remote: {g}"))
|
||||
.unwrap_or_default();
|
||||
panic!("{}: open_url differs from File::open{first}", p.display());
|
||||
}
|
||||
assert!(storage.stats().reads > 0, "read through the cache");
|
||||
same += 1;
|
||||
|
||||
// Cost of "list + read one dataset": with the block cache, and
|
||||
// with none (every read a request).
|
||||
let with = storage_for_url(&url, &quick()).unwrap();
|
||||
server.reset();
|
||||
if let Ok(f) = File::open_storage(with.clone()) {
|
||||
list_and_read_one(&f);
|
||||
}
|
||||
let (cached_requests, cached_bytes) = (server.requests(), server.bytes());
|
||||
let (bare, _) = HttpStorage::open(&url, quick().http).unwrap();
|
||||
let bare = Arc::new(bare);
|
||||
server.reset();
|
||||
if let Ok(f) = File::open_storage(bare.clone()) {
|
||||
list_and_read_one(&f);
|
||||
}
|
||||
let uncached_requests = server.requests();
|
||||
totals[0] += 1 + cached_requests; // + the open probe
|
||||
totals[1] += cached_bytes + with.config().block_size.min(bytes.len() as u64);
|
||||
totals[2] += 1 + uncached_requests;
|
||||
totals[3] += bytes.len() as u64;
|
||||
totals[4] += 1;
|
||||
if report {
|
||||
eprintln!(
|
||||
"requests cached {:>6} uncached {:>8} bytes {:>12} of {:>12} {}",
|
||||
1 + cached_requests,
|
||||
1 + uncached_requests,
|
||||
cached_bytes + with.config().block_size.min(bytes.len() as u64),
|
||||
bytes.len(),
|
||||
p.display()
|
||||
);
|
||||
}
|
||||
// Files within one block: opening fetched everything.
|
||||
if bytes.len() as u64 <= with.config().block_size {
|
||||
assert_eq!(cached_requests, 0, "{}", p.display());
|
||||
}
|
||||
}
|
||||
eprintln!(
|
||||
"list + read one dataset over {} files: {} requests with the 1 MiB block cache \
|
||||
({} bytes transferred, files {} bytes), {} requests without a cache",
|
||||
totals[4], totals[0], totals[1], totals[3], totals[2]
|
||||
);
|
||||
(same, refused)
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn fixtures_read_identically_over_http() {
|
||||
let files = common::fixtures();
|
||||
assert!(files.len() >= 45, "{} fixtures", files.len());
|
||||
let report = std::env::var("CLAWHDF5_REMOTE_REPORT").is_ok_and(|v| v == "1");
|
||||
let (same, _) = compare(&files, report);
|
||||
assert!(same >= 40, "{same}");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn corpus_reads_identically_over_http() {
|
||||
let Some(files) = common::corpus() else {
|
||||
eprintln!("CLAWHDF5_REMOTE_CORPUS not set; skipping the corpus");
|
||||
return;
|
||||
};
|
||||
let report = std::env::var("CLAWHDF5_REMOTE_REPORT").is_ok_and(|v| v == "1");
|
||||
let (same, refused) = compare(&files, report);
|
||||
eprintln!("corpus: {same} files identical, {refused} refused by both");
|
||||
assert!(same > 0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn multi_block_file_values_and_request_budget() {
|
||||
let bytes = multi_block_file();
|
||||
assert!(bytes.len() > 3 << 20, "{}", bytes.len());
|
||||
let server = Server::start(vec![("/m.h5".into(), bytes.clone())]);
|
||||
let url = server.url("/m.h5");
|
||||
let local = File::from_bytes(bytes.clone()).unwrap();
|
||||
let storage = storage_for_url(&url, &quick()).unwrap();
|
||||
assert_eq!(server.requests(), 1, "opening is one request");
|
||||
let remote = File::open_storage(storage.clone()).unwrap();
|
||||
assert!(remote.contiguous_bytes().is_none());
|
||||
let mut names = remote.root().datasets().unwrap();
|
||||
names.sort();
|
||||
assert_eq!(names, ["big", "flat"]);
|
||||
assert_eq!(remote.root().groups().unwrap(), ["grp"]);
|
||||
assert_eq!(
|
||||
remote.dataset("grp/small").unwrap().read_f64().unwrap(),
|
||||
[1.0, 2.0, 3.0]
|
||||
);
|
||||
let big = remote.dataset("big").unwrap().read_f64().unwrap();
|
||||
assert_eq!(big, local.dataset("big").unwrap().read_f64().unwrap());
|
||||
let flat = remote.dataset("flat").unwrap().read_f64().unwrap();
|
||||
assert_eq!(flat.len(), 300_000);
|
||||
assert_eq!(flat[299_999], 299_999.0);
|
||||
// Every byte was fetched at most once, in whole 1 MiB blocks.
|
||||
let log = server.log();
|
||||
let mut blocks: Vec<u64> = Vec::new();
|
||||
for (_, r) in &log {
|
||||
let (a, b) = r.expect("every request is ranged");
|
||||
assert_eq!(a % (1 << 20), 0, "block-aligned");
|
||||
blocks.extend(a >> 20..=b >> 20);
|
||||
}
|
||||
let n = blocks.len();
|
||||
blocks.sort_unstable();
|
||||
blocks.dedup();
|
||||
assert_eq!(n, blocks.len(), "a block was fetched twice: {log:?}");
|
||||
assert!(
|
||||
server.requests() <= (bytes.len() as u64 >> 20) + 2,
|
||||
"{} requests for a {} byte file",
|
||||
server.requests(),
|
||||
bytes.len()
|
||||
);
|
||||
// Reading again costs nothing.
|
||||
let before = server.requests();
|
||||
assert_eq!(transcript(&remote), transcript(&local));
|
||||
assert_eq!(server.requests(), before);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn h5py_written_file_over_http() {
|
||||
if !common::have_h5py() {
|
||||
return;
|
||||
}
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let path = dir.path().join("py.h5");
|
||||
common::run_python(
|
||||
r#"
|
||||
import sys, h5py, numpy as np
|
||||
with h5py.File(sys.argv[1], "w", libver="latest") as f:
|
||||
f.attrs["title"] = "h5py remote test"
|
||||
g = f.create_group("sensors")
|
||||
for i in range(40):
|
||||
d = g.create_dataset(f"s{i:02d}", data=np.arange(1000, dtype="<i4") * (i + 1))
|
||||
d.attrs["index"] = i
|
||||
t = f.create_dataset("temps", data=np.sin(np.arange(2_000_000) / 1000.0),
|
||||
chunks=(50_000,), compression="gzip", shuffle=True)
|
||||
t.attrs["units"] = "K"
|
||||
f.create_dataset("names", data=[b"alpha", b"beta", b"gamma"])
|
||||
f.create_dataset("vl", data=["one", "two", "three"], dtype=h5py.string_dtype())
|
||||
f["link"] = h5py.SoftLink("/sensors/s03")
|
||||
"#,
|
||||
&[path.to_str().unwrap()],
|
||||
);
|
||||
// What libhdf5 reads, for the values the test checks.
|
||||
let sums = common::run_python(
|
||||
r#"
|
||||
import sys, h5py, numpy as np
|
||||
with h5py.File(sys.argv[1], "r") as f:
|
||||
print(repr(float(f["temps"][:].sum())), int(f["sensors/s39"][:].sum()), f["vl"].asstr()[1])
|
||||
"#,
|
||||
&[path.to_str().unwrap()],
|
||||
);
|
||||
let bytes = std::fs::read(&path).unwrap();
|
||||
let server = Server::start(vec![("/py.h5".into(), bytes.clone())]);
|
||||
let remote = open_url_with(&server.url("/py.h5"), &quick()).unwrap();
|
||||
let local = File::open(&path).unwrap();
|
||||
assert_eq!(transcript(&remote), transcript(&local));
|
||||
let temps = remote.dataset("temps").unwrap().read_f64().unwrap();
|
||||
let s39: i64 = remote
|
||||
.dataset("sensors/s39")
|
||||
.unwrap()
|
||||
.read_i64()
|
||||
.unwrap()
|
||||
.iter()
|
||||
.sum();
|
||||
let vl = remote.dataset("vl").unwrap().read_string().unwrap();
|
||||
let mut it = sums.split_whitespace();
|
||||
let want_sum: f64 = it.next().unwrap().parse().unwrap();
|
||||
let got_sum: f64 = temps.iter().sum();
|
||||
assert!((got_sum - want_sum).abs() <= 1e-6 * want_sum.abs().max(1.0));
|
||||
assert_eq!(s39, it.next().unwrap().parse::<i64>().unwrap());
|
||||
assert_eq!(vl[1], it.next().unwrap());
|
||||
assert_eq!(
|
||||
remote.dataset("link").unwrap().read_i32().unwrap(),
|
||||
local.dataset("sensors/s03").unwrap().read_i32().unwrap()
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_server_that_ignores_range_is_refused_or_downloaded_when_allowed() {
|
||||
let bytes = multi_block_file();
|
||||
let server = Server::start(vec![("/m.h5".into(), bytes.clone())]);
|
||||
server.shared.ignore_range.store(true, Ordering::SeqCst);
|
||||
let url = server.url("/m.h5");
|
||||
let err = open_url_with(&url, &quick()).unwrap_err();
|
||||
assert!(
|
||||
matches!(err, Error::Remote(RemoteError::RangeNotSupported(_))),
|
||||
"{err}"
|
||||
);
|
||||
assert!(
|
||||
server.bytes() < bytes.len() as u64,
|
||||
"refusing must not download the file ({} bytes read)",
|
||||
server.bytes()
|
||||
);
|
||||
let mut opts = quick();
|
||||
opts.http.allow_full_download = true;
|
||||
let storage = storage_for_url(&url, &opts).unwrap();
|
||||
let f = File::open_storage(storage.clone()).unwrap();
|
||||
assert_eq!(f.contiguous_bytes(), Some(&bytes[..]), "read from memory");
|
||||
assert_eq!(server.requests(), 2);
|
||||
let local = File::from_bytes(bytes).unwrap();
|
||||
assert_eq!(transcript(&f), transcript(&local));
|
||||
assert_eq!(server.requests(), 2, "no further requests");
|
||||
// Too large for the download limit.
|
||||
opts.http.max_full_download = 1000;
|
||||
assert!(open_url_with(&url, &opts).is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_file_replaced_mid_read_is_an_error_not_mixed_data() {
|
||||
for mode in ["etag", "last-modified", "none"] {
|
||||
let bytes = multi_block_file();
|
||||
let server = Server::start(vec![("/m.h5".into(), bytes.clone())]);
|
||||
match mode {
|
||||
"last-modified" => server.shared.no_etag.store(true, Ordering::SeqCst),
|
||||
"none" => server.shared.no_validators.store(true, Ordering::SeqCst),
|
||||
_ => {}
|
||||
}
|
||||
let url = server.url("/m.h5");
|
||||
let f = open_url_with(&url, &quick()).unwrap();
|
||||
assert_eq!(f.root().groups().unwrap(), ["grp"], "{mode}");
|
||||
// Replace the file: same layout, other values (and, for "none",
|
||||
// one byte longer, which is all that can be checked).
|
||||
let mut other = bytes.clone();
|
||||
let n = other.len();
|
||||
for b in &mut other[n / 2..n / 2 + 1000] {
|
||||
*b ^= 0xff;
|
||||
}
|
||||
if mode == "none" {
|
||||
other.push(0);
|
||||
}
|
||||
server.shared.put("/m.h5", other);
|
||||
let err = f
|
||||
.dataset("big")
|
||||
.unwrap()
|
||||
.read_f64()
|
||||
.unwrap_err()
|
||||
.to_string();
|
||||
assert!(changed(&err), "{mode}: {err}");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_weak_etag_or_no_validator_is_refused_when_required() {
|
||||
let server = Server::start(vec![("/m.h5".into(), multi_block_file())]);
|
||||
server.shared.weak_etag.store(true, Ordering::SeqCst);
|
||||
let mut opts = quick();
|
||||
opts.http.require_validator = true;
|
||||
assert!(open_url_with(&server.url("/m.h5"), &opts).is_err());
|
||||
opts.http.require_validator = false;
|
||||
assert!(open_url_with(&server.url("/m.h5"), &opts).is_ok());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn truncated_bodies_are_retried_then_an_error() {
|
||||
let bytes = multi_block_file();
|
||||
let server = Server::start(vec![("/m.h5".into(), bytes.clone())]);
|
||||
let url = server.url("/m.h5");
|
||||
let local = File::from_bytes(bytes).unwrap();
|
||||
let want = local.dataset("big").unwrap().read_f64().unwrap();
|
||||
|
||||
// One cut-off body: retried, right values.
|
||||
let (http, first) = HttpStorage::open(&url, quick().http).unwrap();
|
||||
assert_eq!(first.len(), 1 << 20);
|
||||
let storage = Arc::new(clawhdf5_remote::BlockCache::new(
|
||||
http,
|
||||
CacheConfig::default(),
|
||||
));
|
||||
let f = File::open_storage(storage.clone()).unwrap();
|
||||
server.shared.truncate_next.store(1, Ordering::SeqCst);
|
||||
assert_eq!(f.dataset("big").unwrap().read_f64().unwrap(), want);
|
||||
assert_eq!(storage.inner().stats().retries, 1);
|
||||
|
||||
// Every body cut off: an error, never partial data.
|
||||
let f = open_url_with(&url, &quick()).unwrap();
|
||||
server.shared.truncate_next.store(1000, Ordering::SeqCst);
|
||||
let err = f
|
||||
.dataset("big")
|
||||
.unwrap()
|
||||
.read_f64()
|
||||
.unwrap_err()
|
||||
.to_string();
|
||||
assert!(
|
||||
err.contains("network error") || err.contains("bad response"),
|
||||
"{err}"
|
||||
);
|
||||
server.shared.truncate_next.store(0, Ordering::SeqCst);
|
||||
// The failure was not cached: the next read succeeds.
|
||||
assert_eq!(f.dataset("big").unwrap().read_f64().unwrap(), want);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn server_errors_are_retried_with_backoff() {
|
||||
let bytes = multi_block_file();
|
||||
let server = Server::start(vec![("/m.h5".into(), bytes.clone())]);
|
||||
server.shared.fail_next.store(2, Ordering::SeqCst);
|
||||
let url = server.url("/m.h5");
|
||||
let f = open_url_with(&url, &quick()).unwrap();
|
||||
let local = File::from_bytes(bytes).unwrap();
|
||||
server.shared.fail_next.store(3, Ordering::SeqCst);
|
||||
assert_eq!(
|
||||
f.dataset("flat").unwrap().read_f64().unwrap(),
|
||||
local.dataset("flat").unwrap().read_f64().unwrap()
|
||||
);
|
||||
// More failures than retries: an error.
|
||||
let f = open_url_with(&url, &quick()).unwrap();
|
||||
server.shared.fail_next.store(100, Ordering::SeqCst);
|
||||
let err = f
|
||||
.dataset("big")
|
||||
.unwrap()
|
||||
.read_f64()
|
||||
.unwrap_err()
|
||||
.to_string();
|
||||
assert!(err.contains("503"), "{err}");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn slow_server_concurrent_readers_fetch_each_block_once() {
|
||||
let bytes = multi_block_file();
|
||||
let server = Server::start(vec![("/m.h5".into(), bytes.clone())]);
|
||||
let url = server.url("/m.h5");
|
||||
let opts = Options {
|
||||
cache: CacheConfig {
|
||||
block_size: 256 << 10,
|
||||
max_request: 256 << 10,
|
||||
..CacheConfig::default()
|
||||
},
|
||||
..quick()
|
||||
};
|
||||
let storage = storage_for_url(&url, &opts).unwrap();
|
||||
let file = File::open_storage(storage.clone()).unwrap();
|
||||
let local = File::from_bytes(bytes).unwrap();
|
||||
let want_big = local.dataset("big").unwrap().read_f64().unwrap();
|
||||
let want_flat = local.dataset("flat").unwrap().read_f64().unwrap();
|
||||
server.shared.delay_ms.store(40, Ordering::SeqCst);
|
||||
server.reset();
|
||||
std::thread::scope(|s| {
|
||||
for t in 0..8 {
|
||||
let (file, want_big, want_flat) = (&file, &want_big, &want_flat);
|
||||
s.spawn(move || {
|
||||
if t % 2 == 0 {
|
||||
assert_eq!(&file.dataset("big").unwrap().read_f64().unwrap(), want_big);
|
||||
} else {
|
||||
assert_eq!(
|
||||
&file.dataset("flat").unwrap().read_f64().unwrap(),
|
||||
want_flat
|
||||
);
|
||||
}
|
||||
});
|
||||
}
|
||||
});
|
||||
let log = server.log();
|
||||
let mut starts: Vec<u64> = log.iter().map(|(_, r)| r.unwrap().0).collect();
|
||||
let n = starts.len();
|
||||
starts.sort_unstable();
|
||||
starts.dedup();
|
||||
assert_eq!(n, starts.len(), "a block was fetched twice");
|
||||
assert!(storage.stats().waits > 0, "readers shared fetches");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn bad_urls_and_statuses_are_clean_errors() {
|
||||
let server = Server::start(vec![("/m.h5".into(), multi_block_file())]);
|
||||
let err = open_url_with(&server.url("/missing.h5"), &quick()).unwrap_err();
|
||||
assert!(
|
||||
matches!(err, Error::Remote(RemoteError::Status { code: 404, .. })),
|
||||
"{err}"
|
||||
);
|
||||
assert!(matches!(
|
||||
open_url("ftp://example.com/a.h5"),
|
||||
Err(Error::Remote(RemoteError::UnsupportedScheme(_)))
|
||||
));
|
||||
assert!(matches!(
|
||||
open_url("no-scheme"),
|
||||
Err(Error::Remote(RemoteError::InvalidUrl(_)))
|
||||
));
|
||||
assert!(matches!(
|
||||
open_url("s3://bucket/key.h5"),
|
||||
Err(Error::Remote(RemoteError::UnsupportedScheme(_)))
|
||||
));
|
||||
#[cfg(not(feature = "https"))]
|
||||
{
|
||||
let e = open_url("https://example.com/a.h5")
|
||||
.unwrap_err()
|
||||
.to_string();
|
||||
assert!(e.contains("`https` feature"), "{e}");
|
||||
}
|
||||
// A content-encoded body is not a byte range.
|
||||
let server = Server::start(vec![("/m.h5".into(), multi_block_file())]);
|
||||
server.shared.gzip_label.store(true, Ordering::SeqCst);
|
||||
let e = open_url(&server.url("/m.h5")).unwrap_err().to_string();
|
||||
assert!(e.contains("gzip-encoded"), "{e}");
|
||||
// Not HDF5.
|
||||
let server = Server::start(vec![("/x.h5".into(), vec![7u8; 5000])]);
|
||||
assert!(matches!(
|
||||
open_url(&server.url("/x.h5")),
|
||||
Err(Error::Hdf5(_))
|
||||
));
|
||||
// A server that closes every connection unanswered: a network error
|
||||
// after the retries. (Not a closed port: another test's server could
|
||||
// take it meanwhile.)
|
||||
let port = {
|
||||
let l = std::net::TcpListener::bind("127.0.0.1:0").unwrap();
|
||||
let port = l.local_addr().unwrap().port();
|
||||
std::thread::spawn(move || {
|
||||
for c in l.incoming() {
|
||||
drop(c);
|
||||
}
|
||||
});
|
||||
port
|
||||
};
|
||||
let err = open_url_with(&format!("http://127.0.0.1:{port}/a.h5"), &quick()).unwrap_err();
|
||||
assert!(
|
||||
matches!(err, Error::Remote(RemoteError::Transport(_))),
|
||||
"{err}"
|
||||
);
|
||||
}
|
||||
Reference in New Issue
Block a user