clawhdf5-remote: request timeouts scale with the body
timeout_global (60 s) covered a whole request, and a request can carry 8 MiB (max_request): below about 140 KB/s every block run timed out, was retried from scratch and failed, so a slow link could not read remote files at all. HttpOptions::timeout (now 30 s) bounds connecting and receiving the response headers; the body gets timeout + its size at the new HttpOptions::min_speed (16 KiB/s by default: 94 s for a 1 MiB block). A slow but moving link is not cut off; a stalled one still fails. (ureq has no idle timeout; its body timeout is a total budget.) The test server can throttle bodies and stall mid-body. Test: a 256 KiB block at 256 KiB/s reads with a 300 ms timeout (it failed before), and a body stalled for 20 s fails in under 5 s. Co-Authored-By: Claude Opus 5.5 (1M context) <[email protected]>
This commit is contained in:
@@ -800,3 +800,37 @@ fn a_200_covering_the_requested_range_is_the_whole_file() {
|
||||
let e = HttpStorage::open(&server.url("/t.h5"), opts).unwrap_err();
|
||||
assert!(matches!(e, RemoteError::RangeNotSupported(_)), "{e}");
|
||||
}
|
||||
|
||||
/// A slow link is not cut off: the body's time budget grows with its size
|
||||
/// (`timeout` + size at `min_speed`), so a 256 KiB block at 256 KiB/s
|
||||
/// (1 s) reads with a 300 ms `timeout`. A stalled body still fails, soon.
|
||||
#[test]
|
||||
fn slow_links_read_and_stalled_ones_fail() {
|
||||
let bytes = multi_block_file();
|
||||
let server = Server::start(vec![("/m.h5".into(), bytes.clone())]);
|
||||
let url = server.url("/m.h5");
|
||||
let mut opts = quick();
|
||||
opts.cache.block_size = 256 << 10;
|
||||
opts.cache.max_request = 256 << 10;
|
||||
opts.http.timeout = Duration::from_millis(300);
|
||||
server
|
||||
.shared
|
||||
.throttle_bps
|
||||
.store(256 << 10, Ordering::SeqCst);
|
||||
let f = open_url_with(&url, &opts).unwrap();
|
||||
assert_eq!(f.root().groups().unwrap(), ["grp"]);
|
||||
assert_eq!(
|
||||
f.dataset("grp/small").unwrap().read_f64().unwrap(),
|
||||
[1.0, 2.0, 3.0]
|
||||
);
|
||||
server.shared.throttle_bps.store(0, Ordering::SeqCst);
|
||||
|
||||
// Stalled mid-body for 20 s: a timeout well before that.
|
||||
server.shared.stall_ms.store(20_000, Ordering::SeqCst);
|
||||
opts.http.retries = 0;
|
||||
opts.http.min_speed = 64 << 20;
|
||||
let t = std::time::Instant::now();
|
||||
let e = open_url_with(&url, &opts).unwrap_err();
|
||||
assert!(matches!(e, Error::Remote(RemoteError::Transport(_))), "{e}");
|
||||
assert!(t.elapsed() < Duration::from_secs(5), "{:?}", t.elapsed());
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user