quantumclaw/clawhdf5
3
1
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 1 Wiki Activity

wasm: openUrl takes Headers, aborts sibling requests on failure, checks parallel

Browse Source 
- opts.headers is read as fetch reads it (a Headers, [name, value] pairs
  or a plain object); it was spread as an object, which silently dropped
  a Headers instance (a common way to pass Authorization). A caller's
  Range is not sent.
- When one range request of a batch fails, the others in flight are
  aborted (one AbortController per batch, its signal passed to fetch)
  and no new ones start; the first failure is the error. The workers
  used to go on issuing requests nobody waited for.
- parallel must be an integer from 1 to 1024 (openUrl) or a positive
  integer (fetchRanges): a non-number gave NaN workers, so none ran and
  fetchRanges returned nothing.

Tests (test.mjs): headers as object, Headers and pairs reach the fetch;
bad parallel values are option errors; fetchRanges with a 500 on the
third of 20 ranges at parallel 3 starts 3 requests and aborts the 2 in
flight. Before: the Headers case sent no header, 17 requests started
after the failure with none aborted, and parallel "x" returned.

Co-Authored-By: Claude Opus 5.5 (1M context) <[email protected]>
This commit is contained in:
osobh
2026-09-27 07:53:22 -05:00
co-authored by Claude Opus 5.5
parent 581c6ddef8
commit 2379e6163e
3 changed files with 127 additions and 44 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/clawhdf5-wasm/js/remote.js
+66 -31
View File
@@ -17,8 +17,28 @@ function fetcher(opts) {
return f;
}
function init(opts, extra, method = "GET") {
return { method, headers: { ...(opts?.headers ?? {}), ...extra }, credentials: opts?.credentials };
// The caller's headers (`opts.headers`: a Headers, [name, value] pairs or a
// plain object, as fetch takes them; names come out in lower case) with
// `extra` over them. A Range of the caller's is dropped: this file asks for
// the ranges.
function init(opts, extra, method = "GET", signal = undefined) {
const headers = {};
if (opts?.headers != null) {
for (const [name, value] of new Headers(opts.headers)) {
if (name !== "range") headers[name] = value;
}
}
Object.assign(headers, extra);
return { method, headers, credentials: opts?.credentials, signal };
}
// `opts.parallel`: range requests in flight at once.
function parallelism(opts) {
const p = opts?.parallel ?? DEFAULT_PARALLEL;
if (!Number.isSafeInteger(p) || p < 1) {
throw new Error(`openUrl: parallel must be a positive integer, got ${String(p)}`);
}
return p;
}
// "bytes a-b/total" -> { start, end (exclusive), total | null }; null when
@@ -108,6 +128,7 @@ function readLimited(resp, limit, url, start) {
*/
export async function probe(url, firstLen, opts) {
const f = fetcher(opts);
parallelism(opts);
const resp = await f(url, init(opts, { Range: `bytes=0-${firstLen - 1}` }));
if (resp.status === 206) {
const cr = contentRange(resp, url);
@@ -157,44 +178,58 @@ export async function probe(url, firstLen, opts) {
* Fetch `ranges` ([start0, end0, start1, end1, ...], ends exclusive) of a
* file opened by `probe`, at most `opts.parallel` (default 6) at a time.
* Every answer must be a 206 with exactly the bytes asked for, from the same
* file (validator and length).
* file (validator and length). When one request fails, the others in
* flight are aborted and no more are made; that failure is the error.
*/
export async function fetchRanges(url, ranges, opts, validator, length) {
const f = fetcher(opts);
const parallel = parallelism(opts);
const n = ranges.length / 2;
const out = new Array(n);
const abort = new AbortController();
let next = 0;
async function one(i) {
const start = ranges[2 * i];
const end = ranges[2 * i + 1];
const resp = await f(url, init(opts, { Range: `bytes=${start}-${end - 1}` }, "GET", abort.signal));
if (resp.status !== 206) {
await discard(resp);
throw new Error(resp.status === 200
? `${url}: the server stopped honouring range requests`
: `${url}: HTTP ${resp.status} ${resp.statusText ?? ""}`.trim());
}
const cr = contentRange(resp, url);
const v = validatorOf(resp);
if ((validator != null && v !== null && v !== validator) ||
(cr?.total != null && cr.total !== length)) {
await discard(resp);
throw new Error(`${url} changed on the server since it was opened`);
}
if (cr && (cr.start !== start || cr.end !== end)) {
await discard(resp);
throw new Error(`${url}: asked for bytes ${start}-${end - 1}, the server sent ${cr.start}-${cr.end - 1}`);
}
const body = await readLimited(resp, end - start, url, start);
if (body.length !== end - start) {
throw new Error(`${url}: asked for ${end - start} bytes at offset ${start}, got ${body.length}`);
}
out[i] = body;
}
async function worker() {
while (next < n) {
const i = next++;
const start = ranges[2 * i];
const end = ranges[2 * i + 1];
const resp = await f(url, init(opts, { Range: `bytes=${start}-${end - 1}` }));
if (resp.status !== 206) {
await discard(resp);
throw new Error(resp.status === 200
? `${url}: the server stopped honouring range requests`
: `${url}: HTTP ${resp.status} ${resp.statusText ?? ""}`.trim());
while (next < n && !abort.signal.aborted) {
try {
await one(next++);
} catch (e) {
// The first failure stops the rest: requests in flight are aborted
// (their AbortErrors are not reported) and no new ones start.
if (!abort.signal.aborted) {
abort.abort();
throw e;
}
return;
}
const cr = contentRange(resp, url);
const v = validatorOf(resp);
if ((validator !== null && v !== null && v !== validator) ||
(cr?.total != null && cr.total !== length)) {
await discard(resp);
throw new Error(`${url} changed on the server since it was opened`);
}
if (cr && (cr.start !== start || cr.end !== end)) {
await discard(resp);
throw new Error(`${url}: asked for bytes ${start}-${end - 1}, the server sent ${cr.start}-${cr.end - 1}`);
}
const body = await readLimited(resp, end - start, url, start);
if (body.length !== end - start) {
throw new Error(`${url}: asked for ${end - start} bytes at offset ${start}, got ${body.length}`);
}
out[i] = body;
}
}
const workers = Math.max(1, Math.min(opts?.parallel ?? DEFAULT_PARALLEL, n));
await Promise.all(Array.from({ length: workers }, worker));
await Promise.all(Array.from({ length: Math.min(parallel, n) }, worker));
return out;
}
crates/clawhdf5-wasm/src/lib.rs
+6 -2
View File
@@ -461,6 +461,8 @@ fn config_from(opts: &JsValue) -> Result<LazyConfig, JsError> {
// Read by remote.js; checked here so a value wasm32 cannot hold is an
// option error rather than a download that cannot be kept.
int_opt(opts, "maxDownload", 0.0, MAX_FETCH_LIMIT as f64)?;
// Also read by remote.js (which checks it too, for direct callers).
int_opt(opts, "parallel", 1.0, 1024.0)?;
Ok(c)
}
@@ -477,8 +479,10 @@ fn config_from(opts: &JsValue) -> Result<LazyConfig, JsError> {
/// - `fallback` — `"download"` (default) reads the whole file when the
/// server ignores `Range` (answers 200), up to `maxDownload` bytes
/// (default 512 MiB, at most 1 GiB); `"error"` refuses such a server;
/// - `headers`, `credentials` — passed to every `fetch`;
/// - `parallel` — range requests in flight at once (default 6);
/// - `headers`, `credentials` — passed to every `fetch` (`headers` as
/// `fetch` takes them: a `Headers`, `[name, value]` pairs or an object);
/// - `parallel` — range requests in flight at once, 1 to 1024 (default
/// 6); when one fails the others are aborted;
/// - `fetch` — a `fetch`-compatible function to use instead of the global.
///
/// Cross-origin servers must allow CORS and expose `Content-Range` (or