quantumclaw/clawhdf5
3
1
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 1 Wiki Activity

wasm: openUrl reads remote files by HTTP range requests (range-read M4)

Browse Source 
openUrl(url, opts) returns a RemoteFile with the methods of H5File
(kind, list, info, attrs, attrErrors, read, readHyperslab), each a
promise, and stats(). It runs every call through the restartable
LazyStorage: a pass that misses reports the byte ranges, js/remote.js
fetches them with fetch() and Range headers (six at a time), and the
pass is re-run. This keeps the main thread free without a Worker or
synchronous XHR (h5wasm's lazy files need both), as the design doc
recommends; the cost is re-running a pass per wave of misses.

Every answer is checked: a 206 with exactly the bytes asked for, and
the same ETag/Last-Modified and length as at open, else an error (never
data). A server that ignores Range (200) is downloaded whole, up to
maxDownload (512 MiB), unless fallback: "error". Options: blockSize,
cacheSize, headers, credentials, parallel, fetch.

test/serve.py is a range-capable static server with request counting
(and /norange/ for a server without range support). test.mjs repeats
every fixture check on files opened by URL (1 MiB and 512 B blocks),
checks the request budget on a 200 MB h5py file (list, three small
reads and a window of the big dataset: 5 requests, 6 MiB), the
download fallback, and HTTP errors, changed files and wrong answers;
with CLAWHDF5_WASM_CORPUS every corpus file is compared with open(bytes).

Co-Authored-By: Claude Opus 5.5 (1M context) <[email protected]>
This commit is contained in:
osobh
2026-09-27 06:44:45 -05:00
co-authored by Claude Opus 5.5
parent 076feb089a
commit 5107583b97
8 changed files with 1209 additions and 101 deletions
Download Patch File Download Diff File Expand all files Collapse all files
examples/wasm-viewer/test/make_fixture.py
+37
View File
@@ -14,6 +14,7 @@ encoded as strings so JSON.parse keeps 64-bit values exact.
"""
import json
import os
import sys
import warnings
from pathlib import Path
@@ -201,3 +202,39 @@ def slab_for(obj):
json.dump({"fixture.h5": describe(h5), "fixture.nc": describe(nc)},
open(out / "expected.json", "w"), indent=1, ensure_ascii=False)
def write_big(path, megabytes):
"""A large file for the range-request tests (`openUrl`): `/big`, about
`megabytes` MB of float64 in 1 MiB chunks, written after a small
dataset and a group, so listing and reading `/small` touch a few blocks
of the file and a window of `/big` one chunk. Returns what h5py reads
back."""
n = megabytes * 1_000_000 // 8
chunk = 1 << 17
with h5py.File(path, "w") as f:
f.attrs["note"] = "large file for range reads"
f.create_dataset("small", data=np.array([1.5, -2.0, 3.25]))
g = f.create_group("meta")
g.attrs["units"] = "m"
g.create_dataset("ids", data=np.arange(10, dtype="<i4"))
big = f.create_dataset("big", shape=(n,), dtype="<f8", chunks=(chunk,))
for s in range(0, n, 1 << 22):
e = min(n, s + (1 << 22))
big[s:e] = np.arange(s, e, dtype="<f8") * 0.5
start = n // 2 + 12_345
with h5py.File(path, "r") as f:
return {
"size": path.stat().st_size,
"list": {"groups": ["meta"], "datasets": ["big", "small"]},
"small": [float(x) for x in f["small"][()]],
"ids": [str(int(x)) for x in f["meta/ids"][()]],
"big_shape": list(f["big"].shape),
"window": {"start": start, "count": 10,
"values": [float(x) for x in f["big"][start:start + 10]]},
}
big_mb = int(os.environ.get("WASM_BIG_MB", "0"))
if big_mb > 0:
json.dump(write_big(out / "big.h5", big_mb), open(out / "big.json", "w"), indent=1)
examples/wasm-viewer/test/run.sh
+26 -5
View File
@@ -1,11 +1,17 @@
#!/usr/bin/env bash
# Build the wasm package (../build.sh), test it under Node against files
# written by h5py and netCDF4 (make_fixture.py), then load the viewer page
# in headless Chromium if one is found (browser.sh).
# written by h5py and netCDF4 (make_fixture.py) — opened from bytes, and
# opened by URL from a local range-capable HTTP server (serve.py) — then
# load the viewer page in headless Chromium if one is found (browser.sh).
#
# Needs node, the wasm-bindgen CLI (see ../build.sh) and a Python with h5py,
# netCDF4 and numpy: CLAWHDF5_PYTHON names it (default python3). Without that
# Python the test is skipped, unless CLAWHDF5_REQUIRE_INTEROP=1.
#
# WASM_BIG_MB (default 200) sizes the large file of the range-request
# budget test (0 leaves it out); it is written under TMPDIR.
# CLAWHDF5_WASM_CORPUS=DIR also compares every HDF5 file under DIR (up to
# 16 MiB) read over HTTP with the same file read from bytes.
set -euo pipefail
HERE="$(cd "$(dirname "$0")" && pwd)"
@@ -23,9 +29,24 @@ fi
bash "$HERE/../build.sh"
fix="$(mktemp -d)"
trap 'rm -rf "$fix"' EXIT
"$PY" "$HERE/make_fixture.py" "$fix"
node "$HERE/test.mjs" "$HERE/../pkg" "$fix"
server=""
cleanup() {
[ -n "$server" ] && kill "$server" 2>/dev/null || true
rm -rf "$fix"
}
trap cleanup EXIT
WASM_BIG_MB="${WASM_BIG_MB:-200}" "$PY" "$HERE/make_fixture.py" "$fix"
# The fixtures (and the corpus) over HTTP with range support.
roots=(--root "fix=$fix")
[ -n "${CLAWHDF5_WASM_CORPUS:-}" ] && roots+=(--root "corpus=$CLAWHDF5_WASM_CORPUS")
"$PY" "$HERE/serve.py" "${roots[@]}" > "$fix/port" &
server=$!
for _ in $(seq 50); do
[ -s "$fix/port" ] && break
sleep 0.1
done
node "$HERE/test.mjs" "$HERE/../pkg" "$fix" "http://127.0.0.1:$(head -1 "$fix/port")"
# The page itself, in headless Chromium when one is available.
status=0
examples/wasm-viewer/test/serve.py
+201
View File
@@ -0,0 +1,201 @@
"""A static HTTP server for the wasm tests, with HTTP Range support and
request counting.
python serve.py [--root PREFIX=DIR ...]
Serves each DIR under URL PREFIX (the first match wins; PREFIX "" is the
site root), prints the port on its first line of stdout, and runs until
killed. No symlinks or copies are made: files are read where they are.
- `Range: bytes=a-b`, `bytes=a-` and `bytes=-n` get 206 with Content-Range,
an unsatisfiable range 416; every file answer carries an ETag, and CORS
headers exposing Content-Range, so a page on another origin can use it.
- Under `/norange/...` the same files are served but Range is ignored
(200 with the whole file), as by a server without range support.
- `GET /__stats` returns `{"requests": n, "bytes": n, "log": [...]}` for
file requests since the last `GET /__reset`, which zeroes them.
"""
import argparse
import hashlib
import json
import os
import posixpath
import sys
import threading
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
from urllib.parse import unquote, urlsplit
TYPES = {
".html": "text/html; charset=utf-8",
".js": "text/javascript; charset=utf-8",
".mjs": "text/javascript; charset=utf-8",
".wasm": "application/wasm",
".json": "application/json",
".ts": "text/plain; charset=utf-8",
}
lock = threading.Lock()
stats = {"requests": 0, "bytes": 0, "log": []}
def resolve(roots, path):
"""The file for URL `path`, or None. `..` never leaves a root."""
parts = [p for p in posixpath.normpath(unquote(path)).split("/") if p]
if any(p in (".", "..") for p in parts):
return None
for prefix, root in roots:
pre = [p for p in prefix.split("/") if p]
if parts[: len(pre)] == pre:
rest = parts[len(pre):] or ["index.html"]
f = os.path.join(root, *rest)
if os.path.isfile(f):
return f
return None
def parse_range(header, size):
"""(start, end exclusive) for a single `bytes=` range, "bad" when
unsatisfiable, None when absent or unparsable (served whole)."""
if not header or not header.startswith("bytes=") or "," in header:
return None
a, _, b = header[len("bytes="):].strip().partition("-")
try:
if a == "":
n = int(b)
return (max(0, size - n), size) if n > 0 and size > 0 else "bad"
start = int(a)
end = int(b) + 1 if b else size
except ValueError:
return None
if start >= size or end <= start:
return "bad"
return start, min(end, size)
def make_handler(roots):
class Handler(BaseHTTPRequestHandler):
protocol_version = "HTTP/1.1"
def log_message(self, *args):
pass
def cors(self):
self.send_header("Access-Control-Allow-Origin", "*")
self.send_header("Access-Control-Expose-Headers",
"Content-Range, Content-Length, ETag, Accept-Ranges")
def do_OPTIONS(self):
self.send_response(204)
self.cors()
self.send_header("Access-Control-Allow-Headers", "Range")
self.send_header("Content-Length", "0")
self.end_headers()
def do_HEAD(self):
self.serve(head=True)
def do_GET(self):
self.serve(head=False)
def json(self, obj):
body = json.dumps(obj).encode()
self.send_response(200)
self.cors()
self.send_header("Content-Type", "application/json")
self.send_header("Content-Length", str(len(body)))
self.send_header("Cache-Control", "no-store")
self.end_headers()
self.wfile.write(body)
def serve(self, head):
path = urlsplit(self.path).path
if path == "/__stats":
with lock:
return self.json(stats)
if path == "/__reset":
with lock:
stats.update(requests=0, bytes=0, log=[])
return self.json({})
ranges = True
if path.startswith("/norange/"):
ranges = False
path = path[len("/norange"):]
f = resolve(roots, path)
if f is None:
self.send_response(404)
self.cors()
self.send_header("Content-Length", "0")
self.end_headers()
return
size = os.path.getsize(f)
st = os.stat(f)
etag = '"%s"' % hashlib.sha1(
f"{f}:{size}:{st.st_mtime_ns}".encode()).hexdigest()[:16]
r = parse_range(self.headers.get("Range"), size) if ranges else None
if r == "bad":
self.send_response(416)
self.cors()
self.send_header("Content-Range", f"bytes */{size}")
self.send_header("Content-Length", "0")
self.end_headers()
return
start, end = r if r else (0, size)
self.send_response(206 if r else 200)
self.cors()
ext = os.path.splitext(f)[1]
self.send_header("Content-Type", TYPES.get(ext, "application/octet-stream"))
self.send_header("Content-Length", str(end - start))
self.send_header("ETag", etag)
self.send_header("Cache-Control", "no-store")
if ranges:
self.send_header("Accept-Ranges", "bytes")
if r:
self.send_header("Content-Range", f"bytes {start}-{end - 1}/{size}")
self.end_headers()
if not head:
with lock:
stats["requests"] += 1
stats["bytes"] += end - start
stats["log"].append([path, start, end, 206 if r else 200])
with open(f, "rb") as fh:
fh.seek(start)
left = end - start
try:
while left:
buf = fh.read(min(left, 1 << 20))
if not buf:
break
self.wfile.write(buf)
left -= len(buf)
except (BrokenPipeError, ConnectionResetError):
pass
return Handler
def main():
ap = argparse.ArgumentParser()
ap.add_argument("--root", action="append", default=[],
help="PREFIX=DIR: serve DIR under URL PREFIX")
args = ap.parse_args()
roots = []
for spec in args.root:
prefix, _, d = spec.partition("=")
roots.append((prefix, os.path.abspath(d)))
class Server(ThreadingHTTPServer):
def handle_error(self, request, client_address):
# A client that drops a connection (a cancelled download) is
# not an error of the server.
if not isinstance(sys.exc_info()[1], (ConnectionError, TimeoutError)):
super().handle_error(request, client_address)
httpd = Server(("127.0.0.1", 0), make_handler(roots))
httpd.daemon_threads = True
print(httpd.server_address[1], flush=True)
sys.stdout.close()
httpd.serve_forever()
if __name__ == "__main__":
main()
examples/wasm-viewer/test/test.mjs
+306 -32
View File
@@ -1,20 +1,31 @@
// Node test of the built wasm package (the exact pkg/ the viewer page loads)
// and the viewer's DOM-free helpers. Run by test/run.sh:
// node test.mjs PKG_DIR FIXTURE_DIR
// node test.mjs PKG_DIR FIXTURE_DIR [SERVER_URL]
// FIXTURE_DIR holds fixture.h5, fixture.nc and expected.json from
// make_fixture.py (values as libhdf5 reads them back).
// make_fixture.py (values as libhdf5 reads them back), and big.h5/big.json
// when it was run with WASM_BIG_MB. SERVER_URL is test/serve.py serving
// FIXTURE_DIR under /fix (and $CLAWHDF5_WASM_CORPUS under /corpus): with
// it, every check is repeated on files opened with openUrl (HTTP range
// requests), and the request budget, the full-download fallback and the
// error paths of openUrl are tested.
import assert from "node:assert/strict";
import { readFileSync } from "node:fs";
import { join } from "node:path";
import { existsSync, readFileSync, readdirSync, statSync } from "node:fs";
import { join, relative } from "node:path";
import { pathToFileURL } from "node:url";
const [pkgDir, fixDir] = process.argv.slice(2);
const [pkgDir, fixDir, base] = process.argv.slice(2);
const pkg = await import(pathToFileURL(join(pkgDir, "clawhdf5_wasm.js")));
pkg.initSync({ module: readFileSync(join(pkgDir, "clawhdf5_wasm_bg.wasm")) });
const lib = await import(pathToFileURL(join(import.meta.dirname, "..", "viewer-lib.js")));
let checks = 0;
const eq = (a, b, msg) => { assert.deepEqual(a, b, msg); checks++; };
// A call that must fail: a thrown Error (open) or a rejected promise
// (openUrl), whose message matches `re`.
const fails = async (fn, re, msg) => {
await assert.rejects(async () => fn(), (e) => e instanceof Error && re.test(e.message), msg);
checks++;
};
const ARRAY_TYPES = {
f32: Float32Array, f64: Float64Array, i8: Int8Array, i16: Int16Array, i32: Int32Array,
@@ -54,13 +65,13 @@ function checkAttr(ctx, a, want) {
assert.fail(`${ctx}: unknown expectation ${JSON.stringify(want)}`);
}
const expected = JSON.parse(readFileSync(join(fixDir, "expected.json"), "utf8"));
for (const [name, exp] of Object.entries(expected)) {
const file = pkg.open(new Uint8Array(readFileSync(join(fixDir, name))));
// Every listing, dataset, error and attribute of `file` against what
// libhdf5 reads (expected.json). `file` is an H5File (synchronous methods)
// or a RemoteFile (promises): every call is awaited.
async function checkFile(name, exp, file) {
for (const [path, want] of Object.entries(exp.lists)) {
eq(file.kind(path), "group", `${name}:${path} kind`);
const list = file.list(path);
eq(await file.kind(path), "group", `${name}:${path} kind`);
const list = await file.list(path);
for (const [kind, key] of [["group", "groups"], ["dataset", "datasets"]]) {
eq(list.filter((c) => c.kind === kind).map((c) => c.name).sort(), want[key], `${name}:${path} ${key}`);
}
@@ -68,58 +79,67 @@ for (const [name, exp] of Object.entries(expected)) {
for (const [path, want] of Object.entries(exp.datasets)) {
const ctx = `${name}:${path}`;
eq(file.kind(path), "dataset", `${ctx} kind`);
eq(await file.kind(path), "dataset", `${ctx} kind`);
if (want.unavailable) {
// The wasm build has no zstd (it links C): a clear error, no data.
assert.throws(() => file.read(path), (e) => e.message.includes(want.unavailable), ctx);
checks++;
await fails(() => file.read(path), new RegExp(want.unavailable), ctx);
continue;
}
const info = file.info(path);
const info = await file.info(path);
eq([...info.shape, ...info.elementShape], want.shape, `${ctx} info shape`);
const r = file.read(path);
const r = await file.read(path);
eq(r.shape, want.shape, `${ctx} shape`);
eq(r.dtype, info.dtype, `${ctx} dtype`);
assert.ok(r.data instanceof ARRAY_TYPES[want.kind], `${ctx}: ${r.data.constructor.name} for ${want.kind}`);
eq(values(want.kind, r.data), want.values, ctx);
if (want.slab) {
const s = want.slab;
const part = file.readHyperslab(path, s.start, s.count, s.stride);
const part = await file.readHyperslab(path, s.start, s.count, s.stride);
eq(part.shape, s.shape, `${ctx} slab shape`);
eq(values(want.kind, part.data), s.values, `${ctx} slab`);
}
}
for (const [path, what] of Object.entries(exp.errors)) {
assert.throws(() => file.read(path), (e) => e instanceof Error && e.message.includes(what), `${name}:${path}`);
checks++;
await fails(() => file.read(path), new RegExp(what), `${name}:${path}`);
}
for (const [path, want] of Object.entries(exp.attrs)) {
const attrs = file.attrs(path);
eq(file.attrErrors(path), [], `${name}:${path} attr errors`);
const attrs = await file.attrs(path);
eq(await file.attrErrors(path), [], `${name}:${path} attr errors`);
const seen = attrs.filter((a) => !a.name.startsWith("_") && !exp.skip_attrs.includes(a.name));
eq(seen.map((a) => a.name).sort(), Object.keys(want).sort(), `${name}:${path} attr names`);
for (const a of seen) checkAttr(`${name}:${path}@${a.name}`, a, want[a.name]);
}
}
// The error paths of the reader, for an H5File or a RemoteFile of
// fixture.h5.
async function checkErrors(h5) {
await fails(() => h5.read("/nope"), /./);
await fails(() => h5.list("/grid"), /not a group/);
await fails(() => h5.readHyperslab("/grid", [0], [1]), /dimensions/);
await fails(() => h5.readHyperslab("/grid", [5, 0], [2, 1]), /exceeds/);
await fails(() => h5.readHyperslab("/grid", [-1, 0], [1, 1]), /non-negative integers/);
await fails(() => h5.readHyperslab("/grid", [0.5, 0], [1, 1]), /non-negative integers/);
// Big integers stay exact.
eq((await h5.read("/u64")).data[0], 18446744073709551615n, "u64 max");
}
const expected = JSON.parse(readFileSync(join(fixDir, "expected.json"), "utf8"));
for (const [name, exp] of Object.entries(expected)) {
const file = pkg.open(new Uint8Array(readFileSync(join(fixDir, name))));
await checkFile(name, exp, file);
file.free();
}
// Errors reach JavaScript as thrown Errors, never as data.
const h5 = pkg.open(new Uint8Array(readFileSync(join(fixDir, "fixture.h5"))));
const throwsMsg = (fn, re) => { assert.throws(fn, (e) => e instanceof Error && re.test(e.message)); checks++; };
throwsMsg(() => pkg.open(new Uint8Array(64)), /./);
throwsMsg(() => h5.read("/nope"), /./);
throwsMsg(() => h5.list("/grid"), /not a group/);
throwsMsg(() => h5.readHyperslab("/grid", [0], [1]), /dimensions/);
throwsMsg(() => h5.readHyperslab("/grid", [5, 0], [2, 1]), /exceeds/);
throwsMsg(() => h5.readHyperslab("/grid", [-1, 0], [1, 1]), /non-negative integers/);
throwsMsg(() => h5.readHyperslab("/grid", [0.5, 0], [1, 1]), /non-negative integers/);
await fails(() => pkg.open(new Uint8Array(64)), /./);
await checkErrors(h5);
// Info for a dataset with an unlimited dimension (netCDF "time").
const nc = pkg.open(new Uint8Array(readFileSync(join(fixDir, "fixture.nc"))));
eq(nc.info("/time").maxshape, [null], "unlimited dimension is null");
// Big integers stay exact.
eq(h5.read("/u64").data[0], 18446744073709551615n, "u64 max");
eq(typeof pkg.version(), "string", "version");
// Viewer helpers.
@@ -137,7 +157,261 @@ eq(lib.toRows(pairs.data, 2, 1, lib.perElement([2])), [["[2, 3]"], ["[4, 5]"]],
eq(lib.formatValue(0.1 + 0.2), "0.3", "float formatting");
eq(lib.formatValue(2n ** 64n - 1n), "18446744073709551615", "bigint formatting");
eq(lib.formatValue("x"), '"x"', "string formatting");
eq(lib.formatBytes(0), "0 B", "bytes");
eq(lib.formatBytes(1536), "1.5 KiB", "KiB");
eq(lib.formatBytes(200 * 1024 * 1024), "200 MiB", "MiB");
eq(lib.formatStats({ lazy: true, requests: 3, bytesFetched: 2 << 20, size: 200 << 20 }),
"3 requests, 2 MiB of 200 MiB fetched (1.0%)", "stats line");
eq(lib.formatStats({ lazy: false, requests: 1, bytesFetched: 1024, size: 1024 }),
"downloaded whole (1 KiB): the server does not support range requests", "stats line, no ranges");
h5.free();
nc.free();
console.log(`wasm package: ${checks} checks passed`);
if (base) await remoteTests();
async function serverStats() {
return (await fetch(`${base}/__stats`)).json();
}
async function remoteTests() {
const before = checks;
await fetch(`${base}/__reset`);
// The same checks over HTTP range requests, at the default block size
// and at 512-byte blocks with a 4 KiB cache (almost every structure read
// a miss, evictions between calls).
for (const opts of [undefined, { blockSize: 512, cacheSize: 4096 }]) {
for (const [name, exp] of Object.entries(expected)) {
const f = await pkg.openUrl(`${base}/fix/${name}`, opts);
await checkFile(`${name} (openUrl ${JSON.stringify(opts ?? {})})`, exp, f);
const st = f.stats();
eq(st.lazy, true, "read by ranges");
eq(st.size, statSync(join(fixDir, name)).size, "size");
f.free();
}
}
const remote = await pkg.openUrl(`${base}/fix/fixture.h5`);
await checkErrors(remote);
eq((await (await pkg.openUrl(`${base}/fix/fixture.nc`)).info("/time")).maxshape, [null], "remote unlimited");
// What the page counts is what the server served.
await fetch(`${base}/__reset`);
const counted = await pkg.openUrl(`${base}/fix/fixture.nc`, { blockSize: 1024 });
await counted.read("/temp");
const server = await serverStats();
eq(counted.stats().requests, server.requests, "requests counted");
eq(counted.stats().bytesFetched, server.bytes, "bytes counted");
// A custom fetch is used for every request, with the caller's headers.
let calls = 0;
const seen = new Set();
const viaCustom = await pkg.openUrl(`${base}/fix/fixture.h5`, {
blockSize: 4096,
headers: { "X-Test": "1" },
fetch: (url, init) => { calls++; seen.add(init.headers["X-Test"]); return fetch(url, init); },
});
eq((await viaCustom.read("/sensors/temp")).data[0], 21.5, "custom fetch values");
eq(calls, viaCustom.stats().requests, "custom fetch calls");
eq([...seen], ["1"], "headers passed");
// Calls in flight at once share the cache (a 1 KiB budget: nothing is
// evicted while any of them runs) and each gets its own answer.
const both = await pkg.openUrl(`${base}/fix/fixture.h5`, { blockSize: 512, cacheSize: 1024 });
const [g, t, l, a] = await Promise.all([
both.read("/grid"), both.read("/sensors/temp"), both.list("/sensors"), both.attrs("/"),
]);
const exp = expected["fixture.h5"];
eq(Array.from(g.data), exp.datasets["/grid"].values, "concurrent /grid");
eq(Array.from(t.data), exp.datasets["/sensors/temp"].values, "concurrent /sensors/temp");
eq(l.map((c) => c.name).sort(), [...exp.lists["/sensors"].groups, ...exp.lists["/sensors"].datasets].sort(), "concurrent list");
eq(a.length > 0, true, "concurrent attrs");
assert.ok(both.stats().cachedBytes <= 1024, "trimmed to the budget when idle");
// Listing and reading small things of a large file fetches a few blocks,
// not the file.
if (existsSync(join(fixDir, "big.json"))) {
const big = JSON.parse(readFileSync(join(fixDir, "big.json"), "utf8"));
await fetch(`${base}/__reset`);
const f = await pkg.openUrl(`${base}/fix/big.h5`);
const list = await f.list("/");
eq(list.filter((c) => c.kind === "group").map((c) => c.name), big.list.groups, "big: groups");
eq(list.filter((c) => c.kind === "dataset").map((c) => c.name).sort(), big.list.datasets, "big: datasets");
eq(Array.from((await f.read("/small")).data), big.small, "big: /small");
eq(Array.from((await f.read("/meta/ids")).data, String), big.ids, "big: /meta/ids");
eq((await f.info("/big")).shape, big.big_shape, "big: shape");
eq((await f.attrs("/meta"))[0].value, "m", "big: group attribute");
const win = await f.readHyperslab("/big", [big.window.start], [big.window.count]);
eq(Array.from(win.data), big.window.values, "big: window");
const st = f.stats();
const server = await serverStats();
eq(st.requests, server.requests, "big: requests counted");
eq(st.bytesFetched, server.bytes, "big: bytes counted");
console.log(`big.h5 (${big.size} bytes): listed, 3 small reads and a window in ` +
`${server.requests} requests, ${server.bytes} bytes (${(100 * server.bytes / big.size).toFixed(2)}%)`);
assert.ok(server.requests <= 8, `big: ${server.requests} requests`);
assert.ok(server.bytes * 20 < big.size, `big: ${server.bytes} bytes fetched`);
checks += 2;
}
// A server without range support: downloaded whole (the default), or
// refused.
const whole = await pkg.openUrl(`${base}/norange/fix/fixture.h5`);
await checkFile("fixture.h5 (no range support)", expected["fixture.h5"], whole);
eq(whole.stats().lazy, false, "downloaded whole");
eq(whole.stats().requests, 1, "one request");
await fails(() => pkg.openUrl(`${base}/norange/fix/fixture.h5`, { fallback: "error" }),
/does not support HTTP range requests/, "fallback: error");
await fails(() => pkg.openUrl(`${base}/norange/fix/fixture.h5`, { maxDownload: 1000 }),
/more than maxDownload/, "maxDownload");
// Without a Content-Length the body is streamed, and stopped at the limit.
const undeclared = async (url, init) => {
const r = await fetch(url, init);
return new Response(r.body, { status: r.status });
};
await fails(() => pkg.openUrl(`${base}/norange/fix/fixture.h5`, { maxDownload: 1000, fetch: undeclared }),
/more than maxDownload/, "maxDownload, streamed");
const streamed = await pkg.openUrl(`${base}/norange/fix/fixture.h5`, { fetch: undeclared });
eq(Array.from((await streamed.read("/sensors/temp")).data), [21.5, 22, 22.25], "streamed download");
// Errors: HTTP status, not HDF5, bad options, a file that changes, a
// server that answers with the wrong bytes.
await fails(() => pkg.openUrl(`${base}/fix/missing.h5`), /HTTP 404/, "404");
await fails(() => pkg.openUrl(`${base}/fix/expected.json`), /./, "not HDF5");
await fails(() => pkg.openUrl(`${base}/fix/fixture.h5`, { blockSize: 100 }), /blockSize/, "blockSize");
const tamper = (edit) => async (url, init) => {
const r = await fetch(url, init);
return init.headers.Range === "bytes=0-511" ? r : edit(r);
};
const withHeaders = async (r, headers) => {
const h = new Headers(r.headers);
for (const [k, v] of Object.entries(headers)) h.set(k, v);
return new Response(await r.arrayBuffer(), { status: r.status, headers: h });
};
await fails(async () => {
const f = await pkg.openUrl(`${base}/fix/fixture.h5`, { blockSize: 512, fetch: tamper((r) => withHeaders(r, { ETag: '"other"' })) });
await f.read("/grid");
}, /changed on the server/, "changed file");
await fails(async () => {
const f = await pkg.openUrl(`${base}/fix/fixture.h5`, {
blockSize: 512,
fetch: tamper(async (r) => new Response((await r.arrayBuffer()).slice(1), { status: 206 })),
});
await f.read("/grid");
}, /got \d+/, "short answer");
await fails(async () => {
const f = await pkg.openUrl(`${base}/fix/fixture.h5`, {
blockSize: 512,
fetch: tamper(async (r) => new Response(await r.arrayBuffer(), { status: 200 })),
});
await f.read("/grid");
}, /stopped honouring range requests/, "200 mid-file");
await fails(async () => {
const f = await pkg.openUrl(`${base}/fix/fixture.h5`, {
blockSize: 512,
fetch: tamper(async (r) => withHeaders(r, { "Content-Range": "bytes 0-511/25752" })),
});
await f.read("/grid");
}, /the server sent 0-511/, "wrong range");
await fails(async () => {
const f = await pkg.openUrl(`${base}/fix/fixture.h5`, {
blockSize: 512,
fetch: tamper(async (r) => withHeaders(r, { "Content-Range": "bytes */25752" })),
});
await f.read("/grid");
}, /unusable Content-Range/, "unusable Content-Range");
// Corpus files: what the viewer can show of each is the same read by
// ranges as in memory (an error wherever it gives one).
const corpus = process.env.CLAWHDF5_WASM_CORPUS;
if (corpus) await corpusTests(corpus);
console.log(`openUrl: ${checks - before} checks passed`);
}
function hdf5Files(dir, out) {
for (const e of readdirSync(dir, { withFileTypes: true })) {
const p = join(dir, e.name);
let st;
try {
st = statSync(p);
} catch {
continue;
}
if (st.isDirectory()) hdf5Files(p, out);
else if (st.size <= 16 << 20) {
const b = readFileSync(p);
const sig = [0x89, 0x48, 0x44, 0x46, 0x0d, 0x0a, 0x1a, 0x0a];
for (let at = 0; at + 8 <= b.length; at = at === 0 ? 512 : at * 2) {
if (sig.every((x, i) => b[at + i] === x)) { out.push(p); break; }
}
}
}
return out;
}
function show(v) {
return JSON.stringify(v, (_, x) => {
if (typeof x === "bigint") return `${x}n`;
if (ArrayBuffer.isView(x)) return Array.from(x, (y) => (typeof y === "bigint" ? `${y}n` : Number.isNaN(y) ? "NaN" : y));
return x;
});
}
async function transcript(f) {
const out = [];
const call = async (what, fn) => {
try {
out.push(`${what}: ${show(await fn())}`);
} catch {
out.push(`${what}: Err`);
}
};
const todo = [["/", 0]];
while (todo.length && out.length < 3000) {
const [path, depth] = todo.pop();
let kind = null;
await call(`${path} kind`, async () => (kind = await f.kind(path)));
await call(`${path} attrs`, () => f.attrs(path));
if (kind === "group") {
let list = [];
await call(`${path} list`, async () => (list = await f.list(path)));
if (depth < 12) for (const c of list.reverse()) todo.push([lib.joinPath(path, c.name), depth + 1]);
} else if (kind === "dataset") {
let info = null;
await call(`${path} info`, async () => (info = await f.info(path)));
if (info && [...info.shape, ...info.elementShape].reduce((a, b) => a * b, 1) <= 1 << 20) {
await call(`${path} read`, () => f.read(path));
}
}
}
return out;
}
async function corpusTests(corpus) {
const files = hdf5Files(corpus, []).sort();
let opened = 0;
let bytes = 0;
let size = 0;
for (const p of files) {
const rel = relative(corpus, p).split("/").map(encodeURIComponent).join("/");
let local;
try {
local = pkg.open(new Uint8Array(readFileSync(p)));
} catch {
await fails(() => pkg.openUrl(`${base}/corpus/${rel}`, { blockSize: 65536 }), /./, `${rel}: opens in neither`);
continue;
}
const remote = await pkg.openUrl(`${base}/corpus/${rel}`, { blockSize: 65536 });
const want = await transcript(local);
const got = await transcript(remote);
// Errors are compared as errors: a malformed file can fail at another
// check, with another message, when read by ranges.
eq(got, want, `${rel}: transcript`);
opened++;
bytes += remote.stats().bytesFetched;
size += remote.stats().size;
local.free();
remote.free();
}
console.log(`corpus: ${opened} of ${files.length} files agree over HTTP (${bytes} of ${size} bytes fetched)`);
}