quantumclaw/clawhdf5
3
1
Fork 0
Code Issues Pull Requests 3 Actions Packages Projects Releases 1 Wiki Activity

wasm: sizes a server or a dataset names are errors, not aborts

Browse Source 
A read longer than isize::MAX (2 GiB on wasm32) aborted the module in
LazyStorage::assemble (capacity_overflow), taking every open file on the
page with it, and a hostile server only had to claim a large length and
serve a heap collection of 2 GiB + 4 KiB to get there (after fetching
2 GiB). Reading a large u8 dataset whole aborted the same way when its
values were widened to 64 bits.

- LazyConfig::max_fetch (openUrl option maxFetch, default 512 MiB, at
  most 1 GiB): a read longer than it fails at once, before anything is
  fetched, and an operation whose passes would fetch more than it fails
  before fetching (Operation::charge). assemble reserves fallibly.
- Reader::read refuses a read that would use more than 1 GiB while
  decoding (core::MAX_READ_BYTES: stored bytes + 64-bit values + result)
  with an error naming readHyperslab, before reading.
- openUrl refuses a file of 4 GiB or more at open on wasm32: the format
  code turns offsets into usize, so nothing past 4 GiB can be read there
  (shown by a new test: data at 3 GiB reads, a 4 GiB file is refused).
  maxDownload is bounded to 1 GiB.

Tests: make_fixture.py writes limits.h5 (a sparse 2^28 + 1024 byte u8
dataset), hostile_vl.h5 (the reviewer's collection) and far.h5 (data at
3 GiB); test.mjs (wasm32) and tests/lazy.rs (native) check each is an
error or reads, and that the module survives. Before: RuntimeError:
unreachable in Node; the native test read the huge dataset and fetched
2 GiB.

Co-Authored-By: Claude Opus 5.5 (1M context) <[email protected]>
This commit is contained in:
osobh
2026-09-27 07:36:21 -05:00
co-authored by Claude Opus 5.5
parent f825a89e23
commit dbafa952ac
7 changed files with 470 additions and 26 deletions
Download Patch File Download Diff File Expand all files Collapse all files
examples/wasm-viewer/test/make_fixture.py
+69 -1
View File
@@ -3,7 +3,9 @@ reads back from them, for the clawhdf5-wasm tests.
python make_fixture.py OUT_DIR
writes OUT_DIR/fixture.h5, OUT_DIR/fixture.nc and OUT_DIR/expected.json.
writes OUT_DIR/fixture.h5, OUT_DIR/fixture.nc and OUT_DIR/expected.json,
and the limit-test files OUT_DIR/limits.h5 and OUT_DIR/hostile_vl.h5 (see
write_limits).
Both the Rust test (crates/clawhdf5-wasm/tests/h5py_interop.rs, native) and
the Node test (test.mjs, the built wasm package) compare against the same
expected.json, so the two check the same values.
@@ -204,6 +206,72 @@ json.dump({"fixture.h5": describe(h5), "fixture.nc": describe(nc)},
open(out / "expected.json", "w"), indent=1, ensure_ascii=False)
HUGE_U8 = 2**28 + 1024
# The collection size hostile_vl.h5 claims: past 2 GiB, which a wasm32
# buffer cannot hold.
HOSTILE_GCOL_SIZE = 2**31 + 4096
# The file length a server claims for hostile_vl.h5 (the tests' mock fetch
# answers every range with zeros past the real bytes): 3 GiB, within what
# wasm32 opens, and room for the collection.
HOSTILE_LENGTH = 3 << 30
def write_limits(out):
"""Files for the size limits (the tests must get errors, not aborts):
- limits.h5: /huge_u8, 2^28 + 1024 bytes of u8 in compressed chunks
(a small file): read whole it would take over 2 GiB while decoding;
its last value is 7.
- hostile_vl.h5: a variable-length string dataset /a whose global heap
collection claims HOSTILE_GCOL_SIZE bytes, with the superblock's end
of file set to HOSTILE_LENGTH (libhdf5 cannot read it; it is only
served by a mock that claims that length).
- far.h5 and far.json: /x, 16 float64 values, whose contiguous data
address is moved FAR_SHIFT bytes on (past 2 GiB, the sign bit of a
wasm32 isize) in a file whose end of file is moved as far; the tests'
mock serves the data there, to show offsets up to 4 GiB work on
wasm32.
"""
with h5py.File(out / "limits.h5", "w") as f:
d = f.create_dataset("huge_u8", shape=(HUGE_U8,), dtype="u1",
chunks=(1 << 20,), compression="gzip")
d[-1] = 7
path = out / "hostile_vl.h5"
with h5py.File(path, "w", libver="earliest") as f:
f.create_dataset("a", data=["x", "yy"], dtype=h5py.string_dtype())
b = bytearray(path.read_bytes())
assert b[8] == 0, "a version 0 superblock"
b[40:48] = HOSTILE_LENGTH.to_bytes(8, "little") # end of file address
at = b.index(b"GCOL")
b[at + 8:at + 16] = HOSTILE_GCOL_SIZE.to_bytes(8, "little")
path.write_bytes(bytes(b))
path = out / "far.h5"
values = np.arange(16, dtype="<f8") * 1.5
with h5py.File(path, "w", libver="earliest") as f:
f.create_dataset("x", data=values)
data_at = f["x"].id.get_offset()
b = bytearray(path.read_bytes())
# The layout message: the data's address, then its size.
old = data_at.to_bytes(8, "little") + (values.nbytes).to_bytes(8, "little")
assert b.count(old) == 1
at = b.index(old)
b[at:at + 8] = (data_at + FAR_SHIFT).to_bytes(8, "little")
length = len(b) + FAR_SHIFT
b[40:48] = length.to_bytes(8, "little")
path.write_bytes(bytes(b))
json.dump({"data_at": data_at, "far_at": data_at + FAR_SHIFT,
"nbytes": values.nbytes, "length": length,
"values": [float(x) for x in values]},
open(out / "far.json", "w"))
# far.h5's data moves this far: past 2 GiB, below 4 GiB.
FAR_SHIFT = 3 << 30
write_limits(out)
def write_big(path, megabytes):
"""A large file for the range-request tests (`openUrl`): `/big`, about
`megabytes` MB of float64 in 1 MiB chunks, written after a small
examples/wasm-viewer/test/test.mjs
+83
View File
@@ -321,6 +321,8 @@ async function remoteTests() {
await f.read("/grid");
}, /unusable Content-Range/, "unusable Content-Range");
await limitTests();
// Corpus files: what the viewer can show of each is the same read by
// ranges as in memory (an error wherever it gives one).
const corpus = process.env.CLAWHDF5_WASM_CORPUS;
@@ -328,6 +330,87 @@ async function remoteTests() {
console.log(`openUrl: ${checks - before} checks passed`);
}
// A fetch that serves `buf` as a file of `total` bytes (zeros past the end
// of `buf`, and `extra` = [[offset, bytes], ...] laid over them), counting
// its calls. `hide` leaves out Content-Range and the validators, as a
// cross-origin server that exposes neither does; HEAD then gives the length.
function mockFetch(buf, { total = buf.length, extra = [], hide = false } = {}) {
const f = async (url, init) => {
f.calls++;
if (init.method === "HEAD") {
return new Response(null, { status: 200, headers: { "Content-Length": String(total) } });
}
const m = /^bytes=(\d+)-(\d+)$/.exec(new Headers(init.headers).get("Range"));
const a = Number(m[1]);
const b = Math.min(Number(m[2]) + 1, total);
const out = new Uint8Array(b - a);
for (const [at, bytes] of [[0, buf], ...extra]) {
const from = Math.max(a, at);
const to = Math.min(b, at + bytes.length);
if (from < to) out.set(bytes.subarray(from - at, to - at), from - a);
}
const headers = { "Content-Length": String(b - a) };
if (!hide) headers["Content-Range"] = `bytes ${a}-${b - 1}/${total}`;
return new Response(out, { status: 206, headers });
};
f.calls = 0;
return f;
}
// Sizes a hostile server or a large dataset can name are errors, never an
// allocation that aborts the module (which would take every open file on
// the page with it); see write_limits in make_fixture.py.
async function limitTests() {
// Read whole, /huge_u8 (2^28 + 1024 bytes) would take over 2 GiB while
// decoding: refused before its chunks are fetched; a window reads.
const n = 2 ** 28 + 1024;
const limits = readFileSync(join(fixDir, "limits.h5"));
const local = pkg.open(new Uint8Array(limits));
await fails(() => local.read("/huge_u8"), /readHyperslab/, "huge read, in memory");
const remote = await pkg.openUrl(`${base}/fix/limits.h5`);
const before = remote.stats().requests;
await fails(() => remote.read("/huge_u8"), /readHyperslab/, "huge read, openUrl");
eq(remote.stats().requests, before, "huge read fetched nothing");
eq(Array.from((await remote.readHyperslab("/huge_u8", [n - 4], [4])).data), [0, 0, 0, 7], "huge window");
eq(Array.from(local.readHyperslab("/huge_u8", [n - 4], [4]).data), [0, 0, 0, 7], "huge window, in memory");
local.free();
// A server claiming 3 GiB, a heap collection claiming 2 GiB + 4 KiB: an
// error after a few requests (it used to fetch 2 GiB, then abort).
const hostile = mockFetch(new Uint8Array(readFileSync(join(fixDir, "hostile_vl.h5"))), { total: 3 * 2 ** 30 });
const h = await pkg.openUrl("http://hostile.invalid/h.h5", { fetch: hostile });
await fails(() => h.read("/a"), /maxFetch/, "hostile collection size");
assert.ok(hostile.calls <= 4, `hostile: ${hostile.calls} requests`);
// The module survived: files open and read.
eq((await (await pkg.openUrl(`${base}/fix/fixture.h5`)).read("/sensors/temp")).data[0], 21.5, "alive after hostile");
// A call that would fetch more than maxFetch fails before fetching it.
await fails(async () => {
const f = await pkg.openUrl(`${base}/fix/fixture.h5`, { blockSize: 512, maxFetch: 1024 });
await f.read("/grid");
}, /maxFetch/, "maxFetch");
await fails(() => pkg.openUrl(`${base}/fix/fixture.h5`, { maxFetch: 2 ** 31 }), /maxFetch/, "maxFetch range");
await fails(() => pkg.openUrl(`${base}/fix/fixture.h5`, { maxDownload: 2 ** 31 }), /maxDownload/, "maxDownload range");
// Offsets past 2 GiB work on wasm32: far.h5's data sits at 3 GiB.
const far = JSON.parse(readFileSync(join(fixDir, "far.json"), "utf8"));
const farBytes = new Uint8Array(readFileSync(join(fixDir, "far.h5")));
const farData = farBytes.subarray(far.data_at, far.data_at + far.nbytes);
const ff = await pkg.openUrl("http://far.invalid/far.h5", {
fetch: mockFetch(farBytes, { total: far.length, extra: [[far.far_at, farData]] }),
});
eq(Array.from((await ff.read("/x")).data), far.values, "data past 2 GiB");
eq(ff.stats().size, far.length, "size past 2 GiB");
// wasm32's reader holds file offsets in 32 bits: a file of 4 GiB or more
// is refused at open, with the limit in the message (past 2^53 - 1 bytes
// a JavaScript number is not even exact).
for (const total of [2 ** 32, 2 ** 40, 2 ** 53 + 2]) {
await fails(() => pkg.openUrl("http://huge.invalid/x.h5", { fetch: mockFetch(farBytes, { total }) }),
total > 2 ** 53 ? /2\^53 - 1/ : /4 GiB/, `length ${total}`);
}
}
function hdf5Files(dir, out) {
for (const e of readdirSync(dir, { withFileTypes: true })) {
const p = join(dir, e.name);