clawhdf5/examples/wasm-viewer/test/test.mjs

// Node test of the built wasm package (the exact pkg/ the viewer page loads)
// and the viewer's DOM-free helpers. Run by test/run.sh:
//   node test.mjs PKG_DIR FIXTURE_DIR [SERVER_URL]
// FIXTURE_DIR holds fixture.h5, fixture.nc and expected.json from
// make_fixture.py (values as libhdf5 reads them back), and big.h5/big.json
// when it was run with WASM_BIG_MB. SERVER_URL is test/serve.py serving
// FIXTURE_DIR under /fix (and $CLAWHDF5_WASM_CORPUS under /corpus): with
// it, every check is repeated on files opened with openUrl (HTTP range
// requests), and the request budget, the full-download fallback and the
// error paths of openUrl are tested.
import assert from "node:assert/strict";
import { existsSync, readFileSync, readdirSync, statSync } from "node:fs";
import { join, relative } from "node:path";
import { pathToFileURL } from "node:url";

const [pkgDir, fixDir, base] = process.argv.slice(2);
const pkg = await import(pathToFileURL(join(pkgDir, "clawhdf5_wasm.js")));
pkg.initSync({ module: readFileSync(join(pkgDir, "clawhdf5_wasm_bg.wasm")) });
const lib = await import(pathToFileURL(join(import.meta.dirname, "..", "viewer-lib.js")));

let checks = 0;
const eq = (a, b, msg) => { assert.deepEqual(a, b, msg); checks++; };
// A call that must fail: a thrown Error (open) or a rejected promise
// (openUrl), whose message matches `re`.
const fails = async (fn, re, msg) => {
  await assert.rejects(async () => fn(), (e) => e instanceof Error && re.test(e.message), msg);
  checks++;
};

const ARRAY_TYPES = {
  f32: Float32Array, f64: Float64Array, i8: Int8Array, i16: Int16Array, i32: Int32Array,
  i64: BigInt64Array, u8: Uint8Array, u16: Uint16Array, u32: Uint32Array, u64: BigUint64Array,
  strings: Array,
};

function values(kind, data) {
  const arr = Array.from(data);
  if (kind === "f32" || kind === "f64" || kind === "strings") return arr;
  return arr.map(String);
}

function checkAttr(ctx, a, want) {
  const v = a.value;
  if ("raw" in want) {
    eq(v, null, ctx);
    assert.ok(a.dtype.includes(want.raw), `${ctx}: ${a.dtype}`);
    return;
  }
  eq(a.dtype, null, `${ctx} dtype`);
  if ("string" in want) return eq(v, want.string, ctx);
  if ("strings" in want) return eq(v, want.strings, ctx);
  if ("int" in want) {
    if (want.scalar) {
      assert.ok(typeof v === "number" || typeof v === "bigint", ctx);
      return eq([String(v)], want.int, ctx);
    }
    assert.ok(v instanceof BigInt64Array || v instanceof BigUint64Array, ctx);
    return eq(Array.from(v, String), want.int, ctx);
  }
  if ("float" in want) {
    if (want.scalar) return eq([v], want.float, ctx);
    assert.ok(v instanceof Float64Array, ctx);
    return eq(Array.from(v), want.float, ctx);
  }
  assert.fail(`${ctx}: unknown expectation ${JSON.stringify(want)}`);
}

// Every listing, dataset, error and attribute of `file` against what
// libhdf5 reads (expected.json). `file` is an H5File (synchronous methods)
// or a RemoteFile (promises): every call is awaited.
async function checkFile(name, exp, file) {
  for (const [path, want] of Object.entries(exp.lists)) {
    eq(await file.kind(path), "group", `${name}:${path} kind`);
    const list = await file.list(path);
    for (const [kind, key] of [["group", "groups"], ["dataset", "datasets"]]) {
      eq(list.filter((c) => c.kind === kind).map((c) => c.name).sort(), want[key], `${name}:${path} ${key}`);
    }
  }

  for (const [path, want] of Object.entries(exp.datasets)) {
    const ctx = `${name}:${path}`;
    eq(await file.kind(path), "dataset", `${ctx} kind`);
    if (want.unavailable) {
      // The wasm build has no zstd (it links C): a clear error, no data.
      await fails(() => file.read(path), new RegExp(want.unavailable), ctx);
      continue;
    }
    const info = await file.info(path);
    eq([...info.shape, ...info.elementShape], want.shape, `${ctx} info shape`);
    const r = await file.read(path);
    eq(r.shape, want.shape, `${ctx} shape`);
    eq(r.dtype, info.dtype, `${ctx} dtype`);
    assert.ok(r.data instanceof ARRAY_TYPES[want.kind], `${ctx}: ${r.data.constructor.name} for ${want.kind}`);
    eq(values(want.kind, r.data), want.values, ctx);
    if (want.slab) {
      const s = want.slab;
      const part = await file.readHyperslab(path, s.start, s.count, s.stride);
      eq(part.shape, s.shape, `${ctx} slab shape`);
      eq(values(want.kind, part.data), s.values, `${ctx} slab`);
    }
  }

  for (const [path, what] of Object.entries(exp.errors)) {
    await fails(() => file.read(path), new RegExp(what), `${name}:${path}`);
  }

  for (const [path, want] of Object.entries(exp.attrs)) {
    const attrs = await file.attrs(path);
    eq(await file.attrErrors(path), [], `${name}:${path} attr errors`);
    const seen = attrs.filter((a) => !a.name.startsWith("_") && !exp.skip_attrs.includes(a.name));
    eq(seen.map((a) => a.name).sort(), Object.keys(want).sort(), `${name}:${path} attr names`);
    for (const a of seen) checkAttr(`${name}:${path}@${a.name}`, a, want[a.name]);
  }
}

// The error paths of the reader, for an H5File or a RemoteFile of
// fixture.h5.
async function checkErrors(h5) {
  await fails(() => h5.read("/nope"), /./);
  await fails(() => h5.list("/grid"), /not a group/);
  await fails(() => h5.readHyperslab("/grid", [0], [1]), /dimensions/);
  await fails(() => h5.readHyperslab("/grid", [5, 0], [2, 1]), /exceeds/);
  await fails(() => h5.readHyperslab("/grid", [-1, 0], [1, 1]), /non-negative integers/);
  await fails(() => h5.readHyperslab("/grid", [0.5, 0], [1, 1]), /non-negative integers/);
  // Big integers stay exact.
  eq((await h5.read("/u64")).data[0], 18446744073709551615n, "u64 max");
}

const expected = JSON.parse(readFileSync(join(fixDir, "expected.json"), "utf8"));
for (const [name, exp] of Object.entries(expected)) {
  const file = pkg.open(new Uint8Array(readFileSync(join(fixDir, name))));
  await checkFile(name, exp, file);
  file.free();
}

// Errors reach JavaScript as thrown Errors, never as data.
const h5 = pkg.open(new Uint8Array(readFileSync(join(fixDir, "fixture.h5"))));
await fails(() => pkg.open(new Uint8Array(64)), /./);
await checkErrors(h5);
// Info for a dataset with an unlimited dimension (netCDF "time").
const nc = pkg.open(new Uint8Array(readFileSync(join(fixDir, "fixture.nc"))));
eq(nc.info("/time").maxshape, [null], "unlimited dimension is null");
eq(typeof pkg.version(), "string", "version");

// Viewer helpers.
eq(lib.joinPath("/", "a"), "/a", "joinPath root");
eq(lib.joinPath("/a", "b"), "/a/b", "joinPath nested");
eq(lib.viewWindow([], {}), null, "scalar window");
eq(lib.viewWindow([7], { row: 5, rows: 50 }), { start: [5], count: [2], rows: 2, cols: 1, row: 5, col: 0 }, "1-D window");
const w = lib.viewWindow([2, 5, 6], { row: 1, col: 4, rows: 3, cols: 5, fixed: [1] });
eq(w, { start: [1, 1, 4], count: [1, 3, 2], rows: 3, cols: 2, row: 1, col: 4 }, "3-D window");
// The window the page would request reads the same values as a direct slab.
const cube = h5.readHyperslab("/cube", w.start, w.count);
eq(lib.toRows(cube.data, w.rows, w.cols), [["40", "41"], ["46", "47"], ["52", "53"]], "cube window cells");
const pairs = h5.readHyperslab("/pairs", [1], [2]);
eq(lib.toRows(pairs.data, 2, 1, lib.perElement([2])), [["[2, 3]"], ["[4, 5]"]], "array-type cells");
eq(lib.formatValue(0.1 + 0.2), "0.3", "float formatting");
eq(lib.formatValue(2n ** 64n - 1n), "18446744073709551615", "bigint formatting");
eq(lib.formatValue("x"), '"x"', "string formatting");
eq(lib.formatBytes(0), "0 B", "bytes");
eq(lib.formatBytes(1536), "1.5 KiB", "KiB");
eq(lib.formatBytes(200 * 1024 * 1024), "200 MiB", "MiB");
eq(lib.formatStats({ lazy: true, requests: 3, bytesFetched: 2 << 20, size: 200 << 20 }),
  "3 requests, 2 MiB of 200 MiB fetched (1.0%)", "stats line");
eq(lib.formatStats({ lazy: false, requests: 1, bytesFetched: 1024, size: 1024 }),
  "downloaded whole (1 KiB): the server does not support range requests", "stats line, no ranges");
h5.free();
nc.free();
console.log(`wasm package: ${checks} checks passed`);

if (base) await remoteTests();

async function serverStats() {
  return (await fetch(`${base}/__stats`)).json();
}

async function remoteTests() {
  const before = checks;
  await fetch(`${base}/__reset`);

  // The same checks over HTTP range requests, at the default block size
  // and at 512-byte blocks with a 4 KiB cache (almost every structure read
  // a miss, evictions between calls).
  for (const opts of [undefined, { blockSize: 512, cacheSize: 4096 }]) {
    for (const [name, exp] of Object.entries(expected)) {
      const f = await pkg.openUrl(`${base}/fix/${name}`, opts);
      await checkFile(`${name} (openUrl ${JSON.stringify(opts ?? {})})`, exp, f);
      const st = f.stats();
      eq(st.lazy, true, "read by ranges");
      eq(st.size, statSync(join(fixDir, name)).size, "size");
      f.free();
    }
  }
  const remote = await pkg.openUrl(`${base}/fix/fixture.h5`);
  await checkErrors(remote);
  eq((await (await pkg.openUrl(`${base}/fix/fixture.nc`)).info("/time")).maxshape, [null], "remote unlimited");

  // What the page counts is what the server served.
  await fetch(`${base}/__reset`);
  const counted = await pkg.openUrl(`${base}/fix/fixture.nc`, { blockSize: 1024 });
  await counted.read("/temp");
  const server = await serverStats();
  eq(counted.stats().requests, server.requests, "requests counted");
  eq(counted.stats().bytesFetched, server.bytes, "bytes counted");

  // A custom fetch is used for every request, with the caller's headers.
  let calls = 0;
  const seen = new Set();
  const viaCustom = await pkg.openUrl(`${base}/fix/fixture.h5`, {
    blockSize: 4096,
    headers: { "X-Test": "1" },
    fetch: (url, init) => { calls++; seen.add(init.headers["X-Test"]); return fetch(url, init); },
  });
  eq((await viaCustom.read("/sensors/temp")).data[0], 21.5, "custom fetch values");
  eq(calls, viaCustom.stats().requests, "custom fetch calls");
  eq([...seen], ["1"], "headers passed");

  // Calls in flight at once share the cache (a 1 KiB budget: nothing is
  // evicted while any of them runs) and each gets its own answer.
  const both = await pkg.openUrl(`${base}/fix/fixture.h5`, { blockSize: 512, cacheSize: 1024 });
  const [g, t, l, a] = await Promise.all([
    both.read("/grid"), both.read("/sensors/temp"), both.list("/sensors"), both.attrs("/"),
  ]);
  const exp = expected["fixture.h5"];
  eq(Array.from(g.data), exp.datasets["/grid"].values, "concurrent /grid");
  eq(Array.from(t.data), exp.datasets["/sensors/temp"].values, "concurrent /sensors/temp");
  eq(l.map((c) => c.name).sort(), [...exp.lists["/sensors"].groups, ...exp.lists["/sensors"].datasets].sort(), "concurrent list");
  eq(a.length > 0, true, "concurrent attrs");
  assert.ok(both.stats().cachedBytes <= 1024, "trimmed to the budget when idle");

  // Listing and reading small things of a large file fetches a few blocks,
  // not the file.
  if (existsSync(join(fixDir, "big.json"))) {
    const big = JSON.parse(readFileSync(join(fixDir, "big.json"), "utf8"));
    await fetch(`${base}/__reset`);
    const f = await pkg.openUrl(`${base}/fix/big.h5`);
    const list = await f.list("/");
    eq(list.filter((c) => c.kind === "group").map((c) => c.name), big.list.groups, "big: groups");
    eq(list.filter((c) => c.kind === "dataset").map((c) => c.name).sort(), big.list.datasets, "big: datasets");
    eq(Array.from((await f.read("/small")).data), big.small, "big: /small");
    eq(Array.from((await f.read("/meta/ids")).data, String), big.ids, "big: /meta/ids");
    eq((await f.info("/big")).shape, big.big_shape, "big: shape");
    eq((await f.attrs("/meta"))[0].value, "m", "big: group attribute");
    const win = await f.readHyperslab("/big", [big.window.start], [big.window.count]);
    eq(Array.from(win.data), big.window.values, "big: window");
    const st = f.stats();
    const server = await serverStats();
    eq(st.requests, server.requests, "big: requests counted");
    eq(st.bytesFetched, server.bytes, "big: bytes counted");
    console.log(`big.h5 (${big.size} bytes): listed, 3 small reads and a window in ` +
      `${server.requests} requests, ${server.bytes} bytes (${(100 * server.bytes / big.size).toFixed(2)}%)`);
    assert.ok(server.requests <= 8, `big: ${server.requests} requests`);
    assert.ok(server.bytes * 20 < big.size, `big: ${server.bytes} bytes fetched`);
    checks += 2;
  }

  // A server without range support: downloaded whole (the default), or
  // refused.
  const whole = await pkg.openUrl(`${base}/norange/fix/fixture.h5`);
  await checkFile("fixture.h5 (no range support)", expected["fixture.h5"], whole);
  eq(whole.stats().lazy, false, "downloaded whole");
  eq(whole.stats().requests, 1, "one request");
  await fails(() => pkg.openUrl(`${base}/norange/fix/fixture.h5`, { fallback: "error" }),
    /does not support HTTP range requests/, "fallback: error");
  await fails(() => pkg.openUrl(`${base}/norange/fix/fixture.h5`, { maxDownload: 1000 }),
    /more than maxDownload/, "maxDownload");
  // Without a Content-Length the body is streamed, and stopped at the limit.
  const undeclared = async (url, init) => {
    const r = await fetch(url, init);
    return new Response(r.body, { status: r.status });
  };
  await fails(() => pkg.openUrl(`${base}/norange/fix/fixture.h5`, { maxDownload: 1000, fetch: undeclared }),
    /more than maxDownload/, "maxDownload, streamed");
  const streamed = await pkg.openUrl(`${base}/norange/fix/fixture.h5`, { fetch: undeclared });
  eq(Array.from((await streamed.read("/sensors/temp")).data), [21.5, 22, 22.25], "streamed download");

  // Errors: HTTP status, not HDF5, bad options, a file that changes, a
  // server that answers with the wrong bytes.
  await fails(() => pkg.openUrl(`${base}/fix/missing.h5`), /HTTP 404/, "404");
  await fails(() => pkg.openUrl(`${base}/fix/expected.json`), /./, "not HDF5");
  await fails(() => pkg.openUrl(`${base}/fix/fixture.h5`, { blockSize: 100 }), /blockSize/, "blockSize");
  const tamper = (edit) => async (url, init) => {
    const r = await fetch(url, init);
    return init.headers.Range === "bytes=0-511" ? r : edit(r);
  };
  const withHeaders = async (r, headers) => {
    const h = new Headers(r.headers);
    for (const [k, v] of Object.entries(headers)) h.set(k, v);
    return new Response(await r.arrayBuffer(), { status: r.status, headers: h });
  };
  await fails(async () => {
    const f = await pkg.openUrl(`${base}/fix/fixture.h5`, { blockSize: 512, fetch: tamper((r) => withHeaders(r, { ETag: '"other"' })) });
    await f.read("/grid");
  }, /changed on the server/, "changed file");
  await fails(async () => {
    const f = await pkg.openUrl(`${base}/fix/fixture.h5`, {
      blockSize: 512,
      fetch: tamper(async (r) => new Response((await r.arrayBuffer()).slice(1), { status: 206 })),
    });
    await f.read("/grid");
  }, /got \d+/, "short answer");
  await fails(async () => {
    const f = await pkg.openUrl(`${base}/fix/fixture.h5`, {
      blockSize: 512,
      fetch: tamper(async (r) => new Response(await r.arrayBuffer(), { status: 200 })),
    });
    await f.read("/grid");
  }, /stopped honouring range requests/, "200 mid-file");
  await fails(async () => {
    const f = await pkg.openUrl(`${base}/fix/fixture.h5`, {
      blockSize: 512,
      fetch: tamper(async (r) => withHeaders(r, { "Content-Range": "bytes 0-511/25752" })),
    });
    await f.read("/grid");
  }, /the server sent 0-511/, "wrong range");
  await fails(async () => {
    const f = await pkg.openUrl(`${base}/fix/fixture.h5`, {
      blockSize: 512,
      fetch: tamper(async (r) => withHeaders(r, { "Content-Range": "bytes */25752" })),
    });
    await f.read("/grid");
  }, /unusable Content-Range/, "unusable Content-Range");

  // Corpus files: what the viewer can show of each is the same read by
  // ranges as in memory (an error wherever it gives one).
  const corpus = process.env.CLAWHDF5_WASM_CORPUS;
  if (corpus) await corpusTests(corpus);
  console.log(`openUrl: ${checks - before} checks passed`);
}

function hdf5Files(dir, out) {
  for (const e of readdirSync(dir, { withFileTypes: true })) {
    const p = join(dir, e.name);
    let st;
    try {
      st = statSync(p);
    } catch {
      continue;
    }
    if (st.isDirectory()) hdf5Files(p, out);
    else if (st.size <= 16 << 20) {
      const b = readFileSync(p);
      const sig = [0x89, 0x48, 0x44, 0x46, 0x0d, 0x0a, 0x1a, 0x0a];
      for (let at = 0; at + 8 <= b.length; at = at === 0 ? 512 : at * 2) {
        if (sig.every((x, i) => b[at + i] === x)) { out.push(p); break; }
      }
    }
  }
  return out;
}

function show(v) {
  return JSON.stringify(v, (_, x) => {
    if (typeof x === "bigint") return `${x}n`;
    if (ArrayBuffer.isView(x)) return Array.from(x, (y) => (typeof y === "bigint" ? `${y}n` : Number.isNaN(y) ? "NaN" : y));
    return x;
  });
}

async function transcript(f) {
  const out = [];
  const call = async (what, fn) => {
    try {
      out.push(`${what}: ${show(await fn())}`);
    } catch {
      out.push(`${what}: Err`);
    }
  };
  const todo = [["/", 0]];
  while (todo.length && out.length < 3000) {
    const [path, depth] = todo.pop();
    let kind = null;
    await call(`${path} kind`, async () => (kind = await f.kind(path)));
    await call(`${path} attrs`, () => f.attrs(path));
    if (kind === "group") {
      let list = [];
      await call(`${path} list`, async () => (list = await f.list(path)));
      if (depth < 12) for (const c of list.reverse()) todo.push([lib.joinPath(path, c.name), depth + 1]);
    } else if (kind === "dataset") {
      let info = null;
      await call(`${path} info`, async () => (info = await f.info(path)));
      if (info && [...info.shape, ...info.elementShape].reduce((a, b) => a * b, 1) <= 1 << 20) {
        await call(`${path} read`, () => f.read(path));
      }
    }
  }
  return out;
}

async function corpusTests(corpus) {
  const files = hdf5Files(corpus, []).sort();
  let opened = 0;
  let bytes = 0;
  let size = 0;
  for (const p of files) {
    const rel = relative(corpus, p).split("/").map(encodeURIComponent).join("/");
    let local;
    try {
      local = pkg.open(new Uint8Array(readFileSync(p)));
    } catch {
      await fails(() => pkg.openUrl(`${base}/corpus/${rel}`, { blockSize: 65536 }), /./, `${rel}: opens in neither`);
      continue;
    }
    const remote = await pkg.openUrl(`${base}/corpus/${rel}`, { blockSize: 65536 });
    const want = await transcript(local);
    const got = await transcript(remote);
    // Errors are compared as errors: a malformed file can fail at another
    // check, with another message, when read by ranges.
    eq(got, want, `${rel}: transcript`);
    opened++;
    bytes += remote.stats().bytesFetched;
    size += remote.stats().size;
    local.free();
    remote.free();
  }
  console.log(`corpus: ${opened} of ${files.length} files agree over HTTP (${bytes} of ${size} bytes fetched)`);
}