feat(wasm): examples/wasm-viewer, an HDF5/NetCDF-4 viewer page
Drop a file (or pass ?file=<url>&path=<object>), browse the tree lazily, see a dataset's type, shape, max shape and attributes, and page through its values as 50x12 hyperslab windows (leading dims of 3-D+ data held at chosen indices). build.sh produces pkg/ (not committed) with wasm-bindgen --target web and checks the CLI matches the crate version. test/run.sh builds it and runs test.mjs under Node against the h5py/ netCDF4 fixture (250 checks: every dataset whole and as a strided hyperslab, listings, attributes, error paths, the page's DOM-free helpers), then browser.sh renders the page in headless Chromium for eight objects and checks the DOM. The fixture gains LZ4 (read) and Zstd (refused: links C) datasets and a compound attribute (value null plus its type). ci-test.sh runs it when node and wasm-bindgen exist; the CI container has neither, so CI relies on the native h5py_interop test. Co-Authored-By: Claude Opus 5.5 (1M context) <[email protected]>
This commit is contained in:
osobh
@@ -0,0 +1,96 @@
|
||||
#!/usr/bin/env bash
|
||||
# Load the viewer page in headless Chromium and check what it renders.
|
||||
#
|
||||
# browser.sh FIXTURE_DIR
|
||||
#
|
||||
# FIXTURE_DIR holds fixture.h5 from make_fixture.py; ../pkg must be built.
|
||||
# The page is opened with ?file=fixture.h5&path=<object>, which fetches the
|
||||
# file, builds the tree down to <object> and shows it; the rendered DOM is
|
||||
# dumped and checked for the values libhdf5 reads.
|
||||
#
|
||||
# Browser: $CHROME, else chromium/google-chrome on PATH, else a Playwright
|
||||
# download under ~/.cache/ms-playwright. Exit 3 when none is found.
|
||||
# BROWSER_DEBUG=/some/prefix saves each rendered page as prefix.<path>.html.
|
||||
set -euo pipefail
|
||||
|
||||
HERE="$(cd "$(dirname "$0")" && pwd)"
|
||||
FIX="$(cd "$1" && pwd)"
|
||||
PY="${CLAWHDF5_PYTHON:-python3}"
|
||||
|
||||
chrome="${CHROME:-}"
|
||||
if [ -z "$chrome" ]; then
|
||||
for c in chromium chromium-browser google-chrome chrome-headless-shell; do
|
||||
if command -v "$c" >/dev/null; then chrome="$(command -v "$c")"; break; fi
|
||||
done
|
||||
fi
|
||||
if [ -z "$chrome" ]; then
|
||||
chrome="$(ls -d "$HOME"/.cache/ms-playwright/chromium_headless_shell-*/chrome-headless-shell-linux64/chrome-headless-shell 2>/dev/null | tail -1 || true)"
|
||||
fi
|
||||
if [ -z "$chrome" ] || [ ! -x "$chrome" ]; then
|
||||
echo "no Chromium found (set CHROME)" >&2
|
||||
exit 3
|
||||
fi
|
||||
|
||||
root="$(mktemp -d)"
|
||||
server=""
|
||||
cleanup() {
|
||||
[ -n "$server" ] && kill "$server" 2>/dev/null || true
|
||||
rm -rf "$root"
|
||||
}
|
||||
trap cleanup EXIT
|
||||
ln -s "$HERE/../index.html" "$HERE/../viewer-lib.js" "$HERE/../pkg" "$FIX/fixture.h5" "$root/"
|
||||
|
||||
port=$("$PY" -c 'import socket; s = socket.socket(); s.bind(("127.0.0.1", 0)); print(s.getsockname()[1])')
|
||||
"$PY" -m http.server --bind 127.0.0.1 --directory "$root" "$port" >/dev/null 2>&1 &
|
||||
server=$!
|
||||
for _ in $(seq 50); do
|
||||
"$PY" -c "import urllib.request; urllib.request.urlopen('http://127.0.0.1:$port/index.html')" 2>/dev/null && break
|
||||
sleep 0.1
|
||||
done
|
||||
|
||||
fails=0
|
||||
# A fresh profile per page: a second instance on the same profile fails.
|
||||
render() {
|
||||
local profile
|
||||
profile="$(mktemp -d "$root/profile.XXXXXX")"
|
||||
"$chrome" --headless --no-sandbox --disable-gpu --user-data-dir="$profile" \
|
||||
--virtual-time-budget=20000 \
|
||||
--dump-dom "http://127.0.0.1:$port/index.html?file=fixture.h5&path=$1" 2>/dev/null
|
||||
}
|
||||
# expect PATH TEXT...: every TEXT appears in the page rendered for PATH.
|
||||
expect() {
|
||||
local path="$1" dom
|
||||
shift
|
||||
dom="$(render "$path")"
|
||||
[ -n "${BROWSER_DEBUG:-}" ] && printf "%s\n" "$dom" > "$BROWSER_DEBUG.$(echo "$path" | tr / _).html"
|
||||
for text in "$@"; do
|
||||
if ! grep -qF -- "$text" <<<"$dom"; then
|
||||
echo "FAIL: page for $path lacks: $text" >&2
|
||||
fails=$((fails + 1))
|
||||
fi
|
||||
done
|
||||
echo "rendered $path"
|
||||
}
|
||||
|
||||
# Tree (root expanded; the group row carries its path) and root attributes.
|
||||
expect "/" 'data-path="/sensors"' 'data-path="/grid"' '<td>title</td><td>"wasm fixture"</td>' \
|
||||
'<td>big</td><td>9223372036854775813</td>' '(compound{x: f64, n: i32})'
|
||||
# A chunked, deflated 2-D dataset: type, shape, the first window of values.
|
||||
expect "/grid" '<dd>f64</dd>' '<dd>(6, 10)</dd>' '<th>9</th>' '<td>0.25</td>' '<td>14.75</td>' \
|
||||
'showing rows 0–5, columns 0–9 of 60 values'
|
||||
# Nested path revealed through the tree; big-endian float32.
|
||||
expect "/sensors/temp" 'data-path="/sensors/temp"' '<dd>f32</dd>' '<td>21.5</td>' '<td>22.25</td>'
|
||||
# 64-bit integers stay exact; strings; array datatype cells.
|
||||
expect "/u64" '<td>18446744073709551615</td>'
|
||||
expect "/vlen_str" '<td>"двa"</td>' '<dd>vlen string</dd>'
|
||||
expect "/pairs" '<td>[2, 3]</td>' '<dd>array[2]<i32></dd>'
|
||||
# 3-D: leading dimension held at 0, window over the last two.
|
||||
expect "/cube" '<dd>(2, 5, 6)</dd>' '<td>29</td>' 'dim 0'
|
||||
# Unsupported type: an error, not values.
|
||||
expect "/table" 'class="error"' 'reading compound{x: f64, n: i32} datasets is not supported'
|
||||
|
||||
if [ "$fails" -gt 0 ]; then
|
||||
echo "browser: $fails checks failed" >&2
|
||||
exit 1
|
||||
fi
|
||||
echo "browser: all checks passed ($chrome)"
|
||||
@@ -22,6 +22,11 @@ import h5py
|
||||
import netCDF4
|
||||
import numpy as np
|
||||
|
||||
try: # registers the LZ4/Zstd filters with libhdf5; optional
|
||||
import hdf5plugin
|
||||
except ImportError:
|
||||
hdf5plugin = None
|
||||
|
||||
# netCDF4 1.7 trips numpy 2.5's shape-setting deprecation on assignment.
|
||||
warnings.filterwarnings("ignore", category=DeprecationWarning)
|
||||
|
||||
@@ -37,6 +42,8 @@ with h5py.File(h5, "w") as f:
|
||||
f.attrs["scale"] = np.array([0.5, 2.0])
|
||||
f.attrs["big"] = np.uint64(2**63 + 5)
|
||||
f.attrs.create("vlen_note", "héllo", dtype=h5py.string_dtype())
|
||||
# No plain JavaScript form: listed with value null and its type.
|
||||
f.attrs["origin"] = np.array((1.5, 2), dtype=[("x", "<f8"), ("n", "<i4")])
|
||||
f.create_dataset(
|
||||
"grid", data=np.arange(60, dtype="<f8").reshape(6, 10) / 4,
|
||||
chunks=(4, 3), compression="gzip", shuffle=True,
|
||||
@@ -62,6 +69,12 @@ with h5py.File(h5, "w") as f:
|
||||
"cube", data=np.arange(2 * 5 * 6, dtype="<i4").reshape(2, 5, 6),
|
||||
chunks=(1, 2, 3), compression="gzip",
|
||||
)
|
||||
if hdf5plugin is not None:
|
||||
# LZ4 is built into clawhdf5-wasm; Zstd links C and is not.
|
||||
f.create_dataset("lz4", data=np.arange(40, dtype="<i4"), chunks=(10,),
|
||||
**hdf5plugin.LZ4())
|
||||
f.create_dataset("zstd", data=np.arange(40, dtype="<i4"), chunks=(10,),
|
||||
**hdf5plugin.Zstd())
|
||||
comp = np.zeros(2, dtype=[("x", "<f8"), ("n", "<i4")])
|
||||
f.create_dataset("table", data=comp)
|
||||
g = f.create_group("sensors")
|
||||
@@ -123,6 +136,8 @@ def entry(ds, slab=None):
|
||||
|
||||
def attr(v):
|
||||
v = np.asarray(v) if not isinstance(v, (str, bytes)) else v
|
||||
if isinstance(v, np.ndarray) and v.dtype.names:
|
||||
return {"raw": "compound"}
|
||||
if isinstance(v, bytes):
|
||||
return {"string": v.decode()}
|
||||
if isinstance(v, str):
|
||||
@@ -160,6 +175,8 @@ def describe(path):
|
||||
walk(key.rstrip("/") + "/" + n, o)
|
||||
elif obj.dtype.names:
|
||||
expected["errors"][key] = "compound"
|
||||
elif key == "/zstd":
|
||||
expected["errors"][key] = "unsupported filter: 32015"
|
||||
else:
|
||||
expected["datasets"][key] = entry(obj, slab_for(obj))
|
||||
|
||||
|
||||
Executable
+37
@@ -0,0 +1,37 @@
|
||||
#!/usr/bin/env bash
|
||||
# Build the wasm package (../build.sh), test it under Node against files
|
||||
# written by h5py and netCDF4 (make_fixture.py), then load the viewer page
|
||||
# in headless Chromium if one is found (browser.sh).
|
||||
#
|
||||
# Needs node, the wasm-bindgen CLI (see ../build.sh) and a Python with h5py,
|
||||
# netCDF4 and numpy: CLAWHDF5_PYTHON names it (default python3). Without that
|
||||
# Python the test is skipped, unless CLAWHDF5_REQUIRE_INTEROP=1.
|
||||
set -euo pipefail
|
||||
|
||||
HERE="$(cd "$(dirname "$0")" && pwd)"
|
||||
PY="${CLAWHDF5_PYTHON:-python3}"
|
||||
|
||||
command -v node >/dev/null || { echo "node not found" >&2; exit 1; }
|
||||
if ! "$PY" -c "import h5py, netCDF4, numpy" >/dev/null 2>&1; then
|
||||
if [ "${CLAWHDF5_REQUIRE_INTEROP:-0}" = "1" ]; then
|
||||
echo "CLAWHDF5_REQUIRE_INTEROP=1 but $PY lacks h5py/netCDF4/numpy" >&2
|
||||
exit 1
|
||||
fi
|
||||
echo "SKIP: $PY lacks h5py/netCDF4/numpy"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
bash "$HERE/../build.sh"
|
||||
fix="$(mktemp -d)"
|
||||
trap 'rm -rf "$fix"' EXIT
|
||||
"$PY" "$HERE/make_fixture.py" "$fix"
|
||||
node "$HERE/test.mjs" "$HERE/../pkg" "$fix"
|
||||
|
||||
# The page itself, in headless Chromium when one is available.
|
||||
status=0
|
||||
bash "$HERE/browser.sh" "$fix" || status=$?
|
||||
if [ "$status" = 3 ]; then
|
||||
echo "SKIP: viewer page in a browser (no Chromium; set CHROME)"
|
||||
elif [ "$status" != 0 ]; then
|
||||
exit "$status"
|
||||
fi
|
||||
@@ -0,0 +1,137 @@
|
||||
// Node test of the built wasm package (the exact pkg/ the viewer page loads)
|
||||
// and the viewer's DOM-free helpers. Run by test/run.sh:
|
||||
// node test.mjs PKG_DIR FIXTURE_DIR
|
||||
// FIXTURE_DIR holds fixture.h5, fixture.nc and expected.json from
|
||||
// make_fixture.py (values as libhdf5 reads them back).
|
||||
import assert from "node:assert/strict";
|
||||
import { readFileSync } from "node:fs";
|
||||
import { join } from "node:path";
|
||||
import { pathToFileURL } from "node:url";
|
||||
|
||||
const [pkgDir, fixDir] = process.argv.slice(2);
|
||||
const pkg = await import(pathToFileURL(join(pkgDir, "clawhdf5_wasm.js")));
|
||||
pkg.initSync({ module: readFileSync(join(pkgDir, "clawhdf5_wasm_bg.wasm")) });
|
||||
const lib = await import(pathToFileURL(join(import.meta.dirname, "..", "viewer-lib.js")));
|
||||
|
||||
let checks = 0;
|
||||
const eq = (a, b, msg) => { assert.deepEqual(a, b, msg); checks++; };
|
||||
|
||||
const ARRAY_TYPES = {
|
||||
f32: Float32Array, f64: Float64Array, i8: Int8Array, i16: Int16Array, i32: Int32Array,
|
||||
i64: BigInt64Array, u8: Uint8Array, u16: Uint16Array, u32: Uint32Array, u64: BigUint64Array,
|
||||
strings: Array,
|
||||
};
|
||||
|
||||
function values(kind, data) {
|
||||
const arr = Array.from(data);
|
||||
if (kind === "f32" || kind === "f64" || kind === "strings") return arr;
|
||||
return arr.map(String);
|
||||
}
|
||||
|
||||
function checkAttr(ctx, a, want) {
|
||||
const v = a.value;
|
||||
if ("raw" in want) {
|
||||
eq(v, null, ctx);
|
||||
assert.ok(a.dtype.includes(want.raw), `${ctx}: ${a.dtype}`);
|
||||
return;
|
||||
}
|
||||
eq(a.dtype, null, `${ctx} dtype`);
|
||||
if ("string" in want) return eq(v, want.string, ctx);
|
||||
if ("strings" in want) return eq(v, want.strings, ctx);
|
||||
if ("int" in want) {
|
||||
if (want.scalar) {
|
||||
assert.ok(typeof v === "number" || typeof v === "bigint", ctx);
|
||||
return eq([String(v)], want.int, ctx);
|
||||
}
|
||||
assert.ok(v instanceof BigInt64Array || v instanceof BigUint64Array, ctx);
|
||||
return eq(Array.from(v, String), want.int, ctx);
|
||||
}
|
||||
if ("float" in want) {
|
||||
if (want.scalar) return eq([v], want.float, ctx);
|
||||
assert.ok(v instanceof Float64Array, ctx);
|
||||
return eq(Array.from(v), want.float, ctx);
|
||||
}
|
||||
assert.fail(`${ctx}: unknown expectation ${JSON.stringify(want)}`);
|
||||
}
|
||||
|
||||
const expected = JSON.parse(readFileSync(join(fixDir, "expected.json"), "utf8"));
|
||||
for (const [name, exp] of Object.entries(expected)) {
|
||||
const file = pkg.open(new Uint8Array(readFileSync(join(fixDir, name))));
|
||||
|
||||
for (const [path, want] of Object.entries(exp.lists)) {
|
||||
eq(file.kind(path), "group", `${name}:${path} kind`);
|
||||
const list = file.list(path);
|
||||
for (const [kind, key] of [["group", "groups"], ["dataset", "datasets"]]) {
|
||||
eq(list.filter((c) => c.kind === kind).map((c) => c.name).sort(), want[key], `${name}:${path} ${key}`);
|
||||
}
|
||||
}
|
||||
|
||||
for (const [path, want] of Object.entries(exp.datasets)) {
|
||||
const ctx = `${name}:${path}`;
|
||||
eq(file.kind(path), "dataset", `${ctx} kind`);
|
||||
const info = file.info(path);
|
||||
eq([...info.shape, ...info.elementShape], want.shape, `${ctx} info shape`);
|
||||
const r = file.read(path);
|
||||
eq(r.shape, want.shape, `${ctx} shape`);
|
||||
eq(r.dtype, info.dtype, `${ctx} dtype`);
|
||||
assert.ok(r.data instanceof ARRAY_TYPES[want.kind], `${ctx}: ${r.data.constructor.name} for ${want.kind}`);
|
||||
eq(values(want.kind, r.data), want.values, ctx);
|
||||
if (want.slab) {
|
||||
const s = want.slab;
|
||||
const part = file.readHyperslab(path, s.start, s.count, s.stride);
|
||||
eq(part.shape, s.shape, `${ctx} slab shape`);
|
||||
eq(values(want.kind, part.data), s.values, `${ctx} slab`);
|
||||
}
|
||||
}
|
||||
|
||||
for (const [path, what] of Object.entries(exp.errors)) {
|
||||
assert.throws(() => file.read(path), (e) => e instanceof Error && e.message.includes(what), `${name}:${path}`);
|
||||
checks++;
|
||||
}
|
||||
|
||||
for (const [path, want] of Object.entries(exp.attrs)) {
|
||||
const attrs = file.attrs(path);
|
||||
eq(file.attrErrors(path), [], `${name}:${path} attr errors`);
|
||||
const seen = attrs.filter((a) => !a.name.startsWith("_") && !exp.skip_attrs.includes(a.name));
|
||||
eq(seen.map((a) => a.name).sort(), Object.keys(want).sort(), `${name}:${path} attr names`);
|
||||
for (const a of seen) checkAttr(`${name}:${path}@${a.name}`, a, want[a.name]);
|
||||
}
|
||||
file.free();
|
||||
}
|
||||
|
||||
// Errors reach JavaScript as thrown Errors, never as data.
|
||||
const h5 = pkg.open(new Uint8Array(readFileSync(join(fixDir, "fixture.h5"))));
|
||||
const throwsMsg = (fn, re) => { assert.throws(fn, (e) => e instanceof Error && re.test(e.message)); checks++; };
|
||||
throwsMsg(() => pkg.open(new Uint8Array(64)), /./);
|
||||
throwsMsg(() => h5.read("/nope"), /./);
|
||||
throwsMsg(() => h5.list("/grid"), /not a group/);
|
||||
throwsMsg(() => h5.readHyperslab("/grid", [0], [1]), /dimensions/);
|
||||
throwsMsg(() => h5.readHyperslab("/grid", [5, 0], [2, 1]), /exceeds/);
|
||||
throwsMsg(() => h5.readHyperslab("/grid", [-1, 0], [1, 1]), /non-negative integers/);
|
||||
throwsMsg(() => h5.readHyperslab("/grid", [0.5, 0], [1, 1]), /non-negative integers/);
|
||||
// Info for a dataset with an unlimited dimension (netCDF "time").
|
||||
const nc = pkg.open(new Uint8Array(readFileSync(join(fixDir, "fixture.nc"))));
|
||||
eq(nc.info("/time").maxshape, [null], "unlimited dimension is null");
|
||||
// Big integers stay exact.
|
||||
eq(h5.read("/u64").data[0], 18446744073709551615n, "u64 max");
|
||||
eq(typeof pkg.version(), "string", "version");
|
||||
|
||||
// Viewer helpers.
|
||||
eq(lib.joinPath("/", "a"), "/a", "joinPath root");
|
||||
eq(lib.joinPath("/a", "b"), "/a/b", "joinPath nested");
|
||||
eq(lib.viewWindow([], {}), null, "scalar window");
|
||||
eq(lib.viewWindow([7], { row: 5, rows: 50 }), { start: [5], count: [2], rows: 2, cols: 1, row: 5, col: 0 }, "1-D window");
|
||||
const w = lib.viewWindow([2, 5, 6], { row: 1, col: 4, rows: 3, cols: 5, fixed: [1] });
|
||||
eq(w, { start: [1, 1, 4], count: [1, 3, 2], rows: 3, cols: 2, row: 1, col: 4 }, "3-D window");
|
||||
// The window the page would request reads the same values as a direct slab.
|
||||
const cube = h5.readHyperslab("/cube", w.start, w.count);
|
||||
eq(lib.toRows(cube.data, w.rows, w.cols), [["40", "41"], ["46", "47"], ["52", "53"]], "cube window cells");
|
||||
const pairs = h5.readHyperslab("/pairs", [1], [2]);
|
||||
eq(lib.toRows(pairs.data, 2, 1, lib.perElement([2])), [["[2, 3]"], ["[4, 5]"]], "array-type cells");
|
||||
eq(lib.formatValue(0.1 + 0.2), "0.3", "float formatting");
|
||||
eq(lib.formatValue(2n ** 64n - 1n), "18446744073709551615", "bigint formatting");
|
||||
eq(lib.formatValue("x"), '"x"', "string formatting");
|
||||
h5.free();
|
||||
nc.free();
|
||||
|
||||
console.log(`wasm package: ${checks} checks passed`);
|
||||
Reference in New Issue
Block a user