From e815eb922fa5a9e8211b0356a41898bcc48192f6 Mon Sep 17 00:00:00 2001 From: osobh Date: Fri, 25 Sep 2026 23:51:20 -0500 Subject: [PATCH 1/7] feat(facade): Dataset::raw_datatype returns the full stored datatype dtype() simplifies the type (no byte order, string padding or member offsets), so callers could not decode read_selection's bytes for types the typed read_* methods skip. raw_datatype() returns the parsed Datatype, committed types resolved, for use with data_read. Co-Authored-By: Claude Opus 5.5 (1M context) --- crates/clawhdf5/src/lib.rs | 29 +++++++++++++++++++++++++++++ crates/clawhdf5/src/reader.rs | 9 +++++++++ 2 files changed, 38 insertions(+) diff --git a/crates/clawhdf5/src/lib.rs b/crates/clawhdf5/src/lib.rs index 40ae557..12e2f69 100644 --- a/crates/clawhdf5/src/lib.rs +++ b/crates/clawhdf5/src/lib.rs @@ -144,6 +144,35 @@ mod tests { assert_eq!(ds.dtype().unwrap(), DType::I32); } + #[test] + fn dataset_raw_datatype() { + use clawhdf5_format::datatype::Datatype; + let bytes = make_simple_file(); + let file = File::from_bytes(bytes).unwrap(); + let dt = file.dataset("counts").unwrap().raw_datatype().unwrap(); + assert!( + matches!( + dt, + Datatype::FixedPoint { + size: 4, + signed: true, + .. + } + ), + "{dt:?}" + ); + // The full type pairs with read_selection's bytes. + let raw = file + .dataset("counts") + .unwrap() + .read_selection(&Selection::All) + .unwrap(); + assert_eq!( + clawhdf5_format::data_read::read_as_i64(&raw, &dt).unwrap(), + vec![10, 20, 30] + ); + } + #[test] fn root_group_datasets() { let bytes = make_simple_file(); diff --git a/crates/clawhdf5/src/reader.rs b/crates/clawhdf5/src/reader.rs index 65c0292..b0c8d9c 100644 --- a/crates/clawhdf5/src/reader.rs +++ b/crates/clawhdf5/src/reader.rs @@ -416,6 +416,15 @@ impl<'f> Dataset<'f> { Ok(classify_datatype(&dt)) } + /// Returns the dataset's full datatype as stored in the file (byte order, + /// string padding, compound layout, ...), with a committed datatype + /// resolved. Use it with the `clawhdf5_format::data_read` converters on + /// the bytes [`read_selection`](Self::read_selection) returns, for types + /// the typed `read_*` methods do not cover. + pub fn raw_datatype(&self) -> Result { + self.datatype() + } + /// Read all data as `f64` values. pub fn read_f64(&self) -> Result, Error> { let dt = self.datatype()?; From 74f9f5008675ee607150b8ccf44a7e79202644c4 Mon Sep 17 00:00:00 2001 From: osobh Date: Sat, 26 Sep 2026 00:00:11 -0500 Subject: [PATCH 2/7] ci: build the clawhdf5 read path for wasm32-unknown-unknown The facade already builds for the browser target without mmap (and with it: memmap2 compiles there and File::open just fails, as std::fs does). Nothing needed gating; keep it that way with a ci-test.sh step, and install the target in the CI container. Co-Authored-By: Claude Opus 5.5 (1M context) --- .gitea/workflows/ci.yml | 3 +++ scripts/ci-test.sh | 8 ++++++++ 2 files changed, 11 insertions(+) diff --git a/.gitea/workflows/ci.yml b/.gitea/workflows/ci.yml index 1a7cdde..853a729 100644 --- a/.gitea/workflows/ci.yml +++ b/.gitea/workflows/ci.yml @@ -22,6 +22,9 @@ jobs: run: rustup component add rustfmt clippy - name: Install thumbv7em-none-eabihf target run: rustup target add thumbv7em-none-eabihf + - name: Install wasm32-unknown-unknown target + # ci-test.sh builds the reader and clawhdf5-wasm for the browser. + run: rustup target add wasm32-unknown-unknown - name: Install Python interop dependencies # The interop suites used to skip silently when python3/h5py were # missing, so they never ran in CI. Install them and make a missing diff --git a/scripts/ci-test.sh b/scripts/ci-test.sh index 1d7a7f9..6cad171 100755 --- a/scripts/ci-test.sh +++ b/scripts/ci-test.sh @@ -106,6 +106,14 @@ no_c_in_default_build() { } run_step "no C in the default build (core crates)" no_c_in_default_build +# The reader in the browser: the facade's read path must build for +# wasm32-unknown-unknown (no mmap, no threads, no file system). Needs +# `rustup target add wasm32-unknown-unknown`. +run_step "wasm32 build (clawhdf5, no default features)" cargo build \ + -p clawhdf5 \ + --target wasm32-unknown-unknown \ + --no-default-features + # The workspace declares a minimum Rust version (rust-version in Cargo.toml); # check that it really builds there, so the README badge and the manifests # cannot drift from the truth. Separate target dir: a different toolchain From a42b6466899d6a3b0456214dd95cc3cc7e3f4a58 Mon Sep 17 00:00:00 2001 From: osobh Date: Sat, 26 Sep 2026 00:00:46 -0500 Subject: [PATCH 3/7] feat(wasm): clawhdf5-wasm, the reader for JavaScript via wasm-bindgen open(bytes) -> H5File with kind/list/info/attrs/attrErrors/read/ readHyperslab. Numeric data comes back in the typed array of the stored width (Int16Array for i16, BigInt64Array for i64, Float32Array for f32/f16, ...), strings and enum names as string arrays, array datatypes flattened with their dims appended to the shape. Compound, reference, opaque and VL-sequence datasets are refused with an error naming the type; nothing is returned as reinterpreted bytes. The logic is in a plain-Rust core module, tested natively: unit tests, and h5py_interop, which compares every dataset, hyperslab, listing and attribute of an h5py- and a netCDF4-written file with what libhdf5 reads back (generator shared with the Node test of the built package). No mmap, no threads; lz4 is on, zstd/szip (C) are not. A wasm-release profile (opt-level s, LTO) serves the browser build. ci-test.sh lints the crate for wasm32 and checks it builds no C. Co-Authored-By: Claude Opus 5.5 (1M context) --- CLAUDE.md | 3 +- Cargo.toml | 10 + README.md | 5 +- crates/clawhdf5-wasm/Cargo.toml | 30 + crates/clawhdf5-wasm/src/core.rs | 603 +++++++++++++++++++++ crates/clawhdf5-wasm/src/lib.rs | 244 +++++++++ crates/clawhdf5-wasm/tests/h5py_interop.rs | 252 +++++++++ examples/wasm-viewer/test/make_fixture.py | 184 +++++++ scripts/ci-test.sh | 11 +- 9 files changed, 1337 insertions(+), 5 deletions(-) create mode 100644 crates/clawhdf5-wasm/Cargo.toml create mode 100644 crates/clawhdf5-wasm/src/core.rs create mode 100644 crates/clawhdf5-wasm/src/lib.rs create mode 100644 crates/clawhdf5-wasm/tests/h5py_interop.rs create mode 100644 examples/wasm-viewer/test/make_fixture.py diff --git a/CLAUDE.md b/CLAUDE.md index 3f8d9c1..4969c6a 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -5,7 +5,7 @@ Pure-Rust HDF5 format implementation with HNSW vector search, WAL-backed persist ## Architecture -Cargo workspace with 16 crates under `crates/` (plus `libaec-sys`, an internal FFI bindings crate for the optional `szip` feature): +Cargo workspace with 17 crates under `crates/` (plus `libaec-sys`, an internal FFI bindings crate for the optional `szip` feature): | Crate | Role | |-------|------| @@ -24,6 +24,7 @@ Cargo workspace with 16 crates under `crates/` (plus `libaec-sys`, an internal F | `clawhdf5-cli` | Command-line interface | | `clawhdf5-napi` | Node.js native addon bindings | | `clawhdf5-py` | PyO3 Python bindings | +| `clawhdf5-wasm` | WebAssembly (wasm-bindgen) reader for the browser; demo in `examples/wasm-viewer/` | | `clawhdf5-bench` | Benchmark suite | ## Key Features diff --git a/Cargo.toml b/Cargo.toml index 2e1f6fe..3fa2124 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -16,6 +16,7 @@ members = [ "crates/clawhdf5-cli", "crates/clawhdf5-napi", "crates/clawhdf5-bench", + "crates/clawhdf5-wasm", "crates/libaec-sys", ] resolver = "2" @@ -34,3 +35,12 @@ tempfile = "3" criterion = { version = "0.5", features = ["html_reports"] } half = "2.7" serde = { version = "1", features = ["derive"] } + +# The browser build of clawhdf5-wasm (examples/wasm-viewer/build.sh): size +# over speed, whole-program optimisation. Native profiles are unaffected. +[profile.wasm-release] +inherits = "release" +opt-level = "s" +lto = true +codegen-units = 1 +panic = "abort" diff --git a/README.md b/README.md index 5d4c707..eb2b27a 100644 --- a/README.md +++ b/README.md @@ -587,7 +587,7 @@ let exported = backend.export_markdown("MEMORY.md")?; ## Crate Map ``` -clawhdf5 workspace (16 crates, ~86K lines of Rust in src/, ~104K with tests +clawhdf5 workspace (17 crates, ~86K lines of Rust in src/, ~104K with tests and benches; plus libaec-sys, an internal FFI bindings crate for the optional szip feature) │ @@ -610,7 +610,8 @@ clawhdf5 workspace (16 crates, ~86K lines of Rust in src/, ~104K with tests │ ├── Bindings │ ├── clawhdf5-py — Python (PyO3) -│ └── clawhdf5-napi — Node.js (napi-rs) +│ ├── clawhdf5-napi — Node.js (napi-rs) +│ └── clawhdf5-wasm — Browser (WebAssembly, wasm-bindgen; read-only) │ └── Tooling └── clawhdf5-bench — Benchmark suite diff --git a/crates/clawhdf5-wasm/Cargo.toml b/crates/clawhdf5-wasm/Cargo.toml new file mode 100644 index 0000000..303837a --- /dev/null +++ b/crates/clawhdf5-wasm/Cargo.toml @@ -0,0 +1,30 @@ +[package] +name = "clawhdf5-wasm" +version.workspace = true +edition.workspace = true +rust-version.workspace = true +description = "Read HDF5 and NetCDF-4 files in the browser: clawhdf5's reader compiled to WebAssembly" +license.workspace = true +repository.workspace = true +keywords = ["hdf5", "netcdf", "wasm", "browser"] +categories = ["wasm", "parser-implementations", "science"] +# Distributed as the wasm-bindgen package built by examples/wasm-viewer/build.sh. +publish = false + +[lib] +# cdylib for wasm-bindgen; rlib so the pure-Rust core is tested natively. +crate-type = ["cdylib", "rlib"] + +[dependencies] +# No mmap (there is no file system) and no `parallel` (no threads on +# wasm32-unknown-unknown). lz4 is pure Rust; zstd and szip link C and are +# left out, so such datasets fail with a clear filter error. +clawhdf5 = { path = "../clawhdf5", version = "2.7.0", default-features = false, features = ["lz4"] } +clawhdf5-format = { path = "../clawhdf5-format", version = "2.7.0" } +# Must match the wasm-bindgen CLI exactly; build.sh checks. +wasm-bindgen = "0.2.129" +js-sys = "0.3.106" + +[dev-dependencies] +serde_json = "1" +tempfile = { workspace = true } diff --git a/crates/clawhdf5-wasm/src/core.rs b/crates/clawhdf5-wasm/src/core.rs new file mode 100644 index 0000000..baadcda --- /dev/null +++ b/crates/clawhdf5-wasm/src/core.rs @@ -0,0 +1,603 @@ +//! The reader behind the JavaScript API, in plain Rust so it is tested +//! natively. The `wasm_bindgen` layer in `lib.rs` only converts these types +//! to JavaScript values. +//! +//! Every read either returns the dataset's values or an error: a datatype +//! with no typed-array mapping (compound, reference, opaque, ...) is refused +//! with a message naming it, never returned as reinterpreted bytes. + +use clawhdf5::{AttrValue, File, Selection}; +use clawhdf5_format::data_read; +use clawhdf5_format::datatype::{Datatype, DatatypeByteOrder}; + +/// Errors are reported to JavaScript as messages. +pub type Result = std::result::Result; + +fn err(e: impl std::fmt::Display) -> String { + e.to_string() +} + +/// What a path names. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Kind { + Group, + Dataset, +} + +impl Kind { + pub fn as_str(self) -> &'static str { + match self { + Kind::Group => "group", + Kind::Dataset => "dataset", + } + } +} + +/// One entry of a group listing. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct Child { + pub name: String, + pub kind: Kind, +} + +/// A dataset's metadata. +#[derive(Debug, Clone, PartialEq)] +pub struct DatasetInfo { + /// Dataspace dimensions (empty for a scalar). + pub shape: Vec, + /// Maximum dimensions, `None` per unlimited dimension; `None` overall + /// when the dataspace records none. + pub maxshape: Option>>, + /// Human-readable datatype, e.g. `f64`, `i16 (big-endian)`, `string[8]`. + pub dtype: String, + /// Dimensions of an array datatype's elements, appended to the shape of + /// what [`Reader::read`] returns (empty otherwise). + pub element_shape: Vec, +} + +/// Decoded values, one variant per JavaScript typed array. +#[derive(Debug, Clone, PartialEq)] +pub enum Data { + F32(Vec), + F64(Vec), + I8(Vec), + I16(Vec), + I32(Vec), + I64(Vec), + U8(Vec), + U16(Vec), + U32(Vec), + U64(Vec), + /// Fixed- and variable-length strings, and enumeration member names. + Strings(Vec), +} + +impl Data { + pub fn len(&self) -> usize { + match self { + Data::F32(v) => v.len(), + Data::F64(v) => v.len(), + Data::I8(v) => v.len(), + Data::I16(v) => v.len(), + Data::I32(v) => v.len(), + Data::I64(v) => v.len(), + Data::U8(v) => v.len(), + Data::U16(v) => v.len(), + Data::U32(v) => v.len(), + Data::U64(v) => v.len(), + Data::Strings(v) => v.len(), + } + } + + pub fn is_empty(&self) -> bool { + self.len() == 0 + } +} + +/// Values in row-major order with their shape. +#[derive(Debug, Clone, PartialEq)] +pub struct Array { + pub shape: Vec, + pub data: Data, +} + +/// A regular hyperslab, as in `H5Sselect_hyperslab`. `stride` and `block` +/// default to 1 in every dimension. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct Hyperslab { + pub start: Vec, + pub count: Vec, + pub stride: Option>, + pub block: Option>, +} + +/// An attribute: its value, or why it has none. +#[derive(Debug, Clone)] +pub struct Attr { + pub name: String, + pub value: AttrValue, +} + +/// An open file, held in memory. +pub struct Reader { + file: File, +} + +impl Reader { + /// Parse a file from its bytes (the browser hands over the whole file). + pub fn open(bytes: Vec) -> Result { + Ok(Self { + file: File::from_bytes(bytes).map_err(err)?, + }) + } + + /// Whether `path` names a group or a dataset. + pub fn kind(&self, path: &str) -> Result { + match self.file.dataset(path) { + Ok(_) => Ok(Kind::Dataset), + Err(clawhdf5::Error::NotADataset(_)) => Ok(Kind::Group), + Err(e) => Err(err(e)), + } + } + + /// The groups, then the datasets, in the group at `path` (`/` is the + /// root). Soft links are listed as their targets; external and dangling + /// links, and named datatypes, are left out. + pub fn list(&self, path: &str) -> Result> { + if self.kind(path)? != Kind::Group { + return Err(format!("not a group: {path}")); + } + let group = self.file.group(path).map_err(err)?; + let mut out: Vec = group + .groups() + .map_err(err)? + .into_iter() + .map(|name| Child { + name, + kind: Kind::Group, + }) + .collect(); + out.extend( + group + .datasets() + .map_err(err)? + .into_iter() + .map(|name| Child { + name, + kind: Kind::Dataset, + }), + ); + Ok(out) + } + + /// Shape, max shape and datatype of the dataset at `path`. + pub fn info(&self, path: &str) -> Result { + let ds = self.file.dataset(path).map_err(err)?; + let dt = ds.raw_datatype().map_err(err)?; + let maxshape = ds.max_dimensions().map_err(err)?.map(|dims| { + dims.into_iter() + .map(|d| (d != u64::MAX).then_some(d)) + .collect() + }); + Ok(DatasetInfo { + shape: ds.shape().map_err(err)?, + maxshape, + dtype: describe(&dt), + element_shape: element_shape(&dt), + }) + } + + /// The attributes of the group or dataset at `path`, sorted by name, and + /// one message per attribute that could not be read at all. An attribute + /// whose type has no plain JavaScript form is returned as + /// [`AttrValue::Raw`]. + pub fn attrs(&self, path: &str) -> Result<(Vec, Vec)> { + let (map, errors) = match self.kind(path)? { + Kind::Dataset => self + .file + .dataset(path) + .and_then(|d| d.attrs_with_errors()) + .map_err(err)?, + Kind::Group => self + .file + .group(path) + .and_then(|g| g.attrs_with_errors()) + .map_err(err)?, + }; + let mut attrs: Vec = map + .into_iter() + .map(|(name, value)| Attr { name, value }) + .collect(); + attrs.sort_by(|a, b| a.name.cmp(&b.name)); + Ok((attrs, errors.into_iter().map(err).collect())) + } + + /// Read the dataset at `path`, whole or a hyperslab of it. + pub fn read(&self, path: &str, slab: Option<&Hyperslab>) -> Result { + let ds = self.file.dataset(path).map_err(err)?; + let dt = ds.raw_datatype().map_err(err)?; + let shape = ds.shape().map_err(err)?; + let (selection, mut out_shape) = match slab { + None => (Selection::All, shape.clone()), + Some(h) => hyperslab_selection(h, &shape)?, + }; + let raw = ds.read_selection(&selection).map_err(err)?; + let data = self.decode(&raw, &dt)?; + out_shape.extend(element_shape(&dt)); + let expected = out_shape + .iter() + .try_fold(1u64, |acc, &d| acc.checked_mul(d)) + .ok_or("selection size overflows")?; + if data.len() as u64 != expected { + return Err(format!( + "read {} values for shape {out_shape:?} ({expected} expected)", + data.len() + )); + } + Ok(Array { + shape: out_shape, + data, + }) + } + + fn decode(&self, raw: &[u8], dt: &Datatype) -> Result { + let base = array_base(dt); + let is_array = !std::ptr::eq(base, dt); + Ok(match base { + Datatype::FloatingPoint { size, .. } if *size <= 4 => { + Data::F32(data_read::read_as_f32(raw, dt).map_err(err)?) + } + Datatype::FloatingPoint { .. } => { + Data::F64(data_read::read_as_f64(raw, dt).map_err(err)?) + } + Datatype::FixedPoint { size, signed, .. } => { + let signed_ints = || data_read::read_as_i64(raw, dt).map_err(err); + let unsigned_ints = || data_read::read_as_u64(raw, dt).map_err(err); + match (size, signed) { + (1, true) => Data::I8(narrow(signed_ints()?)?), + (2, true) => Data::I16(narrow(signed_ints()?)?), + (4, true) => Data::I32(narrow(signed_ints()?)?), + (_, true) => Data::I64(signed_ints()?), + (1, false) => Data::U8(narrow(unsigned_ints()?)?), + (2, false) => Data::U16(narrow(unsigned_ints()?)?), + (4, false) => Data::U32(narrow(unsigned_ints()?)?), + (_, false) => Data::U64(unsigned_ints()?), + } + } + Datatype::String { .. } if !is_array => { + Data::Strings(data_read::read_as_strings(raw, dt).map_err(err)?) + } + Datatype::VariableLength { + is_string: true, .. + } if !is_array => { + let size = dt.type_size() as usize; + if size == 0 || !raw.len().is_multiple_of(size) { + return Err(format!( + "{} bytes is not a whole number of {size}-byte string references", + raw.len() + )); + } + let sb = self.file.superblock(); + Data::Strings( + clawhdf5_format::vl_data::read_vl_strings( + self.file.as_bytes(), + raw, + (raw.len() / size) as u64, + sb.offset_size, + sb.length_size, + ) + .map_err(err)?, + ) + } + Datatype::Enumeration { .. } if !is_array => { + Data::Strings(data_read::read_enum_names(raw, dt).map_err(err)?) + } + _ => { + return Err(format!( + "reading {} datasets is not supported", + describe(dt) + )); + } + }) + } +} + +/// Narrow integers read at 64 bits to the dataset's own width. The source is +/// that width, so this cannot fail on correct input; it is checked anyway. +fn narrow>(v: Vec) -> Result> { + v.into_iter() + .map(|x| T::try_from(x).map_err(|_| format!("value {x} out of range"))) + .collect() +} + +/// Innermost element type of (possibly nested) array datatypes. +fn array_base(dt: &Datatype) -> &Datatype { + match dt { + Datatype::Array { base_type, .. } => array_base(base_type), + _ => dt, + } +} + +fn element_shape(dt: &Datatype) -> Vec { + match dt { + Datatype::Array { + base_type, + dimensions, + } => { + let mut dims: Vec = dimensions.iter().map(|&d| u64::from(d)).collect(); + dims.extend(element_shape(base_type)); + dims + } + _ => Vec::new(), + } +} + +fn hyperslab_selection(h: &Hyperslab, shape: &[u64]) -> Result<(Selection, Vec)> { + let rank = shape.len(); + let ones = vec![1u64; rank]; + let stride = h.stride.clone().unwrap_or_else(|| ones.clone()); + let block = h.block.clone().unwrap_or(ones); + for (what, v) in [ + ("start", &h.start), + ("count", &h.count), + ("stride", &stride), + ("block", &block), + ] { + if v.len() != rank { + return Err(format!( + "hyperslab {what} has {} dimensions, the dataset has {rank}", + v.len() + )); + } + } + let mut out = Vec::with_capacity(rank); + for d in 0..rank { + if stride[d] == 0 || block[d] == 0 { + return Err(format!("hyperslab stride and block must be >= 1 (dim {d})")); + } + if h.count[d] > 1 && block[d] > stride[d] { + return Err(format!( + "hyperslab blocks overlap in dim {d}: block {} > stride {}", + block[d], stride[d] + )); + } + // Last element selected: start + (count-1)*stride + block - 1. + if h.count[d] > 0 { + let last = (h.count[d] - 1) + .checked_mul(stride[d]) + .and_then(|x| x.checked_add(h.start[d])) + .and_then(|x| x.checked_add(block[d] - 1)); + match last { + Some(l) if l < shape[d] => {} + _ => { + return Err(format!( + "hyperslab exceeds dimension {d} (extent {})", + shape[d] + )); + } + } + } + out.push( + h.count[d] + .checked_mul(block[d]) + .ok_or("selection size overflows")?, + ); + } + Ok(( + Selection::Hyperslab { + start: h.start.clone(), + stride, + count: h.count.clone(), + block, + }, + out, + )) +} + +/// A short, human-readable datatype name. +pub fn describe(dt: &Datatype) -> String { + fn endian(order: &DatatypeByteOrder) -> &'static str { + match order { + DatatypeByteOrder::BigEndian => " (big-endian)", + DatatypeByteOrder::Vax => " (VAX)", + _ => "", + } + } + match dt { + Datatype::FixedPoint { + size, + signed, + byte_order, + .. + } => format!( + "{}{}{}", + if *signed { "i" } else { "u" }, + size * 8, + endian(byte_order) + ), + Datatype::FloatingPoint { + size, byte_order, .. + } => format!("f{}{}", size * 8, endian(byte_order)), + Datatype::Time { size, .. } => format!("time{}", size * 8), + Datatype::String { size, .. } => format!("string[{size}]"), + Datatype::BitField { size, .. } => format!("bitfield{}", size * 8), + Datatype::Opaque { size, .. } => format!("opaque[{size}]"), + Datatype::Compound { members, .. } => { + let fields: Vec = members + .iter() + .map(|m| format!("{}: {}", m.name, describe(&m.datatype))) + .collect(); + format!("compound{{{}}}", fields.join(", ")) + } + Datatype::Reference { .. } => "reference".to_string(), + Datatype::Enumeration { + base_type, members, .. + } => { + let names: Vec<&str> = members.iter().map(|m| m.name.as_str()).collect(); + format!("enum<{}>{{{}}}", describe(base_type), names.join(", ")) + } + Datatype::VariableLength { + is_string: true, .. + } => "vlen string".to_string(), + Datatype::VariableLength { base_type, .. } => { + format!("vlen<{}>", describe(base_type)) + } + Datatype::Array { + base_type, + dimensions, + } => format!("array{dimensions:?}<{}>", describe(base_type)), + } +} + +#[cfg(test)] +mod tests { + use super::*; + use clawhdf5::FileBuilder; + + fn sample() -> Reader { + let mut b = FileBuilder::new(); + b.create_dataset("grid") + .with_f64_data(&(0..12).map(f64::from).collect::>()) + .with_shape(&[3, 4]) + .with_chunks(&[2, 2]) + .with_deflate(4); + b.create_dataset("bytes").with_u8_data(&[1, 2, 250]); + let mut g = b.create_group("sensors"); + g.create_dataset("temp").with_f32_data(&[1.5, -2.25]); + g.set_attr("location", AttrValue::String("lab".into())); + b.add_group(g.finish()); + b.set_attr("version", AttrValue::I64(3)); + b.set_attr("scale", AttrValue::F64Array(vec![0.5, 2.0])); + Reader::open(b.finish().unwrap()).unwrap() + } + + #[test] + fn lists_groups_then_datasets() { + let r = sample(); + let names: Vec<(String, Kind)> = r + .list("/") + .unwrap() + .into_iter() + .map(|c| (c.name, c.kind)) + .collect(); + assert_eq!(names[0], ("sensors".to_string(), Kind::Group)); + let mut ds: Vec<&str> = names[1..].iter().map(|(n, _)| n.as_str()).collect(); + ds.sort(); + assert_eq!(ds, ["bytes", "grid"]); + assert_eq!( + r.list("sensors").unwrap(), + vec![Child { + name: "temp".into(), + kind: Kind::Dataset + }] + ); + assert!(r.list("grid").unwrap_err().contains("not a group")); + assert!(r.list("missing").is_err()); + } + + #[test] + fn info_reports_shape_and_dtype() { + let r = sample(); + let i = r.info("grid").unwrap(); + assert_eq!(i.shape, vec![3, 4]); + assert_eq!(i.dtype, "f64"); + assert!(i.element_shape.is_empty()); + assert_eq!(r.info("sensors/temp").unwrap().dtype, "f32"); + assert_eq!(r.kind("/sensors").unwrap(), Kind::Group); + assert_eq!(r.kind("/sensors/temp").unwrap(), Kind::Dataset); + } + + #[test] + fn reads_whole_and_hyperslab() { + let r = sample(); + let all = r.read("grid", None).unwrap(); + assert_eq!(all.shape, vec![3, 4]); + assert_eq!(all.data, Data::F64((0..12).map(f64::from).collect())); + + let slab = Hyperslab { + start: vec![1, 0], + count: vec![2, 2], + stride: Some(vec![1, 2]), + block: None, + }; + let part = r.read("grid", Some(&slab)).unwrap(); + assert_eq!(part.shape, vec![2, 2]); + assert_eq!(part.data, Data::F64(vec![4.0, 6.0, 8.0, 10.0])); + + assert_eq!( + r.read("bytes", None).unwrap().data, + Data::U8(vec![1, 2, 250]) + ); + assert_eq!( + r.read("sensors/temp", None).unwrap().data, + Data::F32(vec![1.5, -2.25]) + ); + } + + #[test] + fn bad_hyperslabs_are_refused() { + let r = sample(); + let mk = |start: Vec, count: Vec| Hyperslab { + start, + count, + stride: None, + block: None, + }; + assert!( + r.read("grid", Some(&mk(vec![0], vec![1]))) + .unwrap_err() + .contains("dimensions") + ); + assert!( + r.read("grid", Some(&mk(vec![2, 0], vec![2, 1]))) + .unwrap_err() + .contains("exceeds") + ); + let overlap = Hyperslab { + start: vec![0, 0], + count: vec![2, 1], + stride: Some(vec![1, 1]), + block: Some(vec![2, 1]), + }; + assert!( + r.read("grid", Some(&overlap)) + .unwrap_err() + .contains("overlap") + ); + } + + #[test] + fn attrs_are_sorted() { + let r = sample(); + let (attrs, errors) = r.attrs("/").unwrap(); + assert!(errors.is_empty()); + let names: Vec<&str> = attrs.iter().map(|a| a.name.as_str()).collect(); + assert_eq!(names, ["scale", "version"]); + let (g, _) = r.attrs("sensors").unwrap(); + assert!(matches!(&g[0].value, AttrValue::String(s) if s == "lab")); + } + + #[test] + fn compound_is_refused_not_reinterpreted() { + use clawhdf5::CompoundTypeBuilder; + let ct = CompoundTypeBuilder::new() + .f64_field("x") + .i32_field("n") + .build(); + let mut rec = Vec::new(); + rec.extend_from_slice(&1.0f64.to_le_bytes()); + rec.extend_from_slice(&7i32.to_le_bytes()); + let mut b = FileBuilder::new(); + b.create_dataset("table").with_compound_data(ct, rec, 1); + let r = Reader::open(b.finish().unwrap()).unwrap(); + let e = r.read("table", None).unwrap_err(); + assert!(e.contains("compound{x: f64, n: i32}"), "{e}"); + assert!(e.contains("not supported"), "{e}"); + } + + #[test] + fn garbage_is_an_error() { + assert!(Reader::open(vec![0u8; 64]).is_err()); + assert!(Reader::open(Vec::new()).is_err()); + } +} diff --git a/crates/clawhdf5-wasm/src/lib.rs b/crates/clawhdf5-wasm/src/lib.rs new file mode 100644 index 0000000..7c6b4a9 --- /dev/null +++ b/crates/clawhdf5-wasm/src/lib.rs @@ -0,0 +1,244 @@ +//! clawhdf5's HDF5 reader for JavaScript, via `wasm-bindgen`. +//! +//! ```js +//! import init, { open } from "./pkg/clawhdf5_wasm.js"; +//! await init(); +//! const file = open(new Uint8Array(await blob.arrayBuffer())); +//! file.list("/"); // [{ name, kind: "group" | "dataset" }] +//! file.info("/x"); // { shape, maxshape, dtype, elementShape } +//! file.attrs("/x"); // [{ name, value, dtype }] +//! file.read("/x"); // { shape, dtype, data: Float64Array | ... | string[] } +//! file.readHyperslab("/x", [0, 0], [10, 10]); // stride, block optional +//! file.free(); +//! ``` +//! +//! Numeric data comes back in the typed array of the stored width +//! (`Int16Array` for `i16`, `BigInt64Array` for `i64`, `Float32Array` for +//! `f32` and `f16`, ...); strings and enumeration names as arrays of strings. +//! Anything else is a thrown `Error` naming the datatype. Only the reader is +//! exposed: nothing here writes files. +//! +//! The logic lives in [`core`], which is plain Rust and tested natively. + +pub mod core; + +use clawhdf5::AttrValue; +use js_sys::{Array, Object, Reflect}; +use wasm_bindgen::prelude::*; + +use crate::core::{Data, Hyperslab, Reader}; + +/// JavaScript numbers are exact up to 2^53. +const MAX_SAFE_INTEGER: f64 = 9_007_199_254_740_991.0; + +fn js_err(msg: String) -> JsError { + JsError::new(&msg) +} + +fn set(obj: &Object, key: &str, value: impl Into) { + // Defining a property on a fresh plain object cannot fail. + Reflect::set(obj, &JsValue::from_str(key), &value.into()).unwrap_throw(); +} + +fn shape_to_js(shape: &[u64]) -> Array { + shape.iter().map(|&d| JsValue::from_f64(d as f64)).collect() +} + +fn indices_from_js(what: &str, v: &[f64]) -> Result, JsError> { + v.iter() + .map(|&x| { + if x.is_finite() && x >= 0.0 && x.fract() == 0.0 && x <= MAX_SAFE_INTEGER { + Ok(x as u64) + } else { + Err(js_err(format!( + "{what} must hold non-negative integers, got {x}" + ))) + } + }) + .collect() +} + +fn data_to_js(data: Data) -> JsValue { + match data { + Data::F32(v) => js_sys::Float32Array::from(&v[..]).into(), + Data::F64(v) => js_sys::Float64Array::from(&v[..]).into(), + Data::I8(v) => js_sys::Int8Array::from(&v[..]).into(), + Data::I16(v) => js_sys::Int16Array::from(&v[..]).into(), + Data::I32(v) => js_sys::Int32Array::from(&v[..]).into(), + Data::I64(v) => js_sys::BigInt64Array::from(&v[..]).into(), + Data::U8(v) => js_sys::Uint8Array::from(&v[..]).into(), + Data::U16(v) => js_sys::Uint16Array::from(&v[..]).into(), + Data::U32(v) => js_sys::Uint32Array::from(&v[..]).into(), + Data::U64(v) => js_sys::BigUint64Array::from(&v[..]).into(), + Data::Strings(v) => v + .into_iter() + .map(|s| JsValue::from_str(&s)) + .collect::() + .into(), + } +} + +/// A scalar integer as a `number` when exact, else a `bigint`. +fn int_to_js(x: i128) -> JsValue { + if (x as f64).abs() <= MAX_SAFE_INTEGER { + JsValue::from_f64(x as f64) + } else if let Ok(v) = i64::try_from(x) { + JsValue::from(v) + } else { + JsValue::from(x as u64) + } +} + +/// An attribute value, and the datatype of one that has no JavaScript form. +fn attr_to_js(value: AttrValue) -> (JsValue, Option) { + match value { + AttrValue::F64(x) => (JsValue::from_f64(x), None), + AttrValue::F64Array(v) => (js_sys::Float64Array::from(&v[..]).into(), None), + AttrValue::I64(x) => (int_to_js(i128::from(x)), None), + AttrValue::I64Array(v) => (js_sys::BigInt64Array::from(&v[..]).into(), None), + AttrValue::U64(x) => (int_to_js(i128::from(x)), None), + AttrValue::U64Array(v) => (js_sys::BigUint64Array::from(&v[..]).into(), None), + AttrValue::String(s) => (JsValue::from_str(&s), None), + AttrValue::StringArray(v) => ( + v.iter() + .map(|s| JsValue::from_str(s)) + .collect::() + .into(), + None, + ), + AttrValue::Raw { datatype, .. } => (JsValue::NULL, Some(core::describe(&datatype))), + } +} + +/// An open HDF5 (or NetCDF-4) file. +#[wasm_bindgen] +pub struct H5File { + inner: Reader, +} + +/// Open a file from its bytes. Throws if they are not an HDF5 file. +#[wasm_bindgen] +pub fn open(bytes: Vec) -> Result { + H5File::new(bytes) +} + +/// The clawhdf5 version this module was built from. +#[wasm_bindgen] +pub fn version() -> String { + env!("CARGO_PKG_VERSION").to_string() +} + +#[wasm_bindgen] +impl H5File { + /// Same as [`open`]. + #[wasm_bindgen(constructor)] + pub fn new(bytes: Vec) -> Result { + Ok(H5File { + inner: Reader::open(bytes).map_err(js_err)?, + }) + } + + /// `"group"` or `"dataset"`. + pub fn kind(&self, path: &str) -> Result { + Ok(self.inner.kind(path).map_err(js_err)?.as_str().to_string()) + } + + /// The group's members: `[{ name, kind }]`, groups first. + pub fn list(&self, path: &str) -> Result { + Ok(self + .inner + .list(path) + .map_err(js_err)? + .into_iter() + .map(|c| { + let o = Object::new(); + set(&o, "name", c.name); + set(&o, "kind", c.kind.as_str()); + JsValue::from(o) + }) + .collect()) + } + + /// `{ shape, maxshape, dtype, elementShape }`. `maxshape` is `null` + /// when not recorded, with `null` for each unlimited dimension. + pub fn info(&self, path: &str) -> Result { + let i = self.inner.info(path).map_err(js_err)?; + let o = Object::new(); + set(&o, "shape", shape_to_js(&i.shape)); + let max: JsValue = match i.maxshape { + None => JsValue::NULL, + Some(dims) => dims + .into_iter() + .map(|d| d.map_or(JsValue::NULL, |d| JsValue::from_f64(d as f64))) + .collect::() + .into(), + }; + set(&o, "maxshape", max); + set(&o, "dtype", i.dtype); + set(&o, "elementShape", shape_to_js(&i.element_shape)); + Ok(o) + } + + /// `[{ name, value, dtype }]`, sorted by name. Scalars are `number` + /// (`bigint` beyond 2^53) or `string`; arrays are typed arrays or + /// `string[]`. An attribute with no JavaScript form has `value: null` + /// and its `dtype`; one that could not be read at all is reported by + /// [`attrErrors`](Self::attr_errors). + pub fn attrs(&self, path: &str) -> Result { + let (attrs, _) = self.inner.attrs(path).map_err(js_err)?; + Ok(attrs + .into_iter() + .map(|a| { + let o = Object::new(); + set(&o, "name", a.name); + let (value, dtype) = attr_to_js(a.value); + set(&o, "value", value); + set(&o, "dtype", dtype.map_or(JsValue::NULL, JsValue::from)); + JsValue::from(o) + }) + .collect()) + } + + /// Messages for attributes that could not be read. + #[wasm_bindgen(js_name = attrErrors)] + pub fn attr_errors(&self, path: &str) -> Result { + let (_, errors) = self.inner.attrs(path).map_err(js_err)?; + Ok(errors.into_iter().map(JsValue::from).collect()) + } + + /// The whole dataset: `{ shape, dtype, data }`, `data` in row-major + /// order. + pub fn read(&self, path: &str) -> Result { + self.read_impl(path, None) + } + + /// A regular hyperslab (`H5Sselect_hyperslab`): `stride` and `block` + /// default to 1. The result's shape is `count * block` per dimension. + #[wasm_bindgen(js_name = readHyperslab)] + pub fn read_hyperslab( + &self, + path: &str, + start: Vec, + count: Vec, + stride: Option>, + block: Option>, + ) -> Result { + let slab = Hyperslab { + start: indices_from_js("start", &start)?, + count: indices_from_js("count", &count)?, + stride: stride.map(|s| indices_from_js("stride", &s)).transpose()?, + block: block.map(|b| indices_from_js("block", &b)).transpose()?, + }; + self.read_impl(path, Some(&slab)) + } + + fn read_impl(&self, path: &str, slab: Option<&Hyperslab>) -> Result { + let dtype = self.inner.info(path).map_err(js_err)?.dtype; + let a = self.inner.read(path, slab).map_err(js_err)?; + let o = Object::new(); + set(&o, "shape", shape_to_js(&a.shape)); + set(&o, "dtype", dtype); + set(&o, "data", data_to_js(a.data)); + Ok(o) + } +} diff --git a/crates/clawhdf5-wasm/tests/h5py_interop.rs b/crates/clawhdf5-wasm/tests/h5py_interop.rs new file mode 100644 index 0000000..7d71a92 --- /dev/null +++ b/crates/clawhdf5-wasm/tests/h5py_interop.rs @@ -0,0 +1,252 @@ +//! The wasm reader's core against files written by h5py and netCDF4, with +//! the values libhdf5 reads back as the reference. The generator, +//! `examples/wasm-viewer/test/make_fixture.py`, is shared with the Node test +//! of the built wasm package, so both compare against the same expectations. +//! +//! Skipped when python3 with h5py/netCDF4 is missing, unless +//! `CLAWHDF5_REQUIRE_INTEROP=1`. `CLAWHDF5_PYTHON` names the interpreter. + +use std::path::{Path, PathBuf}; +use std::process::Command; + +use clawhdf5::AttrValue; +use clawhdf5_wasm::core::{Data, Hyperslab, Kind, Reader}; +use serde_json::Value; + +fn python() -> String { + std::env::var("CLAWHDF5_PYTHON").unwrap_or_else(|_| "python3".to_string()) +} + +fn interop_required() -> bool { + std::env::var("CLAWHDF5_REQUIRE_INTEROP").is_ok_and(|v| v == "1") +} + +fn python_available() -> bool { + Command::new(python()) + .args(["-c", "import h5py, netCDF4, numpy"]) + .output() + .map(|o| o.status.success()) + .unwrap_or(false) +} + +fn generator() -> PathBuf { + Path::new(env!("CARGO_MANIFEST_DIR")).join("../../examples/wasm-viewer/test/make_fixture.py") +} + +/// Values as comparable strings: integers exactly, floats by their f64 +/// value (an f32 widens exactly), strings as themselves. +fn data_strings(d: &Data) -> (&'static str, Vec) { + fn s(v: &[T]) -> Vec { + v.iter().map(ToString::to_string).collect() + } + fn f>(v: &[T]) -> Vec { + v.iter().map(|&x| format!("{:?}", x.into())).collect() + } + match d { + Data::F32(v) => ("f32", f(v)), + Data::F64(v) => ("f64", f(v)), + Data::I8(v) => ("i8", s(v)), + Data::I16(v) => ("i16", s(v)), + Data::I32(v) => ("i32", s(v)), + Data::I64(v) => ("i64", s(v)), + Data::U8(v) => ("u8", s(v)), + Data::U16(v) => ("u16", s(v)), + Data::U32(v) => ("u32", s(v)), + Data::U64(v) => ("u64", s(v)), + Data::Strings(v) => ("strings", v.clone()), + } +} + +fn expected_strings(kind: &str, values: &Value) -> Vec { + values + .as_array() + .unwrap() + .iter() + .map(|v| match (kind, v) { + ("f32" | "f64", Value::Number(n)) => format!("{:?}", n.as_f64().unwrap()), + (_, Value::String(s)) => s.clone(), + other => panic!("unexpected expected value {other:?}"), + }) + .collect() +} + +fn shape(v: &Value) -> Vec { + v.as_array() + .unwrap() + .iter() + .map(|x| x.as_u64().unwrap()) + .collect() +} + +fn check_attr(file: &str, path: &str, name: &str, got: &AttrValue, want: &Value) { + let ctx = format!("{file}:{path}@{name}"); + let ints = |v: &Value| -> Vec { + v.as_array() + .unwrap() + .iter() + .map(|x| x.as_str().unwrap().to_string()) + .collect() + }; + let scalar = want["scalar"].as_bool().unwrap_or(false); + match got { + AttrValue::String(s) => assert_eq!(want["string"].as_str(), Some(s.as_str()), "{ctx}"), + AttrValue::StringArray(v) => { + let w: Vec<&str> = want["strings"] + .as_array() + .unwrap_or_else(|| panic!("{ctx}: got {v:?}")) + .iter() + .map(|x| x.as_str().unwrap()) + .collect(); + assert_eq!(v, &w, "{ctx}"); + } + AttrValue::I64(x) => { + assert!(scalar, "{ctx}"); + assert_eq!(vec![x.to_string()], ints(&want["int"]), "{ctx}"); + } + AttrValue::U64(x) => { + assert!(scalar, "{ctx}"); + assert_eq!(vec![x.to_string()], ints(&want["int"]), "{ctx}"); + } + AttrValue::I64Array(v) => assert_eq!( + v.iter().map(ToString::to_string).collect::>(), + ints(&want["int"]), + "{ctx}" + ), + AttrValue::U64Array(v) => assert_eq!( + v.iter().map(ToString::to_string).collect::>(), + ints(&want["int"]), + "{ctx}" + ), + AttrValue::F64(x) => { + assert!(scalar, "{ctx}"); + assert_eq!(Some(*x), want["float"][0].as_f64(), "{ctx}"); + } + AttrValue::F64Array(v) => { + let w: Vec = want["float"] + .as_array() + .unwrap() + .iter() + .map(|x| x.as_f64().unwrap()) + .collect(); + assert_eq!(v, &w, "{ctx}"); + } + AttrValue::Raw { datatype, .. } => panic!("{ctx}: undecoded {datatype:?}"), + } +} + +fn check_file(dir: &Path, file: &str, exp: &Value) { + let r = Reader::open(std::fs::read(dir.join(file)).unwrap()).unwrap(); + + for (path, want) in exp["lists"].as_object().unwrap() { + let list = r + .list(path) + .unwrap_or_else(|e| panic!("{file}:{path}: {e}")); + for (kind, key) in [(Kind::Group, "groups"), (Kind::Dataset, "datasets")] { + let mut got: Vec<&str> = list + .iter() + .filter(|c| c.kind == kind) + .map(|c| c.name.as_str()) + .collect(); + got.sort(); + let w: Vec<&str> = want[key] + .as_array() + .unwrap() + .iter() + .map(|x| x.as_str().unwrap()) + .collect(); + assert_eq!(got, w, "{file}:{path} {key}"); + } + } + + for (path, want) in exp["datasets"].as_object().unwrap() { + let kind = want["kind"].as_str().unwrap(); + let a = r + .read(path, None) + .unwrap_or_else(|e| panic!("{file}:{path}: {e}")); + assert_eq!(a.shape, shape(&want["shape"]), "{file}:{path} shape"); + let (got_kind, got) = data_strings(&a.data); + assert_eq!(got_kind, kind, "{file}:{path} kind"); + assert_eq!( + got, + expected_strings(kind, &want["values"]), + "{file}:{path}" + ); + + if let Some(slab) = want.get("slab") { + let h = Hyperslab { + start: shape(&slab["start"]), + count: shape(&slab["count"]), + stride: Some(shape(&slab["stride"])), + block: None, + }; + let a = r + .read(path, Some(&h)) + .unwrap_or_else(|e| panic!("{file}:{path} {h:?}: {e}")); + assert_eq!(a.shape, shape(&slab["shape"]), "{file}:{path} slab shape"); + assert_eq!( + data_strings(&a.data).1, + expected_strings(kind, &slab["values"]), + "{file}:{path} slab" + ); + } + } + + for (path, what) in exp["errors"].as_object().unwrap() { + let e = r.read(path, None).expect_err(path); + assert!(e.contains(what.as_str().unwrap()), "{file}:{path}: {e}"); + } + + let skip: Vec<&str> = exp["skip_attrs"] + .as_array() + .unwrap() + .iter() + .map(|x| x.as_str().unwrap()) + .collect(); + for (path, want) in exp["attrs"].as_object().unwrap() { + let (attrs, errors) = r + .attrs(path) + .unwrap_or_else(|e| panic!("{file}:{path}: {e}")); + assert!(errors.is_empty(), "{file}:{path}: {errors:?}"); + let want = want.as_object().unwrap(); + let mut compared = 0; + for a in &attrs { + if a.name.starts_with('_') || skip.contains(&a.name.as_str()) { + continue; + } + let w = want + .get(&a.name) + .unwrap_or_else(|| panic!("{file}:{path}: unexpected attribute {}", a.name)); + check_attr(file, path, &a.name, &a.value, w); + compared += 1; + } + assert_eq!(compared, want.len(), "{file}:{path}: attributes missing"); + } +} + +#[test] +fn reads_what_h5py_and_netcdf4_wrote() { + if !python_available() { + assert!( + !interop_required(), + "CLAWHDF5_REQUIRE_INTEROP=1 but python with h5py, netCDF4 and numpy is not available" + ); + eprintln!("SKIP: python with h5py/netCDF4 not available"); + return; + } + let dir = tempfile::tempdir().unwrap(); + let out = Command::new(python()) + .arg(generator()) + .arg(dir.path()) + .output() + .expect("run python"); + assert!( + out.status.success(), + "fixture generator failed:\n{}", + String::from_utf8_lossy(&out.stderr) + ); + let exp: Value = + serde_json::from_slice(&std::fs::read(dir.path().join("expected.json")).unwrap()).unwrap(); + for file in ["fixture.h5", "fixture.nc"] { + check_file(dir.path(), file, &exp[file]); + } +} diff --git a/examples/wasm-viewer/test/make_fixture.py b/examples/wasm-viewer/test/make_fixture.py new file mode 100644 index 0000000..50b893b --- /dev/null +++ b/examples/wasm-viewer/test/make_fixture.py @@ -0,0 +1,184 @@ +"""Write HDF5 and NetCDF-4 test files with h5py/netCDF4, and what libhdf5 +reads back from them, for the clawhdf5-wasm tests. + + python make_fixture.py OUT_DIR + +writes OUT_DIR/fixture.h5, OUT_DIR/fixture.nc and OUT_DIR/expected.json. +Both the Rust test (crates/clawhdf5-wasm/tests/h5py_interop.rs, native) and +the Node test (test.mjs, the built wasm package) compare against the same +expected.json, so the two check the same values. + +Every expected value comes from h5py reading the file back (numpy slicing +for hyperslabs), never from the arrays that were written. Integers are +encoded as strings so JSON.parse keeps 64-bit values exact. +""" + +import json +import sys +import warnings +from pathlib import Path + +import h5py +import netCDF4 +import numpy as np + +# netCDF4 1.7 trips numpy 2.5's shape-setting deprecation on assignment. +warnings.filterwarnings("ignore", category=DeprecationWarning) + +out = Path(sys.argv[1]) +out.mkdir(parents=True, exist_ok=True) +h5 = out / "fixture.h5" +nc = out / "fixture.nc" + +rng = np.random.default_rng(7) +with h5py.File(h5, "w") as f: + f.attrs["title"] = "wasm fixture" + f.attrs["version"] = np.int64(3) + f.attrs["scale"] = np.array([0.5, 2.0]) + f.attrs["big"] = np.uint64(2**63 + 5) + f.attrs.create("vlen_note", "héllo", dtype=h5py.string_dtype()) + f.create_dataset( + "grid", data=np.arange(60, dtype="f4")) + f.create_dataset("f16", data=np.array([0.5, -1.25, 65504], dtype=" 2 else 1] + [2] * (obj.ndim - 1) + count = [min(c, (n - s - 1) // st + 1) + for c, s, st, n in zip(count, start, stride, obj.shape)] + return (start, count, stride) + + +json.dump({"fixture.h5": describe(h5), "fixture.nc": describe(nc)}, + open(out / "expected.json", "w"), indent=1, ensure_ascii=False) diff --git a/scripts/ci-test.sh b/scripts/ci-test.sh index 6cad171..9b93039 100755 --- a/scripts/ci-test.sh +++ b/scripts/ci-test.sh @@ -92,7 +92,8 @@ run_step "cargo clippy (fast-deflate / zlib-ng)" cargo clippy \ no_c_in_default_build() { local crate found=0 for crate in clawhdf5-format clawhdf5-io clawhdf5-filters clawhdf5 \ - clawhdf5-agent clawhdf5-ann clawhdf5-accel clawhdf5-netcdf4 clawhdf5-cli; do + clawhdf5-agent clawhdf5-ann clawhdf5-accel clawhdf5-netcdf4 clawhdf5-cli \ + clawhdf5-wasm; do local c_deps c_deps=$(cargo tree -q -p "$crate" -e normal,build --prefix none \ | grep -E '^([a-z0-9_-]+-sys|cc|cmake) v' | sort -u) @@ -107,12 +108,18 @@ no_c_in_default_build() { run_step "no C in the default build (core crates)" no_c_in_default_build # The reader in the browser: the facade's read path must build for -# wasm32-unknown-unknown (no mmap, no threads, no file system). Needs +# wasm32-unknown-unknown (no mmap, no threads, no file system), and the +# wasm-bindgen crate must build and lint there. Needs # `rustup target add wasm32-unknown-unknown`. run_step "wasm32 build (clawhdf5, no default features)" cargo build \ -p clawhdf5 \ --target wasm32-unknown-unknown \ --no-default-features +run_step "wasm32 clippy (clawhdf5-wasm)" cargo clippy \ + -p clawhdf5-wasm \ + --target wasm32-unknown-unknown \ + --all-targets \ + -- -D warnings # The workspace declares a minimum Rust version (rust-version in Cargo.toml); # check that it really builds there, so the README badge and the manifests From 1abd93e0f8c157428d823e059b62cf0432a62073 Mon Sep 17 00:00:00 2001 From: osobh Date: Sat, 26 Sep 2026 00:05:34 -0500 Subject: [PATCH 4/7] feat(wasm): examples/wasm-viewer, an HDF5/NetCDF-4 viewer page Drop a file (or pass ?file=&path=), browse the tree lazily, see a dataset's type, shape, max shape and attributes, and page through its values as 50x12 hyperslab windows (leading dims of 3-D+ data held at chosen indices). build.sh produces pkg/ (not committed) with wasm-bindgen --target web and checks the CLI matches the crate version. test/run.sh builds it and runs test.mjs under Node against the h5py/ netCDF4 fixture (250 checks: every dataset whole and as a strided hyperslab, listings, attributes, error paths, the page's DOM-free helpers), then browser.sh renders the page in headless Chromium for eight objects and checks the DOM. The fixture gains LZ4 (read) and Zstd (refused: links C) datasets and a compound attribute (value null plus its type). ci-test.sh runs it when node and wasm-bindgen exist; the CI container has neither, so CI relies on the native h5py_interop test. Co-Authored-By: Claude Opus 5.5 (1M context) --- crates/clawhdf5-wasm/tests/h5py_interop.rs | 8 +- examples/wasm-viewer/.gitignore | 2 + examples/wasm-viewer/README.md | 97 +++++++ examples/wasm-viewer/build.sh | 39 +++ examples/wasm-viewer/index.html | 278 +++++++++++++++++++++ examples/wasm-viewer/test/browser.sh | 96 +++++++ examples/wasm-viewer/test/make_fixture.py | 17 ++ examples/wasm-viewer/test/run.sh | 37 +++ examples/wasm-viewer/test/test.mjs | 137 ++++++++++ examples/wasm-viewer/viewer-lib.js | 77 ++++++ scripts/ci-test.sh | 13 + 11 files changed, 800 insertions(+), 1 deletion(-) create mode 100644 examples/wasm-viewer/.gitignore create mode 100644 examples/wasm-viewer/README.md create mode 100755 examples/wasm-viewer/build.sh create mode 100644 examples/wasm-viewer/index.html create mode 100644 examples/wasm-viewer/test/browser.sh create mode 100755 examples/wasm-viewer/test/run.sh create mode 100644 examples/wasm-viewer/test/test.mjs create mode 100644 examples/wasm-viewer/viewer-lib.js diff --git a/crates/clawhdf5-wasm/tests/h5py_interop.rs b/crates/clawhdf5-wasm/tests/h5py_interop.rs index 7d71a92..a70d8bd 100644 --- a/crates/clawhdf5-wasm/tests/h5py_interop.rs +++ b/crates/clawhdf5-wasm/tests/h5py_interop.rs @@ -130,7 +130,13 @@ fn check_attr(file: &str, path: &str, name: &str, got: &AttrValue, want: &Value) .collect(); assert_eq!(v, &w, "{ctx}"); } - AttrValue::Raw { datatype, .. } => panic!("{ctx}: undecoded {datatype:?}"), + AttrValue::Raw { datatype, .. } => { + let want = want["raw"] + .as_str() + .unwrap_or_else(|| panic!("{ctx}: undecoded {datatype:?}")); + let d = clawhdf5_wasm::core::describe(datatype); + assert!(d.contains(want), "{ctx}: {d}"); + } } } diff --git a/examples/wasm-viewer/.gitignore b/examples/wasm-viewer/.gitignore new file mode 100644 index 0000000..f57bf2b --- /dev/null +++ b/examples/wasm-viewer/.gitignore @@ -0,0 +1,2 @@ +# Generated by build.sh. +/pkg/ diff --git a/examples/wasm-viewer/README.md b/examples/wasm-viewer/README.md new file mode 100644 index 0000000..1a5c47d --- /dev/null +++ b/examples/wasm-viewer/README.md @@ -0,0 +1,97 @@ +# HDF5 viewer in the browser + +A single page that opens an HDF5 or NetCDF-4 file entirely in the browser +with `clawhdf5-wasm` (clawhdf5's reader compiled to WebAssembly): drop a +file, browse its groups, and look at a dataset's type, shape, attributes +and values (a 50 x 12 window at a time, read as a hyperslab, with the +leading dimensions of a 3-D+ dataset held at chosen indices). The file never +leaves the page. + +## Build and open + +```bash +rustup target add wasm32-unknown-unknown +cargo install wasm-bindgen-cli --version 0.2.129 # must equal the crate version; build.sh checks +bash examples/wasm-viewer/build.sh # writes examples/wasm-viewer/pkg/ (not committed) +python3 -m http.server -d examples/wasm-viewer 8000 # wasm cannot load from file:// +``` + +Then open . `?file=&path=` opens a +file from a URL (same origin, or one serving CORS headers) and selects an +object in it, e.g. `?file=data/run1.h5&path=/results/energy`. + +## JavaScript API + +```js +import init, { open } from "./pkg/clawhdf5_wasm.js"; +await init(); +const f = open(new Uint8Array(await blob.arrayBuffer())); +f.list("/"); // [{ name, kind: "group" | "dataset" }], groups first +f.info("/grid"); // { shape, maxshape, dtype, elementShape } +f.attrs("/grid"); // [{ name, value, dtype }] +f.read("/grid"); // { shape, dtype, data } +f.readHyperslab("/grid", [0, 0], [10, 5], [2, 1]); // start, count, stride?, block? +f.free(); +``` + +`data` is the typed array of the stored width (`Float64Array`, +`Float32Array` also for `f16`, `Int8Array` ... `BigInt64Array`, +`BigUint64Array`), or an array of strings for fixed- and variable-length +strings and enumerations (h5py booleans read as `"TRUE"`/`"FALSE"`). Array +datatypes are flattened, their dimensions appended to `shape`. Anything +else throws an `Error` naming the type. + +## Limits + +- Read-only, and the whole file is held in memory (no range requests). +- Compound, reference, opaque and variable-length-sequence datasets are + refused with an error. Attributes of those types are listed with + `value: null` and their `dtype`. +- No Zstd or SZIP filters (they link C): such a dataset fails with + `unsupported filter`. Deflate, shuffle, Fletcher-32, LZ4, N-Bit and + scale-offset are read (within the limits in `docs/known-issues.md`). +- Virtual datasets whose sources are in other files, and external links, + cannot be followed: there is no file system. + +## Tests + +`test/run.sh` builds the package, writes `fixture.h5` (h5py) and +`fixture.nc` (netCDF4) with `test/make_fixture.py`, then: + +- runs `test/test.mjs` under Node: every dataset (whole and a strided + hyperslab), listing and attribute is compared with what libhdf5 reads + back, error paths are checked, and so are the page's DOM-free helpers + (`viewer-lib.js`); +- runs `test/browser.sh`: loads the page in headless Chromium with + `?file=fixture.h5&path=...` for eight objects and checks the rendered tree, + types, shapes, attribute and value cells, and the error shown for an + unsupported type. Skipped when no Chromium is found (`CHROME` names one; + a Playwright download under `~/.cache/ms-playwright` is picked up). + Drag-and-drop and the file picker are not driven by it; they share + `load()` with the `?file=` path. + +The same expectations are checked natively, without Node, by +`crates/clawhdf5-wasm/tests/h5py_interop.rs`, which is what CI runs (the CI +container has no Node or browser). + +## Size + +Measured 2026-09-26 on tank (rustc 1.98.1, wasm-bindgen 0.2.129, gzip 1.14, +`gzip -9 -n`), after `bash examples/wasm-viewer/build.sh`: + +| | raw | gzip -9 | +|---|---:|---:| +| `pkg/clawhdf5_wasm_bg.wasm` (profile `wasm-release`, opt-level `s`) | 627,501 B | 191,639 B | +| `pkg/clawhdf5_wasm.js` (wasm-bindgen glue) | 21,826 B | 4,487 B | +| same wasm at opt-level `z` | 693,068 B | 192,550 B | +| same wasm at opt-level `3` | 544,035 B | 198,803 B | +| h5wasm 0.10.3: wasm embedded in `dist/esm/hdf5_util.js` | 3,544,184 B | 907,096 B | +| h5wasm 0.10.3: `dist/esm/hdf5_util.js` as shipped | 4,150,134 B | 986,699 B | + +h5wasm figures: `npm pack h5wasm@0.10.3` (npm reports +`dist.unpackedSize` 14,731,385 B for the whole package), wasm extracted from +the `binaryDecode` literal in `hdf5_util.js`. h5wasm is the whole of libhdf5 +(writing, every datatype, plugins), so this compares download size, not +equal functionality. No `wasm-opt` pass was applied (binaryen is not +installed on tank). opt-level `s` is used because it is the smallest +compressed. diff --git a/examples/wasm-viewer/build.sh b/examples/wasm-viewer/build.sh new file mode 100755 index 0000000..6af81bf --- /dev/null +++ b/examples/wasm-viewer/build.sh @@ -0,0 +1,39 @@ +#!/usr/bin/env bash +# Build the clawhdf5-wasm package the viewer loads, into examples/wasm-viewer/pkg/. +# +# Needs the wasm32-unknown-unknown target and the wasm-bindgen CLI at the +# exact version cargo resolves for the wasm-bindgen crate: +# rustup target add wasm32-unknown-unknown +# cargo install wasm-bindgen-cli --version +# +# Then serve this directory over HTTP (browsers do not load wasm modules from +# file://) and open it: +# python3 -m http.server -d examples/wasm-viewer 8000 +set -euo pipefail + +HERE="$(cd "$(dirname "$0")" && pwd)" +ROOT="$(cd "$HERE/../.." && pwd)" +cd "$ROOT" + +# The resolved crate version (Cargo.lock is not committed, so ask cargo). +want=$(cargo pkgid wasm-bindgen | sed 's/.*[@#]//') +if ! command -v wasm-bindgen >/dev/null; then + echo "wasm-bindgen CLI not found: cargo install wasm-bindgen-cli --version $want" >&2 + exit 1 +fi +have=$(wasm-bindgen --version | awk '{print $2}') +if [ "$want" != "$have" ]; then + echo "wasm-bindgen CLI is $have but the crate is $want:" >&2 + echo " cargo install wasm-bindgen-cli --version $want" >&2 + exit 1 +fi + +cargo build -p clawhdf5-wasm --target wasm32-unknown-unknown --profile wasm-release + +target_dir=$(cargo metadata --format-version 1 --no-deps \ + | sed -n 's/.*"target_directory":"\([^"]*\)".*/\1/p') +wasm="$target_dir/wasm32-unknown-unknown/wasm-release/clawhdf5_wasm.wasm" + +rm -rf "$HERE/pkg" +wasm-bindgen --target web --out-dir "$HERE/pkg" "$wasm" +echo "built $HERE/pkg ($(wc -c < "$HERE/pkg/clawhdf5_wasm_bg.wasm") bytes of wasm)" diff --git a/examples/wasm-viewer/index.html b/examples/wasm-viewer/index.html new file mode 100644 index 0000000..d477e49 --- /dev/null +++ b/examples/wasm-viewer/index.html @@ -0,0 +1,278 @@ + + + + + +HDF5 Viewer + + + +
+

HDF5 Viewer

+ no file + +
+
+ +
+
+

Drop an HDF5 or NetCDF-4 file here, or use “Open file…”.

+

The file is read in this page by clawhdf5 compiled to WebAssembly; it is not uploaded anywhere.

+

+
+
+
+ + + diff --git a/examples/wasm-viewer/test/browser.sh b/examples/wasm-viewer/test/browser.sh new file mode 100644 index 0000000..a9ea5aa --- /dev/null +++ b/examples/wasm-viewer/test/browser.sh @@ -0,0 +1,96 @@ +#!/usr/bin/env bash +# Load the viewer page in headless Chromium and check what it renders. +# +# browser.sh FIXTURE_DIR +# +# FIXTURE_DIR holds fixture.h5 from make_fixture.py; ../pkg must be built. +# The page is opened with ?file=fixture.h5&path=, which fetches the +# file, builds the tree down to and shows it; the rendered DOM is +# dumped and checked for the values libhdf5 reads. +# +# Browser: $CHROME, else chromium/google-chrome on PATH, else a Playwright +# download under ~/.cache/ms-playwright. Exit 3 when none is found. +# BROWSER_DEBUG=/some/prefix saves each rendered page as prefix..html. +set -euo pipefail + +HERE="$(cd "$(dirname "$0")" && pwd)" +FIX="$(cd "$1" && pwd)" +PY="${CLAWHDF5_PYTHON:-python3}" + +chrome="${CHROME:-}" +if [ -z "$chrome" ]; then + for c in chromium chromium-browser google-chrome chrome-headless-shell; do + if command -v "$c" >/dev/null; then chrome="$(command -v "$c")"; break; fi + done +fi +if [ -z "$chrome" ]; then + chrome="$(ls -d "$HOME"/.cache/ms-playwright/chromium_headless_shell-*/chrome-headless-shell-linux64/chrome-headless-shell 2>/dev/null | tail -1 || true)" +fi +if [ -z "$chrome" ] || [ ! -x "$chrome" ]; then + echo "no Chromium found (set CHROME)" >&2 + exit 3 +fi + +root="$(mktemp -d)" +server="" +cleanup() { + [ -n "$server" ] && kill "$server" 2>/dev/null || true + rm -rf "$root" +} +trap cleanup EXIT +ln -s "$HERE/../index.html" "$HERE/../viewer-lib.js" "$HERE/../pkg" "$FIX/fixture.h5" "$root/" + +port=$("$PY" -c 'import socket; s = socket.socket(); s.bind(("127.0.0.1", 0)); print(s.getsockname()[1])') +"$PY" -m http.server --bind 127.0.0.1 --directory "$root" "$port" >/dev/null 2>&1 & +server=$! +for _ in $(seq 50); do + "$PY" -c "import urllib.request; urllib.request.urlopen('http://127.0.0.1:$port/index.html')" 2>/dev/null && break + sleep 0.1 +done + +fails=0 +# A fresh profile per page: a second instance on the same profile fails. +render() { + local profile + profile="$(mktemp -d "$root/profile.XXXXXX")" + "$chrome" --headless --no-sandbox --disable-gpu --user-data-dir="$profile" \ + --virtual-time-budget=20000 \ + --dump-dom "http://127.0.0.1:$port/index.html?file=fixture.h5&path=$1" 2>/dev/null +} +# expect PATH TEXT...: every TEXT appears in the page rendered for PATH. +expect() { + local path="$1" dom + shift + dom="$(render "$path")" + [ -n "${BROWSER_DEBUG:-}" ] && printf "%s\n" "$dom" > "$BROWSER_DEBUG.$(echo "$path" | tr / _).html" + for text in "$@"; do + if ! grep -qF -- "$text" <<<"$dom"; then + echo "FAIL: page for $path lacks: $text" >&2 + fails=$((fails + 1)) + fi + done + echo "rendered $path" +} + +# Tree (root expanded; the group row carries its path) and root attributes. +expect "/" 'data-path="/sensors"' 'data-path="/grid"' 'title"wasm fixture"' \ + 'big9223372036854775813' '(compound{x: f64, n: i32})' +# A chunked, deflated 2-D dataset: type, shape, the first window of values. +expect "/grid" '
f64
' '
(6, 10)
' '9' '0.25' '14.75' \ + 'showing rows 0–5, columns 0–9 of 60 values' +# Nested path revealed through the tree; big-endian float32. +expect "/sensors/temp" 'data-path="/sensors/temp"' '
f32
' '21.5' '22.25' +# 64-bit integers stay exact; strings; array datatype cells. +expect "/u64" '18446744073709551615' +expect "/vlen_str" '"двa"' '
vlen string
' +expect "/pairs" '[2, 3]' '
array[2]<i32>
' +# 3-D: leading dimension held at 0, window over the last two. +expect "/cube" '
(2, 5, 6)
' '29' 'dim 0' +# Unsupported type: an error, not values. +expect "/table" 'class="error"' 'reading compound{x: f64, n: i32} datasets is not supported' + +if [ "$fails" -gt 0 ]; then + echo "browser: $fails checks failed" >&2 + exit 1 +fi +echo "browser: all checks passed ($chrome)" diff --git a/examples/wasm-viewer/test/make_fixture.py b/examples/wasm-viewer/test/make_fixture.py index 50b893b..c8d023a 100644 --- a/examples/wasm-viewer/test/make_fixture.py +++ b/examples/wasm-viewer/test/make_fixture.py @@ -22,6 +22,11 @@ import h5py import netCDF4 import numpy as np +try: # registers the LZ4/Zstd filters with libhdf5; optional + import hdf5plugin +except ImportError: + hdf5plugin = None + # netCDF4 1.7 trips numpy 2.5's shape-setting deprecation on assignment. warnings.filterwarnings("ignore", category=DeprecationWarning) @@ -37,6 +42,8 @@ with h5py.File(h5, "w") as f: f.attrs["scale"] = np.array([0.5, 2.0]) f.attrs["big"] = np.uint64(2**63 + 5) f.attrs.create("vlen_note", "héllo", dtype=h5py.string_dtype()) + # No plain JavaScript form: listed with value null and its type. + f.attrs["origin"] = np.array((1.5, 2), dtype=[("x", "/dev/null || { echo "node not found" >&2; exit 1; } +if ! "$PY" -c "import h5py, netCDF4, numpy" >/dev/null 2>&1; then + if [ "${CLAWHDF5_REQUIRE_INTEROP:-0}" = "1" ]; then + echo "CLAWHDF5_REQUIRE_INTEROP=1 but $PY lacks h5py/netCDF4/numpy" >&2 + exit 1 + fi + echo "SKIP: $PY lacks h5py/netCDF4/numpy" + exit 0 +fi + +bash "$HERE/../build.sh" +fix="$(mktemp -d)" +trap 'rm -rf "$fix"' EXIT +"$PY" "$HERE/make_fixture.py" "$fix" +node "$HERE/test.mjs" "$HERE/../pkg" "$fix" + +# The page itself, in headless Chromium when one is available. +status=0 +bash "$HERE/browser.sh" "$fix" || status=$? +if [ "$status" = 3 ]; then + echo "SKIP: viewer page in a browser (no Chromium; set CHROME)" +elif [ "$status" != 0 ]; then + exit "$status" +fi diff --git a/examples/wasm-viewer/test/test.mjs b/examples/wasm-viewer/test/test.mjs new file mode 100644 index 0000000..1386f4c --- /dev/null +++ b/examples/wasm-viewer/test/test.mjs @@ -0,0 +1,137 @@ +// Node test of the built wasm package (the exact pkg/ the viewer page loads) +// and the viewer's DOM-free helpers. Run by test/run.sh: +// node test.mjs PKG_DIR FIXTURE_DIR +// FIXTURE_DIR holds fixture.h5, fixture.nc and expected.json from +// make_fixture.py (values as libhdf5 reads them back). +import assert from "node:assert/strict"; +import { readFileSync } from "node:fs"; +import { join } from "node:path"; +import { pathToFileURL } from "node:url"; + +const [pkgDir, fixDir] = process.argv.slice(2); +const pkg = await import(pathToFileURL(join(pkgDir, "clawhdf5_wasm.js"))); +pkg.initSync({ module: readFileSync(join(pkgDir, "clawhdf5_wasm_bg.wasm")) }); +const lib = await import(pathToFileURL(join(import.meta.dirname, "..", "viewer-lib.js"))); + +let checks = 0; +const eq = (a, b, msg) => { assert.deepEqual(a, b, msg); checks++; }; + +const ARRAY_TYPES = { + f32: Float32Array, f64: Float64Array, i8: Int8Array, i16: Int16Array, i32: Int32Array, + i64: BigInt64Array, u8: Uint8Array, u16: Uint16Array, u32: Uint32Array, u64: BigUint64Array, + strings: Array, +}; + +function values(kind, data) { + const arr = Array.from(data); + if (kind === "f32" || kind === "f64" || kind === "strings") return arr; + return arr.map(String); +} + +function checkAttr(ctx, a, want) { + const v = a.value; + if ("raw" in want) { + eq(v, null, ctx); + assert.ok(a.dtype.includes(want.raw), `${ctx}: ${a.dtype}`); + return; + } + eq(a.dtype, null, `${ctx} dtype`); + if ("string" in want) return eq(v, want.string, ctx); + if ("strings" in want) return eq(v, want.strings, ctx); + if ("int" in want) { + if (want.scalar) { + assert.ok(typeof v === "number" || typeof v === "bigint", ctx); + return eq([String(v)], want.int, ctx); + } + assert.ok(v instanceof BigInt64Array || v instanceof BigUint64Array, ctx); + return eq(Array.from(v, String), want.int, ctx); + } + if ("float" in want) { + if (want.scalar) return eq([v], want.float, ctx); + assert.ok(v instanceof Float64Array, ctx); + return eq(Array.from(v), want.float, ctx); + } + assert.fail(`${ctx}: unknown expectation ${JSON.stringify(want)}`); +} + +const expected = JSON.parse(readFileSync(join(fixDir, "expected.json"), "utf8")); +for (const [name, exp] of Object.entries(expected)) { + const file = pkg.open(new Uint8Array(readFileSync(join(fixDir, name)))); + + for (const [path, want] of Object.entries(exp.lists)) { + eq(file.kind(path), "group", `${name}:${path} kind`); + const list = file.list(path); + for (const [kind, key] of [["group", "groups"], ["dataset", "datasets"]]) { + eq(list.filter((c) => c.kind === kind).map((c) => c.name).sort(), want[key], `${name}:${path} ${key}`); + } + } + + for (const [path, want] of Object.entries(exp.datasets)) { + const ctx = `${name}:${path}`; + eq(file.kind(path), "dataset", `${ctx} kind`); + const info = file.info(path); + eq([...info.shape, ...info.elementShape], want.shape, `${ctx} info shape`); + const r = file.read(path); + eq(r.shape, want.shape, `${ctx} shape`); + eq(r.dtype, info.dtype, `${ctx} dtype`); + assert.ok(r.data instanceof ARRAY_TYPES[want.kind], `${ctx}: ${r.data.constructor.name} for ${want.kind}`); + eq(values(want.kind, r.data), want.values, ctx); + if (want.slab) { + const s = want.slab; + const part = file.readHyperslab(path, s.start, s.count, s.stride); + eq(part.shape, s.shape, `${ctx} slab shape`); + eq(values(want.kind, part.data), s.values, `${ctx} slab`); + } + } + + for (const [path, what] of Object.entries(exp.errors)) { + assert.throws(() => file.read(path), (e) => e instanceof Error && e.message.includes(what), `${name}:${path}`); + checks++; + } + + for (const [path, want] of Object.entries(exp.attrs)) { + const attrs = file.attrs(path); + eq(file.attrErrors(path), [], `${name}:${path} attr errors`); + const seen = attrs.filter((a) => !a.name.startsWith("_") && !exp.skip_attrs.includes(a.name)); + eq(seen.map((a) => a.name).sort(), Object.keys(want).sort(), `${name}:${path} attr names`); + for (const a of seen) checkAttr(`${name}:${path}@${a.name}`, a, want[a.name]); + } + file.free(); +} + +// Errors reach JavaScript as thrown Errors, never as data. +const h5 = pkg.open(new Uint8Array(readFileSync(join(fixDir, "fixture.h5")))); +const throwsMsg = (fn, re) => { assert.throws(fn, (e) => e instanceof Error && re.test(e.message)); checks++; }; +throwsMsg(() => pkg.open(new Uint8Array(64)), /./); +throwsMsg(() => h5.read("/nope"), /./); +throwsMsg(() => h5.list("/grid"), /not a group/); +throwsMsg(() => h5.readHyperslab("/grid", [0], [1]), /dimensions/); +throwsMsg(() => h5.readHyperslab("/grid", [5, 0], [2, 1]), /exceeds/); +throwsMsg(() => h5.readHyperslab("/grid", [-1, 0], [1, 1]), /non-negative integers/); +throwsMsg(() => h5.readHyperslab("/grid", [0.5, 0], [1, 1]), /non-negative integers/); +// Info for a dataset with an unlimited dimension (netCDF "time"). +const nc = pkg.open(new Uint8Array(readFileSync(join(fixDir, "fixture.nc")))); +eq(nc.info("/time").maxshape, [null], "unlimited dimension is null"); +// Big integers stay exact. +eq(h5.read("/u64").data[0], 18446744073709551615n, "u64 max"); +eq(typeof pkg.version(), "string", "version"); + +// Viewer helpers. +eq(lib.joinPath("/", "a"), "/a", "joinPath root"); +eq(lib.joinPath("/a", "b"), "/a/b", "joinPath nested"); +eq(lib.viewWindow([], {}), null, "scalar window"); +eq(lib.viewWindow([7], { row: 5, rows: 50 }), { start: [5], count: [2], rows: 2, cols: 1, row: 5, col: 0 }, "1-D window"); +const w = lib.viewWindow([2, 5, 6], { row: 1, col: 4, rows: 3, cols: 5, fixed: [1] }); +eq(w, { start: [1, 1, 4], count: [1, 3, 2], rows: 3, cols: 2, row: 1, col: 4 }, "3-D window"); +// The window the page would request reads the same values as a direct slab. +const cube = h5.readHyperslab("/cube", w.start, w.count); +eq(lib.toRows(cube.data, w.rows, w.cols), [["40", "41"], ["46", "47"], ["52", "53"]], "cube window cells"); +const pairs = h5.readHyperslab("/pairs", [1], [2]); +eq(lib.toRows(pairs.data, 2, 1, lib.perElement([2])), [["[2, 3]"], ["[4, 5]"]], "array-type cells"); +eq(lib.formatValue(0.1 + 0.2), "0.3", "float formatting"); +eq(lib.formatValue(2n ** 64n - 1n), "18446744073709551615", "bigint formatting"); +eq(lib.formatValue("x"), '"x"', "string formatting"); +h5.free(); +nc.free(); + +console.log(`wasm package: ${checks} checks passed`); diff --git a/examples/wasm-viewer/viewer-lib.js b/examples/wasm-viewer/viewer-lib.js new file mode 100644 index 0000000..72c5bf3 --- /dev/null +++ b/examples/wasm-viewer/viewer-lib.js @@ -0,0 +1,77 @@ +// DOM-free helpers for the viewer, so Node can test them (test/test.mjs). + +/** Child path of `name` in the group at `parent`. */ +export function joinPath(parent, name) { + return parent === "/" ? `/${name}` : `${parent}/${name}`; +} + +/** One value as display text. */ +export function formatValue(v) { + if (v === null || v === undefined) return ""; + if (typeof v === "bigint") return v.toString(); + if (typeof v === "number") { + if (Number.isInteger(v)) return String(v); + return String(Number(v.toPrecision(7))); + } + if (typeof v === "string") return JSON.stringify(v); + if (ArrayBuffer.isView(v) || Array.isArray(v)) { + const items = Array.from(v.slice(0, 16), formatValue); + if (v.length > 16) items.push(`… (${v.length} values)`); + return `[${items.join(", ")}]`; + } + return String(v); +} + +/** + * The window of a dataset to show: a hyperslab over its `shape` with the + * last dimension as columns, the one before as rows, and any leading + * dimensions held at `fixed` indices. `row`/`col` are the window's top-left + * corner. Returns null for a scalar (read it whole). + */ +export function viewWindow(shape, { row = 0, col = 0, rows = 50, cols = 12, fixed = [] } = {}) { + const rank = shape.length; + if (rank === 0) return null; + const clamp = (x, n) => Math.max(0, Math.min(x, Math.max(0, n - 1))); + if (rank === 1) { + const r0 = clamp(row, shape[0]); + const n = Math.max(0, Math.min(rows, shape[0] - r0)); + return { start: [r0], count: [n], rows: n, cols: 1, row: r0, col: 0 }; + } + const lead = shape.slice(0, rank - 2).map((n, i) => clamp(fixed[i] ?? 0, n)); + const nr = shape[rank - 2]; + const nc = shape[rank - 1]; + const r0 = clamp(row, nr); + const c0 = clamp(col, nc); + const r = Math.max(0, Math.min(rows, nr - r0)); + const c = Math.max(0, Math.min(cols, nc - c0)); + return { + start: [...lead, r0, c0], + count: [...lead.map(() => 1), r, c], + rows: r, + cols: c, + row: r0, + col: c0, + }; +} + +/** + * Split row-major `data` into `rows` x `cols` cells of display text; each + * cell holds `per` consecutive values (the elements of an array datatype). + */ +export function toRows(data, rows, cols, per = 1) { + const out = []; + for (let r = 0; r < rows; r++) { + const row = []; + for (let c = 0; c < cols; c++) { + const i = (r * cols + c) * per; + row.push(per === 1 ? formatValue(data[i]) : formatValue(data.slice(i, i + per))); + } + out.push(row); + } + return out; +} + +/** Number of values an array datatype packs into each element. */ +export function perElement(elementShape) { + return elementShape.reduce((a, b) => a * b, 1); +} diff --git a/scripts/ci-test.sh b/scripts/ci-test.sh index 9b93039..52e3548 100755 --- a/scripts/ci-test.sh +++ b/scripts/ci-test.sh @@ -121,6 +121,19 @@ run_step "wasm32 clippy (clawhdf5-wasm)" cargo clippy \ --all-targets \ -- -D warnings +# The built wasm package, run under Node against h5py/netCDF4-written files, +# and the viewer page in headless Chromium when one is found. +# Needs node and the wasm-bindgen CLI, which the CI container does not have; +# the same expectations are checked natively by clawhdf5-wasm's h5py_interop +# test in the cargo test step. +if command -v node >/dev/null && command -v wasm-bindgen >/dev/null; then + run_step "wasm package under Node (+ browser)" bash "$SCRIPT_DIR/../examples/wasm-viewer/test/run.sh" +else + echo "" + echo "==> [wasm package under Node] SKIPPED: needs node and wasm-bindgen" + STEPS+=("SKIP: wasm package under Node") +fi + # The workspace declares a minimum Rust version (rust-version in Cargo.toml); # check that it really builds there, so the README badge and the manifests # cannot drift from the truth. Separate target dir: a different toolchain From b58d61cfb7a6e6964ed576f975373840f6fe4bfc Mon Sep 17 00:00:00 2001 From: osobh Date: Sat, 26 Sep 2026 00:06:26 -0500 Subject: [PATCH 5/7] test(wasm): accept a zstd read when the build has the filter cargo test --workspace unifies clawhdf5-format/zstd on (another member enables it), so the native interop test read the Zstd dataset that the wasm build refuses. The fixture now records its values plus the error the wasm build must give; the native test accepts either, the Node test of the real wasm package still requires the error. Co-Authored-By: Claude Opus 5.5 (1M context) --- crates/clawhdf5-wasm/tests/h5py_interop.rs | 13 ++++++++++--- examples/wasm-viewer/test/make_fixture.py | 6 ++++-- examples/wasm-viewer/test/test.mjs | 6 ++++++ 3 files changed, 20 insertions(+), 5 deletions(-) diff --git a/crates/clawhdf5-wasm/tests/h5py_interop.rs b/crates/clawhdf5-wasm/tests/h5py_interop.rs index a70d8bd..7ef60ab 100644 --- a/crates/clawhdf5-wasm/tests/h5py_interop.rs +++ b/crates/clawhdf5-wasm/tests/h5py_interop.rs @@ -166,9 +166,16 @@ fn check_file(dir: &Path, file: &str, exp: &Value) { for (path, want) in exp["datasets"].as_object().unwrap() { let kind = want["kind"].as_str().unwrap(); - let a = r - .read(path, None) - .unwrap_or_else(|e| panic!("{file}:{path}: {e}")); + let a = match (r.read(path, None), want["unavailable"].as_str()) { + (Ok(a), _) => a, + // A filter this build may lack (zstd: the wasm build has it + // off, a workspace build may unify it on) must fail clearly. + (Err(e), Some(why)) => { + assert!(e.contains(why), "{file}:{path}: {e}"); + continue; + } + (Err(e), None) => panic!("{file}:{path}: {e}"), + }; assert_eq!(a.shape, shape(&want["shape"]), "{file}:{path} shape"); let (got_kind, got) = data_strings(&a.data); assert_eq!(got_kind, kind, "{file}:{path} kind"); diff --git a/examples/wasm-viewer/test/make_fixture.py b/examples/wasm-viewer/test/make_fixture.py index c8d023a..8c7722c 100644 --- a/examples/wasm-viewer/test/make_fixture.py +++ b/examples/wasm-viewer/test/make_fixture.py @@ -175,10 +175,12 @@ def describe(path): walk(key.rstrip("/") + "/" + n, o) elif obj.dtype.names: expected["errors"][key] = "compound" - elif key == "/zstd": - expected["errors"][key] = "unsupported filter: 32015" else: expected["datasets"][key] = entry(obj, slab_for(obj)) + if key == "/zstd": + # Refused by the wasm build (no zstd); a native build + # that unifies in clawhdf5-format/zstd reads it. + expected["datasets"][key]["unavailable"] = "unsupported filter: 32015" walk("/", f) return expected diff --git a/examples/wasm-viewer/test/test.mjs b/examples/wasm-viewer/test/test.mjs index 1386f4c..80ca30a 100644 --- a/examples/wasm-viewer/test/test.mjs +++ b/examples/wasm-viewer/test/test.mjs @@ -69,6 +69,12 @@ for (const [name, exp] of Object.entries(expected)) { for (const [path, want] of Object.entries(exp.datasets)) { const ctx = `${name}:${path}`; eq(file.kind(path), "dataset", `${ctx} kind`); + if (want.unavailable) { + // The wasm build has no zstd (it links C): a clear error, no data. + assert.throws(() => file.read(path), (e) => e.message.includes(want.unavailable), ctx); + checks++; + continue; + } const info = file.info(path); eq([...info.shape, ...info.elementShape], want.shape, `${ctx} info shape`); const r = file.read(path); From 34987ec194ef8c887d28745bb76a3e69d9ba4ca6 Mon Sep 17 00:00:00 2001 From: osobh Date: Sat, 26 Sep 2026 00:06:58 -0500 Subject: [PATCH 6/7] docs: clawhdf5-wasm in the changelog, known issues and CLAUDE.md Changelog entry with the sizes measured on tank on 2026-09-26 (and the h5wasm 0.10.3 comparison), the browser build's limits as a known-issues entry, and where its tests run. Co-Authored-By: Claude Opus 5.5 (1M context) --- CHANGELOG.md | 26 ++++++++++++++++++++++++++ CLAUDE.md | 8 ++++++++ docs/known-issues.md | 17 +++++++++++++++++ 3 files changed, 51 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1fc429d..7c44dbf 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -280,6 +280,32 @@ infinity; batches are all or nothing. CLI: `create --float16`. See `BENCHMARKS.md`, "float16 embedding storage". +### Browser (WebAssembly) +- **New crate `clawhdf5-wasm`:** the reader compiled to + `wasm32-unknown-unknown` with a wasm-bindgen JavaScript API — + `open(bytes)`, `list`, `info`, `attrs`, `read`, `readHyperslab` — returning + typed arrays of the stored width (`BigInt64Array` for 64-bit integers), + string arrays for strings and enums, and a thrown `Error` for types with no + typed-array form (compound, reference, opaque, VL sequences) or filters the + build lacks (Zstd, SZIP). Read-only; the file is held in memory. +- **`examples/wasm-viewer/`:** a drop-a-file HDF5/NetCDF-4 viewer page (tree, + type/shape/attributes, values paged as hyperslabs; `?file=&path=` opens a + URL). `build.sh` produces the package; `test/run.sh` checks it under Node + (251 checks against values h5py/libhdf5 read back from an h5py- and a + netCDF4-written file) and renders the page in headless Chromium. Size, + measured 2026-09-26 on tank (`gzip -9 -n`): 627,501 B of wasm, 191,639 B + gzipped, plus 21,826 B (4,487 B) of JS glue; h5wasm 0.10.3's embedded wasm + is 3,544,184 B (907,096 B) — full libhdf5, so not equal functionality. See + `examples/wasm-viewer/README.md`. +- The facade's read path already built for `wasm32-unknown-unknown` (nothing + needed gating); `ci-test.sh` now builds it (`--no-default-features`) and + lints `clawhdf5-wasm` for that target, and CI installs the target. The Node + and browser tests run in `ci-test.sh` only where `node` and `wasm-bindgen` + exist (not the CI container); CI checks the same expectations natively + (`clawhdf5-wasm`'s `h5py_interop` test). +- `Dataset::raw_datatype()` (facade) returns the full stored datatype, for + decoding `read_selection` bytes with `clawhdf5_format::data_read`. + ### Build - **Pure-Rust default.** `clawhdf5-format`, `clawhdf5-filters` and the `clawhdf5` facade default to the `zlib-rs` deflate backend; `fast-deflate` diff --git a/CLAUDE.md b/CLAUDE.md index 4969c6a..c21af88 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -150,6 +150,14 @@ Cargo workspace with 17 crates under `crates/` (plus `libaec-sys`, an internal F `MemorySource` for this bookkeeping is inferred from the caller-supplied `source_channel` string (a heuristic, not an authenticated trust boundary). - GPU-accelerated vector distance computation (`clawhdf5-gpu`, wgpu); HDF5 I/O itself is CPU-only +- Browser: `clawhdf5-wasm` (wasm-bindgen, read-only, file held in memory; + no Zstd/SZIP since they link C) and the `examples/wasm-viewer/` page. + `examples/wasm-viewer/test/run.sh` builds the package (needs the + `wasm-bindgen` CLI at the crate's exact version) and tests it under Node + and headless Chromium (a Playwright download in `~/.cache/ms-playwright` + on tank); the CI container has neither, so CI runs the native + `clawhdf5-wasm` `h5py_interop` test on the same fixture. Size numbers are + in the example's README. - Python and Node.js bindings for cross-language use - NetCDF-4 compatibility for scientific data interop diff --git a/docs/known-issues.md b/docs/known-issues.md index 4ea7518..ca4282b 100644 --- a/docs/known-issues.md +++ b/docs/known-issues.md @@ -422,6 +422,23 @@ the same agent-store interop test. **Fix:** an empty contiguous dataset gets the undefined address (all `0xff`), which is what libhdf5 itself writes. +## `clawhdf5-wasm` (browser) limits + +**Status:** open (by design for now; added 2026-09-26). + +- The whole file is held in memory: `open()` takes its bytes. There are no + HTTP range reads, so a multi-GB file does not fit a browser tab. +- Compound, reference, opaque, bitfield, time and VL-sequence datasets are + refused with an error naming the type; attributes of those types come back + as `value: null` with their `dtype`. +- No Zstd or SZIP (both link C): such datasets fail with + `unsupported filter: 32015` / `: 4`. pcodec is not enabled either. +- External links and virtual-dataset sources in other files cannot be + followed (no file system). +- Variable-length string datasets are read by decoding `read_selection`'s + bytes with `clawhdf5_format::vl_data` in the wasm crate; `File` itself still + cannot (see the audit gaps above). + ## The Node.js package (`packages/clawhdf5-node`) does not work **Status:** open (found 2026-09-25). Unpublished; not built or tested in CI. From 5461a13984b3ad03fc4386d88acc11bc421a5af3 Mon Sep 17 00:00:00 2001 From: osobh Date: Sat, 26 Sep 2026 00:14:30 -0500 Subject: [PATCH 7/7] ci: js-sys is not C in the no-C check The check matches any *-sys crate, and clawhdf5-wasm pulls in js-sys, wasm-bindgen's bindings to JavaScript, which compiles no C. Exempt it by name so the check keeps catching real C for the wasm crate. Co-Authored-By: Claude Opus 5.5 (1M context) --- scripts/ci-test.sh | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/scripts/ci-test.sh b/scripts/ci-test.sh index 52e3548..3c557d8 100755 --- a/scripts/ci-test.sh +++ b/scripts/ci-test.sh @@ -89,6 +89,7 @@ run_step "cargo clippy (fast-deflate / zlib-ng)" cargo clippy \ # that: fail if a crate that compiles C (a *-sys crate, cc or cmake) enters the # default dependency tree of any of them. clawhdf5-migrate (bundled SQLite), # clawhdf5-napi (Node) and clawhdf5-gpu (graphics drivers) are exempt. +# js-sys (clawhdf5-wasm's bindings to JavaScript) builds no C. no_c_in_default_build() { local crate found=0 for crate in clawhdf5-format clawhdf5-io clawhdf5-filters clawhdf5 \ @@ -96,7 +97,8 @@ no_c_in_default_build() { clawhdf5-wasm; do local c_deps c_deps=$(cargo tree -q -p "$crate" -e normal,build --prefix none \ - | grep -E '^([a-z0-9_-]+-sys|cc|cmake) v' | sort -u) + | grep -E '^([a-z0-9_-]+-sys|cc|cmake) v' \ + | grep -v '^js-sys v' | sort -u) if [ -n "$c_deps" ]; then echo "$crate pulls in C by default:" echo "$c_deps" | sed 's/^/ /'