diff --git a/.gitea/workflows/ci.yml b/.gitea/workflows/ci.yml index f57d8e4..cfd9624 100644 --- a/.gitea/workflows/ci.yml +++ b/.gitea/workflows/ci.yml @@ -22,6 +22,9 @@ jobs: run: rustup component add rustfmt clippy - name: Install thumbv7em-none-eabihf target run: rustup target add thumbv7em-none-eabihf + - name: Install wasm32-unknown-unknown target + # ci-test.sh builds the reader and clawhdf5-wasm for the browser. + run: rustup target add wasm32-unknown-unknown - name: Install Python interop dependencies # The interop suites used to skip silently when python3/h5py were # missing, so they never ran in CI. Install them and make a missing diff --git a/CHANGELOG.md b/CHANGELOG.md index 44ca7c0..1df18a3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -397,6 +397,32 @@ infinity; batches are all or nothing. CLI: `create --float16`. See `BENCHMARKS.md`, "float16 embedding storage". +### Browser (WebAssembly) +- **New crate `clawhdf5-wasm`:** the reader compiled to + `wasm32-unknown-unknown` with a wasm-bindgen JavaScript API — + `open(bytes)`, `list`, `info`, `attrs`, `read`, `readHyperslab` — returning + typed arrays of the stored width (`BigInt64Array` for 64-bit integers), + string arrays for strings and enums, and a thrown `Error` for types with no + typed-array form (compound, reference, opaque, VL sequences) or filters the + build lacks (Zstd, SZIP). Read-only; the file is held in memory. +- **`examples/wasm-viewer/`:** a drop-a-file HDF5/NetCDF-4 viewer page (tree, + type/shape/attributes, values paged as hyperslabs; `?file=&path=` opens a + URL). `build.sh` produces the package; `test/run.sh` checks it under Node + (251 checks against values h5py/libhdf5 read back from an h5py- and a + netCDF4-written file) and renders the page in headless Chromium. Size, + measured 2026-09-26 on tank (`gzip -9 -n`): 627,501 B of wasm, 191,639 B + gzipped, plus 21,826 B (4,487 B) of JS glue; h5wasm 0.10.3's embedded wasm + is 3,544,184 B (907,096 B) — full libhdf5, so not equal functionality. See + `examples/wasm-viewer/README.md`. +- The facade's read path already built for `wasm32-unknown-unknown` (nothing + needed gating); `ci-test.sh` now builds it (`--no-default-features`) and + lints `clawhdf5-wasm` for that target, and CI installs the target. The Node + and browser tests run in `ci-test.sh` only where `node` and `wasm-bindgen` + exist (not the CI container); CI checks the same expectations natively + (`clawhdf5-wasm`'s `h5py_interop` test). +- `Dataset::raw_datatype()` (facade) returns the full stored datatype, for + decoding `read_selection` bytes with `clawhdf5_format::data_read`. + ### Build - **Pure-Rust default.** `clawhdf5-format`, `clawhdf5-filters` and the `clawhdf5` facade default to the `zlib-rs` deflate backend; `fast-deflate` diff --git a/CLAUDE.md b/CLAUDE.md index 472ee3a..1cb7aba 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -5,7 +5,7 @@ Pure-Rust HDF5 format implementation with HNSW vector search, WAL-backed persist ## Architecture -Cargo workspace with 17 crates under `crates/` (plus `libaec-sys`, an internal FFI bindings crate for the optional `szip` feature): +Cargo workspace with 18 crates under `crates/` (plus `libaec-sys`, an internal FFI bindings crate for the optional `szip` feature): | Crate | Role | |-------|------| @@ -25,6 +25,7 @@ Cargo workspace with 17 crates under `crates/` (plus `libaec-sys`, an internal F | `clawhdf5-tools` | `h5rs`: pure-Rust HDF5 tools — `ls`, `dump` (DDL / hdf5-json), `stat`, `diff`, `check` (structural + checksum validator) | | `clawhdf5-napi` | Node.js native addon bindings | | `clawhdf5-py` | PyO3 Python bindings | +| `clawhdf5-wasm` | WebAssembly (wasm-bindgen) reader for the browser; demo in `examples/wasm-viewer/` | | `clawhdf5-bench` | Benchmark suite | ## Key Features @@ -150,6 +151,14 @@ Cargo workspace with 17 crates under `crates/` (plus `libaec-sys`, an internal F `MemorySource` for this bookkeeping is inferred from the caller-supplied `source_channel` string (a heuristic, not an authenticated trust boundary). - GPU-accelerated vector distance computation (`clawhdf5-gpu`, wgpu); HDF5 I/O itself is CPU-only +- Browser: `clawhdf5-wasm` (wasm-bindgen, read-only, file held in memory; + no Zstd/SZIP since they link C) and the `examples/wasm-viewer/` page. + `examples/wasm-viewer/test/run.sh` builds the package (needs the + `wasm-bindgen` CLI at the crate's exact version) and tests it under Node + and headless Chromium (a Playwright download in `~/.cache/ms-playwright` + on tank); the CI container has neither, so CI runs the native + `clawhdf5-wasm` `h5py_interop` test on the same fixture. Size numbers are + in the example's README. - Python and Node.js bindings for cross-language use - NetCDF-4 compatibility for scientific data interop diff --git a/Cargo.toml b/Cargo.toml index b86d8b6..0801b3d 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -17,6 +17,7 @@ members = [ "crates/clawhdf5-napi", "crates/clawhdf5-bench", "crates/clawhdf5-tools", + "crates/clawhdf5-wasm", "crates/libaec-sys", ] resolver = "2" @@ -35,3 +36,12 @@ tempfile = "3" criterion = { version = "0.5", features = ["html_reports"] } half = "2.7" serde = { version = "1", features = ["derive"] } + +# The browser build of clawhdf5-wasm (examples/wasm-viewer/build.sh): size +# over speed, whole-program optimisation. Native profiles are unaffected. +[profile.wasm-release] +inherits = "release" +opt-level = "s" +lto = true +codegen-units = 1 +panic = "abort" diff --git a/README.md b/README.md index 1088194..b9e3aa1 100644 --- a/README.md +++ b/README.md @@ -587,7 +587,7 @@ let exported = backend.export_markdown("MEMORY.md")?; ## Crate Map ``` -clawhdf5 workspace (16 crates, ~86K lines of Rust in src/, ~104K with tests +clawhdf5 workspace (17 crates, ~86K lines of Rust in src/, ~104K with tests and benches; plus libaec-sys, an internal FFI bindings crate for the optional szip feature) │ @@ -610,7 +610,8 @@ clawhdf5 workspace (16 crates, ~86K lines of Rust in src/, ~104K with tests │ ├── Bindings │ ├── clawhdf5-py — Python (PyO3) -│ └── clawhdf5-napi — Node.js (napi-rs) +│ ├── clawhdf5-napi — Node.js (napi-rs) +│ └── clawhdf5-wasm — Browser (WebAssembly, wasm-bindgen; read-only) │ └── Tooling └── clawhdf5-bench — Benchmark suite diff --git a/crates/clawhdf5-wasm/Cargo.toml b/crates/clawhdf5-wasm/Cargo.toml new file mode 100644 index 0000000..303837a --- /dev/null +++ b/crates/clawhdf5-wasm/Cargo.toml @@ -0,0 +1,30 @@ +[package] +name = "clawhdf5-wasm" +version.workspace = true +edition.workspace = true +rust-version.workspace = true +description = "Read HDF5 and NetCDF-4 files in the browser: clawhdf5's reader compiled to WebAssembly" +license.workspace = true +repository.workspace = true +keywords = ["hdf5", "netcdf", "wasm", "browser"] +categories = ["wasm", "parser-implementations", "science"] +# Distributed as the wasm-bindgen package built by examples/wasm-viewer/build.sh. +publish = false + +[lib] +# cdylib for wasm-bindgen; rlib so the pure-Rust core is tested natively. +crate-type = ["cdylib", "rlib"] + +[dependencies] +# No mmap (there is no file system) and no `parallel` (no threads on +# wasm32-unknown-unknown). lz4 is pure Rust; zstd and szip link C and are +# left out, so such datasets fail with a clear filter error. +clawhdf5 = { path = "../clawhdf5", version = "2.7.0", default-features = false, features = ["lz4"] } +clawhdf5-format = { path = "../clawhdf5-format", version = "2.7.0" } +# Must match the wasm-bindgen CLI exactly; build.sh checks. +wasm-bindgen = "0.2.129" +js-sys = "0.3.106" + +[dev-dependencies] +serde_json = "1" +tempfile = { workspace = true } diff --git a/crates/clawhdf5-wasm/src/core.rs b/crates/clawhdf5-wasm/src/core.rs new file mode 100644 index 0000000..baadcda --- /dev/null +++ b/crates/clawhdf5-wasm/src/core.rs @@ -0,0 +1,603 @@ +//! The reader behind the JavaScript API, in plain Rust so it is tested +//! natively. The `wasm_bindgen` layer in `lib.rs` only converts these types +//! to JavaScript values. +//! +//! Every read either returns the dataset's values or an error: a datatype +//! with no typed-array mapping (compound, reference, opaque, ...) is refused +//! with a message naming it, never returned as reinterpreted bytes. + +use clawhdf5::{AttrValue, File, Selection}; +use clawhdf5_format::data_read; +use clawhdf5_format::datatype::{Datatype, DatatypeByteOrder}; + +/// Errors are reported to JavaScript as messages. +pub type Result = std::result::Result; + +fn err(e: impl std::fmt::Display) -> String { + e.to_string() +} + +/// What a path names. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Kind { + Group, + Dataset, +} + +impl Kind { + pub fn as_str(self) -> &'static str { + match self { + Kind::Group => "group", + Kind::Dataset => "dataset", + } + } +} + +/// One entry of a group listing. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct Child { + pub name: String, + pub kind: Kind, +} + +/// A dataset's metadata. +#[derive(Debug, Clone, PartialEq)] +pub struct DatasetInfo { + /// Dataspace dimensions (empty for a scalar). + pub shape: Vec, + /// Maximum dimensions, `None` per unlimited dimension; `None` overall + /// when the dataspace records none. + pub maxshape: Option>>, + /// Human-readable datatype, e.g. `f64`, `i16 (big-endian)`, `string[8]`. + pub dtype: String, + /// Dimensions of an array datatype's elements, appended to the shape of + /// what [`Reader::read`] returns (empty otherwise). + pub element_shape: Vec, +} + +/// Decoded values, one variant per JavaScript typed array. +#[derive(Debug, Clone, PartialEq)] +pub enum Data { + F32(Vec), + F64(Vec), + I8(Vec), + I16(Vec), + I32(Vec), + I64(Vec), + U8(Vec), + U16(Vec), + U32(Vec), + U64(Vec), + /// Fixed- and variable-length strings, and enumeration member names. + Strings(Vec), +} + +impl Data { + pub fn len(&self) -> usize { + match self { + Data::F32(v) => v.len(), + Data::F64(v) => v.len(), + Data::I8(v) => v.len(), + Data::I16(v) => v.len(), + Data::I32(v) => v.len(), + Data::I64(v) => v.len(), + Data::U8(v) => v.len(), + Data::U16(v) => v.len(), + Data::U32(v) => v.len(), + Data::U64(v) => v.len(), + Data::Strings(v) => v.len(), + } + } + + pub fn is_empty(&self) -> bool { + self.len() == 0 + } +} + +/// Values in row-major order with their shape. +#[derive(Debug, Clone, PartialEq)] +pub struct Array { + pub shape: Vec, + pub data: Data, +} + +/// A regular hyperslab, as in `H5Sselect_hyperslab`. `stride` and `block` +/// default to 1 in every dimension. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct Hyperslab { + pub start: Vec, + pub count: Vec, + pub stride: Option>, + pub block: Option>, +} + +/// An attribute: its value, or why it has none. +#[derive(Debug, Clone)] +pub struct Attr { + pub name: String, + pub value: AttrValue, +} + +/// An open file, held in memory. +pub struct Reader { + file: File, +} + +impl Reader { + /// Parse a file from its bytes (the browser hands over the whole file). + pub fn open(bytes: Vec) -> Result { + Ok(Self { + file: File::from_bytes(bytes).map_err(err)?, + }) + } + + /// Whether `path` names a group or a dataset. + pub fn kind(&self, path: &str) -> Result { + match self.file.dataset(path) { + Ok(_) => Ok(Kind::Dataset), + Err(clawhdf5::Error::NotADataset(_)) => Ok(Kind::Group), + Err(e) => Err(err(e)), + } + } + + /// The groups, then the datasets, in the group at `path` (`/` is the + /// root). Soft links are listed as their targets; external and dangling + /// links, and named datatypes, are left out. + pub fn list(&self, path: &str) -> Result> { + if self.kind(path)? != Kind::Group { + return Err(format!("not a group: {path}")); + } + let group = self.file.group(path).map_err(err)?; + let mut out: Vec = group + .groups() + .map_err(err)? + .into_iter() + .map(|name| Child { + name, + kind: Kind::Group, + }) + .collect(); + out.extend( + group + .datasets() + .map_err(err)? + .into_iter() + .map(|name| Child { + name, + kind: Kind::Dataset, + }), + ); + Ok(out) + } + + /// Shape, max shape and datatype of the dataset at `path`. + pub fn info(&self, path: &str) -> Result { + let ds = self.file.dataset(path).map_err(err)?; + let dt = ds.raw_datatype().map_err(err)?; + let maxshape = ds.max_dimensions().map_err(err)?.map(|dims| { + dims.into_iter() + .map(|d| (d != u64::MAX).then_some(d)) + .collect() + }); + Ok(DatasetInfo { + shape: ds.shape().map_err(err)?, + maxshape, + dtype: describe(&dt), + element_shape: element_shape(&dt), + }) + } + + /// The attributes of the group or dataset at `path`, sorted by name, and + /// one message per attribute that could not be read at all. An attribute + /// whose type has no plain JavaScript form is returned as + /// [`AttrValue::Raw`]. + pub fn attrs(&self, path: &str) -> Result<(Vec, Vec)> { + let (map, errors) = match self.kind(path)? { + Kind::Dataset => self + .file + .dataset(path) + .and_then(|d| d.attrs_with_errors()) + .map_err(err)?, + Kind::Group => self + .file + .group(path) + .and_then(|g| g.attrs_with_errors()) + .map_err(err)?, + }; + let mut attrs: Vec = map + .into_iter() + .map(|(name, value)| Attr { name, value }) + .collect(); + attrs.sort_by(|a, b| a.name.cmp(&b.name)); + Ok((attrs, errors.into_iter().map(err).collect())) + } + + /// Read the dataset at `path`, whole or a hyperslab of it. + pub fn read(&self, path: &str, slab: Option<&Hyperslab>) -> Result { + let ds = self.file.dataset(path).map_err(err)?; + let dt = ds.raw_datatype().map_err(err)?; + let shape = ds.shape().map_err(err)?; + let (selection, mut out_shape) = match slab { + None => (Selection::All, shape.clone()), + Some(h) => hyperslab_selection(h, &shape)?, + }; + let raw = ds.read_selection(&selection).map_err(err)?; + let data = self.decode(&raw, &dt)?; + out_shape.extend(element_shape(&dt)); + let expected = out_shape + .iter() + .try_fold(1u64, |acc, &d| acc.checked_mul(d)) + .ok_or("selection size overflows")?; + if data.len() as u64 != expected { + return Err(format!( + "read {} values for shape {out_shape:?} ({expected} expected)", + data.len() + )); + } + Ok(Array { + shape: out_shape, + data, + }) + } + + fn decode(&self, raw: &[u8], dt: &Datatype) -> Result { + let base = array_base(dt); + let is_array = !std::ptr::eq(base, dt); + Ok(match base { + Datatype::FloatingPoint { size, .. } if *size <= 4 => { + Data::F32(data_read::read_as_f32(raw, dt).map_err(err)?) + } + Datatype::FloatingPoint { .. } => { + Data::F64(data_read::read_as_f64(raw, dt).map_err(err)?) + } + Datatype::FixedPoint { size, signed, .. } => { + let signed_ints = || data_read::read_as_i64(raw, dt).map_err(err); + let unsigned_ints = || data_read::read_as_u64(raw, dt).map_err(err); + match (size, signed) { + (1, true) => Data::I8(narrow(signed_ints()?)?), + (2, true) => Data::I16(narrow(signed_ints()?)?), + (4, true) => Data::I32(narrow(signed_ints()?)?), + (_, true) => Data::I64(signed_ints()?), + (1, false) => Data::U8(narrow(unsigned_ints()?)?), + (2, false) => Data::U16(narrow(unsigned_ints()?)?), + (4, false) => Data::U32(narrow(unsigned_ints()?)?), + (_, false) => Data::U64(unsigned_ints()?), + } + } + Datatype::String { .. } if !is_array => { + Data::Strings(data_read::read_as_strings(raw, dt).map_err(err)?) + } + Datatype::VariableLength { + is_string: true, .. + } if !is_array => { + let size = dt.type_size() as usize; + if size == 0 || !raw.len().is_multiple_of(size) { + return Err(format!( + "{} bytes is not a whole number of {size}-byte string references", + raw.len() + )); + } + let sb = self.file.superblock(); + Data::Strings( + clawhdf5_format::vl_data::read_vl_strings( + self.file.as_bytes(), + raw, + (raw.len() / size) as u64, + sb.offset_size, + sb.length_size, + ) + .map_err(err)?, + ) + } + Datatype::Enumeration { .. } if !is_array => { + Data::Strings(data_read::read_enum_names(raw, dt).map_err(err)?) + } + _ => { + return Err(format!( + "reading {} datasets is not supported", + describe(dt) + )); + } + }) + } +} + +/// Narrow integers read at 64 bits to the dataset's own width. The source is +/// that width, so this cannot fail on correct input; it is checked anyway. +fn narrow>(v: Vec) -> Result> { + v.into_iter() + .map(|x| T::try_from(x).map_err(|_| format!("value {x} out of range"))) + .collect() +} + +/// Innermost element type of (possibly nested) array datatypes. +fn array_base(dt: &Datatype) -> &Datatype { + match dt { + Datatype::Array { base_type, .. } => array_base(base_type), + _ => dt, + } +} + +fn element_shape(dt: &Datatype) -> Vec { + match dt { + Datatype::Array { + base_type, + dimensions, + } => { + let mut dims: Vec = dimensions.iter().map(|&d| u64::from(d)).collect(); + dims.extend(element_shape(base_type)); + dims + } + _ => Vec::new(), + } +} + +fn hyperslab_selection(h: &Hyperslab, shape: &[u64]) -> Result<(Selection, Vec)> { + let rank = shape.len(); + let ones = vec![1u64; rank]; + let stride = h.stride.clone().unwrap_or_else(|| ones.clone()); + let block = h.block.clone().unwrap_or(ones); + for (what, v) in [ + ("start", &h.start), + ("count", &h.count), + ("stride", &stride), + ("block", &block), + ] { + if v.len() != rank { + return Err(format!( + "hyperslab {what} has {} dimensions, the dataset has {rank}", + v.len() + )); + } + } + let mut out = Vec::with_capacity(rank); + for d in 0..rank { + if stride[d] == 0 || block[d] == 0 { + return Err(format!("hyperslab stride and block must be >= 1 (dim {d})")); + } + if h.count[d] > 1 && block[d] > stride[d] { + return Err(format!( + "hyperslab blocks overlap in dim {d}: block {} > stride {}", + block[d], stride[d] + )); + } + // Last element selected: start + (count-1)*stride + block - 1. + if h.count[d] > 0 { + let last = (h.count[d] - 1) + .checked_mul(stride[d]) + .and_then(|x| x.checked_add(h.start[d])) + .and_then(|x| x.checked_add(block[d] - 1)); + match last { + Some(l) if l < shape[d] => {} + _ => { + return Err(format!( + "hyperslab exceeds dimension {d} (extent {})", + shape[d] + )); + } + } + } + out.push( + h.count[d] + .checked_mul(block[d]) + .ok_or("selection size overflows")?, + ); + } + Ok(( + Selection::Hyperslab { + start: h.start.clone(), + stride, + count: h.count.clone(), + block, + }, + out, + )) +} + +/// A short, human-readable datatype name. +pub fn describe(dt: &Datatype) -> String { + fn endian(order: &DatatypeByteOrder) -> &'static str { + match order { + DatatypeByteOrder::BigEndian => " (big-endian)", + DatatypeByteOrder::Vax => " (VAX)", + _ => "", + } + } + match dt { + Datatype::FixedPoint { + size, + signed, + byte_order, + .. + } => format!( + "{}{}{}", + if *signed { "i" } else { "u" }, + size * 8, + endian(byte_order) + ), + Datatype::FloatingPoint { + size, byte_order, .. + } => format!("f{}{}", size * 8, endian(byte_order)), + Datatype::Time { size, .. } => format!("time{}", size * 8), + Datatype::String { size, .. } => format!("string[{size}]"), + Datatype::BitField { size, .. } => format!("bitfield{}", size * 8), + Datatype::Opaque { size, .. } => format!("opaque[{size}]"), + Datatype::Compound { members, .. } => { + let fields: Vec = members + .iter() + .map(|m| format!("{}: {}", m.name, describe(&m.datatype))) + .collect(); + format!("compound{{{}}}", fields.join(", ")) + } + Datatype::Reference { .. } => "reference".to_string(), + Datatype::Enumeration { + base_type, members, .. + } => { + let names: Vec<&str> = members.iter().map(|m| m.name.as_str()).collect(); + format!("enum<{}>{{{}}}", describe(base_type), names.join(", ")) + } + Datatype::VariableLength { + is_string: true, .. + } => "vlen string".to_string(), + Datatype::VariableLength { base_type, .. } => { + format!("vlen<{}>", describe(base_type)) + } + Datatype::Array { + base_type, + dimensions, + } => format!("array{dimensions:?}<{}>", describe(base_type)), + } +} + +#[cfg(test)] +mod tests { + use super::*; + use clawhdf5::FileBuilder; + + fn sample() -> Reader { + let mut b = FileBuilder::new(); + b.create_dataset("grid") + .with_f64_data(&(0..12).map(f64::from).collect::>()) + .with_shape(&[3, 4]) + .with_chunks(&[2, 2]) + .with_deflate(4); + b.create_dataset("bytes").with_u8_data(&[1, 2, 250]); + let mut g = b.create_group("sensors"); + g.create_dataset("temp").with_f32_data(&[1.5, -2.25]); + g.set_attr("location", AttrValue::String("lab".into())); + b.add_group(g.finish()); + b.set_attr("version", AttrValue::I64(3)); + b.set_attr("scale", AttrValue::F64Array(vec![0.5, 2.0])); + Reader::open(b.finish().unwrap()).unwrap() + } + + #[test] + fn lists_groups_then_datasets() { + let r = sample(); + let names: Vec<(String, Kind)> = r + .list("/") + .unwrap() + .into_iter() + .map(|c| (c.name, c.kind)) + .collect(); + assert_eq!(names[0], ("sensors".to_string(), Kind::Group)); + let mut ds: Vec<&str> = names[1..].iter().map(|(n, _)| n.as_str()).collect(); + ds.sort(); + assert_eq!(ds, ["bytes", "grid"]); + assert_eq!( + r.list("sensors").unwrap(), + vec![Child { + name: "temp".into(), + kind: Kind::Dataset + }] + ); + assert!(r.list("grid").unwrap_err().contains("not a group")); + assert!(r.list("missing").is_err()); + } + + #[test] + fn info_reports_shape_and_dtype() { + let r = sample(); + let i = r.info("grid").unwrap(); + assert_eq!(i.shape, vec![3, 4]); + assert_eq!(i.dtype, "f64"); + assert!(i.element_shape.is_empty()); + assert_eq!(r.info("sensors/temp").unwrap().dtype, "f32"); + assert_eq!(r.kind("/sensors").unwrap(), Kind::Group); + assert_eq!(r.kind("/sensors/temp").unwrap(), Kind::Dataset); + } + + #[test] + fn reads_whole_and_hyperslab() { + let r = sample(); + let all = r.read("grid", None).unwrap(); + assert_eq!(all.shape, vec![3, 4]); + assert_eq!(all.data, Data::F64((0..12).map(f64::from).collect())); + + let slab = Hyperslab { + start: vec![1, 0], + count: vec![2, 2], + stride: Some(vec![1, 2]), + block: None, + }; + let part = r.read("grid", Some(&slab)).unwrap(); + assert_eq!(part.shape, vec![2, 2]); + assert_eq!(part.data, Data::F64(vec![4.0, 6.0, 8.0, 10.0])); + + assert_eq!( + r.read("bytes", None).unwrap().data, + Data::U8(vec![1, 2, 250]) + ); + assert_eq!( + r.read("sensors/temp", None).unwrap().data, + Data::F32(vec![1.5, -2.25]) + ); + } + + #[test] + fn bad_hyperslabs_are_refused() { + let r = sample(); + let mk = |start: Vec, count: Vec| Hyperslab { + start, + count, + stride: None, + block: None, + }; + assert!( + r.read("grid", Some(&mk(vec![0], vec![1]))) + .unwrap_err() + .contains("dimensions") + ); + assert!( + r.read("grid", Some(&mk(vec![2, 0], vec![2, 1]))) + .unwrap_err() + .contains("exceeds") + ); + let overlap = Hyperslab { + start: vec![0, 0], + count: vec![2, 1], + stride: Some(vec![1, 1]), + block: Some(vec![2, 1]), + }; + assert!( + r.read("grid", Some(&overlap)) + .unwrap_err() + .contains("overlap") + ); + } + + #[test] + fn attrs_are_sorted() { + let r = sample(); + let (attrs, errors) = r.attrs("/").unwrap(); + assert!(errors.is_empty()); + let names: Vec<&str> = attrs.iter().map(|a| a.name.as_str()).collect(); + assert_eq!(names, ["scale", "version"]); + let (g, _) = r.attrs("sensors").unwrap(); + assert!(matches!(&g[0].value, AttrValue::String(s) if s == "lab")); + } + + #[test] + fn compound_is_refused_not_reinterpreted() { + use clawhdf5::CompoundTypeBuilder; + let ct = CompoundTypeBuilder::new() + .f64_field("x") + .i32_field("n") + .build(); + let mut rec = Vec::new(); + rec.extend_from_slice(&1.0f64.to_le_bytes()); + rec.extend_from_slice(&7i32.to_le_bytes()); + let mut b = FileBuilder::new(); + b.create_dataset("table").with_compound_data(ct, rec, 1); + let r = Reader::open(b.finish().unwrap()).unwrap(); + let e = r.read("table", None).unwrap_err(); + assert!(e.contains("compound{x: f64, n: i32}"), "{e}"); + assert!(e.contains("not supported"), "{e}"); + } + + #[test] + fn garbage_is_an_error() { + assert!(Reader::open(vec![0u8; 64]).is_err()); + assert!(Reader::open(Vec::new()).is_err()); + } +} diff --git a/crates/clawhdf5-wasm/src/lib.rs b/crates/clawhdf5-wasm/src/lib.rs new file mode 100644 index 0000000..7c6b4a9 --- /dev/null +++ b/crates/clawhdf5-wasm/src/lib.rs @@ -0,0 +1,244 @@ +//! clawhdf5's HDF5 reader for JavaScript, via `wasm-bindgen`. +//! +//! ```js +//! import init, { open } from "./pkg/clawhdf5_wasm.js"; +//! await init(); +//! const file = open(new Uint8Array(await blob.arrayBuffer())); +//! file.list("/"); // [{ name, kind: "group" | "dataset" }] +//! file.info("/x"); // { shape, maxshape, dtype, elementShape } +//! file.attrs("/x"); // [{ name, value, dtype }] +//! file.read("/x"); // { shape, dtype, data: Float64Array | ... | string[] } +//! file.readHyperslab("/x", [0, 0], [10, 10]); // stride, block optional +//! file.free(); +//! ``` +//! +//! Numeric data comes back in the typed array of the stored width +//! (`Int16Array` for `i16`, `BigInt64Array` for `i64`, `Float32Array` for +//! `f32` and `f16`, ...); strings and enumeration names as arrays of strings. +//! Anything else is a thrown `Error` naming the datatype. Only the reader is +//! exposed: nothing here writes files. +//! +//! The logic lives in [`core`], which is plain Rust and tested natively. + +pub mod core; + +use clawhdf5::AttrValue; +use js_sys::{Array, Object, Reflect}; +use wasm_bindgen::prelude::*; + +use crate::core::{Data, Hyperslab, Reader}; + +/// JavaScript numbers are exact up to 2^53. +const MAX_SAFE_INTEGER: f64 = 9_007_199_254_740_991.0; + +fn js_err(msg: String) -> JsError { + JsError::new(&msg) +} + +fn set(obj: &Object, key: &str, value: impl Into) { + // Defining a property on a fresh plain object cannot fail. + Reflect::set(obj, &JsValue::from_str(key), &value.into()).unwrap_throw(); +} + +fn shape_to_js(shape: &[u64]) -> Array { + shape.iter().map(|&d| JsValue::from_f64(d as f64)).collect() +} + +fn indices_from_js(what: &str, v: &[f64]) -> Result, JsError> { + v.iter() + .map(|&x| { + if x.is_finite() && x >= 0.0 && x.fract() == 0.0 && x <= MAX_SAFE_INTEGER { + Ok(x as u64) + } else { + Err(js_err(format!( + "{what} must hold non-negative integers, got {x}" + ))) + } + }) + .collect() +} + +fn data_to_js(data: Data) -> JsValue { + match data { + Data::F32(v) => js_sys::Float32Array::from(&v[..]).into(), + Data::F64(v) => js_sys::Float64Array::from(&v[..]).into(), + Data::I8(v) => js_sys::Int8Array::from(&v[..]).into(), + Data::I16(v) => js_sys::Int16Array::from(&v[..]).into(), + Data::I32(v) => js_sys::Int32Array::from(&v[..]).into(), + Data::I64(v) => js_sys::BigInt64Array::from(&v[..]).into(), + Data::U8(v) => js_sys::Uint8Array::from(&v[..]).into(), + Data::U16(v) => js_sys::Uint16Array::from(&v[..]).into(), + Data::U32(v) => js_sys::Uint32Array::from(&v[..]).into(), + Data::U64(v) => js_sys::BigUint64Array::from(&v[..]).into(), + Data::Strings(v) => v + .into_iter() + .map(|s| JsValue::from_str(&s)) + .collect::() + .into(), + } +} + +/// A scalar integer as a `number` when exact, else a `bigint`. +fn int_to_js(x: i128) -> JsValue { + if (x as f64).abs() <= MAX_SAFE_INTEGER { + JsValue::from_f64(x as f64) + } else if let Ok(v) = i64::try_from(x) { + JsValue::from(v) + } else { + JsValue::from(x as u64) + } +} + +/// An attribute value, and the datatype of one that has no JavaScript form. +fn attr_to_js(value: AttrValue) -> (JsValue, Option) { + match value { + AttrValue::F64(x) => (JsValue::from_f64(x), None), + AttrValue::F64Array(v) => (js_sys::Float64Array::from(&v[..]).into(), None), + AttrValue::I64(x) => (int_to_js(i128::from(x)), None), + AttrValue::I64Array(v) => (js_sys::BigInt64Array::from(&v[..]).into(), None), + AttrValue::U64(x) => (int_to_js(i128::from(x)), None), + AttrValue::U64Array(v) => (js_sys::BigUint64Array::from(&v[..]).into(), None), + AttrValue::String(s) => (JsValue::from_str(&s), None), + AttrValue::StringArray(v) => ( + v.iter() + .map(|s| JsValue::from_str(s)) + .collect::() + .into(), + None, + ), + AttrValue::Raw { datatype, .. } => (JsValue::NULL, Some(core::describe(&datatype))), + } +} + +/// An open HDF5 (or NetCDF-4) file. +#[wasm_bindgen] +pub struct H5File { + inner: Reader, +} + +/// Open a file from its bytes. Throws if they are not an HDF5 file. +#[wasm_bindgen] +pub fn open(bytes: Vec) -> Result { + H5File::new(bytes) +} + +/// The clawhdf5 version this module was built from. +#[wasm_bindgen] +pub fn version() -> String { + env!("CARGO_PKG_VERSION").to_string() +} + +#[wasm_bindgen] +impl H5File { + /// Same as [`open`]. + #[wasm_bindgen(constructor)] + pub fn new(bytes: Vec) -> Result { + Ok(H5File { + inner: Reader::open(bytes).map_err(js_err)?, + }) + } + + /// `"group"` or `"dataset"`. + pub fn kind(&self, path: &str) -> Result { + Ok(self.inner.kind(path).map_err(js_err)?.as_str().to_string()) + } + + /// The group's members: `[{ name, kind }]`, groups first. + pub fn list(&self, path: &str) -> Result { + Ok(self + .inner + .list(path) + .map_err(js_err)? + .into_iter() + .map(|c| { + let o = Object::new(); + set(&o, "name", c.name); + set(&o, "kind", c.kind.as_str()); + JsValue::from(o) + }) + .collect()) + } + + /// `{ shape, maxshape, dtype, elementShape }`. `maxshape` is `null` + /// when not recorded, with `null` for each unlimited dimension. + pub fn info(&self, path: &str) -> Result { + let i = self.inner.info(path).map_err(js_err)?; + let o = Object::new(); + set(&o, "shape", shape_to_js(&i.shape)); + let max: JsValue = match i.maxshape { + None => JsValue::NULL, + Some(dims) => dims + .into_iter() + .map(|d| d.map_or(JsValue::NULL, |d| JsValue::from_f64(d as f64))) + .collect::() + .into(), + }; + set(&o, "maxshape", max); + set(&o, "dtype", i.dtype); + set(&o, "elementShape", shape_to_js(&i.element_shape)); + Ok(o) + } + + /// `[{ name, value, dtype }]`, sorted by name. Scalars are `number` + /// (`bigint` beyond 2^53) or `string`; arrays are typed arrays or + /// `string[]`. An attribute with no JavaScript form has `value: null` + /// and its `dtype`; one that could not be read at all is reported by + /// [`attrErrors`](Self::attr_errors). + pub fn attrs(&self, path: &str) -> Result { + let (attrs, _) = self.inner.attrs(path).map_err(js_err)?; + Ok(attrs + .into_iter() + .map(|a| { + let o = Object::new(); + set(&o, "name", a.name); + let (value, dtype) = attr_to_js(a.value); + set(&o, "value", value); + set(&o, "dtype", dtype.map_or(JsValue::NULL, JsValue::from)); + JsValue::from(o) + }) + .collect()) + } + + /// Messages for attributes that could not be read. + #[wasm_bindgen(js_name = attrErrors)] + pub fn attr_errors(&self, path: &str) -> Result { + let (_, errors) = self.inner.attrs(path).map_err(js_err)?; + Ok(errors.into_iter().map(JsValue::from).collect()) + } + + /// The whole dataset: `{ shape, dtype, data }`, `data` in row-major + /// order. + pub fn read(&self, path: &str) -> Result { + self.read_impl(path, None) + } + + /// A regular hyperslab (`H5Sselect_hyperslab`): `stride` and `block` + /// default to 1. The result's shape is `count * block` per dimension. + #[wasm_bindgen(js_name = readHyperslab)] + pub fn read_hyperslab( + &self, + path: &str, + start: Vec, + count: Vec, + stride: Option>, + block: Option>, + ) -> Result { + let slab = Hyperslab { + start: indices_from_js("start", &start)?, + count: indices_from_js("count", &count)?, + stride: stride.map(|s| indices_from_js("stride", &s)).transpose()?, + block: block.map(|b| indices_from_js("block", &b)).transpose()?, + }; + self.read_impl(path, Some(&slab)) + } + + fn read_impl(&self, path: &str, slab: Option<&Hyperslab>) -> Result { + let dtype = self.inner.info(path).map_err(js_err)?.dtype; + let a = self.inner.read(path, slab).map_err(js_err)?; + let o = Object::new(); + set(&o, "shape", shape_to_js(&a.shape)); + set(&o, "dtype", dtype); + set(&o, "data", data_to_js(a.data)); + Ok(o) + } +} diff --git a/crates/clawhdf5-wasm/tests/h5py_interop.rs b/crates/clawhdf5-wasm/tests/h5py_interop.rs new file mode 100644 index 0000000..7ef60ab --- /dev/null +++ b/crates/clawhdf5-wasm/tests/h5py_interop.rs @@ -0,0 +1,265 @@ +//! The wasm reader's core against files written by h5py and netCDF4, with +//! the values libhdf5 reads back as the reference. The generator, +//! `examples/wasm-viewer/test/make_fixture.py`, is shared with the Node test +//! of the built wasm package, so both compare against the same expectations. +//! +//! Skipped when python3 with h5py/netCDF4 is missing, unless +//! `CLAWHDF5_REQUIRE_INTEROP=1`. `CLAWHDF5_PYTHON` names the interpreter. + +use std::path::{Path, PathBuf}; +use std::process::Command; + +use clawhdf5::AttrValue; +use clawhdf5_wasm::core::{Data, Hyperslab, Kind, Reader}; +use serde_json::Value; + +fn python() -> String { + std::env::var("CLAWHDF5_PYTHON").unwrap_or_else(|_| "python3".to_string()) +} + +fn interop_required() -> bool { + std::env::var("CLAWHDF5_REQUIRE_INTEROP").is_ok_and(|v| v == "1") +} + +fn python_available() -> bool { + Command::new(python()) + .args(["-c", "import h5py, netCDF4, numpy"]) + .output() + .map(|o| o.status.success()) + .unwrap_or(false) +} + +fn generator() -> PathBuf { + Path::new(env!("CARGO_MANIFEST_DIR")).join("../../examples/wasm-viewer/test/make_fixture.py") +} + +/// Values as comparable strings: integers exactly, floats by their f64 +/// value (an f32 widens exactly), strings as themselves. +fn data_strings(d: &Data) -> (&'static str, Vec) { + fn s(v: &[T]) -> Vec { + v.iter().map(ToString::to_string).collect() + } + fn f>(v: &[T]) -> Vec { + v.iter().map(|&x| format!("{:?}", x.into())).collect() + } + match d { + Data::F32(v) => ("f32", f(v)), + Data::F64(v) => ("f64", f(v)), + Data::I8(v) => ("i8", s(v)), + Data::I16(v) => ("i16", s(v)), + Data::I32(v) => ("i32", s(v)), + Data::I64(v) => ("i64", s(v)), + Data::U8(v) => ("u8", s(v)), + Data::U16(v) => ("u16", s(v)), + Data::U32(v) => ("u32", s(v)), + Data::U64(v) => ("u64", s(v)), + Data::Strings(v) => ("strings", v.clone()), + } +} + +fn expected_strings(kind: &str, values: &Value) -> Vec { + values + .as_array() + .unwrap() + .iter() + .map(|v| match (kind, v) { + ("f32" | "f64", Value::Number(n)) => format!("{:?}", n.as_f64().unwrap()), + (_, Value::String(s)) => s.clone(), + other => panic!("unexpected expected value {other:?}"), + }) + .collect() +} + +fn shape(v: &Value) -> Vec { + v.as_array() + .unwrap() + .iter() + .map(|x| x.as_u64().unwrap()) + .collect() +} + +fn check_attr(file: &str, path: &str, name: &str, got: &AttrValue, want: &Value) { + let ctx = format!("{file}:{path}@{name}"); + let ints = |v: &Value| -> Vec { + v.as_array() + .unwrap() + .iter() + .map(|x| x.as_str().unwrap().to_string()) + .collect() + }; + let scalar = want["scalar"].as_bool().unwrap_or(false); + match got { + AttrValue::String(s) => assert_eq!(want["string"].as_str(), Some(s.as_str()), "{ctx}"), + AttrValue::StringArray(v) => { + let w: Vec<&str> = want["strings"] + .as_array() + .unwrap_or_else(|| panic!("{ctx}: got {v:?}")) + .iter() + .map(|x| x.as_str().unwrap()) + .collect(); + assert_eq!(v, &w, "{ctx}"); + } + AttrValue::I64(x) => { + assert!(scalar, "{ctx}"); + assert_eq!(vec![x.to_string()], ints(&want["int"]), "{ctx}"); + } + AttrValue::U64(x) => { + assert!(scalar, "{ctx}"); + assert_eq!(vec![x.to_string()], ints(&want["int"]), "{ctx}"); + } + AttrValue::I64Array(v) => assert_eq!( + v.iter().map(ToString::to_string).collect::>(), + ints(&want["int"]), + "{ctx}" + ), + AttrValue::U64Array(v) => assert_eq!( + v.iter().map(ToString::to_string).collect::>(), + ints(&want["int"]), + "{ctx}" + ), + AttrValue::F64(x) => { + assert!(scalar, "{ctx}"); + assert_eq!(Some(*x), want["float"][0].as_f64(), "{ctx}"); + } + AttrValue::F64Array(v) => { + let w: Vec = want["float"] + .as_array() + .unwrap() + .iter() + .map(|x| x.as_f64().unwrap()) + .collect(); + assert_eq!(v, &w, "{ctx}"); + } + AttrValue::Raw { datatype, .. } => { + let want = want["raw"] + .as_str() + .unwrap_or_else(|| panic!("{ctx}: undecoded {datatype:?}")); + let d = clawhdf5_wasm::core::describe(datatype); + assert!(d.contains(want), "{ctx}: {d}"); + } + } +} + +fn check_file(dir: &Path, file: &str, exp: &Value) { + let r = Reader::open(std::fs::read(dir.join(file)).unwrap()).unwrap(); + + for (path, want) in exp["lists"].as_object().unwrap() { + let list = r + .list(path) + .unwrap_or_else(|e| panic!("{file}:{path}: {e}")); + for (kind, key) in [(Kind::Group, "groups"), (Kind::Dataset, "datasets")] { + let mut got: Vec<&str> = list + .iter() + .filter(|c| c.kind == kind) + .map(|c| c.name.as_str()) + .collect(); + got.sort(); + let w: Vec<&str> = want[key] + .as_array() + .unwrap() + .iter() + .map(|x| x.as_str().unwrap()) + .collect(); + assert_eq!(got, w, "{file}:{path} {key}"); + } + } + + for (path, want) in exp["datasets"].as_object().unwrap() { + let kind = want["kind"].as_str().unwrap(); + let a = match (r.read(path, None), want["unavailable"].as_str()) { + (Ok(a), _) => a, + // A filter this build may lack (zstd: the wasm build has it + // off, a workspace build may unify it on) must fail clearly. + (Err(e), Some(why)) => { + assert!(e.contains(why), "{file}:{path}: {e}"); + continue; + } + (Err(e), None) => panic!("{file}:{path}: {e}"), + }; + assert_eq!(a.shape, shape(&want["shape"]), "{file}:{path} shape"); + let (got_kind, got) = data_strings(&a.data); + assert_eq!(got_kind, kind, "{file}:{path} kind"); + assert_eq!( + got, + expected_strings(kind, &want["values"]), + "{file}:{path}" + ); + + if let Some(slab) = want.get("slab") { + let h = Hyperslab { + start: shape(&slab["start"]), + count: shape(&slab["count"]), + stride: Some(shape(&slab["stride"])), + block: None, + }; + let a = r + .read(path, Some(&h)) + .unwrap_or_else(|e| panic!("{file}:{path} {h:?}: {e}")); + assert_eq!(a.shape, shape(&slab["shape"]), "{file}:{path} slab shape"); + assert_eq!( + data_strings(&a.data).1, + expected_strings(kind, &slab["values"]), + "{file}:{path} slab" + ); + } + } + + for (path, what) in exp["errors"].as_object().unwrap() { + let e = r.read(path, None).expect_err(path); + assert!(e.contains(what.as_str().unwrap()), "{file}:{path}: {e}"); + } + + let skip: Vec<&str> = exp["skip_attrs"] + .as_array() + .unwrap() + .iter() + .map(|x| x.as_str().unwrap()) + .collect(); + for (path, want) in exp["attrs"].as_object().unwrap() { + let (attrs, errors) = r + .attrs(path) + .unwrap_or_else(|e| panic!("{file}:{path}: {e}")); + assert!(errors.is_empty(), "{file}:{path}: {errors:?}"); + let want = want.as_object().unwrap(); + let mut compared = 0; + for a in &attrs { + if a.name.starts_with('_') || skip.contains(&a.name.as_str()) { + continue; + } + let w = want + .get(&a.name) + .unwrap_or_else(|| panic!("{file}:{path}: unexpected attribute {}", a.name)); + check_attr(file, path, &a.name, &a.value, w); + compared += 1; + } + assert_eq!(compared, want.len(), "{file}:{path}: attributes missing"); + } +} + +#[test] +fn reads_what_h5py_and_netcdf4_wrote() { + if !python_available() { + assert!( + !interop_required(), + "CLAWHDF5_REQUIRE_INTEROP=1 but python with h5py, netCDF4 and numpy is not available" + ); + eprintln!("SKIP: python with h5py/netCDF4 not available"); + return; + } + let dir = tempfile::tempdir().unwrap(); + let out = Command::new(python()) + .arg(generator()) + .arg(dir.path()) + .output() + .expect("run python"); + assert!( + out.status.success(), + "fixture generator failed:\n{}", + String::from_utf8_lossy(&out.stderr) + ); + let exp: Value = + serde_json::from_slice(&std::fs::read(dir.path().join("expected.json")).unwrap()).unwrap(); + for file in ["fixture.h5", "fixture.nc"] { + check_file(dir.path(), file, &exp[file]); + } +} diff --git a/crates/clawhdf5/src/lib.rs b/crates/clawhdf5/src/lib.rs index 40ae557..12e2f69 100644 --- a/crates/clawhdf5/src/lib.rs +++ b/crates/clawhdf5/src/lib.rs @@ -144,6 +144,35 @@ mod tests { assert_eq!(ds.dtype().unwrap(), DType::I32); } + #[test] + fn dataset_raw_datatype() { + use clawhdf5_format::datatype::Datatype; + let bytes = make_simple_file(); + let file = File::from_bytes(bytes).unwrap(); + let dt = file.dataset("counts").unwrap().raw_datatype().unwrap(); + assert!( + matches!( + dt, + Datatype::FixedPoint { + size: 4, + signed: true, + .. + } + ), + "{dt:?}" + ); + // The full type pairs with read_selection's bytes. + let raw = file + .dataset("counts") + .unwrap() + .read_selection(&Selection::All) + .unwrap(); + assert_eq!( + clawhdf5_format::data_read::read_as_i64(&raw, &dt).unwrap(), + vec![10, 20, 30] + ); + } + #[test] fn root_group_datasets() { let bytes = make_simple_file(); diff --git a/crates/clawhdf5/src/reader.rs b/crates/clawhdf5/src/reader.rs index 6e506b4..83cbfa8 100644 --- a/crates/clawhdf5/src/reader.rs +++ b/crates/clawhdf5/src/reader.rs @@ -424,6 +424,15 @@ impl<'f> Dataset<'f> { Ok(classify_datatype(&dt)) } + /// Returns the dataset's full datatype as stored in the file (byte order, + /// string padding, compound layout, ...), with a committed datatype + /// resolved. Use it with the `clawhdf5_format::data_read` converters on + /// the bytes [`read_selection`](Self::read_selection) returns, for types + /// the typed `read_*` methods do not cover. + pub fn raw_datatype(&self) -> Result { + self.datatype() + } + /// Read all data as `f64` values. pub fn read_f64(&self) -> Result, Error> { let dt = self.datatype()?; diff --git a/docs/known-issues.md b/docs/known-issues.md index 13ebc32..9a2ae44 100644 --- a/docs/known-issues.md +++ b/docs/known-issues.md @@ -473,6 +473,23 @@ the same agent-store interop test. **Fix:** an empty contiguous dataset gets the undefined address (all `0xff`), which is what libhdf5 itself writes. +## `clawhdf5-wasm` (browser) limits + +**Status:** open (by design for now; added 2026-09-26). + +- The whole file is held in memory: `open()` takes its bytes. There are no + HTTP range reads, so a multi-GB file does not fit a browser tab. +- Compound, reference, opaque, bitfield, time and VL-sequence datasets are + refused with an error naming the type; attributes of those types come back + as `value: null` with their `dtype`. +- No Zstd or SZIP (both link C): such datasets fail with + `unsupported filter: 32015` / `: 4`. pcodec is not enabled either. +- External links and virtual-dataset sources in other files cannot be + followed (no file system). +- Variable-length string datasets are read by decoding `read_selection`'s + bytes with `clawhdf5_format::vl_data` in the wasm crate; `File` itself still + cannot (see the audit gaps above). + ## The Node.js package (`packages/clawhdf5-node`) does not work **Status:** open (found 2026-09-25). Unpublished; not built or tested in CI. diff --git a/examples/wasm-viewer/.gitignore b/examples/wasm-viewer/.gitignore new file mode 100644 index 0000000..f57bf2b --- /dev/null +++ b/examples/wasm-viewer/.gitignore @@ -0,0 +1,2 @@ +# Generated by build.sh. +/pkg/ diff --git a/examples/wasm-viewer/README.md b/examples/wasm-viewer/README.md new file mode 100644 index 0000000..1a5c47d --- /dev/null +++ b/examples/wasm-viewer/README.md @@ -0,0 +1,97 @@ +# HDF5 viewer in the browser + +A single page that opens an HDF5 or NetCDF-4 file entirely in the browser +with `clawhdf5-wasm` (clawhdf5's reader compiled to WebAssembly): drop a +file, browse its groups, and look at a dataset's type, shape, attributes +and values (a 50 x 12 window at a time, read as a hyperslab, with the +leading dimensions of a 3-D+ dataset held at chosen indices). The file never +leaves the page. + +## Build and open + +```bash +rustup target add wasm32-unknown-unknown +cargo install wasm-bindgen-cli --version 0.2.129 # must equal the crate version; build.sh checks +bash examples/wasm-viewer/build.sh # writes examples/wasm-viewer/pkg/ (not committed) +python3 -m http.server -d examples/wasm-viewer 8000 # wasm cannot load from file:// +``` + +Then open . `?file=&path=` opens a +file from a URL (same origin, or one serving CORS headers) and selects an +object in it, e.g. `?file=data/run1.h5&path=/results/energy`. + +## JavaScript API + +```js +import init, { open } from "./pkg/clawhdf5_wasm.js"; +await init(); +const f = open(new Uint8Array(await blob.arrayBuffer())); +f.list("/"); // [{ name, kind: "group" | "dataset" }], groups first +f.info("/grid"); // { shape, maxshape, dtype, elementShape } +f.attrs("/grid"); // [{ name, value, dtype }] +f.read("/grid"); // { shape, dtype, data } +f.readHyperslab("/grid", [0, 0], [10, 5], [2, 1]); // start, count, stride?, block? +f.free(); +``` + +`data` is the typed array of the stored width (`Float64Array`, +`Float32Array` also for `f16`, `Int8Array` ... `BigInt64Array`, +`BigUint64Array`), or an array of strings for fixed- and variable-length +strings and enumerations (h5py booleans read as `"TRUE"`/`"FALSE"`). Array +datatypes are flattened, their dimensions appended to `shape`. Anything +else throws an `Error` naming the type. + +## Limits + +- Read-only, and the whole file is held in memory (no range requests). +- Compound, reference, opaque and variable-length-sequence datasets are + refused with an error. Attributes of those types are listed with + `value: null` and their `dtype`. +- No Zstd or SZIP filters (they link C): such a dataset fails with + `unsupported filter`. Deflate, shuffle, Fletcher-32, LZ4, N-Bit and + scale-offset are read (within the limits in `docs/known-issues.md`). +- Virtual datasets whose sources are in other files, and external links, + cannot be followed: there is no file system. + +## Tests + +`test/run.sh` builds the package, writes `fixture.h5` (h5py) and +`fixture.nc` (netCDF4) with `test/make_fixture.py`, then: + +- runs `test/test.mjs` under Node: every dataset (whole and a strided + hyperslab), listing and attribute is compared with what libhdf5 reads + back, error paths are checked, and so are the page's DOM-free helpers + (`viewer-lib.js`); +- runs `test/browser.sh`: loads the page in headless Chromium with + `?file=fixture.h5&path=...` for eight objects and checks the rendered tree, + types, shapes, attribute and value cells, and the error shown for an + unsupported type. Skipped when no Chromium is found (`CHROME` names one; + a Playwright download under `~/.cache/ms-playwright` is picked up). + Drag-and-drop and the file picker are not driven by it; they share + `load()` with the `?file=` path. + +The same expectations are checked natively, without Node, by +`crates/clawhdf5-wasm/tests/h5py_interop.rs`, which is what CI runs (the CI +container has no Node or browser). + +## Size + +Measured 2026-09-26 on tank (rustc 1.98.1, wasm-bindgen 0.2.129, gzip 1.14, +`gzip -9 -n`), after `bash examples/wasm-viewer/build.sh`: + +| | raw | gzip -9 | +|---|---:|---:| +| `pkg/clawhdf5_wasm_bg.wasm` (profile `wasm-release`, opt-level `s`) | 627,501 B | 191,639 B | +| `pkg/clawhdf5_wasm.js` (wasm-bindgen glue) | 21,826 B | 4,487 B | +| same wasm at opt-level `z` | 693,068 B | 192,550 B | +| same wasm at opt-level `3` | 544,035 B | 198,803 B | +| h5wasm 0.10.3: wasm embedded in `dist/esm/hdf5_util.js` | 3,544,184 B | 907,096 B | +| h5wasm 0.10.3: `dist/esm/hdf5_util.js` as shipped | 4,150,134 B | 986,699 B | + +h5wasm figures: `npm pack h5wasm@0.10.3` (npm reports +`dist.unpackedSize` 14,731,385 B for the whole package), wasm extracted from +the `binaryDecode` literal in `hdf5_util.js`. h5wasm is the whole of libhdf5 +(writing, every datatype, plugins), so this compares download size, not +equal functionality. No `wasm-opt` pass was applied (binaryen is not +installed on tank). opt-level `s` is used because it is the smallest +compressed. diff --git a/examples/wasm-viewer/build.sh b/examples/wasm-viewer/build.sh new file mode 100755 index 0000000..6af81bf --- /dev/null +++ b/examples/wasm-viewer/build.sh @@ -0,0 +1,39 @@ +#!/usr/bin/env bash +# Build the clawhdf5-wasm package the viewer loads, into examples/wasm-viewer/pkg/. +# +# Needs the wasm32-unknown-unknown target and the wasm-bindgen CLI at the +# exact version cargo resolves for the wasm-bindgen crate: +# rustup target add wasm32-unknown-unknown +# cargo install wasm-bindgen-cli --version +# +# Then serve this directory over HTTP (browsers do not load wasm modules from +# file://) and open it: +# python3 -m http.server -d examples/wasm-viewer 8000 +set -euo pipefail + +HERE="$(cd "$(dirname "$0")" && pwd)" +ROOT="$(cd "$HERE/../.." && pwd)" +cd "$ROOT" + +# The resolved crate version (Cargo.lock is not committed, so ask cargo). +want=$(cargo pkgid wasm-bindgen | sed 's/.*[@#]//') +if ! command -v wasm-bindgen >/dev/null; then + echo "wasm-bindgen CLI not found: cargo install wasm-bindgen-cli --version $want" >&2 + exit 1 +fi +have=$(wasm-bindgen --version | awk '{print $2}') +if [ "$want" != "$have" ]; then + echo "wasm-bindgen CLI is $have but the crate is $want:" >&2 + echo " cargo install wasm-bindgen-cli --version $want" >&2 + exit 1 +fi + +cargo build -p clawhdf5-wasm --target wasm32-unknown-unknown --profile wasm-release + +target_dir=$(cargo metadata --format-version 1 --no-deps \ + | sed -n 's/.*"target_directory":"\([^"]*\)".*/\1/p') +wasm="$target_dir/wasm32-unknown-unknown/wasm-release/clawhdf5_wasm.wasm" + +rm -rf "$HERE/pkg" +wasm-bindgen --target web --out-dir "$HERE/pkg" "$wasm" +echo "built $HERE/pkg ($(wc -c < "$HERE/pkg/clawhdf5_wasm_bg.wasm") bytes of wasm)" diff --git a/examples/wasm-viewer/index.html b/examples/wasm-viewer/index.html new file mode 100644 index 0000000..d477e49 --- /dev/null +++ b/examples/wasm-viewer/index.html @@ -0,0 +1,278 @@ + + + + + +HDF5 Viewer + + + +
+

HDF5 Viewer

+ no file + +
+
+ +
+
+

Drop an HDF5 or NetCDF-4 file here, or use “Open file…”.

+

The file is read in this page by clawhdf5 compiled to WebAssembly; it is not uploaded anywhere.

+

+
+
+
+ + + diff --git a/examples/wasm-viewer/test/browser.sh b/examples/wasm-viewer/test/browser.sh new file mode 100644 index 0000000..a9ea5aa --- /dev/null +++ b/examples/wasm-viewer/test/browser.sh @@ -0,0 +1,96 @@ +#!/usr/bin/env bash +# Load the viewer page in headless Chromium and check what it renders. +# +# browser.sh FIXTURE_DIR +# +# FIXTURE_DIR holds fixture.h5 from make_fixture.py; ../pkg must be built. +# The page is opened with ?file=fixture.h5&path=, which fetches the +# file, builds the tree down to and shows it; the rendered DOM is +# dumped and checked for the values libhdf5 reads. +# +# Browser: $CHROME, else chromium/google-chrome on PATH, else a Playwright +# download under ~/.cache/ms-playwright. Exit 3 when none is found. +# BROWSER_DEBUG=/some/prefix saves each rendered page as prefix..html. +set -euo pipefail + +HERE="$(cd "$(dirname "$0")" && pwd)" +FIX="$(cd "$1" && pwd)" +PY="${CLAWHDF5_PYTHON:-python3}" + +chrome="${CHROME:-}" +if [ -z "$chrome" ]; then + for c in chromium chromium-browser google-chrome chrome-headless-shell; do + if command -v "$c" >/dev/null; then chrome="$(command -v "$c")"; break; fi + done +fi +if [ -z "$chrome" ]; then + chrome="$(ls -d "$HOME"/.cache/ms-playwright/chromium_headless_shell-*/chrome-headless-shell-linux64/chrome-headless-shell 2>/dev/null | tail -1 || true)" +fi +if [ -z "$chrome" ] || [ ! -x "$chrome" ]; then + echo "no Chromium found (set CHROME)" >&2 + exit 3 +fi + +root="$(mktemp -d)" +server="" +cleanup() { + [ -n "$server" ] && kill "$server" 2>/dev/null || true + rm -rf "$root" +} +trap cleanup EXIT +ln -s "$HERE/../index.html" "$HERE/../viewer-lib.js" "$HERE/../pkg" "$FIX/fixture.h5" "$root/" + +port=$("$PY" -c 'import socket; s = socket.socket(); s.bind(("127.0.0.1", 0)); print(s.getsockname()[1])') +"$PY" -m http.server --bind 127.0.0.1 --directory "$root" "$port" >/dev/null 2>&1 & +server=$! +for _ in $(seq 50); do + "$PY" -c "import urllib.request; urllib.request.urlopen('http://127.0.0.1:$port/index.html')" 2>/dev/null && break + sleep 0.1 +done + +fails=0 +# A fresh profile per page: a second instance on the same profile fails. +render() { + local profile + profile="$(mktemp -d "$root/profile.XXXXXX")" + "$chrome" --headless --no-sandbox --disable-gpu --user-data-dir="$profile" \ + --virtual-time-budget=20000 \ + --dump-dom "http://127.0.0.1:$port/index.html?file=fixture.h5&path=$1" 2>/dev/null +} +# expect PATH TEXT...: every TEXT appears in the page rendered for PATH. +expect() { + local path="$1" dom + shift + dom="$(render "$path")" + [ -n "${BROWSER_DEBUG:-}" ] && printf "%s\n" "$dom" > "$BROWSER_DEBUG.$(echo "$path" | tr / _).html" + for text in "$@"; do + if ! grep -qF -- "$text" <<<"$dom"; then + echo "FAIL: page for $path lacks: $text" >&2 + fails=$((fails + 1)) + fi + done + echo "rendered $path" +} + +# Tree (root expanded; the group row carries its path) and root attributes. +expect "/" 'data-path="/sensors"' 'data-path="/grid"' 'title"wasm fixture"' \ + 'big9223372036854775813' '(compound{x: f64, n: i32})' +# A chunked, deflated 2-D dataset: type, shape, the first window of values. +expect "/grid" '
f64
' '
(6, 10)
' '9' '0.25' '14.75' \ + 'showing rows 0–5, columns 0–9 of 60 values' +# Nested path revealed through the tree; big-endian float32. +expect "/sensors/temp" 'data-path="/sensors/temp"' '
f32
' '21.5' '22.25' +# 64-bit integers stay exact; strings; array datatype cells. +expect "/u64" '18446744073709551615' +expect "/vlen_str" '"двa"' '
vlen string
' +expect "/pairs" '[2, 3]' '
array[2]<i32>
' +# 3-D: leading dimension held at 0, window over the last two. +expect "/cube" '
(2, 5, 6)
' '29' 'dim 0' +# Unsupported type: an error, not values. +expect "/table" 'class="error"' 'reading compound{x: f64, n: i32} datasets is not supported' + +if [ "$fails" -gt 0 ]; then + echo "browser: $fails checks failed" >&2 + exit 1 +fi +echo "browser: all checks passed ($chrome)" diff --git a/examples/wasm-viewer/test/make_fixture.py b/examples/wasm-viewer/test/make_fixture.py new file mode 100644 index 0000000..8c7722c --- /dev/null +++ b/examples/wasm-viewer/test/make_fixture.py @@ -0,0 +1,203 @@ +"""Write HDF5 and NetCDF-4 test files with h5py/netCDF4, and what libhdf5 +reads back from them, for the clawhdf5-wasm tests. + + python make_fixture.py OUT_DIR + +writes OUT_DIR/fixture.h5, OUT_DIR/fixture.nc and OUT_DIR/expected.json. +Both the Rust test (crates/clawhdf5-wasm/tests/h5py_interop.rs, native) and +the Node test (test.mjs, the built wasm package) compare against the same +expected.json, so the two check the same values. + +Every expected value comes from h5py reading the file back (numpy slicing +for hyperslabs), never from the arrays that were written. Integers are +encoded as strings so JSON.parse keeps 64-bit values exact. +""" + +import json +import sys +import warnings +from pathlib import Path + +import h5py +import netCDF4 +import numpy as np + +try: # registers the LZ4/Zstd filters with libhdf5; optional + import hdf5plugin +except ImportError: + hdf5plugin = None + +# netCDF4 1.7 trips numpy 2.5's shape-setting deprecation on assignment. +warnings.filterwarnings("ignore", category=DeprecationWarning) + +out = Path(sys.argv[1]) +out.mkdir(parents=True, exist_ok=True) +h5 = out / "fixture.h5" +nc = out / "fixture.nc" + +rng = np.random.default_rng(7) +with h5py.File(h5, "w") as f: + f.attrs["title"] = "wasm fixture" + f.attrs["version"] = np.int64(3) + f.attrs["scale"] = np.array([0.5, 2.0]) + f.attrs["big"] = np.uint64(2**63 + 5) + f.attrs.create("vlen_note", "héllo", dtype=h5py.string_dtype()) + # No plain JavaScript form: listed with value null and its type. + f.attrs["origin"] = np.array((1.5, 2), dtype=[("x", "f4")) + f.create_dataset("f16", data=np.array([0.5, -1.25, 65504], dtype=" 2 else 1] + [2] * (obj.ndim - 1) + count = [min(c, (n - s - 1) // st + 1) + for c, s, st, n in zip(count, start, stride, obj.shape)] + return (start, count, stride) + + +json.dump({"fixture.h5": describe(h5), "fixture.nc": describe(nc)}, + open(out / "expected.json", "w"), indent=1, ensure_ascii=False) diff --git a/examples/wasm-viewer/test/run.sh b/examples/wasm-viewer/test/run.sh new file mode 100755 index 0000000..c1cdae0 --- /dev/null +++ b/examples/wasm-viewer/test/run.sh @@ -0,0 +1,37 @@ +#!/usr/bin/env bash +# Build the wasm package (../build.sh), test it under Node against files +# written by h5py and netCDF4 (make_fixture.py), then load the viewer page +# in headless Chromium if one is found (browser.sh). +# +# Needs node, the wasm-bindgen CLI (see ../build.sh) and a Python with h5py, +# netCDF4 and numpy: CLAWHDF5_PYTHON names it (default python3). Without that +# Python the test is skipped, unless CLAWHDF5_REQUIRE_INTEROP=1. +set -euo pipefail + +HERE="$(cd "$(dirname "$0")" && pwd)" +PY="${CLAWHDF5_PYTHON:-python3}" + +command -v node >/dev/null || { echo "node not found" >&2; exit 1; } +if ! "$PY" -c "import h5py, netCDF4, numpy" >/dev/null 2>&1; then + if [ "${CLAWHDF5_REQUIRE_INTEROP:-0}" = "1" ]; then + echo "CLAWHDF5_REQUIRE_INTEROP=1 but $PY lacks h5py/netCDF4/numpy" >&2 + exit 1 + fi + echo "SKIP: $PY lacks h5py/netCDF4/numpy" + exit 0 +fi + +bash "$HERE/../build.sh" +fix="$(mktemp -d)" +trap 'rm -rf "$fix"' EXIT +"$PY" "$HERE/make_fixture.py" "$fix" +node "$HERE/test.mjs" "$HERE/../pkg" "$fix" + +# The page itself, in headless Chromium when one is available. +status=0 +bash "$HERE/browser.sh" "$fix" || status=$? +if [ "$status" = 3 ]; then + echo "SKIP: viewer page in a browser (no Chromium; set CHROME)" +elif [ "$status" != 0 ]; then + exit "$status" +fi diff --git a/examples/wasm-viewer/test/test.mjs b/examples/wasm-viewer/test/test.mjs new file mode 100644 index 0000000..80ca30a --- /dev/null +++ b/examples/wasm-viewer/test/test.mjs @@ -0,0 +1,143 @@ +// Node test of the built wasm package (the exact pkg/ the viewer page loads) +// and the viewer's DOM-free helpers. Run by test/run.sh: +// node test.mjs PKG_DIR FIXTURE_DIR +// FIXTURE_DIR holds fixture.h5, fixture.nc and expected.json from +// make_fixture.py (values as libhdf5 reads them back). +import assert from "node:assert/strict"; +import { readFileSync } from "node:fs"; +import { join } from "node:path"; +import { pathToFileURL } from "node:url"; + +const [pkgDir, fixDir] = process.argv.slice(2); +const pkg = await import(pathToFileURL(join(pkgDir, "clawhdf5_wasm.js"))); +pkg.initSync({ module: readFileSync(join(pkgDir, "clawhdf5_wasm_bg.wasm")) }); +const lib = await import(pathToFileURL(join(import.meta.dirname, "..", "viewer-lib.js"))); + +let checks = 0; +const eq = (a, b, msg) => { assert.deepEqual(a, b, msg); checks++; }; + +const ARRAY_TYPES = { + f32: Float32Array, f64: Float64Array, i8: Int8Array, i16: Int16Array, i32: Int32Array, + i64: BigInt64Array, u8: Uint8Array, u16: Uint16Array, u32: Uint32Array, u64: BigUint64Array, + strings: Array, +}; + +function values(kind, data) { + const arr = Array.from(data); + if (kind === "f32" || kind === "f64" || kind === "strings") return arr; + return arr.map(String); +} + +function checkAttr(ctx, a, want) { + const v = a.value; + if ("raw" in want) { + eq(v, null, ctx); + assert.ok(a.dtype.includes(want.raw), `${ctx}: ${a.dtype}`); + return; + } + eq(a.dtype, null, `${ctx} dtype`); + if ("string" in want) return eq(v, want.string, ctx); + if ("strings" in want) return eq(v, want.strings, ctx); + if ("int" in want) { + if (want.scalar) { + assert.ok(typeof v === "number" || typeof v === "bigint", ctx); + return eq([String(v)], want.int, ctx); + } + assert.ok(v instanceof BigInt64Array || v instanceof BigUint64Array, ctx); + return eq(Array.from(v, String), want.int, ctx); + } + if ("float" in want) { + if (want.scalar) return eq([v], want.float, ctx); + assert.ok(v instanceof Float64Array, ctx); + return eq(Array.from(v), want.float, ctx); + } + assert.fail(`${ctx}: unknown expectation ${JSON.stringify(want)}`); +} + +const expected = JSON.parse(readFileSync(join(fixDir, "expected.json"), "utf8")); +for (const [name, exp] of Object.entries(expected)) { + const file = pkg.open(new Uint8Array(readFileSync(join(fixDir, name)))); + + for (const [path, want] of Object.entries(exp.lists)) { + eq(file.kind(path), "group", `${name}:${path} kind`); + const list = file.list(path); + for (const [kind, key] of [["group", "groups"], ["dataset", "datasets"]]) { + eq(list.filter((c) => c.kind === kind).map((c) => c.name).sort(), want[key], `${name}:${path} ${key}`); + } + } + + for (const [path, want] of Object.entries(exp.datasets)) { + const ctx = `${name}:${path}`; + eq(file.kind(path), "dataset", `${ctx} kind`); + if (want.unavailable) { + // The wasm build has no zstd (it links C): a clear error, no data. + assert.throws(() => file.read(path), (e) => e.message.includes(want.unavailable), ctx); + checks++; + continue; + } + const info = file.info(path); + eq([...info.shape, ...info.elementShape], want.shape, `${ctx} info shape`); + const r = file.read(path); + eq(r.shape, want.shape, `${ctx} shape`); + eq(r.dtype, info.dtype, `${ctx} dtype`); + assert.ok(r.data instanceof ARRAY_TYPES[want.kind], `${ctx}: ${r.data.constructor.name} for ${want.kind}`); + eq(values(want.kind, r.data), want.values, ctx); + if (want.slab) { + const s = want.slab; + const part = file.readHyperslab(path, s.start, s.count, s.stride); + eq(part.shape, s.shape, `${ctx} slab shape`); + eq(values(want.kind, part.data), s.values, `${ctx} slab`); + } + } + + for (const [path, what] of Object.entries(exp.errors)) { + assert.throws(() => file.read(path), (e) => e instanceof Error && e.message.includes(what), `${name}:${path}`); + checks++; + } + + for (const [path, want] of Object.entries(exp.attrs)) { + const attrs = file.attrs(path); + eq(file.attrErrors(path), [], `${name}:${path} attr errors`); + const seen = attrs.filter((a) => !a.name.startsWith("_") && !exp.skip_attrs.includes(a.name)); + eq(seen.map((a) => a.name).sort(), Object.keys(want).sort(), `${name}:${path} attr names`); + for (const a of seen) checkAttr(`${name}:${path}@${a.name}`, a, want[a.name]); + } + file.free(); +} + +// Errors reach JavaScript as thrown Errors, never as data. +const h5 = pkg.open(new Uint8Array(readFileSync(join(fixDir, "fixture.h5")))); +const throwsMsg = (fn, re) => { assert.throws(fn, (e) => e instanceof Error && re.test(e.message)); checks++; }; +throwsMsg(() => pkg.open(new Uint8Array(64)), /./); +throwsMsg(() => h5.read("/nope"), /./); +throwsMsg(() => h5.list("/grid"), /not a group/); +throwsMsg(() => h5.readHyperslab("/grid", [0], [1]), /dimensions/); +throwsMsg(() => h5.readHyperslab("/grid", [5, 0], [2, 1]), /exceeds/); +throwsMsg(() => h5.readHyperslab("/grid", [-1, 0], [1, 1]), /non-negative integers/); +throwsMsg(() => h5.readHyperslab("/grid", [0.5, 0], [1, 1]), /non-negative integers/); +// Info for a dataset with an unlimited dimension (netCDF "time"). +const nc = pkg.open(new Uint8Array(readFileSync(join(fixDir, "fixture.nc")))); +eq(nc.info("/time").maxshape, [null], "unlimited dimension is null"); +// Big integers stay exact. +eq(h5.read("/u64").data[0], 18446744073709551615n, "u64 max"); +eq(typeof pkg.version(), "string", "version"); + +// Viewer helpers. +eq(lib.joinPath("/", "a"), "/a", "joinPath root"); +eq(lib.joinPath("/a", "b"), "/a/b", "joinPath nested"); +eq(lib.viewWindow([], {}), null, "scalar window"); +eq(lib.viewWindow([7], { row: 5, rows: 50 }), { start: [5], count: [2], rows: 2, cols: 1, row: 5, col: 0 }, "1-D window"); +const w = lib.viewWindow([2, 5, 6], { row: 1, col: 4, rows: 3, cols: 5, fixed: [1] }); +eq(w, { start: [1, 1, 4], count: [1, 3, 2], rows: 3, cols: 2, row: 1, col: 4 }, "3-D window"); +// The window the page would request reads the same values as a direct slab. +const cube = h5.readHyperslab("/cube", w.start, w.count); +eq(lib.toRows(cube.data, w.rows, w.cols), [["40", "41"], ["46", "47"], ["52", "53"]], "cube window cells"); +const pairs = h5.readHyperslab("/pairs", [1], [2]); +eq(lib.toRows(pairs.data, 2, 1, lib.perElement([2])), [["[2, 3]"], ["[4, 5]"]], "array-type cells"); +eq(lib.formatValue(0.1 + 0.2), "0.3", "float formatting"); +eq(lib.formatValue(2n ** 64n - 1n), "18446744073709551615", "bigint formatting"); +eq(lib.formatValue("x"), '"x"', "string formatting"); +h5.free(); +nc.free(); + +console.log(`wasm package: ${checks} checks passed`); diff --git a/examples/wasm-viewer/viewer-lib.js b/examples/wasm-viewer/viewer-lib.js new file mode 100644 index 0000000..72c5bf3 --- /dev/null +++ b/examples/wasm-viewer/viewer-lib.js @@ -0,0 +1,77 @@ +// DOM-free helpers for the viewer, so Node can test them (test/test.mjs). + +/** Child path of `name` in the group at `parent`. */ +export function joinPath(parent, name) { + return parent === "/" ? `/${name}` : `${parent}/${name}`; +} + +/** One value as display text. */ +export function formatValue(v) { + if (v === null || v === undefined) return ""; + if (typeof v === "bigint") return v.toString(); + if (typeof v === "number") { + if (Number.isInteger(v)) return String(v); + return String(Number(v.toPrecision(7))); + } + if (typeof v === "string") return JSON.stringify(v); + if (ArrayBuffer.isView(v) || Array.isArray(v)) { + const items = Array.from(v.slice(0, 16), formatValue); + if (v.length > 16) items.push(`… (${v.length} values)`); + return `[${items.join(", ")}]`; + } + return String(v); +} + +/** + * The window of a dataset to show: a hyperslab over its `shape` with the + * last dimension as columns, the one before as rows, and any leading + * dimensions held at `fixed` indices. `row`/`col` are the window's top-left + * corner. Returns null for a scalar (read it whole). + */ +export function viewWindow(shape, { row = 0, col = 0, rows = 50, cols = 12, fixed = [] } = {}) { + const rank = shape.length; + if (rank === 0) return null; + const clamp = (x, n) => Math.max(0, Math.min(x, Math.max(0, n - 1))); + if (rank === 1) { + const r0 = clamp(row, shape[0]); + const n = Math.max(0, Math.min(rows, shape[0] - r0)); + return { start: [r0], count: [n], rows: n, cols: 1, row: r0, col: 0 }; + } + const lead = shape.slice(0, rank - 2).map((n, i) => clamp(fixed[i] ?? 0, n)); + const nr = shape[rank - 2]; + const nc = shape[rank - 1]; + const r0 = clamp(row, nr); + const c0 = clamp(col, nc); + const r = Math.max(0, Math.min(rows, nr - r0)); + const c = Math.max(0, Math.min(cols, nc - c0)); + return { + start: [...lead, r0, c0], + count: [...lead.map(() => 1), r, c], + rows: r, + cols: c, + row: r0, + col: c0, + }; +} + +/** + * Split row-major `data` into `rows` x `cols` cells of display text; each + * cell holds `per` consecutive values (the elements of an array datatype). + */ +export function toRows(data, rows, cols, per = 1) { + const out = []; + for (let r = 0; r < rows; r++) { + const row = []; + for (let c = 0; c < cols; c++) { + const i = (r * cols + c) * per; + row.push(per === 1 ? formatValue(data[i]) : formatValue(data.slice(i, i + per))); + } + out.push(row); + } + return out; +} + +/** Number of values an array datatype packs into each element. */ +export function perElement(elementShape) { + return elementShape.reduce((a, b) => a * b, 1); +} diff --git a/scripts/ci-test.sh b/scripts/ci-test.sh index df9ecdd..bd64a50 100755 --- a/scripts/ci-test.sh +++ b/scripts/ci-test.sh @@ -108,14 +108,17 @@ run_step "cargo clippy (fast-deflate / zlib-ng)" cargo clippy \ # that: fail if a crate that compiles C (a *-sys crate, cc or cmake) enters the # default dependency tree of any of them. clawhdf5-migrate (bundled SQLite), # clawhdf5-napi (Node) and clawhdf5-gpu (graphics drivers) are exempt. +# js-sys (clawhdf5-wasm's bindings to JavaScript) builds no C. no_c_in_default_build() { local crate found=0 for crate in clawhdf5-format clawhdf5-io clawhdf5-filters clawhdf5 \ clawhdf5-agent clawhdf5-ann clawhdf5-accel clawhdf5-netcdf4 clawhdf5-cli \ - clawhdf5-tools; do + clawhdf5-tools \ + clawhdf5-wasm; do local c_deps c_deps=$(cargo tree -q -p "$crate" -e normal,build --prefix none \ - | grep -E '^([a-z0-9_-]+-sys|cc|cmake) v' | sort -u) + | grep -E '^([a-z0-9_-]+-sys|cc|cmake) v' \ + | grep -v '^js-sys v' | sort -u) if [ -n "$c_deps" ]; then echo "$crate pulls in C by default:" echo "$c_deps" | sed 's/^/ /' @@ -126,6 +129,33 @@ no_c_in_default_build() { } run_step "no C in the default build (core crates)" no_c_in_default_build +# The reader in the browser: the facade's read path must build for +# wasm32-unknown-unknown (no mmap, no threads, no file system), and the +# wasm-bindgen crate must build and lint there. Needs +# `rustup target add wasm32-unknown-unknown`. +run_step "wasm32 build (clawhdf5, no default features)" cargo build \ + -p clawhdf5 \ + --target wasm32-unknown-unknown \ + --no-default-features +run_step "wasm32 clippy (clawhdf5-wasm)" cargo clippy \ + -p clawhdf5-wasm \ + --target wasm32-unknown-unknown \ + --all-targets \ + -- -D warnings + +# The built wasm package, run under Node against h5py/netCDF4-written files, +# and the viewer page in headless Chromium when one is found. +# Needs node and the wasm-bindgen CLI, which the CI container does not have; +# the same expectations are checked natively by clawhdf5-wasm's h5py_interop +# test in the cargo test step. +if command -v node >/dev/null && command -v wasm-bindgen >/dev/null; then + run_step "wasm package under Node (+ browser)" bash "$SCRIPT_DIR/../examples/wasm-viewer/test/run.sh" +else + echo "" + echo "==> [wasm package under Node] SKIPPED: needs node and wasm-bindgen" + STEPS+=("SKIP: wasm package under Node") +fi + # The workspace declares a minimum Rust version (rust-version in Cargo.toml); # check that it really builds there, so the README badge and the manifests # cannot drift from the truth. Separate target dir: a different toolchain