From a42b6466899d6a3b0456214dd95cc3cc7e3f4a58 Mon Sep 17 00:00:00 2001 From: osobh Date: Sat, 26 Sep 2026 00:00:46 -0500 Subject: [PATCH] feat(wasm): clawhdf5-wasm, the reader for JavaScript via wasm-bindgen open(bytes) -> H5File with kind/list/info/attrs/attrErrors/read/ readHyperslab. Numeric data comes back in the typed array of the stored width (Int16Array for i16, BigInt64Array for i64, Float32Array for f32/f16, ...), strings and enum names as string arrays, array datatypes flattened with their dims appended to the shape. Compound, reference, opaque and VL-sequence datasets are refused with an error naming the type; nothing is returned as reinterpreted bytes. The logic is in a plain-Rust core module, tested natively: unit tests, and h5py_interop, which compares every dataset, hyperslab, listing and attribute of an h5py- and a netCDF4-written file with what libhdf5 reads back (generator shared with the Node test of the built package). No mmap, no threads; lz4 is on, zstd/szip (C) are not. A wasm-release profile (opt-level s, LTO) serves the browser build. ci-test.sh lints the crate for wasm32 and checks it builds no C. Co-Authored-By: Claude Opus 5.5 (1M context) --- CLAUDE.md | 3 +- Cargo.toml | 10 + README.md | 5 +- crates/clawhdf5-wasm/Cargo.toml | 30 + crates/clawhdf5-wasm/src/core.rs | 603 +++++++++++++++++++++ crates/clawhdf5-wasm/src/lib.rs | 244 +++++++++ crates/clawhdf5-wasm/tests/h5py_interop.rs | 252 +++++++++ examples/wasm-viewer/test/make_fixture.py | 184 +++++++ scripts/ci-test.sh | 11 +- 9 files changed, 1337 insertions(+), 5 deletions(-) create mode 100644 crates/clawhdf5-wasm/Cargo.toml create mode 100644 crates/clawhdf5-wasm/src/core.rs create mode 100644 crates/clawhdf5-wasm/src/lib.rs create mode 100644 crates/clawhdf5-wasm/tests/h5py_interop.rs create mode 100644 examples/wasm-viewer/test/make_fixture.py diff --git a/CLAUDE.md b/CLAUDE.md index 3f8d9c1..4969c6a 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -5,7 +5,7 @@ Pure-Rust HDF5 format implementation with HNSW vector search, WAL-backed persist ## Architecture -Cargo workspace with 16 crates under `crates/` (plus `libaec-sys`, an internal FFI bindings crate for the optional `szip` feature): +Cargo workspace with 17 crates under `crates/` (plus `libaec-sys`, an internal FFI bindings crate for the optional `szip` feature): | Crate | Role | |-------|------| @@ -24,6 +24,7 @@ Cargo workspace with 16 crates under `crates/` (plus `libaec-sys`, an internal F | `clawhdf5-cli` | Command-line interface | | `clawhdf5-napi` | Node.js native addon bindings | | `clawhdf5-py` | PyO3 Python bindings | +| `clawhdf5-wasm` | WebAssembly (wasm-bindgen) reader for the browser; demo in `examples/wasm-viewer/` | | `clawhdf5-bench` | Benchmark suite | ## Key Features diff --git a/Cargo.toml b/Cargo.toml index 2e1f6fe..3fa2124 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -16,6 +16,7 @@ members = [ "crates/clawhdf5-cli", "crates/clawhdf5-napi", "crates/clawhdf5-bench", + "crates/clawhdf5-wasm", "crates/libaec-sys", ] resolver = "2" @@ -34,3 +35,12 @@ tempfile = "3" criterion = { version = "0.5", features = ["html_reports"] } half = "2.7" serde = { version = "1", features = ["derive"] } + +# The browser build of clawhdf5-wasm (examples/wasm-viewer/build.sh): size +# over speed, whole-program optimisation. Native profiles are unaffected. +[profile.wasm-release] +inherits = "release" +opt-level = "s" +lto = true +codegen-units = 1 +panic = "abort" diff --git a/README.md b/README.md index 5d4c707..eb2b27a 100644 --- a/README.md +++ b/README.md @@ -587,7 +587,7 @@ let exported = backend.export_markdown("MEMORY.md")?; ## Crate Map ``` -clawhdf5 workspace (16 crates, ~86K lines of Rust in src/, ~104K with tests +clawhdf5 workspace (17 crates, ~86K lines of Rust in src/, ~104K with tests and benches; plus libaec-sys, an internal FFI bindings crate for the optional szip feature) │ @@ -610,7 +610,8 @@ clawhdf5 workspace (16 crates, ~86K lines of Rust in src/, ~104K with tests │ ├── Bindings │ ├── clawhdf5-py — Python (PyO3) -│ └── clawhdf5-napi — Node.js (napi-rs) +│ ├── clawhdf5-napi — Node.js (napi-rs) +│ └── clawhdf5-wasm — Browser (WebAssembly, wasm-bindgen; read-only) │ └── Tooling └── clawhdf5-bench — Benchmark suite diff --git a/crates/clawhdf5-wasm/Cargo.toml b/crates/clawhdf5-wasm/Cargo.toml new file mode 100644 index 0000000..303837a --- /dev/null +++ b/crates/clawhdf5-wasm/Cargo.toml @@ -0,0 +1,30 @@ +[package] +name = "clawhdf5-wasm" +version.workspace = true +edition.workspace = true +rust-version.workspace = true +description = "Read HDF5 and NetCDF-4 files in the browser: clawhdf5's reader compiled to WebAssembly" +license.workspace = true +repository.workspace = true +keywords = ["hdf5", "netcdf", "wasm", "browser"] +categories = ["wasm", "parser-implementations", "science"] +# Distributed as the wasm-bindgen package built by examples/wasm-viewer/build.sh. +publish = false + +[lib] +# cdylib for wasm-bindgen; rlib so the pure-Rust core is tested natively. +crate-type = ["cdylib", "rlib"] + +[dependencies] +# No mmap (there is no file system) and no `parallel` (no threads on +# wasm32-unknown-unknown). lz4 is pure Rust; zstd and szip link C and are +# left out, so such datasets fail with a clear filter error. +clawhdf5 = { path = "../clawhdf5", version = "2.7.0", default-features = false, features = ["lz4"] } +clawhdf5-format = { path = "../clawhdf5-format", version = "2.7.0" } +# Must match the wasm-bindgen CLI exactly; build.sh checks. +wasm-bindgen = "0.2.129" +js-sys = "0.3.106" + +[dev-dependencies] +serde_json = "1" +tempfile = { workspace = true } diff --git a/crates/clawhdf5-wasm/src/core.rs b/crates/clawhdf5-wasm/src/core.rs new file mode 100644 index 0000000..baadcda --- /dev/null +++ b/crates/clawhdf5-wasm/src/core.rs @@ -0,0 +1,603 @@ +//! The reader behind the JavaScript API, in plain Rust so it is tested +//! natively. The `wasm_bindgen` layer in `lib.rs` only converts these types +//! to JavaScript values. +//! +//! Every read either returns the dataset's values or an error: a datatype +//! with no typed-array mapping (compound, reference, opaque, ...) is refused +//! with a message naming it, never returned as reinterpreted bytes. + +use clawhdf5::{AttrValue, File, Selection}; +use clawhdf5_format::data_read; +use clawhdf5_format::datatype::{Datatype, DatatypeByteOrder}; + +/// Errors are reported to JavaScript as messages. +pub type Result = std::result::Result; + +fn err(e: impl std::fmt::Display) -> String { + e.to_string() +} + +/// What a path names. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Kind { + Group, + Dataset, +} + +impl Kind { + pub fn as_str(self) -> &'static str { + match self { + Kind::Group => "group", + Kind::Dataset => "dataset", + } + } +} + +/// One entry of a group listing. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct Child { + pub name: String, + pub kind: Kind, +} + +/// A dataset's metadata. +#[derive(Debug, Clone, PartialEq)] +pub struct DatasetInfo { + /// Dataspace dimensions (empty for a scalar). + pub shape: Vec, + /// Maximum dimensions, `None` per unlimited dimension; `None` overall + /// when the dataspace records none. + pub maxshape: Option>>, + /// Human-readable datatype, e.g. `f64`, `i16 (big-endian)`, `string[8]`. + pub dtype: String, + /// Dimensions of an array datatype's elements, appended to the shape of + /// what [`Reader::read`] returns (empty otherwise). + pub element_shape: Vec, +} + +/// Decoded values, one variant per JavaScript typed array. +#[derive(Debug, Clone, PartialEq)] +pub enum Data { + F32(Vec), + F64(Vec), + I8(Vec), + I16(Vec), + I32(Vec), + I64(Vec), + U8(Vec), + U16(Vec), + U32(Vec), + U64(Vec), + /// Fixed- and variable-length strings, and enumeration member names. + Strings(Vec), +} + +impl Data { + pub fn len(&self) -> usize { + match self { + Data::F32(v) => v.len(), + Data::F64(v) => v.len(), + Data::I8(v) => v.len(), + Data::I16(v) => v.len(), + Data::I32(v) => v.len(), + Data::I64(v) => v.len(), + Data::U8(v) => v.len(), + Data::U16(v) => v.len(), + Data::U32(v) => v.len(), + Data::U64(v) => v.len(), + Data::Strings(v) => v.len(), + } + } + + pub fn is_empty(&self) -> bool { + self.len() == 0 + } +} + +/// Values in row-major order with their shape. +#[derive(Debug, Clone, PartialEq)] +pub struct Array { + pub shape: Vec, + pub data: Data, +} + +/// A regular hyperslab, as in `H5Sselect_hyperslab`. `stride` and `block` +/// default to 1 in every dimension. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct Hyperslab { + pub start: Vec, + pub count: Vec, + pub stride: Option>, + pub block: Option>, +} + +/// An attribute: its value, or why it has none. +#[derive(Debug, Clone)] +pub struct Attr { + pub name: String, + pub value: AttrValue, +} + +/// An open file, held in memory. +pub struct Reader { + file: File, +} + +impl Reader { + /// Parse a file from its bytes (the browser hands over the whole file). + pub fn open(bytes: Vec) -> Result { + Ok(Self { + file: File::from_bytes(bytes).map_err(err)?, + }) + } + + /// Whether `path` names a group or a dataset. + pub fn kind(&self, path: &str) -> Result { + match self.file.dataset(path) { + Ok(_) => Ok(Kind::Dataset), + Err(clawhdf5::Error::NotADataset(_)) => Ok(Kind::Group), + Err(e) => Err(err(e)), + } + } + + /// The groups, then the datasets, in the group at `path` (`/` is the + /// root). Soft links are listed as their targets; external and dangling + /// links, and named datatypes, are left out. + pub fn list(&self, path: &str) -> Result> { + if self.kind(path)? != Kind::Group { + return Err(format!("not a group: {path}")); + } + let group = self.file.group(path).map_err(err)?; + let mut out: Vec = group + .groups() + .map_err(err)? + .into_iter() + .map(|name| Child { + name, + kind: Kind::Group, + }) + .collect(); + out.extend( + group + .datasets() + .map_err(err)? + .into_iter() + .map(|name| Child { + name, + kind: Kind::Dataset, + }), + ); + Ok(out) + } + + /// Shape, max shape and datatype of the dataset at `path`. + pub fn info(&self, path: &str) -> Result { + let ds = self.file.dataset(path).map_err(err)?; + let dt = ds.raw_datatype().map_err(err)?; + let maxshape = ds.max_dimensions().map_err(err)?.map(|dims| { + dims.into_iter() + .map(|d| (d != u64::MAX).then_some(d)) + .collect() + }); + Ok(DatasetInfo { + shape: ds.shape().map_err(err)?, + maxshape, + dtype: describe(&dt), + element_shape: element_shape(&dt), + }) + } + + /// The attributes of the group or dataset at `path`, sorted by name, and + /// one message per attribute that could not be read at all. An attribute + /// whose type has no plain JavaScript form is returned as + /// [`AttrValue::Raw`]. + pub fn attrs(&self, path: &str) -> Result<(Vec, Vec)> { + let (map, errors) = match self.kind(path)? { + Kind::Dataset => self + .file + .dataset(path) + .and_then(|d| d.attrs_with_errors()) + .map_err(err)?, + Kind::Group => self + .file + .group(path) + .and_then(|g| g.attrs_with_errors()) + .map_err(err)?, + }; + let mut attrs: Vec = map + .into_iter() + .map(|(name, value)| Attr { name, value }) + .collect(); + attrs.sort_by(|a, b| a.name.cmp(&b.name)); + Ok((attrs, errors.into_iter().map(err).collect())) + } + + /// Read the dataset at `path`, whole or a hyperslab of it. + pub fn read(&self, path: &str, slab: Option<&Hyperslab>) -> Result { + let ds = self.file.dataset(path).map_err(err)?; + let dt = ds.raw_datatype().map_err(err)?; + let shape = ds.shape().map_err(err)?; + let (selection, mut out_shape) = match slab { + None => (Selection::All, shape.clone()), + Some(h) => hyperslab_selection(h, &shape)?, + }; + let raw = ds.read_selection(&selection).map_err(err)?; + let data = self.decode(&raw, &dt)?; + out_shape.extend(element_shape(&dt)); + let expected = out_shape + .iter() + .try_fold(1u64, |acc, &d| acc.checked_mul(d)) + .ok_or("selection size overflows")?; + if data.len() as u64 != expected { + return Err(format!( + "read {} values for shape {out_shape:?} ({expected} expected)", + data.len() + )); + } + Ok(Array { + shape: out_shape, + data, + }) + } + + fn decode(&self, raw: &[u8], dt: &Datatype) -> Result { + let base = array_base(dt); + let is_array = !std::ptr::eq(base, dt); + Ok(match base { + Datatype::FloatingPoint { size, .. } if *size <= 4 => { + Data::F32(data_read::read_as_f32(raw, dt).map_err(err)?) + } + Datatype::FloatingPoint { .. } => { + Data::F64(data_read::read_as_f64(raw, dt).map_err(err)?) + } + Datatype::FixedPoint { size, signed, .. } => { + let signed_ints = || data_read::read_as_i64(raw, dt).map_err(err); + let unsigned_ints = || data_read::read_as_u64(raw, dt).map_err(err); + match (size, signed) { + (1, true) => Data::I8(narrow(signed_ints()?)?), + (2, true) => Data::I16(narrow(signed_ints()?)?), + (4, true) => Data::I32(narrow(signed_ints()?)?), + (_, true) => Data::I64(signed_ints()?), + (1, false) => Data::U8(narrow(unsigned_ints()?)?), + (2, false) => Data::U16(narrow(unsigned_ints()?)?), + (4, false) => Data::U32(narrow(unsigned_ints()?)?), + (_, false) => Data::U64(unsigned_ints()?), + } + } + Datatype::String { .. } if !is_array => { + Data::Strings(data_read::read_as_strings(raw, dt).map_err(err)?) + } + Datatype::VariableLength { + is_string: true, .. + } if !is_array => { + let size = dt.type_size() as usize; + if size == 0 || !raw.len().is_multiple_of(size) { + return Err(format!( + "{} bytes is not a whole number of {size}-byte string references", + raw.len() + )); + } + let sb = self.file.superblock(); + Data::Strings( + clawhdf5_format::vl_data::read_vl_strings( + self.file.as_bytes(), + raw, + (raw.len() / size) as u64, + sb.offset_size, + sb.length_size, + ) + .map_err(err)?, + ) + } + Datatype::Enumeration { .. } if !is_array => { + Data::Strings(data_read::read_enum_names(raw, dt).map_err(err)?) + } + _ => { + return Err(format!( + "reading {} datasets is not supported", + describe(dt) + )); + } + }) + } +} + +/// Narrow integers read at 64 bits to the dataset's own width. The source is +/// that width, so this cannot fail on correct input; it is checked anyway. +fn narrow>(v: Vec) -> Result> { + v.into_iter() + .map(|x| T::try_from(x).map_err(|_| format!("value {x} out of range"))) + .collect() +} + +/// Innermost element type of (possibly nested) array datatypes. +fn array_base(dt: &Datatype) -> &Datatype { + match dt { + Datatype::Array { base_type, .. } => array_base(base_type), + _ => dt, + } +} + +fn element_shape(dt: &Datatype) -> Vec { + match dt { + Datatype::Array { + base_type, + dimensions, + } => { + let mut dims: Vec = dimensions.iter().map(|&d| u64::from(d)).collect(); + dims.extend(element_shape(base_type)); + dims + } + _ => Vec::new(), + } +} + +fn hyperslab_selection(h: &Hyperslab, shape: &[u64]) -> Result<(Selection, Vec)> { + let rank = shape.len(); + let ones = vec![1u64; rank]; + let stride = h.stride.clone().unwrap_or_else(|| ones.clone()); + let block = h.block.clone().unwrap_or(ones); + for (what, v) in [ + ("start", &h.start), + ("count", &h.count), + ("stride", &stride), + ("block", &block), + ] { + if v.len() != rank { + return Err(format!( + "hyperslab {what} has {} dimensions, the dataset has {rank}", + v.len() + )); + } + } + let mut out = Vec::with_capacity(rank); + for d in 0..rank { + if stride[d] == 0 || block[d] == 0 { + return Err(format!("hyperslab stride and block must be >= 1 (dim {d})")); + } + if h.count[d] > 1 && block[d] > stride[d] { + return Err(format!( + "hyperslab blocks overlap in dim {d}: block {} > stride {}", + block[d], stride[d] + )); + } + // Last element selected: start + (count-1)*stride + block - 1. + if h.count[d] > 0 { + let last = (h.count[d] - 1) + .checked_mul(stride[d]) + .and_then(|x| x.checked_add(h.start[d])) + .and_then(|x| x.checked_add(block[d] - 1)); + match last { + Some(l) if l < shape[d] => {} + _ => { + return Err(format!( + "hyperslab exceeds dimension {d} (extent {})", + shape[d] + )); + } + } + } + out.push( + h.count[d] + .checked_mul(block[d]) + .ok_or("selection size overflows")?, + ); + } + Ok(( + Selection::Hyperslab { + start: h.start.clone(), + stride, + count: h.count.clone(), + block, + }, + out, + )) +} + +/// A short, human-readable datatype name. +pub fn describe(dt: &Datatype) -> String { + fn endian(order: &DatatypeByteOrder) -> &'static str { + match order { + DatatypeByteOrder::BigEndian => " (big-endian)", + DatatypeByteOrder::Vax => " (VAX)", + _ => "", + } + } + match dt { + Datatype::FixedPoint { + size, + signed, + byte_order, + .. + } => format!( + "{}{}{}", + if *signed { "i" } else { "u" }, + size * 8, + endian(byte_order) + ), + Datatype::FloatingPoint { + size, byte_order, .. + } => format!("f{}{}", size * 8, endian(byte_order)), + Datatype::Time { size, .. } => format!("time{}", size * 8), + Datatype::String { size, .. } => format!("string[{size}]"), + Datatype::BitField { size, .. } => format!("bitfield{}", size * 8), + Datatype::Opaque { size, .. } => format!("opaque[{size}]"), + Datatype::Compound { members, .. } => { + let fields: Vec = members + .iter() + .map(|m| format!("{}: {}", m.name, describe(&m.datatype))) + .collect(); + format!("compound{{{}}}", fields.join(", ")) + } + Datatype::Reference { .. } => "reference".to_string(), + Datatype::Enumeration { + base_type, members, .. + } => { + let names: Vec<&str> = members.iter().map(|m| m.name.as_str()).collect(); + format!("enum<{}>{{{}}}", describe(base_type), names.join(", ")) + } + Datatype::VariableLength { + is_string: true, .. + } => "vlen string".to_string(), + Datatype::VariableLength { base_type, .. } => { + format!("vlen<{}>", describe(base_type)) + } + Datatype::Array { + base_type, + dimensions, + } => format!("array{dimensions:?}<{}>", describe(base_type)), + } +} + +#[cfg(test)] +mod tests { + use super::*; + use clawhdf5::FileBuilder; + + fn sample() -> Reader { + let mut b = FileBuilder::new(); + b.create_dataset("grid") + .with_f64_data(&(0..12).map(f64::from).collect::>()) + .with_shape(&[3, 4]) + .with_chunks(&[2, 2]) + .with_deflate(4); + b.create_dataset("bytes").with_u8_data(&[1, 2, 250]); + let mut g = b.create_group("sensors"); + g.create_dataset("temp").with_f32_data(&[1.5, -2.25]); + g.set_attr("location", AttrValue::String("lab".into())); + b.add_group(g.finish()); + b.set_attr("version", AttrValue::I64(3)); + b.set_attr("scale", AttrValue::F64Array(vec![0.5, 2.0])); + Reader::open(b.finish().unwrap()).unwrap() + } + + #[test] + fn lists_groups_then_datasets() { + let r = sample(); + let names: Vec<(String, Kind)> = r + .list("/") + .unwrap() + .into_iter() + .map(|c| (c.name, c.kind)) + .collect(); + assert_eq!(names[0], ("sensors".to_string(), Kind::Group)); + let mut ds: Vec<&str> = names[1..].iter().map(|(n, _)| n.as_str()).collect(); + ds.sort(); + assert_eq!(ds, ["bytes", "grid"]); + assert_eq!( + r.list("sensors").unwrap(), + vec![Child { + name: "temp".into(), + kind: Kind::Dataset + }] + ); + assert!(r.list("grid").unwrap_err().contains("not a group")); + assert!(r.list("missing").is_err()); + } + + #[test] + fn info_reports_shape_and_dtype() { + let r = sample(); + let i = r.info("grid").unwrap(); + assert_eq!(i.shape, vec![3, 4]); + assert_eq!(i.dtype, "f64"); + assert!(i.element_shape.is_empty()); + assert_eq!(r.info("sensors/temp").unwrap().dtype, "f32"); + assert_eq!(r.kind("/sensors").unwrap(), Kind::Group); + assert_eq!(r.kind("/sensors/temp").unwrap(), Kind::Dataset); + } + + #[test] + fn reads_whole_and_hyperslab() { + let r = sample(); + let all = r.read("grid", None).unwrap(); + assert_eq!(all.shape, vec![3, 4]); + assert_eq!(all.data, Data::F64((0..12).map(f64::from).collect())); + + let slab = Hyperslab { + start: vec![1, 0], + count: vec![2, 2], + stride: Some(vec![1, 2]), + block: None, + }; + let part = r.read("grid", Some(&slab)).unwrap(); + assert_eq!(part.shape, vec![2, 2]); + assert_eq!(part.data, Data::F64(vec![4.0, 6.0, 8.0, 10.0])); + + assert_eq!( + r.read("bytes", None).unwrap().data, + Data::U8(vec![1, 2, 250]) + ); + assert_eq!( + r.read("sensors/temp", None).unwrap().data, + Data::F32(vec![1.5, -2.25]) + ); + } + + #[test] + fn bad_hyperslabs_are_refused() { + let r = sample(); + let mk = |start: Vec, count: Vec| Hyperslab { + start, + count, + stride: None, + block: None, + }; + assert!( + r.read("grid", Some(&mk(vec![0], vec![1]))) + .unwrap_err() + .contains("dimensions") + ); + assert!( + r.read("grid", Some(&mk(vec![2, 0], vec![2, 1]))) + .unwrap_err() + .contains("exceeds") + ); + let overlap = Hyperslab { + start: vec![0, 0], + count: vec![2, 1], + stride: Some(vec![1, 1]), + block: Some(vec![2, 1]), + }; + assert!( + r.read("grid", Some(&overlap)) + .unwrap_err() + .contains("overlap") + ); + } + + #[test] + fn attrs_are_sorted() { + let r = sample(); + let (attrs, errors) = r.attrs("/").unwrap(); + assert!(errors.is_empty()); + let names: Vec<&str> = attrs.iter().map(|a| a.name.as_str()).collect(); + assert_eq!(names, ["scale", "version"]); + let (g, _) = r.attrs("sensors").unwrap(); + assert!(matches!(&g[0].value, AttrValue::String(s) if s == "lab")); + } + + #[test] + fn compound_is_refused_not_reinterpreted() { + use clawhdf5::CompoundTypeBuilder; + let ct = CompoundTypeBuilder::new() + .f64_field("x") + .i32_field("n") + .build(); + let mut rec = Vec::new(); + rec.extend_from_slice(&1.0f64.to_le_bytes()); + rec.extend_from_slice(&7i32.to_le_bytes()); + let mut b = FileBuilder::new(); + b.create_dataset("table").with_compound_data(ct, rec, 1); + let r = Reader::open(b.finish().unwrap()).unwrap(); + let e = r.read("table", None).unwrap_err(); + assert!(e.contains("compound{x: f64, n: i32}"), "{e}"); + assert!(e.contains("not supported"), "{e}"); + } + + #[test] + fn garbage_is_an_error() { + assert!(Reader::open(vec![0u8; 64]).is_err()); + assert!(Reader::open(Vec::new()).is_err()); + } +} diff --git a/crates/clawhdf5-wasm/src/lib.rs b/crates/clawhdf5-wasm/src/lib.rs new file mode 100644 index 0000000..7c6b4a9 --- /dev/null +++ b/crates/clawhdf5-wasm/src/lib.rs @@ -0,0 +1,244 @@ +//! clawhdf5's HDF5 reader for JavaScript, via `wasm-bindgen`. +//! +//! ```js +//! import init, { open } from "./pkg/clawhdf5_wasm.js"; +//! await init(); +//! const file = open(new Uint8Array(await blob.arrayBuffer())); +//! file.list("/"); // [{ name, kind: "group" | "dataset" }] +//! file.info("/x"); // { shape, maxshape, dtype, elementShape } +//! file.attrs("/x"); // [{ name, value, dtype }] +//! file.read("/x"); // { shape, dtype, data: Float64Array | ... | string[] } +//! file.readHyperslab("/x", [0, 0], [10, 10]); // stride, block optional +//! file.free(); +//! ``` +//! +//! Numeric data comes back in the typed array of the stored width +//! (`Int16Array` for `i16`, `BigInt64Array` for `i64`, `Float32Array` for +//! `f32` and `f16`, ...); strings and enumeration names as arrays of strings. +//! Anything else is a thrown `Error` naming the datatype. Only the reader is +//! exposed: nothing here writes files. +//! +//! The logic lives in [`core`], which is plain Rust and tested natively. + +pub mod core; + +use clawhdf5::AttrValue; +use js_sys::{Array, Object, Reflect}; +use wasm_bindgen::prelude::*; + +use crate::core::{Data, Hyperslab, Reader}; + +/// JavaScript numbers are exact up to 2^53. +const MAX_SAFE_INTEGER: f64 = 9_007_199_254_740_991.0; + +fn js_err(msg: String) -> JsError { + JsError::new(&msg) +} + +fn set(obj: &Object, key: &str, value: impl Into) { + // Defining a property on a fresh plain object cannot fail. + Reflect::set(obj, &JsValue::from_str(key), &value.into()).unwrap_throw(); +} + +fn shape_to_js(shape: &[u64]) -> Array { + shape.iter().map(|&d| JsValue::from_f64(d as f64)).collect() +} + +fn indices_from_js(what: &str, v: &[f64]) -> Result, JsError> { + v.iter() + .map(|&x| { + if x.is_finite() && x >= 0.0 && x.fract() == 0.0 && x <= MAX_SAFE_INTEGER { + Ok(x as u64) + } else { + Err(js_err(format!( + "{what} must hold non-negative integers, got {x}" + ))) + } + }) + .collect() +} + +fn data_to_js(data: Data) -> JsValue { + match data { + Data::F32(v) => js_sys::Float32Array::from(&v[..]).into(), + Data::F64(v) => js_sys::Float64Array::from(&v[..]).into(), + Data::I8(v) => js_sys::Int8Array::from(&v[..]).into(), + Data::I16(v) => js_sys::Int16Array::from(&v[..]).into(), + Data::I32(v) => js_sys::Int32Array::from(&v[..]).into(), + Data::I64(v) => js_sys::BigInt64Array::from(&v[..]).into(), + Data::U8(v) => js_sys::Uint8Array::from(&v[..]).into(), + Data::U16(v) => js_sys::Uint16Array::from(&v[..]).into(), + Data::U32(v) => js_sys::Uint32Array::from(&v[..]).into(), + Data::U64(v) => js_sys::BigUint64Array::from(&v[..]).into(), + Data::Strings(v) => v + .into_iter() + .map(|s| JsValue::from_str(&s)) + .collect::() + .into(), + } +} + +/// A scalar integer as a `number` when exact, else a `bigint`. +fn int_to_js(x: i128) -> JsValue { + if (x as f64).abs() <= MAX_SAFE_INTEGER { + JsValue::from_f64(x as f64) + } else if let Ok(v) = i64::try_from(x) { + JsValue::from(v) + } else { + JsValue::from(x as u64) + } +} + +/// An attribute value, and the datatype of one that has no JavaScript form. +fn attr_to_js(value: AttrValue) -> (JsValue, Option) { + match value { + AttrValue::F64(x) => (JsValue::from_f64(x), None), + AttrValue::F64Array(v) => (js_sys::Float64Array::from(&v[..]).into(), None), + AttrValue::I64(x) => (int_to_js(i128::from(x)), None), + AttrValue::I64Array(v) => (js_sys::BigInt64Array::from(&v[..]).into(), None), + AttrValue::U64(x) => (int_to_js(i128::from(x)), None), + AttrValue::U64Array(v) => (js_sys::BigUint64Array::from(&v[..]).into(), None), + AttrValue::String(s) => (JsValue::from_str(&s), None), + AttrValue::StringArray(v) => ( + v.iter() + .map(|s| JsValue::from_str(s)) + .collect::() + .into(), + None, + ), + AttrValue::Raw { datatype, .. } => (JsValue::NULL, Some(core::describe(&datatype))), + } +} + +/// An open HDF5 (or NetCDF-4) file. +#[wasm_bindgen] +pub struct H5File { + inner: Reader, +} + +/// Open a file from its bytes. Throws if they are not an HDF5 file. +#[wasm_bindgen] +pub fn open(bytes: Vec) -> Result { + H5File::new(bytes) +} + +/// The clawhdf5 version this module was built from. +#[wasm_bindgen] +pub fn version() -> String { + env!("CARGO_PKG_VERSION").to_string() +} + +#[wasm_bindgen] +impl H5File { + /// Same as [`open`]. + #[wasm_bindgen(constructor)] + pub fn new(bytes: Vec) -> Result { + Ok(H5File { + inner: Reader::open(bytes).map_err(js_err)?, + }) + } + + /// `"group"` or `"dataset"`. + pub fn kind(&self, path: &str) -> Result { + Ok(self.inner.kind(path).map_err(js_err)?.as_str().to_string()) + } + + /// The group's members: `[{ name, kind }]`, groups first. + pub fn list(&self, path: &str) -> Result { + Ok(self + .inner + .list(path) + .map_err(js_err)? + .into_iter() + .map(|c| { + let o = Object::new(); + set(&o, "name", c.name); + set(&o, "kind", c.kind.as_str()); + JsValue::from(o) + }) + .collect()) + } + + /// `{ shape, maxshape, dtype, elementShape }`. `maxshape` is `null` + /// when not recorded, with `null` for each unlimited dimension. + pub fn info(&self, path: &str) -> Result { + let i = self.inner.info(path).map_err(js_err)?; + let o = Object::new(); + set(&o, "shape", shape_to_js(&i.shape)); + let max: JsValue = match i.maxshape { + None => JsValue::NULL, + Some(dims) => dims + .into_iter() + .map(|d| d.map_or(JsValue::NULL, |d| JsValue::from_f64(d as f64))) + .collect::() + .into(), + }; + set(&o, "maxshape", max); + set(&o, "dtype", i.dtype); + set(&o, "elementShape", shape_to_js(&i.element_shape)); + Ok(o) + } + + /// `[{ name, value, dtype }]`, sorted by name. Scalars are `number` + /// (`bigint` beyond 2^53) or `string`; arrays are typed arrays or + /// `string[]`. An attribute with no JavaScript form has `value: null` + /// and its `dtype`; one that could not be read at all is reported by + /// [`attrErrors`](Self::attr_errors). + pub fn attrs(&self, path: &str) -> Result { + let (attrs, _) = self.inner.attrs(path).map_err(js_err)?; + Ok(attrs + .into_iter() + .map(|a| { + let o = Object::new(); + set(&o, "name", a.name); + let (value, dtype) = attr_to_js(a.value); + set(&o, "value", value); + set(&o, "dtype", dtype.map_or(JsValue::NULL, JsValue::from)); + JsValue::from(o) + }) + .collect()) + } + + /// Messages for attributes that could not be read. + #[wasm_bindgen(js_name = attrErrors)] + pub fn attr_errors(&self, path: &str) -> Result { + let (_, errors) = self.inner.attrs(path).map_err(js_err)?; + Ok(errors.into_iter().map(JsValue::from).collect()) + } + + /// The whole dataset: `{ shape, dtype, data }`, `data` in row-major + /// order. + pub fn read(&self, path: &str) -> Result { + self.read_impl(path, None) + } + + /// A regular hyperslab (`H5Sselect_hyperslab`): `stride` and `block` + /// default to 1. The result's shape is `count * block` per dimension. + #[wasm_bindgen(js_name = readHyperslab)] + pub fn read_hyperslab( + &self, + path: &str, + start: Vec, + count: Vec, + stride: Option>, + block: Option>, + ) -> Result { + let slab = Hyperslab { + start: indices_from_js("start", &start)?, + count: indices_from_js("count", &count)?, + stride: stride.map(|s| indices_from_js("stride", &s)).transpose()?, + block: block.map(|b| indices_from_js("block", &b)).transpose()?, + }; + self.read_impl(path, Some(&slab)) + } + + fn read_impl(&self, path: &str, slab: Option<&Hyperslab>) -> Result { + let dtype = self.inner.info(path).map_err(js_err)?.dtype; + let a = self.inner.read(path, slab).map_err(js_err)?; + let o = Object::new(); + set(&o, "shape", shape_to_js(&a.shape)); + set(&o, "dtype", dtype); + set(&o, "data", data_to_js(a.data)); + Ok(o) + } +} diff --git a/crates/clawhdf5-wasm/tests/h5py_interop.rs b/crates/clawhdf5-wasm/tests/h5py_interop.rs new file mode 100644 index 0000000..7d71a92 --- /dev/null +++ b/crates/clawhdf5-wasm/tests/h5py_interop.rs @@ -0,0 +1,252 @@ +//! The wasm reader's core against files written by h5py and netCDF4, with +//! the values libhdf5 reads back as the reference. The generator, +//! `examples/wasm-viewer/test/make_fixture.py`, is shared with the Node test +//! of the built wasm package, so both compare against the same expectations. +//! +//! Skipped when python3 with h5py/netCDF4 is missing, unless +//! `CLAWHDF5_REQUIRE_INTEROP=1`. `CLAWHDF5_PYTHON` names the interpreter. + +use std::path::{Path, PathBuf}; +use std::process::Command; + +use clawhdf5::AttrValue; +use clawhdf5_wasm::core::{Data, Hyperslab, Kind, Reader}; +use serde_json::Value; + +fn python() -> String { + std::env::var("CLAWHDF5_PYTHON").unwrap_or_else(|_| "python3".to_string()) +} + +fn interop_required() -> bool { + std::env::var("CLAWHDF5_REQUIRE_INTEROP").is_ok_and(|v| v == "1") +} + +fn python_available() -> bool { + Command::new(python()) + .args(["-c", "import h5py, netCDF4, numpy"]) + .output() + .map(|o| o.status.success()) + .unwrap_or(false) +} + +fn generator() -> PathBuf { + Path::new(env!("CARGO_MANIFEST_DIR")).join("../../examples/wasm-viewer/test/make_fixture.py") +} + +/// Values as comparable strings: integers exactly, floats by their f64 +/// value (an f32 widens exactly), strings as themselves. +fn data_strings(d: &Data) -> (&'static str, Vec) { + fn s(v: &[T]) -> Vec { + v.iter().map(ToString::to_string).collect() + } + fn f>(v: &[T]) -> Vec { + v.iter().map(|&x| format!("{:?}", x.into())).collect() + } + match d { + Data::F32(v) => ("f32", f(v)), + Data::F64(v) => ("f64", f(v)), + Data::I8(v) => ("i8", s(v)), + Data::I16(v) => ("i16", s(v)), + Data::I32(v) => ("i32", s(v)), + Data::I64(v) => ("i64", s(v)), + Data::U8(v) => ("u8", s(v)), + Data::U16(v) => ("u16", s(v)), + Data::U32(v) => ("u32", s(v)), + Data::U64(v) => ("u64", s(v)), + Data::Strings(v) => ("strings", v.clone()), + } +} + +fn expected_strings(kind: &str, values: &Value) -> Vec { + values + .as_array() + .unwrap() + .iter() + .map(|v| match (kind, v) { + ("f32" | "f64", Value::Number(n)) => format!("{:?}", n.as_f64().unwrap()), + (_, Value::String(s)) => s.clone(), + other => panic!("unexpected expected value {other:?}"), + }) + .collect() +} + +fn shape(v: &Value) -> Vec { + v.as_array() + .unwrap() + .iter() + .map(|x| x.as_u64().unwrap()) + .collect() +} + +fn check_attr(file: &str, path: &str, name: &str, got: &AttrValue, want: &Value) { + let ctx = format!("{file}:{path}@{name}"); + let ints = |v: &Value| -> Vec { + v.as_array() + .unwrap() + .iter() + .map(|x| x.as_str().unwrap().to_string()) + .collect() + }; + let scalar = want["scalar"].as_bool().unwrap_or(false); + match got { + AttrValue::String(s) => assert_eq!(want["string"].as_str(), Some(s.as_str()), "{ctx}"), + AttrValue::StringArray(v) => { + let w: Vec<&str> = want["strings"] + .as_array() + .unwrap_or_else(|| panic!("{ctx}: got {v:?}")) + .iter() + .map(|x| x.as_str().unwrap()) + .collect(); + assert_eq!(v, &w, "{ctx}"); + } + AttrValue::I64(x) => { + assert!(scalar, "{ctx}"); + assert_eq!(vec![x.to_string()], ints(&want["int"]), "{ctx}"); + } + AttrValue::U64(x) => { + assert!(scalar, "{ctx}"); + assert_eq!(vec![x.to_string()], ints(&want["int"]), "{ctx}"); + } + AttrValue::I64Array(v) => assert_eq!( + v.iter().map(ToString::to_string).collect::>(), + ints(&want["int"]), + "{ctx}" + ), + AttrValue::U64Array(v) => assert_eq!( + v.iter().map(ToString::to_string).collect::>(), + ints(&want["int"]), + "{ctx}" + ), + AttrValue::F64(x) => { + assert!(scalar, "{ctx}"); + assert_eq!(Some(*x), want["float"][0].as_f64(), "{ctx}"); + } + AttrValue::F64Array(v) => { + let w: Vec = want["float"] + .as_array() + .unwrap() + .iter() + .map(|x| x.as_f64().unwrap()) + .collect(); + assert_eq!(v, &w, "{ctx}"); + } + AttrValue::Raw { datatype, .. } => panic!("{ctx}: undecoded {datatype:?}"), + } +} + +fn check_file(dir: &Path, file: &str, exp: &Value) { + let r = Reader::open(std::fs::read(dir.join(file)).unwrap()).unwrap(); + + for (path, want) in exp["lists"].as_object().unwrap() { + let list = r + .list(path) + .unwrap_or_else(|e| panic!("{file}:{path}: {e}")); + for (kind, key) in [(Kind::Group, "groups"), (Kind::Dataset, "datasets")] { + let mut got: Vec<&str> = list + .iter() + .filter(|c| c.kind == kind) + .map(|c| c.name.as_str()) + .collect(); + got.sort(); + let w: Vec<&str> = want[key] + .as_array() + .unwrap() + .iter() + .map(|x| x.as_str().unwrap()) + .collect(); + assert_eq!(got, w, "{file}:{path} {key}"); + } + } + + for (path, want) in exp["datasets"].as_object().unwrap() { + let kind = want["kind"].as_str().unwrap(); + let a = r + .read(path, None) + .unwrap_or_else(|e| panic!("{file}:{path}: {e}")); + assert_eq!(a.shape, shape(&want["shape"]), "{file}:{path} shape"); + let (got_kind, got) = data_strings(&a.data); + assert_eq!(got_kind, kind, "{file}:{path} kind"); + assert_eq!( + got, + expected_strings(kind, &want["values"]), + "{file}:{path}" + ); + + if let Some(slab) = want.get("slab") { + let h = Hyperslab { + start: shape(&slab["start"]), + count: shape(&slab["count"]), + stride: Some(shape(&slab["stride"])), + block: None, + }; + let a = r + .read(path, Some(&h)) + .unwrap_or_else(|e| panic!("{file}:{path} {h:?}: {e}")); + assert_eq!(a.shape, shape(&slab["shape"]), "{file}:{path} slab shape"); + assert_eq!( + data_strings(&a.data).1, + expected_strings(kind, &slab["values"]), + "{file}:{path} slab" + ); + } + } + + for (path, what) in exp["errors"].as_object().unwrap() { + let e = r.read(path, None).expect_err(path); + assert!(e.contains(what.as_str().unwrap()), "{file}:{path}: {e}"); + } + + let skip: Vec<&str> = exp["skip_attrs"] + .as_array() + .unwrap() + .iter() + .map(|x| x.as_str().unwrap()) + .collect(); + for (path, want) in exp["attrs"].as_object().unwrap() { + let (attrs, errors) = r + .attrs(path) + .unwrap_or_else(|e| panic!("{file}:{path}: {e}")); + assert!(errors.is_empty(), "{file}:{path}: {errors:?}"); + let want = want.as_object().unwrap(); + let mut compared = 0; + for a in &attrs { + if a.name.starts_with('_') || skip.contains(&a.name.as_str()) { + continue; + } + let w = want + .get(&a.name) + .unwrap_or_else(|| panic!("{file}:{path}: unexpected attribute {}", a.name)); + check_attr(file, path, &a.name, &a.value, w); + compared += 1; + } + assert_eq!(compared, want.len(), "{file}:{path}: attributes missing"); + } +} + +#[test] +fn reads_what_h5py_and_netcdf4_wrote() { + if !python_available() { + assert!( + !interop_required(), + "CLAWHDF5_REQUIRE_INTEROP=1 but python with h5py, netCDF4 and numpy is not available" + ); + eprintln!("SKIP: python with h5py/netCDF4 not available"); + return; + } + let dir = tempfile::tempdir().unwrap(); + let out = Command::new(python()) + .arg(generator()) + .arg(dir.path()) + .output() + .expect("run python"); + assert!( + out.status.success(), + "fixture generator failed:\n{}", + String::from_utf8_lossy(&out.stderr) + ); + let exp: Value = + serde_json::from_slice(&std::fs::read(dir.path().join("expected.json")).unwrap()).unwrap(); + for file in ["fixture.h5", "fixture.nc"] { + check_file(dir.path(), file, &exp[file]); + } +} diff --git a/examples/wasm-viewer/test/make_fixture.py b/examples/wasm-viewer/test/make_fixture.py new file mode 100644 index 0000000..50b893b --- /dev/null +++ b/examples/wasm-viewer/test/make_fixture.py @@ -0,0 +1,184 @@ +"""Write HDF5 and NetCDF-4 test files with h5py/netCDF4, and what libhdf5 +reads back from them, for the clawhdf5-wasm tests. + + python make_fixture.py OUT_DIR + +writes OUT_DIR/fixture.h5, OUT_DIR/fixture.nc and OUT_DIR/expected.json. +Both the Rust test (crates/clawhdf5-wasm/tests/h5py_interop.rs, native) and +the Node test (test.mjs, the built wasm package) compare against the same +expected.json, so the two check the same values. + +Every expected value comes from h5py reading the file back (numpy slicing +for hyperslabs), never from the arrays that were written. Integers are +encoded as strings so JSON.parse keeps 64-bit values exact. +""" + +import json +import sys +import warnings +from pathlib import Path + +import h5py +import netCDF4 +import numpy as np + +# netCDF4 1.7 trips numpy 2.5's shape-setting deprecation on assignment. +warnings.filterwarnings("ignore", category=DeprecationWarning) + +out = Path(sys.argv[1]) +out.mkdir(parents=True, exist_ok=True) +h5 = out / "fixture.h5" +nc = out / "fixture.nc" + +rng = np.random.default_rng(7) +with h5py.File(h5, "w") as f: + f.attrs["title"] = "wasm fixture" + f.attrs["version"] = np.int64(3) + f.attrs["scale"] = np.array([0.5, 2.0]) + f.attrs["big"] = np.uint64(2**63 + 5) + f.attrs.create("vlen_note", "héllo", dtype=h5py.string_dtype()) + f.create_dataset( + "grid", data=np.arange(60, dtype="f4")) + f.create_dataset("f16", data=np.array([0.5, -1.25, 65504], dtype=" 2 else 1] + [2] * (obj.ndim - 1) + count = [min(c, (n - s - 1) // st + 1) + for c, s, st, n in zip(count, start, stride, obj.shape)] + return (start, count, stride) + + +json.dump({"fixture.h5": describe(h5), "fixture.nc": describe(nc)}, + open(out / "expected.json", "w"), indent=1, ensure_ascii=False) diff --git a/scripts/ci-test.sh b/scripts/ci-test.sh index 6cad171..9b93039 100755 --- a/scripts/ci-test.sh +++ b/scripts/ci-test.sh @@ -92,7 +92,8 @@ run_step "cargo clippy (fast-deflate / zlib-ng)" cargo clippy \ no_c_in_default_build() { local crate found=0 for crate in clawhdf5-format clawhdf5-io clawhdf5-filters clawhdf5 \ - clawhdf5-agent clawhdf5-ann clawhdf5-accel clawhdf5-netcdf4 clawhdf5-cli; do + clawhdf5-agent clawhdf5-ann clawhdf5-accel clawhdf5-netcdf4 clawhdf5-cli \ + clawhdf5-wasm; do local c_deps c_deps=$(cargo tree -q -p "$crate" -e normal,build --prefix none \ | grep -E '^([a-z0-9_-]+-sys|cc|cmake) v' | sort -u) @@ -107,12 +108,18 @@ no_c_in_default_build() { run_step "no C in the default build (core crates)" no_c_in_default_build # The reader in the browser: the facade's read path must build for -# wasm32-unknown-unknown (no mmap, no threads, no file system). Needs +# wasm32-unknown-unknown (no mmap, no threads, no file system), and the +# wasm-bindgen crate must build and lint there. Needs # `rustup target add wasm32-unknown-unknown`. run_step "wasm32 build (clawhdf5, no default features)" cargo build \ -p clawhdf5 \ --target wasm32-unknown-unknown \ --no-default-features +run_step "wasm32 clippy (clawhdf5-wasm)" cargo clippy \ + -p clawhdf5-wasm \ + --target wasm32-unknown-unknown \ + --all-targets \ + -- -D warnings # The workspace declares a minimum Rust version (rust-version in Cargo.toml); # check that it really builds there, so the README badge and the manifests