h5rs tools, browser reader, libhdf5 header checks, plugin filters, concurrency benchmark #14

Merged
osobh merged 60 commits from feat/p1-proof into main 2026-09-26 13:14:39 +00:00
9 changed files with 1337 additions and 5 deletions
Showing only changes of commit a42b646689 - Show all commits
+2 -1
View File
@@ -5,7 +5,7 @@ Pure-Rust HDF5 format implementation with HNSW vector search, WAL-backed persist
## Architecture ## Architecture
Cargo workspace with 16 crates under `crates/` (plus `libaec-sys`, an internal FFI bindings crate for the optional `szip` feature): Cargo workspace with 17 crates under `crates/` (plus `libaec-sys`, an internal FFI bindings crate for the optional `szip` feature):
| Crate | Role | | Crate | Role |
|-------|------| |-------|------|
@@ -24,6 +24,7 @@ Cargo workspace with 16 crates under `crates/` (plus `libaec-sys`, an internal F
| `clawhdf5-cli` | Command-line interface | | `clawhdf5-cli` | Command-line interface |
| `clawhdf5-napi` | Node.js native addon bindings | | `clawhdf5-napi` | Node.js native addon bindings |
| `clawhdf5-py` | PyO3 Python bindings | | `clawhdf5-py` | PyO3 Python bindings |
| `clawhdf5-wasm` | WebAssembly (wasm-bindgen) reader for the browser; demo in `examples/wasm-viewer/` |
| `clawhdf5-bench` | Benchmark suite | | `clawhdf5-bench` | Benchmark suite |
## Key Features ## Key Features
+10
View File
@@ -16,6 +16,7 @@ members = [
"crates/clawhdf5-cli", "crates/clawhdf5-cli",
"crates/clawhdf5-napi", "crates/clawhdf5-napi",
"crates/clawhdf5-bench", "crates/clawhdf5-bench",
"crates/clawhdf5-wasm",
"crates/libaec-sys", "crates/libaec-sys",
] ]
resolver = "2" resolver = "2"
@@ -34,3 +35,12 @@ tempfile = "3"
criterion = { version = "0.5", features = ["html_reports"] } criterion = { version = "0.5", features = ["html_reports"] }
half = "2.7" half = "2.7"
serde = { version = "1", features = ["derive"] } serde = { version = "1", features = ["derive"] }
# The browser build of clawhdf5-wasm (examples/wasm-viewer/build.sh): size
# over speed, whole-program optimisation. Native profiles are unaffected.
[profile.wasm-release]
inherits = "release"
opt-level = "s"
lto = true
codegen-units = 1
panic = "abort"
+3 -2
View File
@@ -587,7 +587,7 @@ let exported = backend.export_markdown("MEMORY.md")?;
## Crate Map ## Crate Map
``` ```
clawhdf5 workspace (16 crates, ~86K lines of Rust in src/, ~104K with tests clawhdf5 workspace (17 crates, ~86K lines of Rust in src/, ~104K with tests
and benches; plus libaec-sys, an internal FFI bindings and benches; plus libaec-sys, an internal FFI bindings
crate for the optional szip feature) crate for the optional szip feature)
│ │
@@ -610,7 +610,8 @@ clawhdf5 workspace (16 crates, ~86K lines of Rust in src/, ~104K with tests
│ │
├── Bindings ├── Bindings
│ ├── clawhdf5-py — Python (PyO3) │ ├── clawhdf5-py — Python (PyO3)
│ └── clawhdf5-napi — Node.js (napi-rs) │ ├── clawhdf5-napi — Node.js (napi-rs)
│ └── clawhdf5-wasm — Browser (WebAssembly, wasm-bindgen; read-only)
│ │
└── Tooling └── Tooling
└── clawhdf5-bench — Benchmark suite └── clawhdf5-bench — Benchmark suite
+30
View File
@@ -0,0 +1,30 @@
[package]
name = "clawhdf5-wasm"
version.workspace = true
edition.workspace = true
rust-version.workspace = true
description = "Read HDF5 and NetCDF-4 files in the browser: clawhdf5's reader compiled to WebAssembly"
license.workspace = true
repository.workspace = true
keywords = ["hdf5", "netcdf", "wasm", "browser"]
categories = ["wasm", "parser-implementations", "science"]
# Distributed as the wasm-bindgen package built by examples/wasm-viewer/build.sh.
publish = false
[lib]
# cdylib for wasm-bindgen; rlib so the pure-Rust core is tested natively.
crate-type = ["cdylib", "rlib"]
[dependencies]
# No mmap (there is no file system) and no `parallel` (no threads on
# wasm32-unknown-unknown). lz4 is pure Rust; zstd and szip link C and are
# left out, so such datasets fail with a clear filter error.
clawhdf5 = { path = "../clawhdf5", version = "2.7.0", default-features = false, features = ["lz4"] }
clawhdf5-format = { path = "../clawhdf5-format", version = "2.7.0" }
# Must match the wasm-bindgen CLI exactly; build.sh checks.
wasm-bindgen = "0.2.129"
js-sys = "0.3.106"
[dev-dependencies]
serde_json = "1"
tempfile = { workspace = true }
+603
View File
@@ -0,0 +1,603 @@
//! The reader behind the JavaScript API, in plain Rust so it is tested
//! natively. The `wasm_bindgen` layer in `lib.rs` only converts these types
//! to JavaScript values.
//!
//! Every read either returns the dataset's values or an error: a datatype
//! with no typed-array mapping (compound, reference, opaque, ...) is refused
//! with a message naming it, never returned as reinterpreted bytes.
use clawhdf5::{AttrValue, File, Selection};
use clawhdf5_format::data_read;
use clawhdf5_format::datatype::{Datatype, DatatypeByteOrder};
/// Errors are reported to JavaScript as messages.
pub type Result<T> = std::result::Result<T, String>;
fn err(e: impl std::fmt::Display) -> String {
e.to_string()
}
/// What a path names.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum Kind {
Group,
Dataset,
}
impl Kind {
pub fn as_str(self) -> &'static str {
match self {
Kind::Group => "group",
Kind::Dataset => "dataset",
}
}
}
/// One entry of a group listing.
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct Child {
pub name: String,
pub kind: Kind,
}
/// A dataset's metadata.
#[derive(Debug, Clone, PartialEq)]
pub struct DatasetInfo {
/// Dataspace dimensions (empty for a scalar).
pub shape: Vec<u64>,
/// Maximum dimensions, `None` per unlimited dimension; `None` overall
/// when the dataspace records none.
pub maxshape: Option<Vec<Option<u64>>>,
/// Human-readable datatype, e.g. `f64`, `i16 (big-endian)`, `string[8]`.
pub dtype: String,
/// Dimensions of an array datatype's elements, appended to the shape of
/// what [`Reader::read`] returns (empty otherwise).
pub element_shape: Vec<u64>,
}
/// Decoded values, one variant per JavaScript typed array.
#[derive(Debug, Clone, PartialEq)]
pub enum Data {
F32(Vec<f32>),
F64(Vec<f64>),
I8(Vec<i8>),
I16(Vec<i16>),
I32(Vec<i32>),
I64(Vec<i64>),
U8(Vec<u8>),
U16(Vec<u16>),
U32(Vec<u32>),
U64(Vec<u64>),
/// Fixed- and variable-length strings, and enumeration member names.
Strings(Vec<String>),
}
impl Data {
pub fn len(&self) -> usize {
match self {
Data::F32(v) => v.len(),
Data::F64(v) => v.len(),
Data::I8(v) => v.len(),
Data::I16(v) => v.len(),
Data::I32(v) => v.len(),
Data::I64(v) => v.len(),
Data::U8(v) => v.len(),
Data::U16(v) => v.len(),
Data::U32(v) => v.len(),
Data::U64(v) => v.len(),
Data::Strings(v) => v.len(),
}
}
pub fn is_empty(&self) -> bool {
self.len() == 0
}
}
/// Values in row-major order with their shape.
#[derive(Debug, Clone, PartialEq)]
pub struct Array {
pub shape: Vec<u64>,
pub data: Data,
}
/// A regular hyperslab, as in `H5Sselect_hyperslab`. `stride` and `block`
/// default to 1 in every dimension.
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct Hyperslab {
pub start: Vec<u64>,
pub count: Vec<u64>,
pub stride: Option<Vec<u64>>,
pub block: Option<Vec<u64>>,
}
/// An attribute: its value, or why it has none.
#[derive(Debug, Clone)]
pub struct Attr {
pub name: String,
pub value: AttrValue,
}
/// An open file, held in memory.
pub struct Reader {
file: File,
}
impl Reader {
/// Parse a file from its bytes (the browser hands over the whole file).
pub fn open(bytes: Vec<u8>) -> Result<Self> {
Ok(Self {
file: File::from_bytes(bytes).map_err(err)?,
})
}
/// Whether `path` names a group or a dataset.
pub fn kind(&self, path: &str) -> Result<Kind> {
match self.file.dataset(path) {
Ok(_) => Ok(Kind::Dataset),
Err(clawhdf5::Error::NotADataset(_)) => Ok(Kind::Group),
Err(e) => Err(err(e)),
}
}
/// The groups, then the datasets, in the group at `path` (`/` is the
/// root). Soft links are listed as their targets; external and dangling
/// links, and named datatypes, are left out.
pub fn list(&self, path: &str) -> Result<Vec<Child>> {
if self.kind(path)? != Kind::Group {
return Err(format!("not a group: {path}"));
}
let group = self.file.group(path).map_err(err)?;
let mut out: Vec<Child> = group
.groups()
.map_err(err)?
.into_iter()
.map(|name| Child {
name,
kind: Kind::Group,
})
.collect();
out.extend(
group
.datasets()
.map_err(err)?
.into_iter()
.map(|name| Child {
name,
kind: Kind::Dataset,
}),
);
Ok(out)
}
/// Shape, max shape and datatype of the dataset at `path`.
pub fn info(&self, path: &str) -> Result<DatasetInfo> {
let ds = self.file.dataset(path).map_err(err)?;
let dt = ds.raw_datatype().map_err(err)?;
let maxshape = ds.max_dimensions().map_err(err)?.map(|dims| {
dims.into_iter()
.map(|d| (d != u64::MAX).then_some(d))
.collect()
});
Ok(DatasetInfo {
shape: ds.shape().map_err(err)?,
maxshape,
dtype: describe(&dt),
element_shape: element_shape(&dt),
})
}
/// The attributes of the group or dataset at `path`, sorted by name, and
/// one message per attribute that could not be read at all. An attribute
/// whose type has no plain JavaScript form is returned as
/// [`AttrValue::Raw`].
pub fn attrs(&self, path: &str) -> Result<(Vec<Attr>, Vec<String>)> {
let (map, errors) = match self.kind(path)? {
Kind::Dataset => self
.file
.dataset(path)
.and_then(|d| d.attrs_with_errors())
.map_err(err)?,
Kind::Group => self
.file
.group(path)
.and_then(|g| g.attrs_with_errors())
.map_err(err)?,
};
let mut attrs: Vec<Attr> = map
.into_iter()
.map(|(name, value)| Attr { name, value })
.collect();
attrs.sort_by(|a, b| a.name.cmp(&b.name));
Ok((attrs, errors.into_iter().map(err).collect()))
}
/// Read the dataset at `path`, whole or a hyperslab of it.
pub fn read(&self, path: &str, slab: Option<&Hyperslab>) -> Result<Array> {
let ds = self.file.dataset(path).map_err(err)?;
let dt = ds.raw_datatype().map_err(err)?;
let shape = ds.shape().map_err(err)?;
let (selection, mut out_shape) = match slab {
None => (Selection::All, shape.clone()),
Some(h) => hyperslab_selection(h, &shape)?,
};
let raw = ds.read_selection(&selection).map_err(err)?;
let data = self.decode(&raw, &dt)?;
out_shape.extend(element_shape(&dt));
let expected = out_shape
.iter()
.try_fold(1u64, |acc, &d| acc.checked_mul(d))
.ok_or("selection size overflows")?;
if data.len() as u64 != expected {
return Err(format!(
"read {} values for shape {out_shape:?} ({expected} expected)",
data.len()
));
}
Ok(Array {
shape: out_shape,
data,
})
}
fn decode(&self, raw: &[u8], dt: &Datatype) -> Result<Data> {
let base = array_base(dt);
let is_array = !std::ptr::eq(base, dt);
Ok(match base {
Datatype::FloatingPoint { size, .. } if *size <= 4 => {
Data::F32(data_read::read_as_f32(raw, dt).map_err(err)?)
}
Datatype::FloatingPoint { .. } => {
Data::F64(data_read::read_as_f64(raw, dt).map_err(err)?)
}
Datatype::FixedPoint { size, signed, .. } => {
let signed_ints = || data_read::read_as_i64(raw, dt).map_err(err);
let unsigned_ints = || data_read::read_as_u64(raw, dt).map_err(err);
match (size, signed) {
(1, true) => Data::I8(narrow(signed_ints()?)?),
(2, true) => Data::I16(narrow(signed_ints()?)?),
(4, true) => Data::I32(narrow(signed_ints()?)?),
(_, true) => Data::I64(signed_ints()?),
(1, false) => Data::U8(narrow(unsigned_ints()?)?),
(2, false) => Data::U16(narrow(unsigned_ints()?)?),
(4, false) => Data::U32(narrow(unsigned_ints()?)?),
(_, false) => Data::U64(unsigned_ints()?),
}
}
Datatype::String { .. } if !is_array => {
Data::Strings(data_read::read_as_strings(raw, dt).map_err(err)?)
}
Datatype::VariableLength {
is_string: true, ..
} if !is_array => {
let size = dt.type_size() as usize;
if size == 0 || !raw.len().is_multiple_of(size) {
return Err(format!(
"{} bytes is not a whole number of {size}-byte string references",
raw.len()
));
}
let sb = self.file.superblock();
Data::Strings(
clawhdf5_format::vl_data::read_vl_strings(
self.file.as_bytes(),
raw,
(raw.len() / size) as u64,
sb.offset_size,
sb.length_size,
)
.map_err(err)?,
)
}
Datatype::Enumeration { .. } if !is_array => {
Data::Strings(data_read::read_enum_names(raw, dt).map_err(err)?)
}
_ => {
return Err(format!(
"reading {} datasets is not supported",
describe(dt)
));
}
})
}
}
/// Narrow integers read at 64 bits to the dataset's own width. The source is
/// that width, so this cannot fail on correct input; it is checked anyway.
fn narrow<S: Copy + std::fmt::Display, T: TryFrom<S>>(v: Vec<S>) -> Result<Vec<T>> {
v.into_iter()
.map(|x| T::try_from(x).map_err(|_| format!("value {x} out of range")))
.collect()
}
/// Innermost element type of (possibly nested) array datatypes.
fn array_base(dt: &Datatype) -> &Datatype {
match dt {
Datatype::Array { base_type, .. } => array_base(base_type),
_ => dt,
}
}
fn element_shape(dt: &Datatype) -> Vec<u64> {
match dt {
Datatype::Array {
base_type,
dimensions,
} => {
let mut dims: Vec<u64> = dimensions.iter().map(|&d| u64::from(d)).collect();
dims.extend(element_shape(base_type));
dims
}
_ => Vec::new(),
}
}
fn hyperslab_selection(h: &Hyperslab, shape: &[u64]) -> Result<(Selection, Vec<u64>)> {
let rank = shape.len();
let ones = vec![1u64; rank];
let stride = h.stride.clone().unwrap_or_else(|| ones.clone());
let block = h.block.clone().unwrap_or(ones);
for (what, v) in [
("start", &h.start),
("count", &h.count),
("stride", &stride),
("block", &block),
] {
if v.len() != rank {
return Err(format!(
"hyperslab {what} has {} dimensions, the dataset has {rank}",
v.len()
));
}
}
let mut out = Vec::with_capacity(rank);
for d in 0..rank {
if stride[d] == 0 || block[d] == 0 {
return Err(format!("hyperslab stride and block must be >= 1 (dim {d})"));
}
if h.count[d] > 1 && block[d] > stride[d] {
return Err(format!(
"hyperslab blocks overlap in dim {d}: block {} > stride {}",
block[d], stride[d]
));
}
// Last element selected: start + (count-1)*stride + block - 1.
if h.count[d] > 0 {
let last = (h.count[d] - 1)
.checked_mul(stride[d])
.and_then(|x| x.checked_add(h.start[d]))
.and_then(|x| x.checked_add(block[d] - 1));
match last {
Some(l) if l < shape[d] => {}
_ => {
return Err(format!(
"hyperslab exceeds dimension {d} (extent {})",
shape[d]
));
}
}
}
out.push(
h.count[d]
.checked_mul(block[d])
.ok_or("selection size overflows")?,
);
}
Ok((
Selection::Hyperslab {
start: h.start.clone(),
stride,
count: h.count.clone(),
block,
},
out,
))
}
/// A short, human-readable datatype name.
pub fn describe(dt: &Datatype) -> String {
fn endian(order: &DatatypeByteOrder) -> &'static str {
match order {
DatatypeByteOrder::BigEndian => " (big-endian)",
DatatypeByteOrder::Vax => " (VAX)",
_ => "",
}
}
match dt {
Datatype::FixedPoint {
size,
signed,
byte_order,
..
} => format!(
"{}{}{}",
if *signed { "i" } else { "u" },
size * 8,
endian(byte_order)
),
Datatype::FloatingPoint {
size, byte_order, ..
} => format!("f{}{}", size * 8, endian(byte_order)),
Datatype::Time { size, .. } => format!("time{}", size * 8),
Datatype::String { size, .. } => format!("string[{size}]"),
Datatype::BitField { size, .. } => format!("bitfield{}", size * 8),
Datatype::Opaque { size, .. } => format!("opaque[{size}]"),
Datatype::Compound { members, .. } => {
let fields: Vec<String> = members
.iter()
.map(|m| format!("{}: {}", m.name, describe(&m.datatype)))
.collect();
format!("compound{{{}}}", fields.join(", "))
}
Datatype::Reference { .. } => "reference".to_string(),
Datatype::Enumeration {
base_type, members, ..
} => {
let names: Vec<&str> = members.iter().map(|m| m.name.as_str()).collect();
format!("enum<{}>{{{}}}", describe(base_type), names.join(", "))
}
Datatype::VariableLength {
is_string: true, ..
} => "vlen string".to_string(),
Datatype::VariableLength { base_type, .. } => {
format!("vlen<{}>", describe(base_type))
}
Datatype::Array {
base_type,
dimensions,
} => format!("array{dimensions:?}<{}>", describe(base_type)),
}
}
#[cfg(test)]
mod tests {
use super::*;
use clawhdf5::FileBuilder;
fn sample() -> Reader {
let mut b = FileBuilder::new();
b.create_dataset("grid")
.with_f64_data(&(0..12).map(f64::from).collect::<Vec<_>>())
.with_shape(&[3, 4])
.with_chunks(&[2, 2])
.with_deflate(4);
b.create_dataset("bytes").with_u8_data(&[1, 2, 250]);
let mut g = b.create_group("sensors");
g.create_dataset("temp").with_f32_data(&[1.5, -2.25]);
g.set_attr("location", AttrValue::String("lab".into()));
b.add_group(g.finish());
b.set_attr("version", AttrValue::I64(3));
b.set_attr("scale", AttrValue::F64Array(vec![0.5, 2.0]));
Reader::open(b.finish().unwrap()).unwrap()
}
#[test]
fn lists_groups_then_datasets() {
let r = sample();
let names: Vec<(String, Kind)> = r
.list("/")
.unwrap()
.into_iter()
.map(|c| (c.name, c.kind))
.collect();
assert_eq!(names[0], ("sensors".to_string(), Kind::Group));
let mut ds: Vec<&str> = names[1..].iter().map(|(n, _)| n.as_str()).collect();
ds.sort();
assert_eq!(ds, ["bytes", "grid"]);
assert_eq!(
r.list("sensors").unwrap(),
vec![Child {
name: "temp".into(),
kind: Kind::Dataset
}]
);
assert!(r.list("grid").unwrap_err().contains("not a group"));
assert!(r.list("missing").is_err());
}
#[test]
fn info_reports_shape_and_dtype() {
let r = sample();
let i = r.info("grid").unwrap();
assert_eq!(i.shape, vec![3, 4]);
assert_eq!(i.dtype, "f64");
assert!(i.element_shape.is_empty());
assert_eq!(r.info("sensors/temp").unwrap().dtype, "f32");
assert_eq!(r.kind("/sensors").unwrap(), Kind::Group);
assert_eq!(r.kind("/sensors/temp").unwrap(), Kind::Dataset);
}
#[test]
fn reads_whole_and_hyperslab() {
let r = sample();
let all = r.read("grid", None).unwrap();
assert_eq!(all.shape, vec![3, 4]);
assert_eq!(all.data, Data::F64((0..12).map(f64::from).collect()));
let slab = Hyperslab {
start: vec![1, 0],
count: vec![2, 2],
stride: Some(vec![1, 2]),
block: None,
};
let part = r.read("grid", Some(&slab)).unwrap();
assert_eq!(part.shape, vec![2, 2]);
assert_eq!(part.data, Data::F64(vec![4.0, 6.0, 8.0, 10.0]));
assert_eq!(
r.read("bytes", None).unwrap().data,
Data::U8(vec![1, 2, 250])
);
assert_eq!(
r.read("sensors/temp", None).unwrap().data,
Data::F32(vec![1.5, -2.25])
);
}
#[test]
fn bad_hyperslabs_are_refused() {
let r = sample();
let mk = |start: Vec<u64>, count: Vec<u64>| Hyperslab {
start,
count,
stride: None,
block: None,
};
assert!(
r.read("grid", Some(&mk(vec![0], vec![1])))
.unwrap_err()
.contains("dimensions")
);
assert!(
r.read("grid", Some(&mk(vec![2, 0], vec![2, 1])))
.unwrap_err()
.contains("exceeds")
);
let overlap = Hyperslab {
start: vec![0, 0],
count: vec![2, 1],
stride: Some(vec![1, 1]),
block: Some(vec![2, 1]),
};
assert!(
r.read("grid", Some(&overlap))
.unwrap_err()
.contains("overlap")
);
}
#[test]
fn attrs_are_sorted() {
let r = sample();
let (attrs, errors) = r.attrs("/").unwrap();
assert!(errors.is_empty());
let names: Vec<&str> = attrs.iter().map(|a| a.name.as_str()).collect();
assert_eq!(names, ["scale", "version"]);
let (g, _) = r.attrs("sensors").unwrap();
assert!(matches!(&g[0].value, AttrValue::String(s) if s == "lab"));
}
#[test]
fn compound_is_refused_not_reinterpreted() {
use clawhdf5::CompoundTypeBuilder;
let ct = CompoundTypeBuilder::new()
.f64_field("x")
.i32_field("n")
.build();
let mut rec = Vec::new();
rec.extend_from_slice(&1.0f64.to_le_bytes());
rec.extend_from_slice(&7i32.to_le_bytes());
let mut b = FileBuilder::new();
b.create_dataset("table").with_compound_data(ct, rec, 1);
let r = Reader::open(b.finish().unwrap()).unwrap();
let e = r.read("table", None).unwrap_err();
assert!(e.contains("compound{x: f64, n: i32}"), "{e}");
assert!(e.contains("not supported"), "{e}");
}
#[test]
fn garbage_is_an_error() {
assert!(Reader::open(vec![0u8; 64]).is_err());
assert!(Reader::open(Vec::new()).is_err());
}
}
+244
View File
@@ -0,0 +1,244 @@
//! clawhdf5's HDF5 reader for JavaScript, via `wasm-bindgen`.
//!
//! ```js
//! import init, { open } from "./pkg/clawhdf5_wasm.js";
//! await init();
//! const file = open(new Uint8Array(await blob.arrayBuffer()));
//! file.list("/"); // [{ name, kind: "group" | "dataset" }]
//! file.info("/x"); // { shape, maxshape, dtype, elementShape }
//! file.attrs("/x"); // [{ name, value, dtype }]
//! file.read("/x"); // { shape, dtype, data: Float64Array | ... | string[] }
//! file.readHyperslab("/x", [0, 0], [10, 10]); // stride, block optional
//! file.free();
//! ```
//!
//! Numeric data comes back in the typed array of the stored width
//! (`Int16Array` for `i16`, `BigInt64Array` for `i64`, `Float32Array` for
//! `f32` and `f16`, ...); strings and enumeration names as arrays of strings.
//! Anything else is a thrown `Error` naming the datatype. Only the reader is
//! exposed: nothing here writes files.
//!
//! The logic lives in [`core`], which is plain Rust and tested natively.
pub mod core;
use clawhdf5::AttrValue;
use js_sys::{Array, Object, Reflect};
use wasm_bindgen::prelude::*;
use crate::core::{Data, Hyperslab, Reader};
/// JavaScript numbers are exact up to 2^53.
const MAX_SAFE_INTEGER: f64 = 9_007_199_254_740_991.0;
fn js_err(msg: String) -> JsError {
JsError::new(&msg)
}
fn set(obj: &Object, key: &str, value: impl Into<JsValue>) {
// Defining a property on a fresh plain object cannot fail.
Reflect::set(obj, &JsValue::from_str(key), &value.into()).unwrap_throw();
}
fn shape_to_js(shape: &[u64]) -> Array {
shape.iter().map(|&d| JsValue::from_f64(d as f64)).collect()
}
fn indices_from_js(what: &str, v: &[f64]) -> Result<Vec<u64>, JsError> {
v.iter()
.map(|&x| {
if x.is_finite() && x >= 0.0 && x.fract() == 0.0 && x <= MAX_SAFE_INTEGER {
Ok(x as u64)
} else {
Err(js_err(format!(
"{what} must hold non-negative integers, got {x}"
)))
}
})
.collect()
}
fn data_to_js(data: Data) -> JsValue {
match data {
Data::F32(v) => js_sys::Float32Array::from(&v[..]).into(),
Data::F64(v) => js_sys::Float64Array::from(&v[..]).into(),
Data::I8(v) => js_sys::Int8Array::from(&v[..]).into(),
Data::I16(v) => js_sys::Int16Array::from(&v[..]).into(),
Data::I32(v) => js_sys::Int32Array::from(&v[..]).into(),
Data::I64(v) => js_sys::BigInt64Array::from(&v[..]).into(),
Data::U8(v) => js_sys::Uint8Array::from(&v[..]).into(),
Data::U16(v) => js_sys::Uint16Array::from(&v[..]).into(),
Data::U32(v) => js_sys::Uint32Array::from(&v[..]).into(),
Data::U64(v) => js_sys::BigUint64Array::from(&v[..]).into(),
Data::Strings(v) => v
.into_iter()
.map(|s| JsValue::from_str(&s))
.collect::<Array>()
.into(),
}
}
/// A scalar integer as a `number` when exact, else a `bigint`.
fn int_to_js(x: i128) -> JsValue {
if (x as f64).abs() <= MAX_SAFE_INTEGER {
JsValue::from_f64(x as f64)
} else if let Ok(v) = i64::try_from(x) {
JsValue::from(v)
} else {
JsValue::from(x as u64)
}
}
/// An attribute value, and the datatype of one that has no JavaScript form.
fn attr_to_js(value: AttrValue) -> (JsValue, Option<String>) {
match value {
AttrValue::F64(x) => (JsValue::from_f64(x), None),
AttrValue::F64Array(v) => (js_sys::Float64Array::from(&v[..]).into(), None),
AttrValue::I64(x) => (int_to_js(i128::from(x)), None),
AttrValue::I64Array(v) => (js_sys::BigInt64Array::from(&v[..]).into(), None),
AttrValue::U64(x) => (int_to_js(i128::from(x)), None),
AttrValue::U64Array(v) => (js_sys::BigUint64Array::from(&v[..]).into(), None),
AttrValue::String(s) => (JsValue::from_str(&s), None),
AttrValue::StringArray(v) => (
v.iter()
.map(|s| JsValue::from_str(s))
.collect::<Array>()
.into(),
None,
),
AttrValue::Raw { datatype, .. } => (JsValue::NULL, Some(core::describe(&datatype))),
}
}
/// An open HDF5 (or NetCDF-4) file.
#[wasm_bindgen]
pub struct H5File {
inner: Reader,
}
/// Open a file from its bytes. Throws if they are not an HDF5 file.
#[wasm_bindgen]
pub fn open(bytes: Vec<u8>) -> Result<H5File, JsError> {
H5File::new(bytes)
}
/// The clawhdf5 version this module was built from.
#[wasm_bindgen]
pub fn version() -> String {
env!("CARGO_PKG_VERSION").to_string()
}
#[wasm_bindgen]
impl H5File {
/// Same as [`open`].
#[wasm_bindgen(constructor)]
pub fn new(bytes: Vec<u8>) -> Result<H5File, JsError> {
Ok(H5File {
inner: Reader::open(bytes).map_err(js_err)?,
})
}
/// `"group"` or `"dataset"`.
pub fn kind(&self, path: &str) -> Result<String, JsError> {
Ok(self.inner.kind(path).map_err(js_err)?.as_str().to_string())
}
/// The group's members: `[{ name, kind }]`, groups first.
pub fn list(&self, path: &str) -> Result<Array, JsError> {
Ok(self
.inner
.list(path)
.map_err(js_err)?
.into_iter()
.map(|c| {
let o = Object::new();
set(&o, "name", c.name);
set(&o, "kind", c.kind.as_str());
JsValue::from(o)
})
.collect())
}
/// `{ shape, maxshape, dtype, elementShape }`. `maxshape` is `null`
/// when not recorded, with `null` for each unlimited dimension.
pub fn info(&self, path: &str) -> Result<Object, JsError> {
let i = self.inner.info(path).map_err(js_err)?;
let o = Object::new();
set(&o, "shape", shape_to_js(&i.shape));
let max: JsValue = match i.maxshape {
None => JsValue::NULL,
Some(dims) => dims
.into_iter()
.map(|d| d.map_or(JsValue::NULL, |d| JsValue::from_f64(d as f64)))
.collect::<Array>()
.into(),
};
set(&o, "maxshape", max);
set(&o, "dtype", i.dtype);
set(&o, "elementShape", shape_to_js(&i.element_shape));
Ok(o)
}
/// `[{ name, value, dtype }]`, sorted by name. Scalars are `number`
/// (`bigint` beyond 2^53) or `string`; arrays are typed arrays or
/// `string[]`. An attribute with no JavaScript form has `value: null`
/// and its `dtype`; one that could not be read at all is reported by
/// [`attrErrors`](Self::attr_errors).
pub fn attrs(&self, path: &str) -> Result<Array, JsError> {
let (attrs, _) = self.inner.attrs(path).map_err(js_err)?;
Ok(attrs
.into_iter()
.map(|a| {
let o = Object::new();
set(&o, "name", a.name);
let (value, dtype) = attr_to_js(a.value);
set(&o, "value", value);
set(&o, "dtype", dtype.map_or(JsValue::NULL, JsValue::from));
JsValue::from(o)
})
.collect())
}
/// Messages for attributes that could not be read.
#[wasm_bindgen(js_name = attrErrors)]
pub fn attr_errors(&self, path: &str) -> Result<Array, JsError> {
let (_, errors) = self.inner.attrs(path).map_err(js_err)?;
Ok(errors.into_iter().map(JsValue::from).collect())
}
/// The whole dataset: `{ shape, dtype, data }`, `data` in row-major
/// order.
pub fn read(&self, path: &str) -> Result<Object, JsError> {
self.read_impl(path, None)
}
/// A regular hyperslab (`H5Sselect_hyperslab`): `stride` and `block`
/// default to 1. The result's shape is `count * block` per dimension.
#[wasm_bindgen(js_name = readHyperslab)]
pub fn read_hyperslab(
&self,
path: &str,
start: Vec<f64>,
count: Vec<f64>,
stride: Option<Vec<f64>>,
block: Option<Vec<f64>>,
) -> Result<Object, JsError> {
let slab = Hyperslab {
start: indices_from_js("start", &start)?,
count: indices_from_js("count", &count)?,
stride: stride.map(|s| indices_from_js("stride", &s)).transpose()?,
block: block.map(|b| indices_from_js("block", &b)).transpose()?,
};
self.read_impl(path, Some(&slab))
}
fn read_impl(&self, path: &str, slab: Option<&Hyperslab>) -> Result<Object, JsError> {
let dtype = self.inner.info(path).map_err(js_err)?.dtype;
let a = self.inner.read(path, slab).map_err(js_err)?;
let o = Object::new();
set(&o, "shape", shape_to_js(&a.shape));
set(&o, "dtype", dtype);
set(&o, "data", data_to_js(a.data));
Ok(o)
}
}
+252
View File
@@ -0,0 +1,252 @@
//! The wasm reader's core against files written by h5py and netCDF4, with
//! the values libhdf5 reads back as the reference. The generator,
//! `examples/wasm-viewer/test/make_fixture.py`, is shared with the Node test
//! of the built wasm package, so both compare against the same expectations.
//!
//! Skipped when python3 with h5py/netCDF4 is missing, unless
//! `CLAWHDF5_REQUIRE_INTEROP=1`. `CLAWHDF5_PYTHON` names the interpreter.
use std::path::{Path, PathBuf};
use std::process::Command;
use clawhdf5::AttrValue;
use clawhdf5_wasm::core::{Data, Hyperslab, Kind, Reader};
use serde_json::Value;
fn python() -> String {
std::env::var("CLAWHDF5_PYTHON").unwrap_or_else(|_| "python3".to_string())
}
fn interop_required() -> bool {
std::env::var("CLAWHDF5_REQUIRE_INTEROP").is_ok_and(|v| v == "1")
}
fn python_available() -> bool {
Command::new(python())
.args(["-c", "import h5py, netCDF4, numpy"])
.output()
.map(|o| o.status.success())
.unwrap_or(false)
}
fn generator() -> PathBuf {
Path::new(env!("CARGO_MANIFEST_DIR")).join("../../examples/wasm-viewer/test/make_fixture.py")
}
/// Values as comparable strings: integers exactly, floats by their f64
/// value (an f32 widens exactly), strings as themselves.
fn data_strings(d: &Data) -> (&'static str, Vec<String>) {
fn s<T: ToString>(v: &[T]) -> Vec<String> {
v.iter().map(ToString::to_string).collect()
}
fn f<T: Copy + Into<f64>>(v: &[T]) -> Vec<String> {
v.iter().map(|&x| format!("{:?}", x.into())).collect()
}
match d {
Data::F32(v) => ("f32", f(v)),
Data::F64(v) => ("f64", f(v)),
Data::I8(v) => ("i8", s(v)),
Data::I16(v) => ("i16", s(v)),
Data::I32(v) => ("i32", s(v)),
Data::I64(v) => ("i64", s(v)),
Data::U8(v) => ("u8", s(v)),
Data::U16(v) => ("u16", s(v)),
Data::U32(v) => ("u32", s(v)),
Data::U64(v) => ("u64", s(v)),
Data::Strings(v) => ("strings", v.clone()),
}
}
fn expected_strings(kind: &str, values: &Value) -> Vec<String> {
values
.as_array()
.unwrap()
.iter()
.map(|v| match (kind, v) {
("f32" | "f64", Value::Number(n)) => format!("{:?}", n.as_f64().unwrap()),
(_, Value::String(s)) => s.clone(),
other => panic!("unexpected expected value {other:?}"),
})
.collect()
}
fn shape(v: &Value) -> Vec<u64> {
v.as_array()
.unwrap()
.iter()
.map(|x| x.as_u64().unwrap())
.collect()
}
fn check_attr(file: &str, path: &str, name: &str, got: &AttrValue, want: &Value) {
let ctx = format!("{file}:{path}@{name}");
let ints = |v: &Value| -> Vec<String> {
v.as_array()
.unwrap()
.iter()
.map(|x| x.as_str().unwrap().to_string())
.collect()
};
let scalar = want["scalar"].as_bool().unwrap_or(false);
match got {
AttrValue::String(s) => assert_eq!(want["string"].as_str(), Some(s.as_str()), "{ctx}"),
AttrValue::StringArray(v) => {
let w: Vec<&str> = want["strings"]
.as_array()
.unwrap_or_else(|| panic!("{ctx}: got {v:?}"))
.iter()
.map(|x| x.as_str().unwrap())
.collect();
assert_eq!(v, &w, "{ctx}");
}
AttrValue::I64(x) => {
assert!(scalar, "{ctx}");
assert_eq!(vec![x.to_string()], ints(&want["int"]), "{ctx}");
}
AttrValue::U64(x) => {
assert!(scalar, "{ctx}");
assert_eq!(vec![x.to_string()], ints(&want["int"]), "{ctx}");
}
AttrValue::I64Array(v) => assert_eq!(
v.iter().map(ToString::to_string).collect::<Vec<_>>(),
ints(&want["int"]),
"{ctx}"
),
AttrValue::U64Array(v) => assert_eq!(
v.iter().map(ToString::to_string).collect::<Vec<_>>(),
ints(&want["int"]),
"{ctx}"
),
AttrValue::F64(x) => {
assert!(scalar, "{ctx}");
assert_eq!(Some(*x), want["float"][0].as_f64(), "{ctx}");
}
AttrValue::F64Array(v) => {
let w: Vec<f64> = want["float"]
.as_array()
.unwrap()
.iter()
.map(|x| x.as_f64().unwrap())
.collect();
assert_eq!(v, &w, "{ctx}");
}
AttrValue::Raw { datatype, .. } => panic!("{ctx}: undecoded {datatype:?}"),
}
}
fn check_file(dir: &Path, file: &str, exp: &Value) {
let r = Reader::open(std::fs::read(dir.join(file)).unwrap()).unwrap();
for (path, want) in exp["lists"].as_object().unwrap() {
let list = r
.list(path)
.unwrap_or_else(|e| panic!("{file}:{path}: {e}"));
for (kind, key) in [(Kind::Group, "groups"), (Kind::Dataset, "datasets")] {
let mut got: Vec<&str> = list
.iter()
.filter(|c| c.kind == kind)
.map(|c| c.name.as_str())
.collect();
got.sort();
let w: Vec<&str> = want[key]
.as_array()
.unwrap()
.iter()
.map(|x| x.as_str().unwrap())
.collect();
assert_eq!(got, w, "{file}:{path} {key}");
}
}
for (path, want) in exp["datasets"].as_object().unwrap() {
let kind = want["kind"].as_str().unwrap();
let a = r
.read(path, None)
.unwrap_or_else(|e| panic!("{file}:{path}: {e}"));
assert_eq!(a.shape, shape(&want["shape"]), "{file}:{path} shape");
let (got_kind, got) = data_strings(&a.data);
assert_eq!(got_kind, kind, "{file}:{path} kind");
assert_eq!(
got,
expected_strings(kind, &want["values"]),
"{file}:{path}"
);
if let Some(slab) = want.get("slab") {
let h = Hyperslab {
start: shape(&slab["start"]),
count: shape(&slab["count"]),
stride: Some(shape(&slab["stride"])),
block: None,
};
let a = r
.read(path, Some(&h))
.unwrap_or_else(|e| panic!("{file}:{path} {h:?}: {e}"));
assert_eq!(a.shape, shape(&slab["shape"]), "{file}:{path} slab shape");
assert_eq!(
data_strings(&a.data).1,
expected_strings(kind, &slab["values"]),
"{file}:{path} slab"
);
}
}
for (path, what) in exp["errors"].as_object().unwrap() {
let e = r.read(path, None).expect_err(path);
assert!(e.contains(what.as_str().unwrap()), "{file}:{path}: {e}");
}
let skip: Vec<&str> = exp["skip_attrs"]
.as_array()
.unwrap()
.iter()
.map(|x| x.as_str().unwrap())
.collect();
for (path, want) in exp["attrs"].as_object().unwrap() {
let (attrs, errors) = r
.attrs(path)
.unwrap_or_else(|e| panic!("{file}:{path}: {e}"));
assert!(errors.is_empty(), "{file}:{path}: {errors:?}");
let want = want.as_object().unwrap();
let mut compared = 0;
for a in &attrs {
if a.name.starts_with('_') || skip.contains(&a.name.as_str()) {
continue;
}
let w = want
.get(&a.name)
.unwrap_or_else(|| panic!("{file}:{path}: unexpected attribute {}", a.name));
check_attr(file, path, &a.name, &a.value, w);
compared += 1;
}
assert_eq!(compared, want.len(), "{file}:{path}: attributes missing");
}
}
#[test]
fn reads_what_h5py_and_netcdf4_wrote() {
if !python_available() {
assert!(
!interop_required(),
"CLAWHDF5_REQUIRE_INTEROP=1 but python with h5py, netCDF4 and numpy is not available"
);
eprintln!("SKIP: python with h5py/netCDF4 not available");
return;
}
let dir = tempfile::tempdir().unwrap();
let out = Command::new(python())
.arg(generator())
.arg(dir.path())
.output()
.expect("run python");
assert!(
out.status.success(),
"fixture generator failed:\n{}",
String::from_utf8_lossy(&out.stderr)
);
let exp: Value =
serde_json::from_slice(&std::fs::read(dir.path().join("expected.json")).unwrap()).unwrap();
for file in ["fixture.h5", "fixture.nc"] {
check_file(dir.path(), file, &exp[file]);
}
}
+184
View File
@@ -0,0 +1,184 @@
"""Write HDF5 and NetCDF-4 test files with h5py/netCDF4, and what libhdf5
reads back from them, for the clawhdf5-wasm tests.
python make_fixture.py OUT_DIR
writes OUT_DIR/fixture.h5, OUT_DIR/fixture.nc and OUT_DIR/expected.json.
Both the Rust test (crates/clawhdf5-wasm/tests/h5py_interop.rs, native) and
the Node test (test.mjs, the built wasm package) compare against the same
expected.json, so the two check the same values.
Every expected value comes from h5py reading the file back (numpy slicing
for hyperslabs), never from the arrays that were written. Integers are
encoded as strings so JSON.parse keeps 64-bit values exact.
"""
import json
import sys
import warnings
from pathlib import Path
import h5py
import netCDF4
import numpy as np
# netCDF4 1.7 trips numpy 2.5's shape-setting deprecation on assignment.
warnings.filterwarnings("ignore", category=DeprecationWarning)
out = Path(sys.argv[1])
out.mkdir(parents=True, exist_ok=True)
h5 = out / "fixture.h5"
nc = out / "fixture.nc"
rng = np.random.default_rng(7)
with h5py.File(h5, "w") as f:
f.attrs["title"] = "wasm fixture"
f.attrs["version"] = np.int64(3)
f.attrs["scale"] = np.array([0.5, 2.0])
f.attrs["big"] = np.uint64(2**63 + 5)
f.attrs.create("vlen_note", "héllo", dtype=h5py.string_dtype())
f.create_dataset(
"grid", data=np.arange(60, dtype="<f8").reshape(6, 10) / 4,
chunks=(4, 3), compression="gzip", shuffle=True,
)
f.create_dataset("f32_be", data=rng.standard_normal(7).astype(">f4"))
f.create_dataset("f16", data=np.array([0.5, -1.25, 65504], dtype="<f2"))
f.create_dataset("i8", data=np.array([-128, -1, 0, 127], dtype="i1"))
f.create_dataset("i16_be", data=np.array([-32768, 5, 32767], dtype=">i2"))
f.create_dataset("u16", data=np.array([0, 40000, 65535], dtype="<u2"))
f.create_dataset("u32", data=np.array([0, 4_000_000_000], dtype="<u4"))
f.create_dataset("i64", data=np.array([-(2**63), 2**53 + 1, 7], dtype="<i8"))
f.create_dataset("u64", data=np.array([2**64 - 1, 1], dtype="<u8"))
f.create_dataset("scalar", data=np.float64(2.5))
f.create_dataset("fixed_str", data=np.array([b"alpha", b"be", b""], dtype="S5"))
f.create_dataset(
"vlen_str", data=["one", "двa", ""], dtype=h5py.string_dtype()
)
f.create_dataset("flags", data=np.array([True, False, True]))
# An array datatype: four elements, each an i4[2].
pairs = f.create_dataset("pairs", shape=(4,), dtype=np.dtype(("<i4", (2,))))
pairs[...] = np.arange(8, dtype="<i4").reshape(4, 2)
f.create_dataset(
"cube", data=np.arange(2 * 5 * 6, dtype="<i4").reshape(2, 5, 6),
chunks=(1, 2, 3), compression="gzip",
)
comp = np.zeros(2, dtype=[("x", "<f8"), ("n", "<i4")])
f.create_dataset("table", data=comp)
g = f.create_group("sensors")
g.attrs["location"] = "lab"
g.create_dataset("temp", data=np.array([21.5, 22.0, 22.25], dtype="<f4"))
g.create_group("empty")
f["alias"] = h5py.SoftLink("/sensors/temp")
with netCDF4.Dataset(nc, "w") as d:
d.title = "nc fixture"
d.createDimension("time", None)
d.createDimension("x", 4)
t = d.createVariable("time", "f8", ("time",))
t.units = "days since 2000-01-01"
v = d.createVariable("temp", "f4", ("time", "x"), zlib=True)
t[:] = np.arange(3)
v[:] = np.arange(12, dtype="f4").reshape(3, 4) + 0.5
def kind(dt):
"""The typed-array kind clawhdf5-wasm returns for a numpy dtype."""
if dt.kind == "b" or h5py.check_enum_dtype(dt) is not None:
return "strings"
if h5py.check_string_dtype(dt) is not None or dt.kind == "S":
return "strings"
if dt.subdtype is not None:
return kind(dt.subdtype[0])
if dt.kind == "f":
return "f64" if dt.itemsize == 8 else "f32"
if dt.kind in "iu":
return f"{dt.kind}{dt.itemsize * 8}"
raise ValueError(dt)
def flat(a, k):
a = np.asarray(a)
if k == "strings":
if a.dtype.kind == "b":
return ["TRUE" if x else "FALSE" for x in a.ravel()]
return [x.decode() if isinstance(x, bytes) else str(x) for x in a.ravel()]
if k.startswith(("i", "u")):
return [str(int(x)) for x in a.ravel()]
return [float(x) for x in a.ravel()]
def entry(ds, slab=None):
k = kind(ds.dtype)
data = ds[()]
e = {"kind": k, "shape": list(np.shape(data)), "values": flat(data, k)}
if slab:
start, count, stride = slab
idx = tuple(slice(s, s + (c - 1) * st + 1, st)
for s, c, st in zip(start, count, stride))
part = ds[idx]
e["slab"] = {"start": start, "count": count, "stride": stride,
"shape": list(part.shape), "values": flat(part, k)}
return e
def attr(v):
v = np.asarray(v) if not isinstance(v, (str, bytes)) else v
if isinstance(v, bytes):
return {"string": v.decode()}
if isinstance(v, str):
return {"string": v}
if v.dtype.kind in "iu":
return {"int": [str(int(x)) for x in v.ravel()], "scalar": v.ndim == 0}
if v.dtype.kind == "f":
return {"float": [float(x) for x in v.ravel()], "scalar": v.ndim == 0}
if v.dtype.kind in "OSU":
items = [x.decode() if isinstance(x, bytes) else str(x) for x in v.ravel()]
return {"string": items[0]} if v.ndim == 0 else {"strings": items}
raise ValueError(v.dtype)
# Attributes the reader returns but the comparison leaves out: netCDF-4's
# internal ones (a leading underscore), and dimension-scale bookkeeping.
SKIP_ATTRS = ["DIMENSION_LIST", "REFERENCE_LIST", "CLASS", "NAME"]
def describe(path):
expected = {"datasets": {}, "errors": {}, "attrs": {}, "lists": {},
"skip_attrs": SKIP_ATTRS}
with h5py.File(path, "r") as f:
def walk(key, obj):
expected["attrs"][key] = {
k: attr(obj.attrs[k]) for k in obj.attrs
if not k.startswith("_") and k not in SKIP_ATTRS}
if isinstance(obj, h5py.Group):
members = {n: obj.get(n) for n in obj}
groups = [n for n, o in members.items() if isinstance(o, h5py.Group)]
sets = [n for n, o in members.items() if isinstance(o, h5py.Dataset)]
expected["lists"][key] = {"groups": sorted(groups),
"datasets": sorted(sets)}
for n, o in members.items():
walk(key.rstrip("/") + "/" + n, o)
elif obj.dtype.names:
expected["errors"][key] = "compound"
else:
expected["datasets"][key] = entry(obj, slab_for(obj))
walk("/", f)
return expected
def slab_for(obj):
"""A strided hyperslab inside the dataset's extent, or None."""
if obj.ndim == 0 or obj.shape[0] < 2 or 0 in obj.shape:
return None
start = [1] + [0] * (obj.ndim - 1)
count = [max(1, (obj.shape[0] - 1) // 2)] + [
max(1, (n + 1) // 2) for n in obj.shape[1:]]
stride = [2 if obj.shape[0] > 2 else 1] + [2] * (obj.ndim - 1)
count = [min(c, (n - s - 1) // st + 1)
for c, s, st, n in zip(count, start, stride, obj.shape)]
return (start, count, stride)
json.dump({"fixture.h5": describe(h5), "fixture.nc": describe(nc)},
open(out / "expected.json", "w"), indent=1, ensure_ascii=False)
+9 -2
View File
@@ -92,7 +92,8 @@ run_step "cargo clippy (fast-deflate / zlib-ng)" cargo clippy \
no_c_in_default_build() { no_c_in_default_build() {
local crate found=0 local crate found=0
for crate in clawhdf5-format clawhdf5-io clawhdf5-filters clawhdf5 \ for crate in clawhdf5-format clawhdf5-io clawhdf5-filters clawhdf5 \
clawhdf5-agent clawhdf5-ann clawhdf5-accel clawhdf5-netcdf4 clawhdf5-cli; do clawhdf5-agent clawhdf5-ann clawhdf5-accel clawhdf5-netcdf4 clawhdf5-cli \
clawhdf5-wasm; do
local c_deps local c_deps
c_deps=$(cargo tree -q -p "$crate" -e normal,build --prefix none \ c_deps=$(cargo tree -q -p "$crate" -e normal,build --prefix none \
| grep -E '^([a-z0-9_-]+-sys|cc|cmake) v' | sort -u) | grep -E '^([a-z0-9_-]+-sys|cc|cmake) v' | sort -u)
@@ -107,12 +108,18 @@ no_c_in_default_build() {
run_step "no C in the default build (core crates)" no_c_in_default_build run_step "no C in the default build (core crates)" no_c_in_default_build
# The reader in the browser: the facade's read path must build for # The reader in the browser: the facade's read path must build for
# wasm32-unknown-unknown (no mmap, no threads, no file system). Needs # wasm32-unknown-unknown (no mmap, no threads, no file system), and the
# wasm-bindgen crate must build and lint there. Needs
# `rustup target add wasm32-unknown-unknown`. # `rustup target add wasm32-unknown-unknown`.
run_step "wasm32 build (clawhdf5, no default features)" cargo build \ run_step "wasm32 build (clawhdf5, no default features)" cargo build \
-p clawhdf5 \ -p clawhdf5 \
--target wasm32-unknown-unknown \ --target wasm32-unknown-unknown \
--no-default-features --no-default-features
run_step "wasm32 clippy (clawhdf5-wasm)" cargo clippy \
-p clawhdf5-wasm \
--target wasm32-unknown-unknown \
--all-targets \
-- -D warnings
# The workspace declares a minimum Rust version (rust-version in Cargo.toml); # The workspace declares a minimum Rust version (rust-version in Cargo.toml);
# check that it really builds there, so the README badge and the manifests # check that it really builds there, so the README badge and the manifests