feat(wasm): clawhdf5-wasm, the reader for JavaScript via wasm-bindgen
open(bytes) -> H5File with kind/list/info/attrs/attrErrors/read/ readHyperslab. Numeric data comes back in the typed array of the stored width (Int16Array for i16, BigInt64Array for i64, Float32Array for f32/f16, ...), strings and enum names as string arrays, array datatypes flattened with their dims appended to the shape. Compound, reference, opaque and VL-sequence datasets are refused with an error naming the type; nothing is returned as reinterpreted bytes. The logic is in a plain-Rust core module, tested natively: unit tests, and h5py_interop, which compares every dataset, hyperslab, listing and attribute of an h5py- and a netCDF4-written file with what libhdf5 reads back (generator shared with the Node test of the built package). No mmap, no threads; lz4 is on, zstd/szip (C) are not. A wasm-release profile (opt-level s, LTO) serves the browser build. ci-test.sh lints the crate for wasm32 and checks it builds no C. Co-Authored-By: Claude Opus 5.5 (1M context) <[email protected]>
This commit is contained in:
osobh
@@ -0,0 +1,252 @@
|
||||
//! The wasm reader's core against files written by h5py and netCDF4, with
|
||||
//! the values libhdf5 reads back as the reference. The generator,
|
||||
//! `examples/wasm-viewer/test/make_fixture.py`, is shared with the Node test
|
||||
//! of the built wasm package, so both compare against the same expectations.
|
||||
//!
|
||||
//! Skipped when python3 with h5py/netCDF4 is missing, unless
|
||||
//! `CLAWHDF5_REQUIRE_INTEROP=1`. `CLAWHDF5_PYTHON` names the interpreter.
|
||||
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::process::Command;
|
||||
|
||||
use clawhdf5::AttrValue;
|
||||
use clawhdf5_wasm::core::{Data, Hyperslab, Kind, Reader};
|
||||
use serde_json::Value;
|
||||
|
||||
fn python() -> String {
|
||||
std::env::var("CLAWHDF5_PYTHON").unwrap_or_else(|_| "python3".to_string())
|
||||
}
|
||||
|
||||
fn interop_required() -> bool {
|
||||
std::env::var("CLAWHDF5_REQUIRE_INTEROP").is_ok_and(|v| v == "1")
|
||||
}
|
||||
|
||||
fn python_available() -> bool {
|
||||
Command::new(python())
|
||||
.args(["-c", "import h5py, netCDF4, numpy"])
|
||||
.output()
|
||||
.map(|o| o.status.success())
|
||||
.unwrap_or(false)
|
||||
}
|
||||
|
||||
fn generator() -> PathBuf {
|
||||
Path::new(env!("CARGO_MANIFEST_DIR")).join("../../examples/wasm-viewer/test/make_fixture.py")
|
||||
}
|
||||
|
||||
/// Values as comparable strings: integers exactly, floats by their f64
|
||||
/// value (an f32 widens exactly), strings as themselves.
|
||||
fn data_strings(d: &Data) -> (&'static str, Vec<String>) {
|
||||
fn s<T: ToString>(v: &[T]) -> Vec<String> {
|
||||
v.iter().map(ToString::to_string).collect()
|
||||
}
|
||||
fn f<T: Copy + Into<f64>>(v: &[T]) -> Vec<String> {
|
||||
v.iter().map(|&x| format!("{:?}", x.into())).collect()
|
||||
}
|
||||
match d {
|
||||
Data::F32(v) => ("f32", f(v)),
|
||||
Data::F64(v) => ("f64", f(v)),
|
||||
Data::I8(v) => ("i8", s(v)),
|
||||
Data::I16(v) => ("i16", s(v)),
|
||||
Data::I32(v) => ("i32", s(v)),
|
||||
Data::I64(v) => ("i64", s(v)),
|
||||
Data::U8(v) => ("u8", s(v)),
|
||||
Data::U16(v) => ("u16", s(v)),
|
||||
Data::U32(v) => ("u32", s(v)),
|
||||
Data::U64(v) => ("u64", s(v)),
|
||||
Data::Strings(v) => ("strings", v.clone()),
|
||||
}
|
||||
}
|
||||
|
||||
fn expected_strings(kind: &str, values: &Value) -> Vec<String> {
|
||||
values
|
||||
.as_array()
|
||||
.unwrap()
|
||||
.iter()
|
||||
.map(|v| match (kind, v) {
|
||||
("f32" | "f64", Value::Number(n)) => format!("{:?}", n.as_f64().unwrap()),
|
||||
(_, Value::String(s)) => s.clone(),
|
||||
other => panic!("unexpected expected value {other:?}"),
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
fn shape(v: &Value) -> Vec<u64> {
|
||||
v.as_array()
|
||||
.unwrap()
|
||||
.iter()
|
||||
.map(|x| x.as_u64().unwrap())
|
||||
.collect()
|
||||
}
|
||||
|
||||
fn check_attr(file: &str, path: &str, name: &str, got: &AttrValue, want: &Value) {
|
||||
let ctx = format!("{file}:{path}@{name}");
|
||||
let ints = |v: &Value| -> Vec<String> {
|
||||
v.as_array()
|
||||
.unwrap()
|
||||
.iter()
|
||||
.map(|x| x.as_str().unwrap().to_string())
|
||||
.collect()
|
||||
};
|
||||
let scalar = want["scalar"].as_bool().unwrap_or(false);
|
||||
match got {
|
||||
AttrValue::String(s) => assert_eq!(want["string"].as_str(), Some(s.as_str()), "{ctx}"),
|
||||
AttrValue::StringArray(v) => {
|
||||
let w: Vec<&str> = want["strings"]
|
||||
.as_array()
|
||||
.unwrap_or_else(|| panic!("{ctx}: got {v:?}"))
|
||||
.iter()
|
||||
.map(|x| x.as_str().unwrap())
|
||||
.collect();
|
||||
assert_eq!(v, &w, "{ctx}");
|
||||
}
|
||||
AttrValue::I64(x) => {
|
||||
assert!(scalar, "{ctx}");
|
||||
assert_eq!(vec![x.to_string()], ints(&want["int"]), "{ctx}");
|
||||
}
|
||||
AttrValue::U64(x) => {
|
||||
assert!(scalar, "{ctx}");
|
||||
assert_eq!(vec![x.to_string()], ints(&want["int"]), "{ctx}");
|
||||
}
|
||||
AttrValue::I64Array(v) => assert_eq!(
|
||||
v.iter().map(ToString::to_string).collect::<Vec<_>>(),
|
||||
ints(&want["int"]),
|
||||
"{ctx}"
|
||||
),
|
||||
AttrValue::U64Array(v) => assert_eq!(
|
||||
v.iter().map(ToString::to_string).collect::<Vec<_>>(),
|
||||
ints(&want["int"]),
|
||||
"{ctx}"
|
||||
),
|
||||
AttrValue::F64(x) => {
|
||||
assert!(scalar, "{ctx}");
|
||||
assert_eq!(Some(*x), want["float"][0].as_f64(), "{ctx}");
|
||||
}
|
||||
AttrValue::F64Array(v) => {
|
||||
let w: Vec<f64> = want["float"]
|
||||
.as_array()
|
||||
.unwrap()
|
||||
.iter()
|
||||
.map(|x| x.as_f64().unwrap())
|
||||
.collect();
|
||||
assert_eq!(v, &w, "{ctx}");
|
||||
}
|
||||
AttrValue::Raw { datatype, .. } => panic!("{ctx}: undecoded {datatype:?}"),
|
||||
}
|
||||
}
|
||||
|
||||
fn check_file(dir: &Path, file: &str, exp: &Value) {
|
||||
let r = Reader::open(std::fs::read(dir.join(file)).unwrap()).unwrap();
|
||||
|
||||
for (path, want) in exp["lists"].as_object().unwrap() {
|
||||
let list = r
|
||||
.list(path)
|
||||
.unwrap_or_else(|e| panic!("{file}:{path}: {e}"));
|
||||
for (kind, key) in [(Kind::Group, "groups"), (Kind::Dataset, "datasets")] {
|
||||
let mut got: Vec<&str> = list
|
||||
.iter()
|
||||
.filter(|c| c.kind == kind)
|
||||
.map(|c| c.name.as_str())
|
||||
.collect();
|
||||
got.sort();
|
||||
let w: Vec<&str> = want[key]
|
||||
.as_array()
|
||||
.unwrap()
|
||||
.iter()
|
||||
.map(|x| x.as_str().unwrap())
|
||||
.collect();
|
||||
assert_eq!(got, w, "{file}:{path} {key}");
|
||||
}
|
||||
}
|
||||
|
||||
for (path, want) in exp["datasets"].as_object().unwrap() {
|
||||
let kind = want["kind"].as_str().unwrap();
|
||||
let a = r
|
||||
.read(path, None)
|
||||
.unwrap_or_else(|e| panic!("{file}:{path}: {e}"));
|
||||
assert_eq!(a.shape, shape(&want["shape"]), "{file}:{path} shape");
|
||||
let (got_kind, got) = data_strings(&a.data);
|
||||
assert_eq!(got_kind, kind, "{file}:{path} kind");
|
||||
assert_eq!(
|
||||
got,
|
||||
expected_strings(kind, &want["values"]),
|
||||
"{file}:{path}"
|
||||
);
|
||||
|
||||
if let Some(slab) = want.get("slab") {
|
||||
let h = Hyperslab {
|
||||
start: shape(&slab["start"]),
|
||||
count: shape(&slab["count"]),
|
||||
stride: Some(shape(&slab["stride"])),
|
||||
block: None,
|
||||
};
|
||||
let a = r
|
||||
.read(path, Some(&h))
|
||||
.unwrap_or_else(|e| panic!("{file}:{path} {h:?}: {e}"));
|
||||
assert_eq!(a.shape, shape(&slab["shape"]), "{file}:{path} slab shape");
|
||||
assert_eq!(
|
||||
data_strings(&a.data).1,
|
||||
expected_strings(kind, &slab["values"]),
|
||||
"{file}:{path} slab"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
for (path, what) in exp["errors"].as_object().unwrap() {
|
||||
let e = r.read(path, None).expect_err(path);
|
||||
assert!(e.contains(what.as_str().unwrap()), "{file}:{path}: {e}");
|
||||
}
|
||||
|
||||
let skip: Vec<&str> = exp["skip_attrs"]
|
||||
.as_array()
|
||||
.unwrap()
|
||||
.iter()
|
||||
.map(|x| x.as_str().unwrap())
|
||||
.collect();
|
||||
for (path, want) in exp["attrs"].as_object().unwrap() {
|
||||
let (attrs, errors) = r
|
||||
.attrs(path)
|
||||
.unwrap_or_else(|e| panic!("{file}:{path}: {e}"));
|
||||
assert!(errors.is_empty(), "{file}:{path}: {errors:?}");
|
||||
let want = want.as_object().unwrap();
|
||||
let mut compared = 0;
|
||||
for a in &attrs {
|
||||
if a.name.starts_with('_') || skip.contains(&a.name.as_str()) {
|
||||
continue;
|
||||
}
|
||||
let w = want
|
||||
.get(&a.name)
|
||||
.unwrap_or_else(|| panic!("{file}:{path}: unexpected attribute {}", a.name));
|
||||
check_attr(file, path, &a.name, &a.value, w);
|
||||
compared += 1;
|
||||
}
|
||||
assert_eq!(compared, want.len(), "{file}:{path}: attributes missing");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn reads_what_h5py_and_netcdf4_wrote() {
|
||||
if !python_available() {
|
||||
assert!(
|
||||
!interop_required(),
|
||||
"CLAWHDF5_REQUIRE_INTEROP=1 but python with h5py, netCDF4 and numpy is not available"
|
||||
);
|
||||
eprintln!("SKIP: python with h5py/netCDF4 not available");
|
||||
return;
|
||||
}
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let out = Command::new(python())
|
||||
.arg(generator())
|
||||
.arg(dir.path())
|
||||
.output()
|
||||
.expect("run python");
|
||||
assert!(
|
||||
out.status.success(),
|
||||
"fixture generator failed:\n{}",
|
||||
String::from_utf8_lossy(&out.stderr)
|
||||
);
|
||||
let exp: Value =
|
||||
serde_json::from_slice(&std::fs::read(dir.path().join("expected.json")).unwrap()).unwrap();
|
||||
for file in ["fixture.h5", "fixture.nc"] {
|
||||
check_file(dir.path(), file, &exp[file]);
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user