feat(wasm): clawhdf5-wasm, the reader for JavaScript via wasm-bindgen
open(bytes) -> H5File with kind/list/info/attrs/attrErrors/read/ readHyperslab. Numeric data comes back in the typed array of the stored width (Int16Array for i16, BigInt64Array for i64, Float32Array for f32/f16, ...), strings and enum names as string arrays, array datatypes flattened with their dims appended to the shape. Compound, reference, opaque and VL-sequence datasets are refused with an error naming the type; nothing is returned as reinterpreted bytes. The logic is in a plain-Rust core module, tested natively: unit tests, and h5py_interop, which compares every dataset, hyperslab, listing and attribute of an h5py- and a netCDF4-written file with what libhdf5 reads back (generator shared with the Node test of the built package). No mmap, no threads; lz4 is on, zstd/szip (C) are not. A wasm-release profile (opt-level s, LTO) serves the browser build. ci-test.sh lints the crate for wasm32 and checks it builds no C. Co-Authored-By: Claude Opus 5.5 (1M context) <[email protected]>
This commit is contained in:
osobh
@@ -0,0 +1,603 @@
|
||||
//! The reader behind the JavaScript API, in plain Rust so it is tested
|
||||
//! natively. The `wasm_bindgen` layer in `lib.rs` only converts these types
|
||||
//! to JavaScript values.
|
||||
//!
|
||||
//! Every read either returns the dataset's values or an error: a datatype
|
||||
//! with no typed-array mapping (compound, reference, opaque, ...) is refused
|
||||
//! with a message naming it, never returned as reinterpreted bytes.
|
||||
|
||||
use clawhdf5::{AttrValue, File, Selection};
|
||||
use clawhdf5_format::data_read;
|
||||
use clawhdf5_format::datatype::{Datatype, DatatypeByteOrder};
|
||||
|
||||
/// Errors are reported to JavaScript as messages.
|
||||
pub type Result<T> = std::result::Result<T, String>;
|
||||
|
||||
fn err(e: impl std::fmt::Display) -> String {
|
||||
e.to_string()
|
||||
}
|
||||
|
||||
/// What a path names.
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub enum Kind {
|
||||
Group,
|
||||
Dataset,
|
||||
}
|
||||
|
||||
impl Kind {
|
||||
pub fn as_str(self) -> &'static str {
|
||||
match self {
|
||||
Kind::Group => "group",
|
||||
Kind::Dataset => "dataset",
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// One entry of a group listing.
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct Child {
|
||||
pub name: String,
|
||||
pub kind: Kind,
|
||||
}
|
||||
|
||||
/// A dataset's metadata.
|
||||
#[derive(Debug, Clone, PartialEq)]
|
||||
pub struct DatasetInfo {
|
||||
/// Dataspace dimensions (empty for a scalar).
|
||||
pub shape: Vec<u64>,
|
||||
/// Maximum dimensions, `None` per unlimited dimension; `None` overall
|
||||
/// when the dataspace records none.
|
||||
pub maxshape: Option<Vec<Option<u64>>>,
|
||||
/// Human-readable datatype, e.g. `f64`, `i16 (big-endian)`, `string[8]`.
|
||||
pub dtype: String,
|
||||
/// Dimensions of an array datatype's elements, appended to the shape of
|
||||
/// what [`Reader::read`] returns (empty otherwise).
|
||||
pub element_shape: Vec<u64>,
|
||||
}
|
||||
|
||||
/// Decoded values, one variant per JavaScript typed array.
|
||||
#[derive(Debug, Clone, PartialEq)]
|
||||
pub enum Data {
|
||||
F32(Vec<f32>),
|
||||
F64(Vec<f64>),
|
||||
I8(Vec<i8>),
|
||||
I16(Vec<i16>),
|
||||
I32(Vec<i32>),
|
||||
I64(Vec<i64>),
|
||||
U8(Vec<u8>),
|
||||
U16(Vec<u16>),
|
||||
U32(Vec<u32>),
|
||||
U64(Vec<u64>),
|
||||
/// Fixed- and variable-length strings, and enumeration member names.
|
||||
Strings(Vec<String>),
|
||||
}
|
||||
|
||||
impl Data {
|
||||
pub fn len(&self) -> usize {
|
||||
match self {
|
||||
Data::F32(v) => v.len(),
|
||||
Data::F64(v) => v.len(),
|
||||
Data::I8(v) => v.len(),
|
||||
Data::I16(v) => v.len(),
|
||||
Data::I32(v) => v.len(),
|
||||
Data::I64(v) => v.len(),
|
||||
Data::U8(v) => v.len(),
|
||||
Data::U16(v) => v.len(),
|
||||
Data::U32(v) => v.len(),
|
||||
Data::U64(v) => v.len(),
|
||||
Data::Strings(v) => v.len(),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn is_empty(&self) -> bool {
|
||||
self.len() == 0
|
||||
}
|
||||
}
|
||||
|
||||
/// Values in row-major order with their shape.
|
||||
#[derive(Debug, Clone, PartialEq)]
|
||||
pub struct Array {
|
||||
pub shape: Vec<u64>,
|
||||
pub data: Data,
|
||||
}
|
||||
|
||||
/// A regular hyperslab, as in `H5Sselect_hyperslab`. `stride` and `block`
|
||||
/// default to 1 in every dimension.
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct Hyperslab {
|
||||
pub start: Vec<u64>,
|
||||
pub count: Vec<u64>,
|
||||
pub stride: Option<Vec<u64>>,
|
||||
pub block: Option<Vec<u64>>,
|
||||
}
|
||||
|
||||
/// An attribute: its value, or why it has none.
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct Attr {
|
||||
pub name: String,
|
||||
pub value: AttrValue,
|
||||
}
|
||||
|
||||
/// An open file, held in memory.
|
||||
pub struct Reader {
|
||||
file: File,
|
||||
}
|
||||
|
||||
impl Reader {
|
||||
/// Parse a file from its bytes (the browser hands over the whole file).
|
||||
pub fn open(bytes: Vec<u8>) -> Result<Self> {
|
||||
Ok(Self {
|
||||
file: File::from_bytes(bytes).map_err(err)?,
|
||||
})
|
||||
}
|
||||
|
||||
/// Whether `path` names a group or a dataset.
|
||||
pub fn kind(&self, path: &str) -> Result<Kind> {
|
||||
match self.file.dataset(path) {
|
||||
Ok(_) => Ok(Kind::Dataset),
|
||||
Err(clawhdf5::Error::NotADataset(_)) => Ok(Kind::Group),
|
||||
Err(e) => Err(err(e)),
|
||||
}
|
||||
}
|
||||
|
||||
/// The groups, then the datasets, in the group at `path` (`/` is the
|
||||
/// root). Soft links are listed as their targets; external and dangling
|
||||
/// links, and named datatypes, are left out.
|
||||
pub fn list(&self, path: &str) -> Result<Vec<Child>> {
|
||||
if self.kind(path)? != Kind::Group {
|
||||
return Err(format!("not a group: {path}"));
|
||||
}
|
||||
let group = self.file.group(path).map_err(err)?;
|
||||
let mut out: Vec<Child> = group
|
||||
.groups()
|
||||
.map_err(err)?
|
||||
.into_iter()
|
||||
.map(|name| Child {
|
||||
name,
|
||||
kind: Kind::Group,
|
||||
})
|
||||
.collect();
|
||||
out.extend(
|
||||
group
|
||||
.datasets()
|
||||
.map_err(err)?
|
||||
.into_iter()
|
||||
.map(|name| Child {
|
||||
name,
|
||||
kind: Kind::Dataset,
|
||||
}),
|
||||
);
|
||||
Ok(out)
|
||||
}
|
||||
|
||||
/// Shape, max shape and datatype of the dataset at `path`.
|
||||
pub fn info(&self, path: &str) -> Result<DatasetInfo> {
|
||||
let ds = self.file.dataset(path).map_err(err)?;
|
||||
let dt = ds.raw_datatype().map_err(err)?;
|
||||
let maxshape = ds.max_dimensions().map_err(err)?.map(|dims| {
|
||||
dims.into_iter()
|
||||
.map(|d| (d != u64::MAX).then_some(d))
|
||||
.collect()
|
||||
});
|
||||
Ok(DatasetInfo {
|
||||
shape: ds.shape().map_err(err)?,
|
||||
maxshape,
|
||||
dtype: describe(&dt),
|
||||
element_shape: element_shape(&dt),
|
||||
})
|
||||
}
|
||||
|
||||
/// The attributes of the group or dataset at `path`, sorted by name, and
|
||||
/// one message per attribute that could not be read at all. An attribute
|
||||
/// whose type has no plain JavaScript form is returned as
|
||||
/// [`AttrValue::Raw`].
|
||||
pub fn attrs(&self, path: &str) -> Result<(Vec<Attr>, Vec<String>)> {
|
||||
let (map, errors) = match self.kind(path)? {
|
||||
Kind::Dataset => self
|
||||
.file
|
||||
.dataset(path)
|
||||
.and_then(|d| d.attrs_with_errors())
|
||||
.map_err(err)?,
|
||||
Kind::Group => self
|
||||
.file
|
||||
.group(path)
|
||||
.and_then(|g| g.attrs_with_errors())
|
||||
.map_err(err)?,
|
||||
};
|
||||
let mut attrs: Vec<Attr> = map
|
||||
.into_iter()
|
||||
.map(|(name, value)| Attr { name, value })
|
||||
.collect();
|
||||
attrs.sort_by(|a, b| a.name.cmp(&b.name));
|
||||
Ok((attrs, errors.into_iter().map(err).collect()))
|
||||
}
|
||||
|
||||
/// Read the dataset at `path`, whole or a hyperslab of it.
|
||||
pub fn read(&self, path: &str, slab: Option<&Hyperslab>) -> Result<Array> {
|
||||
let ds = self.file.dataset(path).map_err(err)?;
|
||||
let dt = ds.raw_datatype().map_err(err)?;
|
||||
let shape = ds.shape().map_err(err)?;
|
||||
let (selection, mut out_shape) = match slab {
|
||||
None => (Selection::All, shape.clone()),
|
||||
Some(h) => hyperslab_selection(h, &shape)?,
|
||||
};
|
||||
let raw = ds.read_selection(&selection).map_err(err)?;
|
||||
let data = self.decode(&raw, &dt)?;
|
||||
out_shape.extend(element_shape(&dt));
|
||||
let expected = out_shape
|
||||
.iter()
|
||||
.try_fold(1u64, |acc, &d| acc.checked_mul(d))
|
||||
.ok_or("selection size overflows")?;
|
||||
if data.len() as u64 != expected {
|
||||
return Err(format!(
|
||||
"read {} values for shape {out_shape:?} ({expected} expected)",
|
||||
data.len()
|
||||
));
|
||||
}
|
||||
Ok(Array {
|
||||
shape: out_shape,
|
||||
data,
|
||||
})
|
||||
}
|
||||
|
||||
fn decode(&self, raw: &[u8], dt: &Datatype) -> Result<Data> {
|
||||
let base = array_base(dt);
|
||||
let is_array = !std::ptr::eq(base, dt);
|
||||
Ok(match base {
|
||||
Datatype::FloatingPoint { size, .. } if *size <= 4 => {
|
||||
Data::F32(data_read::read_as_f32(raw, dt).map_err(err)?)
|
||||
}
|
||||
Datatype::FloatingPoint { .. } => {
|
||||
Data::F64(data_read::read_as_f64(raw, dt).map_err(err)?)
|
||||
}
|
||||
Datatype::FixedPoint { size, signed, .. } => {
|
||||
let signed_ints = || data_read::read_as_i64(raw, dt).map_err(err);
|
||||
let unsigned_ints = || data_read::read_as_u64(raw, dt).map_err(err);
|
||||
match (size, signed) {
|
||||
(1, true) => Data::I8(narrow(signed_ints()?)?),
|
||||
(2, true) => Data::I16(narrow(signed_ints()?)?),
|
||||
(4, true) => Data::I32(narrow(signed_ints()?)?),
|
||||
(_, true) => Data::I64(signed_ints()?),
|
||||
(1, false) => Data::U8(narrow(unsigned_ints()?)?),
|
||||
(2, false) => Data::U16(narrow(unsigned_ints()?)?),
|
||||
(4, false) => Data::U32(narrow(unsigned_ints()?)?),
|
||||
(_, false) => Data::U64(unsigned_ints()?),
|
||||
}
|
||||
}
|
||||
Datatype::String { .. } if !is_array => {
|
||||
Data::Strings(data_read::read_as_strings(raw, dt).map_err(err)?)
|
||||
}
|
||||
Datatype::VariableLength {
|
||||
is_string: true, ..
|
||||
} if !is_array => {
|
||||
let size = dt.type_size() as usize;
|
||||
if size == 0 || !raw.len().is_multiple_of(size) {
|
||||
return Err(format!(
|
||||
"{} bytes is not a whole number of {size}-byte string references",
|
||||
raw.len()
|
||||
));
|
||||
}
|
||||
let sb = self.file.superblock();
|
||||
Data::Strings(
|
||||
clawhdf5_format::vl_data::read_vl_strings(
|
||||
self.file.as_bytes(),
|
||||
raw,
|
||||
(raw.len() / size) as u64,
|
||||
sb.offset_size,
|
||||
sb.length_size,
|
||||
)
|
||||
.map_err(err)?,
|
||||
)
|
||||
}
|
||||
Datatype::Enumeration { .. } if !is_array => {
|
||||
Data::Strings(data_read::read_enum_names(raw, dt).map_err(err)?)
|
||||
}
|
||||
_ => {
|
||||
return Err(format!(
|
||||
"reading {} datasets is not supported",
|
||||
describe(dt)
|
||||
));
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
/// Narrow integers read at 64 bits to the dataset's own width. The source is
|
||||
/// that width, so this cannot fail on correct input; it is checked anyway.
|
||||
fn narrow<S: Copy + std::fmt::Display, T: TryFrom<S>>(v: Vec<S>) -> Result<Vec<T>> {
|
||||
v.into_iter()
|
||||
.map(|x| T::try_from(x).map_err(|_| format!("value {x} out of range")))
|
||||
.collect()
|
||||
}
|
||||
|
||||
/// Innermost element type of (possibly nested) array datatypes.
|
||||
fn array_base(dt: &Datatype) -> &Datatype {
|
||||
match dt {
|
||||
Datatype::Array { base_type, .. } => array_base(base_type),
|
||||
_ => dt,
|
||||
}
|
||||
}
|
||||
|
||||
fn element_shape(dt: &Datatype) -> Vec<u64> {
|
||||
match dt {
|
||||
Datatype::Array {
|
||||
base_type,
|
||||
dimensions,
|
||||
} => {
|
||||
let mut dims: Vec<u64> = dimensions.iter().map(|&d| u64::from(d)).collect();
|
||||
dims.extend(element_shape(base_type));
|
||||
dims
|
||||
}
|
||||
_ => Vec::new(),
|
||||
}
|
||||
}
|
||||
|
||||
fn hyperslab_selection(h: &Hyperslab, shape: &[u64]) -> Result<(Selection, Vec<u64>)> {
|
||||
let rank = shape.len();
|
||||
let ones = vec![1u64; rank];
|
||||
let stride = h.stride.clone().unwrap_or_else(|| ones.clone());
|
||||
let block = h.block.clone().unwrap_or(ones);
|
||||
for (what, v) in [
|
||||
("start", &h.start),
|
||||
("count", &h.count),
|
||||
("stride", &stride),
|
||||
("block", &block),
|
||||
] {
|
||||
if v.len() != rank {
|
||||
return Err(format!(
|
||||
"hyperslab {what} has {} dimensions, the dataset has {rank}",
|
||||
v.len()
|
||||
));
|
||||
}
|
||||
}
|
||||
let mut out = Vec::with_capacity(rank);
|
||||
for d in 0..rank {
|
||||
if stride[d] == 0 || block[d] == 0 {
|
||||
return Err(format!("hyperslab stride and block must be >= 1 (dim {d})"));
|
||||
}
|
||||
if h.count[d] > 1 && block[d] > stride[d] {
|
||||
return Err(format!(
|
||||
"hyperslab blocks overlap in dim {d}: block {} > stride {}",
|
||||
block[d], stride[d]
|
||||
));
|
||||
}
|
||||
// Last element selected: start + (count-1)*stride + block - 1.
|
||||
if h.count[d] > 0 {
|
||||
let last = (h.count[d] - 1)
|
||||
.checked_mul(stride[d])
|
||||
.and_then(|x| x.checked_add(h.start[d]))
|
||||
.and_then(|x| x.checked_add(block[d] - 1));
|
||||
match last {
|
||||
Some(l) if l < shape[d] => {}
|
||||
_ => {
|
||||
return Err(format!(
|
||||
"hyperslab exceeds dimension {d} (extent {})",
|
||||
shape[d]
|
||||
));
|
||||
}
|
||||
}
|
||||
}
|
||||
out.push(
|
||||
h.count[d]
|
||||
.checked_mul(block[d])
|
||||
.ok_or("selection size overflows")?,
|
||||
);
|
||||
}
|
||||
Ok((
|
||||
Selection::Hyperslab {
|
||||
start: h.start.clone(),
|
||||
stride,
|
||||
count: h.count.clone(),
|
||||
block,
|
||||
},
|
||||
out,
|
||||
))
|
||||
}
|
||||
|
||||
/// A short, human-readable datatype name.
|
||||
pub fn describe(dt: &Datatype) -> String {
|
||||
fn endian(order: &DatatypeByteOrder) -> &'static str {
|
||||
match order {
|
||||
DatatypeByteOrder::BigEndian => " (big-endian)",
|
||||
DatatypeByteOrder::Vax => " (VAX)",
|
||||
_ => "",
|
||||
}
|
||||
}
|
||||
match dt {
|
||||
Datatype::FixedPoint {
|
||||
size,
|
||||
signed,
|
||||
byte_order,
|
||||
..
|
||||
} => format!(
|
||||
"{}{}{}",
|
||||
if *signed { "i" } else { "u" },
|
||||
size * 8,
|
||||
endian(byte_order)
|
||||
),
|
||||
Datatype::FloatingPoint {
|
||||
size, byte_order, ..
|
||||
} => format!("f{}{}", size * 8, endian(byte_order)),
|
||||
Datatype::Time { size, .. } => format!("time{}", size * 8),
|
||||
Datatype::String { size, .. } => format!("string[{size}]"),
|
||||
Datatype::BitField { size, .. } => format!("bitfield{}", size * 8),
|
||||
Datatype::Opaque { size, .. } => format!("opaque[{size}]"),
|
||||
Datatype::Compound { members, .. } => {
|
||||
let fields: Vec<String> = members
|
||||
.iter()
|
||||
.map(|m| format!("{}: {}", m.name, describe(&m.datatype)))
|
||||
.collect();
|
||||
format!("compound{{{}}}", fields.join(", "))
|
||||
}
|
||||
Datatype::Reference { .. } => "reference".to_string(),
|
||||
Datatype::Enumeration {
|
||||
base_type, members, ..
|
||||
} => {
|
||||
let names: Vec<&str> = members.iter().map(|m| m.name.as_str()).collect();
|
||||
format!("enum<{}>{{{}}}", describe(base_type), names.join(", "))
|
||||
}
|
||||
Datatype::VariableLength {
|
||||
is_string: true, ..
|
||||
} => "vlen string".to_string(),
|
||||
Datatype::VariableLength { base_type, .. } => {
|
||||
format!("vlen<{}>", describe(base_type))
|
||||
}
|
||||
Datatype::Array {
|
||||
base_type,
|
||||
dimensions,
|
||||
} => format!("array{dimensions:?}<{}>", describe(base_type)),
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use clawhdf5::FileBuilder;
|
||||
|
||||
fn sample() -> Reader {
|
||||
let mut b = FileBuilder::new();
|
||||
b.create_dataset("grid")
|
||||
.with_f64_data(&(0..12).map(f64::from).collect::<Vec<_>>())
|
||||
.with_shape(&[3, 4])
|
||||
.with_chunks(&[2, 2])
|
||||
.with_deflate(4);
|
||||
b.create_dataset("bytes").with_u8_data(&[1, 2, 250]);
|
||||
let mut g = b.create_group("sensors");
|
||||
g.create_dataset("temp").with_f32_data(&[1.5, -2.25]);
|
||||
g.set_attr("location", AttrValue::String("lab".into()));
|
||||
b.add_group(g.finish());
|
||||
b.set_attr("version", AttrValue::I64(3));
|
||||
b.set_attr("scale", AttrValue::F64Array(vec![0.5, 2.0]));
|
||||
Reader::open(b.finish().unwrap()).unwrap()
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn lists_groups_then_datasets() {
|
||||
let r = sample();
|
||||
let names: Vec<(String, Kind)> = r
|
||||
.list("/")
|
||||
.unwrap()
|
||||
.into_iter()
|
||||
.map(|c| (c.name, c.kind))
|
||||
.collect();
|
||||
assert_eq!(names[0], ("sensors".to_string(), Kind::Group));
|
||||
let mut ds: Vec<&str> = names[1..].iter().map(|(n, _)| n.as_str()).collect();
|
||||
ds.sort();
|
||||
assert_eq!(ds, ["bytes", "grid"]);
|
||||
assert_eq!(
|
||||
r.list("sensors").unwrap(),
|
||||
vec![Child {
|
||||
name: "temp".into(),
|
||||
kind: Kind::Dataset
|
||||
}]
|
||||
);
|
||||
assert!(r.list("grid").unwrap_err().contains("not a group"));
|
||||
assert!(r.list("missing").is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn info_reports_shape_and_dtype() {
|
||||
let r = sample();
|
||||
let i = r.info("grid").unwrap();
|
||||
assert_eq!(i.shape, vec![3, 4]);
|
||||
assert_eq!(i.dtype, "f64");
|
||||
assert!(i.element_shape.is_empty());
|
||||
assert_eq!(r.info("sensors/temp").unwrap().dtype, "f32");
|
||||
assert_eq!(r.kind("/sensors").unwrap(), Kind::Group);
|
||||
assert_eq!(r.kind("/sensors/temp").unwrap(), Kind::Dataset);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn reads_whole_and_hyperslab() {
|
||||
let r = sample();
|
||||
let all = r.read("grid", None).unwrap();
|
||||
assert_eq!(all.shape, vec![3, 4]);
|
||||
assert_eq!(all.data, Data::F64((0..12).map(f64::from).collect()));
|
||||
|
||||
let slab = Hyperslab {
|
||||
start: vec![1, 0],
|
||||
count: vec![2, 2],
|
||||
stride: Some(vec![1, 2]),
|
||||
block: None,
|
||||
};
|
||||
let part = r.read("grid", Some(&slab)).unwrap();
|
||||
assert_eq!(part.shape, vec![2, 2]);
|
||||
assert_eq!(part.data, Data::F64(vec![4.0, 6.0, 8.0, 10.0]));
|
||||
|
||||
assert_eq!(
|
||||
r.read("bytes", None).unwrap().data,
|
||||
Data::U8(vec![1, 2, 250])
|
||||
);
|
||||
assert_eq!(
|
||||
r.read("sensors/temp", None).unwrap().data,
|
||||
Data::F32(vec![1.5, -2.25])
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn bad_hyperslabs_are_refused() {
|
||||
let r = sample();
|
||||
let mk = |start: Vec<u64>, count: Vec<u64>| Hyperslab {
|
||||
start,
|
||||
count,
|
||||
stride: None,
|
||||
block: None,
|
||||
};
|
||||
assert!(
|
||||
r.read("grid", Some(&mk(vec![0], vec![1])))
|
||||
.unwrap_err()
|
||||
.contains("dimensions")
|
||||
);
|
||||
assert!(
|
||||
r.read("grid", Some(&mk(vec![2, 0], vec![2, 1])))
|
||||
.unwrap_err()
|
||||
.contains("exceeds")
|
||||
);
|
||||
let overlap = Hyperslab {
|
||||
start: vec![0, 0],
|
||||
count: vec![2, 1],
|
||||
stride: Some(vec![1, 1]),
|
||||
block: Some(vec![2, 1]),
|
||||
};
|
||||
assert!(
|
||||
r.read("grid", Some(&overlap))
|
||||
.unwrap_err()
|
||||
.contains("overlap")
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn attrs_are_sorted() {
|
||||
let r = sample();
|
||||
let (attrs, errors) = r.attrs("/").unwrap();
|
||||
assert!(errors.is_empty());
|
||||
let names: Vec<&str> = attrs.iter().map(|a| a.name.as_str()).collect();
|
||||
assert_eq!(names, ["scale", "version"]);
|
||||
let (g, _) = r.attrs("sensors").unwrap();
|
||||
assert!(matches!(&g[0].value, AttrValue::String(s) if s == "lab"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn compound_is_refused_not_reinterpreted() {
|
||||
use clawhdf5::CompoundTypeBuilder;
|
||||
let ct = CompoundTypeBuilder::new()
|
||||
.f64_field("x")
|
||||
.i32_field("n")
|
||||
.build();
|
||||
let mut rec = Vec::new();
|
||||
rec.extend_from_slice(&1.0f64.to_le_bytes());
|
||||
rec.extend_from_slice(&7i32.to_le_bytes());
|
||||
let mut b = FileBuilder::new();
|
||||
b.create_dataset("table").with_compound_data(ct, rec, 1);
|
||||
let r = Reader::open(b.finish().unwrap()).unwrap();
|
||||
let e = r.read("table", None).unwrap_err();
|
||||
assert!(e.contains("compound{x: f64, n: i32}"), "{e}");
|
||||
assert!(e.contains("not supported"), "{e}");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn garbage_is_an_error() {
|
||||
assert!(Reader::open(vec![0u8; 64]).is_err());
|
||||
assert!(Reader::open(Vec::new()).is_err());
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,244 @@
|
||||
//! clawhdf5's HDF5 reader for JavaScript, via `wasm-bindgen`.
|
||||
//!
|
||||
//! ```js
|
||||
//! import init, { open } from "./pkg/clawhdf5_wasm.js";
|
||||
//! await init();
|
||||
//! const file = open(new Uint8Array(await blob.arrayBuffer()));
|
||||
//! file.list("/"); // [{ name, kind: "group" | "dataset" }]
|
||||
//! file.info("/x"); // { shape, maxshape, dtype, elementShape }
|
||||
//! file.attrs("/x"); // [{ name, value, dtype }]
|
||||
//! file.read("/x"); // { shape, dtype, data: Float64Array | ... | string[] }
|
||||
//! file.readHyperslab("/x", [0, 0], [10, 10]); // stride, block optional
|
||||
//! file.free();
|
||||
//! ```
|
||||
//!
|
||||
//! Numeric data comes back in the typed array of the stored width
|
||||
//! (`Int16Array` for `i16`, `BigInt64Array` for `i64`, `Float32Array` for
|
||||
//! `f32` and `f16`, ...); strings and enumeration names as arrays of strings.
|
||||
//! Anything else is a thrown `Error` naming the datatype. Only the reader is
|
||||
//! exposed: nothing here writes files.
|
||||
//!
|
||||
//! The logic lives in [`core`], which is plain Rust and tested natively.
|
||||
|
||||
pub mod core;
|
||||
|
||||
use clawhdf5::AttrValue;
|
||||
use js_sys::{Array, Object, Reflect};
|
||||
use wasm_bindgen::prelude::*;
|
||||
|
||||
use crate::core::{Data, Hyperslab, Reader};
|
||||
|
||||
/// JavaScript numbers are exact up to 2^53.
|
||||
const MAX_SAFE_INTEGER: f64 = 9_007_199_254_740_991.0;
|
||||
|
||||
fn js_err(msg: String) -> JsError {
|
||||
JsError::new(&msg)
|
||||
}
|
||||
|
||||
fn set(obj: &Object, key: &str, value: impl Into<JsValue>) {
|
||||
// Defining a property on a fresh plain object cannot fail.
|
||||
Reflect::set(obj, &JsValue::from_str(key), &value.into()).unwrap_throw();
|
||||
}
|
||||
|
||||
fn shape_to_js(shape: &[u64]) -> Array {
|
||||
shape.iter().map(|&d| JsValue::from_f64(d as f64)).collect()
|
||||
}
|
||||
|
||||
fn indices_from_js(what: &str, v: &[f64]) -> Result<Vec<u64>, JsError> {
|
||||
v.iter()
|
||||
.map(|&x| {
|
||||
if x.is_finite() && x >= 0.0 && x.fract() == 0.0 && x <= MAX_SAFE_INTEGER {
|
||||
Ok(x as u64)
|
||||
} else {
|
||||
Err(js_err(format!(
|
||||
"{what} must hold non-negative integers, got {x}"
|
||||
)))
|
||||
}
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
fn data_to_js(data: Data) -> JsValue {
|
||||
match data {
|
||||
Data::F32(v) => js_sys::Float32Array::from(&v[..]).into(),
|
||||
Data::F64(v) => js_sys::Float64Array::from(&v[..]).into(),
|
||||
Data::I8(v) => js_sys::Int8Array::from(&v[..]).into(),
|
||||
Data::I16(v) => js_sys::Int16Array::from(&v[..]).into(),
|
||||
Data::I32(v) => js_sys::Int32Array::from(&v[..]).into(),
|
||||
Data::I64(v) => js_sys::BigInt64Array::from(&v[..]).into(),
|
||||
Data::U8(v) => js_sys::Uint8Array::from(&v[..]).into(),
|
||||
Data::U16(v) => js_sys::Uint16Array::from(&v[..]).into(),
|
||||
Data::U32(v) => js_sys::Uint32Array::from(&v[..]).into(),
|
||||
Data::U64(v) => js_sys::BigUint64Array::from(&v[..]).into(),
|
||||
Data::Strings(v) => v
|
||||
.into_iter()
|
||||
.map(|s| JsValue::from_str(&s))
|
||||
.collect::<Array>()
|
||||
.into(),
|
||||
}
|
||||
}
|
||||
|
||||
/// A scalar integer as a `number` when exact, else a `bigint`.
|
||||
fn int_to_js(x: i128) -> JsValue {
|
||||
if (x as f64).abs() <= MAX_SAFE_INTEGER {
|
||||
JsValue::from_f64(x as f64)
|
||||
} else if let Ok(v) = i64::try_from(x) {
|
||||
JsValue::from(v)
|
||||
} else {
|
||||
JsValue::from(x as u64)
|
||||
}
|
||||
}
|
||||
|
||||
/// An attribute value, and the datatype of one that has no JavaScript form.
|
||||
fn attr_to_js(value: AttrValue) -> (JsValue, Option<String>) {
|
||||
match value {
|
||||
AttrValue::F64(x) => (JsValue::from_f64(x), None),
|
||||
AttrValue::F64Array(v) => (js_sys::Float64Array::from(&v[..]).into(), None),
|
||||
AttrValue::I64(x) => (int_to_js(i128::from(x)), None),
|
||||
AttrValue::I64Array(v) => (js_sys::BigInt64Array::from(&v[..]).into(), None),
|
||||
AttrValue::U64(x) => (int_to_js(i128::from(x)), None),
|
||||
AttrValue::U64Array(v) => (js_sys::BigUint64Array::from(&v[..]).into(), None),
|
||||
AttrValue::String(s) => (JsValue::from_str(&s), None),
|
||||
AttrValue::StringArray(v) => (
|
||||
v.iter()
|
||||
.map(|s| JsValue::from_str(s))
|
||||
.collect::<Array>()
|
||||
.into(),
|
||||
None,
|
||||
),
|
||||
AttrValue::Raw { datatype, .. } => (JsValue::NULL, Some(core::describe(&datatype))),
|
||||
}
|
||||
}
|
||||
|
||||
/// An open HDF5 (or NetCDF-4) file.
|
||||
#[wasm_bindgen]
|
||||
pub struct H5File {
|
||||
inner: Reader,
|
||||
}
|
||||
|
||||
/// Open a file from its bytes. Throws if they are not an HDF5 file.
|
||||
#[wasm_bindgen]
|
||||
pub fn open(bytes: Vec<u8>) -> Result<H5File, JsError> {
|
||||
H5File::new(bytes)
|
||||
}
|
||||
|
||||
/// The clawhdf5 version this module was built from.
|
||||
#[wasm_bindgen]
|
||||
pub fn version() -> String {
|
||||
env!("CARGO_PKG_VERSION").to_string()
|
||||
}
|
||||
|
||||
#[wasm_bindgen]
|
||||
impl H5File {
|
||||
/// Same as [`open`].
|
||||
#[wasm_bindgen(constructor)]
|
||||
pub fn new(bytes: Vec<u8>) -> Result<H5File, JsError> {
|
||||
Ok(H5File {
|
||||
inner: Reader::open(bytes).map_err(js_err)?,
|
||||
})
|
||||
}
|
||||
|
||||
/// `"group"` or `"dataset"`.
|
||||
pub fn kind(&self, path: &str) -> Result<String, JsError> {
|
||||
Ok(self.inner.kind(path).map_err(js_err)?.as_str().to_string())
|
||||
}
|
||||
|
||||
/// The group's members: `[{ name, kind }]`, groups first.
|
||||
pub fn list(&self, path: &str) -> Result<Array, JsError> {
|
||||
Ok(self
|
||||
.inner
|
||||
.list(path)
|
||||
.map_err(js_err)?
|
||||
.into_iter()
|
||||
.map(|c| {
|
||||
let o = Object::new();
|
||||
set(&o, "name", c.name);
|
||||
set(&o, "kind", c.kind.as_str());
|
||||
JsValue::from(o)
|
||||
})
|
||||
.collect())
|
||||
}
|
||||
|
||||
/// `{ shape, maxshape, dtype, elementShape }`. `maxshape` is `null`
|
||||
/// when not recorded, with `null` for each unlimited dimension.
|
||||
pub fn info(&self, path: &str) -> Result<Object, JsError> {
|
||||
let i = self.inner.info(path).map_err(js_err)?;
|
||||
let o = Object::new();
|
||||
set(&o, "shape", shape_to_js(&i.shape));
|
||||
let max: JsValue = match i.maxshape {
|
||||
None => JsValue::NULL,
|
||||
Some(dims) => dims
|
||||
.into_iter()
|
||||
.map(|d| d.map_or(JsValue::NULL, |d| JsValue::from_f64(d as f64)))
|
||||
.collect::<Array>()
|
||||
.into(),
|
||||
};
|
||||
set(&o, "maxshape", max);
|
||||
set(&o, "dtype", i.dtype);
|
||||
set(&o, "elementShape", shape_to_js(&i.element_shape));
|
||||
Ok(o)
|
||||
}
|
||||
|
||||
/// `[{ name, value, dtype }]`, sorted by name. Scalars are `number`
|
||||
/// (`bigint` beyond 2^53) or `string`; arrays are typed arrays or
|
||||
/// `string[]`. An attribute with no JavaScript form has `value: null`
|
||||
/// and its `dtype`; one that could not be read at all is reported by
|
||||
/// [`attrErrors`](Self::attr_errors).
|
||||
pub fn attrs(&self, path: &str) -> Result<Array, JsError> {
|
||||
let (attrs, _) = self.inner.attrs(path).map_err(js_err)?;
|
||||
Ok(attrs
|
||||
.into_iter()
|
||||
.map(|a| {
|
||||
let o = Object::new();
|
||||
set(&o, "name", a.name);
|
||||
let (value, dtype) = attr_to_js(a.value);
|
||||
set(&o, "value", value);
|
||||
set(&o, "dtype", dtype.map_or(JsValue::NULL, JsValue::from));
|
||||
JsValue::from(o)
|
||||
})
|
||||
.collect())
|
||||
}
|
||||
|
||||
/// Messages for attributes that could not be read.
|
||||
#[wasm_bindgen(js_name = attrErrors)]
|
||||
pub fn attr_errors(&self, path: &str) -> Result<Array, JsError> {
|
||||
let (_, errors) = self.inner.attrs(path).map_err(js_err)?;
|
||||
Ok(errors.into_iter().map(JsValue::from).collect())
|
||||
}
|
||||
|
||||
/// The whole dataset: `{ shape, dtype, data }`, `data` in row-major
|
||||
/// order.
|
||||
pub fn read(&self, path: &str) -> Result<Object, JsError> {
|
||||
self.read_impl(path, None)
|
||||
}
|
||||
|
||||
/// A regular hyperslab (`H5Sselect_hyperslab`): `stride` and `block`
|
||||
/// default to 1. The result's shape is `count * block` per dimension.
|
||||
#[wasm_bindgen(js_name = readHyperslab)]
|
||||
pub fn read_hyperslab(
|
||||
&self,
|
||||
path: &str,
|
||||
start: Vec<f64>,
|
||||
count: Vec<f64>,
|
||||
stride: Option<Vec<f64>>,
|
||||
block: Option<Vec<f64>>,
|
||||
) -> Result<Object, JsError> {
|
||||
let slab = Hyperslab {
|
||||
start: indices_from_js("start", &start)?,
|
||||
count: indices_from_js("count", &count)?,
|
||||
stride: stride.map(|s| indices_from_js("stride", &s)).transpose()?,
|
||||
block: block.map(|b| indices_from_js("block", &b)).transpose()?,
|
||||
};
|
||||
self.read_impl(path, Some(&slab))
|
||||
}
|
||||
|
||||
fn read_impl(&self, path: &str, slab: Option<&Hyperslab>) -> Result<Object, JsError> {
|
||||
let dtype = self.inner.info(path).map_err(js_err)?.dtype;
|
||||
let a = self.inner.read(path, slab).map_err(js_err)?;
|
||||
let o = Object::new();
|
||||
set(&o, "shape", shape_to_js(&a.shape));
|
||||
set(&o, "dtype", dtype);
|
||||
set(&o, "data", data_to_js(a.data));
|
||||
Ok(o)
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user