Merge branch 'feat/p1-wasm' into feat/p1-proof
# Conflicts: # Cargo.toml # scripts/ci-test.sh
This commit is contained in:
osobh
@@ -0,0 +1,30 @@
|
||||
[package]
|
||||
name = "clawhdf5-wasm"
|
||||
version.workspace = true
|
||||
edition.workspace = true
|
||||
rust-version.workspace = true
|
||||
description = "Read HDF5 and NetCDF-4 files in the browser: clawhdf5's reader compiled to WebAssembly"
|
||||
license.workspace = true
|
||||
repository.workspace = true
|
||||
keywords = ["hdf5", "netcdf", "wasm", "browser"]
|
||||
categories = ["wasm", "parser-implementations", "science"]
|
||||
# Distributed as the wasm-bindgen package built by examples/wasm-viewer/build.sh.
|
||||
publish = false
|
||||
|
||||
[lib]
|
||||
# cdylib for wasm-bindgen; rlib so the pure-Rust core is tested natively.
|
||||
crate-type = ["cdylib", "rlib"]
|
||||
|
||||
[dependencies]
|
||||
# No mmap (there is no file system) and no `parallel` (no threads on
|
||||
# wasm32-unknown-unknown). lz4 is pure Rust; zstd and szip link C and are
|
||||
# left out, so such datasets fail with a clear filter error.
|
||||
clawhdf5 = { path = "../clawhdf5", version = "2.7.0", default-features = false, features = ["lz4"] }
|
||||
clawhdf5-format = { path = "../clawhdf5-format", version = "2.7.0" }
|
||||
# Must match the wasm-bindgen CLI exactly; build.sh checks.
|
||||
wasm-bindgen = "0.2.129"
|
||||
js-sys = "0.3.106"
|
||||
|
||||
[dev-dependencies]
|
||||
serde_json = "1"
|
||||
tempfile = { workspace = true }
|
||||
@@ -0,0 +1,603 @@
|
||||
//! The reader behind the JavaScript API, in plain Rust so it is tested
|
||||
//! natively. The `wasm_bindgen` layer in `lib.rs` only converts these types
|
||||
//! to JavaScript values.
|
||||
//!
|
||||
//! Every read either returns the dataset's values or an error: a datatype
|
||||
//! with no typed-array mapping (compound, reference, opaque, ...) is refused
|
||||
//! with a message naming it, never returned as reinterpreted bytes.
|
||||
|
||||
use clawhdf5::{AttrValue, File, Selection};
|
||||
use clawhdf5_format::data_read;
|
||||
use clawhdf5_format::datatype::{Datatype, DatatypeByteOrder};
|
||||
|
||||
/// Errors are reported to JavaScript as messages.
|
||||
pub type Result<T> = std::result::Result<T, String>;
|
||||
|
||||
fn err(e: impl std::fmt::Display) -> String {
|
||||
e.to_string()
|
||||
}
|
||||
|
||||
/// What a path names.
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub enum Kind {
|
||||
Group,
|
||||
Dataset,
|
||||
}
|
||||
|
||||
impl Kind {
|
||||
pub fn as_str(self) -> &'static str {
|
||||
match self {
|
||||
Kind::Group => "group",
|
||||
Kind::Dataset => "dataset",
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// One entry of a group listing.
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct Child {
|
||||
pub name: String,
|
||||
pub kind: Kind,
|
||||
}
|
||||
|
||||
/// A dataset's metadata.
|
||||
#[derive(Debug, Clone, PartialEq)]
|
||||
pub struct DatasetInfo {
|
||||
/// Dataspace dimensions (empty for a scalar).
|
||||
pub shape: Vec<u64>,
|
||||
/// Maximum dimensions, `None` per unlimited dimension; `None` overall
|
||||
/// when the dataspace records none.
|
||||
pub maxshape: Option<Vec<Option<u64>>>,
|
||||
/// Human-readable datatype, e.g. `f64`, `i16 (big-endian)`, `string[8]`.
|
||||
pub dtype: String,
|
||||
/// Dimensions of an array datatype's elements, appended to the shape of
|
||||
/// what [`Reader::read`] returns (empty otherwise).
|
||||
pub element_shape: Vec<u64>,
|
||||
}
|
||||
|
||||
/// Decoded values, one variant per JavaScript typed array.
|
||||
#[derive(Debug, Clone, PartialEq)]
|
||||
pub enum Data {
|
||||
F32(Vec<f32>),
|
||||
F64(Vec<f64>),
|
||||
I8(Vec<i8>),
|
||||
I16(Vec<i16>),
|
||||
I32(Vec<i32>),
|
||||
I64(Vec<i64>),
|
||||
U8(Vec<u8>),
|
||||
U16(Vec<u16>),
|
||||
U32(Vec<u32>),
|
||||
U64(Vec<u64>),
|
||||
/// Fixed- and variable-length strings, and enumeration member names.
|
||||
Strings(Vec<String>),
|
||||
}
|
||||
|
||||
impl Data {
|
||||
pub fn len(&self) -> usize {
|
||||
match self {
|
||||
Data::F32(v) => v.len(),
|
||||
Data::F64(v) => v.len(),
|
||||
Data::I8(v) => v.len(),
|
||||
Data::I16(v) => v.len(),
|
||||
Data::I32(v) => v.len(),
|
||||
Data::I64(v) => v.len(),
|
||||
Data::U8(v) => v.len(),
|
||||
Data::U16(v) => v.len(),
|
||||
Data::U32(v) => v.len(),
|
||||
Data::U64(v) => v.len(),
|
||||
Data::Strings(v) => v.len(),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn is_empty(&self) -> bool {
|
||||
self.len() == 0
|
||||
}
|
||||
}
|
||||
|
||||
/// Values in row-major order with their shape.
|
||||
#[derive(Debug, Clone, PartialEq)]
|
||||
pub struct Array {
|
||||
pub shape: Vec<u64>,
|
||||
pub data: Data,
|
||||
}
|
||||
|
||||
/// A regular hyperslab, as in `H5Sselect_hyperslab`. `stride` and `block`
|
||||
/// default to 1 in every dimension.
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct Hyperslab {
|
||||
pub start: Vec<u64>,
|
||||
pub count: Vec<u64>,
|
||||
pub stride: Option<Vec<u64>>,
|
||||
pub block: Option<Vec<u64>>,
|
||||
}
|
||||
|
||||
/// An attribute: its value, or why it has none.
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct Attr {
|
||||
pub name: String,
|
||||
pub value: AttrValue,
|
||||
}
|
||||
|
||||
/// An open file, held in memory.
|
||||
pub struct Reader {
|
||||
file: File,
|
||||
}
|
||||
|
||||
impl Reader {
|
||||
/// Parse a file from its bytes (the browser hands over the whole file).
|
||||
pub fn open(bytes: Vec<u8>) -> Result<Self> {
|
||||
Ok(Self {
|
||||
file: File::from_bytes(bytes).map_err(err)?,
|
||||
})
|
||||
}
|
||||
|
||||
/// Whether `path` names a group or a dataset.
|
||||
pub fn kind(&self, path: &str) -> Result<Kind> {
|
||||
match self.file.dataset(path) {
|
||||
Ok(_) => Ok(Kind::Dataset),
|
||||
Err(clawhdf5::Error::NotADataset(_)) => Ok(Kind::Group),
|
||||
Err(e) => Err(err(e)),
|
||||
}
|
||||
}
|
||||
|
||||
/// The groups, then the datasets, in the group at `path` (`/` is the
|
||||
/// root). Soft links are listed as their targets; external and dangling
|
||||
/// links, and named datatypes, are left out.
|
||||
pub fn list(&self, path: &str) -> Result<Vec<Child>> {
|
||||
if self.kind(path)? != Kind::Group {
|
||||
return Err(format!("not a group: {path}"));
|
||||
}
|
||||
let group = self.file.group(path).map_err(err)?;
|
||||
let mut out: Vec<Child> = group
|
||||
.groups()
|
||||
.map_err(err)?
|
||||
.into_iter()
|
||||
.map(|name| Child {
|
||||
name,
|
||||
kind: Kind::Group,
|
||||
})
|
||||
.collect();
|
||||
out.extend(
|
||||
group
|
||||
.datasets()
|
||||
.map_err(err)?
|
||||
.into_iter()
|
||||
.map(|name| Child {
|
||||
name,
|
||||
kind: Kind::Dataset,
|
||||
}),
|
||||
);
|
||||
Ok(out)
|
||||
}
|
||||
|
||||
/// Shape, max shape and datatype of the dataset at `path`.
|
||||
pub fn info(&self, path: &str) -> Result<DatasetInfo> {
|
||||
let ds = self.file.dataset(path).map_err(err)?;
|
||||
let dt = ds.raw_datatype().map_err(err)?;
|
||||
let maxshape = ds.max_dimensions().map_err(err)?.map(|dims| {
|
||||
dims.into_iter()
|
||||
.map(|d| (d != u64::MAX).then_some(d))
|
||||
.collect()
|
||||
});
|
||||
Ok(DatasetInfo {
|
||||
shape: ds.shape().map_err(err)?,
|
||||
maxshape,
|
||||
dtype: describe(&dt),
|
||||
element_shape: element_shape(&dt),
|
||||
})
|
||||
}
|
||||
|
||||
/// The attributes of the group or dataset at `path`, sorted by name, and
|
||||
/// one message per attribute that could not be read at all. An attribute
|
||||
/// whose type has no plain JavaScript form is returned as
|
||||
/// [`AttrValue::Raw`].
|
||||
pub fn attrs(&self, path: &str) -> Result<(Vec<Attr>, Vec<String>)> {
|
||||
let (map, errors) = match self.kind(path)? {
|
||||
Kind::Dataset => self
|
||||
.file
|
||||
.dataset(path)
|
||||
.and_then(|d| d.attrs_with_errors())
|
||||
.map_err(err)?,
|
||||
Kind::Group => self
|
||||
.file
|
||||
.group(path)
|
||||
.and_then(|g| g.attrs_with_errors())
|
||||
.map_err(err)?,
|
||||
};
|
||||
let mut attrs: Vec<Attr> = map
|
||||
.into_iter()
|
||||
.map(|(name, value)| Attr { name, value })
|
||||
.collect();
|
||||
attrs.sort_by(|a, b| a.name.cmp(&b.name));
|
||||
Ok((attrs, errors.into_iter().map(err).collect()))
|
||||
}
|
||||
|
||||
/// Read the dataset at `path`, whole or a hyperslab of it.
|
||||
pub fn read(&self, path: &str, slab: Option<&Hyperslab>) -> Result<Array> {
|
||||
let ds = self.file.dataset(path).map_err(err)?;
|
||||
let dt = ds.raw_datatype().map_err(err)?;
|
||||
let shape = ds.shape().map_err(err)?;
|
||||
let (selection, mut out_shape) = match slab {
|
||||
None => (Selection::All, shape.clone()),
|
||||
Some(h) => hyperslab_selection(h, &shape)?,
|
||||
};
|
||||
let raw = ds.read_selection(&selection).map_err(err)?;
|
||||
let data = self.decode(&raw, &dt)?;
|
||||
out_shape.extend(element_shape(&dt));
|
||||
let expected = out_shape
|
||||
.iter()
|
||||
.try_fold(1u64, |acc, &d| acc.checked_mul(d))
|
||||
.ok_or("selection size overflows")?;
|
||||
if data.len() as u64 != expected {
|
||||
return Err(format!(
|
||||
"read {} values for shape {out_shape:?} ({expected} expected)",
|
||||
data.len()
|
||||
));
|
||||
}
|
||||
Ok(Array {
|
||||
shape: out_shape,
|
||||
data,
|
||||
})
|
||||
}
|
||||
|
||||
fn decode(&self, raw: &[u8], dt: &Datatype) -> Result<Data> {
|
||||
let base = array_base(dt);
|
||||
let is_array = !std::ptr::eq(base, dt);
|
||||
Ok(match base {
|
||||
Datatype::FloatingPoint { size, .. } if *size <= 4 => {
|
||||
Data::F32(data_read::read_as_f32(raw, dt).map_err(err)?)
|
||||
}
|
||||
Datatype::FloatingPoint { .. } => {
|
||||
Data::F64(data_read::read_as_f64(raw, dt).map_err(err)?)
|
||||
}
|
||||
Datatype::FixedPoint { size, signed, .. } => {
|
||||
let signed_ints = || data_read::read_as_i64(raw, dt).map_err(err);
|
||||
let unsigned_ints = || data_read::read_as_u64(raw, dt).map_err(err);
|
||||
match (size, signed) {
|
||||
(1, true) => Data::I8(narrow(signed_ints()?)?),
|
||||
(2, true) => Data::I16(narrow(signed_ints()?)?),
|
||||
(4, true) => Data::I32(narrow(signed_ints()?)?),
|
||||
(_, true) => Data::I64(signed_ints()?),
|
||||
(1, false) => Data::U8(narrow(unsigned_ints()?)?),
|
||||
(2, false) => Data::U16(narrow(unsigned_ints()?)?),
|
||||
(4, false) => Data::U32(narrow(unsigned_ints()?)?),
|
||||
(_, false) => Data::U64(unsigned_ints()?),
|
||||
}
|
||||
}
|
||||
Datatype::String { .. } if !is_array => {
|
||||
Data::Strings(data_read::read_as_strings(raw, dt).map_err(err)?)
|
||||
}
|
||||
Datatype::VariableLength {
|
||||
is_string: true, ..
|
||||
} if !is_array => {
|
||||
let size = dt.type_size() as usize;
|
||||
if size == 0 || !raw.len().is_multiple_of(size) {
|
||||
return Err(format!(
|
||||
"{} bytes is not a whole number of {size}-byte string references",
|
||||
raw.len()
|
||||
));
|
||||
}
|
||||
let sb = self.file.superblock();
|
||||
Data::Strings(
|
||||
clawhdf5_format::vl_data::read_vl_strings(
|
||||
self.file.as_bytes(),
|
||||
raw,
|
||||
(raw.len() / size) as u64,
|
||||
sb.offset_size,
|
||||
sb.length_size,
|
||||
)
|
||||
.map_err(err)?,
|
||||
)
|
||||
}
|
||||
Datatype::Enumeration { .. } if !is_array => {
|
||||
Data::Strings(data_read::read_enum_names(raw, dt).map_err(err)?)
|
||||
}
|
||||
_ => {
|
||||
return Err(format!(
|
||||
"reading {} datasets is not supported",
|
||||
describe(dt)
|
||||
));
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
/// Narrow integers read at 64 bits to the dataset's own width. The source is
|
||||
/// that width, so this cannot fail on correct input; it is checked anyway.
|
||||
fn narrow<S: Copy + std::fmt::Display, T: TryFrom<S>>(v: Vec<S>) -> Result<Vec<T>> {
|
||||
v.into_iter()
|
||||
.map(|x| T::try_from(x).map_err(|_| format!("value {x} out of range")))
|
||||
.collect()
|
||||
}
|
||||
|
||||
/// Innermost element type of (possibly nested) array datatypes.
|
||||
fn array_base(dt: &Datatype) -> &Datatype {
|
||||
match dt {
|
||||
Datatype::Array { base_type, .. } => array_base(base_type),
|
||||
_ => dt,
|
||||
}
|
||||
}
|
||||
|
||||
fn element_shape(dt: &Datatype) -> Vec<u64> {
|
||||
match dt {
|
||||
Datatype::Array {
|
||||
base_type,
|
||||
dimensions,
|
||||
} => {
|
||||
let mut dims: Vec<u64> = dimensions.iter().map(|&d| u64::from(d)).collect();
|
||||
dims.extend(element_shape(base_type));
|
||||
dims
|
||||
}
|
||||
_ => Vec::new(),
|
||||
}
|
||||
}
|
||||
|
||||
fn hyperslab_selection(h: &Hyperslab, shape: &[u64]) -> Result<(Selection, Vec<u64>)> {
|
||||
let rank = shape.len();
|
||||
let ones = vec![1u64; rank];
|
||||
let stride = h.stride.clone().unwrap_or_else(|| ones.clone());
|
||||
let block = h.block.clone().unwrap_or(ones);
|
||||
for (what, v) in [
|
||||
("start", &h.start),
|
||||
("count", &h.count),
|
||||
("stride", &stride),
|
||||
("block", &block),
|
||||
] {
|
||||
if v.len() != rank {
|
||||
return Err(format!(
|
||||
"hyperslab {what} has {} dimensions, the dataset has {rank}",
|
||||
v.len()
|
||||
));
|
||||
}
|
||||
}
|
||||
let mut out = Vec::with_capacity(rank);
|
||||
for d in 0..rank {
|
||||
if stride[d] == 0 || block[d] == 0 {
|
||||
return Err(format!("hyperslab stride and block must be >= 1 (dim {d})"));
|
||||
}
|
||||
if h.count[d] > 1 && block[d] > stride[d] {
|
||||
return Err(format!(
|
||||
"hyperslab blocks overlap in dim {d}: block {} > stride {}",
|
||||
block[d], stride[d]
|
||||
));
|
||||
}
|
||||
// Last element selected: start + (count-1)*stride + block - 1.
|
||||
if h.count[d] > 0 {
|
||||
let last = (h.count[d] - 1)
|
||||
.checked_mul(stride[d])
|
||||
.and_then(|x| x.checked_add(h.start[d]))
|
||||
.and_then(|x| x.checked_add(block[d] - 1));
|
||||
match last {
|
||||
Some(l) if l < shape[d] => {}
|
||||
_ => {
|
||||
return Err(format!(
|
||||
"hyperslab exceeds dimension {d} (extent {})",
|
||||
shape[d]
|
||||
));
|
||||
}
|
||||
}
|
||||
}
|
||||
out.push(
|
||||
h.count[d]
|
||||
.checked_mul(block[d])
|
||||
.ok_or("selection size overflows")?,
|
||||
);
|
||||
}
|
||||
Ok((
|
||||
Selection::Hyperslab {
|
||||
start: h.start.clone(),
|
||||
stride,
|
||||
count: h.count.clone(),
|
||||
block,
|
||||
},
|
||||
out,
|
||||
))
|
||||
}
|
||||
|
||||
/// A short, human-readable datatype name.
|
||||
pub fn describe(dt: &Datatype) -> String {
|
||||
fn endian(order: &DatatypeByteOrder) -> &'static str {
|
||||
match order {
|
||||
DatatypeByteOrder::BigEndian => " (big-endian)",
|
||||
DatatypeByteOrder::Vax => " (VAX)",
|
||||
_ => "",
|
||||
}
|
||||
}
|
||||
match dt {
|
||||
Datatype::FixedPoint {
|
||||
size,
|
||||
signed,
|
||||
byte_order,
|
||||
..
|
||||
} => format!(
|
||||
"{}{}{}",
|
||||
if *signed { "i" } else { "u" },
|
||||
size * 8,
|
||||
endian(byte_order)
|
||||
),
|
||||
Datatype::FloatingPoint {
|
||||
size, byte_order, ..
|
||||
} => format!("f{}{}", size * 8, endian(byte_order)),
|
||||
Datatype::Time { size, .. } => format!("time{}", size * 8),
|
||||
Datatype::String { size, .. } => format!("string[{size}]"),
|
||||
Datatype::BitField { size, .. } => format!("bitfield{}", size * 8),
|
||||
Datatype::Opaque { size, .. } => format!("opaque[{size}]"),
|
||||
Datatype::Compound { members, .. } => {
|
||||
let fields: Vec<String> = members
|
||||
.iter()
|
||||
.map(|m| format!("{}: {}", m.name, describe(&m.datatype)))
|
||||
.collect();
|
||||
format!("compound{{{}}}", fields.join(", "))
|
||||
}
|
||||
Datatype::Reference { .. } => "reference".to_string(),
|
||||
Datatype::Enumeration {
|
||||
base_type, members, ..
|
||||
} => {
|
||||
let names: Vec<&str> = members.iter().map(|m| m.name.as_str()).collect();
|
||||
format!("enum<{}>{{{}}}", describe(base_type), names.join(", "))
|
||||
}
|
||||
Datatype::VariableLength {
|
||||
is_string: true, ..
|
||||
} => "vlen string".to_string(),
|
||||
Datatype::VariableLength { base_type, .. } => {
|
||||
format!("vlen<{}>", describe(base_type))
|
||||
}
|
||||
Datatype::Array {
|
||||
base_type,
|
||||
dimensions,
|
||||
} => format!("array{dimensions:?}<{}>", describe(base_type)),
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use clawhdf5::FileBuilder;
|
||||
|
||||
fn sample() -> Reader {
|
||||
let mut b = FileBuilder::new();
|
||||
b.create_dataset("grid")
|
||||
.with_f64_data(&(0..12).map(f64::from).collect::<Vec<_>>())
|
||||
.with_shape(&[3, 4])
|
||||
.with_chunks(&[2, 2])
|
||||
.with_deflate(4);
|
||||
b.create_dataset("bytes").with_u8_data(&[1, 2, 250]);
|
||||
let mut g = b.create_group("sensors");
|
||||
g.create_dataset("temp").with_f32_data(&[1.5, -2.25]);
|
||||
g.set_attr("location", AttrValue::String("lab".into()));
|
||||
b.add_group(g.finish());
|
||||
b.set_attr("version", AttrValue::I64(3));
|
||||
b.set_attr("scale", AttrValue::F64Array(vec![0.5, 2.0]));
|
||||
Reader::open(b.finish().unwrap()).unwrap()
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn lists_groups_then_datasets() {
|
||||
let r = sample();
|
||||
let names: Vec<(String, Kind)> = r
|
||||
.list("/")
|
||||
.unwrap()
|
||||
.into_iter()
|
||||
.map(|c| (c.name, c.kind))
|
||||
.collect();
|
||||
assert_eq!(names[0], ("sensors".to_string(), Kind::Group));
|
||||
let mut ds: Vec<&str> = names[1..].iter().map(|(n, _)| n.as_str()).collect();
|
||||
ds.sort();
|
||||
assert_eq!(ds, ["bytes", "grid"]);
|
||||
assert_eq!(
|
||||
r.list("sensors").unwrap(),
|
||||
vec![Child {
|
||||
name: "temp".into(),
|
||||
kind: Kind::Dataset
|
||||
}]
|
||||
);
|
||||
assert!(r.list("grid").unwrap_err().contains("not a group"));
|
||||
assert!(r.list("missing").is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn info_reports_shape_and_dtype() {
|
||||
let r = sample();
|
||||
let i = r.info("grid").unwrap();
|
||||
assert_eq!(i.shape, vec![3, 4]);
|
||||
assert_eq!(i.dtype, "f64");
|
||||
assert!(i.element_shape.is_empty());
|
||||
assert_eq!(r.info("sensors/temp").unwrap().dtype, "f32");
|
||||
assert_eq!(r.kind("/sensors").unwrap(), Kind::Group);
|
||||
assert_eq!(r.kind("/sensors/temp").unwrap(), Kind::Dataset);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn reads_whole_and_hyperslab() {
|
||||
let r = sample();
|
||||
let all = r.read("grid", None).unwrap();
|
||||
assert_eq!(all.shape, vec![3, 4]);
|
||||
assert_eq!(all.data, Data::F64((0..12).map(f64::from).collect()));
|
||||
|
||||
let slab = Hyperslab {
|
||||
start: vec![1, 0],
|
||||
count: vec![2, 2],
|
||||
stride: Some(vec![1, 2]),
|
||||
block: None,
|
||||
};
|
||||
let part = r.read("grid", Some(&slab)).unwrap();
|
||||
assert_eq!(part.shape, vec![2, 2]);
|
||||
assert_eq!(part.data, Data::F64(vec![4.0, 6.0, 8.0, 10.0]));
|
||||
|
||||
assert_eq!(
|
||||
r.read("bytes", None).unwrap().data,
|
||||
Data::U8(vec![1, 2, 250])
|
||||
);
|
||||
assert_eq!(
|
||||
r.read("sensors/temp", None).unwrap().data,
|
||||
Data::F32(vec![1.5, -2.25])
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn bad_hyperslabs_are_refused() {
|
||||
let r = sample();
|
||||
let mk = |start: Vec<u64>, count: Vec<u64>| Hyperslab {
|
||||
start,
|
||||
count,
|
||||
stride: None,
|
||||
block: None,
|
||||
};
|
||||
assert!(
|
||||
r.read("grid", Some(&mk(vec![0], vec![1])))
|
||||
.unwrap_err()
|
||||
.contains("dimensions")
|
||||
);
|
||||
assert!(
|
||||
r.read("grid", Some(&mk(vec![2, 0], vec![2, 1])))
|
||||
.unwrap_err()
|
||||
.contains("exceeds")
|
||||
);
|
||||
let overlap = Hyperslab {
|
||||
start: vec![0, 0],
|
||||
count: vec![2, 1],
|
||||
stride: Some(vec![1, 1]),
|
||||
block: Some(vec![2, 1]),
|
||||
};
|
||||
assert!(
|
||||
r.read("grid", Some(&overlap))
|
||||
.unwrap_err()
|
||||
.contains("overlap")
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn attrs_are_sorted() {
|
||||
let r = sample();
|
||||
let (attrs, errors) = r.attrs("/").unwrap();
|
||||
assert!(errors.is_empty());
|
||||
let names: Vec<&str> = attrs.iter().map(|a| a.name.as_str()).collect();
|
||||
assert_eq!(names, ["scale", "version"]);
|
||||
let (g, _) = r.attrs("sensors").unwrap();
|
||||
assert!(matches!(&g[0].value, AttrValue::String(s) if s == "lab"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn compound_is_refused_not_reinterpreted() {
|
||||
use clawhdf5::CompoundTypeBuilder;
|
||||
let ct = CompoundTypeBuilder::new()
|
||||
.f64_field("x")
|
||||
.i32_field("n")
|
||||
.build();
|
||||
let mut rec = Vec::new();
|
||||
rec.extend_from_slice(&1.0f64.to_le_bytes());
|
||||
rec.extend_from_slice(&7i32.to_le_bytes());
|
||||
let mut b = FileBuilder::new();
|
||||
b.create_dataset("table").with_compound_data(ct, rec, 1);
|
||||
let r = Reader::open(b.finish().unwrap()).unwrap();
|
||||
let e = r.read("table", None).unwrap_err();
|
||||
assert!(e.contains("compound{x: f64, n: i32}"), "{e}");
|
||||
assert!(e.contains("not supported"), "{e}");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn garbage_is_an_error() {
|
||||
assert!(Reader::open(vec![0u8; 64]).is_err());
|
||||
assert!(Reader::open(Vec::new()).is_err());
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,244 @@
|
||||
//! clawhdf5's HDF5 reader for JavaScript, via `wasm-bindgen`.
|
||||
//!
|
||||
//! ```js
|
||||
//! import init, { open } from "./pkg/clawhdf5_wasm.js";
|
||||
//! await init();
|
||||
//! const file = open(new Uint8Array(await blob.arrayBuffer()));
|
||||
//! file.list("/"); // [{ name, kind: "group" | "dataset" }]
|
||||
//! file.info("/x"); // { shape, maxshape, dtype, elementShape }
|
||||
//! file.attrs("/x"); // [{ name, value, dtype }]
|
||||
//! file.read("/x"); // { shape, dtype, data: Float64Array | ... | string[] }
|
||||
//! file.readHyperslab("/x", [0, 0], [10, 10]); // stride, block optional
|
||||
//! file.free();
|
||||
//! ```
|
||||
//!
|
||||
//! Numeric data comes back in the typed array of the stored width
|
||||
//! (`Int16Array` for `i16`, `BigInt64Array` for `i64`, `Float32Array` for
|
||||
//! `f32` and `f16`, ...); strings and enumeration names as arrays of strings.
|
||||
//! Anything else is a thrown `Error` naming the datatype. Only the reader is
|
||||
//! exposed: nothing here writes files.
|
||||
//!
|
||||
//! The logic lives in [`core`], which is plain Rust and tested natively.
|
||||
|
||||
pub mod core;
|
||||
|
||||
use clawhdf5::AttrValue;
|
||||
use js_sys::{Array, Object, Reflect};
|
||||
use wasm_bindgen::prelude::*;
|
||||
|
||||
use crate::core::{Data, Hyperslab, Reader};
|
||||
|
||||
/// JavaScript numbers are exact up to 2^53.
|
||||
const MAX_SAFE_INTEGER: f64 = 9_007_199_254_740_991.0;
|
||||
|
||||
fn js_err(msg: String) -> JsError {
|
||||
JsError::new(&msg)
|
||||
}
|
||||
|
||||
fn set(obj: &Object, key: &str, value: impl Into<JsValue>) {
|
||||
// Defining a property on a fresh plain object cannot fail.
|
||||
Reflect::set(obj, &JsValue::from_str(key), &value.into()).unwrap_throw();
|
||||
}
|
||||
|
||||
fn shape_to_js(shape: &[u64]) -> Array {
|
||||
shape.iter().map(|&d| JsValue::from_f64(d as f64)).collect()
|
||||
}
|
||||
|
||||
fn indices_from_js(what: &str, v: &[f64]) -> Result<Vec<u64>, JsError> {
|
||||
v.iter()
|
||||
.map(|&x| {
|
||||
if x.is_finite() && x >= 0.0 && x.fract() == 0.0 && x <= MAX_SAFE_INTEGER {
|
||||
Ok(x as u64)
|
||||
} else {
|
||||
Err(js_err(format!(
|
||||
"{what} must hold non-negative integers, got {x}"
|
||||
)))
|
||||
}
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
fn data_to_js(data: Data) -> JsValue {
|
||||
match data {
|
||||
Data::F32(v) => js_sys::Float32Array::from(&v[..]).into(),
|
||||
Data::F64(v) => js_sys::Float64Array::from(&v[..]).into(),
|
||||
Data::I8(v) => js_sys::Int8Array::from(&v[..]).into(),
|
||||
Data::I16(v) => js_sys::Int16Array::from(&v[..]).into(),
|
||||
Data::I32(v) => js_sys::Int32Array::from(&v[..]).into(),
|
||||
Data::I64(v) => js_sys::BigInt64Array::from(&v[..]).into(),
|
||||
Data::U8(v) => js_sys::Uint8Array::from(&v[..]).into(),
|
||||
Data::U16(v) => js_sys::Uint16Array::from(&v[..]).into(),
|
||||
Data::U32(v) => js_sys::Uint32Array::from(&v[..]).into(),
|
||||
Data::U64(v) => js_sys::BigUint64Array::from(&v[..]).into(),
|
||||
Data::Strings(v) => v
|
||||
.into_iter()
|
||||
.map(|s| JsValue::from_str(&s))
|
||||
.collect::<Array>()
|
||||
.into(),
|
||||
}
|
||||
}
|
||||
|
||||
/// A scalar integer as a `number` when exact, else a `bigint`.
|
||||
fn int_to_js(x: i128) -> JsValue {
|
||||
if (x as f64).abs() <= MAX_SAFE_INTEGER {
|
||||
JsValue::from_f64(x as f64)
|
||||
} else if let Ok(v) = i64::try_from(x) {
|
||||
JsValue::from(v)
|
||||
} else {
|
||||
JsValue::from(x as u64)
|
||||
}
|
||||
}
|
||||
|
||||
/// An attribute value, and the datatype of one that has no JavaScript form.
|
||||
fn attr_to_js(value: AttrValue) -> (JsValue, Option<String>) {
|
||||
match value {
|
||||
AttrValue::F64(x) => (JsValue::from_f64(x), None),
|
||||
AttrValue::F64Array(v) => (js_sys::Float64Array::from(&v[..]).into(), None),
|
||||
AttrValue::I64(x) => (int_to_js(i128::from(x)), None),
|
||||
AttrValue::I64Array(v) => (js_sys::BigInt64Array::from(&v[..]).into(), None),
|
||||
AttrValue::U64(x) => (int_to_js(i128::from(x)), None),
|
||||
AttrValue::U64Array(v) => (js_sys::BigUint64Array::from(&v[..]).into(), None),
|
||||
AttrValue::String(s) => (JsValue::from_str(&s), None),
|
||||
AttrValue::StringArray(v) => (
|
||||
v.iter()
|
||||
.map(|s| JsValue::from_str(s))
|
||||
.collect::<Array>()
|
||||
.into(),
|
||||
None,
|
||||
),
|
||||
AttrValue::Raw { datatype, .. } => (JsValue::NULL, Some(core::describe(&datatype))),
|
||||
}
|
||||
}
|
||||
|
||||
/// An open HDF5 (or NetCDF-4) file.
|
||||
#[wasm_bindgen]
|
||||
pub struct H5File {
|
||||
inner: Reader,
|
||||
}
|
||||
|
||||
/// Open a file from its bytes. Throws if they are not an HDF5 file.
|
||||
#[wasm_bindgen]
|
||||
pub fn open(bytes: Vec<u8>) -> Result<H5File, JsError> {
|
||||
H5File::new(bytes)
|
||||
}
|
||||
|
||||
/// The clawhdf5 version this module was built from.
|
||||
#[wasm_bindgen]
|
||||
pub fn version() -> String {
|
||||
env!("CARGO_PKG_VERSION").to_string()
|
||||
}
|
||||
|
||||
#[wasm_bindgen]
|
||||
impl H5File {
|
||||
/// Same as [`open`].
|
||||
#[wasm_bindgen(constructor)]
|
||||
pub fn new(bytes: Vec<u8>) -> Result<H5File, JsError> {
|
||||
Ok(H5File {
|
||||
inner: Reader::open(bytes).map_err(js_err)?,
|
||||
})
|
||||
}
|
||||
|
||||
/// `"group"` or `"dataset"`.
|
||||
pub fn kind(&self, path: &str) -> Result<String, JsError> {
|
||||
Ok(self.inner.kind(path).map_err(js_err)?.as_str().to_string())
|
||||
}
|
||||
|
||||
/// The group's members: `[{ name, kind }]`, groups first.
|
||||
pub fn list(&self, path: &str) -> Result<Array, JsError> {
|
||||
Ok(self
|
||||
.inner
|
||||
.list(path)
|
||||
.map_err(js_err)?
|
||||
.into_iter()
|
||||
.map(|c| {
|
||||
let o = Object::new();
|
||||
set(&o, "name", c.name);
|
||||
set(&o, "kind", c.kind.as_str());
|
||||
JsValue::from(o)
|
||||
})
|
||||
.collect())
|
||||
}
|
||||
|
||||
/// `{ shape, maxshape, dtype, elementShape }`. `maxshape` is `null`
|
||||
/// when not recorded, with `null` for each unlimited dimension.
|
||||
pub fn info(&self, path: &str) -> Result<Object, JsError> {
|
||||
let i = self.inner.info(path).map_err(js_err)?;
|
||||
let o = Object::new();
|
||||
set(&o, "shape", shape_to_js(&i.shape));
|
||||
let max: JsValue = match i.maxshape {
|
||||
None => JsValue::NULL,
|
||||
Some(dims) => dims
|
||||
.into_iter()
|
||||
.map(|d| d.map_or(JsValue::NULL, |d| JsValue::from_f64(d as f64)))
|
||||
.collect::<Array>()
|
||||
.into(),
|
||||
};
|
||||
set(&o, "maxshape", max);
|
||||
set(&o, "dtype", i.dtype);
|
||||
set(&o, "elementShape", shape_to_js(&i.element_shape));
|
||||
Ok(o)
|
||||
}
|
||||
|
||||
/// `[{ name, value, dtype }]`, sorted by name. Scalars are `number`
|
||||
/// (`bigint` beyond 2^53) or `string`; arrays are typed arrays or
|
||||
/// `string[]`. An attribute with no JavaScript form has `value: null`
|
||||
/// and its `dtype`; one that could not be read at all is reported by
|
||||
/// [`attrErrors`](Self::attr_errors).
|
||||
pub fn attrs(&self, path: &str) -> Result<Array, JsError> {
|
||||
let (attrs, _) = self.inner.attrs(path).map_err(js_err)?;
|
||||
Ok(attrs
|
||||
.into_iter()
|
||||
.map(|a| {
|
||||
let o = Object::new();
|
||||
set(&o, "name", a.name);
|
||||
let (value, dtype) = attr_to_js(a.value);
|
||||
set(&o, "value", value);
|
||||
set(&o, "dtype", dtype.map_or(JsValue::NULL, JsValue::from));
|
||||
JsValue::from(o)
|
||||
})
|
||||
.collect())
|
||||
}
|
||||
|
||||
/// Messages for attributes that could not be read.
|
||||
#[wasm_bindgen(js_name = attrErrors)]
|
||||
pub fn attr_errors(&self, path: &str) -> Result<Array, JsError> {
|
||||
let (_, errors) = self.inner.attrs(path).map_err(js_err)?;
|
||||
Ok(errors.into_iter().map(JsValue::from).collect())
|
||||
}
|
||||
|
||||
/// The whole dataset: `{ shape, dtype, data }`, `data` in row-major
|
||||
/// order.
|
||||
pub fn read(&self, path: &str) -> Result<Object, JsError> {
|
||||
self.read_impl(path, None)
|
||||
}
|
||||
|
||||
/// A regular hyperslab (`H5Sselect_hyperslab`): `stride` and `block`
|
||||
/// default to 1. The result's shape is `count * block` per dimension.
|
||||
#[wasm_bindgen(js_name = readHyperslab)]
|
||||
pub fn read_hyperslab(
|
||||
&self,
|
||||
path: &str,
|
||||
start: Vec<f64>,
|
||||
count: Vec<f64>,
|
||||
stride: Option<Vec<f64>>,
|
||||
block: Option<Vec<f64>>,
|
||||
) -> Result<Object, JsError> {
|
||||
let slab = Hyperslab {
|
||||
start: indices_from_js("start", &start)?,
|
||||
count: indices_from_js("count", &count)?,
|
||||
stride: stride.map(|s| indices_from_js("stride", &s)).transpose()?,
|
||||
block: block.map(|b| indices_from_js("block", &b)).transpose()?,
|
||||
};
|
||||
self.read_impl(path, Some(&slab))
|
||||
}
|
||||
|
||||
fn read_impl(&self, path: &str, slab: Option<&Hyperslab>) -> Result<Object, JsError> {
|
||||
let dtype = self.inner.info(path).map_err(js_err)?.dtype;
|
||||
let a = self.inner.read(path, slab).map_err(js_err)?;
|
||||
let o = Object::new();
|
||||
set(&o, "shape", shape_to_js(&a.shape));
|
||||
set(&o, "dtype", dtype);
|
||||
set(&o, "data", data_to_js(a.data));
|
||||
Ok(o)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,265 @@
|
||||
//! The wasm reader's core against files written by h5py and netCDF4, with
|
||||
//! the values libhdf5 reads back as the reference. The generator,
|
||||
//! `examples/wasm-viewer/test/make_fixture.py`, is shared with the Node test
|
||||
//! of the built wasm package, so both compare against the same expectations.
|
||||
//!
|
||||
//! Skipped when python3 with h5py/netCDF4 is missing, unless
|
||||
//! `CLAWHDF5_REQUIRE_INTEROP=1`. `CLAWHDF5_PYTHON` names the interpreter.
|
||||
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::process::Command;
|
||||
|
||||
use clawhdf5::AttrValue;
|
||||
use clawhdf5_wasm::core::{Data, Hyperslab, Kind, Reader};
|
||||
use serde_json::Value;
|
||||
|
||||
fn python() -> String {
|
||||
std::env::var("CLAWHDF5_PYTHON").unwrap_or_else(|_| "python3".to_string())
|
||||
}
|
||||
|
||||
fn interop_required() -> bool {
|
||||
std::env::var("CLAWHDF5_REQUIRE_INTEROP").is_ok_and(|v| v == "1")
|
||||
}
|
||||
|
||||
fn python_available() -> bool {
|
||||
Command::new(python())
|
||||
.args(["-c", "import h5py, netCDF4, numpy"])
|
||||
.output()
|
||||
.map(|o| o.status.success())
|
||||
.unwrap_or(false)
|
||||
}
|
||||
|
||||
fn generator() -> PathBuf {
|
||||
Path::new(env!("CARGO_MANIFEST_DIR")).join("../../examples/wasm-viewer/test/make_fixture.py")
|
||||
}
|
||||
|
||||
/// Values as comparable strings: integers exactly, floats by their f64
|
||||
/// value (an f32 widens exactly), strings as themselves.
|
||||
fn data_strings(d: &Data) -> (&'static str, Vec<String>) {
|
||||
fn s<T: ToString>(v: &[T]) -> Vec<String> {
|
||||
v.iter().map(ToString::to_string).collect()
|
||||
}
|
||||
fn f<T: Copy + Into<f64>>(v: &[T]) -> Vec<String> {
|
||||
v.iter().map(|&x| format!("{:?}", x.into())).collect()
|
||||
}
|
||||
match d {
|
||||
Data::F32(v) => ("f32", f(v)),
|
||||
Data::F64(v) => ("f64", f(v)),
|
||||
Data::I8(v) => ("i8", s(v)),
|
||||
Data::I16(v) => ("i16", s(v)),
|
||||
Data::I32(v) => ("i32", s(v)),
|
||||
Data::I64(v) => ("i64", s(v)),
|
||||
Data::U8(v) => ("u8", s(v)),
|
||||
Data::U16(v) => ("u16", s(v)),
|
||||
Data::U32(v) => ("u32", s(v)),
|
||||
Data::U64(v) => ("u64", s(v)),
|
||||
Data::Strings(v) => ("strings", v.clone()),
|
||||
}
|
||||
}
|
||||
|
||||
fn expected_strings(kind: &str, values: &Value) -> Vec<String> {
|
||||
values
|
||||
.as_array()
|
||||
.unwrap()
|
||||
.iter()
|
||||
.map(|v| match (kind, v) {
|
||||
("f32" | "f64", Value::Number(n)) => format!("{:?}", n.as_f64().unwrap()),
|
||||
(_, Value::String(s)) => s.clone(),
|
||||
other => panic!("unexpected expected value {other:?}"),
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
fn shape(v: &Value) -> Vec<u64> {
|
||||
v.as_array()
|
||||
.unwrap()
|
||||
.iter()
|
||||
.map(|x| x.as_u64().unwrap())
|
||||
.collect()
|
||||
}
|
||||
|
||||
fn check_attr(file: &str, path: &str, name: &str, got: &AttrValue, want: &Value) {
|
||||
let ctx = format!("{file}:{path}@{name}");
|
||||
let ints = |v: &Value| -> Vec<String> {
|
||||
v.as_array()
|
||||
.unwrap()
|
||||
.iter()
|
||||
.map(|x| x.as_str().unwrap().to_string())
|
||||
.collect()
|
||||
};
|
||||
let scalar = want["scalar"].as_bool().unwrap_or(false);
|
||||
match got {
|
||||
AttrValue::String(s) => assert_eq!(want["string"].as_str(), Some(s.as_str()), "{ctx}"),
|
||||
AttrValue::StringArray(v) => {
|
||||
let w: Vec<&str> = want["strings"]
|
||||
.as_array()
|
||||
.unwrap_or_else(|| panic!("{ctx}: got {v:?}"))
|
||||
.iter()
|
||||
.map(|x| x.as_str().unwrap())
|
||||
.collect();
|
||||
assert_eq!(v, &w, "{ctx}");
|
||||
}
|
||||
AttrValue::I64(x) => {
|
||||
assert!(scalar, "{ctx}");
|
||||
assert_eq!(vec![x.to_string()], ints(&want["int"]), "{ctx}");
|
||||
}
|
||||
AttrValue::U64(x) => {
|
||||
assert!(scalar, "{ctx}");
|
||||
assert_eq!(vec![x.to_string()], ints(&want["int"]), "{ctx}");
|
||||
}
|
||||
AttrValue::I64Array(v) => assert_eq!(
|
||||
v.iter().map(ToString::to_string).collect::<Vec<_>>(),
|
||||
ints(&want["int"]),
|
||||
"{ctx}"
|
||||
),
|
||||
AttrValue::U64Array(v) => assert_eq!(
|
||||
v.iter().map(ToString::to_string).collect::<Vec<_>>(),
|
||||
ints(&want["int"]),
|
||||
"{ctx}"
|
||||
),
|
||||
AttrValue::F64(x) => {
|
||||
assert!(scalar, "{ctx}");
|
||||
assert_eq!(Some(*x), want["float"][0].as_f64(), "{ctx}");
|
||||
}
|
||||
AttrValue::F64Array(v) => {
|
||||
let w: Vec<f64> = want["float"]
|
||||
.as_array()
|
||||
.unwrap()
|
||||
.iter()
|
||||
.map(|x| x.as_f64().unwrap())
|
||||
.collect();
|
||||
assert_eq!(v, &w, "{ctx}");
|
||||
}
|
||||
AttrValue::Raw { datatype, .. } => {
|
||||
let want = want["raw"]
|
||||
.as_str()
|
||||
.unwrap_or_else(|| panic!("{ctx}: undecoded {datatype:?}"));
|
||||
let d = clawhdf5_wasm::core::describe(datatype);
|
||||
assert!(d.contains(want), "{ctx}: {d}");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn check_file(dir: &Path, file: &str, exp: &Value) {
|
||||
let r = Reader::open(std::fs::read(dir.join(file)).unwrap()).unwrap();
|
||||
|
||||
for (path, want) in exp["lists"].as_object().unwrap() {
|
||||
let list = r
|
||||
.list(path)
|
||||
.unwrap_or_else(|e| panic!("{file}:{path}: {e}"));
|
||||
for (kind, key) in [(Kind::Group, "groups"), (Kind::Dataset, "datasets")] {
|
||||
let mut got: Vec<&str> = list
|
||||
.iter()
|
||||
.filter(|c| c.kind == kind)
|
||||
.map(|c| c.name.as_str())
|
||||
.collect();
|
||||
got.sort();
|
||||
let w: Vec<&str> = want[key]
|
||||
.as_array()
|
||||
.unwrap()
|
||||
.iter()
|
||||
.map(|x| x.as_str().unwrap())
|
||||
.collect();
|
||||
assert_eq!(got, w, "{file}:{path} {key}");
|
||||
}
|
||||
}
|
||||
|
||||
for (path, want) in exp["datasets"].as_object().unwrap() {
|
||||
let kind = want["kind"].as_str().unwrap();
|
||||
let a = match (r.read(path, None), want["unavailable"].as_str()) {
|
||||
(Ok(a), _) => a,
|
||||
// A filter this build may lack (zstd: the wasm build has it
|
||||
// off, a workspace build may unify it on) must fail clearly.
|
||||
(Err(e), Some(why)) => {
|
||||
assert!(e.contains(why), "{file}:{path}: {e}");
|
||||
continue;
|
||||
}
|
||||
(Err(e), None) => panic!("{file}:{path}: {e}"),
|
||||
};
|
||||
assert_eq!(a.shape, shape(&want["shape"]), "{file}:{path} shape");
|
||||
let (got_kind, got) = data_strings(&a.data);
|
||||
assert_eq!(got_kind, kind, "{file}:{path} kind");
|
||||
assert_eq!(
|
||||
got,
|
||||
expected_strings(kind, &want["values"]),
|
||||
"{file}:{path}"
|
||||
);
|
||||
|
||||
if let Some(slab) = want.get("slab") {
|
||||
let h = Hyperslab {
|
||||
start: shape(&slab["start"]),
|
||||
count: shape(&slab["count"]),
|
||||
stride: Some(shape(&slab["stride"])),
|
||||
block: None,
|
||||
};
|
||||
let a = r
|
||||
.read(path, Some(&h))
|
||||
.unwrap_or_else(|e| panic!("{file}:{path} {h:?}: {e}"));
|
||||
assert_eq!(a.shape, shape(&slab["shape"]), "{file}:{path} slab shape");
|
||||
assert_eq!(
|
||||
data_strings(&a.data).1,
|
||||
expected_strings(kind, &slab["values"]),
|
||||
"{file}:{path} slab"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
for (path, what) in exp["errors"].as_object().unwrap() {
|
||||
let e = r.read(path, None).expect_err(path);
|
||||
assert!(e.contains(what.as_str().unwrap()), "{file}:{path}: {e}");
|
||||
}
|
||||
|
||||
let skip: Vec<&str> = exp["skip_attrs"]
|
||||
.as_array()
|
||||
.unwrap()
|
||||
.iter()
|
||||
.map(|x| x.as_str().unwrap())
|
||||
.collect();
|
||||
for (path, want) in exp["attrs"].as_object().unwrap() {
|
||||
let (attrs, errors) = r
|
||||
.attrs(path)
|
||||
.unwrap_or_else(|e| panic!("{file}:{path}: {e}"));
|
||||
assert!(errors.is_empty(), "{file}:{path}: {errors:?}");
|
||||
let want = want.as_object().unwrap();
|
||||
let mut compared = 0;
|
||||
for a in &attrs {
|
||||
if a.name.starts_with('_') || skip.contains(&a.name.as_str()) {
|
||||
continue;
|
||||
}
|
||||
let w = want
|
||||
.get(&a.name)
|
||||
.unwrap_or_else(|| panic!("{file}:{path}: unexpected attribute {}", a.name));
|
||||
check_attr(file, path, &a.name, &a.value, w);
|
||||
compared += 1;
|
||||
}
|
||||
assert_eq!(compared, want.len(), "{file}:{path}: attributes missing");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn reads_what_h5py_and_netcdf4_wrote() {
|
||||
if !python_available() {
|
||||
assert!(
|
||||
!interop_required(),
|
||||
"CLAWHDF5_REQUIRE_INTEROP=1 but python with h5py, netCDF4 and numpy is not available"
|
||||
);
|
||||
eprintln!("SKIP: python with h5py/netCDF4 not available");
|
||||
return;
|
||||
}
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let out = Command::new(python())
|
||||
.arg(generator())
|
||||
.arg(dir.path())
|
||||
.output()
|
||||
.expect("run python");
|
||||
assert!(
|
||||
out.status.success(),
|
||||
"fixture generator failed:\n{}",
|
||||
String::from_utf8_lossy(&out.stderr)
|
||||
);
|
||||
let exp: Value =
|
||||
serde_json::from_slice(&std::fs::read(dir.path().join("expected.json")).unwrap()).unwrap();
|
||||
for file in ["fixture.h5", "fixture.nc"] {
|
||||
check_file(dir.path(), file, &exp[file]);
|
||||
}
|
||||
}
|
||||
@@ -144,6 +144,35 @@ mod tests {
|
||||
assert_eq!(ds.dtype().unwrap(), DType::I32);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn dataset_raw_datatype() {
|
||||
use clawhdf5_format::datatype::Datatype;
|
||||
let bytes = make_simple_file();
|
||||
let file = File::from_bytes(bytes).unwrap();
|
||||
let dt = file.dataset("counts").unwrap().raw_datatype().unwrap();
|
||||
assert!(
|
||||
matches!(
|
||||
dt,
|
||||
Datatype::FixedPoint {
|
||||
size: 4,
|
||||
signed: true,
|
||||
..
|
||||
}
|
||||
),
|
||||
"{dt:?}"
|
||||
);
|
||||
// The full type pairs with read_selection's bytes.
|
||||
let raw = file
|
||||
.dataset("counts")
|
||||
.unwrap()
|
||||
.read_selection(&Selection::All)
|
||||
.unwrap();
|
||||
assert_eq!(
|
||||
clawhdf5_format::data_read::read_as_i64(&raw, &dt).unwrap(),
|
||||
vec![10, 20, 30]
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn root_group_datasets() {
|
||||
let bytes = make_simple_file();
|
||||
|
||||
@@ -424,6 +424,15 @@ impl<'f> Dataset<'f> {
|
||||
Ok(classify_datatype(&dt))
|
||||
}
|
||||
|
||||
/// Returns the dataset's full datatype as stored in the file (byte order,
|
||||
/// string padding, compound layout, ...), with a committed datatype
|
||||
/// resolved. Use it with the `clawhdf5_format::data_read` converters on
|
||||
/// the bytes [`read_selection`](Self::read_selection) returns, for types
|
||||
/// the typed `read_*` methods do not cover.
|
||||
pub fn raw_datatype(&self) -> Result<Datatype, Error> {
|
||||
self.datatype()
|
||||
}
|
||||
|
||||
/// Read all data as `f64` values.
|
||||
pub fn read_f64(&self) -> Result<Vec<f64>, Error> {
|
||||
let dt = self.datatype()?;
|
||||
|
||||
Reference in New Issue
Block a user