osobhandClaude Opus 5.5
d345ffbf80
h5rs (dump, ls, diff, check --data) kept its own lenient VL decoder: a heap object longer than its element was cut to the element's length (libhdf5 and h5py refuse it), a null string printed "" where h5dump prints NULL, the stored element size was trusted, and every heap collection was kept as an owned copy for the whole run. It now resolves each element with VlResolver::element / string_element (new: one element in place, borrowing from the file), and refuses a VL type whose stored element size is not 4 + offset size + 4, as File does. H5::heap_object and its cache are gone. h5diff compares a null VL string equal to an empty one; so does h5rs diff. clawhdf5-wasm already resolved VL strings with read_vl_strings; it now uses VlResolver and checks the stored element size before reading, as File::read_string does. Tests (h5py writes the files, patched for "a\0b", a null element and mis-sized heap objects, with 8- and 4-byte offsets): - h5rs_interop dump_prints_vl_data_like_h5dump: byte-identical to h5dump; - dump_json_vl_values_match_h5py: h5py's values, errors where h5py fails; - check_data_flags_mis_sized_vl_heap_objects; - clawhdf5-wasm tests/vl_strings.rs: wasm, File and h5py agree. All four fail before. check --data over the 150 cve_hdf5 CVE and fuzzer files now passes 15 (h5dump rejects 8 of them), was 16 and 9: the stored-size check flags cve-2024-32608. h5rs-check-ok-files.sh --data: 0 of 422 flagged; h5rs-fuzz.sh: clean on 180 files. Co-Authored-By: Claude Opus 5.5 (1M context) <[email protected]>
908 lines
32 KiB
Rust
908 lines
32 KiB
Rust
//! `h5rs diff`: compare two files (or two objects) like h5diff.
|
|
|
|
use std::cell::OnceCell;
|
|
use std::collections::{BTreeMap, HashMap};
|
|
use std::rc::Rc;
|
|
|
|
use clawhdf5_format::attribute::AttributeMessage;
|
|
use clawhdf5_format::dataspace::{Dataspace, DataspaceType};
|
|
use clawhdf5_format::datatype::Datatype;
|
|
use clawhdf5_format::object_header::ObjectHeader;
|
|
|
|
use crate::cli::{Args, Out};
|
|
use crate::h5::{H5, Kind, Link, LinkKind};
|
|
use crate::value::{self, Decoder, Value};
|
|
|
|
pub const USAGE: &str = "\
|
|
usage: h5rs diff [options] FILE1 FILE2 [OBJ1 [OBJ2]]
|
|
|
|
Compare FILE1 and FILE2 (or OBJ1 in FILE1 with OBJ2 in FILE2, and everything
|
|
below them): the objects present, their kinds, datatypes, shapes, attribute
|
|
sets and values, and soft/external link targets (an OBJ that is itself a
|
|
soft link is compared as a link, as h5diff does, unless --follow-symlinks).
|
|
|
|
-r, --report list every differing element (position, values, difference)
|
|
-q, --quiet print nothing; only the exit status
|
|
--follow-symlinks compare the objects soft links lead to (and walk into
|
|
soft-linked groups) instead of the links' target paths;
|
|
two dangling links are the same
|
|
-d, --delta D numbers differ only when |a - b| > D
|
|
-p, --relative R numbers differ only when |a - b| / |a| > R (an R below
|
|
the f64 epsilon, 2.2e-16, compares exactly, as h5diff)
|
|
-n, --count N list at most N differing elements per object with -r
|
|
-c, --compare list objects that are not comparable (always done;
|
|
accepted for h5diff compatibility)
|
|
--max-bytes N largest dataset read (default 1 GiB); a larger one is an error
|
|
|
|
Two NaNs compare equal. Unlike h5diff, objects that cannot be compared
|
|
(different kinds, datatype classes or shapes) count as a difference.
|
|
|
|
Exit status: 0 no differences, 1 differences found, 2 error (a file or
|
|
object could not be opened or read).";
|
|
|
|
#[derive(Clone, Copy)]
|
|
enum Tol {
|
|
Exact,
|
|
Delta(f64),
|
|
Relative(f64),
|
|
}
|
|
|
|
struct Opts {
|
|
report: bool,
|
|
quiet: bool,
|
|
tol: Tol,
|
|
count: usize,
|
|
/// Compare the objects soft links lead to, not the links' targets.
|
|
follow: bool,
|
|
}
|
|
|
|
/// What a relative path names in one file.
|
|
#[derive(Clone)]
|
|
enum Entry {
|
|
Obj(u64, Kind),
|
|
Soft(String),
|
|
External(String, String),
|
|
UserDefined(u8),
|
|
Broken(String),
|
|
}
|
|
|
|
struct Side<'a> {
|
|
h5: &'a H5,
|
|
label: String,
|
|
base: String,
|
|
/// Relative path -> what it is.
|
|
entries: BTreeMap<String, Entry>,
|
|
/// Address -> relative path, for comparing references.
|
|
rel_of: OnceCell<HashMap<u64, String>>,
|
|
}
|
|
|
|
impl Side<'_> {
|
|
fn full(&self, rel: &str) -> String {
|
|
if rel.is_empty() {
|
|
if self.base.is_empty() {
|
|
"/".into()
|
|
} else {
|
|
self.base.clone()
|
|
}
|
|
} else {
|
|
format!("{}{rel}", self.base)
|
|
}
|
|
}
|
|
|
|
fn rel_paths(&self) -> &HashMap<u64, String> {
|
|
self.rel_of.get_or_init(|| {
|
|
let mut m = HashMap::new();
|
|
let _ = self.h5.walk(|it| {
|
|
if let (Some(a), None) = (it.addr, it.first_path) {
|
|
m.entry(a).or_insert_with(|| it.path.to_string());
|
|
}
|
|
});
|
|
m
|
|
})
|
|
}
|
|
}
|
|
|
|
struct Diff {
|
|
opts: Opts,
|
|
diffs: u64,
|
|
/// Differences already reported under an object heading.
|
|
per_object: u64,
|
|
errors: u64,
|
|
}
|
|
|
|
pub fn run(args: &mut Args, out: &mut Out) -> std::io::Result<i32> {
|
|
let mut opts = Opts {
|
|
report: false,
|
|
quiet: false,
|
|
tol: Tol::Exact,
|
|
count: usize::MAX,
|
|
follow: false,
|
|
};
|
|
let mut max_bytes = None;
|
|
let mut pos = Vec::new();
|
|
while let Some(a) = args.next() {
|
|
match a.as_str() {
|
|
"-r" | "--report" => opts.report = true,
|
|
"-q" | "--quiet" => opts.quiet = true,
|
|
"--follow-symlinks" => opts.follow = true,
|
|
"-d" | "--delta" => match args.number::<f64>() {
|
|
Some(d) if d >= 0.0 => opts.tol = Tol::Delta(d),
|
|
_ => return args.usage_error(out, "--delta needs a number >= 0", USAGE),
|
|
},
|
|
"-p" | "--relative" => match args.number::<f64>() {
|
|
// As h5diff: a relative tolerance below the f64 epsilon
|
|
// cannot be told from rounding, so it compares exactly.
|
|
Some(r) if r >= 0.0 => {
|
|
opts.tol = if r < f64::EPSILON {
|
|
Tol::Exact
|
|
} else {
|
|
Tol::Relative(r)
|
|
}
|
|
}
|
|
_ => return args.usage_error(out, "--relative needs a number >= 0", USAGE),
|
|
},
|
|
// h5diff's -c: h5rs always lists them.
|
|
"-c" | "--compare" => {}
|
|
"-n" | "--count" => match args.number() {
|
|
Some(n) => opts.count = n,
|
|
None => return args.usage_error(out, "--count needs a number", USAGE),
|
|
},
|
|
"--max-bytes" => match args.number() {
|
|
Some(n) => max_bytes = Some(n),
|
|
None => return args.usage_error(out, "--max-bytes needs a number", USAGE),
|
|
},
|
|
"-h" | "--help" => {
|
|
writeln!(out.o, "{USAGE}")?;
|
|
return Ok(0);
|
|
}
|
|
// h5diff's --count=N, --delta=D, --relative=R forms.
|
|
s if s.starts_with("--") && s.contains('=') => {
|
|
let (k, v) = s.split_once('=').unwrap_or((s, ""));
|
|
if !matches!(k, "--count" | "--delta" | "--relative" | "--max-bytes") {
|
|
return args.usage_error(out, &format!("unknown option {a}"), USAGE);
|
|
}
|
|
args.push_front(k.to_string(), v.to_string());
|
|
}
|
|
s if s.starts_with('-') && s.len() > 1 => {
|
|
return args.usage_error(out, &format!("unknown option {a}"), USAGE);
|
|
}
|
|
_ => pos.push(a),
|
|
}
|
|
}
|
|
if pos.len() < 2 || pos.len() > 4 {
|
|
return args.usage_error(out, "expected FILE1 FILE2 [OBJ1 [OBJ2]]", USAGE);
|
|
}
|
|
let mut files = Vec::new();
|
|
for f in &pos[..2] {
|
|
match H5::open(std::path::Path::new(f)) {
|
|
Ok(mut h) => {
|
|
if let Some(m) = max_bytes {
|
|
h.max_bytes = m;
|
|
}
|
|
files.push(h);
|
|
}
|
|
Err(e) => {
|
|
writeln!(out.e, "h5rs diff: {e}")?;
|
|
return Ok(2);
|
|
}
|
|
}
|
|
}
|
|
let obj1 = pos.get(2).cloned().unwrap_or_else(|| "/".into());
|
|
let obj2 = pos.get(3).cloned().unwrap_or_else(|| obj1.clone());
|
|
let mut sides = Vec::new();
|
|
for (h5, (obj, f)) in files.iter().zip([(&obj1, &pos[0]), (&obj2, &pos[1])]) {
|
|
let start = match start(h5, obj, opts.follow) {
|
|
Ok(x) => x,
|
|
Err(_) => {
|
|
writeln!(
|
|
out.e,
|
|
"h5rs diff: object <{obj}> could not be found in <{f}>"
|
|
)?;
|
|
return Ok(2);
|
|
}
|
|
};
|
|
let base = obj.trim_end_matches('/').to_string();
|
|
let base = if base.is_empty() || base.starts_with('/') {
|
|
base
|
|
} else {
|
|
format!("/{base}")
|
|
};
|
|
let collected = match start {
|
|
Start::Obj(addr) => collect(h5, addr, &base, opts.follow),
|
|
Start::Link(e) => Ok(BTreeMap::from([(String::new(), e)])),
|
|
};
|
|
let entries = match collected {
|
|
Ok(e) => e,
|
|
Err(e) => {
|
|
writeln!(out.e, "h5rs diff: {f}: {e}")?;
|
|
return Ok(2);
|
|
}
|
|
};
|
|
sides.push(Side {
|
|
h5,
|
|
label: f.clone(),
|
|
base,
|
|
entries,
|
|
rel_of: OnceCell::new(),
|
|
});
|
|
}
|
|
let (a, b) = (&sides[0], &sides[1]);
|
|
let mut d = Diff {
|
|
opts,
|
|
diffs: 0,
|
|
per_object: 0,
|
|
errors: 0,
|
|
};
|
|
let mut names: Vec<&String> = a.entries.keys().chain(b.entries.keys()).collect();
|
|
names.sort();
|
|
names.dedup();
|
|
for rel in names {
|
|
match (a.entries.get(rel), b.entries.get(rel)) {
|
|
(Some(_), None) => {
|
|
d.diffs += 1;
|
|
d.say(
|
|
out,
|
|
&format!("<{}> exists only in <{}>", a.full(rel), a.label),
|
|
)?;
|
|
}
|
|
(None, Some(_)) => {
|
|
d.diffs += 1;
|
|
d.say(
|
|
out,
|
|
&format!("<{}> exists only in <{}>", b.full(rel), b.label),
|
|
)?;
|
|
}
|
|
(Some(ea), Some(eb)) => d.entry(out, a, b, rel, ea, eb)?,
|
|
(None, None) => {}
|
|
}
|
|
}
|
|
// Per-object counts were printed with each object; add a total when
|
|
// they do not already tell the whole story.
|
|
if !d.opts.quiet && d.diffs > 0 && d.diffs != d.per_object {
|
|
writeln!(out.o, "{} difference(s) found in total", d.diffs)?;
|
|
}
|
|
Ok(if d.errors > 0 {
|
|
2
|
|
} else if d.diffs > 0 {
|
|
1
|
|
} else {
|
|
0
|
|
})
|
|
}
|
|
|
|
/// What OBJ1/OBJ2 names.
|
|
enum Start {
|
|
Obj(u64),
|
|
/// A soft (not followed, or dangling), external or user-defined link.
|
|
Link(Entry),
|
|
}
|
|
|
|
/// Resolve OBJ1/OBJ2. Soft links on the way to it are followed, as in any
|
|
/// HDF5 path; a soft link that *is* the named object is compared as a link
|
|
/// (its target path), as h5diff does, unless `follow`.
|
|
fn start(h5: &H5, obj: &str, follow: bool) -> crate::h5::Result<Start> {
|
|
let p = obj.trim_matches('/');
|
|
if !p.is_empty() {
|
|
let (parent, name) = p.rsplit_once('/').unwrap_or(("", p));
|
|
let link = h5
|
|
.resolve(parent)
|
|
.and_then(|a| h5.header(a))
|
|
.and_then(|h| h5.links(&h))
|
|
.ok()
|
|
.and_then(|ls| ls.into_iter().find(|l| l.name == name));
|
|
if let Some(l) = link {
|
|
return Ok(match l.kind {
|
|
LinkKind::Hard(a) => Start::Obj(a),
|
|
LinkKind::Soft(t) if follow => match soft_target(h5, &format!("/{parent}"), &t) {
|
|
Ok(a) => Start::Obj(a),
|
|
Err(_) => Start::Link(Entry::Soft(t)),
|
|
},
|
|
LinkKind::Soft(t) => Start::Link(Entry::Soft(t)),
|
|
LinkKind::External { file, path } => Start::Link(Entry::External(file, path)),
|
|
LinkKind::UserDefined(t) => Start::Link(Entry::UserDefined(t)),
|
|
});
|
|
}
|
|
}
|
|
h5.resolve(obj).map(Start::Obj)
|
|
}
|
|
|
|
/// The object a soft link in group `parent` (an absolute path) leads to.
|
|
fn soft_target(h5: &H5, parent: &str, target: &str) -> crate::h5::Result<u64> {
|
|
if target.starts_with('/') {
|
|
h5.resolve(target)
|
|
} else {
|
|
h5.resolve(&format!("{}/{target}", parent.trim_end_matches('/')))
|
|
}
|
|
}
|
|
|
|
/// One object as the path walk sees it: what its paths compare as, and
|
|
/// (for a group) its links.
|
|
struct Node {
|
|
entry: Entry,
|
|
links: Vec<Link>,
|
|
}
|
|
|
|
/// Every path below the object at `start`, relative to it (`""` is `start`
|
|
/// itself). Each hard link is its own path, so an object linked under two
|
|
/// names is compared under both, with everything below it: two files that
|
|
/// hold the same values are equal whether one shares an object between
|
|
/// names and the other stores copies. A hard link back to an ancestor (a
|
|
/// cycle) is recorded, not descended into. With `follow`, a soft link is
|
|
/// walked like a hard link to its target (a dangling one stays a link);
|
|
/// `base` is the absolute path of `start`, for relative link targets.
|
|
fn collect(
|
|
h5: &H5,
|
|
start: u64,
|
|
base: &str,
|
|
follow: bool,
|
|
) -> crate::h5::Result<BTreeMap<String, Entry>> {
|
|
let mut cache: HashMap<u64, Rc<Node>> = HashMap::new();
|
|
let mut node = |addr: u64| -> Rc<Node> {
|
|
cache
|
|
.entry(addr)
|
|
.or_insert_with(|| {
|
|
Rc::new(match h5.header(addr) {
|
|
Err(e) => Node {
|
|
entry: Entry::Broken(e.to_string()),
|
|
links: Vec::new(),
|
|
},
|
|
Ok(h) => match Kind::of(&h) {
|
|
Kind::Group => match h5.links(&h) {
|
|
Ok(links) => Node {
|
|
entry: Entry::Obj(addr, Kind::Group),
|
|
links,
|
|
},
|
|
Err(e) => Node {
|
|
entry: Entry::Broken(format!("links: {e}")),
|
|
links: Vec::new(),
|
|
},
|
|
},
|
|
k => Node {
|
|
entry: Entry::Obj(addr, k),
|
|
links: Vec::new(),
|
|
},
|
|
},
|
|
})
|
|
})
|
|
.clone()
|
|
};
|
|
let mut entries = BTreeMap::new();
|
|
// (address, relative path, addresses of the groups above it)
|
|
let mut stack: Vec<(u64, String, Rc<Vec<u64>>)> =
|
|
vec![(start, String::new(), Rc::new(Vec::new()))];
|
|
while let Some((addr, path, above)) = stack.pop() {
|
|
if entries.len() >= crate::h5::MAX_OBJECTS {
|
|
return Err(crate::h5::Error::new(format!(
|
|
"more than {} paths; stopped walking",
|
|
crate::h5::MAX_OBJECTS
|
|
)));
|
|
}
|
|
let n = node(addr);
|
|
entries.insert(path.clone(), n.entry.clone());
|
|
if n.links.is_empty() || above.contains(&addr) {
|
|
continue;
|
|
}
|
|
let mut chain = (*above).clone();
|
|
chain.push(addr);
|
|
let chain = Rc::new(chain);
|
|
for l in &n.links {
|
|
let child = format!("{path}/{}", l.name);
|
|
match &l.kind {
|
|
LinkKind::Hard(a) => stack.push((*a, child, chain.clone())),
|
|
LinkKind::Soft(t) if follow => match soft_target(h5, &format!("{base}{path}"), t) {
|
|
Ok(a) => stack.push((a, child, chain.clone())),
|
|
Err(_) => {
|
|
entries.insert(child, Entry::Soft(t.clone()));
|
|
}
|
|
},
|
|
LinkKind::Soft(t) => {
|
|
entries.insert(child, Entry::Soft(t.clone()));
|
|
}
|
|
LinkKind::External { file, path } => {
|
|
entries.insert(child, Entry::External(file.clone(), path.clone()));
|
|
}
|
|
LinkKind::UserDefined(t) => {
|
|
entries.insert(child, Entry::UserDefined(*t));
|
|
}
|
|
}
|
|
}
|
|
}
|
|
Ok(entries)
|
|
}
|
|
|
|
fn kind_word(k: Kind) -> &'static str {
|
|
match k {
|
|
Kind::Group => "group",
|
|
Kind::Dataset => "dataset",
|
|
Kind::Datatype => "datatype",
|
|
Kind::Unknown => "object",
|
|
}
|
|
}
|
|
|
|
impl Diff {
|
|
fn say(&self, out: &mut Out, msg: &str) -> std::io::Result<()> {
|
|
if self.opts.quiet {
|
|
Ok(())
|
|
} else {
|
|
writeln!(out.o, "{msg}")
|
|
}
|
|
}
|
|
|
|
fn error(&mut self, out: &mut Out, msg: &str) -> std::io::Result<()> {
|
|
self.errors += 1;
|
|
writeln!(out.e, "h5rs diff: {msg}")
|
|
}
|
|
|
|
fn entry(
|
|
&mut self,
|
|
out: &mut Out,
|
|
a: &Side,
|
|
b: &Side,
|
|
rel: &str,
|
|
ea: &Entry,
|
|
eb: &Entry,
|
|
) -> std::io::Result<()> {
|
|
let (pa, pb) = (a.full(rel), b.full(rel));
|
|
match (ea, eb) {
|
|
(Entry::Broken(e), _) => self.error(out, &format!("<{pa}> in <{}>: {e}", a.label)),
|
|
(_, Entry::Broken(e)) => self.error(out, &format!("<{pb}> in <{}>: {e}", b.label)),
|
|
(Entry::Soft(x), Entry::Soft(y)) => {
|
|
// Followed links that are left are dangling on both sides,
|
|
// which h5diff counts as the same.
|
|
if x != y && !self.opts.follow {
|
|
self.diffs += 1;
|
|
self.say(out, &format!("soft link: <{pa}> -> {x} and <{pb}> -> {y}"))?;
|
|
}
|
|
Ok(())
|
|
}
|
|
(Entry::External(f1, p1), Entry::External(f2, p2)) => {
|
|
if (f1, p1) != (f2, p2) {
|
|
self.diffs += 1;
|
|
self.say(
|
|
out,
|
|
&format!("external link: <{pa}> -> {f1}:{p1} and <{pb}> -> {f2}:{p2}"),
|
|
)?;
|
|
}
|
|
Ok(())
|
|
}
|
|
(Entry::UserDefined(x), Entry::UserDefined(y)) => {
|
|
if x != y {
|
|
self.diffs += 1;
|
|
self.say(out, &format!("user-defined link: <{pa}> and <{pb}> differ"))?;
|
|
}
|
|
Ok(())
|
|
}
|
|
(Entry::Obj(aa, ka), Entry::Obj(ab, kb)) if ka == kb => {
|
|
let (ha, hb) = match (a.h5.header(*aa), b.h5.header(*ab)) {
|
|
(Ok(x), Ok(y)) => (x, y),
|
|
(Err(e), _) | (_, Err(e)) => return self.error(out, &format!("<{pa}>: {e}")),
|
|
};
|
|
let before = self.diffs;
|
|
let mut rows = Vec::new();
|
|
match ka {
|
|
Kind::Dataset => self.dataset(out, a, b, &pa, &pb, &ha, &hb, &mut rows)?,
|
|
Kind::Datatype => match (a.h5.datatype(&ha), b.h5.datatype(&hb)) {
|
|
(Ok(x), Ok(y)) => {
|
|
if x != y {
|
|
self.diffs += 1;
|
|
rows.push("datatypes differ".to_string());
|
|
}
|
|
}
|
|
(Err(e), _) | (_, Err(e)) => self.error(out, &format!("<{pa}>: {e}"))?,
|
|
},
|
|
_ => {}
|
|
}
|
|
self.attributes(out, a, b, &pa, &ha, &hb, &mut rows)?;
|
|
let n = self.diffs - before;
|
|
if n > 0 && !self.opts.quiet {
|
|
writeln!(out.o, "{}: <{pa}> and <{pb}>", kind_word(*ka))?;
|
|
for r in &rows {
|
|
writeln!(out.o, "{r}")?;
|
|
}
|
|
writeln!(out.o, "{n} difference(s) found")?;
|
|
self.per_object += n;
|
|
}
|
|
Ok(())
|
|
}
|
|
_ => {
|
|
self.diffs += 1;
|
|
self.say(
|
|
out,
|
|
&format!(
|
|
"Not comparable: <{pa}> is a {} and <{pb}> is a {}",
|
|
entry_word(ea),
|
|
entry_word(eb)
|
|
),
|
|
)
|
|
}
|
|
}
|
|
}
|
|
|
|
#[allow(clippy::too_many_arguments)]
|
|
fn dataset(
|
|
&mut self,
|
|
out: &mut Out,
|
|
a: &Side,
|
|
b: &Side,
|
|
pa: &str,
|
|
pb: &str,
|
|
ha: &ObjectHeader,
|
|
hb: &ObjectHeader,
|
|
rows: &mut Vec<String>,
|
|
) -> std::io::Result<()> {
|
|
let got = (|| -> crate::h5::Result<_> {
|
|
Ok((
|
|
a.h5.datatype(ha)?,
|
|
a.h5.resolved_dataspace(pa, ha)?,
|
|
b.h5.datatype(hb)?,
|
|
b.h5.resolved_dataspace(pb, hb)?,
|
|
))
|
|
})();
|
|
let (dta, dsa, dtb, dsb) = match got {
|
|
Ok(x) => x,
|
|
Err(e) => return self.error(out, &format!("<{pa}>: {e}")),
|
|
};
|
|
if let Some(why) = not_comparable(&dta, &dsa, &dtb, &dsb) {
|
|
self.diffs += 1;
|
|
rows.push(format!("Not comparable: {why}"));
|
|
return Ok(());
|
|
}
|
|
let raw =
|
|
a.h5.read_dataset(pa, &dta, &dsa)
|
|
.and_then(|x| Ok((x, b.h5.read_dataset(pb, &dtb, &dsb)?)));
|
|
let (ra, rb) = match raw {
|
|
Ok(x) => x,
|
|
Err(e) => return self.error(out, &format!("<{pa}>: {e}")),
|
|
};
|
|
self.values(out, a, b, pa, (&dta, &ra), (&dtb, &rb), &dsa, rows)
|
|
}
|
|
|
|
#[allow(clippy::too_many_arguments)]
|
|
fn values(
|
|
&mut self,
|
|
out: &mut Out,
|
|
a: &Side,
|
|
b: &Side,
|
|
what: &str,
|
|
(dta, ra): (&Datatype, &[u8]),
|
|
(dtb, rb): (&Datatype, &[u8]),
|
|
ds: &Dataspace,
|
|
rows: &mut Vec<String>,
|
|
) -> std::io::Result<()> {
|
|
let n = crate::h5::num_elements(ds).unwrap_or(0) as usize;
|
|
let (da, db) = (Decoder::new(a.h5), Decoder::new(b.h5));
|
|
let dims: Vec<u64> = match ds.space_type {
|
|
DataspaceType::Simple => ds.dimensions.clone(),
|
|
_ => vec![1],
|
|
};
|
|
let mut found = 0u64;
|
|
let mut header = false;
|
|
for i in 0..n {
|
|
let va = da.element(dta, ra, i);
|
|
let vb = db.element(dtb, rb, i);
|
|
if let (Value::Error(e), _) | (_, Value::Error(e)) = (&va, &vb) {
|
|
return self.error(out, &format!("<{what}> element {i}: {e}"));
|
|
}
|
|
if self.equal(a, b, &va, &vb) {
|
|
continue;
|
|
}
|
|
found += 1;
|
|
if self.opts.report && (found as usize) <= self.opts.count {
|
|
if !header {
|
|
header = true;
|
|
rows.push(format!(
|
|
"{:<24}{:<24}{:<24}{}",
|
|
"position", "value 1", "value 2", "difference"
|
|
));
|
|
rows.push("-".repeat(80));
|
|
}
|
|
let pos = format!("[ {} ]", index(i as u64, &dims));
|
|
let ta = value::text(&va, &|_| None);
|
|
let tb = value::text(&vb, &|_| None);
|
|
let dif = match (int_of(&va), int_of(&vb), number(&va), number(&vb)) {
|
|
(Some(x), Some(y), ..) => x.abs_diff(y).to_string(),
|
|
(_, _, Some(x), Some(y)) => value::fmt_float((x - y).abs(), 64),
|
|
_ => String::new(),
|
|
};
|
|
rows.push(format!("{pos:<24}{ta:<24}{tb:<24}{dif}"));
|
|
}
|
|
}
|
|
self.diffs += found;
|
|
Ok(())
|
|
}
|
|
|
|
#[allow(clippy::too_many_arguments)]
|
|
fn attributes(
|
|
&mut self,
|
|
out: &mut Out,
|
|
a: &Side,
|
|
b: &Side,
|
|
path: &str,
|
|
ha: &ObjectHeader,
|
|
hb: &ObjectHeader,
|
|
rows: &mut Vec<String>,
|
|
) -> std::io::Result<()> {
|
|
let (la, lb) = match (a.h5.attributes(ha), b.h5.attributes(hb)) {
|
|
(Ok(x), Ok(y)) => (x, y),
|
|
(Err(e), _) | (_, Err(e)) => return self.error(out, &format!("<{path}>: {e}")),
|
|
};
|
|
for e in la.1.iter().chain(lb.1.iter()) {
|
|
self.error(out, &format!("<{path}>: attribute: {e}"))?;
|
|
}
|
|
let ma: BTreeMap<&str, &AttributeMessage> =
|
|
la.0.iter().map(|x| (x.name.as_str(), x)).collect();
|
|
let mb: BTreeMap<&str, &AttributeMessage> =
|
|
lb.0.iter().map(|x| (x.name.as_str(), x)).collect();
|
|
let mut names: Vec<&str> = ma.keys().chain(mb.keys()).copied().collect();
|
|
names.sort_unstable();
|
|
names.dedup();
|
|
for n in names {
|
|
match (ma.get(n), mb.get(n)) {
|
|
(Some(x), Some(y)) => {
|
|
if let Some(why) =
|
|
not_comparable(&x.datatype, &x.dataspace, &y.datatype, &y.dataspace)
|
|
{
|
|
self.diffs += 1;
|
|
rows.push(format!("attribute \"{n}\": not comparable: {why}"));
|
|
continue;
|
|
}
|
|
for (m, h5) in [(x, a.h5), (y, b.h5)] {
|
|
let need =
|
|
crate::h5::byte_len(&m.dataspace, &m.datatype).unwrap_or(u64::MAX);
|
|
if need > h5.max_bytes || (m.raw_data.len() as u64) < need {
|
|
return self.error(
|
|
out,
|
|
&format!(
|
|
"<{path}> attribute \"{n}\": value is truncated or too large"
|
|
),
|
|
);
|
|
}
|
|
}
|
|
let before = rows.len();
|
|
let what = format!("{path}\" attribute \"{n}");
|
|
self.values(
|
|
out,
|
|
a,
|
|
b,
|
|
&what,
|
|
(&x.datatype, &x.raw_data),
|
|
(&y.datatype, &y.raw_data),
|
|
&x.dataspace,
|
|
rows,
|
|
)?;
|
|
if rows.len() > before {
|
|
rows.insert(before, format!("attribute \"{n}\":"));
|
|
}
|
|
}
|
|
(Some(_), None) => {
|
|
self.diffs += 1;
|
|
rows.push(format!("attribute \"{n}\" exists only in <{}>", a.label));
|
|
}
|
|
(None, Some(_)) => {
|
|
self.diffs += 1;
|
|
rows.push(format!("attribute \"{n}\" exists only in <{}>", b.label));
|
|
}
|
|
(None, None) => {}
|
|
}
|
|
}
|
|
Ok(())
|
|
}
|
|
|
|
fn equal(&self, a: &Side, b: &Side, x: &Value, y: &Value) -> bool {
|
|
// Integers in integer arithmetic: through f64 they lose precision
|
|
// above 2^53, and values that differ would compare equal.
|
|
if let (Some(p), Some(q)) = (int_of(x), int_of(y)) {
|
|
return int_close(p, q, self.opts.tol);
|
|
}
|
|
if let (Some(p), Some(q)) = (number(x), number(y)) {
|
|
return close(p, q, self.opts.tol);
|
|
}
|
|
match (x, y) {
|
|
(Value::Str(p), Value::Str(q)) => p == q,
|
|
// h5diff compares a null VL string equal to an empty one.
|
|
(Value::NullStr, Value::NullStr) => true,
|
|
(Value::NullStr, Value::Str(s)) | (Value::Str(s), Value::NullStr) => s.is_empty(),
|
|
(Value::Bytes(p), Value::Bytes(q)) | (Value::OtherRef(p), Value::OtherRef(q)) => p == q,
|
|
(Value::Compound(p), Value::Compound(q)) => {
|
|
p.len() == q.len()
|
|
&& p.iter()
|
|
.zip(q)
|
|
.all(|((_, u), (_, v))| self.equal(a, b, u, v))
|
|
}
|
|
(Value::Array(p), Value::Array(q)) | (Value::Seq(p), Value::Seq(q)) => {
|
|
p.len() == q.len() && p.iter().zip(q).all(|(u, v)| self.equal(a, b, u, v))
|
|
}
|
|
(Value::Ref(None), Value::Ref(None)) => true,
|
|
(Value::Ref(Some(p)), Value::Ref(Some(q))) => {
|
|
// Addresses mean nothing across files: compare the paths the
|
|
// references lead to.
|
|
let (pp, qq) = (a.rel_paths().get(p), b.rel_paths().get(q));
|
|
pp.is_some() && pp == qq
|
|
}
|
|
_ => false,
|
|
}
|
|
}
|
|
}
|
|
|
|
fn int_of(v: &Value) -> Option<i128> {
|
|
match v {
|
|
Value::Int(i) | Value::Enum(_, i) => Some(*i),
|
|
_ => None,
|
|
}
|
|
}
|
|
|
|
fn number(v: &Value) -> Option<f64> {
|
|
match v {
|
|
Value::Int(i) | Value::Enum(_, i) => Some(*i as f64),
|
|
Value::Float(f, _) => Some(*f),
|
|
_ => None,
|
|
}
|
|
}
|
|
|
|
fn close(a: f64, b: f64, tol: Tol) -> bool {
|
|
if a.is_nan() || b.is_nan() {
|
|
return a.is_nan() && b.is_nan();
|
|
}
|
|
if a == b {
|
|
return true;
|
|
}
|
|
let d = (a - b).abs();
|
|
match tol {
|
|
Tol::Exact => false,
|
|
Tol::Delta(t) => d <= t,
|
|
Tol::Relative(r) => a != 0.0 && d / a.abs() <= r,
|
|
}
|
|
}
|
|
|
|
/// `close` for integers, exactly: the difference is taken in i128, so no
|
|
/// precision is lost at any 64-bit magnitude.
|
|
fn int_close(a: i128, b: i128, tol: Tol) -> bool {
|
|
if a == b {
|
|
return true;
|
|
}
|
|
let d = a.abs_diff(b);
|
|
match tol {
|
|
Tol::Exact => false,
|
|
// d is whole, so d <= t exactly when d <= floor(t) (saturating).
|
|
Tol::Delta(t) => d <= t.floor() as u128,
|
|
// d >= 1 here, so the quotient is never rounded to 0.
|
|
Tol::Relative(r) => a != 0 && d as f64 / a.unsigned_abs() as f64 <= r,
|
|
}
|
|
}
|
|
|
|
fn entry_word(e: &Entry) -> &'static str {
|
|
match e {
|
|
Entry::Obj(_, k) => kind_word(*k),
|
|
Entry::Soft(_) => "soft link",
|
|
Entry::External(..) => "external link",
|
|
Entry::UserDefined(_) => "user-defined link",
|
|
Entry::Broken(_) => "unreadable object",
|
|
}
|
|
}
|
|
|
|
/// Why two datasets/attributes cannot be compared element by element.
|
|
fn not_comparable(
|
|
dta: &Datatype,
|
|
dsa: &Dataspace,
|
|
dtb: &Datatype,
|
|
dsb: &Dataspace,
|
|
) -> Option<String> {
|
|
let rank = |d: &Dataspace| match d.space_type {
|
|
DataspaceType::Simple => Some(d.dimensions.clone()),
|
|
DataspaceType::Scalar => Some(Vec::new()),
|
|
DataspaceType::Null => None,
|
|
};
|
|
let (sa, sb) = (rank(dsa), rank(dsb));
|
|
if sa != sb {
|
|
let show = |s: &Option<Vec<u64>>| match s {
|
|
None => "null".to_string(),
|
|
Some(d) if d.is_empty() => "scalar".to_string(),
|
|
Some(d) => format!("{d:?}"),
|
|
};
|
|
return Some(format!("shapes differ: {} and {}", show(&sa), show(&sb)));
|
|
}
|
|
if !types_comparable(dta, dtb) {
|
|
return Some(format!(
|
|
"datatypes differ: {} and {}",
|
|
crate::dtype::short(dta),
|
|
crate::dtype::short(dtb)
|
|
));
|
|
}
|
|
None
|
|
}
|
|
|
|
fn types_comparable(a: &Datatype, b: &Datatype) -> bool {
|
|
use crate::dtype::class;
|
|
let numeric = |t: &Datatype| matches!(class(t), "integer" | "float");
|
|
if numeric(a) && numeric(b) {
|
|
return class(a) == class(b);
|
|
}
|
|
if class(a) != class(b) {
|
|
return false;
|
|
}
|
|
match (a, b) {
|
|
(Datatype::Compound { members: ma, .. }, Datatype::Compound { members: mb, .. }) => {
|
|
ma.len() == mb.len()
|
|
&& ma
|
|
.iter()
|
|
.zip(mb)
|
|
.all(|(x, y)| x.name == y.name && types_comparable(&x.datatype, &y.datatype))
|
|
}
|
|
(
|
|
Datatype::Array {
|
|
base_type: x,
|
|
dimensions: dx,
|
|
},
|
|
Datatype::Array {
|
|
base_type: y,
|
|
dimensions: dy,
|
|
},
|
|
) => dx == dy && types_comparable(x, y),
|
|
(
|
|
Datatype::VariableLength { base_type: x, .. },
|
|
Datatype::VariableLength { base_type: y, .. },
|
|
) => class(a) == "string" || types_comparable(x, y),
|
|
(
|
|
Datatype::Enumeration { base_type: x, .. },
|
|
Datatype::Enumeration { base_type: y, .. },
|
|
) => types_comparable(x, y),
|
|
_ => true,
|
|
}
|
|
}
|
|
|
|
fn index(mut i: u64, dims: &[u64]) -> String {
|
|
let mut idx = vec![0u64; dims.len()];
|
|
for (k, &d) in dims.iter().enumerate().rev() {
|
|
let d = d.max(1);
|
|
idx[k] = i % d;
|
|
i /= d;
|
|
}
|
|
idx.iter()
|
|
.map(|x| x.to_string())
|
|
.collect::<Vec<_>>()
|
|
.join(" ")
|
|
}
|
|
|
|
#[cfg(test)]
|
|
mod tests {
|
|
use super::*;
|
|
|
|
#[test]
|
|
fn tolerances() {
|
|
assert!(close(1.0, 1.0, Tol::Exact));
|
|
assert!(!close(1.0, 1.001, Tol::Exact));
|
|
assert!(close(1.0, 1.001, Tol::Delta(0.01)));
|
|
assert!(!close(1.0, 1.1, Tol::Delta(0.01)));
|
|
assert!(close(100.0, 101.0, Tol::Relative(0.02)));
|
|
assert!(!close(0.0, 1e-9, Tol::Relative(0.5)));
|
|
assert!(close(f64::NAN, f64::NAN, Tol::Exact));
|
|
assert!(!close(f64::NAN, 1.0, Tol::Delta(1e9)));
|
|
}
|
|
|
|
#[test]
|
|
fn integer_tolerances_are_exact_above_2_pow_53() {
|
|
let big = 1i128 << 60;
|
|
assert!(!int_close(big, big + 1, Tol::Delta(0.0)));
|
|
assert!(!int_close(big, big + 1, Tol::Delta(0.5)));
|
|
assert!(int_close(big, big + 1, Tol::Delta(1.0)));
|
|
assert!(!int_close(big, big + 200, Tol::Delta(199.9)));
|
|
assert!(!int_close(big, big + 1, Tol::Relative(0.0)));
|
|
assert!(!int_close(big, big + 1, Tol::Relative(1e-19)));
|
|
assert!(int_close(big, big + 1, Tol::Relative(1e-18)));
|
|
let umax = i128::from(u64::MAX);
|
|
assert!(!int_close(umax, umax - 1, Tol::Delta(0.0)));
|
|
assert!(int_close(
|
|
i128::from(i64::MIN),
|
|
umax,
|
|
Tol::Delta(f64::INFINITY)
|
|
));
|
|
assert!(!int_close(0, 1, Tol::Relative(1e9)));
|
|
}
|
|
|
|
#[test]
|
|
fn positions() {
|
|
assert_eq!(index(5, &[3, 4]), "1 1");
|
|
assert_eq!(index(0, &[1]), "0");
|
|
}
|
|
}
|