diff --git a/CHANGELOG.md b/CHANGELOG.md index 71373fc..e3fbff7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -131,7 +131,10 @@ - `h5rs diff [-r] [-q] [-d D] [-p R] A B [OBJ1 [OBJ2]]` compares objects, kinds, datatypes, shapes, attributes, values and link targets; exit status 0/1/2 as h5diff's. Every path is compared, including every name - of a hard-linked object and the members of a hard-linked group. Objects + of a hard-linked object and the members of a hard-linked group; with a + `-d`/`-p` tolerance, integers are compared exactly in integer + arithmetic (no loss above 2^53), and a `-p` below the f64 epsilon + compares exactly, as h5diff's. Objects that cannot be compared count as a difference (h5diff exits 0 for them), and NaN equals NaN. - `h5rs check [--data] FILE` is a structural validator: it walks every diff --git a/crates/clawhdf5-tools/README.md b/crates/clawhdf5-tools/README.md index ab51651..b1fa0cf 100644 --- a/crates/clawhdf5-tools/README.md +++ b/crates/clawhdf5-tools/README.md @@ -142,6 +142,11 @@ $ h5rs diff -d 0.001 a.h5 b.h5 # |a - b| > 0.001 is a difference $ h5rs diff -p 0.01 a.h5 b.h5 # |a - b| / |a| > 1% is a difference ``` +Integers are compared in integer arithmetic, with or without a tolerance, so +64-bit values beyond 2^53 lose no precision (`-d 0` tells 2^60 from +2^60 + 1). A relative tolerance below the f64 epsilon (2.2e-16) compares +exactly, as h5diff's does. + Exit status: 0 no differences, 1 differences, 2 error — the same as h5diff's on the cases `diff_exit_codes_match_h5diff` runs. Compared: which objects exist, their kinds, datatypes and shapes, attribute sets and values, dataset diff --git a/crates/clawhdf5-tools/src/diff.rs b/crates/clawhdf5-tools/src/diff.rs index 8c3a793..7379265 100644 --- a/crates/clawhdf5-tools/src/diff.rs +++ b/crates/clawhdf5-tools/src/diff.rs @@ -23,7 +23,8 @@ sets and values, and soft/external link targets. -r, --report list every differing element (position, values, difference) -q, --quiet print nothing; only the exit status -d, --delta D numbers differ only when |a - b| > D - -p, --relative R numbers differ only when |a - b| / |a| > R + -p, --relative R numbers differ only when |a - b| / |a| > R (an R below + the f64 epsilon, 2.2e-16, compares exactly, as h5diff) -c, --count N list at most N differing elements per object with -r --max-bytes N largest dataset read (default 1 GiB); a larger one is an error @@ -119,7 +120,15 @@ pub fn run(args: &mut Args, out: &mut Out) -> std::io::Result { _ => return args.usage_error(out, "--delta needs a number >= 0", USAGE), }, "-p" | "--relative" => match args.number::() { - Some(r) if r >= 0.0 => opts.tol = Tol::Relative(r), + // As h5diff: a relative tolerance below the f64 epsilon + // cannot be told from rounding, so it compares exactly. + Some(r) if r >= 0.0 => { + opts.tol = if r < f64::EPSILON { + Tol::Exact + } else { + Tol::Relative(r) + } + } _ => return args.usage_error(out, "--relative needs a number >= 0", USAGE), }, "-c" | "--count" => match args.number() { @@ -506,8 +515,9 @@ impl Diff { let pos = format!("[ {} ]", index(i as u64, &dims)); let ta = value::text(&va, &|_| None); let tb = value::text(&vb, &|_| None); - let dif = match (number(&va), number(&vb)) { - (Some(x), Some(y)) => value::fmt_float((x - y).abs(), 64), + let dif = match (int_of(&va), int_of(&vb), number(&va), number(&vb)) { + (Some(x), Some(y), ..) => x.abs_diff(y).to_string(), + (_, _, Some(x), Some(y)) => value::fmt_float((x - y).abs(), 64), _ => String::new(), }; rows.push(format!("{pos:<24}{ta:<24}{tb:<24}{dif}")); @@ -595,12 +605,12 @@ impl Diff { } fn equal(&self, a: &Side, b: &Side, x: &Value, y: &Value) -> bool { + // Integers in integer arithmetic: through f64 they lose precision + // above 2^53, and values that differ would compare equal. + if let (Some(p), Some(q)) = (int_of(x), int_of(y)) { + return int_close(p, q, self.opts.tol); + } if let (Some(p), Some(q)) = (number(x), number(y)) { - let exact_ints = matches!((x, y), (Value::Int(_), Value::Int(_))) - || matches!((x, y), (Value::Enum(..), Value::Enum(..))); - if exact_ints && matches!(self.opts.tol, Tol::Exact) { - return int_of(x) == int_of(y); - } return close(p, q, self.opts.tol); } match (x, y) { @@ -657,6 +667,22 @@ fn close(a: f64, b: f64, tol: Tol) -> bool { } } +/// `close` for integers, exactly: the difference is taken in i128, so no +/// precision is lost at any 64-bit magnitude. +fn int_close(a: i128, b: i128, tol: Tol) -> bool { + if a == b { + return true; + } + let d = a.abs_diff(b); + match tol { + Tol::Exact => false, + // d is whole, so d <= t exactly when d <= floor(t) (saturating). + Tol::Delta(t) => d <= t.floor() as u128, + // d >= 1 here, so the quotient is never rounded to 0. + Tol::Relative(r) => a != 0 && d as f64 / a.unsigned_abs() as f64 <= r, + } +} + fn entry_word(e: &Entry) -> &'static str { match e { Entry::Obj(_, k) => kind_word(*k), @@ -766,6 +792,26 @@ mod tests { assert!(!close(f64::NAN, 1.0, Tol::Delta(1e9))); } + #[test] + fn integer_tolerances_are_exact_above_2_pow_53() { + let big = 1i128 << 60; + assert!(!int_close(big, big + 1, Tol::Delta(0.0))); + assert!(!int_close(big, big + 1, Tol::Delta(0.5))); + assert!(int_close(big, big + 1, Tol::Delta(1.0))); + assert!(!int_close(big, big + 200, Tol::Delta(199.9))); + assert!(!int_close(big, big + 1, Tol::Relative(0.0))); + assert!(!int_close(big, big + 1, Tol::Relative(1e-19))); + assert!(int_close(big, big + 1, Tol::Relative(1e-18))); + let umax = i128::from(u64::MAX); + assert!(!int_close(umax, umax - 1, Tol::Delta(0.0))); + assert!(int_close( + i128::from(i64::MIN), + umax, + Tol::Delta(f64::INFINITY) + )); + assert!(!int_close(0, 1, Tol::Relative(1e9))); + } + #[test] fn positions() { assert_eq!(index(5, &[3, 4]), "1 1"); diff --git a/crates/clawhdf5-tools/tests/gen_files.py b/crates/clawhdf5-tools/tests/gen_files.py index cdd3226..c32aa47 100644 --- a/crates/clawhdf5-tools/tests/gen_files.py +++ b/crates/clawhdf5-tools/tests/gen_files.py @@ -126,6 +126,12 @@ for name, last in (("copied.h5", 1), ("copied_changed.h5", 9)): g["d"] = np.arange(3) g.create_group("s")["e"] = np.array([0, last if gname == "h" else 1]) +# 64-bit integers one apart, beyond f64's 2^53 integer precision. +for name, d in (("big1.h5", 0), ("big2.h5", 1)): + with h5py.File(os.path.join(out, name), "w") as f: + f["i"] = np.array([2**60 + d, -(2**62) - d], dtype="i8") + f["u"] = np.array([2**64 - 1 - d], dtype="u8") + with h5py.File(os.path.join(out, "userblock.h5"), "w", userblock_size=1024) as f: f["d"] = np.arange(10) diff --git a/crates/clawhdf5-tools/tests/h5rs_interop.rs b/crates/clawhdf5-tools/tests/h5rs_interop.rs index 4e7e1bd..96618e0 100644 --- a/crates/clawhdf5-tools/tests/h5rs_interop.rs +++ b/crates/clawhdf5-tools/tests/h5rs_interop.rs @@ -380,6 +380,38 @@ fn diff_exit_codes_match_h5diff() { assert!(s.contains("2 difference(s) found"), "{s}"); } +/// Tolerances on 64-bit integers one apart, far beyond f64's integer +/// precision: compared exactly, as h5diff does. +#[test] +fn diff_tolerances_compare_large_integers_exactly() { + let Some(f) = generate() else { return }; + let (a, b) = (f.p("big1.h5"), f.p("big2.h5")); + let cases: Vec<(Vec<&str>, i32)> = vec![ + (vec![&a, &b], 1), + (vec!["-d", "0", &a, &b], 1), + (vec!["-d", "0.5", &a, &b], 1), + (vec!["-d", "1", &a, &b], 0), + (vec!["-d", "0", &a, &b, "/u"], 1), + (vec!["-p", "0", &a, &b], 1), + (vec!["-p", "1e-18", &a, &b, "/i"], 1), + (vec!["-p", "1e-15", &a, &b], 0), + ]; + let have_h5diff = tool_available("h5diff"); + for (args, want) in cases { + if have_h5diff { + assert_eq!(code(&run("h5diff", &args)), want, "h5diff {args:?}"); + } + let o = h5rs(&[&["diff"], args.as_slice()].concat()); + assert_eq!(code(&o), want, "h5rs diff {args:?}: {}", stdout(&o)); + } + // The reported difference is exact too. + let s = stdout(&h5rs(&["diff", "-r", "-d", "0", &a, &b, "/u"])); + assert!( + s.contains("18446744073709551615 18446744073709551614 1\n"), + "{s}" + ); +} + /// An object hard-linked under two names is compared under both, with /// everything below it, so a file that shares one object between two names /// equals a file that stores two identical copies.