From f325d111f3f63d99259696060e3e6b89169578aa Mon Sep 17 00:00:00 2001 From: osobh Date: Sat, 26 Sep 2026 01:16:09 -0500 Subject: [PATCH] fix(tools): h5rs diff compares integers exactly under -d/-p With a tolerance, integers were converted to f64 before comparing, so int64/uint64 values above 2^53 that differ compared equal: -d 0 on 2^60 and 2^60 + 1 exited 0, where h5diff exits 1. Integer pairs are now compared in i128 (the delta against floor(D), the relative quotient from an exact difference), and the report prints the exact difference. h5diff compares exactly when -p is below the f64 epsilon (2^60 and 2^60 + 1 differ at -p 1e-18, and nextafter(2, 0) and 2 at -p 1.5e-16); h5rs now does the same. Co-Authored-By: Claude Opus 5.5 (1M context) --- CHANGELOG.md | 5 +- crates/clawhdf5-tools/README.md | 5 ++ crates/clawhdf5-tools/src/diff.rs | 64 ++++++++++++++++++--- crates/clawhdf5-tools/tests/gen_files.py | 6 ++ crates/clawhdf5-tools/tests/h5rs_interop.rs | 32 +++++++++++ 5 files changed, 102 insertions(+), 10 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 71373fc..e3fbff7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -131,7 +131,10 @@ - `h5rs diff [-r] [-q] [-d D] [-p R] A B [OBJ1 [OBJ2]]` compares objects, kinds, datatypes, shapes, attributes, values and link targets; exit status 0/1/2 as h5diff's. Every path is compared, including every name - of a hard-linked object and the members of a hard-linked group. Objects + of a hard-linked object and the members of a hard-linked group; with a + `-d`/`-p` tolerance, integers are compared exactly in integer + arithmetic (no loss above 2^53), and a `-p` below the f64 epsilon + compares exactly, as h5diff's. Objects that cannot be compared count as a difference (h5diff exits 0 for them), and NaN equals NaN. - `h5rs check [--data] FILE` is a structural validator: it walks every diff --git a/crates/clawhdf5-tools/README.md b/crates/clawhdf5-tools/README.md index ab51651..b1fa0cf 100644 --- a/crates/clawhdf5-tools/README.md +++ b/crates/clawhdf5-tools/README.md @@ -142,6 +142,11 @@ $ h5rs diff -d 0.001 a.h5 b.h5 # |a - b| > 0.001 is a difference $ h5rs diff -p 0.01 a.h5 b.h5 # |a - b| / |a| > 1% is a difference ``` +Integers are compared in integer arithmetic, with or without a tolerance, so +64-bit values beyond 2^53 lose no precision (`-d 0` tells 2^60 from +2^60 + 1). A relative tolerance below the f64 epsilon (2.2e-16) compares +exactly, as h5diff's does. + Exit status: 0 no differences, 1 differences, 2 error — the same as h5diff's on the cases `diff_exit_codes_match_h5diff` runs. Compared: which objects exist, their kinds, datatypes and shapes, attribute sets and values, dataset diff --git a/crates/clawhdf5-tools/src/diff.rs b/crates/clawhdf5-tools/src/diff.rs index 8c3a793..7379265 100644 --- a/crates/clawhdf5-tools/src/diff.rs +++ b/crates/clawhdf5-tools/src/diff.rs @@ -23,7 +23,8 @@ sets and values, and soft/external link targets. -r, --report list every differing element (position, values, difference) -q, --quiet print nothing; only the exit status -d, --delta D numbers differ only when |a - b| > D - -p, --relative R numbers differ only when |a - b| / |a| > R + -p, --relative R numbers differ only when |a - b| / |a| > R (an R below + the f64 epsilon, 2.2e-16, compares exactly, as h5diff) -c, --count N list at most N differing elements per object with -r --max-bytes N largest dataset read (default 1 GiB); a larger one is an error @@ -119,7 +120,15 @@ pub fn run(args: &mut Args, out: &mut Out) -> std::io::Result { _ => return args.usage_error(out, "--delta needs a number >= 0", USAGE), }, "-p" | "--relative" => match args.number::() { - Some(r) if r >= 0.0 => opts.tol = Tol::Relative(r), + // As h5diff: a relative tolerance below the f64 epsilon + // cannot be told from rounding, so it compares exactly. + Some(r) if r >= 0.0 => { + opts.tol = if r < f64::EPSILON { + Tol::Exact + } else { + Tol::Relative(r) + } + } _ => return args.usage_error(out, "--relative needs a number >= 0", USAGE), }, "-c" | "--count" => match args.number() { @@ -506,8 +515,9 @@ impl Diff { let pos = format!("[ {} ]", index(i as u64, &dims)); let ta = value::text(&va, &|_| None); let tb = value::text(&vb, &|_| None); - let dif = match (number(&va), number(&vb)) { - (Some(x), Some(y)) => value::fmt_float((x - y).abs(), 64), + let dif = match (int_of(&va), int_of(&vb), number(&va), number(&vb)) { + (Some(x), Some(y), ..) => x.abs_diff(y).to_string(), + (_, _, Some(x), Some(y)) => value::fmt_float((x - y).abs(), 64), _ => String::new(), }; rows.push(format!("{pos:<24}{ta:<24}{tb:<24}{dif}")); @@ -595,12 +605,12 @@ impl Diff { } fn equal(&self, a: &Side, b: &Side, x: &Value, y: &Value) -> bool { + // Integers in integer arithmetic: through f64 they lose precision + // above 2^53, and values that differ would compare equal. + if let (Some(p), Some(q)) = (int_of(x), int_of(y)) { + return int_close(p, q, self.opts.tol); + } if let (Some(p), Some(q)) = (number(x), number(y)) { - let exact_ints = matches!((x, y), (Value::Int(_), Value::Int(_))) - || matches!((x, y), (Value::Enum(..), Value::Enum(..))); - if exact_ints && matches!(self.opts.tol, Tol::Exact) { - return int_of(x) == int_of(y); - } return close(p, q, self.opts.tol); } match (x, y) { @@ -657,6 +667,22 @@ fn close(a: f64, b: f64, tol: Tol) -> bool { } } +/// `close` for integers, exactly: the difference is taken in i128, so no +/// precision is lost at any 64-bit magnitude. +fn int_close(a: i128, b: i128, tol: Tol) -> bool { + if a == b { + return true; + } + let d = a.abs_diff(b); + match tol { + Tol::Exact => false, + // d is whole, so d <= t exactly when d <= floor(t) (saturating). + Tol::Delta(t) => d <= t.floor() as u128, + // d >= 1 here, so the quotient is never rounded to 0. + Tol::Relative(r) => a != 0 && d as f64 / a.unsigned_abs() as f64 <= r, + } +} + fn entry_word(e: &Entry) -> &'static str { match e { Entry::Obj(_, k) => kind_word(*k), @@ -766,6 +792,26 @@ mod tests { assert!(!close(f64::NAN, 1.0, Tol::Delta(1e9))); } + #[test] + fn integer_tolerances_are_exact_above_2_pow_53() { + let big = 1i128 << 60; + assert!(!int_close(big, big + 1, Tol::Delta(0.0))); + assert!(!int_close(big, big + 1, Tol::Delta(0.5))); + assert!(int_close(big, big + 1, Tol::Delta(1.0))); + assert!(!int_close(big, big + 200, Tol::Delta(199.9))); + assert!(!int_close(big, big + 1, Tol::Relative(0.0))); + assert!(!int_close(big, big + 1, Tol::Relative(1e-19))); + assert!(int_close(big, big + 1, Tol::Relative(1e-18))); + let umax = i128::from(u64::MAX); + assert!(!int_close(umax, umax - 1, Tol::Delta(0.0))); + assert!(int_close( + i128::from(i64::MIN), + umax, + Tol::Delta(f64::INFINITY) + )); + assert!(!int_close(0, 1, Tol::Relative(1e9))); + } + #[test] fn positions() { assert_eq!(index(5, &[3, 4]), "1 1"); diff --git a/crates/clawhdf5-tools/tests/gen_files.py b/crates/clawhdf5-tools/tests/gen_files.py index cdd3226..c32aa47 100644 --- a/crates/clawhdf5-tools/tests/gen_files.py +++ b/crates/clawhdf5-tools/tests/gen_files.py @@ -126,6 +126,12 @@ for name, last in (("copied.h5", 1), ("copied_changed.h5", 9)): g["d"] = np.arange(3) g.create_group("s")["e"] = np.array([0, last if gname == "h" else 1]) +# 64-bit integers one apart, beyond f64's 2^53 integer precision. +for name, d in (("big1.h5", 0), ("big2.h5", 1)): + with h5py.File(os.path.join(out, name), "w") as f: + f["i"] = np.array([2**60 + d, -(2**62) - d], dtype="i8") + f["u"] = np.array([2**64 - 1 - d], dtype="u8") + with h5py.File(os.path.join(out, "userblock.h5"), "w", userblock_size=1024) as f: f["d"] = np.arange(10) diff --git a/crates/clawhdf5-tools/tests/h5rs_interop.rs b/crates/clawhdf5-tools/tests/h5rs_interop.rs index 4e7e1bd..96618e0 100644 --- a/crates/clawhdf5-tools/tests/h5rs_interop.rs +++ b/crates/clawhdf5-tools/tests/h5rs_interop.rs @@ -380,6 +380,38 @@ fn diff_exit_codes_match_h5diff() { assert!(s.contains("2 difference(s) found"), "{s}"); } +/// Tolerances on 64-bit integers one apart, far beyond f64's integer +/// precision: compared exactly, as h5diff does. +#[test] +fn diff_tolerances_compare_large_integers_exactly() { + let Some(f) = generate() else { return }; + let (a, b) = (f.p("big1.h5"), f.p("big2.h5")); + let cases: Vec<(Vec<&str>, i32)> = vec![ + (vec![&a, &b], 1), + (vec!["-d", "0", &a, &b], 1), + (vec!["-d", "0.5", &a, &b], 1), + (vec!["-d", "1", &a, &b], 0), + (vec!["-d", "0", &a, &b, "/u"], 1), + (vec!["-p", "0", &a, &b], 1), + (vec!["-p", "1e-18", &a, &b, "/i"], 1), + (vec!["-p", "1e-15", &a, &b], 0), + ]; + let have_h5diff = tool_available("h5diff"); + for (args, want) in cases { + if have_h5diff { + assert_eq!(code(&run("h5diff", &args)), want, "h5diff {args:?}"); + } + let o = h5rs(&[&["diff"], args.as_slice()].concat()); + assert_eq!(code(&o), want, "h5rs diff {args:?}: {}", stdout(&o)); + } + // The reported difference is exact too. + let s = stdout(&h5rs(&["diff", "-r", "-d", "0", &a, &b, "/u"])); + assert!( + s.contains("18446744073709551615 18446744073709551614 1\n"), + "{s}" + ); +} + /// An object hard-linked under two names is compared under both, with /// everything below it, so a file that shares one object between two names /// equals a file that stores two identical copies.