fix(tools): h5rs diff compares integers exactly under -d/-p

With a tolerance, integers were converted to f64 before comparing, so
int64/uint64 values above 2^53 that differ compared equal: -d 0 on 2^60
and 2^60 + 1 exited 0, where h5diff exits 1. Integer pairs are now
compared in i128 (the delta against floor(D), the relative quotient from
an exact difference), and the report prints the exact difference.

h5diff compares exactly when -p is below the f64 epsilon (2^60 and
2^60 + 1 differ at -p 1e-18, and nextafter(2, 0) and 2 at -p 1.5e-16);
h5rs now does the same.

Co-Authored-By: Claude Opus 5.5 (1M context) <[email protected]>
This commit is contained in:
osobh
2026-09-26 01:16:09 -05:00
co-authored by Claude Opus 5.5
parent 699ee9c447
commit f325d111f3
5 changed files with 102 additions and 10 deletions
+4 -1
View File
@@ -131,7 +131,10 @@
- `h5rs diff [-r] [-q] [-d D] [-p R] A B [OBJ1 [OBJ2]]` compares objects, - `h5rs diff [-r] [-q] [-d D] [-p R] A B [OBJ1 [OBJ2]]` compares objects,
kinds, datatypes, shapes, attributes, values and link targets; exit kinds, datatypes, shapes, attributes, values and link targets; exit
status 0/1/2 as h5diff's. Every path is compared, including every name status 0/1/2 as h5diff's. Every path is compared, including every name
of a hard-linked object and the members of a hard-linked group. Objects of a hard-linked object and the members of a hard-linked group; with a
`-d`/`-p` tolerance, integers are compared exactly in integer
arithmetic (no loss above 2^53), and a `-p` below the f64 epsilon
compares exactly, as h5diff's. Objects
that cannot be compared count as a difference (h5diff exits 0 for them), that cannot be compared count as a difference (h5diff exits 0 for them),
and NaN equals NaN. and NaN equals NaN.
- `h5rs check [--data] FILE` is a structural validator: it walks every - `h5rs check [--data] FILE` is a structural validator: it walks every
+5
View File
@@ -142,6 +142,11 @@ $ h5rs diff -d 0.001 a.h5 b.h5 # |a - b| > 0.001 is a difference
$ h5rs diff -p 0.01 a.h5 b.h5 # |a - b| / |a| > 1% is a difference $ h5rs diff -p 0.01 a.h5 b.h5 # |a - b| / |a| > 1% is a difference
``` ```
Integers are compared in integer arithmetic, with or without a tolerance, so
64-bit values beyond 2^53 lose no precision (`-d 0` tells 2^60 from
2^60 + 1). A relative tolerance below the f64 epsilon (2.2e-16) compares
exactly, as h5diff's does.
Exit status: 0 no differences, 1 differences, 2 error — the same as h5diff's Exit status: 0 no differences, 1 differences, 2 error — the same as h5diff's
on the cases `diff_exit_codes_match_h5diff` runs. Compared: which objects on the cases `diff_exit_codes_match_h5diff` runs. Compared: which objects
exist, their kinds, datatypes and shapes, attribute sets and values, dataset exist, their kinds, datatypes and shapes, attribute sets and values, dataset
+55 -9
View File
@@ -23,7 +23,8 @@ sets and values, and soft/external link targets.
-r, --report list every differing element (position, values, difference) -r, --report list every differing element (position, values, difference)
-q, --quiet print nothing; only the exit status -q, --quiet print nothing; only the exit status
-d, --delta D numbers differ only when |a - b| > D -d, --delta D numbers differ only when |a - b| > D
-p, --relative R numbers differ only when |a - b| / |a| > R -p, --relative R numbers differ only when |a - b| / |a| > R (an R below
the f64 epsilon, 2.2e-16, compares exactly, as h5diff)
-c, --count N list at most N differing elements per object with -r -c, --count N list at most N differing elements per object with -r
--max-bytes N largest dataset read (default 1 GiB); a larger one is an error --max-bytes N largest dataset read (default 1 GiB); a larger one is an error
@@ -119,7 +120,15 @@ pub fn run(args: &mut Args, out: &mut Out) -> std::io::Result<i32> {
_ => return args.usage_error(out, "--delta needs a number >= 0", USAGE), _ => return args.usage_error(out, "--delta needs a number >= 0", USAGE),
}, },
"-p" | "--relative" => match args.number::<f64>() { "-p" | "--relative" => match args.number::<f64>() {
Some(r) if r >= 0.0 => opts.tol = Tol::Relative(r), // As h5diff: a relative tolerance below the f64 epsilon
// cannot be told from rounding, so it compares exactly.
Some(r) if r >= 0.0 => {
opts.tol = if r < f64::EPSILON {
Tol::Exact
} else {
Tol::Relative(r)
}
}
_ => return args.usage_error(out, "--relative needs a number >= 0", USAGE), _ => return args.usage_error(out, "--relative needs a number >= 0", USAGE),
}, },
"-c" | "--count" => match args.number() { "-c" | "--count" => match args.number() {
@@ -506,8 +515,9 @@ impl Diff {
let pos = format!("[ {} ]", index(i as u64, &dims)); let pos = format!("[ {} ]", index(i as u64, &dims));
let ta = value::text(&va, &|_| None); let ta = value::text(&va, &|_| None);
let tb = value::text(&vb, &|_| None); let tb = value::text(&vb, &|_| None);
let dif = match (number(&va), number(&vb)) { let dif = match (int_of(&va), int_of(&vb), number(&va), number(&vb)) {
(Some(x), Some(y)) => value::fmt_float((x - y).abs(), 64), (Some(x), Some(y), ..) => x.abs_diff(y).to_string(),
(_, _, Some(x), Some(y)) => value::fmt_float((x - y).abs(), 64),
_ => String::new(), _ => String::new(),
}; };
rows.push(format!("{pos:<24}{ta:<24}{tb:<24}{dif}")); rows.push(format!("{pos:<24}{ta:<24}{tb:<24}{dif}"));
@@ -595,12 +605,12 @@ impl Diff {
} }
fn equal(&self, a: &Side, b: &Side, x: &Value, y: &Value) -> bool { fn equal(&self, a: &Side, b: &Side, x: &Value, y: &Value) -> bool {
// Integers in integer arithmetic: through f64 they lose precision
// above 2^53, and values that differ would compare equal.
if let (Some(p), Some(q)) = (int_of(x), int_of(y)) {
return int_close(p, q, self.opts.tol);
}
if let (Some(p), Some(q)) = (number(x), number(y)) { if let (Some(p), Some(q)) = (number(x), number(y)) {
let exact_ints = matches!((x, y), (Value::Int(_), Value::Int(_)))
|| matches!((x, y), (Value::Enum(..), Value::Enum(..)));
if exact_ints && matches!(self.opts.tol, Tol::Exact) {
return int_of(x) == int_of(y);
}
return close(p, q, self.opts.tol); return close(p, q, self.opts.tol);
} }
match (x, y) { match (x, y) {
@@ -657,6 +667,22 @@ fn close(a: f64, b: f64, tol: Tol) -> bool {
} }
} }
/// `close` for integers, exactly: the difference is taken in i128, so no
/// precision is lost at any 64-bit magnitude.
fn int_close(a: i128, b: i128, tol: Tol) -> bool {
if a == b {
return true;
}
let d = a.abs_diff(b);
match tol {
Tol::Exact => false,
// d is whole, so d <= t exactly when d <= floor(t) (saturating).
Tol::Delta(t) => d <= t.floor() as u128,
// d >= 1 here, so the quotient is never rounded to 0.
Tol::Relative(r) => a != 0 && d as f64 / a.unsigned_abs() as f64 <= r,
}
}
fn entry_word(e: &Entry) -> &'static str { fn entry_word(e: &Entry) -> &'static str {
match e { match e {
Entry::Obj(_, k) => kind_word(*k), Entry::Obj(_, k) => kind_word(*k),
@@ -766,6 +792,26 @@ mod tests {
assert!(!close(f64::NAN, 1.0, Tol::Delta(1e9))); assert!(!close(f64::NAN, 1.0, Tol::Delta(1e9)));
} }
#[test]
fn integer_tolerances_are_exact_above_2_pow_53() {
let big = 1i128 << 60;
assert!(!int_close(big, big + 1, Tol::Delta(0.0)));
assert!(!int_close(big, big + 1, Tol::Delta(0.5)));
assert!(int_close(big, big + 1, Tol::Delta(1.0)));
assert!(!int_close(big, big + 200, Tol::Delta(199.9)));
assert!(!int_close(big, big + 1, Tol::Relative(0.0)));
assert!(!int_close(big, big + 1, Tol::Relative(1e-19)));
assert!(int_close(big, big + 1, Tol::Relative(1e-18)));
let umax = i128::from(u64::MAX);
assert!(!int_close(umax, umax - 1, Tol::Delta(0.0)));
assert!(int_close(
i128::from(i64::MIN),
umax,
Tol::Delta(f64::INFINITY)
));
assert!(!int_close(0, 1, Tol::Relative(1e9)));
}
#[test] #[test]
fn positions() { fn positions() {
assert_eq!(index(5, &[3, 4]), "1 1"); assert_eq!(index(5, &[3, 4]), "1 1");
+6
View File
@@ -126,6 +126,12 @@ for name, last in (("copied.h5", 1), ("copied_changed.h5", 9)):
g["d"] = np.arange(3) g["d"] = np.arange(3)
g.create_group("s")["e"] = np.array([0, last if gname == "h" else 1]) g.create_group("s")["e"] = np.array([0, last if gname == "h" else 1])
# 64-bit integers one apart, beyond f64's 2^53 integer precision.
for name, d in (("big1.h5", 0), ("big2.h5", 1)):
with h5py.File(os.path.join(out, name), "w") as f:
f["i"] = np.array([2**60 + d, -(2**62) - d], dtype="i8")
f["u"] = np.array([2**64 - 1 - d], dtype="u8")
with h5py.File(os.path.join(out, "userblock.h5"), "w", userblock_size=1024) as f: with h5py.File(os.path.join(out, "userblock.h5"), "w", userblock_size=1024) as f:
f["d"] = np.arange(10) f["d"] = np.arange(10)
@@ -380,6 +380,38 @@ fn diff_exit_codes_match_h5diff() {
assert!(s.contains("2 difference(s) found"), "{s}"); assert!(s.contains("2 difference(s) found"), "{s}");
} }
/// Tolerances on 64-bit integers one apart, far beyond f64's integer
/// precision: compared exactly, as h5diff does.
#[test]
fn diff_tolerances_compare_large_integers_exactly() {
let Some(f) = generate() else { return };
let (a, b) = (f.p("big1.h5"), f.p("big2.h5"));
let cases: Vec<(Vec<&str>, i32)> = vec![
(vec![&a, &b], 1),
(vec!["-d", "0", &a, &b], 1),
(vec!["-d", "0.5", &a, &b], 1),
(vec!["-d", "1", &a, &b], 0),
(vec!["-d", "0", &a, &b, "/u"], 1),
(vec!["-p", "0", &a, &b], 1),
(vec!["-p", "1e-18", &a, &b, "/i"], 1),
(vec!["-p", "1e-15", &a, &b], 0),
];
let have_h5diff = tool_available("h5diff");
for (args, want) in cases {
if have_h5diff {
assert_eq!(code(&run("h5diff", &args)), want, "h5diff {args:?}");
}
let o = h5rs(&[&["diff"], args.as_slice()].concat());
assert_eq!(code(&o), want, "h5rs diff {args:?}: {}", stdout(&o));
}
// The reported difference is exact too.
let s = stdout(&h5rs(&["diff", "-r", "-d", "0", &a, &b, "/u"]));
assert!(
s.contains("18446744073709551615 18446744073709551614 1\n"),
"{s}"
);
}
/// An object hard-linked under two names is compared under both, with /// An object hard-linked under two names is compared under both, with
/// everything below it, so a file that shares one object between two names /// everything below it, so a file that shares one object between two names
/// equals a file that stores two identical copies. /// equals a file that stores two identical copies.