From b5e43bacd738535f4975be5f94f1bfbfefa8ef36 Mon Sep 17 00:00:00 2001 From: osobh Date: Sat, 26 Sep 2026 01:18:13 -0500 Subject: [PATCH] fix(tools): h5rs diff compares soft links by target, like h5diff An OBJ that was a soft link was resolved and its target object compared, so two files whose /g/s both point at /z differed when /z did: exit 1, where h5diff (without --follow-symlinks) compares the links' target paths and exits 0. A soft link is now compared as a link wherever it is, OBJ included. --follow-symlinks compares the objects soft links lead to instead, walks into soft-linked groups, resolves relative targets against the link's group, and treats two dangling links as the same; exit codes equal h5diff's on 14 cases. External links are never followed (documented). Co-Authored-By: Claude Opus 5.5 (1M context) --- CHANGELOG.md | 20 ++--- crates/clawhdf5-tools/README.md | 10 ++- crates/clawhdf5-tools/src/diff.rs | 86 +++++++++++++++++++-- crates/clawhdf5-tools/tests/gen_files.py | 17 ++++ crates/clawhdf5-tools/tests/h5rs_interop.rs | 39 ++++++++++ 5 files changed, 155 insertions(+), 17 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e3fbff7..885da0f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -128,15 +128,17 @@ - `h5rs stat FILE` reports h5stat's object, link, rank, layout, filter, attribute, raw-data and file-size figures (equal to h5stat's on the test files); metadata space is one figure, not broken down. - - `h5rs diff [-r] [-q] [-d D] [-p R] A B [OBJ1 [OBJ2]]` compares objects, - kinds, datatypes, shapes, attributes, values and link targets; exit - status 0/1/2 as h5diff's. Every path is compared, including every name - of a hard-linked object and the members of a hard-linked group; with a - `-d`/`-p` tolerance, integers are compared exactly in integer - arithmetic (no loss above 2^53), and a `-p` below the f64 epsilon - compares exactly, as h5diff's. Objects - that cannot be compared count as a difference (h5diff exits 0 for them), - and NaN equals NaN. + - `h5rs diff [-r] [-q] [-d D] [-p R] [--follow-symlinks] A B [OBJ1 [OBJ2]]` + compares objects, kinds, datatypes, shapes, attributes, values and link + targets; exit status 0/1/2 as h5diff's. Soft links are compared by + target path, as h5diff's default, or with `--follow-symlinks` by the + objects they lead to (external links are never followed). Every path is + compared, including every name of a hard-linked object and the members + of a hard-linked group; with a `-d`/`-p` tolerance, integers are + compared exactly in integer arithmetic (no loss above 2^53), and a `-p` + below the f64 epsilon compares exactly, as h5diff's. Objects that cannot + be compared count as a difference (h5diff exits 0 for them), and NaN + equals NaN. - `h5rs check [--data] FILE` is a structural validator: it walks every object, parses every header message, verifies the checksums of every version 2+ structure it meets (superblock, object headers and diff --git a/crates/clawhdf5-tools/README.md b/crates/clawhdf5-tools/README.md index b1fa0cf..633da68 100644 --- a/crates/clawhdf5-tools/README.md +++ b/crates/clawhdf5-tools/README.md @@ -140,6 +140,7 @@ $ h5rs diff a.h5 b.h5 /x /y # different paths in each $ h5rs diff -r a.h5 b.h5 /d # list every differing element $ h5rs diff -d 0.001 a.h5 b.h5 # |a - b| > 0.001 is a difference $ h5rs diff -p 0.01 a.h5 b.h5 # |a - b| / |a| > 1% is a difference +$ h5rs diff --follow-symlinks a.h5 b.h5 /lnk # what the soft link /lnk leads to ``` Integers are compared in integer arithmetic, with or without a tolerance, so @@ -150,7 +151,14 @@ exactly, as h5diff's does. Exit status: 0 no differences, 1 differences, 2 error — the same as h5diff's on the cases `diff_exit_codes_match_h5diff` runs. Compared: which objects exist, their kinds, datatypes and shapes, attribute sets and values, dataset -values, and soft/external link targets. Every path is compared: an object +values, and soft/external link targets. A soft link — including an OBJ +that is itself a soft link — is compared as a link, by its target path, as +h5diff does; `--follow-symlinks` compares the objects soft links lead to +instead (and walks into soft-linked groups), and two dangling links are then +the same, as in h5diff. External links are always compared by target (file +and path): `--follow-symlinks` does not open other files, where h5diff's +does. Two dangling soft links with different targets are a difference +(h5diff reports them with `-r`/`-v` but exits 0 without). Every path is compared: an object hard-linked under two names is compared under both, with everything below it, so a file that shares one object between two names equals a file that stores two identical copies. Differences from h5diff, on purpose: objects diff --git a/crates/clawhdf5-tools/src/diff.rs b/crates/clawhdf5-tools/src/diff.rs index 7379265..8a52029 100644 --- a/crates/clawhdf5-tools/src/diff.rs +++ b/crates/clawhdf5-tools/src/diff.rs @@ -18,10 +18,14 @@ usage: h5rs diff [options] FILE1 FILE2 [OBJ1 [OBJ2]] Compare FILE1 and FILE2 (or OBJ1 in FILE1 with OBJ2 in FILE2, and everything below them): the objects present, their kinds, datatypes, shapes, attribute -sets and values, and soft/external link targets. +sets and values, and soft/external link targets (an OBJ that is itself a +soft link is compared as a link, as h5diff does, unless --follow-symlinks). -r, --report list every differing element (position, values, difference) -q, --quiet print nothing; only the exit status + --follow-symlinks compare the objects soft links lead to (and walk into + soft-linked groups) instead of the links' target paths; + two dangling links are the same -d, --delta D numbers differ only when |a - b| > D -p, --relative R numbers differ only when |a - b| / |a| > R (an R below the f64 epsilon, 2.2e-16, compares exactly, as h5diff) @@ -46,6 +50,8 @@ struct Opts { quiet: bool, tol: Tol, count: usize, + /// Compare the objects soft links lead to, not the links' targets. + follow: bool, } /// What a relative path names in one file. @@ -108,6 +114,7 @@ pub fn run(args: &mut Args, out: &mut Out) -> std::io::Result { quiet: false, tol: Tol::Exact, count: usize::MAX, + follow: false, }; let mut max_bytes = None; let mut pos = Vec::new(); @@ -115,6 +122,7 @@ pub fn run(args: &mut Args, out: &mut Out) -> std::io::Result { match a.as_str() { "-r" | "--report" => opts.report = true, "-q" | "--quiet" => opts.quiet = true, + "--follow-symlinks" => opts.follow = true, "-d" | "--delta" => match args.number::() { Some(d) if d >= 0.0 => opts.tol = Tol::Delta(d), _ => return args.usage_error(out, "--delta needs a number >= 0", USAGE), @@ -171,8 +179,8 @@ pub fn run(args: &mut Args, out: &mut Out) -> std::io::Result { let obj2 = pos.get(3).cloned().unwrap_or_else(|| obj1.clone()); let mut sides = Vec::new(); for (h5, (obj, f)) in files.iter().zip([(&obj1, &pos[0]), (&obj2, &pos[1])]) { - let addr = match h5.resolve(obj) { - Ok(a) => a, + let start = match start(h5, obj, opts.follow) { + Ok(x) => x, Err(_) => { writeln!( out.e, @@ -187,7 +195,11 @@ pub fn run(args: &mut Args, out: &mut Out) -> std::io::Result { } else { format!("/{base}") }; - let entries = match collect(h5, addr) { + let collected = match start { + Start::Obj(addr) => collect(h5, addr, &base, opts.follow), + Start::Link(e) => Ok(BTreeMap::from([(String::new(), e)])), + }; + let entries = match collected { Ok(e) => e, Err(e) => { writeln!(out.e, "h5rs diff: {f}: {e}")?; @@ -246,6 +258,51 @@ pub fn run(args: &mut Args, out: &mut Out) -> std::io::Result { }) } +/// What OBJ1/OBJ2 names. +enum Start { + Obj(u64), + /// A soft (not followed, or dangling), external or user-defined link. + Link(Entry), +} + +/// Resolve OBJ1/OBJ2. Soft links on the way to it are followed, as in any +/// HDF5 path; a soft link that *is* the named object is compared as a link +/// (its target path), as h5diff does, unless `follow`. +fn start(h5: &H5, obj: &str, follow: bool) -> crate::h5::Result { + let p = obj.trim_matches('/'); + if !p.is_empty() { + let (parent, name) = p.rsplit_once('/').unwrap_or(("", p)); + let link = h5 + .resolve(parent) + .and_then(|a| h5.header(a)) + .and_then(|h| h5.links(&h)) + .ok() + .and_then(|ls| ls.into_iter().find(|l| l.name == name)); + if let Some(l) = link { + return Ok(match l.kind { + LinkKind::Hard(a) => Start::Obj(a), + LinkKind::Soft(t) if follow => match soft_target(h5, &format!("/{parent}"), &t) { + Ok(a) => Start::Obj(a), + Err(_) => Start::Link(Entry::Soft(t)), + }, + LinkKind::Soft(t) => Start::Link(Entry::Soft(t)), + LinkKind::External { file, path } => Start::Link(Entry::External(file, path)), + LinkKind::UserDefined(t) => Start::Link(Entry::UserDefined(t)), + }); + } + } + h5.resolve(obj).map(Start::Obj) +} + +/// The object a soft link in group `parent` (an absolute path) leads to. +fn soft_target(h5: &H5, parent: &str, target: &str) -> crate::h5::Result { + if target.starts_with('/') { + h5.resolve(target) + } else { + h5.resolve(&format!("{}/{target}", parent.trim_end_matches('/'))) + } +} + /// One object as the path walk sees it: what its paths compare as, and /// (for a group) its links. struct Node { @@ -258,8 +315,15 @@ struct Node { /// names is compared under both, with everything below it: two files that /// hold the same values are equal whether one shares an object between /// names and the other stores copies. A hard link back to an ancestor (a -/// cycle) is recorded, not descended into. -fn collect(h5: &H5, start: u64) -> crate::h5::Result> { +/// cycle) is recorded, not descended into. With `follow`, a soft link is +/// walked like a hard link to its target (a dangling one stays a link); +/// `base` is the absolute path of `start`, for relative link targets. +fn collect( + h5: &H5, + start: u64, + base: &str, + follow: bool, +) -> crate::h5::Result> { let mut cache: HashMap> = HashMap::new(); let mut node = |addr: u64| -> Rc { cache @@ -313,6 +377,12 @@ fn collect(h5: &H5, start: u64) -> crate::h5::Result> { let child = format!("{path}/{}", l.name); match &l.kind { LinkKind::Hard(a) => stack.push((*a, child, chain.clone())), + LinkKind::Soft(t) if follow => match soft_target(h5, &format!("{base}{path}"), t) { + Ok(a) => stack.push((a, child, chain.clone())), + Err(_) => { + entries.insert(child, Entry::Soft(t.clone())); + } + }, LinkKind::Soft(t) => { entries.insert(child, Entry::Soft(t.clone())); } @@ -365,7 +435,9 @@ impl Diff { (Entry::Broken(e), _) => self.error(out, &format!("<{pa}> in <{}>: {e}", a.label)), (_, Entry::Broken(e)) => self.error(out, &format!("<{pb}> in <{}>: {e}", b.label)), (Entry::Soft(x), Entry::Soft(y)) => { - if x != y { + // Followed links that are left are dangling on both sides, + // which h5diff counts as the same. + if x != y && !self.opts.follow { self.diffs += 1; self.say(out, &format!("soft link: <{pa}> -> {x} and <{pb}> -> {y}"))?; } diff --git a/crates/clawhdf5-tools/tests/gen_files.py b/crates/clawhdf5-tools/tests/gen_files.py index c32aa47..bc18574 100644 --- a/crates/clawhdf5-tools/tests/gen_files.py +++ b/crates/clawhdf5-tools/tests/gen_files.py @@ -132,6 +132,23 @@ for name, d in (("big1.h5", 0), ("big2.h5", 1)): f["i"] = np.array([2**60 + d, -(2**62) - d], dtype="i8") f["u"] = np.array([2**64 - 1 - d], dtype="u8") +# Soft links: the same link targets, whose target objects differ. +for name, v in (("soft1.h5", 0), ("soft2.h5", 1)): + with h5py.File(os.path.join(out, name), "w") as f: + f["z"] = np.arange(4) + v + f.create_group("g")["s"] = h5py.SoftLink("/z") + grp = f.create_group("grp") + grp["d"] = np.arange(3) + v + f["lnk"] = h5py.SoftLink("/grp") + f["dang"] = h5py.SoftLink("/nowhere") +# Soft links: different link targets, whose target objects are equal. +for name, t in (("target1.h5", "/a"), ("target2.h5", "/b")): + with h5py.File(os.path.join(out, name), "w") as f: + f["a"] = np.arange(4) + f["b"] = np.arange(4) + f["s"] = h5py.SoftLink(t) + f["rel"] = h5py.SoftLink("a") + with h5py.File(os.path.join(out, "userblock.h5"), "w", userblock_size=1024) as f: f["d"] = np.arange(10) diff --git a/crates/clawhdf5-tools/tests/h5rs_interop.rs b/crates/clawhdf5-tools/tests/h5rs_interop.rs index 96618e0..0e00e0a 100644 --- a/crates/clawhdf5-tools/tests/h5rs_interop.rs +++ b/crates/clawhdf5-tools/tests/h5rs_interop.rs @@ -412,6 +412,45 @@ fn diff_tolerances_compare_large_integers_exactly() { ); } +/// A soft link is compared as a link (its target path), as h5diff does, +/// unless `--follow-symlinks`, which compares (and walks into) what it +/// leads to; two dangling links are then the same. +#[test] +fn diff_soft_links_like_h5diff() { + let Some(f) = generate() else { return }; + let (s1, s2, t1, t2) = ( + f.p("soft1.h5"), + f.p("soft2.h5"), + f.p("target1.h5"), + f.p("target2.h5"), + ); + let fl = "--follow-symlinks"; + let cases: Vec<(Vec<&str>, i32)> = vec![ + (vec![&s1, &s2, "/g/s"], 0), + (vec![&s1, &s2, "/g"], 0), + (vec![&s1, &s2, "/lnk"], 0), + (vec![&s1, &s2, "/dang"], 0), + (vec![&s1, &s2, "/lnk/d"], 1), + (vec![&t1, &t2, "/s"], 1), + (vec![&t1, &t2, "/s", "/rel"], 1), + (vec![fl, &s1, &s2, "/g/s"], 1), + (vec![fl, &s1, &s2, "/g"], 1), + (vec![fl, &s1, &s2, "/lnk"], 1), + (vec![fl, &s1, &s2, "/dang"], 0), + (vec![fl, &s1, &s1], 0), + (vec![fl, &t1, &t2, "/s"], 0), + (vec![fl, &t1, &t2, "/s", "/rel"], 0), + ]; + let have_h5diff = tool_available("h5diff"); + for (args, want) in cases { + if have_h5diff { + assert_eq!(code(&run("h5diff", &args)), want, "h5diff {args:?}"); + } + let o = h5rs(&[&["diff"], args.as_slice()].concat()); + assert_eq!(code(&o), want, "h5rs diff {args:?}: {}", stdout(&o)); + } +} + /// An object hard-linked under two names is compared under both, with /// everything below it, so a file that shares one object between two names /// equals a file that stores two identical copies.