From 699ee9c447b86c77c1ee49cea145436aa0ff1179 Mon Sep 17 00:00:00 2001 From: osobh Date: Sat, 26 Sep 2026 01:14:53 -0500 Subject: [PATCH] fix(tools): h5rs diff compares every name of a hard-linked object The path walk skipped the second hard link to an object, so a file that shares one dataset between /x and /y differed from a file holding two identical copies: " exists only in ", exit 1, where h5diff exits 0. For a hard-linked group every member was reported the same way. diff now enumerates every path below the start object (a hard link back to an ancestor is recorded but not descended into), so each name is compared. A group whose links cannot be read is now an error instead of an empty group. Co-Authored-By: Claude Opus 5.5 (1M context) --- CHANGELOG.md | 6 +- crates/clawhdf5-tools/README.md | 12 ++- crates/clawhdf5-tools/src/diff.rs | 111 ++++++++++++++++---- crates/clawhdf5-tools/tests/gen_files.py | 17 +++ crates/clawhdf5-tools/tests/h5rs_interop.rs | 36 +++++++ 5 files changed, 156 insertions(+), 26 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index cb97d9a..71373fc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -130,8 +130,10 @@ files); metadata space is one figure, not broken down. - `h5rs diff [-r] [-q] [-d D] [-p R] A B [OBJ1 [OBJ2]]` compares objects, kinds, datatypes, shapes, attributes, values and link targets; exit - status 0/1/2 as h5diff's. Objects that cannot be compared count as a - difference (h5diff exits 0 for them), and NaN equals NaN. + status 0/1/2 as h5diff's. Every path is compared, including every name + of a hard-linked object and the members of a hard-linked group. Objects + that cannot be compared count as a difference (h5diff exits 0 for them), + and NaN equals NaN. - `h5rs check [--data] FILE` is a structural validator: it walks every object, parses every header message, verifies the checksums of every version 2+ structure it meets (superblock, object headers and diff --git a/crates/clawhdf5-tools/README.md b/crates/clawhdf5-tools/README.md index d40c486..ab51651 100644 --- a/crates/clawhdf5-tools/README.md +++ b/crates/clawhdf5-tools/README.md @@ -145,10 +145,14 @@ $ h5rs diff -p 0.01 a.h5 b.h5 # |a - b| / |a| > 1% is a difference Exit status: 0 no differences, 1 differences, 2 error — the same as h5diff's on the cases `diff_exit_codes_match_h5diff` runs. Compared: which objects exist, their kinds, datatypes and shapes, attribute sets and values, dataset -values, and soft/external link targets. Two differences from h5diff, on -purpose: objects that cannot be compared (different shapes or datatype -classes) count as a difference (h5diff warns and exits 0), and two NaNs are -equal. +values, and soft/external link targets. Every path is compared: an object +hard-linked under two names is compared under both, with everything below +it, so a file that shares one object between two names equals a file that +stores two identical copies. Differences from h5diff, on purpose: objects +that cannot be compared (different shapes or datatype classes) count as a +difference (h5diff warns and exits 0); two NaNs are equal; and the members +of a group reached by a second hard link are compared (h5diff lists them in +one file only and exits 1). ## `h5rs check` diff --git a/crates/clawhdf5-tools/src/diff.rs b/crates/clawhdf5-tools/src/diff.rs index ba8d324..8c3a793 100644 --- a/crates/clawhdf5-tools/src/diff.rs +++ b/crates/clawhdf5-tools/src/diff.rs @@ -2,6 +2,7 @@ use std::cell::OnceCell; use std::collections::{BTreeMap, HashMap}; +use std::rc::Rc; use clawhdf5_format::attribute::AttributeMessage; use clawhdf5_format::dataspace::{Dataspace, DataspaceType}; @@ -9,7 +10,7 @@ use clawhdf5_format::datatype::Datatype; use clawhdf5_format::object_header::ObjectHeader; use crate::cli::{Args, Out}; -use crate::h5::{H5, Kind, LinkKind}; +use crate::h5::{H5, Kind, Link, LinkKind}; use crate::value::{self, Decoder, Value}; pub const USAGE: &str = "\ @@ -177,25 +178,13 @@ pub fn run(args: &mut Args, out: &mut Out) -> std::io::Result { } else { format!("/{base}") }; - let mut entries = BTreeMap::new(); - let walk = h5.walk_from(addr, "", &mut |it| { - let e = match (it.link.map(|l| &l.kind), it.header) { - (Some(LinkKind::Soft(t)), _) => Entry::Soft(t.clone()), - (Some(LinkKind::External { file, path }), _) => { - Entry::External(file.clone(), path.clone()) - } - (Some(LinkKind::UserDefined(t)), _) => Entry::UserDefined(*t), - (_, Some(Err(e))) => Entry::Broken(e.to_string()), - (_, Some(Ok(h))) => Entry::Obj(it.addr.unwrap_or(0), Kind::of(h)), - // A second hard link to an object already compared. - (_, None) => return, - }; - entries.insert(it.path.to_string(), e); - }); - if let Err(e) = walk { - writeln!(out.e, "h5rs diff: {f}: {e}")?; - return Ok(2); - } + let entries = match collect(h5, addr) { + Ok(e) => e, + Err(e) => { + writeln!(out.e, "h5rs diff: {f}: {e}")?; + return Ok(2); + } + }; sides.push(Side { h5, label: f.clone(), @@ -248,6 +237,88 @@ pub fn run(args: &mut Args, out: &mut Out) -> std::io::Result { }) } +/// One object as the path walk sees it: what its paths compare as, and +/// (for a group) its links. +struct Node { + entry: Entry, + links: Vec, +} + +/// Every path below the object at `start`, relative to it (`""` is `start` +/// itself). Each hard link is its own path, so an object linked under two +/// names is compared under both, with everything below it: two files that +/// hold the same values are equal whether one shares an object between +/// names and the other stores copies. A hard link back to an ancestor (a +/// cycle) is recorded, not descended into. +fn collect(h5: &H5, start: u64) -> crate::h5::Result> { + let mut cache: HashMap> = HashMap::new(); + let mut node = |addr: u64| -> Rc { + cache + .entry(addr) + .or_insert_with(|| { + Rc::new(match h5.header(addr) { + Err(e) => Node { + entry: Entry::Broken(e.to_string()), + links: Vec::new(), + }, + Ok(h) => match Kind::of(&h) { + Kind::Group => match h5.links(&h) { + Ok(links) => Node { + entry: Entry::Obj(addr, Kind::Group), + links, + }, + Err(e) => Node { + entry: Entry::Broken(format!("links: {e}")), + links: Vec::new(), + }, + }, + k => Node { + entry: Entry::Obj(addr, k), + links: Vec::new(), + }, + }, + }) + }) + .clone() + }; + let mut entries = BTreeMap::new(); + // (address, relative path, addresses of the groups above it) + let mut stack: Vec<(u64, String, Rc>)> = + vec![(start, String::new(), Rc::new(Vec::new()))]; + while let Some((addr, path, above)) = stack.pop() { + if entries.len() >= crate::h5::MAX_OBJECTS { + return Err(crate::h5::Error::new(format!( + "more than {} paths; stopped walking", + crate::h5::MAX_OBJECTS + ))); + } + let n = node(addr); + entries.insert(path.clone(), n.entry.clone()); + if n.links.is_empty() || above.contains(&addr) { + continue; + } + let mut chain = (*above).clone(); + chain.push(addr); + let chain = Rc::new(chain); + for l in &n.links { + let child = format!("{path}/{}", l.name); + match &l.kind { + LinkKind::Hard(a) => stack.push((*a, child, chain.clone())), + LinkKind::Soft(t) => { + entries.insert(child, Entry::Soft(t.clone())); + } + LinkKind::External { file, path } => { + entries.insert(child, Entry::External(file.clone(), path.clone())); + } + LinkKind::UserDefined(t) => { + entries.insert(child, Entry::UserDefined(*t)); + } + } + } + } + Ok(entries) +} + fn kind_word(k: Kind) -> &'static str { match k { Kind::Group => "group", diff --git a/crates/clawhdf5-tools/tests/gen_files.py b/crates/clawhdf5-tools/tests/gen_files.py index 988cb22..cdd3226 100644 --- a/crates/clawhdf5-tools/tests/gen_files.py +++ b/crates/clawhdf5-tools/tests/gen_files.py @@ -109,6 +109,23 @@ small("attr.h5", attr=True) small("reshaped.h5", shape=(4, 3)) small("int.h5", dtype="i4") +# One object under two names (a hard link) against two separate copies. +with h5py.File(os.path.join(out, "hardlinked.h5"), "w") as f: + f["x"] = np.arange(5) + f["y"] = f["x"] + g = f.create_group("g") + g["d"] = np.arange(3) + g.create_group("s")["e"] = np.arange(2) + f["h"] = g +for name, last in (("copied.h5", 1), ("copied_changed.h5", 9)): + with h5py.File(os.path.join(out, name), "w") as f: + f["x"] = np.arange(5) + f["y"] = np.arange(5) + for gname in ("g", "h"): + g = f.create_group(gname) + g["d"] = np.arange(3) + g.create_group("s")["e"] = np.array([0, last if gname == "h" else 1]) + with h5py.File(os.path.join(out, "userblock.h5"), "w", userblock_size=1024) as f: f["d"] = np.arange(10) diff --git a/crates/clawhdf5-tools/tests/h5rs_interop.rs b/crates/clawhdf5-tools/tests/h5rs_interop.rs index 2e03b45..4e7e1bd 100644 --- a/crates/clawhdf5-tools/tests/h5rs_interop.rs +++ b/crates/clawhdf5-tools/tests/h5rs_interop.rs @@ -380,6 +380,42 @@ fn diff_exit_codes_match_h5diff() { assert!(s.contains("2 difference(s) found"), "{s}"); } +/// An object hard-linked under two names is compared under both, with +/// everything below it, so a file that shares one object between two names +/// equals a file that stores two identical copies. +#[test] +fn diff_compares_every_hard_link_path() { + let Some(f) = generate() else { return }; + let (linked, copied, changed) = ( + f.p("hardlinked.h5"), + f.p("copied.h5"), + f.p("copied_changed.h5"), + ); + // A hard-linked dataset: h5diff agrees (exit 0). + for (a, b) in [(&linked, &copied), (&copied, &linked)] { + let o = h5rs(&["diff", a, b, "/y"]); + assert_eq!(code(&o), 0, "{a} {b} /y: {}", stdout(&o)); + if tool_available("h5diff") { + assert_eq!(code(&run("h5diff", &[a, b, "/y"])), 0, "h5diff /y"); + } + // The whole file, including a hard-linked group's members (h5diff + // does not descend a group's second name, so it reports those + // members as present in one file only and exits 1). + let o = h5rs(&["diff", a, b]); + assert_eq!(code(&o), 0, "{a} {b}: {}", stdout(&o)); + assert!(!stdout(&o).contains("exists only"), "{}", stdout(&o)); + } + // A value under the second name is compared, not skipped. + let o = h5rs(&["diff", "-r", &linked, &changed]); + assert_eq!(code(&o), 1, "{}", stdout(&o)); + assert!( + stdout(&o).contains("dataset: and "), + "{}", + stdout(&o) + ); + assert!(!stdout(&o).contains(""), "{}", stdout(&o)); +} + /// Where h5rs deliberately differs from h5diff: objects that cannot be /// compared (another shape, another datatype class) are a difference. #[test]