diff --git a/CHANGELOG.md b/CHANGELOG.md index cb97d9a..71373fc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -130,8 +130,10 @@ files); metadata space is one figure, not broken down. - `h5rs diff [-r] [-q] [-d D] [-p R] A B [OBJ1 [OBJ2]]` compares objects, kinds, datatypes, shapes, attributes, values and link targets; exit - status 0/1/2 as h5diff's. Objects that cannot be compared count as a - difference (h5diff exits 0 for them), and NaN equals NaN. + status 0/1/2 as h5diff's. Every path is compared, including every name + of a hard-linked object and the members of a hard-linked group. Objects + that cannot be compared count as a difference (h5diff exits 0 for them), + and NaN equals NaN. - `h5rs check [--data] FILE` is a structural validator: it walks every object, parses every header message, verifies the checksums of every version 2+ structure it meets (superblock, object headers and diff --git a/crates/clawhdf5-tools/README.md b/crates/clawhdf5-tools/README.md index d40c486..ab51651 100644 --- a/crates/clawhdf5-tools/README.md +++ b/crates/clawhdf5-tools/README.md @@ -145,10 +145,14 @@ $ h5rs diff -p 0.01 a.h5 b.h5 # |a - b| / |a| > 1% is a difference Exit status: 0 no differences, 1 differences, 2 error — the same as h5diff's on the cases `diff_exit_codes_match_h5diff` runs. Compared: which objects exist, their kinds, datatypes and shapes, attribute sets and values, dataset -values, and soft/external link targets. Two differences from h5diff, on -purpose: objects that cannot be compared (different shapes or datatype -classes) count as a difference (h5diff warns and exits 0), and two NaNs are -equal. +values, and soft/external link targets. Every path is compared: an object +hard-linked under two names is compared under both, with everything below +it, so a file that shares one object between two names equals a file that +stores two identical copies. Differences from h5diff, on purpose: objects +that cannot be compared (different shapes or datatype classes) count as a +difference (h5diff warns and exits 0); two NaNs are equal; and the members +of a group reached by a second hard link are compared (h5diff lists them in +one file only and exits 1). ## `h5rs check` diff --git a/crates/clawhdf5-tools/src/diff.rs b/crates/clawhdf5-tools/src/diff.rs index ba8d324..8c3a793 100644 --- a/crates/clawhdf5-tools/src/diff.rs +++ b/crates/clawhdf5-tools/src/diff.rs @@ -2,6 +2,7 @@ use std::cell::OnceCell; use std::collections::{BTreeMap, HashMap}; +use std::rc::Rc; use clawhdf5_format::attribute::AttributeMessage; use clawhdf5_format::dataspace::{Dataspace, DataspaceType}; @@ -9,7 +10,7 @@ use clawhdf5_format::datatype::Datatype; use clawhdf5_format::object_header::ObjectHeader; use crate::cli::{Args, Out}; -use crate::h5::{H5, Kind, LinkKind}; +use crate::h5::{H5, Kind, Link, LinkKind}; use crate::value::{self, Decoder, Value}; pub const USAGE: &str = "\ @@ -177,25 +178,13 @@ pub fn run(args: &mut Args, out: &mut Out) -> std::io::Result { } else { format!("/{base}") }; - let mut entries = BTreeMap::new(); - let walk = h5.walk_from(addr, "", &mut |it| { - let e = match (it.link.map(|l| &l.kind), it.header) { - (Some(LinkKind::Soft(t)), _) => Entry::Soft(t.clone()), - (Some(LinkKind::External { file, path }), _) => { - Entry::External(file.clone(), path.clone()) - } - (Some(LinkKind::UserDefined(t)), _) => Entry::UserDefined(*t), - (_, Some(Err(e))) => Entry::Broken(e.to_string()), - (_, Some(Ok(h))) => Entry::Obj(it.addr.unwrap_or(0), Kind::of(h)), - // A second hard link to an object already compared. - (_, None) => return, - }; - entries.insert(it.path.to_string(), e); - }); - if let Err(e) = walk { - writeln!(out.e, "h5rs diff: {f}: {e}")?; - return Ok(2); - } + let entries = match collect(h5, addr) { + Ok(e) => e, + Err(e) => { + writeln!(out.e, "h5rs diff: {f}: {e}")?; + return Ok(2); + } + }; sides.push(Side { h5, label: f.clone(), @@ -248,6 +237,88 @@ pub fn run(args: &mut Args, out: &mut Out) -> std::io::Result { }) } +/// One object as the path walk sees it: what its paths compare as, and +/// (for a group) its links. +struct Node { + entry: Entry, + links: Vec, +} + +/// Every path below the object at `start`, relative to it (`""` is `start` +/// itself). Each hard link is its own path, so an object linked under two +/// names is compared under both, with everything below it: two files that +/// hold the same values are equal whether one shares an object between +/// names and the other stores copies. A hard link back to an ancestor (a +/// cycle) is recorded, not descended into. +fn collect(h5: &H5, start: u64) -> crate::h5::Result> { + let mut cache: HashMap> = HashMap::new(); + let mut node = |addr: u64| -> Rc { + cache + .entry(addr) + .or_insert_with(|| { + Rc::new(match h5.header(addr) { + Err(e) => Node { + entry: Entry::Broken(e.to_string()), + links: Vec::new(), + }, + Ok(h) => match Kind::of(&h) { + Kind::Group => match h5.links(&h) { + Ok(links) => Node { + entry: Entry::Obj(addr, Kind::Group), + links, + }, + Err(e) => Node { + entry: Entry::Broken(format!("links: {e}")), + links: Vec::new(), + }, + }, + k => Node { + entry: Entry::Obj(addr, k), + links: Vec::new(), + }, + }, + }) + }) + .clone() + }; + let mut entries = BTreeMap::new(); + // (address, relative path, addresses of the groups above it) + let mut stack: Vec<(u64, String, Rc>)> = + vec![(start, String::new(), Rc::new(Vec::new()))]; + while let Some((addr, path, above)) = stack.pop() { + if entries.len() >= crate::h5::MAX_OBJECTS { + return Err(crate::h5::Error::new(format!( + "more than {} paths; stopped walking", + crate::h5::MAX_OBJECTS + ))); + } + let n = node(addr); + entries.insert(path.clone(), n.entry.clone()); + if n.links.is_empty() || above.contains(&addr) { + continue; + } + let mut chain = (*above).clone(); + chain.push(addr); + let chain = Rc::new(chain); + for l in &n.links { + let child = format!("{path}/{}", l.name); + match &l.kind { + LinkKind::Hard(a) => stack.push((*a, child, chain.clone())), + LinkKind::Soft(t) => { + entries.insert(child, Entry::Soft(t.clone())); + } + LinkKind::External { file, path } => { + entries.insert(child, Entry::External(file.clone(), path.clone())); + } + LinkKind::UserDefined(t) => { + entries.insert(child, Entry::UserDefined(*t)); + } + } + } + } + Ok(entries) +} + fn kind_word(k: Kind) -> &'static str { match k { Kind::Group => "group", diff --git a/crates/clawhdf5-tools/tests/gen_files.py b/crates/clawhdf5-tools/tests/gen_files.py index 988cb22..cdd3226 100644 --- a/crates/clawhdf5-tools/tests/gen_files.py +++ b/crates/clawhdf5-tools/tests/gen_files.py @@ -109,6 +109,23 @@ small("attr.h5", attr=True) small("reshaped.h5", shape=(4, 3)) small("int.h5", dtype="i4") +# One object under two names (a hard link) against two separate copies. +with h5py.File(os.path.join(out, "hardlinked.h5"), "w") as f: + f["x"] = np.arange(5) + f["y"] = f["x"] + g = f.create_group("g") + g["d"] = np.arange(3) + g.create_group("s")["e"] = np.arange(2) + f["h"] = g +for name, last in (("copied.h5", 1), ("copied_changed.h5", 9)): + with h5py.File(os.path.join(out, name), "w") as f: + f["x"] = np.arange(5) + f["y"] = np.arange(5) + for gname in ("g", "h"): + g = f.create_group(gname) + g["d"] = np.arange(3) + g.create_group("s")["e"] = np.array([0, last if gname == "h" else 1]) + with h5py.File(os.path.join(out, "userblock.h5"), "w", userblock_size=1024) as f: f["d"] = np.arange(10) diff --git a/crates/clawhdf5-tools/tests/h5rs_interop.rs b/crates/clawhdf5-tools/tests/h5rs_interop.rs index 2e03b45..4e7e1bd 100644 --- a/crates/clawhdf5-tools/tests/h5rs_interop.rs +++ b/crates/clawhdf5-tools/tests/h5rs_interop.rs @@ -380,6 +380,42 @@ fn diff_exit_codes_match_h5diff() { assert!(s.contains("2 difference(s) found"), "{s}"); } +/// An object hard-linked under two names is compared under both, with +/// everything below it, so a file that shares one object between two names +/// equals a file that stores two identical copies. +#[test] +fn diff_compares_every_hard_link_path() { + let Some(f) = generate() else { return }; + let (linked, copied, changed) = ( + f.p("hardlinked.h5"), + f.p("copied.h5"), + f.p("copied_changed.h5"), + ); + // A hard-linked dataset: h5diff agrees (exit 0). + for (a, b) in [(&linked, &copied), (&copied, &linked)] { + let o = h5rs(&["diff", a, b, "/y"]); + assert_eq!(code(&o), 0, "{a} {b} /y: {}", stdout(&o)); + if tool_available("h5diff") { + assert_eq!(code(&run("h5diff", &[a, b, "/y"])), 0, "h5diff /y"); + } + // The whole file, including a hard-linked group's members (h5diff + // does not descend a group's second name, so it reports those + // members as present in one file only and exits 1). + let o = h5rs(&["diff", a, b]); + assert_eq!(code(&o), 0, "{a} {b}: {}", stdout(&o)); + assert!(!stdout(&o).contains("exists only"), "{}", stdout(&o)); + } + // A value under the second name is compared, not skipped. + let o = h5rs(&["diff", "-r", &linked, &changed]); + assert_eq!(code(&o), 1, "{}", stdout(&o)); + assert!( + stdout(&o).contains("dataset: and "), + "{}", + stdout(&o) + ); + assert!(!stdout(&o).contains(""), "{}", stdout(&o)); +} + /// Where h5rs deliberately differs from h5diff: objects that cannot be /// compared (another shape, another datatype class) are a difference. #[test]