osobhandClaude Opus 5.5
b5e43bacd7
An OBJ that was a soft link was resolved and its target object compared, so two files whose /g/s both point at /z differed when /z did: exit 1, where h5diff (without --follow-symlinks) compares the links' target paths and exits 0. A soft link is now compared as a link wherever it is, OBJ included. --follow-symlinks compares the objects soft links lead to instead, walks into soft-linked groups, resolves relative targets against the link's group, and treats two dangling links as the same; exit codes equal h5diff's on 14 cases. External links are never followed (documented). Co-Authored-By: Claude Opus 5.5 (1M context) <[email protected]>
677 lines
24 KiB
Rust
677 lines
24 KiB
Rust
//! `h5rs` against libhdf5's own tools, on files h5py writes
|
|
//! (`tests/gen_files.py`): `ls` against h5ls, `stat` against h5stat, `dump`
|
|
//! against h5dump, `diff` exit codes against h5diff, `dump --json` values
|
|
//! against h5py, and `check` on valid and deliberately corrupted files.
|
|
//!
|
|
//! Needs python3 with h5py (`CLAWHDF5_PYTHON`) and, for the comparisons,
|
|
//! the libhdf5 command-line tools (h5ls, h5stat, h5dump, h5diff) on `PATH`.
|
|
//! Each test skips when what it needs is missing, unless
|
|
//! `CLAWHDF5_REQUIRE_INTEROP=1`, which turns a skip into a failure.
|
|
|
|
use std::collections::BTreeMap;
|
|
use std::path::{Path, PathBuf};
|
|
use std::process::{Command, Output};
|
|
|
|
use clawhdf5_format::checksum::jenkins_lookup3;
|
|
|
|
fn python() -> String {
|
|
std::env::var("CLAWHDF5_PYTHON").unwrap_or_else(|_| "python3".to_string())
|
|
}
|
|
|
|
fn interop_required() -> bool {
|
|
std::env::var("CLAWHDF5_REQUIRE_INTEROP").is_ok_and(|v| v == "1")
|
|
}
|
|
|
|
fn python_available() -> bool {
|
|
Command::new(python())
|
|
.args(["-c", "import h5py, numpy"])
|
|
.output()
|
|
.map(|o| o.status.success())
|
|
.unwrap_or(false)
|
|
}
|
|
|
|
fn tool_available(name: &str) -> bool {
|
|
Command::new(name)
|
|
.arg("--version")
|
|
.output()
|
|
.map(|o| o.status.success())
|
|
.unwrap_or(false)
|
|
}
|
|
|
|
/// Skip (return true) when `what` is unavailable, or fail when interop is
|
|
/// required.
|
|
fn missing(ok: bool, what: &str) -> bool {
|
|
if ok {
|
|
return false;
|
|
}
|
|
assert!(
|
|
!interop_required(),
|
|
"CLAWHDF5_REQUIRE_INTEROP=1 but {what} is not available"
|
|
);
|
|
eprintln!("SKIP: {what} not available");
|
|
true
|
|
}
|
|
|
|
fn h5rs(args: &[&str]) -> Output {
|
|
Command::new(env!("CARGO_BIN_EXE_h5rs"))
|
|
.args(args)
|
|
.output()
|
|
.expect("run h5rs")
|
|
}
|
|
|
|
fn run(tool: &str, args: &[&str]) -> Output {
|
|
Command::new(tool).args(args).output().expect("run tool")
|
|
}
|
|
|
|
fn stdout(o: &Output) -> String {
|
|
String::from_utf8_lossy(&o.stdout).into_owned()
|
|
}
|
|
|
|
/// The generated files and what h5py read from them.
|
|
struct Files {
|
|
dir: tempfile::TempDir,
|
|
values: serde_json::Value,
|
|
}
|
|
|
|
impl Files {
|
|
fn path(&self, name: &str) -> PathBuf {
|
|
self.dir.path().join(name)
|
|
}
|
|
|
|
fn p(&self, name: &str) -> String {
|
|
self.path(name).to_string_lossy().into_owned()
|
|
}
|
|
}
|
|
|
|
fn generate() -> Option<Files> {
|
|
if missing(python_available(), "python3 with h5py") {
|
|
return None;
|
|
}
|
|
let dir = tempfile::tempdir().unwrap();
|
|
let script = Path::new(env!("CARGO_MANIFEST_DIR")).join("tests/gen_files.py");
|
|
let out = Command::new(python())
|
|
.arg(&script)
|
|
.arg(dir.path())
|
|
.output()
|
|
.expect("run gen_files.py");
|
|
assert!(
|
|
out.status.success(),
|
|
"gen_files.py failed:\n{}",
|
|
String::from_utf8_lossy(&out.stderr)
|
|
);
|
|
let values = serde_json::from_slice(&out.stdout).expect("gen_files.py output");
|
|
Some(Files { dir, values })
|
|
}
|
|
|
|
const LIBVERS: [&str; 2] = ["earliest.h5", "latest.h5"];
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// ls
|
|
// ---------------------------------------------------------------------------
|
|
|
|
/// Every h5ls line must be the start of the matching h5rs line (h5rs adds
|
|
/// the datatype after a dataset's shape).
|
|
fn assert_ls_like(ours: &str, reference: &str, what: &str) {
|
|
let ours: Vec<&str> = ours.lines().collect();
|
|
let refs: Vec<&str> = reference.lines().collect();
|
|
assert_eq!(
|
|
ours.len(),
|
|
refs.len(),
|
|
"{what}: line count\nh5rs:\n{}\nh5ls:\n{}",
|
|
ours.join("\n"),
|
|
refs.join("\n")
|
|
);
|
|
for (o, r) in ours.iter().zip(&refs) {
|
|
let r = r.trim_end();
|
|
assert!(
|
|
o.starts_with(r) && (o.len() == r.len() || r.contains("Dataset {")),
|
|
"{what}:\n h5rs: {o}\n h5ls: {r}"
|
|
);
|
|
}
|
|
}
|
|
|
|
#[test]
|
|
fn ls_matches_h5ls() {
|
|
let Some(f) = generate() else { return };
|
|
if missing(tool_available("h5ls"), "h5ls") {
|
|
return;
|
|
}
|
|
for name in LIBVERS.iter().chain(&["userblock.h5"]) {
|
|
let p = f.p(name);
|
|
for (args, what) in [(vec!["-r"], "recursive"), (vec![], "root")] {
|
|
let mut a: Vec<&str> = args.clone();
|
|
a.push(&p);
|
|
let ours = h5rs(&[&["ls"], a.as_slice()].concat());
|
|
assert!(ours.status.success(), "{name} {what}: {:?}", ours);
|
|
let reference = run("h5ls", &a);
|
|
assert_ls_like(
|
|
&stdout(&ours),
|
|
&stdout(&reference),
|
|
&format!("{name} {what}"),
|
|
);
|
|
}
|
|
}
|
|
// A group, and a dataset, named after the file.
|
|
for obj in ["/grp", "/grp/sub", "/contig", "/grp/ext2"] {
|
|
let arg = format!("{}{obj}", f.p("latest.h5"));
|
|
let ours = h5rs(&["ls", &arg]);
|
|
let reference = run("h5ls", &[&arg]);
|
|
assert_ls_like(&stdout(&ours), &stdout(&reference), &arg);
|
|
let ours = h5rs(&["ls", "-r", &arg]);
|
|
let reference = run("h5ls", &["-r", &arg]);
|
|
assert_ls_like(&stdout(&ours), &stdout(&reference), &format!("-r {arg}"));
|
|
}
|
|
}
|
|
|
|
#[test]
|
|
fn ls_verbose_describes_layout_filters_and_attributes() {
|
|
let Some(f) = generate() else { return };
|
|
let o = h5rs(&["ls", "-r", "-v", &f.p("latest.h5")]);
|
|
assert!(o.status.success());
|
|
let s = stdout(&o);
|
|
for want in [
|
|
"Layout: chunked (extensible array index)",
|
|
"Layout: chunked (v2 B-tree index)",
|
|
"Layout: chunked (fixed array index)",
|
|
"Layout: chunked (single chunk index)",
|
|
"Filter-0: shuffle-2",
|
|
"Filter-1: deflate-1",
|
|
"Filter-2: fletcher32-3",
|
|
"Chunks: {100} 400 bytes",
|
|
"Attribute: units scalar",
|
|
"Data: \"m\"",
|
|
] {
|
|
assert!(s.contains(want), "missing {want:?} in\n{s}");
|
|
}
|
|
let o = h5rs(&["ls", "-v", &format!("{}/grp", f.p("earliest.h5"))]);
|
|
assert!(stdout(&o).contains("Layout: chunked (v1 B-tree index)"));
|
|
}
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// stat
|
|
// ---------------------------------------------------------------------------
|
|
|
|
/// `key: value` lines of an h5stat-style report.
|
|
fn stat_facts(s: &str) -> BTreeMap<String, String> {
|
|
s.lines()
|
|
.filter_map(|l| {
|
|
let (k, v) = l.rsplit_once(':')?;
|
|
Some((k.trim().to_string(), v.trim().to_string()))
|
|
})
|
|
.collect()
|
|
}
|
|
|
|
#[test]
|
|
fn stat_matches_h5stat() {
|
|
let Some(f) = generate() else { return };
|
|
if missing(tool_available("h5stat"), "h5stat") {
|
|
return;
|
|
}
|
|
let keys = [
|
|
"# of unique groups",
|
|
"# of unique datasets",
|
|
"# of unique named datatypes",
|
|
"# of unique links",
|
|
"# of unique other",
|
|
"Max. # of links to object",
|
|
"Max. # of objects in group",
|
|
"Max. rank of datasets",
|
|
"Max. dimension size of 1-D datasets",
|
|
"Total raw data size",
|
|
"Dataset layout counts[COMPACT]",
|
|
"Dataset layout counts[CONTIG]",
|
|
"Dataset layout counts[CHUNKED]",
|
|
"Dataset layout counts[VIRTUAL]",
|
|
"NO filter",
|
|
"GZIP filter",
|
|
"SHUFFLE filter",
|
|
"FLETCHER32 filter",
|
|
"SZIP filter",
|
|
"NBIT filter",
|
|
"SCALEOFFSET filter",
|
|
"USER-DEFINED filter",
|
|
"Max. # of attributes to objects",
|
|
"Total space",
|
|
];
|
|
for name in LIBVERS.iter().chain(&["userblock.h5", "base.h5"]) {
|
|
let p = f.p(name);
|
|
let ours = h5rs(&["stat", &p]);
|
|
assert!(ours.status.success(), "{name}: {ours:?}");
|
|
let reference = run("h5stat", &[&p]);
|
|
assert!(reference.status.success(), "h5stat {name} failed");
|
|
let (o, r) = (stat_facts(&stdout(&ours)), stat_facts(&stdout(&reference)));
|
|
for k in keys {
|
|
// h5stat leaves out the attribute summary when nothing has one.
|
|
let Some(want) = r.get(k) else { continue };
|
|
assert_eq!(o.get(k), Some(want), "{name}: {k}");
|
|
}
|
|
}
|
|
}
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// dump
|
|
// ---------------------------------------------------------------------------
|
|
|
|
#[test]
|
|
fn dump_matches_h5dump() {
|
|
let Some(f) = generate() else { return };
|
|
if missing(tool_available("h5dump"), "h5dump") {
|
|
return;
|
|
}
|
|
for name in LIBVERS.iter().chain(&["userblock.h5", "changed.h5"]) {
|
|
let p = f.p(name);
|
|
for args in [vec![], vec!["-A"]] {
|
|
let mut a = args.clone();
|
|
a.push(p.as_str());
|
|
let ours = h5rs(&[&["dump"], a.as_slice()].concat());
|
|
assert!(ours.status.success(), "{name} {args:?}: {ours:?}");
|
|
let reference = run("h5dump", &a);
|
|
// h5dump names the file as given; h5rs by its file name.
|
|
let r = stdout(&reference).replacen(&p, name, 1);
|
|
assert_eq!(stdout(&ours), r, "{name} {args:?}");
|
|
}
|
|
}
|
|
let p = f.p("latest.h5");
|
|
let ours = h5rs(&["dump", "-d", "/grp/ext2", &p]);
|
|
let reference = run("h5dump", &["-d", "/grp/ext2", &p]);
|
|
assert_eq!(
|
|
stdout(&ours),
|
|
stdout(&reference).replacen(&p, "latest.h5", 1)
|
|
);
|
|
}
|
|
|
|
#[test]
|
|
fn dump_json_values_match_h5py() {
|
|
let Some(f) = generate() else { return };
|
|
for name in LIBVERS {
|
|
let o = h5rs(&["dump", "--json", &f.p(name)]);
|
|
assert!(o.status.success(), "{name}: {o:?}");
|
|
let doc: serde_json::Value = serde_json::from_slice(&o.stdout).unwrap();
|
|
assert_eq!(doc["apiVersion"], "1.1.1");
|
|
let root = doc["root"].as_str().unwrap();
|
|
assert!(doc["groups"][root].is_object());
|
|
let mut by_path = BTreeMap::new();
|
|
for (_, d) in doc["datasets"].as_object().unwrap() {
|
|
for a in d["alias"].as_array().unwrap() {
|
|
by_path.insert(a.as_str().unwrap().to_string(), d.clone());
|
|
}
|
|
}
|
|
let want = f.values[name.trim_end_matches(".h5")].as_object().unwrap();
|
|
assert!(want.len() > 10);
|
|
for (path, v) in want {
|
|
let d = by_path
|
|
.get(path)
|
|
.unwrap_or_else(|| panic!("{name}: no {path}"));
|
|
let got = &d["value"];
|
|
// JSON numbers: compare as f64 (h5py's ints and floats both
|
|
// round-trip exactly through JSON here).
|
|
assert_eq!(flatten(got), flatten(v), "{name}: {path}");
|
|
}
|
|
let links = doc["groups"][root]["links"].as_array().unwrap();
|
|
let soft = links.iter().find(|l| l["title"] == "soft").unwrap();
|
|
assert_eq!(soft["class"], "H5L_TYPE_SOFT");
|
|
assert_eq!(soft["h5path"], "/contig");
|
|
let ext = links.iter().find(|l| l["title"] == "external").unwrap();
|
|
assert_eq!(ext["class"], "H5L_TYPE_EXTERNAL");
|
|
assert_eq!(ext["file"], "other.h5");
|
|
}
|
|
}
|
|
|
|
fn flatten(v: &serde_json::Value) -> Vec<f64> {
|
|
match v {
|
|
serde_json::Value::Array(a) => a.iter().flat_map(flatten).collect(),
|
|
serde_json::Value::Number(n) => vec![n.as_f64().unwrap()],
|
|
other => panic!("not numeric: {other}"),
|
|
}
|
|
}
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// diff
|
|
// ---------------------------------------------------------------------------
|
|
|
|
fn code(o: &Output) -> i32 {
|
|
o.status.code().expect("exit code")
|
|
}
|
|
|
|
#[test]
|
|
fn diff_exit_codes_match_h5diff() {
|
|
let Some(f) = generate() else { return };
|
|
if missing(tool_available("h5diff"), "h5diff") {
|
|
return;
|
|
}
|
|
let (b, same, changed, extra, attr) = (
|
|
f.p("base.h5"),
|
|
f.p("same.h5"),
|
|
f.p("changed.h5"),
|
|
f.p("extra.h5"),
|
|
f.p("attr.h5"),
|
|
);
|
|
let missing_file = f.p("does-not-exist.h5");
|
|
let cases: Vec<(Vec<&str>, i32)> = vec![
|
|
(vec![&b, &same], 0),
|
|
(vec![&b, &b], 0),
|
|
(vec![&b, &changed], 1),
|
|
(vec![&b, &changed, "/d"], 1),
|
|
(vec![&b, &changed, "/g"], 0),
|
|
(vec!["-d", "0.01", &b, &changed], 0),
|
|
(vec!["-d", "0.0001", &b, &changed], 1),
|
|
(vec!["-p", "0.01", &b, &changed], 0),
|
|
(vec!["-p", "1e-9", &b, &changed], 1),
|
|
(vec![&b, &extra], 1),
|
|
(vec![&b, &attr], 1),
|
|
(vec![&b, &attr, "/g"], 0),
|
|
(vec![&b, &same, "/nope"], 2),
|
|
(vec![&b, &missing_file], 2),
|
|
(vec![&b, &same, "/d", "/d"], 0),
|
|
];
|
|
for (args, want) in cases {
|
|
let theirs = code(&run("h5diff", &args));
|
|
assert_eq!(theirs, want, "h5diff {args:?} (test expectation)");
|
|
let o = h5rs(&[&["diff"], args.as_slice()].concat());
|
|
assert_eq!(code(&o), want, "h5rs diff {args:?}: {}", stdout(&o));
|
|
let q = h5rs(&[&["diff", "-q"], args.as_slice()].concat());
|
|
assert_eq!(code(&q), want, "h5rs diff -q {args:?}");
|
|
assert!(q.stdout.is_empty(), "-q printed output");
|
|
}
|
|
// The report lists the differing positions.
|
|
let o = h5rs(&["diff", "-r", &b, &changed, "/d"]);
|
|
let s = stdout(&o);
|
|
assert!(s.contains("[ 0 2 ]") && s.contains("[ 2 3 ]"), "{s}");
|
|
assert!(s.contains("2 difference(s) found"), "{s}");
|
|
}
|
|
|
|
/// Tolerances on 64-bit integers one apart, far beyond f64's integer
|
|
/// precision: compared exactly, as h5diff does.
|
|
#[test]
|
|
fn diff_tolerances_compare_large_integers_exactly() {
|
|
let Some(f) = generate() else { return };
|
|
let (a, b) = (f.p("big1.h5"), f.p("big2.h5"));
|
|
let cases: Vec<(Vec<&str>, i32)> = vec![
|
|
(vec![&a, &b], 1),
|
|
(vec!["-d", "0", &a, &b], 1),
|
|
(vec!["-d", "0.5", &a, &b], 1),
|
|
(vec!["-d", "1", &a, &b], 0),
|
|
(vec!["-d", "0", &a, &b, "/u"], 1),
|
|
(vec!["-p", "0", &a, &b], 1),
|
|
(vec!["-p", "1e-18", &a, &b, "/i"], 1),
|
|
(vec!["-p", "1e-15", &a, &b], 0),
|
|
];
|
|
let have_h5diff = tool_available("h5diff");
|
|
for (args, want) in cases {
|
|
if have_h5diff {
|
|
assert_eq!(code(&run("h5diff", &args)), want, "h5diff {args:?}");
|
|
}
|
|
let o = h5rs(&[&["diff"], args.as_slice()].concat());
|
|
assert_eq!(code(&o), want, "h5rs diff {args:?}: {}", stdout(&o));
|
|
}
|
|
// The reported difference is exact too.
|
|
let s = stdout(&h5rs(&["diff", "-r", "-d", "0", &a, &b, "/u"]));
|
|
assert!(
|
|
s.contains("18446744073709551615 18446744073709551614 1\n"),
|
|
"{s}"
|
|
);
|
|
}
|
|
|
|
/// A soft link is compared as a link (its target path), as h5diff does,
|
|
/// unless `--follow-symlinks`, which compares (and walks into) what it
|
|
/// leads to; two dangling links are then the same.
|
|
#[test]
|
|
fn diff_soft_links_like_h5diff() {
|
|
let Some(f) = generate() else { return };
|
|
let (s1, s2, t1, t2) = (
|
|
f.p("soft1.h5"),
|
|
f.p("soft2.h5"),
|
|
f.p("target1.h5"),
|
|
f.p("target2.h5"),
|
|
);
|
|
let fl = "--follow-symlinks";
|
|
let cases: Vec<(Vec<&str>, i32)> = vec![
|
|
(vec![&s1, &s2, "/g/s"], 0),
|
|
(vec![&s1, &s2, "/g"], 0),
|
|
(vec![&s1, &s2, "/lnk"], 0),
|
|
(vec![&s1, &s2, "/dang"], 0),
|
|
(vec![&s1, &s2, "/lnk/d"], 1),
|
|
(vec![&t1, &t2, "/s"], 1),
|
|
(vec![&t1, &t2, "/s", "/rel"], 1),
|
|
(vec![fl, &s1, &s2, "/g/s"], 1),
|
|
(vec![fl, &s1, &s2, "/g"], 1),
|
|
(vec![fl, &s1, &s2, "/lnk"], 1),
|
|
(vec![fl, &s1, &s2, "/dang"], 0),
|
|
(vec![fl, &s1, &s1], 0),
|
|
(vec![fl, &t1, &t2, "/s"], 0),
|
|
(vec![fl, &t1, &t2, "/s", "/rel"], 0),
|
|
];
|
|
let have_h5diff = tool_available("h5diff");
|
|
for (args, want) in cases {
|
|
if have_h5diff {
|
|
assert_eq!(code(&run("h5diff", &args)), want, "h5diff {args:?}");
|
|
}
|
|
let o = h5rs(&[&["diff"], args.as_slice()].concat());
|
|
assert_eq!(code(&o), want, "h5rs diff {args:?}: {}", stdout(&o));
|
|
}
|
|
}
|
|
|
|
/// An object hard-linked under two names is compared under both, with
|
|
/// everything below it, so a file that shares one object between two names
|
|
/// equals a file that stores two identical copies.
|
|
#[test]
|
|
fn diff_compares_every_hard_link_path() {
|
|
let Some(f) = generate() else { return };
|
|
let (linked, copied, changed) = (
|
|
f.p("hardlinked.h5"),
|
|
f.p("copied.h5"),
|
|
f.p("copied_changed.h5"),
|
|
);
|
|
// A hard-linked dataset: h5diff agrees (exit 0).
|
|
for (a, b) in [(&linked, &copied), (&copied, &linked)] {
|
|
let o = h5rs(&["diff", a, b, "/y"]);
|
|
assert_eq!(code(&o), 0, "{a} {b} /y: {}", stdout(&o));
|
|
if tool_available("h5diff") {
|
|
assert_eq!(code(&run("h5diff", &[a, b, "/y"])), 0, "h5diff /y");
|
|
}
|
|
// The whole file, including a hard-linked group's members (h5diff
|
|
// does not descend a group's second name, so it reports those
|
|
// members as present in one file only and exits 1).
|
|
let o = h5rs(&["diff", a, b]);
|
|
assert_eq!(code(&o), 0, "{a} {b}: {}", stdout(&o));
|
|
assert!(!stdout(&o).contains("exists only"), "{}", stdout(&o));
|
|
}
|
|
// A value under the second name is compared, not skipped.
|
|
let o = h5rs(&["diff", "-r", &linked, &changed]);
|
|
assert_eq!(code(&o), 1, "{}", stdout(&o));
|
|
assert!(
|
|
stdout(&o).contains("dataset: </h/s/e> and </h/s/e>"),
|
|
"{}",
|
|
stdout(&o)
|
|
);
|
|
assert!(!stdout(&o).contains("</g/s/e>"), "{}", stdout(&o));
|
|
}
|
|
|
|
/// Where h5rs deliberately differs from h5diff: objects that cannot be
|
|
/// compared (another shape, another datatype class) are a difference.
|
|
#[test]
|
|
fn diff_counts_incomparable_objects_as_different() {
|
|
let Some(f) = generate() else { return };
|
|
let b = f.p("base.h5");
|
|
for other in ["reshaped.h5", "int.h5"] {
|
|
let o = h5rs(&["diff", &b, &f.p(other)]);
|
|
assert_eq!(code(&o), 1, "{other}: {}", stdout(&o));
|
|
assert!(stdout(&o).contains("Not comparable"), "{other}");
|
|
}
|
|
}
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// check
|
|
// ---------------------------------------------------------------------------
|
|
|
|
#[test]
|
|
fn check_accepts_valid_files() {
|
|
let Some(f) = generate() else { return };
|
|
for name in LIBVERS.iter().chain(&[
|
|
"userblock.h5",
|
|
"base.h5",
|
|
"changed.h5",
|
|
"extra.h5",
|
|
"attr.h5",
|
|
]) {
|
|
let o = h5rs(&["check", "--data", &f.p(name)]);
|
|
assert_eq!(code(&o), 0, "{name}:\n{}", stdout(&o));
|
|
assert!(stdout(&o).contains("no problems found"));
|
|
}
|
|
// The newest format's checksummed structures were all verified.
|
|
let s = stdout(&h5rs(&["check", &f.p("latest.h5")]));
|
|
assert!(s.contains("superblock 1, v2 object headers 33,"), "{s}");
|
|
assert!(s.contains("chunk indexes 5"), "{s}");
|
|
}
|
|
|
|
/// Signatures of checksummed structures, and where each one's checksum
|
|
/// is: found by trying every end offset (the checksum is the Jenkins
|
|
/// lookup3 hash of the bytes before it). A fractal heap direct block's
|
|
/// checksum is instead stored near its start, computed over the whole block
|
|
/// with the field zeroed.
|
|
fn checksummed_structures(data: &[u8]) -> Vec<(&'static str, usize, usize)> {
|
|
const SIGS: [&str; 13] = [
|
|
"OHDR", "OCHK", "BTHD", "BTIN", "BTLF", "FRHP", "FHIB", "EAHD", "EAIB", "EASB", "EADB",
|
|
"FAHD", "FADB",
|
|
];
|
|
let mut found = Vec::new();
|
|
for start in 0..data.len().saturating_sub(4) {
|
|
let sig = &data[start..start + 4];
|
|
if let Some(name) = SIGS.iter().find(|s| s.as_bytes() == sig) {
|
|
let limit = data.len().min(start + 65536);
|
|
for end in start + 8..limit.saturating_sub(4) {
|
|
let stored = u32::from_le_bytes(data[end..end + 4].try_into().unwrap());
|
|
if jenkins_lookup3(&data[start..end]) == stored {
|
|
found.push((*name, start, end));
|
|
break;
|
|
}
|
|
}
|
|
} else if sig == b"FHDB" {
|
|
// version(1) heap address(8) block offset(1..8) checksum(4)
|
|
'found: for boff in 1..=8usize {
|
|
let at = start + 5 + 8 + boff;
|
|
let Some(stored) = data.get(at..at + 4) else {
|
|
break;
|
|
};
|
|
let stored = u32::from_le_bytes(stored.try_into().unwrap());
|
|
for size in [512usize, 1024, 2048, 4096, 8192, 16384, 32768, 65536] {
|
|
let Some(block) = data.get(start..start + size) else {
|
|
break;
|
|
};
|
|
let mut b = block.to_vec();
|
|
b[at - start..at - start + 4].fill(0);
|
|
if jenkins_lookup3(&b) == stored {
|
|
found.push(("FHDB", start, at));
|
|
break 'found;
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
found
|
|
}
|
|
|
|
#[test]
|
|
fn check_flags_every_corrupted_checksum() {
|
|
let Some(f) = generate() else { return };
|
|
let path = f.path("latest.h5");
|
|
let data = std::fs::read(&path).unwrap();
|
|
let structures = checksummed_structures(&data);
|
|
let kinds: std::collections::BTreeSet<&str> = structures.iter().map(|s| s.0).collect();
|
|
for want in [
|
|
"OHDR", "OCHK", "BTHD", "BTLF", "FRHP", "FHDB", "EAHD", "EAIB", "EADB", "FAHD", "FADB",
|
|
] {
|
|
assert!(kinds.contains(want), "test file has no {want}: {kinds:?}");
|
|
}
|
|
let bad = f.path("bad.h5");
|
|
for (name, start, at) in structures {
|
|
let mut d = data.clone();
|
|
d[at] ^= 0x01;
|
|
std::fs::write(&bad, &d).unwrap();
|
|
let o = h5rs(&["check", &bad.to_string_lossy()]);
|
|
let s = stdout(&o);
|
|
assert_eq!(
|
|
code(&o),
|
|
1,
|
|
"{name} at {start:#x}: checksum flip not flagged\n{s}"
|
|
);
|
|
assert!(
|
|
s.lines()
|
|
.any(|l| l.starts_with("problem: 0x") && l.contains("checksum")),
|
|
"{name} at {start:#x}: no checksum problem reported\n{s}"
|
|
);
|
|
}
|
|
}
|
|
|
|
#[test]
|
|
fn check_flags_a_truncated_file() {
|
|
let Some(f) = generate() else { return };
|
|
let data = std::fs::read(f.path("latest.h5")).unwrap();
|
|
let bad = f.path("truncated.h5");
|
|
std::fs::write(&bad, &data[..data.len() * 3 / 4]).unwrap();
|
|
let o = h5rs(&["check", &bad.to_string_lossy()]);
|
|
assert_eq!(code(&o), 1);
|
|
assert!(stdout(&o).contains("file is truncated"), "{}", stdout(&o));
|
|
}
|
|
|
|
/// A v1 B-tree chunk index has no checksum; its keys are checked against
|
|
/// the dataset: a chunk offset that is not a multiple of the chunk size.
|
|
#[test]
|
|
fn check_flags_a_misaligned_chunk() {
|
|
let Some(f) = generate() else { return };
|
|
let path = f.path("earliest.h5");
|
|
let mut data = std::fs::read(&path).unwrap();
|
|
// The leaf chunk B-tree of /grp/gz (10 deflated chunks of 100 i32, so
|
|
// each stored in under 400 bytes): "TREE" type(1)=1 level(1)=0
|
|
// entries(2) left(8) right(8), then key, child, key, child, ... where a
|
|
// rank-1 chunk key is size(4) mask(4) offsets(2 x 8) and a child is 8.
|
|
// Key 1 starts at 24 + 24 + 8 = 56; its dimension-0 offset (100) at 64.
|
|
let mut patched = false;
|
|
for start in 0..data.len() - 72 {
|
|
if &data[start..start + 4] == b"TREE" && data[start + 4] == 1 && data[start + 5] == 0 {
|
|
let entries = u16::from_le_bytes(data[start + 6..start + 8].try_into().unwrap());
|
|
let size0 = u32::from_le_bytes(data[start + 24..start + 28].try_into().unwrap());
|
|
let off1 = u64::from_le_bytes(data[start + 64..start + 72].try_into().unwrap());
|
|
if entries == 10 && size0 < 400 && off1 == 100 {
|
|
data[start + 64..start + 72].copy_from_slice(&103u64.to_le_bytes());
|
|
patched = true;
|
|
break;
|
|
}
|
|
}
|
|
}
|
|
assert!(patched, "no chunk B-tree found for /grp/gz");
|
|
let bad = f.path("misaligned.h5");
|
|
std::fs::write(&bad, &data).unwrap();
|
|
let o = h5rs(&["check", &bad.to_string_lossy()]);
|
|
let s = stdout(&o);
|
|
assert_eq!(code(&o), 1, "{s}");
|
|
assert!(
|
|
s.contains("/grp/gz: chunk at [103, 0] offset 103 in dimension 0 is not a multiple of the chunk size 100"),
|
|
"{s}"
|
|
);
|
|
}
|
|
|
|
#[test]
|
|
fn every_subcommand_rejects_a_non_hdf5_file_cleanly() {
|
|
let dir = tempfile::tempdir().unwrap();
|
|
let p = dir.path().join("junk.h5");
|
|
std::fs::write(&p, b"this is not an HDF5 file at all").unwrap();
|
|
let p = p.to_string_lossy().into_owned();
|
|
for args in [
|
|
vec!["ls", p.as_str()],
|
|
vec!["dump", p.as_str()],
|
|
vec!["stat", p.as_str()],
|
|
vec!["diff", p.as_str(), p.as_str()],
|
|
] {
|
|
let o = h5rs(&args);
|
|
assert_eq!(code(&o), 2, "{args:?}");
|
|
assert!(String::from_utf8_lossy(&o.stderr).contains("not an HDF5 file"));
|
|
}
|
|
let o = h5rs(&["check", &p]);
|
|
assert_eq!(code(&o), 1);
|
|
assert!(stdout(&o).contains("no HDF5 signature"));
|
|
assert_eq!(code(&h5rs(&["nonsense"])), 2);
|
|
assert_eq!(code(&h5rs(&["ls"])), 2);
|
|
assert_eq!(code(&h5rs(&["--help"])), 0);
|
|
}
|