osobhandClaude Opus 5.5
f325d111f3
With a tolerance, integers were converted to f64 before comparing, so int64/uint64 values above 2^53 that differ compared equal: -d 0 on 2^60 and 2^60 + 1 exited 0, where h5diff exits 1. Integer pairs are now compared in i128 (the delta against floor(D), the relative quotient from an exact difference), and the report prints the exact difference. h5diff compares exactly when -p is below the f64 epsilon (2^60 and 2^60 + 1 differ at -p 1e-18, and nextafter(2, 0) and 2 at -p 1.5e-16); h5rs now does the same. Co-Authored-By: Claude Opus 5.5 (1M context) <[email protected]>
638 lines
23 KiB
Rust
638 lines
23 KiB
Rust
//! `h5rs` against libhdf5's own tools, on files h5py writes
|
|
//! (`tests/gen_files.py`): `ls` against h5ls, `stat` against h5stat, `dump`
|
|
//! against h5dump, `diff` exit codes against h5diff, `dump --json` values
|
|
//! against h5py, and `check` on valid and deliberately corrupted files.
|
|
//!
|
|
//! Needs python3 with h5py (`CLAWHDF5_PYTHON`) and, for the comparisons,
|
|
//! the libhdf5 command-line tools (h5ls, h5stat, h5dump, h5diff) on `PATH`.
|
|
//! Each test skips when what it needs is missing, unless
|
|
//! `CLAWHDF5_REQUIRE_INTEROP=1`, which turns a skip into a failure.
|
|
|
|
use std::collections::BTreeMap;
|
|
use std::path::{Path, PathBuf};
|
|
use std::process::{Command, Output};
|
|
|
|
use clawhdf5_format::checksum::jenkins_lookup3;
|
|
|
|
fn python() -> String {
|
|
std::env::var("CLAWHDF5_PYTHON").unwrap_or_else(|_| "python3".to_string())
|
|
}
|
|
|
|
fn interop_required() -> bool {
|
|
std::env::var("CLAWHDF5_REQUIRE_INTEROP").is_ok_and(|v| v == "1")
|
|
}
|
|
|
|
fn python_available() -> bool {
|
|
Command::new(python())
|
|
.args(["-c", "import h5py, numpy"])
|
|
.output()
|
|
.map(|o| o.status.success())
|
|
.unwrap_or(false)
|
|
}
|
|
|
|
fn tool_available(name: &str) -> bool {
|
|
Command::new(name)
|
|
.arg("--version")
|
|
.output()
|
|
.map(|o| o.status.success())
|
|
.unwrap_or(false)
|
|
}
|
|
|
|
/// Skip (return true) when `what` is unavailable, or fail when interop is
|
|
/// required.
|
|
fn missing(ok: bool, what: &str) -> bool {
|
|
if ok {
|
|
return false;
|
|
}
|
|
assert!(
|
|
!interop_required(),
|
|
"CLAWHDF5_REQUIRE_INTEROP=1 but {what} is not available"
|
|
);
|
|
eprintln!("SKIP: {what} not available");
|
|
true
|
|
}
|
|
|
|
fn h5rs(args: &[&str]) -> Output {
|
|
Command::new(env!("CARGO_BIN_EXE_h5rs"))
|
|
.args(args)
|
|
.output()
|
|
.expect("run h5rs")
|
|
}
|
|
|
|
fn run(tool: &str, args: &[&str]) -> Output {
|
|
Command::new(tool).args(args).output().expect("run tool")
|
|
}
|
|
|
|
fn stdout(o: &Output) -> String {
|
|
String::from_utf8_lossy(&o.stdout).into_owned()
|
|
}
|
|
|
|
/// The generated files and what h5py read from them.
|
|
struct Files {
|
|
dir: tempfile::TempDir,
|
|
values: serde_json::Value,
|
|
}
|
|
|
|
impl Files {
|
|
fn path(&self, name: &str) -> PathBuf {
|
|
self.dir.path().join(name)
|
|
}
|
|
|
|
fn p(&self, name: &str) -> String {
|
|
self.path(name).to_string_lossy().into_owned()
|
|
}
|
|
}
|
|
|
|
fn generate() -> Option<Files> {
|
|
if missing(python_available(), "python3 with h5py") {
|
|
return None;
|
|
}
|
|
let dir = tempfile::tempdir().unwrap();
|
|
let script = Path::new(env!("CARGO_MANIFEST_DIR")).join("tests/gen_files.py");
|
|
let out = Command::new(python())
|
|
.arg(&script)
|
|
.arg(dir.path())
|
|
.output()
|
|
.expect("run gen_files.py");
|
|
assert!(
|
|
out.status.success(),
|
|
"gen_files.py failed:\n{}",
|
|
String::from_utf8_lossy(&out.stderr)
|
|
);
|
|
let values = serde_json::from_slice(&out.stdout).expect("gen_files.py output");
|
|
Some(Files { dir, values })
|
|
}
|
|
|
|
const LIBVERS: [&str; 2] = ["earliest.h5", "latest.h5"];
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// ls
|
|
// ---------------------------------------------------------------------------
|
|
|
|
/// Every h5ls line must be the start of the matching h5rs line (h5rs adds
|
|
/// the datatype after a dataset's shape).
|
|
fn assert_ls_like(ours: &str, reference: &str, what: &str) {
|
|
let ours: Vec<&str> = ours.lines().collect();
|
|
let refs: Vec<&str> = reference.lines().collect();
|
|
assert_eq!(
|
|
ours.len(),
|
|
refs.len(),
|
|
"{what}: line count\nh5rs:\n{}\nh5ls:\n{}",
|
|
ours.join("\n"),
|
|
refs.join("\n")
|
|
);
|
|
for (o, r) in ours.iter().zip(&refs) {
|
|
let r = r.trim_end();
|
|
assert!(
|
|
o.starts_with(r) && (o.len() == r.len() || r.contains("Dataset {")),
|
|
"{what}:\n h5rs: {o}\n h5ls: {r}"
|
|
);
|
|
}
|
|
}
|
|
|
|
#[test]
|
|
fn ls_matches_h5ls() {
|
|
let Some(f) = generate() else { return };
|
|
if missing(tool_available("h5ls"), "h5ls") {
|
|
return;
|
|
}
|
|
for name in LIBVERS.iter().chain(&["userblock.h5"]) {
|
|
let p = f.p(name);
|
|
for (args, what) in [(vec!["-r"], "recursive"), (vec![], "root")] {
|
|
let mut a: Vec<&str> = args.clone();
|
|
a.push(&p);
|
|
let ours = h5rs(&[&["ls"], a.as_slice()].concat());
|
|
assert!(ours.status.success(), "{name} {what}: {:?}", ours);
|
|
let reference = run("h5ls", &a);
|
|
assert_ls_like(
|
|
&stdout(&ours),
|
|
&stdout(&reference),
|
|
&format!("{name} {what}"),
|
|
);
|
|
}
|
|
}
|
|
// A group, and a dataset, named after the file.
|
|
for obj in ["/grp", "/grp/sub", "/contig", "/grp/ext2"] {
|
|
let arg = format!("{}{obj}", f.p("latest.h5"));
|
|
let ours = h5rs(&["ls", &arg]);
|
|
let reference = run("h5ls", &[&arg]);
|
|
assert_ls_like(&stdout(&ours), &stdout(&reference), &arg);
|
|
let ours = h5rs(&["ls", "-r", &arg]);
|
|
let reference = run("h5ls", &["-r", &arg]);
|
|
assert_ls_like(&stdout(&ours), &stdout(&reference), &format!("-r {arg}"));
|
|
}
|
|
}
|
|
|
|
#[test]
|
|
fn ls_verbose_describes_layout_filters_and_attributes() {
|
|
let Some(f) = generate() else { return };
|
|
let o = h5rs(&["ls", "-r", "-v", &f.p("latest.h5")]);
|
|
assert!(o.status.success());
|
|
let s = stdout(&o);
|
|
for want in [
|
|
"Layout: chunked (extensible array index)",
|
|
"Layout: chunked (v2 B-tree index)",
|
|
"Layout: chunked (fixed array index)",
|
|
"Layout: chunked (single chunk index)",
|
|
"Filter-0: shuffle-2",
|
|
"Filter-1: deflate-1",
|
|
"Filter-2: fletcher32-3",
|
|
"Chunks: {100} 400 bytes",
|
|
"Attribute: units scalar",
|
|
"Data: \"m\"",
|
|
] {
|
|
assert!(s.contains(want), "missing {want:?} in\n{s}");
|
|
}
|
|
let o = h5rs(&["ls", "-v", &format!("{}/grp", f.p("earliest.h5"))]);
|
|
assert!(stdout(&o).contains("Layout: chunked (v1 B-tree index)"));
|
|
}
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// stat
|
|
// ---------------------------------------------------------------------------
|
|
|
|
/// `key: value` lines of an h5stat-style report.
|
|
fn stat_facts(s: &str) -> BTreeMap<String, String> {
|
|
s.lines()
|
|
.filter_map(|l| {
|
|
let (k, v) = l.rsplit_once(':')?;
|
|
Some((k.trim().to_string(), v.trim().to_string()))
|
|
})
|
|
.collect()
|
|
}
|
|
|
|
#[test]
|
|
fn stat_matches_h5stat() {
|
|
let Some(f) = generate() else { return };
|
|
if missing(tool_available("h5stat"), "h5stat") {
|
|
return;
|
|
}
|
|
let keys = [
|
|
"# of unique groups",
|
|
"# of unique datasets",
|
|
"# of unique named datatypes",
|
|
"# of unique links",
|
|
"# of unique other",
|
|
"Max. # of links to object",
|
|
"Max. # of objects in group",
|
|
"Max. rank of datasets",
|
|
"Max. dimension size of 1-D datasets",
|
|
"Total raw data size",
|
|
"Dataset layout counts[COMPACT]",
|
|
"Dataset layout counts[CONTIG]",
|
|
"Dataset layout counts[CHUNKED]",
|
|
"Dataset layout counts[VIRTUAL]",
|
|
"NO filter",
|
|
"GZIP filter",
|
|
"SHUFFLE filter",
|
|
"FLETCHER32 filter",
|
|
"SZIP filter",
|
|
"NBIT filter",
|
|
"SCALEOFFSET filter",
|
|
"USER-DEFINED filter",
|
|
"Max. # of attributes to objects",
|
|
"Total space",
|
|
];
|
|
for name in LIBVERS.iter().chain(&["userblock.h5", "base.h5"]) {
|
|
let p = f.p(name);
|
|
let ours = h5rs(&["stat", &p]);
|
|
assert!(ours.status.success(), "{name}: {ours:?}");
|
|
let reference = run("h5stat", &[&p]);
|
|
assert!(reference.status.success(), "h5stat {name} failed");
|
|
let (o, r) = (stat_facts(&stdout(&ours)), stat_facts(&stdout(&reference)));
|
|
for k in keys {
|
|
// h5stat leaves out the attribute summary when nothing has one.
|
|
let Some(want) = r.get(k) else { continue };
|
|
assert_eq!(o.get(k), Some(want), "{name}: {k}");
|
|
}
|
|
}
|
|
}
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// dump
|
|
// ---------------------------------------------------------------------------
|
|
|
|
#[test]
|
|
fn dump_matches_h5dump() {
|
|
let Some(f) = generate() else { return };
|
|
if missing(tool_available("h5dump"), "h5dump") {
|
|
return;
|
|
}
|
|
for name in LIBVERS.iter().chain(&["userblock.h5", "changed.h5"]) {
|
|
let p = f.p(name);
|
|
for args in [vec![], vec!["-A"]] {
|
|
let mut a = args.clone();
|
|
a.push(p.as_str());
|
|
let ours = h5rs(&[&["dump"], a.as_slice()].concat());
|
|
assert!(ours.status.success(), "{name} {args:?}: {ours:?}");
|
|
let reference = run("h5dump", &a);
|
|
// h5dump names the file as given; h5rs by its file name.
|
|
let r = stdout(&reference).replacen(&p, name, 1);
|
|
assert_eq!(stdout(&ours), r, "{name} {args:?}");
|
|
}
|
|
}
|
|
let p = f.p("latest.h5");
|
|
let ours = h5rs(&["dump", "-d", "/grp/ext2", &p]);
|
|
let reference = run("h5dump", &["-d", "/grp/ext2", &p]);
|
|
assert_eq!(
|
|
stdout(&ours),
|
|
stdout(&reference).replacen(&p, "latest.h5", 1)
|
|
);
|
|
}
|
|
|
|
#[test]
|
|
fn dump_json_values_match_h5py() {
|
|
let Some(f) = generate() else { return };
|
|
for name in LIBVERS {
|
|
let o = h5rs(&["dump", "--json", &f.p(name)]);
|
|
assert!(o.status.success(), "{name}: {o:?}");
|
|
let doc: serde_json::Value = serde_json::from_slice(&o.stdout).unwrap();
|
|
assert_eq!(doc["apiVersion"], "1.1.1");
|
|
let root = doc["root"].as_str().unwrap();
|
|
assert!(doc["groups"][root].is_object());
|
|
let mut by_path = BTreeMap::new();
|
|
for (_, d) in doc["datasets"].as_object().unwrap() {
|
|
for a in d["alias"].as_array().unwrap() {
|
|
by_path.insert(a.as_str().unwrap().to_string(), d.clone());
|
|
}
|
|
}
|
|
let want = f.values[name.trim_end_matches(".h5")].as_object().unwrap();
|
|
assert!(want.len() > 10);
|
|
for (path, v) in want {
|
|
let d = by_path
|
|
.get(path)
|
|
.unwrap_or_else(|| panic!("{name}: no {path}"));
|
|
let got = &d["value"];
|
|
// JSON numbers: compare as f64 (h5py's ints and floats both
|
|
// round-trip exactly through JSON here).
|
|
assert_eq!(flatten(got), flatten(v), "{name}: {path}");
|
|
}
|
|
let links = doc["groups"][root]["links"].as_array().unwrap();
|
|
let soft = links.iter().find(|l| l["title"] == "soft").unwrap();
|
|
assert_eq!(soft["class"], "H5L_TYPE_SOFT");
|
|
assert_eq!(soft["h5path"], "/contig");
|
|
let ext = links.iter().find(|l| l["title"] == "external").unwrap();
|
|
assert_eq!(ext["class"], "H5L_TYPE_EXTERNAL");
|
|
assert_eq!(ext["file"], "other.h5");
|
|
}
|
|
}
|
|
|
|
fn flatten(v: &serde_json::Value) -> Vec<f64> {
|
|
match v {
|
|
serde_json::Value::Array(a) => a.iter().flat_map(flatten).collect(),
|
|
serde_json::Value::Number(n) => vec![n.as_f64().unwrap()],
|
|
other => panic!("not numeric: {other}"),
|
|
}
|
|
}
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// diff
|
|
// ---------------------------------------------------------------------------
|
|
|
|
fn code(o: &Output) -> i32 {
|
|
o.status.code().expect("exit code")
|
|
}
|
|
|
|
#[test]
|
|
fn diff_exit_codes_match_h5diff() {
|
|
let Some(f) = generate() else { return };
|
|
if missing(tool_available("h5diff"), "h5diff") {
|
|
return;
|
|
}
|
|
let (b, same, changed, extra, attr) = (
|
|
f.p("base.h5"),
|
|
f.p("same.h5"),
|
|
f.p("changed.h5"),
|
|
f.p("extra.h5"),
|
|
f.p("attr.h5"),
|
|
);
|
|
let missing_file = f.p("does-not-exist.h5");
|
|
let cases: Vec<(Vec<&str>, i32)> = vec![
|
|
(vec![&b, &same], 0),
|
|
(vec![&b, &b], 0),
|
|
(vec![&b, &changed], 1),
|
|
(vec![&b, &changed, "/d"], 1),
|
|
(vec![&b, &changed, "/g"], 0),
|
|
(vec!["-d", "0.01", &b, &changed], 0),
|
|
(vec!["-d", "0.0001", &b, &changed], 1),
|
|
(vec!["-p", "0.01", &b, &changed], 0),
|
|
(vec!["-p", "1e-9", &b, &changed], 1),
|
|
(vec![&b, &extra], 1),
|
|
(vec![&b, &attr], 1),
|
|
(vec![&b, &attr, "/g"], 0),
|
|
(vec![&b, &same, "/nope"], 2),
|
|
(vec![&b, &missing_file], 2),
|
|
(vec![&b, &same, "/d", "/d"], 0),
|
|
];
|
|
for (args, want) in cases {
|
|
let theirs = code(&run("h5diff", &args));
|
|
assert_eq!(theirs, want, "h5diff {args:?} (test expectation)");
|
|
let o = h5rs(&[&["diff"], args.as_slice()].concat());
|
|
assert_eq!(code(&o), want, "h5rs diff {args:?}: {}", stdout(&o));
|
|
let q = h5rs(&[&["diff", "-q"], args.as_slice()].concat());
|
|
assert_eq!(code(&q), want, "h5rs diff -q {args:?}");
|
|
assert!(q.stdout.is_empty(), "-q printed output");
|
|
}
|
|
// The report lists the differing positions.
|
|
let o = h5rs(&["diff", "-r", &b, &changed, "/d"]);
|
|
let s = stdout(&o);
|
|
assert!(s.contains("[ 0 2 ]") && s.contains("[ 2 3 ]"), "{s}");
|
|
assert!(s.contains("2 difference(s) found"), "{s}");
|
|
}
|
|
|
|
/// Tolerances on 64-bit integers one apart, far beyond f64's integer
|
|
/// precision: compared exactly, as h5diff does.
|
|
#[test]
|
|
fn diff_tolerances_compare_large_integers_exactly() {
|
|
let Some(f) = generate() else { return };
|
|
let (a, b) = (f.p("big1.h5"), f.p("big2.h5"));
|
|
let cases: Vec<(Vec<&str>, i32)> = vec![
|
|
(vec![&a, &b], 1),
|
|
(vec!["-d", "0", &a, &b], 1),
|
|
(vec!["-d", "0.5", &a, &b], 1),
|
|
(vec!["-d", "1", &a, &b], 0),
|
|
(vec!["-d", "0", &a, &b, "/u"], 1),
|
|
(vec!["-p", "0", &a, &b], 1),
|
|
(vec!["-p", "1e-18", &a, &b, "/i"], 1),
|
|
(vec!["-p", "1e-15", &a, &b], 0),
|
|
];
|
|
let have_h5diff = tool_available("h5diff");
|
|
for (args, want) in cases {
|
|
if have_h5diff {
|
|
assert_eq!(code(&run("h5diff", &args)), want, "h5diff {args:?}");
|
|
}
|
|
let o = h5rs(&[&["diff"], args.as_slice()].concat());
|
|
assert_eq!(code(&o), want, "h5rs diff {args:?}: {}", stdout(&o));
|
|
}
|
|
// The reported difference is exact too.
|
|
let s = stdout(&h5rs(&["diff", "-r", "-d", "0", &a, &b, "/u"]));
|
|
assert!(
|
|
s.contains("18446744073709551615 18446744073709551614 1\n"),
|
|
"{s}"
|
|
);
|
|
}
|
|
|
|
/// An object hard-linked under two names is compared under both, with
|
|
/// everything below it, so a file that shares one object between two names
|
|
/// equals a file that stores two identical copies.
|
|
#[test]
|
|
fn diff_compares_every_hard_link_path() {
|
|
let Some(f) = generate() else { return };
|
|
let (linked, copied, changed) = (
|
|
f.p("hardlinked.h5"),
|
|
f.p("copied.h5"),
|
|
f.p("copied_changed.h5"),
|
|
);
|
|
// A hard-linked dataset: h5diff agrees (exit 0).
|
|
for (a, b) in [(&linked, &copied), (&copied, &linked)] {
|
|
let o = h5rs(&["diff", a, b, "/y"]);
|
|
assert_eq!(code(&o), 0, "{a} {b} /y: {}", stdout(&o));
|
|
if tool_available("h5diff") {
|
|
assert_eq!(code(&run("h5diff", &[a, b, "/y"])), 0, "h5diff /y");
|
|
}
|
|
// The whole file, including a hard-linked group's members (h5diff
|
|
// does not descend a group's second name, so it reports those
|
|
// members as present in one file only and exits 1).
|
|
let o = h5rs(&["diff", a, b]);
|
|
assert_eq!(code(&o), 0, "{a} {b}: {}", stdout(&o));
|
|
assert!(!stdout(&o).contains("exists only"), "{}", stdout(&o));
|
|
}
|
|
// A value under the second name is compared, not skipped.
|
|
let o = h5rs(&["diff", "-r", &linked, &changed]);
|
|
assert_eq!(code(&o), 1, "{}", stdout(&o));
|
|
assert!(
|
|
stdout(&o).contains("dataset: </h/s/e> and </h/s/e>"),
|
|
"{}",
|
|
stdout(&o)
|
|
);
|
|
assert!(!stdout(&o).contains("</g/s/e>"), "{}", stdout(&o));
|
|
}
|
|
|
|
/// Where h5rs deliberately differs from h5diff: objects that cannot be
|
|
/// compared (another shape, another datatype class) are a difference.
|
|
#[test]
|
|
fn diff_counts_incomparable_objects_as_different() {
|
|
let Some(f) = generate() else { return };
|
|
let b = f.p("base.h5");
|
|
for other in ["reshaped.h5", "int.h5"] {
|
|
let o = h5rs(&["diff", &b, &f.p(other)]);
|
|
assert_eq!(code(&o), 1, "{other}: {}", stdout(&o));
|
|
assert!(stdout(&o).contains("Not comparable"), "{other}");
|
|
}
|
|
}
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// check
|
|
// ---------------------------------------------------------------------------
|
|
|
|
#[test]
|
|
fn check_accepts_valid_files() {
|
|
let Some(f) = generate() else { return };
|
|
for name in LIBVERS.iter().chain(&[
|
|
"userblock.h5",
|
|
"base.h5",
|
|
"changed.h5",
|
|
"extra.h5",
|
|
"attr.h5",
|
|
]) {
|
|
let o = h5rs(&["check", "--data", &f.p(name)]);
|
|
assert_eq!(code(&o), 0, "{name}:\n{}", stdout(&o));
|
|
assert!(stdout(&o).contains("no problems found"));
|
|
}
|
|
// The newest format's checksummed structures were all verified.
|
|
let s = stdout(&h5rs(&["check", &f.p("latest.h5")]));
|
|
assert!(s.contains("superblock 1, v2 object headers 33,"), "{s}");
|
|
assert!(s.contains("chunk indexes 5"), "{s}");
|
|
}
|
|
|
|
/// Signatures of checksummed structures, and where each one's checksum
|
|
/// is: found by trying every end offset (the checksum is the Jenkins
|
|
/// lookup3 hash of the bytes before it). A fractal heap direct block's
|
|
/// checksum is instead stored near its start, computed over the whole block
|
|
/// with the field zeroed.
|
|
fn checksummed_structures(data: &[u8]) -> Vec<(&'static str, usize, usize)> {
|
|
const SIGS: [&str; 13] = [
|
|
"OHDR", "OCHK", "BTHD", "BTIN", "BTLF", "FRHP", "FHIB", "EAHD", "EAIB", "EASB", "EADB",
|
|
"FAHD", "FADB",
|
|
];
|
|
let mut found = Vec::new();
|
|
for start in 0..data.len().saturating_sub(4) {
|
|
let sig = &data[start..start + 4];
|
|
if let Some(name) = SIGS.iter().find(|s| s.as_bytes() == sig) {
|
|
let limit = data.len().min(start + 65536);
|
|
for end in start + 8..limit.saturating_sub(4) {
|
|
let stored = u32::from_le_bytes(data[end..end + 4].try_into().unwrap());
|
|
if jenkins_lookup3(&data[start..end]) == stored {
|
|
found.push((*name, start, end));
|
|
break;
|
|
}
|
|
}
|
|
} else if sig == b"FHDB" {
|
|
// version(1) heap address(8) block offset(1..8) checksum(4)
|
|
'found: for boff in 1..=8usize {
|
|
let at = start + 5 + 8 + boff;
|
|
let Some(stored) = data.get(at..at + 4) else {
|
|
break;
|
|
};
|
|
let stored = u32::from_le_bytes(stored.try_into().unwrap());
|
|
for size in [512usize, 1024, 2048, 4096, 8192, 16384, 32768, 65536] {
|
|
let Some(block) = data.get(start..start + size) else {
|
|
break;
|
|
};
|
|
let mut b = block.to_vec();
|
|
b[at - start..at - start + 4].fill(0);
|
|
if jenkins_lookup3(&b) == stored {
|
|
found.push(("FHDB", start, at));
|
|
break 'found;
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
found
|
|
}
|
|
|
|
#[test]
|
|
fn check_flags_every_corrupted_checksum() {
|
|
let Some(f) = generate() else { return };
|
|
let path = f.path("latest.h5");
|
|
let data = std::fs::read(&path).unwrap();
|
|
let structures = checksummed_structures(&data);
|
|
let kinds: std::collections::BTreeSet<&str> = structures.iter().map(|s| s.0).collect();
|
|
for want in [
|
|
"OHDR", "OCHK", "BTHD", "BTLF", "FRHP", "FHDB", "EAHD", "EAIB", "EADB", "FAHD", "FADB",
|
|
] {
|
|
assert!(kinds.contains(want), "test file has no {want}: {kinds:?}");
|
|
}
|
|
let bad = f.path("bad.h5");
|
|
for (name, start, at) in structures {
|
|
let mut d = data.clone();
|
|
d[at] ^= 0x01;
|
|
std::fs::write(&bad, &d).unwrap();
|
|
let o = h5rs(&["check", &bad.to_string_lossy()]);
|
|
let s = stdout(&o);
|
|
assert_eq!(
|
|
code(&o),
|
|
1,
|
|
"{name} at {start:#x}: checksum flip not flagged\n{s}"
|
|
);
|
|
assert!(
|
|
s.lines()
|
|
.any(|l| l.starts_with("problem: 0x") && l.contains("checksum")),
|
|
"{name} at {start:#x}: no checksum problem reported\n{s}"
|
|
);
|
|
}
|
|
}
|
|
|
|
#[test]
|
|
fn check_flags_a_truncated_file() {
|
|
let Some(f) = generate() else { return };
|
|
let data = std::fs::read(f.path("latest.h5")).unwrap();
|
|
let bad = f.path("truncated.h5");
|
|
std::fs::write(&bad, &data[..data.len() * 3 / 4]).unwrap();
|
|
let o = h5rs(&["check", &bad.to_string_lossy()]);
|
|
assert_eq!(code(&o), 1);
|
|
assert!(stdout(&o).contains("file is truncated"), "{}", stdout(&o));
|
|
}
|
|
|
|
/// A v1 B-tree chunk index has no checksum; its keys are checked against
|
|
/// the dataset: a chunk offset that is not a multiple of the chunk size.
|
|
#[test]
|
|
fn check_flags_a_misaligned_chunk() {
|
|
let Some(f) = generate() else { return };
|
|
let path = f.path("earliest.h5");
|
|
let mut data = std::fs::read(&path).unwrap();
|
|
// The leaf chunk B-tree of /grp/gz (10 deflated chunks of 100 i32, so
|
|
// each stored in under 400 bytes): "TREE" type(1)=1 level(1)=0
|
|
// entries(2) left(8) right(8), then key, child, key, child, ... where a
|
|
// rank-1 chunk key is size(4) mask(4) offsets(2 x 8) and a child is 8.
|
|
// Key 1 starts at 24 + 24 + 8 = 56; its dimension-0 offset (100) at 64.
|
|
let mut patched = false;
|
|
for start in 0..data.len() - 72 {
|
|
if &data[start..start + 4] == b"TREE" && data[start + 4] == 1 && data[start + 5] == 0 {
|
|
let entries = u16::from_le_bytes(data[start + 6..start + 8].try_into().unwrap());
|
|
let size0 = u32::from_le_bytes(data[start + 24..start + 28].try_into().unwrap());
|
|
let off1 = u64::from_le_bytes(data[start + 64..start + 72].try_into().unwrap());
|
|
if entries == 10 && size0 < 400 && off1 == 100 {
|
|
data[start + 64..start + 72].copy_from_slice(&103u64.to_le_bytes());
|
|
patched = true;
|
|
break;
|
|
}
|
|
}
|
|
}
|
|
assert!(patched, "no chunk B-tree found for /grp/gz");
|
|
let bad = f.path("misaligned.h5");
|
|
std::fs::write(&bad, &data).unwrap();
|
|
let o = h5rs(&["check", &bad.to_string_lossy()]);
|
|
let s = stdout(&o);
|
|
assert_eq!(code(&o), 1, "{s}");
|
|
assert!(
|
|
s.contains("/grp/gz: chunk at [103, 0] offset 103 in dimension 0 is not a multiple of the chunk size 100"),
|
|
"{s}"
|
|
);
|
|
}
|
|
|
|
#[test]
|
|
fn every_subcommand_rejects_a_non_hdf5_file_cleanly() {
|
|
let dir = tempfile::tempdir().unwrap();
|
|
let p = dir.path().join("junk.h5");
|
|
std::fs::write(&p, b"this is not an HDF5 file at all").unwrap();
|
|
let p = p.to_string_lossy().into_owned();
|
|
for args in [
|
|
vec!["ls", p.as_str()],
|
|
vec!["dump", p.as_str()],
|
|
vec!["stat", p.as_str()],
|
|
vec!["diff", p.as_str(), p.as_str()],
|
|
] {
|
|
let o = h5rs(&args);
|
|
assert_eq!(code(&o), 2, "{args:?}");
|
|
assert!(String::from_utf8_lossy(&o.stderr).contains("not an HDF5 file"));
|
|
}
|
|
let o = h5rs(&["check", &p]);
|
|
assert_eq!(code(&o), 1);
|
|
assert!(stdout(&o).contains("no HDF5 signature"));
|
|
assert_eq!(code(&h5rs(&["nonsense"])), 2);
|
|
assert_eq!(code(&h5rs(&["ls"])), 2);
|
|
assert_eq!(code(&h5rs(&["--help"])), 0);
|
|
}
|