osobhandClaude Opus 5.5
72306c6013
With the header checks merged, the library refuses truncated files and misaligned chunk-index keys itself, so check reported only "file cannot be opened" for a truncated file. It now reports the truncation (stored end of file vs file length) or the library's error, and the misaligned chunk test accepts the library's refusal of the key. Co-Authored-By: Claude Opus 5.5 (1M context) <[email protected]>
776 lines
28 KiB
Rust
776 lines
28 KiB
Rust
//! `h5rs` against libhdf5's own tools, on files h5py writes
|
|
//! (`tests/gen_files.py`): `ls` against h5ls, `stat` against h5stat, `dump`
|
|
//! against h5dump, `diff` exit codes against h5diff, `dump --json` values
|
|
//! against h5py, and `check` on valid and deliberately corrupted files.
|
|
//!
|
|
//! Needs python3 with h5py (`CLAWHDF5_PYTHON`) and, for the comparisons,
|
|
//! the libhdf5 command-line tools (h5ls, h5stat, h5dump, h5diff) on `PATH`.
|
|
//! Each test skips when what it needs is missing, unless
|
|
//! `CLAWHDF5_REQUIRE_INTEROP=1`, which turns a skip into a failure.
|
|
|
|
use std::collections::BTreeMap;
|
|
use std::path::{Path, PathBuf};
|
|
use std::process::{Command, Output};
|
|
|
|
use clawhdf5_format::checksum::jenkins_lookup3;
|
|
|
|
fn python() -> String {
|
|
std::env::var("CLAWHDF5_PYTHON").unwrap_or_else(|_| "python3".to_string())
|
|
}
|
|
|
|
fn interop_required() -> bool {
|
|
std::env::var("CLAWHDF5_REQUIRE_INTEROP").is_ok_and(|v| v == "1")
|
|
}
|
|
|
|
fn python_available() -> bool {
|
|
Command::new(python())
|
|
.args(["-c", "import h5py, numpy"])
|
|
.output()
|
|
.map(|o| o.status.success())
|
|
.unwrap_or(false)
|
|
}
|
|
|
|
fn tool_available(name: &str) -> bool {
|
|
Command::new(name)
|
|
.arg("--version")
|
|
.output()
|
|
.map(|o| o.status.success())
|
|
.unwrap_or(false)
|
|
}
|
|
|
|
/// Skip (return true) when `what` is unavailable, or fail when interop is
|
|
/// required.
|
|
fn missing(ok: bool, what: &str) -> bool {
|
|
if ok {
|
|
return false;
|
|
}
|
|
assert!(
|
|
!interop_required(),
|
|
"CLAWHDF5_REQUIRE_INTEROP=1 but {what} is not available"
|
|
);
|
|
eprintln!("SKIP: {what} not available");
|
|
true
|
|
}
|
|
|
|
fn h5rs(args: &[&str]) -> Output {
|
|
Command::new(env!("CARGO_BIN_EXE_h5rs"))
|
|
.args(args)
|
|
.output()
|
|
.expect("run h5rs")
|
|
}
|
|
|
|
fn run(tool: &str, args: &[&str]) -> Output {
|
|
Command::new(tool).args(args).output().expect("run tool")
|
|
}
|
|
|
|
fn stdout(o: &Output) -> String {
|
|
String::from_utf8_lossy(&o.stdout).into_owned()
|
|
}
|
|
|
|
/// The generated files and what h5py read from them.
|
|
struct Files {
|
|
dir: tempfile::TempDir,
|
|
values: serde_json::Value,
|
|
}
|
|
|
|
impl Files {
|
|
fn path(&self, name: &str) -> PathBuf {
|
|
self.dir.path().join(name)
|
|
}
|
|
|
|
fn p(&self, name: &str) -> String {
|
|
self.path(name).to_string_lossy().into_owned()
|
|
}
|
|
}
|
|
|
|
fn generate() -> Option<Files> {
|
|
if missing(python_available(), "python3 with h5py") {
|
|
return None;
|
|
}
|
|
let dir = tempfile::tempdir().unwrap();
|
|
let script = Path::new(env!("CARGO_MANIFEST_DIR")).join("tests/gen_files.py");
|
|
let out = Command::new(python())
|
|
.arg(&script)
|
|
.arg(dir.path())
|
|
.output()
|
|
.expect("run gen_files.py");
|
|
assert!(
|
|
out.status.success(),
|
|
"gen_files.py failed:\n{}",
|
|
String::from_utf8_lossy(&out.stderr)
|
|
);
|
|
let values = serde_json::from_slice(&out.stdout).expect("gen_files.py output");
|
|
Some(Files { dir, values })
|
|
}
|
|
|
|
const LIBVERS: [&str; 2] = ["earliest.h5", "latest.h5"];
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// ls
|
|
// ---------------------------------------------------------------------------
|
|
|
|
/// Every h5ls line must be the start of the matching h5rs line (h5rs adds
|
|
/// the datatype after a dataset's shape).
|
|
fn assert_ls_like(ours: &str, reference: &str, what: &str) {
|
|
let ours: Vec<&str> = ours.lines().collect();
|
|
let refs: Vec<&str> = reference.lines().collect();
|
|
assert_eq!(
|
|
ours.len(),
|
|
refs.len(),
|
|
"{what}: line count\nh5rs:\n{}\nh5ls:\n{}",
|
|
ours.join("\n"),
|
|
refs.join("\n")
|
|
);
|
|
for (o, r) in ours.iter().zip(&refs) {
|
|
let r = r.trim_end();
|
|
assert!(
|
|
o.starts_with(r) && (o.len() == r.len() || r.contains("Dataset {")),
|
|
"{what}:\n h5rs: {o}\n h5ls: {r}"
|
|
);
|
|
}
|
|
}
|
|
|
|
#[test]
|
|
fn ls_matches_h5ls() {
|
|
let Some(f) = generate() else { return };
|
|
if missing(tool_available("h5ls"), "h5ls") {
|
|
return;
|
|
}
|
|
for name in LIBVERS.iter().chain(&["userblock.h5"]) {
|
|
let p = f.p(name);
|
|
for (args, what) in [(vec!["-r"], "recursive"), (vec![], "root")] {
|
|
let mut a: Vec<&str> = args.clone();
|
|
a.push(&p);
|
|
let ours = h5rs(&[&["ls"], a.as_slice()].concat());
|
|
assert!(ours.status.success(), "{name} {what}: {:?}", ours);
|
|
let reference = run("h5ls", &a);
|
|
assert_ls_like(
|
|
&stdout(&ours),
|
|
&stdout(&reference),
|
|
&format!("{name} {what}"),
|
|
);
|
|
}
|
|
}
|
|
// A group, and a dataset, named after the file.
|
|
for obj in ["/grp", "/grp/sub", "/contig", "/grp/ext2"] {
|
|
let arg = format!("{}{obj}", f.p("latest.h5"));
|
|
let ours = h5rs(&["ls", &arg]);
|
|
let reference = run("h5ls", &[&arg]);
|
|
assert_ls_like(&stdout(&ours), &stdout(&reference), &arg);
|
|
let ours = h5rs(&["ls", "-r", &arg]);
|
|
let reference = run("h5ls", &["-r", &arg]);
|
|
assert_ls_like(&stdout(&ours), &stdout(&reference), &format!("-r {arg}"));
|
|
}
|
|
}
|
|
|
|
#[test]
|
|
fn ls_verbose_describes_layout_filters_and_attributes() {
|
|
let Some(f) = generate() else { return };
|
|
let o = h5rs(&["ls", "-r", "-v", &f.p("latest.h5")]);
|
|
assert!(o.status.success());
|
|
let s = stdout(&o);
|
|
for want in [
|
|
"Layout: chunked (extensible array index)",
|
|
"Layout: chunked (v2 B-tree index)",
|
|
"Layout: chunked (fixed array index)",
|
|
"Layout: chunked (single chunk index)",
|
|
"Filter-0: shuffle-2",
|
|
"Filter-1: deflate-1",
|
|
"Filter-2: fletcher32-3",
|
|
"Chunks: {100} 400 bytes",
|
|
"Attribute: units scalar",
|
|
"Data: \"m\"",
|
|
] {
|
|
assert!(s.contains(want), "missing {want:?} in\n{s}");
|
|
}
|
|
let o = h5rs(&["ls", "-v", &format!("{}/grp", f.p("earliest.h5"))]);
|
|
assert!(stdout(&o).contains("Layout: chunked (v1 B-tree index)"));
|
|
}
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// stat
|
|
// ---------------------------------------------------------------------------
|
|
|
|
/// `key: value` lines of an h5stat-style report.
|
|
fn stat_facts(s: &str) -> BTreeMap<String, String> {
|
|
s.lines()
|
|
.filter_map(|l| {
|
|
let (k, v) = l.rsplit_once(':')?;
|
|
Some((k.trim().to_string(), v.trim().to_string()))
|
|
})
|
|
.collect()
|
|
}
|
|
|
|
#[test]
|
|
fn stat_matches_h5stat() {
|
|
let Some(f) = generate() else { return };
|
|
if missing(tool_available("h5stat"), "h5stat") {
|
|
return;
|
|
}
|
|
let keys = [
|
|
"# of unique groups",
|
|
"# of unique datasets",
|
|
"# of unique named datatypes",
|
|
"# of unique links",
|
|
"# of unique other",
|
|
"Max. # of links to object",
|
|
"Max. # of objects in group",
|
|
"Max. rank of datasets",
|
|
"Max. dimension size of 1-D datasets",
|
|
"Total raw data size",
|
|
"Dataset layout counts[COMPACT]",
|
|
"Dataset layout counts[CONTIG]",
|
|
"Dataset layout counts[CHUNKED]",
|
|
"Dataset layout counts[VIRTUAL]",
|
|
"NO filter",
|
|
"GZIP filter",
|
|
"SHUFFLE filter",
|
|
"FLETCHER32 filter",
|
|
"SZIP filter",
|
|
"NBIT filter",
|
|
"SCALEOFFSET filter",
|
|
"USER-DEFINED filter",
|
|
"Max. # of attributes to objects",
|
|
"Total space",
|
|
];
|
|
for name in LIBVERS.iter().chain(&["userblock.h5", "base.h5"]) {
|
|
let p = f.p(name);
|
|
let ours = h5rs(&["stat", &p]);
|
|
assert!(ours.status.success(), "{name}: {ours:?}");
|
|
let reference = run("h5stat", &[&p]);
|
|
assert!(reference.status.success(), "h5stat {name} failed");
|
|
let (o, r) = (stat_facts(&stdout(&ours)), stat_facts(&stdout(&reference)));
|
|
for k in keys {
|
|
// h5stat leaves out the attribute summary when nothing has one.
|
|
let Some(want) = r.get(k) else { continue };
|
|
assert_eq!(o.get(k), Some(want), "{name}: {k}");
|
|
}
|
|
}
|
|
}
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// dump
|
|
// ---------------------------------------------------------------------------
|
|
|
|
#[test]
|
|
fn dump_matches_h5dump() {
|
|
let Some(f) = generate() else { return };
|
|
if missing(tool_available("h5dump"), "h5dump") {
|
|
return;
|
|
}
|
|
for name in LIBVERS.iter().chain(&["userblock.h5", "changed.h5"]) {
|
|
let p = f.p(name);
|
|
for args in [vec![], vec!["-A"]] {
|
|
let mut a = args.clone();
|
|
a.push(p.as_str());
|
|
let ours = h5rs(&[&["dump"], a.as_slice()].concat());
|
|
assert!(ours.status.success(), "{name} {args:?}: {ours:?}");
|
|
let reference = run("h5dump", &a);
|
|
// h5dump names the file as given; h5rs by its file name.
|
|
let r = stdout(&reference).replacen(&p, name, 1);
|
|
assert_eq!(stdout(&ours), r, "{name} {args:?}");
|
|
}
|
|
}
|
|
let p = f.p("latest.h5");
|
|
let ours = h5rs(&["dump", "-d", "/grp/ext2", &p]);
|
|
let reference = run("h5dump", &["-d", "/grp/ext2", &p]);
|
|
assert_eq!(
|
|
stdout(&ours),
|
|
stdout(&reference).replacen(&p, "latest.h5", 1)
|
|
);
|
|
}
|
|
|
|
/// A null-padded fixed string prints every byte (NULs as `\000`), as
|
|
/// h5dump prints it, also inside a compound and an array member.
|
|
#[test]
|
|
fn dump_shows_nul_padding_in_nested_strings() {
|
|
let Some(f) = generate() else { return };
|
|
let p = f.p("nulstrings.h5");
|
|
let ours = stdout(&h5rs(&["dump", &p]));
|
|
for want in [
|
|
r#"(0): "\000\000\000", "ab\000", "a\000b""#,
|
|
r#""\000\000\000","#,
|
|
r#""a\000b","#,
|
|
r#"[ "\000\000", "x\000" ]"#,
|
|
] {
|
|
assert!(ours.contains(want), "no {want} in\n{ours}");
|
|
}
|
|
if missing(tool_available("h5dump"), "h5dump") {
|
|
return;
|
|
}
|
|
let reference = run("h5dump", &[&p]);
|
|
assert_eq!(ours, stdout(&reference).replacen(&p, "nulstrings.h5", 1));
|
|
}
|
|
|
|
#[test]
|
|
fn dump_json_values_match_h5py() {
|
|
let Some(f) = generate() else { return };
|
|
for name in LIBVERS {
|
|
let o = h5rs(&["dump", "--json", &f.p(name)]);
|
|
assert!(o.status.success(), "{name}: {o:?}");
|
|
let doc: serde_json::Value = serde_json::from_slice(&o.stdout).unwrap();
|
|
assert_eq!(doc["apiVersion"], "1.1.1");
|
|
let root = doc["root"].as_str().unwrap();
|
|
assert!(doc["groups"][root].is_object());
|
|
let mut by_path = BTreeMap::new();
|
|
for (_, d) in doc["datasets"].as_object().unwrap() {
|
|
for a in d["alias"].as_array().unwrap() {
|
|
by_path.insert(a.as_str().unwrap().to_string(), d.clone());
|
|
}
|
|
}
|
|
let want = f.values[name.trim_end_matches(".h5")].as_object().unwrap();
|
|
assert!(want.len() > 10);
|
|
for (path, v) in want {
|
|
let d = by_path
|
|
.get(path)
|
|
.unwrap_or_else(|| panic!("{name}: no {path}"));
|
|
let got = &d["value"];
|
|
// JSON numbers: compare as f64 (h5py's ints and floats both
|
|
// round-trip exactly through JSON here).
|
|
assert_eq!(flatten(got), flatten(v), "{name}: {path}");
|
|
}
|
|
let links = doc["groups"][root]["links"].as_array().unwrap();
|
|
let soft = links.iter().find(|l| l["title"] == "soft").unwrap();
|
|
assert_eq!(soft["class"], "H5L_TYPE_SOFT");
|
|
assert_eq!(soft["h5path"], "/contig");
|
|
let ext = links.iter().find(|l| l["title"] == "external").unwrap();
|
|
assert_eq!(ext["class"], "H5L_TYPE_EXTERNAL");
|
|
assert_eq!(ext["file"], "other.h5");
|
|
}
|
|
}
|
|
|
|
fn flatten(v: &serde_json::Value) -> Vec<f64> {
|
|
match v {
|
|
serde_json::Value::Array(a) => a.iter().flat_map(flatten).collect(),
|
|
serde_json::Value::Number(n) => vec![n.as_f64().unwrap()],
|
|
other => panic!("not numeric: {other}"),
|
|
}
|
|
}
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// diff
|
|
// ---------------------------------------------------------------------------
|
|
|
|
fn code(o: &Output) -> i32 {
|
|
o.status.code().expect("exit code")
|
|
}
|
|
|
|
#[test]
|
|
fn diff_exit_codes_match_h5diff() {
|
|
let Some(f) = generate() else { return };
|
|
if missing(tool_available("h5diff"), "h5diff") {
|
|
return;
|
|
}
|
|
let (b, same, changed, extra, attr) = (
|
|
f.p("base.h5"),
|
|
f.p("same.h5"),
|
|
f.p("changed.h5"),
|
|
f.p("extra.h5"),
|
|
f.p("attr.h5"),
|
|
);
|
|
let missing_file = f.p("does-not-exist.h5");
|
|
let cases: Vec<(Vec<&str>, i32)> = vec![
|
|
(vec![&b, &same], 0),
|
|
(vec![&b, &b], 0),
|
|
(vec![&b, &changed], 1),
|
|
(vec![&b, &changed, "/d"], 1),
|
|
(vec![&b, &changed, "/g"], 0),
|
|
(vec!["-d", "0.01", &b, &changed], 0),
|
|
(vec!["-d", "0.0001", &b, &changed], 1),
|
|
(vec!["-p", "0.01", &b, &changed], 0),
|
|
(vec!["-p", "1e-9", &b, &changed], 1),
|
|
(vec![&b, &extra], 1),
|
|
(vec![&b, &attr], 1),
|
|
(vec![&b, &attr, "/g"], 0),
|
|
(vec![&b, &same, "/nope"], 2),
|
|
(vec![&b, &missing_file], 2),
|
|
(vec![&b, &same, "/d", "/d"], 0),
|
|
];
|
|
for (args, want) in cases {
|
|
let theirs = code(&run("h5diff", &args));
|
|
assert_eq!(theirs, want, "h5diff {args:?} (test expectation)");
|
|
let o = h5rs(&[&["diff"], args.as_slice()].concat());
|
|
assert_eq!(code(&o), want, "h5rs diff {args:?}: {}", stdout(&o));
|
|
let q = h5rs(&[&["diff", "-q"], args.as_slice()].concat());
|
|
assert_eq!(code(&q), want, "h5rs diff -q {args:?}");
|
|
assert!(q.stdout.is_empty(), "-q printed output");
|
|
}
|
|
// The report lists the differing positions.
|
|
let o = h5rs(&["diff", "-r", &b, &changed, "/d"]);
|
|
let s = stdout(&o);
|
|
assert!(s.contains("[ 0 2 ]") && s.contains("[ 2 3 ]"), "{s}");
|
|
assert!(s.contains("2 difference(s) found"), "{s}");
|
|
}
|
|
|
|
/// The option names are h5diff's: -c is --compare (a flag), the count is
|
|
/// -n/--count, and the --name=value forms are accepted.
|
|
#[test]
|
|
fn diff_options_are_named_like_h5diff() {
|
|
let Some(f) = generate() else { return };
|
|
let (b, changed) = (f.p("base.h5"), f.p("changed.h5"));
|
|
let cases: Vec<(Vec<&str>, i32)> = vec![
|
|
// "2" is then the first file name, which does not exist.
|
|
(vec!["-r", "-c", "2", &b, &changed], 2),
|
|
(vec!["-c", &b, &changed], 1),
|
|
(vec!["--compare", &b, &b], 0),
|
|
(vec!["-r", "-n", "1", &b, &changed], 1),
|
|
(vec!["-r", "--count=1", &b, &changed], 1),
|
|
(vec!["--delta=0.01", &b, &changed], 0),
|
|
(vec!["--relative=1e-9", &b, &changed], 1),
|
|
];
|
|
let have_h5diff = tool_available("h5diff");
|
|
for (args, want) in cases {
|
|
if have_h5diff {
|
|
assert_eq!(code(&run("h5diff", &args)), want, "h5diff {args:?}");
|
|
}
|
|
let o = h5rs(&[&["diff"], args.as_slice()].concat());
|
|
assert_eq!(code(&o), want, "h5rs diff {args:?}: {}", stdout(&o));
|
|
}
|
|
for count in [vec!["-n", "1"], vec!["--count=1"]] {
|
|
let o = h5rs(&[&["diff", "-r"], count.as_slice(), &[&b, &changed, "/d"]].concat());
|
|
let s = stdout(&o);
|
|
assert!(
|
|
s.contains("[ 0 2 ]") && !s.contains("[ 2 3 ]"),
|
|
"{count:?}: {s}"
|
|
);
|
|
assert!(s.contains("2 difference(s) found"), "{s}");
|
|
}
|
|
assert_eq!(code(&h5rs(&["diff", "--bogus=1", &b, &b])), 2);
|
|
}
|
|
|
|
/// Tolerances on 64-bit integers one apart, far beyond f64's integer
|
|
/// precision: compared exactly, as h5diff does.
|
|
#[test]
|
|
fn diff_tolerances_compare_large_integers_exactly() {
|
|
let Some(f) = generate() else { return };
|
|
let (a, b) = (f.p("big1.h5"), f.p("big2.h5"));
|
|
let cases: Vec<(Vec<&str>, i32)> = vec![
|
|
(vec![&a, &b], 1),
|
|
(vec!["-d", "0", &a, &b], 1),
|
|
(vec!["-d", "0.5", &a, &b], 1),
|
|
(vec!["-d", "1", &a, &b], 0),
|
|
(vec!["-d", "0", &a, &b, "/u"], 1),
|
|
(vec!["-p", "0", &a, &b], 1),
|
|
(vec!["-p", "1e-18", &a, &b, "/i"], 1),
|
|
(vec!["-p", "1e-15", &a, &b], 0),
|
|
];
|
|
let have_h5diff = tool_available("h5diff");
|
|
for (args, want) in cases {
|
|
if have_h5diff {
|
|
assert_eq!(code(&run("h5diff", &args)), want, "h5diff {args:?}");
|
|
}
|
|
let o = h5rs(&[&["diff"], args.as_slice()].concat());
|
|
assert_eq!(code(&o), want, "h5rs diff {args:?}: {}", stdout(&o));
|
|
}
|
|
// The reported difference is exact too.
|
|
let s = stdout(&h5rs(&["diff", "-r", "-d", "0", &a, &b, "/u"]));
|
|
assert!(
|
|
s.contains("18446744073709551615 18446744073709551614 1\n"),
|
|
"{s}"
|
|
);
|
|
}
|
|
|
|
/// A soft link is compared as a link (its target path), as h5diff does,
|
|
/// unless `--follow-symlinks`, which compares (and walks into) what it
|
|
/// leads to; two dangling links are then the same.
|
|
#[test]
|
|
fn diff_soft_links_like_h5diff() {
|
|
let Some(f) = generate() else { return };
|
|
let (s1, s2, t1, t2) = (
|
|
f.p("soft1.h5"),
|
|
f.p("soft2.h5"),
|
|
f.p("target1.h5"),
|
|
f.p("target2.h5"),
|
|
);
|
|
let fl = "--follow-symlinks";
|
|
let cases: Vec<(Vec<&str>, i32)> = vec![
|
|
(vec![&s1, &s2, "/g/s"], 0),
|
|
(vec![&s1, &s2, "/g"], 0),
|
|
(vec![&s1, &s2, "/lnk"], 0),
|
|
(vec![&s1, &s2, "/dang"], 0),
|
|
(vec![&s1, &s2, "/lnk/d"], 1),
|
|
(vec![&t1, &t2, "/s"], 1),
|
|
(vec![&t1, &t2, "/s", "/rel"], 1),
|
|
(vec![fl, &s1, &s2, "/g/s"], 1),
|
|
(vec![fl, &s1, &s2, "/g"], 1),
|
|
(vec![fl, &s1, &s2, "/lnk"], 1),
|
|
(vec![fl, &s1, &s2, "/dang"], 0),
|
|
(vec![fl, &s1, &s1], 0),
|
|
(vec![fl, &t1, &t2, "/s"], 0),
|
|
(vec![fl, &t1, &t2, "/s", "/rel"], 0),
|
|
];
|
|
let have_h5diff = tool_available("h5diff");
|
|
for (args, want) in cases {
|
|
if have_h5diff {
|
|
assert_eq!(code(&run("h5diff", &args)), want, "h5diff {args:?}");
|
|
}
|
|
let o = h5rs(&[&["diff"], args.as_slice()].concat());
|
|
assert_eq!(code(&o), want, "h5rs diff {args:?}: {}", stdout(&o));
|
|
}
|
|
}
|
|
|
|
/// An object hard-linked under two names is compared under both, with
|
|
/// everything below it, so a file that shares one object between two names
|
|
/// equals a file that stores two identical copies.
|
|
#[test]
|
|
fn diff_compares_every_hard_link_path() {
|
|
let Some(f) = generate() else { return };
|
|
let (linked, copied, changed) = (
|
|
f.p("hardlinked.h5"),
|
|
f.p("copied.h5"),
|
|
f.p("copied_changed.h5"),
|
|
);
|
|
// A hard-linked dataset: h5diff agrees (exit 0).
|
|
for (a, b) in [(&linked, &copied), (&copied, &linked)] {
|
|
let o = h5rs(&["diff", a, b, "/y"]);
|
|
assert_eq!(code(&o), 0, "{a} {b} /y: {}", stdout(&o));
|
|
if tool_available("h5diff") {
|
|
assert_eq!(code(&run("h5diff", &[a, b, "/y"])), 0, "h5diff /y");
|
|
}
|
|
// The whole file, including a hard-linked group's members (h5diff
|
|
// does not descend a group's second name, so it reports those
|
|
// members as present in one file only and exits 1).
|
|
let o = h5rs(&["diff", a, b]);
|
|
assert_eq!(code(&o), 0, "{a} {b}: {}", stdout(&o));
|
|
assert!(!stdout(&o).contains("exists only"), "{}", stdout(&o));
|
|
}
|
|
// A value under the second name is compared, not skipped.
|
|
let o = h5rs(&["diff", "-r", &linked, &changed]);
|
|
assert_eq!(code(&o), 1, "{}", stdout(&o));
|
|
assert!(
|
|
stdout(&o).contains("dataset: </h/s/e> and </h/s/e>"),
|
|
"{}",
|
|
stdout(&o)
|
|
);
|
|
assert!(!stdout(&o).contains("</g/s/e>"), "{}", stdout(&o));
|
|
}
|
|
|
|
/// Where h5rs deliberately differs from h5diff: objects that cannot be
|
|
/// compared (another shape, another datatype class) are a difference.
|
|
#[test]
|
|
fn diff_counts_incomparable_objects_as_different() {
|
|
let Some(f) = generate() else { return };
|
|
let b = f.p("base.h5");
|
|
for other in ["reshaped.h5", "int.h5"] {
|
|
let o = h5rs(&["diff", &b, &f.p(other)]);
|
|
assert_eq!(code(&o), 1, "{other}: {}", stdout(&o));
|
|
assert!(stdout(&o).contains("Not comparable"), "{other}");
|
|
}
|
|
}
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// check
|
|
// ---------------------------------------------------------------------------
|
|
|
|
#[test]
|
|
fn check_accepts_valid_files() {
|
|
let Some(f) = generate() else { return };
|
|
for name in LIBVERS.iter().chain(&[
|
|
"userblock.h5",
|
|
"base.h5",
|
|
"changed.h5",
|
|
"extra.h5",
|
|
"attr.h5",
|
|
]) {
|
|
let o = h5rs(&["check", "--data", &f.p(name)]);
|
|
assert_eq!(code(&o), 0, "{name}:\n{}", stdout(&o));
|
|
assert!(stdout(&o).contains("no problems found"));
|
|
}
|
|
// The newest format's checksummed structures were all verified.
|
|
let s = stdout(&h5rs(&["check", &f.p("latest.h5")]));
|
|
assert!(s.contains("superblock 1, v2 object headers 33,"), "{s}");
|
|
assert!(s.contains("chunk indexes 5"), "{s}");
|
|
}
|
|
|
|
/// Signatures of checksummed structures, and where each one's checksum
|
|
/// is: found by trying every end offset (the checksum is the Jenkins
|
|
/// lookup3 hash of the bytes before it). A fractal heap direct block's
|
|
/// checksum is instead stored near its start, computed over the whole block
|
|
/// with the field zeroed.
|
|
fn checksummed_structures(data: &[u8]) -> Vec<(&'static str, usize, usize)> {
|
|
const SIGS: [&str; 13] = [
|
|
"OHDR", "OCHK", "BTHD", "BTIN", "BTLF", "FRHP", "FHIB", "EAHD", "EAIB", "EASB", "EADB",
|
|
"FAHD", "FADB",
|
|
];
|
|
let mut found = Vec::new();
|
|
for start in 0..data.len().saturating_sub(4) {
|
|
let sig = &data[start..start + 4];
|
|
if let Some(name) = SIGS.iter().find(|s| s.as_bytes() == sig) {
|
|
let limit = data.len().min(start + 65536);
|
|
for end in start + 8..limit.saturating_sub(4) {
|
|
let stored = u32::from_le_bytes(data[end..end + 4].try_into().unwrap());
|
|
if jenkins_lookup3(&data[start..end]) == stored {
|
|
found.push((*name, start, end));
|
|
break;
|
|
}
|
|
}
|
|
} else if sig == b"FHDB" {
|
|
// version(1) heap address(8) block offset(1..8) checksum(4)
|
|
'found: for boff in 1..=8usize {
|
|
let at = start + 5 + 8 + boff;
|
|
let Some(stored) = data.get(at..at + 4) else {
|
|
break;
|
|
};
|
|
let stored = u32::from_le_bytes(stored.try_into().unwrap());
|
|
for size in [512usize, 1024, 2048, 4096, 8192, 16384, 32768, 65536] {
|
|
let Some(block) = data.get(start..start + size) else {
|
|
break;
|
|
};
|
|
let mut b = block.to_vec();
|
|
b[at - start..at - start + 4].fill(0);
|
|
if jenkins_lookup3(&b) == stored {
|
|
found.push(("FHDB", start, at));
|
|
break 'found;
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
found
|
|
}
|
|
|
|
#[test]
|
|
fn check_flags_every_corrupted_checksum() {
|
|
let Some(f) = generate() else { return };
|
|
let path = f.path("latest.h5");
|
|
let data = std::fs::read(&path).unwrap();
|
|
let structures = checksummed_structures(&data);
|
|
let kinds: std::collections::BTreeSet<&str> = structures.iter().map(|s| s.0).collect();
|
|
for want in [
|
|
"OHDR", "OCHK", "BTHD", "BTLF", "FRHP", "FHDB", "EAHD", "EAIB", "EADB", "FAHD", "FADB",
|
|
] {
|
|
assert!(kinds.contains(want), "test file has no {want}: {kinds:?}");
|
|
}
|
|
let bad = f.path("bad.h5");
|
|
for (name, start, at) in structures {
|
|
let mut d = data.clone();
|
|
d[at] ^= 0x01;
|
|
std::fs::write(&bad, &d).unwrap();
|
|
let o = h5rs(&["check", &bad.to_string_lossy()]);
|
|
let s = stdout(&o);
|
|
assert_eq!(
|
|
code(&o),
|
|
1,
|
|
"{name} at {start:#x}: checksum flip not flagged\n{s}"
|
|
);
|
|
assert!(
|
|
s.lines()
|
|
.any(|l| l.starts_with("problem: 0x") && l.contains("checksum")),
|
|
"{name} at {start:#x}: no checksum problem reported\n{s}"
|
|
);
|
|
}
|
|
}
|
|
|
|
/// `check --data` follows variable-length elements into the global heap:
|
|
/// a damaged collection is reported at its address, for the dataset and
|
|
/// the attribute that point into it. Without --data it is not read.
|
|
#[test]
|
|
fn check_data_follows_vl_data_into_the_global_heap() {
|
|
let Some(f) = generate() else { return };
|
|
for name in LIBVERS {
|
|
let mut data = std::fs::read(f.path(name)).unwrap();
|
|
let gcols: Vec<usize> = (0..data.len() - 4)
|
|
.filter(|&i| &data[i..i + 4] == b"GCOL")
|
|
.collect();
|
|
assert!(!gcols.is_empty(), "{name}: no global heap");
|
|
// "GCOL" version(1) reserved(3) size(8), then heap object 1:
|
|
// index(2) refcount(2) reserved(4) size(8) — claim 4 GiB.
|
|
let at = gcols[0];
|
|
data[at + 24..at + 28].fill(0xff);
|
|
let bad = f.path(&format!("bad-gcol-{name}"));
|
|
std::fs::write(&bad, &data).unwrap();
|
|
let bad = bad.to_string_lossy().into_owned();
|
|
let o = h5rs(&["check", "--data", &bad]);
|
|
let s = stdout(&o);
|
|
assert_eq!(code(&o), 1, "{name}: {s}");
|
|
let want = format!("problem: {at:#x} /vlstr: variable-length data: global heap");
|
|
assert!(s.contains(&want), "{name}: no {want:?} in\n{s}");
|
|
// h5dump refuses the file too.
|
|
if tool_available("h5dump") {
|
|
assert_ne!(code(&run("h5dump", &[&bad])), 0, "{name}: h5dump read it");
|
|
}
|
|
assert_eq!(code(&h5rs(&["check", &bad])), 0, "{name}: without --data");
|
|
let ok = h5rs(&["check", "--data", &f.p(name)]);
|
|
assert!(
|
|
stdout(&ok).contains(&format!("global heap collections read: {}", gcols.len())),
|
|
"{name}: {}",
|
|
stdout(&ok)
|
|
);
|
|
}
|
|
}
|
|
|
|
#[test]
|
|
fn check_flags_a_truncated_file() {
|
|
let Some(f) = generate() else { return };
|
|
let data = std::fs::read(f.path("latest.h5")).unwrap();
|
|
let bad = f.path("truncated.h5");
|
|
std::fs::write(&bad, &data[..data.len() * 3 / 4]).unwrap();
|
|
let o = h5rs(&["check", &bad.to_string_lossy()]);
|
|
assert_eq!(code(&o), 1);
|
|
assert!(stdout(&o).contains("file is truncated"), "{}", stdout(&o));
|
|
}
|
|
|
|
/// A v1 B-tree chunk index has no checksum; its keys are checked against
|
|
/// the dataset: a chunk offset that is not a multiple of the chunk size.
|
|
#[test]
|
|
fn check_flags_a_misaligned_chunk() {
|
|
let Some(f) = generate() else { return };
|
|
let path = f.path("earliest.h5");
|
|
let mut data = std::fs::read(&path).unwrap();
|
|
// The leaf chunk B-tree of /grp/gz (10 deflated chunks of 100 i32, so
|
|
// each stored in under 400 bytes): "TREE" type(1)=1 level(1)=0
|
|
// entries(2) left(8) right(8), then key, child, key, child, ... where a
|
|
// rank-1 chunk key is size(4) mask(4) offsets(2 x 8) and a child is 8.
|
|
// Key 1 starts at 24 + 24 + 8 = 56; its dimension-0 offset (100) at 64.
|
|
let mut patched = false;
|
|
for start in 0..data.len() - 72 {
|
|
if &data[start..start + 4] == b"TREE" && data[start + 4] == 1 && data[start + 5] == 0 {
|
|
let entries = u16::from_le_bytes(data[start + 6..start + 8].try_into().unwrap());
|
|
let size0 = u32::from_le_bytes(data[start + 24..start + 28].try_into().unwrap());
|
|
let off1 = u64::from_le_bytes(data[start + 64..start + 72].try_into().unwrap());
|
|
if entries == 10 && size0 < 400 && off1 == 100 {
|
|
data[start + 64..start + 72].copy_from_slice(&103u64.to_le_bytes());
|
|
patched = true;
|
|
break;
|
|
}
|
|
}
|
|
}
|
|
assert!(patched, "no chunk B-tree found for /grp/gz");
|
|
let bad = f.path("misaligned.h5");
|
|
std::fs::write(&bad, &data).unwrap();
|
|
let o = h5rs(&["check", &bad.to_string_lossy()]);
|
|
let s = stdout(&o);
|
|
assert_eq!(code(&o), 1, "{s}");
|
|
// The library's chunk-index reader refuses the key first (as libhdf5
|
|
// does); either way the problem is reported against the dataset.
|
|
assert!(
|
|
s.contains("/grp/gz: chunk at [103, 0] offset 103 in dimension 0 is not a multiple of the chunk size 100")
|
|
|| s.contains("/grp/gz: chunk index (v1 B-tree): chunked read error: bad coordinate offset [103, 0] for chunk dimensions [100, 4]"),
|
|
"{s}"
|
|
);
|
|
}
|
|
|
|
#[test]
|
|
fn every_subcommand_rejects_a_non_hdf5_file_cleanly() {
|
|
let dir = tempfile::tempdir().unwrap();
|
|
let p = dir.path().join("junk.h5");
|
|
std::fs::write(&p, b"this is not an HDF5 file at all").unwrap();
|
|
let p = p.to_string_lossy().into_owned();
|
|
for args in [
|
|
vec!["ls", p.as_str()],
|
|
vec!["dump", p.as_str()],
|
|
vec!["stat", p.as_str()],
|
|
vec!["diff", p.as_str(), p.as_str()],
|
|
] {
|
|
let o = h5rs(&args);
|
|
assert_eq!(code(&o), 2, "{args:?}");
|
|
assert!(String::from_utf8_lossy(&o.stderr).contains("not an HDF5 file"));
|
|
}
|
|
let o = h5rs(&["check", &p]);
|
|
assert_eq!(code(&o), 1);
|
|
assert!(stdout(&o).contains("no HDF5 signature"));
|
|
assert_eq!(code(&h5rs(&["nonsense"])), 2);
|
|
assert_eq!(code(&h5rs(&["ls"])), 2);
|
|
assert_eq!(code(&h5rs(&["--help"])), 0);
|
|
}
|