feat(tools): h5rs, pure-Rust HDF5 tools (ls, dump, stat, diff, check)
New workspace crate clawhdf5-tools with one binary, h5rs, built only on the clawhdf5 facade and clawhdf5-format (no libhdf5, no C): - ls [-r] [-v] FILE[/path]: h5ls's listing (same text in its first two columns) plus the datatype; -v adds address, link count, layout and chunk index, chunk size, storage, filters, datatype and attributes. - dump [--json] [-A] [-p] [-d PATH] FILE: h5dump DDL (byte-identical to h5dump 1.14.6 on the test files) or hdf5-json. - stat FILE: h5stat's object/link/rank/layout/filter/attribute counts, raw data and total size. - diff [-r] [-q] [-d D] [-p R] A B [OBJ1 [OBJ2]]: structural and value differences, exit 0/1/2 like h5diff. - check [--data] FILE: walks every object, parses every message, verifies the checksums of every v2+ structure (including the fractal heap blocks the library never checks), checks chunk indexes against their datasets and raw data for out-of-file or overlapping extents; every problem with its address. Values over --max-bytes are reported, not read; dense-storage heaps are verified before objects are read from them; panics are caught (exit 3). Tests compare with h5ls, h5stat, h5dump and h5diff and with h5py's values, and flip the checksum of every checksummed structure in a v1.14-format file. Co-Authored-By: Claude Opus 5.5 (1M context) <[email protected]>
This commit is contained in:
osobh
@@ -0,0 +1,115 @@
|
||||
"""Write the HDF5 files the h5rs interop tests run on.
|
||||
|
||||
usage: gen_files.py OUTDIR
|
||||
|
||||
Writes OUTDIR/{earliest,latest}.h5 (the same content with the oldest and the
|
||||
newest file-format structures: symbol tables and v1 B-trees vs. v2 object
|
||||
headers, fractal heaps, v2 B-trees and the chunk indexes of HDF5 1.10+), the
|
||||
pairs the diff tests compare, and a file with a user block. Prints one JSON
|
||||
object with the values h5py reads back, for the dump tests.
|
||||
"""
|
||||
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
|
||||
import h5py
|
||||
import numpy as np
|
||||
|
||||
out = sys.argv[1]
|
||||
|
||||
|
||||
def content(f, dense):
|
||||
f.attrs["title"] = "h5rs test"
|
||||
f.attrs["version"] = np.int64(3)
|
||||
f.attrs["scale"] = np.array([0.5, 1.5], dtype="f4")
|
||||
f["contig"] = np.arange(12, dtype="f8").reshape(3, 4)
|
||||
dcpl = h5py.h5p.create(h5py.h5p.DATASET_CREATE)
|
||||
dcpl.set_layout(h5py.h5d.COMPACT)
|
||||
space = h5py.h5s.create_simple((4,))
|
||||
h5py.h5d.create(f.id, b"compact", h5py.h5t.STD_I16LE, space, dcpl).write(
|
||||
h5py.h5s.ALL, h5py.h5s.ALL, np.arange(4, dtype="<i2")
|
||||
)
|
||||
g = f.create_group("grp")
|
||||
g.attrs["units"] = "m"
|
||||
g.create_dataset(
|
||||
"gz", data=np.arange(1000, dtype="i4"), chunks=(100,), compression="gzip",
|
||||
shuffle=True, fletcher32=True,
|
||||
)
|
||||
g.create_dataset("ext1", data=np.arange(50, dtype="u2"), chunks=(8,), maxshape=(None,))
|
||||
g.create_dataset(
|
||||
"ext2", data=np.arange(60, dtype="f4").reshape(6, 10), chunks=(4, 4), maxshape=(None, None)
|
||||
)
|
||||
g.create_dataset("fixed", data=np.arange(64, dtype="i8").reshape(8, 8), chunks=(3, 3))
|
||||
g.create_dataset("single", data=np.arange(10, dtype="i4"), chunks=(10,))
|
||||
g.create_dataset("sparse", shape=(100,), dtype="i4", chunks=(10,), fillvalue=-1)
|
||||
g["sparse"][20:30] = 7
|
||||
sub = g.create_group("sub")
|
||||
sub["scalar"] = np.float32(2.5)
|
||||
sub["empty"] = h5py.Empty("f8")
|
||||
f["strings"] = np.array([b"ab", b"cde"])
|
||||
f["vlstr"] = np.array(["x", "yy", "zzz"], dtype=h5py.string_dtype())
|
||||
f["cmp"] = np.array([(1, 2.5), (3, 4.5)], dtype=[("a", "i2"), ("b", ">f4")])
|
||||
f.create_dataset(
|
||||
"enum", data=np.array([0, 1, 1], dtype="u1"),
|
||||
dtype=h5py.enum_dtype({"RED": 0, "GREEN": 1}, basetype="u1"),
|
||||
)
|
||||
f["be"] = np.arange(5, dtype=">i4")
|
||||
f["arr"] = np.array([([1, 2, 3],), ([4, 5, 6],)], dtype=[("v", "3i4")])
|
||||
f["named_t"] = np.dtype("i8")
|
||||
f["soft"] = h5py.SoftLink("/contig")
|
||||
f["dangling"] = h5py.SoftLink("/nowhere")
|
||||
f["external"] = h5py.ExternalLink("other.h5", "/x")
|
||||
f["hard2"] = g["sub"]
|
||||
many = f.create_group("many")
|
||||
for i in range(12 if dense else 4):
|
||||
many[f"d{i:02}"] = np.int32(i)
|
||||
many.attrs[f"a{i:02}"] = i
|
||||
|
||||
|
||||
values = {}
|
||||
# "latest" under HDF5 2.0 writes datatype messages that libhdf5 1.14 tools
|
||||
# cannot read, so the newest format is taken as 1.14's.
|
||||
for libver in ("earliest", "latest"):
|
||||
path = os.path.join(out, f"{libver}.h5")
|
||||
bounds = ("earliest", "v114") if libver == "earliest" else ("v114", "v114")
|
||||
with h5py.File(path, "w", libver=bounds) as f:
|
||||
content(f, True)
|
||||
with h5py.File(path, "r") as f:
|
||||
vals = {}
|
||||
|
||||
def grab(name, obj):
|
||||
if isinstance(obj, h5py.Dataset) and obj.dtype.kind in "iuf" and obj.shape is not None:
|
||||
vals["/" + name] = obj[()].tolist()
|
||||
|
||||
f.visititems(grab)
|
||||
values[libver] = vals
|
||||
|
||||
# diff pairs
|
||||
def small(path, data=None, extra=False, attr=False, shape=(3, 4), dtype="f8"):
|
||||
with h5py.File(os.path.join(out, path), "w") as f:
|
||||
d = np.arange(12, dtype=dtype).reshape(shape) if data is None else data
|
||||
f["d"] = d
|
||||
f["g/x"] = np.arange(3)
|
||||
if extra:
|
||||
f["only_here"] = 1
|
||||
if attr:
|
||||
f["d"].attrs["u"] = 1
|
||||
|
||||
|
||||
base = np.arange(12, dtype="f8").reshape(3, 4)
|
||||
small("base.h5")
|
||||
small("same.h5")
|
||||
changed = base.copy()
|
||||
changed[0, 2] += 0.001
|
||||
changed[2, 3] += 0.001
|
||||
small("changed.h5", data=changed)
|
||||
small("extra.h5", extra=True)
|
||||
small("attr.h5", attr=True)
|
||||
small("reshaped.h5", shape=(4, 3))
|
||||
small("int.h5", dtype="i4")
|
||||
|
||||
with h5py.File(os.path.join(out, "userblock.h5"), "w", userblock_size=1024) as f:
|
||||
f["d"] = np.arange(10)
|
||||
|
||||
print(json.dumps(values))
|
||||
@@ -0,0 +1,569 @@
|
||||
//! `h5rs` against libhdf5's own tools, on files h5py writes
|
||||
//! (`tests/gen_files.py`): `ls` against h5ls, `stat` against h5stat, `dump`
|
||||
//! against h5dump, `diff` exit codes against h5diff, `dump --json` values
|
||||
//! against h5py, and `check` on valid and deliberately corrupted files.
|
||||
//!
|
||||
//! Needs python3 with h5py (`CLAWHDF5_PYTHON`) and, for the comparisons,
|
||||
//! the libhdf5 command-line tools (h5ls, h5stat, h5dump, h5diff) on `PATH`.
|
||||
//! Each test skips when what it needs is missing, unless
|
||||
//! `CLAWHDF5_REQUIRE_INTEROP=1`, which turns a skip into a failure.
|
||||
|
||||
use std::collections::BTreeMap;
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::process::{Command, Output};
|
||||
|
||||
use clawhdf5_format::checksum::jenkins_lookup3;
|
||||
|
||||
fn python() -> String {
|
||||
std::env::var("CLAWHDF5_PYTHON").unwrap_or_else(|_| "python3".to_string())
|
||||
}
|
||||
|
||||
fn interop_required() -> bool {
|
||||
std::env::var("CLAWHDF5_REQUIRE_INTEROP").is_ok_and(|v| v == "1")
|
||||
}
|
||||
|
||||
fn python_available() -> bool {
|
||||
Command::new(python())
|
||||
.args(["-c", "import h5py, numpy"])
|
||||
.output()
|
||||
.map(|o| o.status.success())
|
||||
.unwrap_or(false)
|
||||
}
|
||||
|
||||
fn tool_available(name: &str) -> bool {
|
||||
Command::new(name)
|
||||
.arg("--version")
|
||||
.output()
|
||||
.map(|o| o.status.success())
|
||||
.unwrap_or(false)
|
||||
}
|
||||
|
||||
/// Skip (return true) when `what` is unavailable, or fail when interop is
|
||||
/// required.
|
||||
fn missing(ok: bool, what: &str) -> bool {
|
||||
if ok {
|
||||
return false;
|
||||
}
|
||||
assert!(
|
||||
!interop_required(),
|
||||
"CLAWHDF5_REQUIRE_INTEROP=1 but {what} is not available"
|
||||
);
|
||||
eprintln!("SKIP: {what} not available");
|
||||
true
|
||||
}
|
||||
|
||||
fn h5rs(args: &[&str]) -> Output {
|
||||
Command::new(env!("CARGO_BIN_EXE_h5rs"))
|
||||
.args(args)
|
||||
.output()
|
||||
.expect("run h5rs")
|
||||
}
|
||||
|
||||
fn run(tool: &str, args: &[&str]) -> Output {
|
||||
Command::new(tool).args(args).output().expect("run tool")
|
||||
}
|
||||
|
||||
fn stdout(o: &Output) -> String {
|
||||
String::from_utf8_lossy(&o.stdout).into_owned()
|
||||
}
|
||||
|
||||
/// The generated files and what h5py read from them.
|
||||
struct Files {
|
||||
dir: tempfile::TempDir,
|
||||
values: serde_json::Value,
|
||||
}
|
||||
|
||||
impl Files {
|
||||
fn path(&self, name: &str) -> PathBuf {
|
||||
self.dir.path().join(name)
|
||||
}
|
||||
|
||||
fn p(&self, name: &str) -> String {
|
||||
self.path(name).to_string_lossy().into_owned()
|
||||
}
|
||||
}
|
||||
|
||||
fn generate() -> Option<Files> {
|
||||
if missing(python_available(), "python3 with h5py") {
|
||||
return None;
|
||||
}
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let script = Path::new(env!("CARGO_MANIFEST_DIR")).join("tests/gen_files.py");
|
||||
let out = Command::new(python())
|
||||
.arg(&script)
|
||||
.arg(dir.path())
|
||||
.output()
|
||||
.expect("run gen_files.py");
|
||||
assert!(
|
||||
out.status.success(),
|
||||
"gen_files.py failed:\n{}",
|
||||
String::from_utf8_lossy(&out.stderr)
|
||||
);
|
||||
let values = serde_json::from_slice(&out.stdout).expect("gen_files.py output");
|
||||
Some(Files { dir, values })
|
||||
}
|
||||
|
||||
const LIBVERS: [&str; 2] = ["earliest.h5", "latest.h5"];
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// ls
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/// Every h5ls line must be the start of the matching h5rs line (h5rs adds
|
||||
/// the datatype after a dataset's shape).
|
||||
fn assert_ls_like(ours: &str, reference: &str, what: &str) {
|
||||
let ours: Vec<&str> = ours.lines().collect();
|
||||
let refs: Vec<&str> = reference.lines().collect();
|
||||
assert_eq!(
|
||||
ours.len(),
|
||||
refs.len(),
|
||||
"{what}: line count\nh5rs:\n{}\nh5ls:\n{}",
|
||||
ours.join("\n"),
|
||||
refs.join("\n")
|
||||
);
|
||||
for (o, r) in ours.iter().zip(&refs) {
|
||||
let r = r.trim_end();
|
||||
assert!(
|
||||
o.starts_with(r) && (o.len() == r.len() || r.contains("Dataset {")),
|
||||
"{what}:\n h5rs: {o}\n h5ls: {r}"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn ls_matches_h5ls() {
|
||||
let Some(f) = generate() else { return };
|
||||
if missing(tool_available("h5ls"), "h5ls") {
|
||||
return;
|
||||
}
|
||||
for name in LIBVERS.iter().chain(&["userblock.h5"]) {
|
||||
let p = f.p(name);
|
||||
for (args, what) in [(vec!["-r"], "recursive"), (vec![], "root")] {
|
||||
let mut a: Vec<&str> = args.clone();
|
||||
a.push(&p);
|
||||
let ours = h5rs(&[&["ls"], a.as_slice()].concat());
|
||||
assert!(ours.status.success(), "{name} {what}: {:?}", ours);
|
||||
let reference = run("h5ls", &a);
|
||||
assert_ls_like(
|
||||
&stdout(&ours),
|
||||
&stdout(&reference),
|
||||
&format!("{name} {what}"),
|
||||
);
|
||||
}
|
||||
}
|
||||
// A group, and a dataset, named after the file.
|
||||
for obj in ["/grp", "/grp/sub", "/contig", "/grp/ext2"] {
|
||||
let arg = format!("{}{obj}", f.p("latest.h5"));
|
||||
let ours = h5rs(&["ls", &arg]);
|
||||
let reference = run("h5ls", &[&arg]);
|
||||
assert_ls_like(&stdout(&ours), &stdout(&reference), &arg);
|
||||
let ours = h5rs(&["ls", "-r", &arg]);
|
||||
let reference = run("h5ls", &["-r", &arg]);
|
||||
assert_ls_like(&stdout(&ours), &stdout(&reference), &format!("-r {arg}"));
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn ls_verbose_describes_layout_filters_and_attributes() {
|
||||
let Some(f) = generate() else { return };
|
||||
let o = h5rs(&["ls", "-r", "-v", &f.p("latest.h5")]);
|
||||
assert!(o.status.success());
|
||||
let s = stdout(&o);
|
||||
for want in [
|
||||
"Layout: chunked (extensible array index)",
|
||||
"Layout: chunked (v2 B-tree index)",
|
||||
"Layout: chunked (fixed array index)",
|
||||
"Layout: chunked (single chunk index)",
|
||||
"Filter-0: shuffle-2",
|
||||
"Filter-1: deflate-1",
|
||||
"Filter-2: fletcher32-3",
|
||||
"Chunks: {100} 400 bytes",
|
||||
"Attribute: units scalar",
|
||||
"Data: \"m\"",
|
||||
] {
|
||||
assert!(s.contains(want), "missing {want:?} in\n{s}");
|
||||
}
|
||||
let o = h5rs(&["ls", "-v", &format!("{}/grp", f.p("earliest.h5"))]);
|
||||
assert!(stdout(&o).contains("Layout: chunked (v1 B-tree index)"));
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// stat
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/// `key: value` lines of an h5stat-style report.
|
||||
fn stat_facts(s: &str) -> BTreeMap<String, String> {
|
||||
s.lines()
|
||||
.filter_map(|l| {
|
||||
let (k, v) = l.rsplit_once(':')?;
|
||||
Some((k.trim().to_string(), v.trim().to_string()))
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn stat_matches_h5stat() {
|
||||
let Some(f) = generate() else { return };
|
||||
if missing(tool_available("h5stat"), "h5stat") {
|
||||
return;
|
||||
}
|
||||
let keys = [
|
||||
"# of unique groups",
|
||||
"# of unique datasets",
|
||||
"# of unique named datatypes",
|
||||
"# of unique links",
|
||||
"# of unique other",
|
||||
"Max. # of links to object",
|
||||
"Max. # of objects in group",
|
||||
"Max. rank of datasets",
|
||||
"Max. dimension size of 1-D datasets",
|
||||
"Total raw data size",
|
||||
"Dataset layout counts[COMPACT]",
|
||||
"Dataset layout counts[CONTIG]",
|
||||
"Dataset layout counts[CHUNKED]",
|
||||
"Dataset layout counts[VIRTUAL]",
|
||||
"NO filter",
|
||||
"GZIP filter",
|
||||
"SHUFFLE filter",
|
||||
"FLETCHER32 filter",
|
||||
"SZIP filter",
|
||||
"NBIT filter",
|
||||
"SCALEOFFSET filter",
|
||||
"USER-DEFINED filter",
|
||||
"Max. # of attributes to objects",
|
||||
"Total space",
|
||||
];
|
||||
for name in LIBVERS.iter().chain(&["userblock.h5", "base.h5"]) {
|
||||
let p = f.p(name);
|
||||
let ours = h5rs(&["stat", &p]);
|
||||
assert!(ours.status.success(), "{name}: {ours:?}");
|
||||
let reference = run("h5stat", &[&p]);
|
||||
assert!(reference.status.success(), "h5stat {name} failed");
|
||||
let (o, r) = (stat_facts(&stdout(&ours)), stat_facts(&stdout(&reference)));
|
||||
for k in keys {
|
||||
// h5stat leaves out the attribute summary when nothing has one.
|
||||
let Some(want) = r.get(k) else { continue };
|
||||
assert_eq!(o.get(k), Some(want), "{name}: {k}");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// dump
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
#[test]
|
||||
fn dump_matches_h5dump() {
|
||||
let Some(f) = generate() else { return };
|
||||
if missing(tool_available("h5dump"), "h5dump") {
|
||||
return;
|
||||
}
|
||||
for name in LIBVERS.iter().chain(&["userblock.h5", "changed.h5"]) {
|
||||
let p = f.p(name);
|
||||
for args in [vec![], vec!["-A"]] {
|
||||
let mut a = args.clone();
|
||||
a.push(p.as_str());
|
||||
let ours = h5rs(&[&["dump"], a.as_slice()].concat());
|
||||
assert!(ours.status.success(), "{name} {args:?}: {ours:?}");
|
||||
let reference = run("h5dump", &a);
|
||||
// h5dump names the file as given; h5rs by its file name.
|
||||
let r = stdout(&reference).replacen(&p, name, 1);
|
||||
assert_eq!(stdout(&ours), r, "{name} {args:?}");
|
||||
}
|
||||
}
|
||||
let p = f.p("latest.h5");
|
||||
let ours = h5rs(&["dump", "-d", "/grp/ext2", &p]);
|
||||
let reference = run("h5dump", &["-d", "/grp/ext2", &p]);
|
||||
assert_eq!(
|
||||
stdout(&ours),
|
||||
stdout(&reference).replacen(&p, "latest.h5", 1)
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn dump_json_values_match_h5py() {
|
||||
let Some(f) = generate() else { return };
|
||||
for name in LIBVERS {
|
||||
let o = h5rs(&["dump", "--json", &f.p(name)]);
|
||||
assert!(o.status.success(), "{name}: {o:?}");
|
||||
let doc: serde_json::Value = serde_json::from_slice(&o.stdout).unwrap();
|
||||
assert_eq!(doc["apiVersion"], "1.1.1");
|
||||
let root = doc["root"].as_str().unwrap();
|
||||
assert!(doc["groups"][root].is_object());
|
||||
let mut by_path = BTreeMap::new();
|
||||
for (_, d) in doc["datasets"].as_object().unwrap() {
|
||||
for a in d["alias"].as_array().unwrap() {
|
||||
by_path.insert(a.as_str().unwrap().to_string(), d.clone());
|
||||
}
|
||||
}
|
||||
let want = f.values[name.trim_end_matches(".h5")].as_object().unwrap();
|
||||
assert!(want.len() > 10);
|
||||
for (path, v) in want {
|
||||
let d = by_path
|
||||
.get(path)
|
||||
.unwrap_or_else(|| panic!("{name}: no {path}"));
|
||||
let got = &d["value"];
|
||||
// JSON numbers: compare as f64 (h5py's ints and floats both
|
||||
// round-trip exactly through JSON here).
|
||||
assert_eq!(flatten(got), flatten(v), "{name}: {path}");
|
||||
}
|
||||
let links = doc["groups"][root]["links"].as_array().unwrap();
|
||||
let soft = links.iter().find(|l| l["title"] == "soft").unwrap();
|
||||
assert_eq!(soft["class"], "H5L_TYPE_SOFT");
|
||||
assert_eq!(soft["h5path"], "/contig");
|
||||
let ext = links.iter().find(|l| l["title"] == "external").unwrap();
|
||||
assert_eq!(ext["class"], "H5L_TYPE_EXTERNAL");
|
||||
assert_eq!(ext["file"], "other.h5");
|
||||
}
|
||||
}
|
||||
|
||||
fn flatten(v: &serde_json::Value) -> Vec<f64> {
|
||||
match v {
|
||||
serde_json::Value::Array(a) => a.iter().flat_map(flatten).collect(),
|
||||
serde_json::Value::Number(n) => vec![n.as_f64().unwrap()],
|
||||
other => panic!("not numeric: {other}"),
|
||||
}
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// diff
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
fn code(o: &Output) -> i32 {
|
||||
o.status.code().expect("exit code")
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn diff_exit_codes_match_h5diff() {
|
||||
let Some(f) = generate() else { return };
|
||||
if missing(tool_available("h5diff"), "h5diff") {
|
||||
return;
|
||||
}
|
||||
let (b, same, changed, extra, attr) = (
|
||||
f.p("base.h5"),
|
||||
f.p("same.h5"),
|
||||
f.p("changed.h5"),
|
||||
f.p("extra.h5"),
|
||||
f.p("attr.h5"),
|
||||
);
|
||||
let missing_file = f.p("does-not-exist.h5");
|
||||
let cases: Vec<(Vec<&str>, i32)> = vec![
|
||||
(vec![&b, &same], 0),
|
||||
(vec![&b, &b], 0),
|
||||
(vec![&b, &changed], 1),
|
||||
(vec![&b, &changed, "/d"], 1),
|
||||
(vec![&b, &changed, "/g"], 0),
|
||||
(vec!["-d", "0.01", &b, &changed], 0),
|
||||
(vec!["-d", "0.0001", &b, &changed], 1),
|
||||
(vec!["-p", "0.01", &b, &changed], 0),
|
||||
(vec!["-p", "1e-9", &b, &changed], 1),
|
||||
(vec![&b, &extra], 1),
|
||||
(vec![&b, &attr], 1),
|
||||
(vec![&b, &attr, "/g"], 0),
|
||||
(vec![&b, &same, "/nope"], 2),
|
||||
(vec![&b, &missing_file], 2),
|
||||
(vec![&b, &same, "/d", "/d"], 0),
|
||||
];
|
||||
for (args, want) in cases {
|
||||
let theirs = code(&run("h5diff", &args));
|
||||
assert_eq!(theirs, want, "h5diff {args:?} (test expectation)");
|
||||
let o = h5rs(&[&["diff"], args.as_slice()].concat());
|
||||
assert_eq!(code(&o), want, "h5rs diff {args:?}: {}", stdout(&o));
|
||||
let q = h5rs(&[&["diff", "-q"], args.as_slice()].concat());
|
||||
assert_eq!(code(&q), want, "h5rs diff -q {args:?}");
|
||||
assert!(q.stdout.is_empty(), "-q printed output");
|
||||
}
|
||||
// The report lists the differing positions.
|
||||
let o = h5rs(&["diff", "-r", &b, &changed, "/d"]);
|
||||
let s = stdout(&o);
|
||||
assert!(s.contains("[ 0 2 ]") && s.contains("[ 2 3 ]"), "{s}");
|
||||
assert!(s.contains("2 difference(s) found"), "{s}");
|
||||
}
|
||||
|
||||
/// Where h5rs deliberately differs from h5diff: objects that cannot be
|
||||
/// compared (another shape, another datatype class) are a difference.
|
||||
#[test]
|
||||
fn diff_counts_incomparable_objects_as_different() {
|
||||
let Some(f) = generate() else { return };
|
||||
let b = f.p("base.h5");
|
||||
for other in ["reshaped.h5", "int.h5"] {
|
||||
let o = h5rs(&["diff", &b, &f.p(other)]);
|
||||
assert_eq!(code(&o), 1, "{other}: {}", stdout(&o));
|
||||
assert!(stdout(&o).contains("Not comparable"), "{other}");
|
||||
}
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// check
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
#[test]
|
||||
fn check_accepts_valid_files() {
|
||||
let Some(f) = generate() else { return };
|
||||
for name in LIBVERS.iter().chain(&[
|
||||
"userblock.h5",
|
||||
"base.h5",
|
||||
"changed.h5",
|
||||
"extra.h5",
|
||||
"attr.h5",
|
||||
]) {
|
||||
let o = h5rs(&["check", "--data", &f.p(name)]);
|
||||
assert_eq!(code(&o), 0, "{name}:\n{}", stdout(&o));
|
||||
assert!(stdout(&o).contains("no problems found"));
|
||||
}
|
||||
// The newest format's checksummed structures were all verified.
|
||||
let s = stdout(&h5rs(&["check", &f.p("latest.h5")]));
|
||||
assert!(s.contains("superblock 1, v2 object headers 33,"), "{s}");
|
||||
assert!(s.contains("chunk indexes 5"), "{s}");
|
||||
}
|
||||
|
||||
/// Signatures of checksummed structures, and where each one's checksum
|
||||
/// is: found by trying every end offset (the checksum is the Jenkins
|
||||
/// lookup3 hash of the bytes before it). A fractal heap direct block's
|
||||
/// checksum is instead stored near its start, computed over the whole block
|
||||
/// with the field zeroed.
|
||||
fn checksummed_structures(data: &[u8]) -> Vec<(&'static str, usize, usize)> {
|
||||
const SIGS: [&str; 13] = [
|
||||
"OHDR", "OCHK", "BTHD", "BTIN", "BTLF", "FRHP", "FHIB", "EAHD", "EAIB", "EASB", "EADB",
|
||||
"FAHD", "FADB",
|
||||
];
|
||||
let mut found = Vec::new();
|
||||
for start in 0..data.len().saturating_sub(4) {
|
||||
let sig = &data[start..start + 4];
|
||||
if let Some(name) = SIGS.iter().find(|s| s.as_bytes() == sig) {
|
||||
let limit = data.len().min(start + 65536);
|
||||
for end in start + 8..limit.saturating_sub(4) {
|
||||
let stored = u32::from_le_bytes(data[end..end + 4].try_into().unwrap());
|
||||
if jenkins_lookup3(&data[start..end]) == stored {
|
||||
found.push((*name, start, end));
|
||||
break;
|
||||
}
|
||||
}
|
||||
} else if sig == b"FHDB" {
|
||||
// version(1) heap address(8) block offset(1..8) checksum(4)
|
||||
'found: for boff in 1..=8usize {
|
||||
let at = start + 5 + 8 + boff;
|
||||
let Some(stored) = data.get(at..at + 4) else {
|
||||
break;
|
||||
};
|
||||
let stored = u32::from_le_bytes(stored.try_into().unwrap());
|
||||
for size in [512usize, 1024, 2048, 4096, 8192, 16384, 32768, 65536] {
|
||||
let Some(block) = data.get(start..start + size) else {
|
||||
break;
|
||||
};
|
||||
let mut b = block.to_vec();
|
||||
b[at - start..at - start + 4].fill(0);
|
||||
if jenkins_lookup3(&b) == stored {
|
||||
found.push(("FHDB", start, at));
|
||||
break 'found;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
found
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn check_flags_every_corrupted_checksum() {
|
||||
let Some(f) = generate() else { return };
|
||||
let path = f.path("latest.h5");
|
||||
let data = std::fs::read(&path).unwrap();
|
||||
let structures = checksummed_structures(&data);
|
||||
let kinds: std::collections::BTreeSet<&str> = structures.iter().map(|s| s.0).collect();
|
||||
for want in [
|
||||
"OHDR", "OCHK", "BTHD", "BTLF", "FRHP", "FHDB", "EAHD", "EAIB", "EADB", "FAHD", "FADB",
|
||||
] {
|
||||
assert!(kinds.contains(want), "test file has no {want}: {kinds:?}");
|
||||
}
|
||||
let bad = f.path("bad.h5");
|
||||
for (name, start, at) in structures {
|
||||
let mut d = data.clone();
|
||||
d[at] ^= 0x01;
|
||||
std::fs::write(&bad, &d).unwrap();
|
||||
let o = h5rs(&["check", &bad.to_string_lossy()]);
|
||||
let s = stdout(&o);
|
||||
assert_eq!(
|
||||
code(&o),
|
||||
1,
|
||||
"{name} at {start:#x}: checksum flip not flagged\n{s}"
|
||||
);
|
||||
assert!(
|
||||
s.lines()
|
||||
.any(|l| l.starts_with("problem: 0x") && l.contains("checksum")),
|
||||
"{name} at {start:#x}: no checksum problem reported\n{s}"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn check_flags_a_truncated_file() {
|
||||
let Some(f) = generate() else { return };
|
||||
let data = std::fs::read(f.path("latest.h5")).unwrap();
|
||||
let bad = f.path("truncated.h5");
|
||||
std::fs::write(&bad, &data[..data.len() * 3 / 4]).unwrap();
|
||||
let o = h5rs(&["check", &bad.to_string_lossy()]);
|
||||
assert_eq!(code(&o), 1);
|
||||
assert!(stdout(&o).contains("file is truncated"), "{}", stdout(&o));
|
||||
}
|
||||
|
||||
/// A v1 B-tree chunk index has no checksum; its keys are checked against
|
||||
/// the dataset: a chunk offset that is not a multiple of the chunk size.
|
||||
#[test]
|
||||
fn check_flags_a_misaligned_chunk() {
|
||||
let Some(f) = generate() else { return };
|
||||
let path = f.path("earliest.h5");
|
||||
let mut data = std::fs::read(&path).unwrap();
|
||||
// The leaf chunk B-tree of /grp/gz (10 deflated chunks of 100 i32, so
|
||||
// each stored in under 400 bytes): "TREE" type(1)=1 level(1)=0
|
||||
// entries(2) left(8) right(8), then key, child, key, child, ... where a
|
||||
// rank-1 chunk key is size(4) mask(4) offsets(2 x 8) and a child is 8.
|
||||
// Key 1 starts at 24 + 24 + 8 = 56; its dimension-0 offset (100) at 64.
|
||||
let mut patched = false;
|
||||
for start in 0..data.len() - 72 {
|
||||
if &data[start..start + 4] == b"TREE" && data[start + 4] == 1 && data[start + 5] == 0 {
|
||||
let entries = u16::from_le_bytes(data[start + 6..start + 8].try_into().unwrap());
|
||||
let size0 = u32::from_le_bytes(data[start + 24..start + 28].try_into().unwrap());
|
||||
let off1 = u64::from_le_bytes(data[start + 64..start + 72].try_into().unwrap());
|
||||
if entries == 10 && size0 < 400 && off1 == 100 {
|
||||
data[start + 64..start + 72].copy_from_slice(&103u64.to_le_bytes());
|
||||
patched = true;
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
assert!(patched, "no chunk B-tree found for /grp/gz");
|
||||
let bad = f.path("misaligned.h5");
|
||||
std::fs::write(&bad, &data).unwrap();
|
||||
let o = h5rs(&["check", &bad.to_string_lossy()]);
|
||||
let s = stdout(&o);
|
||||
assert_eq!(code(&o), 1, "{s}");
|
||||
assert!(
|
||||
s.contains("/grp/gz: chunk at [103, 0] offset 103 in dimension 0 is not a multiple of the chunk size 100"),
|
||||
"{s}"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn every_subcommand_rejects_a_non_hdf5_file_cleanly() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let p = dir.path().join("junk.h5");
|
||||
std::fs::write(&p, b"this is not an HDF5 file at all").unwrap();
|
||||
let p = p.to_string_lossy().into_owned();
|
||||
for args in [
|
||||
vec!["ls", p.as_str()],
|
||||
vec!["dump", p.as_str()],
|
||||
vec!["stat", p.as_str()],
|
||||
vec!["diff", p.as_str(), p.as_str()],
|
||||
] {
|
||||
let o = h5rs(&args);
|
||||
assert_eq!(code(&o), 2, "{args:?}");
|
||||
assert!(String::from_utf8_lossy(&o.stderr).contains("not an HDF5 file"));
|
||||
}
|
||||
let o = h5rs(&["check", &p]);
|
||||
assert_eq!(code(&o), 1);
|
||||
assert!(stdout(&o).contains("no HDF5 signature"));
|
||||
assert_eq!(code(&h5rs(&["nonsense"])), 2);
|
||||
assert_eq!(code(&h5rs(&["ls"])), 2);
|
||||
assert_eq!(code(&h5rs(&["--help"])), 0);
|
||||
}
|
||||
Reference in New Issue
Block a user