osobhandClaude Opus 5.5
193a5f8a82
h5py's track_order=True orders attributes as well as links; the writer tracked links only. A tracking object's header now sets the attribute creation order tracked/indexed flags and carries per-message creation orders, an Attribute Info message holds the next order (inline too), and dense storage gets a type-9 creation-order index. The file default applies to datasets, with DatasetBuilder::track_order per dataset; more than 65 535 attributes on a tracking object is an error (libhdf5's counter is 2 bytes). The reader lists such attributes in creation order. h5py lists them in order (inline, dense, 20 000 on one dataset) and keeps numbering in r+ mode, including its inline-to-dense move. Co-Authored-By: Claude Opus 5.5 (1M context) <[email protected]>
1105 lines
42 KiB
Rust
1105 lines
42 KiB
Rust
//! `h5rs` against libhdf5's own tools, on files h5py writes
|
||
//! (`tests/gen_files.py`): `ls` against h5ls, `stat` against h5stat, `dump`
|
||
//! against h5dump, `diff` exit codes against h5diff, `dump --json` values
|
||
//! against h5py, and `check` on valid and deliberately corrupted files.
|
||
//!
|
||
//! Needs python3 with h5py (`CLAWHDF5_PYTHON`) and, for the comparisons,
|
||
//! the libhdf5 command-line tools (h5ls, h5stat, h5dump, h5diff) on `PATH`.
|
||
//! Each test skips when what it needs is missing, unless
|
||
//! `CLAWHDF5_REQUIRE_INTEROP=1`, which turns a skip into a failure.
|
||
|
||
use std::collections::BTreeMap;
|
||
use std::path::{Path, PathBuf};
|
||
use std::process::{Command, Output};
|
||
|
||
use clawhdf5_format::checksum::jenkins_lookup3;
|
||
|
||
fn python() -> String {
|
||
std::env::var("CLAWHDF5_PYTHON").unwrap_or_else(|_| "python3".to_string())
|
||
}
|
||
|
||
fn interop_required() -> bool {
|
||
std::env::var("CLAWHDF5_REQUIRE_INTEROP").is_ok_and(|v| v == "1")
|
||
}
|
||
|
||
fn python_available() -> bool {
|
||
Command::new(python())
|
||
.args(["-c", "import h5py, numpy"])
|
||
.output()
|
||
.map(|o| o.status.success())
|
||
.unwrap_or(false)
|
||
}
|
||
|
||
fn tool_available(name: &str) -> bool {
|
||
Command::new(name)
|
||
.arg("--version")
|
||
.output()
|
||
.map(|o| o.status.success())
|
||
.unwrap_or(false)
|
||
}
|
||
|
||
/// Skip (return true) when `what` is unavailable, or fail when interop is
|
||
/// required.
|
||
fn missing(ok: bool, what: &str) -> bool {
|
||
if ok {
|
||
return false;
|
||
}
|
||
assert!(
|
||
!interop_required(),
|
||
"CLAWHDF5_REQUIRE_INTEROP=1 but {what} is not available"
|
||
);
|
||
eprintln!("SKIP: {what} not available");
|
||
true
|
||
}
|
||
|
||
fn h5rs(args: &[&str]) -> Output {
|
||
Command::new(env!("CARGO_BIN_EXE_h5rs"))
|
||
.args(args)
|
||
.output()
|
||
.expect("run h5rs")
|
||
}
|
||
|
||
fn run(tool: &str, args: &[&str]) -> Output {
|
||
Command::new(tool).args(args).output().expect("run tool")
|
||
}
|
||
|
||
fn stdout(o: &Output) -> String {
|
||
String::from_utf8_lossy(&o.stdout).into_owned()
|
||
}
|
||
|
||
/// The generated files and what h5py read from them.
|
||
struct Files {
|
||
dir: tempfile::TempDir,
|
||
values: serde_json::Value,
|
||
}
|
||
|
||
impl Files {
|
||
fn path(&self, name: &str) -> PathBuf {
|
||
self.dir.path().join(name)
|
||
}
|
||
|
||
fn p(&self, name: &str) -> String {
|
||
self.path(name).to_string_lossy().into_owned()
|
||
}
|
||
}
|
||
|
||
fn generate() -> Option<Files> {
|
||
if missing(python_available(), "python3 with h5py") {
|
||
return None;
|
||
}
|
||
let dir = tempfile::tempdir().unwrap();
|
||
let script = Path::new(env!("CARGO_MANIFEST_DIR")).join("tests/gen_files.py");
|
||
let out = Command::new(python())
|
||
.arg(&script)
|
||
.arg(dir.path())
|
||
.output()
|
||
.expect("run gen_files.py");
|
||
assert!(
|
||
out.status.success(),
|
||
"gen_files.py failed:\n{}",
|
||
String::from_utf8_lossy(&out.stderr)
|
||
);
|
||
let values = serde_json::from_slice(&out.stdout).expect("gen_files.py output");
|
||
Some(Files { dir, values })
|
||
}
|
||
|
||
const LIBVERS: [&str; 2] = ["earliest.h5", "latest.h5"];
|
||
|
||
// ---------------------------------------------------------------------------
|
||
// ls
|
||
// ---------------------------------------------------------------------------
|
||
|
||
/// Every h5ls line must be the start of the matching h5rs line (h5rs adds
|
||
/// the datatype after a dataset's shape).
|
||
fn assert_ls_like(ours: &str, reference: &str, what: &str) {
|
||
let ours: Vec<&str> = ours.lines().collect();
|
||
let refs: Vec<&str> = reference.lines().collect();
|
||
assert_eq!(
|
||
ours.len(),
|
||
refs.len(),
|
||
"{what}: line count\nh5rs:\n{}\nh5ls:\n{}",
|
||
ours.join("\n"),
|
||
refs.join("\n")
|
||
);
|
||
for (o, r) in ours.iter().zip(&refs) {
|
||
let r = r.trim_end();
|
||
assert!(
|
||
o.starts_with(r) && (o.len() == r.len() || r.contains("Dataset {")),
|
||
"{what}:\n h5rs: {o}\n h5ls: {r}"
|
||
);
|
||
}
|
||
}
|
||
|
||
#[test]
|
||
fn ls_matches_h5ls() {
|
||
let Some(f) = generate() else { return };
|
||
if missing(tool_available("h5ls"), "h5ls") {
|
||
return;
|
||
}
|
||
for name in LIBVERS.iter().chain(&["userblock.h5"]) {
|
||
let p = f.p(name);
|
||
for (args, what) in [(vec!["-r"], "recursive"), (vec![], "root")] {
|
||
let mut a: Vec<&str> = args.clone();
|
||
a.push(&p);
|
||
let ours = h5rs(&[&["ls"], a.as_slice()].concat());
|
||
assert!(ours.status.success(), "{name} {what}: {:?}", ours);
|
||
let reference = run("h5ls", &a);
|
||
assert_ls_like(
|
||
&stdout(&ours),
|
||
&stdout(&reference),
|
||
&format!("{name} {what}"),
|
||
);
|
||
}
|
||
}
|
||
// A group, and a dataset, named after the file.
|
||
for obj in ["/grp", "/grp/sub", "/contig", "/grp/ext2"] {
|
||
let arg = format!("{}{obj}", f.p("latest.h5"));
|
||
let ours = h5rs(&["ls", &arg]);
|
||
let reference = run("h5ls", &[&arg]);
|
||
assert_ls_like(&stdout(&ours), &stdout(&reference), &arg);
|
||
let ours = h5rs(&["ls", "-r", &arg]);
|
||
let reference = run("h5ls", &["-r", &arg]);
|
||
assert_ls_like(&stdout(&ours), &stdout(&reference), &format!("-r {arg}"));
|
||
}
|
||
}
|
||
|
||
#[test]
|
||
fn ls_verbose_describes_layout_filters_and_attributes() {
|
||
let Some(f) = generate() else { return };
|
||
let o = h5rs(&["ls", "-r", "-v", &f.p("latest.h5")]);
|
||
assert!(o.status.success());
|
||
let s = stdout(&o);
|
||
for want in [
|
||
"Layout: chunked (extensible array index)",
|
||
"Layout: chunked (v2 B-tree index)",
|
||
"Layout: chunked (fixed array index)",
|
||
"Layout: chunked (single chunk index)",
|
||
"Filter-0: shuffle-2",
|
||
"Filter-1: deflate-1",
|
||
"Filter-2: fletcher32-3",
|
||
"Chunks: {100} 400 bytes",
|
||
"Attribute: units scalar",
|
||
"Data: \"m\"",
|
||
] {
|
||
assert!(s.contains(want), "missing {want:?} in\n{s}");
|
||
}
|
||
let o = h5rs(&["ls", "-v", &format!("{}/grp", f.p("earliest.h5"))]);
|
||
assert!(stdout(&o).contains("Layout: chunked (v1 B-tree index)"));
|
||
}
|
||
|
||
// ---------------------------------------------------------------------------
|
||
// stat
|
||
// ---------------------------------------------------------------------------
|
||
|
||
/// `key: value` lines of an h5stat-style report.
|
||
fn stat_facts(s: &str) -> BTreeMap<String, String> {
|
||
s.lines()
|
||
.filter_map(|l| {
|
||
let (k, v) = l.rsplit_once(':')?;
|
||
Some((k.trim().to_string(), v.trim().to_string()))
|
||
})
|
||
.collect()
|
||
}
|
||
|
||
#[test]
|
||
fn stat_matches_h5stat() {
|
||
let Some(f) = generate() else { return };
|
||
if missing(tool_available("h5stat"), "h5stat") {
|
||
return;
|
||
}
|
||
let keys = [
|
||
"# of unique groups",
|
||
"# of unique datasets",
|
||
"# of unique named datatypes",
|
||
"# of unique links",
|
||
"# of unique other",
|
||
"Max. # of links to object",
|
||
"Max. # of objects in group",
|
||
"Max. rank of datasets",
|
||
"Max. dimension size of 1-D datasets",
|
||
"Total raw data size",
|
||
"Dataset layout counts[COMPACT]",
|
||
"Dataset layout counts[CONTIG]",
|
||
"Dataset layout counts[CHUNKED]",
|
||
"Dataset layout counts[VIRTUAL]",
|
||
"NO filter",
|
||
"GZIP filter",
|
||
"SHUFFLE filter",
|
||
"FLETCHER32 filter",
|
||
"SZIP filter",
|
||
"NBIT filter",
|
||
"SCALEOFFSET filter",
|
||
"USER-DEFINED filter",
|
||
"Max. # of attributes to objects",
|
||
"Total space",
|
||
];
|
||
for name in LIBVERS.iter().chain(&["userblock.h5", "base.h5"]) {
|
||
let p = f.p(name);
|
||
let ours = h5rs(&["stat", &p]);
|
||
assert!(ours.status.success(), "{name}: {ours:?}");
|
||
let reference = run("h5stat", &[&p]);
|
||
assert!(reference.status.success(), "h5stat {name} failed");
|
||
let (o, r) = (stat_facts(&stdout(&ours)), stat_facts(&stdout(&reference)));
|
||
for k in keys {
|
||
// h5stat leaves out the attribute summary when nothing has one.
|
||
let Some(want) = r.get(k) else { continue };
|
||
assert_eq!(o.get(k), Some(want), "{name}: {k}");
|
||
}
|
||
}
|
||
}
|
||
|
||
// ---------------------------------------------------------------------------
|
||
// dump
|
||
// ---------------------------------------------------------------------------
|
||
|
||
#[test]
|
||
fn dump_matches_h5dump() {
|
||
let Some(f) = generate() else { return };
|
||
if missing(tool_available("h5dump"), "h5dump") {
|
||
return;
|
||
}
|
||
for name in LIBVERS.iter().chain(&["userblock.h5", "changed.h5"]) {
|
||
let p = f.p(name);
|
||
for args in [vec![], vec!["-A"]] {
|
||
let mut a = args.clone();
|
||
a.push(p.as_str());
|
||
let ours = h5rs(&[&["dump"], a.as_slice()].concat());
|
||
assert!(ours.status.success(), "{name} {args:?}: {ours:?}");
|
||
let reference = run("h5dump", &a);
|
||
// h5dump names the file as given; h5rs by its file name.
|
||
let r = stdout(&reference).replacen(&p, name, 1);
|
||
assert_eq!(stdout(&ours), r, "{name} {args:?}");
|
||
}
|
||
}
|
||
let p = f.p("latest.h5");
|
||
let ours = h5rs(&["dump", "-d", "/grp/ext2", &p]);
|
||
let reference = run("h5dump", &["-d", "/grp/ext2", &p]);
|
||
assert_eq!(
|
||
stdout(&ours),
|
||
stdout(&reference).replacen(&p, "latest.h5", 1)
|
||
);
|
||
}
|
||
|
||
/// A null-padded fixed string prints every byte (NULs as `\000`), as
|
||
/// h5dump prints it, also inside a compound and an array member.
|
||
#[test]
|
||
fn dump_shows_nul_padding_in_nested_strings() {
|
||
let Some(f) = generate() else { return };
|
||
let p = f.p("nulstrings.h5");
|
||
let ours = stdout(&h5rs(&["dump", &p]));
|
||
for want in [
|
||
r#"(0): "\000\000\000", "ab\000", "a\000b""#,
|
||
r#""\000\000\000","#,
|
||
r#""a\000b","#,
|
||
r#"[ "\000\000", "x\000" ]"#,
|
||
] {
|
||
assert!(ours.contains(want), "no {want} in\n{ours}");
|
||
}
|
||
if missing(tool_available("h5dump"), "h5dump") {
|
||
return;
|
||
}
|
||
let reference = run("h5dump", &[&p]);
|
||
assert_eq!(ours, stdout(&reference).replacen(&p, "nulstrings.h5", 1));
|
||
}
|
||
|
||
#[test]
|
||
fn dump_json_values_match_h5py() {
|
||
let Some(f) = generate() else { return };
|
||
for name in LIBVERS {
|
||
let o = h5rs(&["dump", "--json", &f.p(name)]);
|
||
assert!(o.status.success(), "{name}: {o:?}");
|
||
let doc: serde_json::Value = serde_json::from_slice(&o.stdout).unwrap();
|
||
assert_eq!(doc["apiVersion"], "1.1.1");
|
||
let root = doc["root"].as_str().unwrap();
|
||
assert!(doc["groups"][root].is_object());
|
||
let mut by_path = BTreeMap::new();
|
||
for (_, d) in doc["datasets"].as_object().unwrap() {
|
||
for a in d["alias"].as_array().unwrap() {
|
||
by_path.insert(a.as_str().unwrap().to_string(), d.clone());
|
||
}
|
||
}
|
||
let want = f.values[name.trim_end_matches(".h5")].as_object().unwrap();
|
||
assert!(want.len() > 10);
|
||
for (path, v) in want {
|
||
let d = by_path
|
||
.get(path)
|
||
.unwrap_or_else(|| panic!("{name}: no {path}"));
|
||
let got = &d["value"];
|
||
// JSON numbers: compare as f64 (h5py's ints and floats both
|
||
// round-trip exactly through JSON here).
|
||
assert_eq!(flatten(got), flatten(v), "{name}: {path}");
|
||
}
|
||
let links = doc["groups"][root]["links"].as_array().unwrap();
|
||
let soft = links.iter().find(|l| l["title"] == "soft").unwrap();
|
||
assert_eq!(soft["class"], "H5L_TYPE_SOFT");
|
||
assert_eq!(soft["h5path"], "/contig");
|
||
let ext = links.iter().find(|l| l["title"] == "external").unwrap();
|
||
assert_eq!(ext["class"], "H5L_TYPE_EXTERNAL");
|
||
assert_eq!(ext["file"], "other.h5");
|
||
}
|
||
}
|
||
|
||
fn flatten(v: &serde_json::Value) -> Vec<f64> {
|
||
match v {
|
||
serde_json::Value::Array(a) => a.iter().flat_map(flatten).collect(),
|
||
serde_json::Value::Number(n) => vec![n.as_f64().unwrap()],
|
||
other => panic!("not numeric: {other}"),
|
||
}
|
||
}
|
||
|
||
// ---------------------------------------------------------------------------
|
||
// diff
|
||
// ---------------------------------------------------------------------------
|
||
|
||
fn code(o: &Output) -> i32 {
|
||
o.status.code().expect("exit code")
|
||
}
|
||
|
||
#[test]
|
||
fn diff_exit_codes_match_h5diff() {
|
||
let Some(f) = generate() else { return };
|
||
if missing(tool_available("h5diff"), "h5diff") {
|
||
return;
|
||
}
|
||
let (b, same, changed, extra, attr) = (
|
||
f.p("base.h5"),
|
||
f.p("same.h5"),
|
||
f.p("changed.h5"),
|
||
f.p("extra.h5"),
|
||
f.p("attr.h5"),
|
||
);
|
||
let missing_file = f.p("does-not-exist.h5");
|
||
let cases: Vec<(Vec<&str>, i32)> = vec![
|
||
(vec![&b, &same], 0),
|
||
(vec![&b, &b], 0),
|
||
(vec![&b, &changed], 1),
|
||
(vec![&b, &changed, "/d"], 1),
|
||
(vec![&b, &changed, "/g"], 0),
|
||
(vec!["-d", "0.01", &b, &changed], 0),
|
||
(vec!["-d", "0.0001", &b, &changed], 1),
|
||
(vec!["-p", "0.01", &b, &changed], 0),
|
||
(vec!["-p", "1e-9", &b, &changed], 1),
|
||
(vec![&b, &extra], 1),
|
||
(vec![&b, &attr], 1),
|
||
(vec![&b, &attr, "/g"], 0),
|
||
(vec![&b, &same, "/nope"], 2),
|
||
(vec![&b, &missing_file], 2),
|
||
(vec![&b, &same, "/d", "/d"], 0),
|
||
];
|
||
for (args, want) in cases {
|
||
let theirs = code(&run("h5diff", &args));
|
||
assert_eq!(theirs, want, "h5diff {args:?} (test expectation)");
|
||
let o = h5rs(&[&["diff"], args.as_slice()].concat());
|
||
assert_eq!(code(&o), want, "h5rs diff {args:?}: {}", stdout(&o));
|
||
let q = h5rs(&[&["diff", "-q"], args.as_slice()].concat());
|
||
assert_eq!(code(&q), want, "h5rs diff -q {args:?}");
|
||
assert!(q.stdout.is_empty(), "-q printed output");
|
||
}
|
||
// The report lists the differing positions.
|
||
let o = h5rs(&["diff", "-r", &b, &changed, "/d"]);
|
||
let s = stdout(&o);
|
||
assert!(s.contains("[ 0 2 ]") && s.contains("[ 2 3 ]"), "{s}");
|
||
assert!(s.contains("2 difference(s) found"), "{s}");
|
||
}
|
||
|
||
/// The option names are h5diff's: -c is --compare (a flag), the count is
|
||
/// -n/--count, and the --name=value forms are accepted.
|
||
#[test]
|
||
fn diff_options_are_named_like_h5diff() {
|
||
let Some(f) = generate() else { return };
|
||
let (b, changed) = (f.p("base.h5"), f.p("changed.h5"));
|
||
let cases: Vec<(Vec<&str>, i32)> = vec![
|
||
// "2" is then the first file name, which does not exist.
|
||
(vec!["-r", "-c", "2", &b, &changed], 2),
|
||
(vec!["-c", &b, &changed], 1),
|
||
(vec!["--compare", &b, &b], 0),
|
||
(vec!["-r", "-n", "1", &b, &changed], 1),
|
||
(vec!["-r", "--count=1", &b, &changed], 1),
|
||
(vec!["--delta=0.01", &b, &changed], 0),
|
||
(vec!["--relative=1e-9", &b, &changed], 1),
|
||
];
|
||
let have_h5diff = tool_available("h5diff");
|
||
for (args, want) in cases {
|
||
if have_h5diff {
|
||
assert_eq!(code(&run("h5diff", &args)), want, "h5diff {args:?}");
|
||
}
|
||
let o = h5rs(&[&["diff"], args.as_slice()].concat());
|
||
assert_eq!(code(&o), want, "h5rs diff {args:?}: {}", stdout(&o));
|
||
}
|
||
for count in [vec!["-n", "1"], vec!["--count=1"]] {
|
||
let o = h5rs(&[&["diff", "-r"], count.as_slice(), &[&b, &changed, "/d"]].concat());
|
||
let s = stdout(&o);
|
||
assert!(
|
||
s.contains("[ 0 2 ]") && !s.contains("[ 2 3 ]"),
|
||
"{count:?}: {s}"
|
||
);
|
||
assert!(s.contains("2 difference(s) found"), "{s}");
|
||
}
|
||
assert_eq!(code(&h5rs(&["diff", "--bogus=1", &b, &b])), 2);
|
||
}
|
||
|
||
/// Tolerances on 64-bit integers one apart, far beyond f64's integer
|
||
/// precision: compared exactly, as h5diff does.
|
||
#[test]
|
||
fn diff_tolerances_compare_large_integers_exactly() {
|
||
let Some(f) = generate() else { return };
|
||
let (a, b) = (f.p("big1.h5"), f.p("big2.h5"));
|
||
let cases: Vec<(Vec<&str>, i32)> = vec![
|
||
(vec![&a, &b], 1),
|
||
(vec!["-d", "0", &a, &b], 1),
|
||
(vec!["-d", "0.5", &a, &b], 1),
|
||
(vec!["-d", "1", &a, &b], 0),
|
||
(vec!["-d", "0", &a, &b, "/u"], 1),
|
||
(vec!["-p", "0", &a, &b], 1),
|
||
(vec!["-p", "1e-18", &a, &b, "/i"], 1),
|
||
(vec!["-p", "1e-15", &a, &b], 0),
|
||
];
|
||
let have_h5diff = tool_available("h5diff");
|
||
for (args, want) in cases {
|
||
if have_h5diff {
|
||
assert_eq!(code(&run("h5diff", &args)), want, "h5diff {args:?}");
|
||
}
|
||
let o = h5rs(&[&["diff"], args.as_slice()].concat());
|
||
assert_eq!(code(&o), want, "h5rs diff {args:?}: {}", stdout(&o));
|
||
}
|
||
// The reported difference is exact too.
|
||
let s = stdout(&h5rs(&["diff", "-r", "-d", "0", &a, &b, "/u"]));
|
||
assert!(
|
||
s.contains("18446744073709551615 18446744073709551614 1\n"),
|
||
"{s}"
|
||
);
|
||
}
|
||
|
||
/// A soft link is compared as a link (its target path), as h5diff does,
|
||
/// unless `--follow-symlinks`, which compares (and walks into) what it
|
||
/// leads to; two dangling links are then the same.
|
||
#[test]
|
||
fn diff_soft_links_like_h5diff() {
|
||
let Some(f) = generate() else { return };
|
||
let (s1, s2, t1, t2) = (
|
||
f.p("soft1.h5"),
|
||
f.p("soft2.h5"),
|
||
f.p("target1.h5"),
|
||
f.p("target2.h5"),
|
||
);
|
||
let fl = "--follow-symlinks";
|
||
let cases: Vec<(Vec<&str>, i32)> = vec![
|
||
(vec![&s1, &s2, "/g/s"], 0),
|
||
(vec![&s1, &s2, "/g"], 0),
|
||
(vec![&s1, &s2, "/lnk"], 0),
|
||
(vec![&s1, &s2, "/dang"], 0),
|
||
(vec![&s1, &s2, "/lnk/d"], 1),
|
||
(vec![&t1, &t2, "/s"], 1),
|
||
(vec![&t1, &t2, "/s", "/rel"], 1),
|
||
(vec![fl, &s1, &s2, "/g/s"], 1),
|
||
(vec![fl, &s1, &s2, "/g"], 1),
|
||
(vec![fl, &s1, &s2, "/lnk"], 1),
|
||
(vec![fl, &s1, &s2, "/dang"], 0),
|
||
(vec![fl, &s1, &s1], 0),
|
||
(vec![fl, &t1, &t2, "/s"], 0),
|
||
(vec![fl, &t1, &t2, "/s", "/rel"], 0),
|
||
];
|
||
let have_h5diff = tool_available("h5diff");
|
||
for (args, want) in cases {
|
||
if have_h5diff {
|
||
assert_eq!(code(&run("h5diff", &args)), want, "h5diff {args:?}");
|
||
}
|
||
let o = h5rs(&[&["diff"], args.as_slice()].concat());
|
||
assert_eq!(code(&o), want, "h5rs diff {args:?}: {}", stdout(&o));
|
||
}
|
||
}
|
||
|
||
/// An object hard-linked under two names is compared under both, with
|
||
/// everything below it, so a file that shares one object between two names
|
||
/// equals a file that stores two identical copies.
|
||
#[test]
|
||
fn diff_compares_every_hard_link_path() {
|
||
let Some(f) = generate() else { return };
|
||
let (linked, copied, changed) = (
|
||
f.p("hardlinked.h5"),
|
||
f.p("copied.h5"),
|
||
f.p("copied_changed.h5"),
|
||
);
|
||
// A hard-linked dataset: h5diff agrees (exit 0).
|
||
for (a, b) in [(&linked, &copied), (&copied, &linked)] {
|
||
let o = h5rs(&["diff", a, b, "/y"]);
|
||
assert_eq!(code(&o), 0, "{a} {b} /y: {}", stdout(&o));
|
||
if tool_available("h5diff") {
|
||
assert_eq!(code(&run("h5diff", &[a, b, "/y"])), 0, "h5diff /y");
|
||
}
|
||
// The whole file, including a hard-linked group's members (h5diff
|
||
// does not descend a group's second name, so it reports those
|
||
// members as present in one file only and exits 1).
|
||
let o = h5rs(&["diff", a, b]);
|
||
assert_eq!(code(&o), 0, "{a} {b}: {}", stdout(&o));
|
||
assert!(!stdout(&o).contains("exists only"), "{}", stdout(&o));
|
||
}
|
||
// A value under the second name is compared, not skipped.
|
||
let o = h5rs(&["diff", "-r", &linked, &changed]);
|
||
assert_eq!(code(&o), 1, "{}", stdout(&o));
|
||
assert!(
|
||
stdout(&o).contains("dataset: </h/s/e> and </h/s/e>"),
|
||
"{}",
|
||
stdout(&o)
|
||
);
|
||
assert!(!stdout(&o).contains("</g/s/e>"), "{}", stdout(&o));
|
||
}
|
||
|
||
/// Where h5rs deliberately differs from h5diff: objects that cannot be
|
||
/// compared (another shape, another datatype class) are a difference.
|
||
#[test]
|
||
fn diff_counts_incomparable_objects_as_different() {
|
||
let Some(f) = generate() else { return };
|
||
let b = f.p("base.h5");
|
||
for other in ["reshaped.h5", "int.h5"] {
|
||
let o = h5rs(&["diff", &b, &f.p(other)]);
|
||
assert_eq!(code(&o), 1, "{other}: {}", stdout(&o));
|
||
assert!(stdout(&o).contains("Not comparable"), "{other}");
|
||
}
|
||
}
|
||
|
||
// ---------------------------------------------------------------------------
|
||
// check
|
||
// ---------------------------------------------------------------------------
|
||
|
||
#[test]
|
||
fn check_accepts_valid_files() {
|
||
let Some(f) = generate() else { return };
|
||
for name in LIBVERS.iter().chain(&[
|
||
"userblock.h5",
|
||
"base.h5",
|
||
"changed.h5",
|
||
"extra.h5",
|
||
"attr.h5",
|
||
]) {
|
||
let o = h5rs(&["check", "--data", &f.p(name)]);
|
||
assert_eq!(code(&o), 0, "{name}:\n{}", stdout(&o));
|
||
assert!(stdout(&o).contains("no problems found"));
|
||
}
|
||
// The newest format's checksummed structures were all verified.
|
||
let s = stdout(&h5rs(&["check", &f.p("latest.h5")]));
|
||
assert!(s.contains("superblock 1, v2 object headers 33,"), "{s}");
|
||
assert!(s.contains("chunk indexes 5"), "{s}");
|
||
}
|
||
|
||
/// Signatures of checksummed structures, and where each one's checksum
|
||
/// is: found by trying every end offset (the checksum is the Jenkins
|
||
/// lookup3 hash of the bytes before it). A fractal heap direct block's
|
||
/// checksum is instead stored near its start, computed over the whole block
|
||
/// with the field zeroed.
|
||
fn checksummed_structures(data: &[u8]) -> Vec<(&'static str, usize, usize)> {
|
||
const SIGS: [&str; 13] = [
|
||
"OHDR", "OCHK", "BTHD", "BTIN", "BTLF", "FRHP", "FHIB", "EAHD", "EAIB", "EASB", "EADB",
|
||
"FAHD", "FADB",
|
||
];
|
||
let mut found = Vec::new();
|
||
for start in 0..data.len().saturating_sub(4) {
|
||
let sig = &data[start..start + 4];
|
||
if let Some(name) = SIGS.iter().find(|s| s.as_bytes() == sig) {
|
||
let limit = data.len().min(start + 65536);
|
||
for end in start + 8..limit.saturating_sub(4) {
|
||
let stored = u32::from_le_bytes(data[end..end + 4].try_into().unwrap());
|
||
if jenkins_lookup3(&data[start..end]) == stored {
|
||
found.push((*name, start, end));
|
||
break;
|
||
}
|
||
}
|
||
} else if sig == b"FHDB" {
|
||
// version(1) heap address(8) block offset(1..8) checksum(4)
|
||
'found: for boff in 1..=8usize {
|
||
let at = start + 5 + 8 + boff;
|
||
let Some(stored) = data.get(at..at + 4) else {
|
||
break;
|
||
};
|
||
let stored = u32::from_le_bytes(stored.try_into().unwrap());
|
||
for size in [512usize, 1024, 2048, 4096, 8192, 16384, 32768, 65536] {
|
||
let Some(block) = data.get(start..start + size) else {
|
||
break;
|
||
};
|
||
let mut b = block.to_vec();
|
||
b[at - start..at - start + 4].fill(0);
|
||
if jenkins_lookup3(&b) == stored {
|
||
found.push(("FHDB", start, at));
|
||
break 'found;
|
||
}
|
||
}
|
||
}
|
||
}
|
||
}
|
||
found
|
||
}
|
||
|
||
#[test]
|
||
fn check_flags_every_corrupted_checksum() {
|
||
let Some(f) = generate() else { return };
|
||
let path = f.path("latest.h5");
|
||
let data = std::fs::read(&path).unwrap();
|
||
let structures = checksummed_structures(&data);
|
||
let kinds: std::collections::BTreeSet<&str> = structures.iter().map(|s| s.0).collect();
|
||
for want in [
|
||
"OHDR", "OCHK", "BTHD", "BTLF", "FRHP", "FHDB", "EAHD", "EAIB", "EADB", "FAHD", "FADB",
|
||
] {
|
||
assert!(kinds.contains(want), "test file has no {want}: {kinds:?}");
|
||
}
|
||
let bad = f.path("bad.h5");
|
||
for (name, start, at) in structures {
|
||
let mut d = data.clone();
|
||
d[at] ^= 0x01;
|
||
std::fs::write(&bad, &d).unwrap();
|
||
let o = h5rs(&["check", &bad.to_string_lossy()]);
|
||
let s = stdout(&o);
|
||
assert_eq!(
|
||
code(&o),
|
||
1,
|
||
"{name} at {start:#x}: checksum flip not flagged\n{s}"
|
||
);
|
||
assert!(
|
||
s.lines()
|
||
.any(|l| l.starts_with("problem: 0x") && l.contains("checksum")),
|
||
"{name} at {start:#x}: no checksum problem reported\n{s}"
|
||
);
|
||
}
|
||
}
|
||
|
||
/// `check --data` follows variable-length elements into the global heap:
|
||
/// a damaged collection is reported at its address, for the dataset and
|
||
/// the attribute that point into it. Without --data it is not read.
|
||
#[test]
|
||
fn check_data_follows_vl_data_into_the_global_heap() {
|
||
let Some(f) = generate() else { return };
|
||
for name in LIBVERS {
|
||
let mut data = std::fs::read(f.path(name)).unwrap();
|
||
let gcols: Vec<usize> = (0..data.len() - 4)
|
||
.filter(|&i| &data[i..i + 4] == b"GCOL")
|
||
.collect();
|
||
assert!(!gcols.is_empty(), "{name}: no global heap");
|
||
// "GCOL" version(1) reserved(3) size(8), then heap object 1:
|
||
// index(2) refcount(2) reserved(4) size(8) — claim 4 GiB.
|
||
let at = gcols[0];
|
||
data[at + 24..at + 28].fill(0xff);
|
||
let bad = f.path(&format!("bad-gcol-{name}"));
|
||
std::fs::write(&bad, &data).unwrap();
|
||
let bad = bad.to_string_lossy().into_owned();
|
||
let o = h5rs(&["check", "--data", &bad]);
|
||
let s = stdout(&o);
|
||
assert_eq!(code(&o), 1, "{name}: {s}");
|
||
let want = format!("problem: {at:#x} /vlstr: variable-length data: global heap");
|
||
assert!(s.contains(&want), "{name}: no {want:?} in\n{s}");
|
||
// h5dump refuses the file too.
|
||
if tool_available("h5dump") {
|
||
assert_ne!(code(&run("h5dump", &[&bad])), 0, "{name}: h5dump read it");
|
||
}
|
||
assert_eq!(code(&h5rs(&["check", &bad])), 0, "{name}: without --data");
|
||
let ok = h5rs(&["check", "--data", &f.p(name)]);
|
||
assert!(
|
||
stdout(&ok).contains(&format!("global heap collections read: {}", gcols.len())),
|
||
"{name}: {}",
|
||
stdout(&ok)
|
||
);
|
||
}
|
||
}
|
||
|
||
#[test]
|
||
fn check_flags_a_truncated_file() {
|
||
let Some(f) = generate() else { return };
|
||
let data = std::fs::read(f.path("latest.h5")).unwrap();
|
||
let bad = f.path("truncated.h5");
|
||
std::fs::write(&bad, &data[..data.len() * 3 / 4]).unwrap();
|
||
let o = h5rs(&["check", &bad.to_string_lossy()]);
|
||
assert_eq!(code(&o), 1);
|
||
assert!(stdout(&o).contains("file is truncated"), "{}", stdout(&o));
|
||
}
|
||
|
||
/// A v1 B-tree chunk index has no checksum; its keys are checked against
|
||
/// the dataset: a chunk offset that is not a multiple of the chunk size.
|
||
#[test]
|
||
fn check_flags_a_misaligned_chunk() {
|
||
let Some(f) = generate() else { return };
|
||
let path = f.path("earliest.h5");
|
||
let mut data = std::fs::read(&path).unwrap();
|
||
// The leaf chunk B-tree of /grp/gz (10 deflated chunks of 100 i32, so
|
||
// each stored in under 400 bytes): "TREE" type(1)=1 level(1)=0
|
||
// entries(2) left(8) right(8), then key, child, key, child, ... where a
|
||
// rank-1 chunk key is size(4) mask(4) offsets(2 x 8) and a child is 8.
|
||
// Key 1 starts at 24 + 24 + 8 = 56; its dimension-0 offset (100) at 64.
|
||
let mut patched = false;
|
||
for start in 0..data.len() - 72 {
|
||
if &data[start..start + 4] == b"TREE" && data[start + 4] == 1 && data[start + 5] == 0 {
|
||
let entries = u16::from_le_bytes(data[start + 6..start + 8].try_into().unwrap());
|
||
let size0 = u32::from_le_bytes(data[start + 24..start + 28].try_into().unwrap());
|
||
let off1 = u64::from_le_bytes(data[start + 64..start + 72].try_into().unwrap());
|
||
if entries == 10 && size0 < 400 && off1 == 100 {
|
||
data[start + 64..start + 72].copy_from_slice(&103u64.to_le_bytes());
|
||
patched = true;
|
||
break;
|
||
}
|
||
}
|
||
}
|
||
assert!(patched, "no chunk B-tree found for /grp/gz");
|
||
let bad = f.path("misaligned.h5");
|
||
std::fs::write(&bad, &data).unwrap();
|
||
let o = h5rs(&["check", &bad.to_string_lossy()]);
|
||
let s = stdout(&o);
|
||
assert_eq!(code(&o), 1, "{s}");
|
||
// The library's chunk-index reader refuses the key first (as libhdf5
|
||
// does); either way the problem is reported against the dataset.
|
||
assert!(
|
||
s.contains("/grp/gz: chunk at [103, 0] offset 103 in dimension 0 is not a multiple of the chunk size 100")
|
||
|| s.contains("/grp/gz: chunk index (v1 B-tree): chunked read error: bad coordinate offset [103, 0] for chunk dimensions [100, 4]"),
|
||
"{s}"
|
||
);
|
||
}
|
||
|
||
#[test]
|
||
fn every_subcommand_rejects_a_non_hdf5_file_cleanly() {
|
||
let dir = tempfile::tempdir().unwrap();
|
||
let p = dir.path().join("junk.h5");
|
||
std::fs::write(&p, b"this is not an HDF5 file at all").unwrap();
|
||
let p = p.to_string_lossy().into_owned();
|
||
for args in [
|
||
vec!["ls", p.as_str()],
|
||
vec!["dump", p.as_str()],
|
||
vec!["stat", p.as_str()],
|
||
vec!["diff", p.as_str(), p.as_str()],
|
||
] {
|
||
let o = h5rs(&args);
|
||
assert_eq!(code(&o), 2, "{args:?}");
|
||
assert!(String::from_utf8_lossy(&o.stderr).contains("not an HDF5 file"));
|
||
}
|
||
let o = h5rs(&["check", &p]);
|
||
assert_eq!(code(&o), 1);
|
||
assert!(stdout(&o).contains("no HDF5 signature"));
|
||
assert_eq!(code(&h5rs(&["nonsense"])), 2);
|
||
assert_eq!(code(&h5rs(&["ls"])), 2);
|
||
assert_eq!(code(&h5rs(&["--help"])), 0);
|
||
}
|
||
|
||
// ---------------------------------------------------------------------------
|
||
// variable-length data
|
||
// ---------------------------------------------------------------------------
|
||
|
||
/// Runs `tests/gen_vl_files.py`: VL strings (with an embedded NUL, empty
|
||
/// and null elements), VL sequences, a VL compound member and a VL
|
||
/// attribute, with 8- and 4-byte offsets, plus files whose heap objects
|
||
/// disagree with their elements' lengths.
|
||
fn generate_vl() -> Option<Files> {
|
||
if missing(python_available(), "python3 with h5py") {
|
||
return None;
|
||
}
|
||
let dir = tempfile::tempdir().unwrap();
|
||
let script = Path::new(env!("CARGO_MANIFEST_DIR")).join("tests/gen_vl_files.py");
|
||
let out = Command::new(python())
|
||
.arg(&script)
|
||
.arg(dir.path())
|
||
.output()
|
||
.expect("run gen_vl_files.py");
|
||
assert!(
|
||
out.status.success(),
|
||
"gen_vl_files.py failed:\n{}",
|
||
String::from_utf8_lossy(&out.stderr)
|
||
);
|
||
let values = serde_json::from_slice(&out.stdout).expect("gen_vl_files.py output");
|
||
Some(Files { dir, values })
|
||
}
|
||
|
||
/// `dump` resolves VL elements through the library's `VlResolver`, as
|
||
/// libhdf5 does: "a\0b" prints as "a", a null string as NULL (it printed
|
||
/// ""), and with 4-byte offsets too; the output is h5dump's byte for byte.
|
||
#[test]
|
||
fn dump_prints_vl_data_like_h5dump() {
|
||
let Some(f) = generate_vl() else { return };
|
||
for name in ["vl8.h5", "vl4.h5"] {
|
||
let p = f.p(name);
|
||
let ours = stdout(&h5rs(&["dump", &p]));
|
||
assert!(
|
||
ours.contains(r#"(0): "a", "", NULL, "zz", "hello""#),
|
||
"{name}:\n{ours}"
|
||
);
|
||
assert!(ours.contains(r#"(0): NULL, "w", NULL, NULL"#), "{name}");
|
||
assert!(ours.contains("(0): (1, 2, 3), (), (-5)"), "{name}");
|
||
if missing(tool_available("h5dump"), "h5dump") {
|
||
continue;
|
||
}
|
||
let reference = run("h5dump", &[&p]);
|
||
assert!(reference.status.success(), "{name}: {reference:?}");
|
||
assert_eq!(ours, stdout(&reference).replacen(&p, name, 1), "{name}");
|
||
}
|
||
}
|
||
|
||
/// `dump --json` gives the values h5py reads, element by element; and an
|
||
/// element whose heap object is not its length × base size is an error, as
|
||
/// in h5py, not a truncated value (it printed "cde" and (1, 2)); so is a
|
||
/// length-0 element at the undefined heap address (it printed "").
|
||
#[test]
|
||
fn dump_json_vl_values_match_h5py() {
|
||
let Some(f) = generate_vl() else { return };
|
||
for tag in ["8", "4"] {
|
||
let (good, bad) = (format!("vl{tag}"), format!("bad{tag}"));
|
||
let o = h5rs(&["dump", "--json", &f.p(&format!("{good}.h5"))]);
|
||
assert!(o.status.success(), "{good}: {o:?}");
|
||
let doc: serde_json::Value = serde_json::from_slice(&o.stdout).unwrap();
|
||
let want = &f.values[&good];
|
||
for d in doc["datasets"].as_object().unwrap().values() {
|
||
let path = d["alias"][0].as_str().unwrap();
|
||
assert_eq!(d["value"], want[&path[1..]], "{good}: {path}");
|
||
}
|
||
let attrs = &doc["groups"][doc["root"].as_str().unwrap()]["attributes"];
|
||
assert_eq!(attrs[0]["name"], "va");
|
||
assert_eq!(attrs[0]["value"], want["va"], "{good}: va");
|
||
|
||
let o = h5rs(&["dump", "--json", &f.p(&format!("{bad}.h5"))]);
|
||
let doc: serde_json::Value = serde_json::from_slice(&o.stdout).unwrap();
|
||
let want = &f.values[&bad];
|
||
for d in doc["datasets"].as_object().unwrap().values() {
|
||
let path = d["alias"][0].as_str().unwrap();
|
||
let got = d["value"].as_array().unwrap();
|
||
let want = want[&path[1..]].as_array().unwrap();
|
||
assert_eq!(got.len(), want.len(), "{bad}: {path}");
|
||
for (g, w) in got.iter().zip(want) {
|
||
if w.is_null() {
|
||
// h5py cannot read it: neither can we.
|
||
let e = g["error"]
|
||
.as_str()
|
||
.unwrap_or_else(|| panic!("{bad}: {path}: {g}"));
|
||
let why = if path == "/undef" {
|
||
"undefined"
|
||
} else {
|
||
"holds"
|
||
};
|
||
assert!(e.contains(why), "{bad}: {path}: {e}");
|
||
} else {
|
||
assert_eq!(g, w, "{bad}: {path}");
|
||
}
|
||
}
|
||
}
|
||
}
|
||
}
|
||
|
||
/// `check --data` holds VL elements to libhdf5's rule: a heap object whose
|
||
/// size is not exactly the element's length × base size is a problem (it
|
||
/// only caught objects shorter than the element), and so is an element at
|
||
/// the undefined heap address.
|
||
#[test]
|
||
fn check_data_flags_mis_sized_vl_heap_objects() {
|
||
let Some(f) = generate_vl() else { return };
|
||
for tag in ["8", "4"] {
|
||
let o = h5rs(&["check", "--data", &f.p(&format!("vl{tag}.h5"))]);
|
||
let s = stdout(&o);
|
||
assert_eq!(code(&o), 0, "vl{tag}: {s}");
|
||
assert!(
|
||
s.contains("global heap collections read: 1"),
|
||
"vl{tag}: {s}"
|
||
);
|
||
|
||
let o = h5rs(&["check", "--data", &f.p(&format!("bad{tag}.h5"))]);
|
||
let s = stdout(&o);
|
||
assert_eq!(code(&o), 1, "bad{tag}: {s}");
|
||
let at = f.values[format!("bad{tag}")]["gcol"].as_u64().unwrap();
|
||
for (path, what) in [("/bad", "6 bytes"), ("/badseq", "12 bytes")] {
|
||
let want = format!("problem: {at:#x} {path}: variable-length data: global heap object");
|
||
assert!(s.contains(&want), "bad{tag}: no {want:?} in\n{s}");
|
||
assert!(s.contains(what), "bad{tag}: {s}");
|
||
}
|
||
// A length-0 element at the undefined heap address: libhdf5 fails
|
||
// to read it; check skipped it.
|
||
let undef: u64 = if tag == "8" { u64::MAX } else { 0xffff_ffff };
|
||
let want = format!("problem: {undef:#x} /undef: variable-length data: global heap:");
|
||
assert!(s.contains(&want), "bad{tag}: no {want:?} in\n{s}");
|
||
assert!(s.contains("undefined global heap address"), "bad{tag}: {s}");
|
||
}
|
||
}
|
||
|
||
// ---------------------------------------------------------------------------
|
||
// files clawhdf5 writes: nested groups and links
|
||
// ---------------------------------------------------------------------------
|
||
|
||
/// Nested groups (4 levels, by builders and by path names), soft, hard and
|
||
/// external links, creation-order tracking, and dense link and attribute
|
||
/// storage, as `FileBuilder` writes them.
|
||
fn write_nested_links(dir: &Path) -> Vec<String> {
|
||
use clawhdf5::{AttrValue, FileBuilder};
|
||
let mut b = FileBuilder::new();
|
||
b.set_attr("title", AttrValue::String("links".into()));
|
||
b.create_dataset("x/y").with_f64_data(&[1.0, 2.0]);
|
||
b.create_dataset("a/b/c/d/leaf")
|
||
.with_i32_data(&[4, 5])
|
||
.set_attr("depth", AttrValue::I64(5));
|
||
b.add_soft_link("soft", "/x/y");
|
||
b.add_soft_link("dangling", "/nowhere");
|
||
b.add_hard_link("alias", "/x/y");
|
||
b.add_external_link("ext", "other.h5", "/data");
|
||
let mut g = b.create_group("a/b");
|
||
g.set_attr("merged", AttrValue::I64(1));
|
||
for i in 0..10 {
|
||
g.set_attr(&format!("attr{i}"), AttrValue::F64(i as f64));
|
||
}
|
||
b.add_group(g.finish());
|
||
let mut g = b.create_group("ordered");
|
||
g.track_order(true);
|
||
for i in (0..40).rev() {
|
||
g.create_dataset(&format!("n{i:02}")).with_i32_data(&[i]);
|
||
}
|
||
g.add_hard_link("back", "/a/b/c");
|
||
// Attribute creation order tracked too: dense, with a type-9 index.
|
||
for i in (0..12).rev() {
|
||
g.set_attr(&format!("attr{i:02}"), AttrValue::I64(i));
|
||
}
|
||
b.add_group(g.finish());
|
||
let mut g = b.create_group("compact_ordered");
|
||
g.track_order(true);
|
||
g.create_dataset("z").with_i32_data(&[1]);
|
||
g.create_dataset("a").with_i32_data(&[2]);
|
||
g.set_attr("zz", AttrValue::I64(1));
|
||
g.set_attr("aa", AttrValue::I64(2));
|
||
b.add_group(g.finish());
|
||
let nested = dir.join("nested.h5");
|
||
b.write(&nested).unwrap();
|
||
|
||
let mut b = FileBuilder::new();
|
||
let mut g = b.create_group("many");
|
||
for i in 0..10_000 {
|
||
g.create_dataset(&format!("d{i:05}")).with_i32_data(&[i]);
|
||
}
|
||
b.add_group(g.finish());
|
||
let many = dir.join("many.h5");
|
||
b.write(&many).unwrap();
|
||
[nested, many]
|
||
.iter()
|
||
.map(|p| p.to_string_lossy().into_owned())
|
||
.collect()
|
||
}
|
||
|
||
#[test]
|
||
fn check_and_dump_files_with_nested_groups_and_links() {
|
||
let dir = tempfile::tempdir().unwrap();
|
||
let files = write_nested_links(dir.path());
|
||
// `--data` reads every dataset by path, and a lookup in a dense group
|
||
// scans all of its links: 10 000 datasets take minutes in a debug
|
||
// build, so the big file is checked structurally only (and not dumped).
|
||
for (p, data) in [(&files[0], true), (&files[1], false)] {
|
||
let args: &[&str] = if data {
|
||
&["check", "--data", p]
|
||
} else {
|
||
&["check", p]
|
||
};
|
||
let o = h5rs(args);
|
||
assert_eq!(code(&o), 0, "{p}:\n{}", stdout(&o));
|
||
assert!(stdout(&o).contains("no problems found"), "{}", stdout(&o));
|
||
}
|
||
if missing(tool_available("h5dump"), "h5dump") {
|
||
return;
|
||
}
|
||
for p in &files[..1] {
|
||
let name = Path::new(p).file_name().unwrap().to_string_lossy();
|
||
let ours = h5rs(&["dump", p]);
|
||
assert!(ours.status.success(), "{p}: {ours:?}");
|
||
let reference = run("h5dump", &[p]);
|
||
assert!(reference.status.success(), "h5dump {p}: {reference:?}");
|
||
let r = stdout(&reference).replacen(p.as_str(), &name, 1);
|
||
assert_eq!(stdout(&ours), r, "{name}");
|
||
}
|
||
}
|
||
|
||
#[test]
|
||
fn check_files_with_big_dense_storage() {
|
||
// Dense links and attributes past the 512 KiB the root indirect block's
|
||
// direct blocks hold: the heap then needs child indirect blocks, which
|
||
// the writer used to write as direct blocks ("fractal heap indirect
|
||
// block: bad signature").
|
||
use clawhdf5::{AttrValue, FileBuilder};
|
||
let dir = tempfile::tempdir().unwrap();
|
||
let mut b = FileBuilder::new();
|
||
let x = b.create_dataset("x");
|
||
x.with_i32_data(&[7]);
|
||
for i in 0..150usize {
|
||
let len = if i % 3 == 0 { 7_000 } else { 1 + i };
|
||
x.set_attr(
|
||
&format!("a{i:03}"),
|
||
AttrValue::F64Array(vec![i as f64; len]),
|
||
);
|
||
}
|
||
let mut g = b.create_group("g");
|
||
for i in 0..40_000 {
|
||
g.add_hard_link(&format!("link_{i:06}_{}", "x".repeat(88)), "/x");
|
||
}
|
||
b.add_group(g.finish());
|
||
let p = dir.path().join("big.h5").to_string_lossy().into_owned();
|
||
b.write(&p).unwrap();
|
||
// Structure only: `--data` looks every link up by a linear scan.
|
||
let o = h5rs(&["check", &p]);
|
||
assert_eq!(code(&o), 0, "{p}:\n{}", stdout(&o));
|
||
assert!(stdout(&o).contains("no problems found"), "{}", stdout(&o));
|
||
}
|
||
|
||
/// `(type, depth)` of every v2 B-tree header in a file written with 8-byte
|
||
/// offsets and lengths (found by signature and checksum).
|
||
fn btree_v2_depths(data: &[u8]) -> Vec<(u8, u16)> {
|
||
const LEN: usize = 4 + 1 + 1 + 4 + 2 + 2 + 1 + 1 + 8 + 2 + 8;
|
||
let mut out = Vec::new();
|
||
for at in 0..data.len().saturating_sub(LEN + 4) {
|
||
if &data[at..at + 4] != b"BTHD" {
|
||
continue;
|
||
}
|
||
let stored = u32::from_le_bytes(data[at + LEN..at + LEN + 4].try_into().unwrap());
|
||
if jenkins_lookup3(&data[at..at + LEN]) == stored {
|
||
out.push((
|
||
data[at + 5],
|
||
u16::from_le_bytes([data[at + 12], data[at + 13]]),
|
||
));
|
||
}
|
||
}
|
||
out
|
||
}
|
||
|
||
#[test]
|
||
fn check_files_with_deep_btrees() {
|
||
// Dense indexes and a chunk index too big for one leaf: the writer then
|
||
// builds internal nodes, whose child pointers carry record counts in
|
||
// widths derived from the node size. `check` reads every record through
|
||
// them and compares the count with the header's.
|
||
use clawhdf5::{AttrValue, FileBuilder};
|
||
const U: u64 = u64::MAX;
|
||
let dir = tempfile::tempdir().unwrap();
|
||
let mut b = FileBuilder::new();
|
||
let x = b.create_dataset("x");
|
||
x.with_i32_data(&[7]);
|
||
for i in 0..70_000 {
|
||
x.set_attr(&format!("attr_{i}"), AttrValue::I64(i));
|
||
}
|
||
let mut g = b.create_group("g");
|
||
g.track_order(true);
|
||
for i in 0..100_000 {
|
||
g.add_hard_link(&format!("k{i}"), "/x");
|
||
}
|
||
b.add_group(g.finish());
|
||
let p = dir.path().join("deep.h5").to_string_lossy().into_owned();
|
||
b.write(&p).unwrap();
|
||
let o = h5rs(&["check", &p]);
|
||
assert_eq!(code(&o), 0, "{p}:\n{}", stdout(&o));
|
||
assert!(stdout(&o).contains("no problems found"), "{}", stdout(&o));
|
||
let mut depths = btree_v2_depths(&std::fs::read(&p).unwrap());
|
||
depths.sort();
|
||
assert_eq!(depths, [(5, 3), (6, 3), (8, 3)]);
|
||
|
||
let mut b = FileBuilder::new();
|
||
b.create_dataset("d")
|
||
.with_i32_data(&(0..200_000).collect::<Vec<i32>>())
|
||
.with_shape(&[400, 500])
|
||
.with_chunks(&[1, 1])
|
||
.with_maxshape(&[U, U]);
|
||
b.create_dataset("z")
|
||
.with_i32_data(&(0..70_000).collect::<Vec<i32>>())
|
||
.with_shape(&[70, 1000])
|
||
.with_chunks(&[1, 1])
|
||
.with_maxshape(&[U, U])
|
||
.with_deflate(1);
|
||
let p = dir.path().join("chunks.h5").to_string_lossy().into_owned();
|
||
b.write(&p).unwrap();
|
||
let o = h5rs(&["check", "--data", &p]);
|
||
assert_eq!(code(&o), 0, "{p}:\n{}", stdout(&o));
|
||
assert!(stdout(&o).contains("no problems found"), "{}", stdout(&o));
|
||
let mut depths = btree_v2_depths(&std::fs::read(&p).unwrap());
|
||
depths.sort();
|
||
assert_eq!(depths, [(10, 2), (11, 2)]);
|
||
}
|