//! `h5rs` against libhdf5's own tools, on files h5py writes //! (`tests/gen_files.py`): `ls` against h5ls, `stat` against h5stat, `dump` //! against h5dump, `diff` exit codes against h5diff, `dump --json` values //! against h5py, and `check` on valid and deliberately corrupted files. //! //! Needs python3 with h5py (`CLAWHDF5_PYTHON`) and, for the comparisons, //! the libhdf5 command-line tools (h5ls, h5stat, h5dump, h5diff) on `PATH`. //! Each test skips when what it needs is missing, unless //! `CLAWHDF5_REQUIRE_INTEROP=1`, which turns a skip into a failure. use std::collections::BTreeMap; use std::path::{Path, PathBuf}; use std::process::{Command, Output}; use clawhdf5_format::checksum::jenkins_lookup3; fn python() -> String { std::env::var("CLAWHDF5_PYTHON").unwrap_or_else(|_| "python3".to_string()) } fn interop_required() -> bool { std::env::var("CLAWHDF5_REQUIRE_INTEROP").is_ok_and(|v| v == "1") } fn python_available() -> bool { Command::new(python()) .args(["-c", "import h5py, numpy"]) .output() .map(|o| o.status.success()) .unwrap_or(false) } fn tool_available(name: &str) -> bool { Command::new(name) .arg("--version") .output() .map(|o| o.status.success()) .unwrap_or(false) } /// Skip (return true) when `what` is unavailable, or fail when interop is /// required. fn missing(ok: bool, what: &str) -> bool { if ok { return false; } assert!( !interop_required(), "CLAWHDF5_REQUIRE_INTEROP=1 but {what} is not available" ); eprintln!("SKIP: {what} not available"); true } fn h5rs(args: &[&str]) -> Output { Command::new(env!("CARGO_BIN_EXE_h5rs")) .args(args) .output() .expect("run h5rs") } fn run(tool: &str, args: &[&str]) -> Output { Command::new(tool).args(args).output().expect("run tool") } fn stdout(o: &Output) -> String { String::from_utf8_lossy(&o.stdout).into_owned() } /// The generated files and what h5py read from them. struct Files { dir: tempfile::TempDir, values: serde_json::Value, } impl Files { fn path(&self, name: &str) -> PathBuf { self.dir.path().join(name) } fn p(&self, name: &str) -> String { self.path(name).to_string_lossy().into_owned() } } fn generate() -> Option { if missing(python_available(), "python3 with h5py") { return None; } let dir = tempfile::tempdir().unwrap(); let script = Path::new(env!("CARGO_MANIFEST_DIR")).join("tests/gen_files.py"); let out = Command::new(python()) .arg(&script) .arg(dir.path()) .output() .expect("run gen_files.py"); assert!( out.status.success(), "gen_files.py failed:\n{}", String::from_utf8_lossy(&out.stderr) ); let values = serde_json::from_slice(&out.stdout).expect("gen_files.py output"); Some(Files { dir, values }) } const LIBVERS: [&str; 2] = ["earliest.h5", "latest.h5"]; // --------------------------------------------------------------------------- // ls // --------------------------------------------------------------------------- /// Every h5ls line must be the start of the matching h5rs line (h5rs adds /// the datatype after a dataset's shape). fn assert_ls_like(ours: &str, reference: &str, what: &str) { let ours: Vec<&str> = ours.lines().collect(); let refs: Vec<&str> = reference.lines().collect(); assert_eq!( ours.len(), refs.len(), "{what}: line count\nh5rs:\n{}\nh5ls:\n{}", ours.join("\n"), refs.join("\n") ); for (o, r) in ours.iter().zip(&refs) { let r = r.trim_end(); assert!( o.starts_with(r) && (o.len() == r.len() || r.contains("Dataset {")), "{what}:\n h5rs: {o}\n h5ls: {r}" ); } } #[test] fn ls_matches_h5ls() { let Some(f) = generate() else { return }; if missing(tool_available("h5ls"), "h5ls") { return; } for name in LIBVERS.iter().chain(&["userblock.h5"]) { let p = f.p(name); for (args, what) in [(vec!["-r"], "recursive"), (vec![], "root")] { let mut a: Vec<&str> = args.clone(); a.push(&p); let ours = h5rs(&[&["ls"], a.as_slice()].concat()); assert!(ours.status.success(), "{name} {what}: {:?}", ours); let reference = run("h5ls", &a); assert_ls_like( &stdout(&ours), &stdout(&reference), &format!("{name} {what}"), ); } } // A group, and a dataset, named after the file. for obj in ["/grp", "/grp/sub", "/contig", "/grp/ext2"] { let arg = format!("{}{obj}", f.p("latest.h5")); let ours = h5rs(&["ls", &arg]); let reference = run("h5ls", &[&arg]); assert_ls_like(&stdout(&ours), &stdout(&reference), &arg); let ours = h5rs(&["ls", "-r", &arg]); let reference = run("h5ls", &["-r", &arg]); assert_ls_like(&stdout(&ours), &stdout(&reference), &format!("-r {arg}")); } } #[test] fn ls_verbose_describes_layout_filters_and_attributes() { let Some(f) = generate() else { return }; let o = h5rs(&["ls", "-r", "-v", &f.p("latest.h5")]); assert!(o.status.success()); let s = stdout(&o); for want in [ "Layout: chunked (extensible array index)", "Layout: chunked (v2 B-tree index)", "Layout: chunked (fixed array index)", "Layout: chunked (single chunk index)", "Filter-0: shuffle-2", "Filter-1: deflate-1", "Filter-2: fletcher32-3", "Chunks: {100} 400 bytes", "Attribute: units scalar", "Data: \"m\"", ] { assert!(s.contains(want), "missing {want:?} in\n{s}"); } let o = h5rs(&["ls", "-v", &format!("{}/grp", f.p("earliest.h5"))]); assert!(stdout(&o).contains("Layout: chunked (v1 B-tree index)")); } // --------------------------------------------------------------------------- // stat // --------------------------------------------------------------------------- /// `key: value` lines of an h5stat-style report. fn stat_facts(s: &str) -> BTreeMap { s.lines() .filter_map(|l| { let (k, v) = l.rsplit_once(':')?; Some((k.trim().to_string(), v.trim().to_string())) }) .collect() } #[test] fn stat_matches_h5stat() { let Some(f) = generate() else { return }; if missing(tool_available("h5stat"), "h5stat") { return; } let keys = [ "# of unique groups", "# of unique datasets", "# of unique named datatypes", "# of unique links", "# of unique other", "Max. # of links to object", "Max. # of objects in group", "Max. rank of datasets", "Max. dimension size of 1-D datasets", "Total raw data size", "Dataset layout counts[COMPACT]", "Dataset layout counts[CONTIG]", "Dataset layout counts[CHUNKED]", "Dataset layout counts[VIRTUAL]", "NO filter", "GZIP filter", "SHUFFLE filter", "FLETCHER32 filter", "SZIP filter", "NBIT filter", "SCALEOFFSET filter", "USER-DEFINED filter", "Max. # of attributes to objects", "Total space", ]; for name in LIBVERS.iter().chain(&["userblock.h5", "base.h5"]) { let p = f.p(name); let ours = h5rs(&["stat", &p]); assert!(ours.status.success(), "{name}: {ours:?}"); let reference = run("h5stat", &[&p]); assert!(reference.status.success(), "h5stat {name} failed"); let (o, r) = (stat_facts(&stdout(&ours)), stat_facts(&stdout(&reference))); for k in keys { // h5stat leaves out the attribute summary when nothing has one. let Some(want) = r.get(k) else { continue }; assert_eq!(o.get(k), Some(want), "{name}: {k}"); } } } // --------------------------------------------------------------------------- // dump // --------------------------------------------------------------------------- #[test] fn dump_matches_h5dump() { let Some(f) = generate() else { return }; if missing(tool_available("h5dump"), "h5dump") { return; } for name in LIBVERS.iter().chain(&["userblock.h5", "changed.h5"]) { let p = f.p(name); for args in [vec![], vec!["-A"]] { let mut a = args.clone(); a.push(p.as_str()); let ours = h5rs(&[&["dump"], a.as_slice()].concat()); assert!(ours.status.success(), "{name} {args:?}: {ours:?}"); let reference = run("h5dump", &a); // h5dump names the file as given; h5rs by its file name. let r = stdout(&reference).replacen(&p, name, 1); assert_eq!(stdout(&ours), r, "{name} {args:?}"); } } let p = f.p("latest.h5"); let ours = h5rs(&["dump", "-d", "/grp/ext2", &p]); let reference = run("h5dump", &["-d", "/grp/ext2", &p]); assert_eq!( stdout(&ours), stdout(&reference).replacen(&p, "latest.h5", 1) ); } /// A null-padded fixed string prints every byte (NULs as `\000`), as /// h5dump prints it, also inside a compound and an array member. #[test] fn dump_shows_nul_padding_in_nested_strings() { let Some(f) = generate() else { return }; let p = f.p("nulstrings.h5"); let ours = stdout(&h5rs(&["dump", &p])); for want in [ r#"(0): "\000\000\000", "ab\000", "a\000b""#, r#""\000\000\000","#, r#""a\000b","#, r#"[ "\000\000", "x\000" ]"#, ] { assert!(ours.contains(want), "no {want} in\n{ours}"); } if missing(tool_available("h5dump"), "h5dump") { return; } let reference = run("h5dump", &[&p]); assert_eq!(ours, stdout(&reference).replacen(&p, "nulstrings.h5", 1)); } #[test] fn dump_json_values_match_h5py() { let Some(f) = generate() else { return }; for name in LIBVERS { let o = h5rs(&["dump", "--json", &f.p(name)]); assert!(o.status.success(), "{name}: {o:?}"); let doc: serde_json::Value = serde_json::from_slice(&o.stdout).unwrap(); assert_eq!(doc["apiVersion"], "1.1.1"); let root = doc["root"].as_str().unwrap(); assert!(doc["groups"][root].is_object()); let mut by_path = BTreeMap::new(); for (_, d) in doc["datasets"].as_object().unwrap() { for a in d["alias"].as_array().unwrap() { by_path.insert(a.as_str().unwrap().to_string(), d.clone()); } } let want = f.values[name.trim_end_matches(".h5")].as_object().unwrap(); assert!(want.len() > 10); for (path, v) in want { let d = by_path .get(path) .unwrap_or_else(|| panic!("{name}: no {path}")); let got = &d["value"]; // JSON numbers: compare as f64 (h5py's ints and floats both // round-trip exactly through JSON here). assert_eq!(flatten(got), flatten(v), "{name}: {path}"); } let links = doc["groups"][root]["links"].as_array().unwrap(); let soft = links.iter().find(|l| l["title"] == "soft").unwrap(); assert_eq!(soft["class"], "H5L_TYPE_SOFT"); assert_eq!(soft["h5path"], "/contig"); let ext = links.iter().find(|l| l["title"] == "external").unwrap(); assert_eq!(ext["class"], "H5L_TYPE_EXTERNAL"); assert_eq!(ext["file"], "other.h5"); } } fn flatten(v: &serde_json::Value) -> Vec { match v { serde_json::Value::Array(a) => a.iter().flat_map(flatten).collect(), serde_json::Value::Number(n) => vec![n.as_f64().unwrap()], other => panic!("not numeric: {other}"), } } // --------------------------------------------------------------------------- // diff // --------------------------------------------------------------------------- fn code(o: &Output) -> i32 { o.status.code().expect("exit code") } #[test] fn diff_exit_codes_match_h5diff() { let Some(f) = generate() else { return }; if missing(tool_available("h5diff"), "h5diff") { return; } let (b, same, changed, extra, attr) = ( f.p("base.h5"), f.p("same.h5"), f.p("changed.h5"), f.p("extra.h5"), f.p("attr.h5"), ); let missing_file = f.p("does-not-exist.h5"); let cases: Vec<(Vec<&str>, i32)> = vec![ (vec![&b, &same], 0), (vec![&b, &b], 0), (vec![&b, &changed], 1), (vec![&b, &changed, "/d"], 1), (vec![&b, &changed, "/g"], 0), (vec!["-d", "0.01", &b, &changed], 0), (vec!["-d", "0.0001", &b, &changed], 1), (vec!["-p", "0.01", &b, &changed], 0), (vec!["-p", "1e-9", &b, &changed], 1), (vec![&b, &extra], 1), (vec![&b, &attr], 1), (vec![&b, &attr, "/g"], 0), (vec![&b, &same, "/nope"], 2), (vec![&b, &missing_file], 2), (vec![&b, &same, "/d", "/d"], 0), ]; for (args, want) in cases { let theirs = code(&run("h5diff", &args)); assert_eq!(theirs, want, "h5diff {args:?} (test expectation)"); let o = h5rs(&[&["diff"], args.as_slice()].concat()); assert_eq!(code(&o), want, "h5rs diff {args:?}: {}", stdout(&o)); let q = h5rs(&[&["diff", "-q"], args.as_slice()].concat()); assert_eq!(code(&q), want, "h5rs diff -q {args:?}"); assert!(q.stdout.is_empty(), "-q printed output"); } // The report lists the differing positions. let o = h5rs(&["diff", "-r", &b, &changed, "/d"]); let s = stdout(&o); assert!(s.contains("[ 0 2 ]") && s.contains("[ 2 3 ]"), "{s}"); assert!(s.contains("2 difference(s) found"), "{s}"); } /// The option names are h5diff's: -c is --compare (a flag), the count is /// -n/--count, and the --name=value forms are accepted. #[test] fn diff_options_are_named_like_h5diff() { let Some(f) = generate() else { return }; let (b, changed) = (f.p("base.h5"), f.p("changed.h5")); let cases: Vec<(Vec<&str>, i32)> = vec![ // "2" is then the first file name, which does not exist. (vec!["-r", "-c", "2", &b, &changed], 2), (vec!["-c", &b, &changed], 1), (vec!["--compare", &b, &b], 0), (vec!["-r", "-n", "1", &b, &changed], 1), (vec!["-r", "--count=1", &b, &changed], 1), (vec!["--delta=0.01", &b, &changed], 0), (vec!["--relative=1e-9", &b, &changed], 1), ]; let have_h5diff = tool_available("h5diff"); for (args, want) in cases { if have_h5diff { assert_eq!(code(&run("h5diff", &args)), want, "h5diff {args:?}"); } let o = h5rs(&[&["diff"], args.as_slice()].concat()); assert_eq!(code(&o), want, "h5rs diff {args:?}: {}", stdout(&o)); } for count in [vec!["-n", "1"], vec!["--count=1"]] { let o = h5rs(&[&["diff", "-r"], count.as_slice(), &[&b, &changed, "/d"]].concat()); let s = stdout(&o); assert!( s.contains("[ 0 2 ]") && !s.contains("[ 2 3 ]"), "{count:?}: {s}" ); assert!(s.contains("2 difference(s) found"), "{s}"); } assert_eq!(code(&h5rs(&["diff", "--bogus=1", &b, &b])), 2); } /// Tolerances on 64-bit integers one apart, far beyond f64's integer /// precision: compared exactly, as h5diff does. #[test] fn diff_tolerances_compare_large_integers_exactly() { let Some(f) = generate() else { return }; let (a, b) = (f.p("big1.h5"), f.p("big2.h5")); let cases: Vec<(Vec<&str>, i32)> = vec![ (vec![&a, &b], 1), (vec!["-d", "0", &a, &b], 1), (vec!["-d", "0.5", &a, &b], 1), (vec!["-d", "1", &a, &b], 0), (vec!["-d", "0", &a, &b, "/u"], 1), (vec!["-p", "0", &a, &b], 1), (vec!["-p", "1e-18", &a, &b, "/i"], 1), (vec!["-p", "1e-15", &a, &b], 0), ]; let have_h5diff = tool_available("h5diff"); for (args, want) in cases { if have_h5diff { assert_eq!(code(&run("h5diff", &args)), want, "h5diff {args:?}"); } let o = h5rs(&[&["diff"], args.as_slice()].concat()); assert_eq!(code(&o), want, "h5rs diff {args:?}: {}", stdout(&o)); } // The reported difference is exact too. let s = stdout(&h5rs(&["diff", "-r", "-d", "0", &a, &b, "/u"])); assert!( s.contains("18446744073709551615 18446744073709551614 1\n"), "{s}" ); } /// A soft link is compared as a link (its target path), as h5diff does, /// unless `--follow-symlinks`, which compares (and walks into) what it /// leads to; two dangling links are then the same. #[test] fn diff_soft_links_like_h5diff() { let Some(f) = generate() else { return }; let (s1, s2, t1, t2) = ( f.p("soft1.h5"), f.p("soft2.h5"), f.p("target1.h5"), f.p("target2.h5"), ); let fl = "--follow-symlinks"; let cases: Vec<(Vec<&str>, i32)> = vec![ (vec![&s1, &s2, "/g/s"], 0), (vec![&s1, &s2, "/g"], 0), (vec![&s1, &s2, "/lnk"], 0), (vec![&s1, &s2, "/dang"], 0), (vec![&s1, &s2, "/lnk/d"], 1), (vec![&t1, &t2, "/s"], 1), (vec![&t1, &t2, "/s", "/rel"], 1), (vec![fl, &s1, &s2, "/g/s"], 1), (vec![fl, &s1, &s2, "/g"], 1), (vec![fl, &s1, &s2, "/lnk"], 1), (vec![fl, &s1, &s2, "/dang"], 0), (vec![fl, &s1, &s1], 0), (vec![fl, &t1, &t2, "/s"], 0), (vec![fl, &t1, &t2, "/s", "/rel"], 0), ]; let have_h5diff = tool_available("h5diff"); for (args, want) in cases { if have_h5diff { assert_eq!(code(&run("h5diff", &args)), want, "h5diff {args:?}"); } let o = h5rs(&[&["diff"], args.as_slice()].concat()); assert_eq!(code(&o), want, "h5rs diff {args:?}: {}", stdout(&o)); } } /// An object hard-linked under two names is compared under both, with /// everything below it, so a file that shares one object between two names /// equals a file that stores two identical copies. #[test] fn diff_compares_every_hard_link_path() { let Some(f) = generate() else { return }; let (linked, copied, changed) = ( f.p("hardlinked.h5"), f.p("copied.h5"), f.p("copied_changed.h5"), ); // A hard-linked dataset: h5diff agrees (exit 0). for (a, b) in [(&linked, &copied), (&copied, &linked)] { let o = h5rs(&["diff", a, b, "/y"]); assert_eq!(code(&o), 0, "{a} {b} /y: {}", stdout(&o)); if tool_available("h5diff") { assert_eq!(code(&run("h5diff", &[a, b, "/y"])), 0, "h5diff /y"); } // The whole file, including a hard-linked group's members (h5diff // does not descend a group's second name, so it reports those // members as present in one file only and exits 1). let o = h5rs(&["diff", a, b]); assert_eq!(code(&o), 0, "{a} {b}: {}", stdout(&o)); assert!(!stdout(&o).contains("exists only"), "{}", stdout(&o)); } // A value under the second name is compared, not skipped. let o = h5rs(&["diff", "-r", &linked, &changed]); assert_eq!(code(&o), 1, "{}", stdout(&o)); assert!( stdout(&o).contains("dataset: and "), "{}", stdout(&o) ); assert!(!stdout(&o).contains(""), "{}", stdout(&o)); } /// Where h5rs deliberately differs from h5diff: objects that cannot be /// compared (another shape, another datatype class) are a difference. #[test] fn diff_counts_incomparable_objects_as_different() { let Some(f) = generate() else { return }; let b = f.p("base.h5"); for other in ["reshaped.h5", "int.h5"] { let o = h5rs(&["diff", &b, &f.p(other)]); assert_eq!(code(&o), 1, "{other}: {}", stdout(&o)); assert!(stdout(&o).contains("Not comparable"), "{other}"); } } // --------------------------------------------------------------------------- // check // --------------------------------------------------------------------------- #[test] fn check_accepts_valid_files() { let Some(f) = generate() else { return }; for name in LIBVERS.iter().chain(&[ "userblock.h5", "base.h5", "changed.h5", "extra.h5", "attr.h5", ]) { let o = h5rs(&["check", "--data", &f.p(name)]); assert_eq!(code(&o), 0, "{name}:\n{}", stdout(&o)); assert!(stdout(&o).contains("no problems found")); } // The newest format's checksummed structures were all verified. let s = stdout(&h5rs(&["check", &f.p("latest.h5")])); assert!(s.contains("superblock 1, v2 object headers 33,"), "{s}"); assert!(s.contains("chunk indexes 5"), "{s}"); } /// Signatures of checksummed structures, and where each one's checksum /// is: found by trying every end offset (the checksum is the Jenkins /// lookup3 hash of the bytes before it). A fractal heap direct block's /// checksum is instead stored near its start, computed over the whole block /// with the field zeroed. fn checksummed_structures(data: &[u8]) -> Vec<(&'static str, usize, usize)> { const SIGS: [&str; 13] = [ "OHDR", "OCHK", "BTHD", "BTIN", "BTLF", "FRHP", "FHIB", "EAHD", "EAIB", "EASB", "EADB", "FAHD", "FADB", ]; let mut found = Vec::new(); for start in 0..data.len().saturating_sub(4) { let sig = &data[start..start + 4]; if let Some(name) = SIGS.iter().find(|s| s.as_bytes() == sig) { let limit = data.len().min(start + 65536); for end in start + 8..limit.saturating_sub(4) { let stored = u32::from_le_bytes(data[end..end + 4].try_into().unwrap()); if jenkins_lookup3(&data[start..end]) == stored { found.push((*name, start, end)); break; } } } else if sig == b"FHDB" { // version(1) heap address(8) block offset(1..8) checksum(4) 'found: for boff in 1..=8usize { let at = start + 5 + 8 + boff; let Some(stored) = data.get(at..at + 4) else { break; }; let stored = u32::from_le_bytes(stored.try_into().unwrap()); for size in [512usize, 1024, 2048, 4096, 8192, 16384, 32768, 65536] { let Some(block) = data.get(start..start + size) else { break; }; let mut b = block.to_vec(); b[at - start..at - start + 4].fill(0); if jenkins_lookup3(&b) == stored { found.push(("FHDB", start, at)); break 'found; } } } } } found } #[test] fn check_flags_every_corrupted_checksum() { let Some(f) = generate() else { return }; let path = f.path("latest.h5"); let data = std::fs::read(&path).unwrap(); let structures = checksummed_structures(&data); let kinds: std::collections::BTreeSet<&str> = structures.iter().map(|s| s.0).collect(); for want in [ "OHDR", "OCHK", "BTHD", "BTLF", "FRHP", "FHDB", "EAHD", "EAIB", "EADB", "FAHD", "FADB", ] { assert!(kinds.contains(want), "test file has no {want}: {kinds:?}"); } let bad = f.path("bad.h5"); for (name, start, at) in structures { let mut d = data.clone(); d[at] ^= 0x01; std::fs::write(&bad, &d).unwrap(); let o = h5rs(&["check", &bad.to_string_lossy()]); let s = stdout(&o); assert_eq!( code(&o), 1, "{name} at {start:#x}: checksum flip not flagged\n{s}" ); assert!( s.lines() .any(|l| l.starts_with("problem: 0x") && l.contains("checksum")), "{name} at {start:#x}: no checksum problem reported\n{s}" ); } } /// `check --data` follows variable-length elements into the global heap: /// a damaged collection is reported at its address, for the dataset and /// the attribute that point into it. Without --data it is not read. #[test] fn check_data_follows_vl_data_into_the_global_heap() { let Some(f) = generate() else { return }; for name in LIBVERS { let mut data = std::fs::read(f.path(name)).unwrap(); let gcols: Vec = (0..data.len() - 4) .filter(|&i| &data[i..i + 4] == b"GCOL") .collect(); assert!(!gcols.is_empty(), "{name}: no global heap"); // "GCOL" version(1) reserved(3) size(8), then heap object 1: // index(2) refcount(2) reserved(4) size(8) — claim 4 GiB. let at = gcols[0]; data[at + 24..at + 28].fill(0xff); let bad = f.path(&format!("bad-gcol-{name}")); std::fs::write(&bad, &data).unwrap(); let bad = bad.to_string_lossy().into_owned(); let o = h5rs(&["check", "--data", &bad]); let s = stdout(&o); assert_eq!(code(&o), 1, "{name}: {s}"); let want = format!("problem: {at:#x} /vlstr: variable-length data: global heap"); assert!(s.contains(&want), "{name}: no {want:?} in\n{s}"); // h5dump refuses the file too. if tool_available("h5dump") { assert_ne!(code(&run("h5dump", &[&bad])), 0, "{name}: h5dump read it"); } assert_eq!(code(&h5rs(&["check", &bad])), 0, "{name}: without --data"); let ok = h5rs(&["check", "--data", &f.p(name)]); assert!( stdout(&ok).contains(&format!("global heap collections read: {}", gcols.len())), "{name}: {}", stdout(&ok) ); } } #[test] fn check_flags_a_truncated_file() { let Some(f) = generate() else { return }; let data = std::fs::read(f.path("latest.h5")).unwrap(); let bad = f.path("truncated.h5"); std::fs::write(&bad, &data[..data.len() * 3 / 4]).unwrap(); let o = h5rs(&["check", &bad.to_string_lossy()]); assert_eq!(code(&o), 1); assert!(stdout(&o).contains("file is truncated"), "{}", stdout(&o)); } /// A v1 B-tree chunk index has no checksum; its keys are checked against /// the dataset: a chunk offset that is not a multiple of the chunk size. #[test] fn check_flags_a_misaligned_chunk() { let Some(f) = generate() else { return }; let path = f.path("earliest.h5"); let mut data = std::fs::read(&path).unwrap(); // The leaf chunk B-tree of /grp/gz (10 deflated chunks of 100 i32, so // each stored in under 400 bytes): "TREE" type(1)=1 level(1)=0 // entries(2) left(8) right(8), then key, child, key, child, ... where a // rank-1 chunk key is size(4) mask(4) offsets(2 x 8) and a child is 8. // Key 1 starts at 24 + 24 + 8 = 56; its dimension-0 offset (100) at 64. let mut patched = false; for start in 0..data.len() - 72 { if &data[start..start + 4] == b"TREE" && data[start + 4] == 1 && data[start + 5] == 0 { let entries = u16::from_le_bytes(data[start + 6..start + 8].try_into().unwrap()); let size0 = u32::from_le_bytes(data[start + 24..start + 28].try_into().unwrap()); let off1 = u64::from_le_bytes(data[start + 64..start + 72].try_into().unwrap()); if entries == 10 && size0 < 400 && off1 == 100 { data[start + 64..start + 72].copy_from_slice(&103u64.to_le_bytes()); patched = true; break; } } } assert!(patched, "no chunk B-tree found for /grp/gz"); let bad = f.path("misaligned.h5"); std::fs::write(&bad, &data).unwrap(); let o = h5rs(&["check", &bad.to_string_lossy()]); let s = stdout(&o); assert_eq!(code(&o), 1, "{s}"); assert!( s.contains("/grp/gz: chunk at [103, 0] offset 103 in dimension 0 is not a multiple of the chunk size 100"), "{s}" ); } #[test] fn every_subcommand_rejects_a_non_hdf5_file_cleanly() { let dir = tempfile::tempdir().unwrap(); let p = dir.path().join("junk.h5"); std::fs::write(&p, b"this is not an HDF5 file at all").unwrap(); let p = p.to_string_lossy().into_owned(); for args in [ vec!["ls", p.as_str()], vec!["dump", p.as_str()], vec!["stat", p.as_str()], vec!["diff", p.as_str(), p.as_str()], ] { let o = h5rs(&args); assert_eq!(code(&o), 2, "{args:?}"); assert!(String::from_utf8_lossy(&o.stderr).contains("not an HDF5 file")); } let o = h5rs(&["check", &p]); assert_eq!(code(&o), 1); assert!(stdout(&o).contains("no HDF5 signature")); assert_eq!(code(&h5rs(&["nonsense"])), 2); assert_eq!(code(&h5rs(&["ls"])), 2); assert_eq!(code(&h5rs(&["--help"])), 0); }