osobhandClaude Opus 5.5
04a7f6f6c7
A valid group has one link per name, but a damaged or hand-made one can have two. resolve_child followed the first soft link of the name, the listing skipped a dangling one and listed the name via a later link, and path resolution followed the last symbolic link: three answers. All now take the first link of the name (header message order in a compact group, name index order in a dense one) and ignore the rest, even if the first dangles. That is libhdf5's rule for compact groups (H5G__compact_lookup stops at the first Link message); h5py opens nothing for a dangling first link although a later one resolves. For a dense group libhdf5 binary-searches the index and may land on another of several exact duplicates; documented on first_link_named. find_symbolic_link's v2 branch was dead (only v1 groups reach it) and is now v1-only. Test: an h5py compact group with soft links dup_A (dangling, or to /d) and dup_B (the other), dup_B renamed to dup_A in the header and re-checksummed. Lookup, path and listing through all three readers match h5py for both orders. With the old group_v2.rs the path lookup returned 42 where h5py opens nothing. Co-Authored-By: Claude Opus 5.5 (1M context) <[email protected]>
613 lines
24 KiB
Rust
613 lines
24 KiB
Rust
//! Looking one name up in a dense group (links in a fractal heap, indexed by
|
|
//! a v2 B-tree of name hashes) or in dense attribute storage reads the name
|
|
//! index, not every link: O(log n) index nodes and only the links whose
|
|
//! lookup3 hash equals the name's. Before, every lookup decoded all n links,
|
|
//! so opening each child of a 35 001-link group by name decoded ~1.2e9.
|
|
//!
|
|
//! The file is written by h5py (libhdf5 orders the index), with names whose
|
|
//! hashes collide, and every result is compared with what h5py reads.
|
|
//!
|
|
//! Skipped when python3 with h5py is unavailable, unless
|
|
//! `CLAWHDF5_REQUIRE_INTEROP=1`.
|
|
|
|
use std::collections::{BTreeMap, HashMap};
|
|
use std::process::Command;
|
|
use std::sync::OnceLock;
|
|
use std::time::{Duration, Instant};
|
|
|
|
use clawhdf5::{AttrValue, File, LazyFile, MmapFile};
|
|
use clawhdf5_format::checksum::jenkins_lookup3;
|
|
use clawhdf5_format::error::FormatError;
|
|
use clawhdf5_format::lookup_stats;
|
|
|
|
fn python() -> String {
|
|
std::env::var("CLAWHDF5_PYTHON").unwrap_or_else(|_| "python3".to_string())
|
|
}
|
|
|
|
fn interop_required() -> bool {
|
|
std::env::var("CLAWHDF5_REQUIRE_INTEROP").is_ok_and(|v| v == "1")
|
|
}
|
|
|
|
fn python_available() -> bool {
|
|
Command::new(python())
|
|
.args(["-c", "import h5py"])
|
|
.output()
|
|
.map(|o| o.status.success())
|
|
.unwrap_or(false)
|
|
}
|
|
|
|
macro_rules! skip_if_no_python {
|
|
() => {
|
|
if !python_available() {
|
|
assert!(
|
|
!interop_required(),
|
|
"CLAWHDF5_REQUIRE_INTEROP=1 but python3 with h5py is not available"
|
|
);
|
|
eprintln!("SKIP: python3 with h5py not available");
|
|
return;
|
|
}
|
|
};
|
|
}
|
|
|
|
fn run_python(script: &str) -> String {
|
|
let output = Command::new(python())
|
|
.args(["-c", script])
|
|
.output()
|
|
.expect("failed to run python");
|
|
assert!(
|
|
output.status.success(),
|
|
"Python script failed:\nSTDOUT: {}\nSTDERR: {}",
|
|
String::from_utf8_lossy(&output.stdout),
|
|
String::from_utf8_lossy(&output.stderr)
|
|
);
|
|
String::from_utf8_lossy(&output.stdout).trim().to_string()
|
|
}
|
|
|
|
/// Links in the big group, as libhdf5's `h5stat_newgrat.h5` has.
|
|
const LINKS: usize = 35_001;
|
|
/// Attributes on the dense-attribute dataset.
|
|
const ATTRS: usize = 3_000;
|
|
|
|
/// Pairs of distinct names with equal lookup3 hashes, found by search (the
|
|
/// hash is fixed, so the pairs are too).
|
|
fn colliding_pairs(count: usize) -> Vec<(String, String)> {
|
|
let mut seen: HashMap<u32, String> = HashMap::new();
|
|
let mut pairs = Vec::new();
|
|
for i in 0.. {
|
|
let name = format!("c{i}");
|
|
let h = jenkins_lookup3(name.as_bytes());
|
|
if let Some(first) = seen.insert(h, name.clone()) {
|
|
pairs.push((first, name));
|
|
if pairs.len() == count {
|
|
break;
|
|
}
|
|
}
|
|
}
|
|
pairs
|
|
}
|
|
|
|
/// What h5py reads: link values, attribute values, and which of the
|
|
/// missing names it finds as links and as attributes (none).
|
|
type H5pyView = (
|
|
BTreeMap<String, i64>,
|
|
BTreeMap<String, i64>,
|
|
Vec<String>,
|
|
Vec<String>,
|
|
);
|
|
|
|
struct Fixture {
|
|
_dir: tempfile::TempDir,
|
|
path: String,
|
|
/// Names in the big group, with the value of the scalar dataset each
|
|
/// links to, as h5py reads them.
|
|
links: BTreeMap<String, i64>,
|
|
/// Names that are not links but hash like one that is.
|
|
missing_links: Vec<String>,
|
|
/// Attributes of `/x`, as h5py reads them.
|
|
attrs: BTreeMap<String, i64>,
|
|
missing_attrs: Vec<String>,
|
|
}
|
|
|
|
fn fixture() -> &'static Fixture {
|
|
static FIXTURE: OnceLock<Fixture> = OnceLock::new();
|
|
FIXTURE.get_or_init(|| {
|
|
let pairs = colliding_pairs(6);
|
|
for (a, b) in &pairs {
|
|
assert_ne!(a, b);
|
|
assert_eq!(jenkins_lookup3(a.as_bytes()), jenkins_lookup3(b.as_bytes()));
|
|
}
|
|
// Pairs 0-2 both present (either can be the one libhdf5 orders
|
|
// first), pairs 3-5 only the first: its partner must not be found.
|
|
// "k69209"/"k155448" is the pair the writer once misordered.
|
|
let mut present: Vec<String> = vec!["k69209".into(), "k155448".into()];
|
|
let mut missing: Vec<String> = Vec::new();
|
|
for (i, (a, b)) in pairs.into_iter().enumerate() {
|
|
present.push(a);
|
|
if i < 3 {
|
|
present.push(b);
|
|
} else {
|
|
missing.push(b);
|
|
}
|
|
}
|
|
missing.extend(["", "nope", "n35001x", "N1"].map(String::from));
|
|
let mut links = present.clone();
|
|
let mut i = 0;
|
|
while links.len() < LINKS {
|
|
links.push(format!("n{i}"));
|
|
i += 1;
|
|
}
|
|
let mut attrs = present.clone();
|
|
attrs.extend((0..ATTRS - present.len()).map(|i| format!("a{i}")));
|
|
|
|
let dir = tempfile::tempdir().unwrap();
|
|
let path = dir.path().join("big.h5").display().to_string();
|
|
// The names go through a file: 35 001 of them overflow an argument.
|
|
let names = dir.path().join("names.json");
|
|
std::fs::write(
|
|
&names,
|
|
serde_json::to_string(&(&links, &attrs, &missing)).unwrap(),
|
|
)
|
|
.unwrap();
|
|
let names = names.display();
|
|
let out = run_python(&format!(
|
|
"import h5py, json, numpy as np\n\
|
|
links, attrs, missing = json.load(open(r'{names}'))\n\
|
|
with h5py.File(r'{path}', 'w', libver='latest') as f:\n\
|
|
\x20 g = f.create_group('g')\n\
|
|
\x20 for i, n in enumerate(links):\n\
|
|
\x20 g.create_dataset(n, data=np.int64(i))\n\
|
|
\x20 x = f.create_dataset('x', data=np.int64(0))\n\
|
|
\x20 for i, n in enumerate(attrs):\n\
|
|
\x20 x.attrs[n] = np.int64(1000 + i)\n\
|
|
with h5py.File(r'{path}', 'r') as f:\n\
|
|
\x20 g, a = f['g'], f['x'].attrs\n\
|
|
\x20 print(json.dumps([{{n: int(g[n][()]) for n in g}}, {{n: int(a[n]) for n in a}},\n\
|
|
\x20 [n for n in missing if n and n in g], [n for n in missing if n and n in a]]))",
|
|
));
|
|
let (links, attrs, found_links, found_attrs): H5pyView =
|
|
serde_json::from_str(&out).unwrap();
|
|
assert_eq!(links.len(), LINKS);
|
|
assert_eq!(attrs.len(), ATTRS);
|
|
assert!(found_links.is_empty() && found_attrs.is_empty());
|
|
Fixture {
|
|
_dir: dir,
|
|
path,
|
|
links,
|
|
missing_links: missing.clone(),
|
|
attrs,
|
|
missing_attrs: missing,
|
|
}
|
|
})
|
|
}
|
|
|
|
fn is_not_found(e: &clawhdf5::Error) -> bool {
|
|
matches!(e, clawhdf5::Error::Format(FormatError::PathNotFound(_)))
|
|
}
|
|
|
|
#[test]
|
|
fn one_link_lookup_reads_the_index_not_every_link() {
|
|
skip_if_no_python!();
|
|
let fx = fixture();
|
|
let f = File::open(&fx.path).unwrap();
|
|
let g = f.group("g").unwrap();
|
|
for (name, value) in &fx.links {
|
|
lookup_stats::reset();
|
|
let ds = g.dataset(name).unwrap();
|
|
// One link decoded per lookup, two where hashes collide — not 35 001.
|
|
let read = lookup_stats::heap_objects_read();
|
|
assert!(read <= 2, "looking up {name} read {read} heap objects");
|
|
assert_eq!(ds.read_i64().unwrap(), vec![*value], "{name}");
|
|
}
|
|
|
|
for name in &fx.missing_links {
|
|
lookup_stats::reset();
|
|
let err = g.dataset(name).unwrap_err();
|
|
assert!(is_not_found(&err), "{name:?}: {err:?}");
|
|
assert!(lookup_stats::heap_objects_read() <= 2, "{name:?}");
|
|
}
|
|
|
|
// A path resolves each component the same way.
|
|
for name in ["k155448", "n0", "n34000"] {
|
|
lookup_stats::reset();
|
|
let ds = f.dataset(&format!("/g/{name}")).unwrap();
|
|
assert!(lookup_stats::heap_objects_read() <= 2);
|
|
assert_eq!(ds.read_i64().unwrap(), vec![fx.links[name]]);
|
|
}
|
|
}
|
|
|
|
#[test]
|
|
fn one_attribute_lookup_reads_the_index_not_every_attribute() {
|
|
skip_if_no_python!();
|
|
let fx = fixture();
|
|
let f = File::open(&fx.path).unwrap();
|
|
let x = f.dataset("x").unwrap();
|
|
let all = x.attrs().unwrap();
|
|
assert_eq!(all.len(), ATTRS);
|
|
for (name, value) in &fx.attrs {
|
|
lookup_stats::reset();
|
|
let got = x.attr(name).unwrap();
|
|
assert!(lookup_stats::heap_objects_read() <= 2, "{name}");
|
|
assert!(
|
|
matches!(got, Some(AttrValue::I64(v)) if v == *value),
|
|
"{name}: {got:?}"
|
|
);
|
|
assert!(matches!(all.get(name), Some(AttrValue::I64(v)) if v == value));
|
|
}
|
|
for name in &fx.missing_attrs {
|
|
lookup_stats::reset();
|
|
assert!(x.attr(name).unwrap().is_none(), "{name:?}");
|
|
assert!(lookup_stats::heap_objects_read() <= 2, "{name:?}");
|
|
}
|
|
// Compact attributes (on the root group: none) and a group's attributes.
|
|
assert!(f.root().attr("k69209").unwrap().is_none());
|
|
}
|
|
|
|
/// Every child of the big group opened by name through each file type,
|
|
/// within `limit`: with a scan per lookup this is ~1.2e9 link decodes.
|
|
#[test]
|
|
fn opening_every_child_of_a_35001_link_group_by_name_is_quick() {
|
|
skip_if_no_python!();
|
|
let fx = fixture();
|
|
let limit = Duration::from_secs(120);
|
|
let started = Instant::now();
|
|
let check_time = |n: usize| {
|
|
assert!(
|
|
started.elapsed() < limit,
|
|
"{n} lookups took {:?}",
|
|
started.elapsed()
|
|
);
|
|
};
|
|
|
|
let f = File::open(&fx.path).unwrap();
|
|
let g = f.group("g").unwrap();
|
|
for (n, (name, value)) in fx.links.iter().enumerate() {
|
|
assert_eq!(g.dataset(name).unwrap().read_i64().unwrap(), vec![*value]);
|
|
check_time(n);
|
|
}
|
|
// The listing hands out entries: open each by address.
|
|
let entries = g.entries().unwrap();
|
|
assert_eq!(entries.len(), LINKS);
|
|
for (name, address) in &entries {
|
|
let ds = f.dataset_at(*address).unwrap();
|
|
assert_eq!(ds.read_i64().unwrap(), vec![fx.links[name]]);
|
|
}
|
|
assert!(f.group_at(g_address(&f)).dataset("n0").is_ok());
|
|
|
|
let m = MmapFile::open(&fx.path).unwrap();
|
|
let mg = m.group("g").unwrap();
|
|
for (n, (name, value)) in fx.links.iter().enumerate() {
|
|
assert_eq!(mg.dataset(name).unwrap().read_i64().unwrap(), vec![*value]);
|
|
check_time(n);
|
|
}
|
|
assert!(mg.group("nope").is_err_and(|e| is_not_found(&e)));
|
|
|
|
let l = LazyFile::open_mmap(&fx.path).unwrap();
|
|
let lg = l.group("g").unwrap();
|
|
for (n, (name, value)) in fx.links.iter().enumerate() {
|
|
assert_eq!(lg.dataset(name).unwrap().read_i64().unwrap(), vec![*value]);
|
|
check_time(n);
|
|
}
|
|
let lx = l.dataset("x").unwrap();
|
|
assert!(
|
|
matches!(lx.attr("k155448").unwrap(), Some(AttrValue::I64(v)) if v == fx.attrs["k155448"])
|
|
);
|
|
assert!(
|
|
lg.dataset(&fx.missing_links[0])
|
|
.is_err_and(|e| is_not_found(&e))
|
|
);
|
|
}
|
|
|
|
fn g_address(f: &File) -> u64 {
|
|
f.root()
|
|
.entries()
|
|
.unwrap()
|
|
.into_iter()
|
|
.find(|(n, _)| n == "g")
|
|
.unwrap()
|
|
.1
|
|
}
|
|
|
|
/// Every kind of link, looked up by name in a dense group (through the name
|
|
/// index) and in a compact one, opens what h5py opens and nothing it cannot:
|
|
/// hard links, soft links (absolute, relative, to a group), and not a
|
|
/// dangling soft link, an external link or a missing name.
|
|
#[test]
|
|
fn links_of_every_kind_resolve_by_name_as_in_h5py() {
|
|
skip_if_no_python!();
|
|
let dir = tempfile::tempdir().unwrap();
|
|
let path = dir.path().join("links.h5").display().to_string();
|
|
// For each group and name: "dataset <value>", "group", or "none" as
|
|
// h5py sees it.
|
|
let out = run_python(&format!(
|
|
"import h5py, json, numpy as np\n\
|
|
with h5py.File(r'{path}', 'w', libver='latest') as f:\n\
|
|
\x20 for gname, n in (('dense', 20), ('compact', 2)):\n\
|
|
\x20 g = f.create_group(gname)\n\
|
|
\x20 for i in range(n):\n\
|
|
\x20 g.create_dataset(f'd{{i}}', data=np.int64(100 + i))\n\
|
|
\x20 s = g.create_group('sub')\n\
|
|
\x20 s.create_dataset('x', data=np.int64(7))\n\
|
|
\x20 g['abs'] = h5py.SoftLink(f'/{{gname}}/d1')\n\
|
|
\x20 g['rel'] = h5py.SoftLink('sub/x')\n\
|
|
\x20 g['tosub'] = h5py.SoftLink('sub')\n\
|
|
\x20 g['dangling'] = h5py.SoftLink('/nowhere')\n\
|
|
\x20 g['ext'] = h5py.ExternalLink('other.h5', '/y')\n\
|
|
names = ['d0', 'd1', 'sub', 'abs', 'rel', 'tosub', 'dangling', 'ext', 'nope', '']\n\
|
|
seen = {{}}\n\
|
|
with h5py.File(r'{path}', 'r') as f:\n\
|
|
\x20 for gname in ('dense', 'compact'):\n\
|
|
\x20 g = f[gname]\n\
|
|
\x20 for n in names:\n\
|
|
\x20 try:\n\
|
|
\x20 o = g[n] if n else None\n\
|
|
\x20 except (KeyError, OSError):\n\
|
|
\x20 o = None\n\
|
|
\x20 if isinstance(o, h5py.Dataset):\n\
|
|
\x20 seen[f'{{gname}}/{{n}}'] = f'dataset {{int(o[()])}}'\n\
|
|
\x20 elif isinstance(o, h5py.Group):\n\
|
|
\x20 seen[f'{{gname}}/{{n}}'] = 'group'\n\
|
|
\x20 else:\n\
|
|
\x20 seen[f'{{gname}}/{{n}}'] = 'none'\n\
|
|
print(json.dumps(seen))",
|
|
));
|
|
let seen: BTreeMap<String, String> = serde_json::from_str(&out).unwrap();
|
|
assert_eq!(seen.len(), 20);
|
|
|
|
let f = File::open(&path).unwrap();
|
|
// The dense group's links are in a heap, the compact group's in its
|
|
// header.
|
|
for (gname, dense) in [("dense", true), ("compact", false)] {
|
|
let g = f.group(gname).unwrap();
|
|
lookup_stats::reset();
|
|
g.dataset("d0").unwrap();
|
|
assert_eq!(lookup_stats::heap_objects_read() > 0, dense, "{gname}");
|
|
}
|
|
let m = MmapFile::open(&path).unwrap();
|
|
let l = LazyFile::open_mmap(&path).unwrap();
|
|
for (key, want) in &seen {
|
|
let (gname, name) = key.split_once('/').unwrap();
|
|
let got = {
|
|
let g = f.group(gname).unwrap();
|
|
match (g.dataset(name), g.group(name)) {
|
|
(Ok(ds), _) => format!("dataset {}", ds.read_i64().unwrap()[0]),
|
|
(Err(clawhdf5::Error::NotADataset(_)), Ok(sub)) => {
|
|
// A group: it has the child `x` (checks the address).
|
|
assert!(sub.dataset("x").is_ok() || name == "sub" || name == "tosub");
|
|
"group".to_string()
|
|
}
|
|
(Err(e), Err(e2)) => {
|
|
assert!(
|
|
is_not_found(&e) && is_not_found(&e2),
|
|
"{key}: {e:?} / {e2:?}"
|
|
);
|
|
"none".to_string()
|
|
}
|
|
(Err(e), Ok(_)) => panic!("{key}: dataset {e:?} but group ok"),
|
|
}
|
|
};
|
|
assert_eq!(&got, want, "{key}");
|
|
// The other readers agree, and a path through the group resolves the
|
|
// same way.
|
|
let mg = m.group(gname).unwrap();
|
|
let lg = l.group(gname).unwrap();
|
|
match want.strip_prefix("dataset ") {
|
|
Some(v) => {
|
|
let v: i64 = v.parse().unwrap();
|
|
assert_eq!(mg.dataset(name).unwrap().read_i64().unwrap(), vec![v]);
|
|
assert_eq!(lg.dataset(name).unwrap().read_i64().unwrap(), vec![v]);
|
|
let ds = f.dataset(&format!("/{gname}/{name}")).unwrap();
|
|
assert_eq!(ds.read_i64().unwrap(), vec![v], "{key}");
|
|
}
|
|
None if want == "group" => {
|
|
assert!(mg.group(name).unwrap().dataset("x").is_ok(), "{key}");
|
|
assert!(lg.group(name).unwrap().dataset("x").is_ok(), "{key}");
|
|
let ds = f.dataset(&format!("/{gname}/{name}/x")).unwrap();
|
|
assert_eq!(ds.read_i64().unwrap(), vec![7]);
|
|
}
|
|
None => {
|
|
assert!(mg.dataset(name).is_err_and(|e| is_not_found(&e)), "{key}");
|
|
assert!(lg.group(name).is_err_and(|e| is_not_found(&e)), "{key}");
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
/// The link name index (v2 B-tree, record type 5) of the big group: its
|
|
/// depth and root node address, read from the one type-5 `BTHD` in the file.
|
|
fn name_index_root(bytes: &[u8]) -> (u16, usize) {
|
|
let headers: Vec<usize> = bytes
|
|
.windows(4)
|
|
.enumerate()
|
|
.filter(|(i, w)| *w == b"BTHD" && bytes.get(i + 5) == Some(&5))
|
|
.map(|(i, _)| i)
|
|
.collect();
|
|
assert_eq!(headers.len(), 1, "type-5 B-tree headers at {headers:?}");
|
|
let h = headers[0];
|
|
// signature, version, type, node size (4), record size (2), depth (2),
|
|
// split and merge percent, root address (8).
|
|
let depth = u16::from_le_bytes([bytes[h + 12], bytes[h + 13]]);
|
|
let root = u64::from_le_bytes(bytes[h + 16..h + 24].try_into().unwrap());
|
|
(depth, usize::try_from(root).unwrap())
|
|
}
|
|
|
|
/// One byte changed in a key of the name index's root (an internal node)
|
|
/// must be an error, not a name quietly routed to the wrong child and
|
|
/// reported missing: lookups prune children by those keys. libhdf5 checks
|
|
/// the internal node's checksum and refuses the group; so must we, for a
|
|
/// lookup and for a listing.
|
|
#[test]
|
|
fn a_corrupt_internal_index_node_is_an_error_not_a_missing_name() {
|
|
skip_if_no_python!();
|
|
let fx = fixture();
|
|
let mut bytes = std::fs::read(&fx.path).unwrap();
|
|
let (depth, root) = name_index_root(&bytes);
|
|
assert!(depth >= 2, "want a deep index, got depth {depth}");
|
|
assert_eq!(&bytes[root..root + 4], b"BTIN");
|
|
// Signature, version, type, then record 0: its name hash comes first.
|
|
bytes[root + 6] ^= 0x5a;
|
|
let dir = tempfile::tempdir().unwrap();
|
|
let bad = dir.path().join("bad.h5");
|
|
std::fs::write(&bad, &bytes).unwrap();
|
|
let bad = bad.display().to_string();
|
|
|
|
let is_checksum = |e: &clawhdf5::Error| {
|
|
matches!(
|
|
e,
|
|
clawhdf5::Error::Format(FormatError::ChecksumMismatch { .. })
|
|
)
|
|
};
|
|
let f = File::open(&bad).unwrap();
|
|
let g = f.group("g").unwrap();
|
|
// Every name, present or not, goes through the root.
|
|
for name in fx.links.keys().step_by(97).chain(&fx.missing_links) {
|
|
let err = g.dataset(name).map(|_| ()).unwrap_err();
|
|
assert!(is_checksum(&err), "dataset({name:?}): {err:?}");
|
|
}
|
|
let err = f.dataset("/g/n0").map(|_| ()).unwrap_err();
|
|
assert!(is_checksum(&err), "path: {err:?}");
|
|
let err = g.datasets().unwrap_err();
|
|
assert!(is_checksum(&err), "listing: {err:?}");
|
|
let err = g.entries().unwrap_err();
|
|
assert!(is_checksum(&err), "entries: {err:?}");
|
|
|
|
let m = MmapFile::open(&bad).unwrap();
|
|
let mg = m.group("g").unwrap();
|
|
assert!(mg.dataset("n0").is_err_and(|e| is_checksum(&e)));
|
|
assert!(mg.datasets().is_err_and(|e| is_checksum(&e)));
|
|
let l = LazyFile::open_mmap(&bad).unwrap();
|
|
let lg = l.group("g").unwrap();
|
|
assert!(lg.dataset("n0").is_err_and(|e| is_checksum(&e)));
|
|
assert!(lg.datasets().is_err_and(|e| is_checksum(&e)));
|
|
|
|
// libhdf5 refuses both too.
|
|
let out = run_python(&format!(
|
|
"import h5py\n\
|
|
r = []\n\
|
|
with h5py.File(r'{bad}', 'r') as f:\n\
|
|
\x20 g = f['g']\n\
|
|
\x20 for op in (lambda: g['n0'], lambda: list(g)):\n\
|
|
\x20 try:\n\
|
|
\x20 op()\n\
|
|
\x20 r.append('ok')\n\
|
|
\x20 except Exception as e:\n\
|
|
\x20 r.append('checksum' if 'checksum' in str(e) else repr(e))\n\
|
|
print(' '.join(r))",
|
|
));
|
|
assert_eq!(out, "checksum checksum");
|
|
}
|
|
|
|
/// Rename the one link called `from` to `to` (same length) in `bytes`, and
|
|
/// re-checksum the object header chunk holding it: two links of one name,
|
|
/// which libhdf5 cannot write.
|
|
fn rename_link_in_header(bytes: &mut [u8], from: &[u8], to: &[u8]) {
|
|
assert_eq!(from.len(), to.len());
|
|
let find = |hay: &[u8], needle: &[u8]| hay.windows(needle.len()).position(|w| w == needle);
|
|
let at = find(bytes, from).expect("link name");
|
|
assert!(find(&bytes[at + 1..], from).is_none(), "name not unique");
|
|
bytes[at..at + to.len()].copy_from_slice(to);
|
|
// The v2 object header (chunk 0) holding it.
|
|
let ohdr = bytes[..at]
|
|
.windows(4)
|
|
.rposition(|w| w == b"OHDR")
|
|
.expect("OHDR");
|
|
let flags = bytes[ohdr + 5];
|
|
let mut pos = ohdr + 6;
|
|
if flags & 0x20 != 0 {
|
|
pos += 16; // times
|
|
}
|
|
if flags & 0x10 != 0 {
|
|
pos += 4; // attribute phase change
|
|
}
|
|
let width = 1usize << (flags & 3);
|
|
let mut size = [0u8; 8];
|
|
size[..width].copy_from_slice(&bytes[pos..pos + width]);
|
|
let end = pos + width + usize::try_from(u64::from_le_bytes(size)).unwrap();
|
|
assert!(at < end, "name outside chunk 0");
|
|
let sum = jenkins_lookup3(&bytes[ohdr..end]);
|
|
bytes[end..end + 4].copy_from_slice(&sum.to_le_bytes());
|
|
}
|
|
|
|
/// Two soft links of one name (a damaged or hand-made group; libhdf5
|
|
/// cannot create one), one dangling: only the first counts, as in libhdf5,
|
|
/// which opens the first Link message of a name and fails if it dangles.
|
|
/// Lookup, path and listing agree — before, the listing skipped a dangling
|
|
/// first link and listed the name via the second, which lookup did not
|
|
/// follow, and path resolution followed the last.
|
|
#[test]
|
|
fn of_two_links_with_one_name_the_first_wins_everywhere() {
|
|
skip_if_no_python!();
|
|
let dir = tempfile::tempdir().unwrap();
|
|
for dangling_first in [true, false] {
|
|
let path = dir
|
|
.path()
|
|
.join(format!("dup_{dangling_first}.h5"))
|
|
.display()
|
|
.to_string();
|
|
let (first, second) = if dangling_first {
|
|
("/nowhere_xyz", "/d")
|
|
} else {
|
|
("/d", "/nowhere_xyz")
|
|
};
|
|
run_python(&format!(
|
|
"import h5py, numpy as np\n\
|
|
with h5py.File(r'{path}', 'w', libver='latest') as f:\n\
|
|
\x20 f.create_dataset('d', data=np.int64(42))\n\
|
|
\x20 s = f.create_group('s')\n\
|
|
\x20 s['dup_A'] = h5py.SoftLink('{first}')\n\
|
|
\x20 s['dup_B'] = h5py.SoftLink('{second}')",
|
|
));
|
|
let mut bytes = std::fs::read(&path).unwrap();
|
|
rename_link_in_header(&mut bytes, b"dup_B", b"dup_A");
|
|
std::fs::write(&path, &bytes).unwrap();
|
|
|
|
// What libhdf5 opens under that name: both names listed, first link
|
|
// followed.
|
|
let out = run_python(&format!(
|
|
"import h5py\n\
|
|
with h5py.File(r'{path}', 'r') as f:\n\
|
|
\x20 s = f['s']\n\
|
|
\x20 assert list(s) == ['dup_A', 'dup_A'], list(s)\n\
|
|
\x20 try:\n\
|
|
\x20 print(int(s['dup_A'][()]))\n\
|
|
\x20 except KeyError:\n\
|
|
\x20 print('none')",
|
|
));
|
|
let want = if dangling_first { "none" } else { "42" };
|
|
assert_eq!(out, want, "h5py, dangling first: {dangling_first}");
|
|
let want = (!dangling_first).then_some(42i64);
|
|
|
|
let f = File::open(&path).unwrap();
|
|
let s = f.group("s").unwrap();
|
|
let got = |r: Result<clawhdf5::Dataset<'_>, clawhdf5::Error>| match r {
|
|
Ok(ds) => Some(ds.read_i64().unwrap()[0]),
|
|
Err(e) => {
|
|
assert!(is_not_found(&e), "{e:?}");
|
|
None
|
|
}
|
|
};
|
|
assert_eq!(got(s.dataset("dup_A")), want, "lookup, {dangling_first}");
|
|
assert_eq!(got(f.dataset("/s/dup_A")), want, "path, {dangling_first}");
|
|
let listed = s.datasets().unwrap();
|
|
let listed_n = listed.iter().filter(|n| *n == "dup_A").count();
|
|
assert_eq!(listed_n, usize::from(want.is_some()), "{listed:?}");
|
|
let entries = s.entries().unwrap();
|
|
assert_eq!(entries.len(), listed_n, "{entries:?}");
|
|
|
|
let m = MmapFile::open(&path).unwrap();
|
|
let l = LazyFile::open_mmap(&path).unwrap();
|
|
let (mg, lg) = (m.group("s").unwrap(), l.group("s").unwrap());
|
|
match want {
|
|
Some(v) => {
|
|
assert_eq!(mg.dataset("dup_A").unwrap().read_i64().unwrap(), vec![v]);
|
|
assert_eq!(lg.dataset("dup_A").unwrap().read_i64().unwrap(), vec![v]);
|
|
}
|
|
None => {
|
|
assert!(mg.dataset("dup_A").is_err_and(|e| is_not_found(&e)));
|
|
assert!(lg.dataset("dup_A").is_err_and(|e| is_not_found(&e)));
|
|
}
|
|
}
|
|
assert_eq!(mg.datasets().unwrap(), listed);
|
|
assert_eq!(lg.datasets().unwrap(), listed);
|
|
}
|
|
}
|