read: look names up through the dense name indexes
Finding one link or attribute by name read every entry: Group::dataset and Group::group (File, MmapFile, LazyFile) listed the whole group per call, and path resolution scanned each group's links. Opening every child of a 35 001-link group by name decoded ~1.2e9 links. Now a dense group's v2 B-tree name index (type 5, lookup3 hash of the name) is descended to the records with the name's hash (btree_v2::find_btree_v2_records reads only the nodes whose key interval overlaps), and only those links are read and compared; all hash-equal records are compared, so libhdf5's tie order does not matter. Dense attributes the same through their type 8 index (attribute::find_attribute_in_file, facade attr(name)); huge heap objects through their ID-ordered index. group_v2::resolve_child returns what the listing has under a name (soft links followed, dangling/external ones not found). Group::entries and File::group_at hand out a listing's addresses. The lookup-stats feature counts heap objects read. Tests: one lookup in an h5py-written 35 001-link group with colliding hashes reads at most two links (before: 35 001, failing), attribute lookups likewise (before: 3 000, failing), every child opens through all three readers and matches h5py, every link kind resolves as h5py resolves it in dense and compact groups, 300 huge attributes are found, and a range search matches a full scan at every tree depth. Co-Authored-By: Claude Opus 5.5 (1M context) <[email protected]>
This commit is contained in:
@@ -385,6 +385,59 @@ mod tests {
|
||||
assert!(nodes > 0);
|
||||
}
|
||||
|
||||
/// Descending to a key range finds exactly the records a full read
|
||||
/// holds in it — runs of equal keys that straddle node boundaries
|
||||
/// included — at every depth, and nothing for keys not in the tree.
|
||||
#[test]
|
||||
fn a_key_range_search_matches_a_full_scan() {
|
||||
use crate::btree_v2::find_btree_v2_records;
|
||||
use core::cmp::Ordering;
|
||||
let rs = 11usize;
|
||||
// Keys 0, 0, 0, 2, 2, 2, 4, ...: runs of three, odd keys missing.
|
||||
for n in [1usize, 45, 46, 1150, 30_000] {
|
||||
let mut recs = Vec::with_capacity(n * rs);
|
||||
for i in 0..n {
|
||||
let mut r = vec![0u8; rs];
|
||||
r[..8].copy_from_slice(&((i / 3 * 2) as u64).to_be_bytes());
|
||||
r[8..].copy_from_slice(&[(i % 3) as u8, 0, 0]);
|
||||
recs.extend_from_slice(&r);
|
||||
}
|
||||
let base = 4096u64;
|
||||
let tree = build_btree_v2(params(512, 11), &recs, base, 8, 8).unwrap();
|
||||
let mut file = vec![0u8; base as usize];
|
||||
file.extend_from_slice(&tree);
|
||||
let hdr = BTreeV2Header::parse(&file, base as usize, 8, 8).unwrap();
|
||||
let all = collect_btree_v2_records(&file, &hdr, 8, 8).unwrap();
|
||||
let key = |r: &[u8]| u64::from_be_bytes(r[..8].try_into().unwrap());
|
||||
let last = key(&all[n - 1].data);
|
||||
let probes = (0..=last + 1).step_by(if n > 1000 { 37 } else { 1 });
|
||||
for k in probes.chain([last, last + 1, u64::MAX]) {
|
||||
let found =
|
||||
find_btree_v2_records(&file, &hdr, 8, &mut |r: &[u8]| key(r).cmp(&k)).unwrap();
|
||||
let want: Vec<&[u8]> = all
|
||||
.iter()
|
||||
.map(|r| r.data.as_slice())
|
||||
.filter(|r| key(r) == k)
|
||||
.collect();
|
||||
let got: Vec<&[u8]> = found.iter().map(|r| r.data.as_slice()).collect();
|
||||
assert_eq!(got, want, "n {n} key {k}");
|
||||
assert_eq!(
|
||||
got.len(),
|
||||
if k % 2 == 0 && k <= last {
|
||||
want.len()
|
||||
} else {
|
||||
0
|
||||
}
|
||||
);
|
||||
}
|
||||
// Every record, or none, when the whole tree is in or out of range.
|
||||
let every = find_btree_v2_records(&file, &hdr, 8, &mut |_| Ordering::Equal).unwrap();
|
||||
assert_eq!(every.len(), n);
|
||||
let none = find_btree_v2_records(&file, &hdr, 8, &mut |_| Ordering::Less).unwrap();
|
||||
assert!(none.is_empty());
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_node_too_small_or_too_big_is_an_error() {
|
||||
assert!(build_btree_v2(params(16, 11), &records(1, 11), 0, 8, 8).is_err());
|
||||
|
||||
Reference in New Issue
Block a user