perf(py): datasets and groups keep their address; groups their links
Every ds[...] and g[k] resolved the path from the root again, two or three times per open, and resolving a name in a large group scans its links: visiting a group was O(n^2). 4000 scalar datasets in one group took 39 s (v1 group) and 131 s (dense) to list, read and re-read; now 0.3 s each. A Dataset keeps its object address, a Group (and the file's root) its address and, after the first lookup, its link table. New facade API File::dataset_at(address), tested in integration_tests. Co-Authored-By: Claude Opus 5.5 (1M context) <[email protected]>
This commit is contained in:
@@ -176,6 +176,22 @@ impl File {
|
||||
})
|
||||
}
|
||||
|
||||
/// A `Dataset` handle for the object header at `address` (an address
|
||||
/// from a group listing, or one kept from an earlier lookup), without
|
||||
/// resolving a path. Resolving a path walks every group on it, which in
|
||||
/// a large group costs a scan of its links; keep the address instead to
|
||||
/// open the same dataset repeatedly.
|
||||
pub fn dataset_at(&self, address: u64) -> Result<Dataset<'_>, Error> {
|
||||
let hdr = self.parse_header(address)?;
|
||||
if !has_message(&hdr, MessageType::DataLayout) {
|
||||
return Err(Error::NotADataset(format!("object at address {address}")));
|
||||
}
|
||||
Ok(Dataset {
|
||||
file: self,
|
||||
header: hdr,
|
||||
})
|
||||
}
|
||||
|
||||
/// Resolve a path and return a `Group` handle.
|
||||
///
|
||||
/// The path uses `/` separators (e.g., `"sensors"`).
|
||||
|
||||
@@ -986,3 +986,35 @@ fn u64_data_roundtrip() {
|
||||
values
|
||||
);
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Opening a dataset by address
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
#[test]
|
||||
fn dataset_at_opens_the_same_dataset_as_its_path() {
|
||||
let mut b = FileBuilder::new();
|
||||
let mut g = b.create_group("grp");
|
||||
g.create_dataset("vals").with_f64_data(&[1.0, 2.5, -3.0]);
|
||||
b.add_group(g.finish());
|
||||
let file = File::from_bytes(b.finish().unwrap()).unwrap();
|
||||
|
||||
let addr =
|
||||
clawhdf5_format::group_v2::resolve_path_any(file.as_bytes(), file.superblock(), "grp/vals")
|
||||
.unwrap();
|
||||
let by_addr = file.dataset_at(addr).unwrap();
|
||||
assert_eq!(by_addr.read_f64().unwrap(), vec![1.0, 2.5, -3.0]);
|
||||
assert_eq!(
|
||||
by_addr.shape().unwrap(),
|
||||
file.dataset("grp/vals").unwrap().shape().unwrap()
|
||||
);
|
||||
|
||||
// The group's own header is not a dataset.
|
||||
let group_addr =
|
||||
clawhdf5_format::group_v2::resolve_path_any(file.as_bytes(), file.superblock(), "grp")
|
||||
.unwrap();
|
||||
assert!(matches!(
|
||||
file.dataset_at(group_addr),
|
||||
Err(clawhdf5::Error::NotADataset(_))
|
||||
));
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user