read: look names up through the dense name indexes
Finding one link or attribute by name read every entry: Group::dataset and Group::group (File, MmapFile, LazyFile) listed the whole group per call, and path resolution scanned each group's links. Opening every child of a 35 001-link group by name decoded ~1.2e9 links. Now a dense group's v2 B-tree name index (type 5, lookup3 hash of the name) is descended to the records with the name's hash (btree_v2::find_btree_v2_records reads only the nodes whose key interval overlaps), and only those links are read and compared; all hash-equal records are compared, so libhdf5's tie order does not matter. Dense attributes the same through their type 8 index (attribute::find_attribute_in_file, facade attr(name)); huge heap objects through their ID-ordered index. group_v2::resolve_child returns what the listing has under a name (soft links followed, dangling/external ones not found). Group::entries and File::group_at hand out a listing's addresses. The lookup-stats feature counts heap objects read. Tests: one lookup in an h5py-written 35 001-link group with colliding hashes reads at most two links (before: 35 001, failing), attribute lookups likewise (before: 3 000, failing), every child opens through all three readers and matches h5py, every link kind resolves as h5py resolves it in dense and compact groups, 300 huge attributes are found, and a range search matches a full scan at every tree depth. Co-Authored-By: Claude Opus 5.5 (1M context) <[email protected]>
This commit is contained in:
osobh
@@ -19,7 +19,8 @@ rayon = { version = "1", optional = true }
|
||||
tempfile = { workspace = true }
|
||||
criterion = { workspace = true }
|
||||
clawhdf5-io = { path = "../clawhdf5-io", version = "2.7.0", features = ["mmap"] }
|
||||
clawhdf5-format = { path = "../clawhdf5-format", version = "2.7.0", features = ["parallel", "fast-checksum"] }
|
||||
clawhdf5-format = { path = "../clawhdf5-format", version = "2.7.0", features = ["parallel", "fast-checksum", "lookup-stats"] }
|
||||
serde_json = "1"
|
||||
clawhdf5-filters = { path = "../clawhdf5-filters", version = "2.7.0" }
|
||||
|
||||
[[bench]]
|
||||
|
||||
+44
-13
@@ -28,7 +28,7 @@ use clawhdf5_format::superblock::Superblock;
|
||||
use clawhdf5_io::HDF5Read;
|
||||
|
||||
use crate::error::Error;
|
||||
use crate::types::{AttrValue, DType, classify_datatype, read_attrs};
|
||||
use crate::types::{AttrValue, DType, classify_datatype, read_attr, read_attrs};
|
||||
|
||||
/// A lazy HDF5 file handle that parses metadata on demand.
|
||||
///
|
||||
@@ -304,12 +304,7 @@ impl<'f, R: HDF5Read> LazyGroup<'f, R> {
|
||||
|
||||
/// Get a dataset within this group by name.
|
||||
pub fn dataset(&self, name: &str) -> Result<LazyDataset<'f, R>, Error> {
|
||||
let entries = self.children()?;
|
||||
let entry = entries
|
||||
.iter()
|
||||
.find(|e| e.name == name)
|
||||
.ok_or_else(|| Error::Format(FormatError::PathNotFound(name.to_string())))?;
|
||||
let hdr = self.file.get_or_parse_header(entry.object_header_address)?;
|
||||
let hdr = self.file.get_or_parse_header(self.child_address(name)?)?;
|
||||
if !has_message(&hdr, MessageType::DataLayout) {
|
||||
return Err(Error::NotADataset(name.to_string()));
|
||||
}
|
||||
@@ -322,17 +317,38 @@ impl<'f, R: HDF5Read> LazyGroup<'f, R> {
|
||||
|
||||
/// Get a subgroup within this group by name.
|
||||
pub fn group(&self, name: &str) -> Result<LazyGroup<'f, R>, Error> {
|
||||
let entries = self.children()?;
|
||||
let entry = entries
|
||||
.iter()
|
||||
.find(|e| e.name == name)
|
||||
.ok_or_else(|| Error::Format(FormatError::PathNotFound(name.to_string())))?;
|
||||
Ok(LazyGroup {
|
||||
file: self.file,
|
||||
address: entry.object_header_address,
|
||||
address: self.child_address(name)?,
|
||||
})
|
||||
}
|
||||
|
||||
/// The attribute called `name`, or `None` if it has none by that name
|
||||
/// (or it cannot be read) — the value [`attrs`](Self::attrs) has under
|
||||
/// that name, found without reading the other attributes when they are
|
||||
/// stored densely.
|
||||
pub fn attr(&self, name: &str) -> Result<Option<AttrValue>, Error> {
|
||||
let hdr = self.file.get_or_parse_header(self.address)?;
|
||||
let data = self.file.hdf5_bytes();
|
||||
read_attr(
|
||||
data,
|
||||
&hdr,
|
||||
name,
|
||||
self.file.offset_size(),
|
||||
self.file.length_size(),
|
||||
)
|
||||
}
|
||||
|
||||
/// The object header address of the child called `name`: the entry of
|
||||
/// the group's listing with that name, looked up through the group's
|
||||
/// name index rather than by listing the group (see
|
||||
/// [`group_v2::resolve_child`]).
|
||||
fn child_address(&self, name: &str) -> Result<u64, Error> {
|
||||
let data = self.file.hdf5_bytes();
|
||||
group_v2::resolve_child(data, &self.file.superblock, self.address, name)
|
||||
.map_err(Error::Format)
|
||||
}
|
||||
|
||||
/// This group's links that can be opened: hard links, and soft links
|
||||
/// resolved to their targets (see
|
||||
/// [`group_v2::resolve_group_children`]); dangling, external and
|
||||
@@ -555,6 +571,21 @@ impl<'f, R: HDF5Read> LazyDataset<'f, R> {
|
||||
)
|
||||
}
|
||||
|
||||
/// The attribute called `name`, or `None` if it has none by that name
|
||||
/// (or it cannot be read) — the value [`attrs`](Self::attrs) has under
|
||||
/// that name, found without reading the other attributes when they are
|
||||
/// stored densely.
|
||||
pub fn attr(&self, name: &str) -> Result<Option<AttrValue>, Error> {
|
||||
let data = self.file.hdf5_bytes();
|
||||
read_attr(
|
||||
data,
|
||||
&self.header,
|
||||
name,
|
||||
self.file.offset_size(),
|
||||
self.file.length_size(),
|
||||
)
|
||||
}
|
||||
|
||||
/// A header message's payload, resolved through the shared-message
|
||||
/// indirection when needed (e.g. a committed datatype). See
|
||||
/// [`clawhdf5_format::shared_message::message_data`].
|
||||
|
||||
@@ -23,7 +23,7 @@ use clawhdf5_format::superblock::Superblock;
|
||||
use clawhdf5_io::MmapReader;
|
||||
|
||||
use crate::error::Error;
|
||||
use crate::types::{AttrValue, DType, classify_datatype, read_attrs};
|
||||
use crate::types::{AttrValue, DType, classify_datatype, read_attr, read_attrs};
|
||||
|
||||
/// An HDF5 file opened via memory mapping.
|
||||
///
|
||||
@@ -242,12 +242,7 @@ impl<'f> MmapGroup<'f> {
|
||||
|
||||
/// Get a dataset within this group by name.
|
||||
pub fn dataset(&self, name: &str) -> Result<MmapDataset<'f>, Error> {
|
||||
let entries = self.children()?;
|
||||
let entry = entries
|
||||
.iter()
|
||||
.find(|e| e.name == name)
|
||||
.ok_or_else(|| Error::Format(FormatError::PathNotFound(name.to_string())))?;
|
||||
let hdr = self.file.parse_header(entry.object_header_address)?;
|
||||
let hdr = self.file.parse_header(self.child_address(name)?)?;
|
||||
if !has_message(&hdr, MessageType::DataLayout) {
|
||||
return Err(Error::NotADataset(name.to_string()));
|
||||
}
|
||||
@@ -260,17 +255,38 @@ impl<'f> MmapGroup<'f> {
|
||||
|
||||
/// Get a subgroup within this group by name.
|
||||
pub fn group(&self, name: &str) -> Result<MmapGroup<'f>, Error> {
|
||||
let entries = self.children()?;
|
||||
let entry = entries
|
||||
.iter()
|
||||
.find(|e| e.name == name)
|
||||
.ok_or_else(|| Error::Format(FormatError::PathNotFound(name.to_string())))?;
|
||||
Ok(MmapGroup {
|
||||
file: self.file,
|
||||
address: entry.object_header_address,
|
||||
address: self.child_address(name)?,
|
||||
})
|
||||
}
|
||||
|
||||
/// The attribute called `name`, or `None` if it has none by that name
|
||||
/// (or it cannot be read) — the value [`attrs`](Self::attrs) has under
|
||||
/// that name, found without reading the other attributes when they are
|
||||
/// stored densely.
|
||||
pub fn attr(&self, name: &str) -> Result<Option<AttrValue>, Error> {
|
||||
let hdr = self.file.parse_header(self.address)?;
|
||||
let data = self.file.hdf5_bytes();
|
||||
read_attr(
|
||||
data,
|
||||
&hdr,
|
||||
name,
|
||||
self.file.offset_size(),
|
||||
self.file.length_size(),
|
||||
)
|
||||
}
|
||||
|
||||
/// The object header address of the child called `name`: the entry of
|
||||
/// the group's listing with that name, looked up through the group's
|
||||
/// name index rather than by listing the group (see
|
||||
/// [`group_v2::resolve_child`]).
|
||||
fn child_address(&self, name: &str) -> Result<u64, Error> {
|
||||
let data = self.file.meta()?;
|
||||
group_v2::resolve_child(data, &self.file.superblock, self.address, name)
|
||||
.map_err(Error::Format)
|
||||
}
|
||||
|
||||
/// This group's links that can be opened: hard links, and soft links
|
||||
/// resolved to their targets (see
|
||||
/// [`group_v2::resolve_group_children`]); dangling, external and
|
||||
@@ -506,6 +522,21 @@ impl<'f> MmapDataset<'f> {
|
||||
)
|
||||
}
|
||||
|
||||
/// The attribute called `name`, or `None` if it has none by that name
|
||||
/// (or it cannot be read) — the value [`attrs`](Self::attrs) has under
|
||||
/// that name, found without reading the other attributes when they are
|
||||
/// stored densely.
|
||||
pub fn attr(&self, name: &str) -> Result<Option<AttrValue>, Error> {
|
||||
let data = self.file.hdf5_bytes();
|
||||
read_attr(
|
||||
data,
|
||||
&self.header,
|
||||
name,
|
||||
self.file.offset_size(),
|
||||
self.file.length_size(),
|
||||
)
|
||||
}
|
||||
|
||||
/// A header message's payload, resolved through the shared-message
|
||||
/// indirection when needed (e.g. a committed datatype). See
|
||||
/// [`clawhdf5_format::shared_message::message_data`].
|
||||
|
||||
@@ -23,7 +23,7 @@ use clawhdf5_format::superblock::Superblock;
|
||||
|
||||
use crate::cache_image::{self, ImageView};
|
||||
use crate::error::Error;
|
||||
use crate::types::{AttrValue, DType, classify_datatype, read_attrs};
|
||||
use crate::types::{AttrValue, DType, classify_datatype, read_attr, read_attrs};
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// FileData — internal storage for either owned bytes or an mmap
|
||||
@@ -230,9 +230,10 @@ impl File {
|
||||
|
||||
/// A `Dataset` handle for the object header at `address` (an address
|
||||
/// from a group listing, or one kept from an earlier lookup), without
|
||||
/// resolving a path. Resolving a path walks every group on it, which in
|
||||
/// a large group costs a scan of its links; keep the address instead to
|
||||
/// open the same dataset repeatedly.
|
||||
/// resolving a path. Resolving a path looks each component up in its
|
||||
/// group (through the name index of a dense group; a v1 group's entries
|
||||
/// are scanned); keep the address instead to open the same dataset
|
||||
/// repeatedly.
|
||||
pub fn dataset_at(&self, address: u64) -> Result<Dataset<'_>, Error> {
|
||||
let hdr = self.parse_header(address)?;
|
||||
if !has_message(&hdr, MessageType::DataLayout) {
|
||||
@@ -245,6 +246,17 @@ impl File {
|
||||
.check_open()
|
||||
}
|
||||
|
||||
/// A `Group` handle for the object header at `address` (from
|
||||
/// [`Group::entries`], or kept from an earlier lookup), without
|
||||
/// resolving a path. Like [`group`](Self::group), the object is not
|
||||
/// checked to be a group; a non-group has no children.
|
||||
pub fn group_at(&self, address: u64) -> Group<'_> {
|
||||
Group {
|
||||
file: self,
|
||||
address,
|
||||
}
|
||||
}
|
||||
|
||||
/// Resolve a path and return a `Group` handle.
|
||||
///
|
||||
/// The path uses `/` separators (e.g., `"sensors"`).
|
||||
@@ -483,12 +495,7 @@ impl<'f> Group<'f> {
|
||||
|
||||
/// Get a dataset within this group by name.
|
||||
pub fn dataset(&self, name: &str) -> Result<Dataset<'f>, Error> {
|
||||
let entries = self.children()?;
|
||||
let entry = entries
|
||||
.iter()
|
||||
.find(|e| e.name == name)
|
||||
.ok_or_else(|| Error::Format(FormatError::PathNotFound(name.to_string())))?;
|
||||
let hdr = self.file.parse_header(entry.object_header_address)?;
|
||||
let hdr = self.file.parse_header(self.child_address(name)?)?;
|
||||
if !has_message(&hdr, MessageType::DataLayout) {
|
||||
return Err(Error::NotADataset(name.to_string()));
|
||||
}
|
||||
@@ -501,17 +508,51 @@ impl<'f> Group<'f> {
|
||||
|
||||
/// Get a subgroup within this group by name.
|
||||
pub fn group(&self, name: &str) -> Result<Group<'f>, Error> {
|
||||
let entries = self.children()?;
|
||||
let entry = entries
|
||||
.iter()
|
||||
.find(|e| e.name == name)
|
||||
.ok_or_else(|| Error::Format(FormatError::PathNotFound(name.to_string())))?;
|
||||
Ok(Group {
|
||||
file: self.file,
|
||||
address: entry.object_header_address,
|
||||
address: self.child_address(name)?,
|
||||
})
|
||||
}
|
||||
|
||||
/// The attribute called `name`, or `None` if it has none by that name
|
||||
/// (or it cannot be read) — the value [`attrs`](Self::attrs) has under
|
||||
/// that name, found without reading the other attributes when they are
|
||||
/// stored densely.
|
||||
pub fn attr(&self, name: &str) -> Result<Option<AttrValue>, Error> {
|
||||
let hdr = self.file.parse_header(self.address)?;
|
||||
let data = self.file.data.as_bytes();
|
||||
read_attr(
|
||||
data,
|
||||
&hdr,
|
||||
name,
|
||||
self.file.offset_size(),
|
||||
self.file.length_size(),
|
||||
)
|
||||
}
|
||||
|
||||
/// The object header address of the child called `name`: the entry of
|
||||
/// the group's listing with that name, looked up through the group's
|
||||
/// name index rather than by listing the group (see
|
||||
/// [`group_v2::resolve_child`]).
|
||||
fn child_address(&self, name: &str) -> Result<u64, Error> {
|
||||
let data = self.file.data.meta()?;
|
||||
group_v2::resolve_child(data, &self.file.superblock, self.address, name)
|
||||
.map_err(Error::Format)
|
||||
}
|
||||
|
||||
/// This group's children that can be opened, as `(name, object header
|
||||
/// address)` in listing order — the entries [`datasets`](Self::datasets)
|
||||
/// and [`groups`](Self::groups) are drawn from. Open one with
|
||||
/// [`File::dataset_at`] or [`File::group_at`] to skip looking its name
|
||||
/// up again, or keep the addresses to revisit the objects.
|
||||
pub fn entries(&self) -> Result<Vec<(String, u64)>, Error> {
|
||||
Ok(self
|
||||
.children()?
|
||||
.into_iter()
|
||||
.map(|e| (e.name, e.object_header_address))
|
||||
.collect())
|
||||
}
|
||||
|
||||
/// This group's links that can be opened: hard links, and soft links
|
||||
/// resolved to their targets (see
|
||||
/// [`group_v2::resolve_group_children`]); dangling, external and
|
||||
@@ -1051,6 +1092,21 @@ impl<'f> Dataset<'f> {
|
||||
)
|
||||
}
|
||||
|
||||
/// The attribute called `name`, or `None` if it has none by that name
|
||||
/// (or it cannot be read) — the value [`attrs`](Self::attrs) has under
|
||||
/// that name, found without reading the other attributes when they are
|
||||
/// stored densely.
|
||||
pub fn attr(&self, name: &str) -> Result<Option<AttrValue>, Error> {
|
||||
let data = self.file.data.as_bytes();
|
||||
read_attr(
|
||||
data,
|
||||
&self.header,
|
||||
name,
|
||||
self.file.offset_size(),
|
||||
self.file.length_size(),
|
||||
)
|
||||
}
|
||||
|
||||
/// Verify this dataset's content against its stored provenance hash
|
||||
/// (`_provenance_sha256`, written automatically on save when a
|
||||
/// [`Provenance`](clawhdf5_format::provenance::Provenance) is set — see
|
||||
|
||||
@@ -182,6 +182,35 @@ pub(crate) fn read_attrs(
|
||||
))
|
||||
}
|
||||
|
||||
/// The attribute called `name` on the object with header `header`, decoded
|
||||
/// as [`read_attrs`] decodes it, or `None` (see
|
||||
/// [`find_attribute_in_file`](clawhdf5_format::attribute::find_attribute_in_file)).
|
||||
pub(crate) fn read_attr(
|
||||
file_data: &[u8],
|
||||
header: &clawhdf5_format::object_header::ObjectHeader,
|
||||
name: &str,
|
||||
offset_size: u8,
|
||||
length_size: u8,
|
||||
) -> Result<Option<AttrValue>, crate::Error> {
|
||||
let Some(msg) = clawhdf5_format::attribute::find_attribute_in_file(
|
||||
file_data,
|
||||
header,
|
||||
name,
|
||||
offset_size,
|
||||
length_size,
|
||||
)?
|
||||
else {
|
||||
return Ok(None);
|
||||
};
|
||||
Ok(attrs_to_map(
|
||||
std::slice::from_ref(&msg),
|
||||
file_data,
|
||||
offset_size,
|
||||
length_size,
|
||||
)
|
||||
.remove(name))
|
||||
}
|
||||
|
||||
pub(crate) fn attrs_to_map(
|
||||
attrs: &[clawhdf5_format::attribute::AttributeMessage],
|
||||
file_data: &[u8],
|
||||
|
||||
@@ -211,6 +211,30 @@ fn dense_attribute_stored_as_a_huge_heap_object() {
|
||||
assert!(matches!(&attrs["bigger"], AttrValue::I64Array(v) if *v == bigger));
|
||||
}
|
||||
|
||||
/// Enough huge attributes that the huge-object B-tree has internal nodes:
|
||||
/// each one is found by descending it by heap ID (libhdf5 orders the
|
||||
/// records of indirectly addressed huge objects by ID), and every value
|
||||
/// matches what was written.
|
||||
#[test]
|
||||
fn many_huge_attributes_are_found_through_their_index() {
|
||||
skip_if_no_python!();
|
||||
let (_dir, path) = h5py_file(
|
||||
"d = f.create_dataset('d', data=[1.0])\n\
|
||||
for i in range(300):\n\
|
||||
\x20 d.attrs['h%03d' % i] = np.arange(600, dtype='i8') + i\n",
|
||||
);
|
||||
let f = File::open(&path).unwrap();
|
||||
let d = f.dataset("d").unwrap();
|
||||
let attrs = d.attrs().unwrap();
|
||||
assert_eq!(attrs.len(), 300);
|
||||
for i in 0..300i64 {
|
||||
let name = format!("h{i:03}");
|
||||
let want: Vec<i64> = (0..600).map(|v| v + i).collect();
|
||||
assert!(matches!(&attrs[&name], AttrValue::I64Array(v) if *v == want), "{name}");
|
||||
assert!(matches!(d.attr(&name).unwrap(), Some(AttrValue::I64Array(v)) if v == want), "{name}");
|
||||
}
|
||||
}
|
||||
|
||||
/// A group whose link heap has a deflate I/O filter (set on the group
|
||||
/// creation property list), with 3 000 links and one link whose message is
|
||||
/// larger than the heap's managed-object limit, so it is a huge object.
|
||||
|
||||
@@ -0,0 +1,410 @@
|
||||
//! Looking one name up in a dense group (links in a fractal heap, indexed by
|
||||
//! a v2 B-tree of name hashes) or in dense attribute storage reads the name
|
||||
//! index, not every link: O(log n) index nodes and only the links whose
|
||||
//! lookup3 hash equals the name's. Before, every lookup decoded all n links,
|
||||
//! so opening each child of a 35 001-link group by name decoded ~1.2e9.
|
||||
//!
|
||||
//! The file is written by h5py (libhdf5 orders the index), with names whose
|
||||
//! hashes collide, and every result is compared with what h5py reads.
|
||||
//!
|
||||
//! Skipped when python3 with h5py is unavailable, unless
|
||||
//! `CLAWHDF5_REQUIRE_INTEROP=1`.
|
||||
|
||||
use std::collections::{BTreeMap, HashMap};
|
||||
use std::process::Command;
|
||||
use std::sync::OnceLock;
|
||||
use std::time::{Duration, Instant};
|
||||
|
||||
use clawhdf5::{AttrValue, File, LazyFile, MmapFile};
|
||||
use clawhdf5_format::checksum::jenkins_lookup3;
|
||||
use clawhdf5_format::error::FormatError;
|
||||
use clawhdf5_format::lookup_stats;
|
||||
|
||||
fn python() -> String {
|
||||
std::env::var("CLAWHDF5_PYTHON").unwrap_or_else(|_| "python3".to_string())
|
||||
}
|
||||
|
||||
fn interop_required() -> bool {
|
||||
std::env::var("CLAWHDF5_REQUIRE_INTEROP").is_ok_and(|v| v == "1")
|
||||
}
|
||||
|
||||
fn python_available() -> bool {
|
||||
Command::new(python())
|
||||
.args(["-c", "import h5py"])
|
||||
.output()
|
||||
.map(|o| o.status.success())
|
||||
.unwrap_or(false)
|
||||
}
|
||||
|
||||
macro_rules! skip_if_no_python {
|
||||
() => {
|
||||
if !python_available() {
|
||||
assert!(
|
||||
!interop_required(),
|
||||
"CLAWHDF5_REQUIRE_INTEROP=1 but python3 with h5py is not available"
|
||||
);
|
||||
eprintln!("SKIP: python3 with h5py not available");
|
||||
return;
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
fn run_python(script: &str) -> String {
|
||||
let output = Command::new(python())
|
||||
.args(["-c", script])
|
||||
.output()
|
||||
.expect("failed to run python");
|
||||
assert!(
|
||||
output.status.success(),
|
||||
"Python script failed:\nSTDOUT: {}\nSTDERR: {}",
|
||||
String::from_utf8_lossy(&output.stdout),
|
||||
String::from_utf8_lossy(&output.stderr)
|
||||
);
|
||||
String::from_utf8_lossy(&output.stdout).trim().to_string()
|
||||
}
|
||||
|
||||
/// Links in the big group, as libhdf5's `h5stat_newgrat.h5` has.
|
||||
const LINKS: usize = 35_001;
|
||||
/// Attributes on the dense-attribute dataset.
|
||||
const ATTRS: usize = 3_000;
|
||||
|
||||
/// Pairs of distinct names with equal lookup3 hashes, found by search (the
|
||||
/// hash is fixed, so the pairs are too).
|
||||
fn colliding_pairs(count: usize) -> Vec<(String, String)> {
|
||||
let mut seen: HashMap<u32, String> = HashMap::new();
|
||||
let mut pairs = Vec::new();
|
||||
for i in 0.. {
|
||||
let name = format!("c{i}");
|
||||
let h = jenkins_lookup3(name.as_bytes());
|
||||
if let Some(first) = seen.insert(h, name.clone()) {
|
||||
pairs.push((first, name));
|
||||
if pairs.len() == count {
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
pairs
|
||||
}
|
||||
|
||||
/// What h5py reads: link values, attribute values, and which of the
|
||||
/// missing names it finds as links and as attributes (none).
|
||||
type H5pyView = (
|
||||
BTreeMap<String, i64>,
|
||||
BTreeMap<String, i64>,
|
||||
Vec<String>,
|
||||
Vec<String>,
|
||||
);
|
||||
|
||||
struct Fixture {
|
||||
_dir: tempfile::TempDir,
|
||||
path: String,
|
||||
/// Names in the big group, with the value of the scalar dataset each
|
||||
/// links to, as h5py reads them.
|
||||
links: BTreeMap<String, i64>,
|
||||
/// Names that are not links but hash like one that is.
|
||||
missing_links: Vec<String>,
|
||||
/// Attributes of `/x`, as h5py reads them.
|
||||
attrs: BTreeMap<String, i64>,
|
||||
missing_attrs: Vec<String>,
|
||||
}
|
||||
|
||||
fn fixture() -> &'static Fixture {
|
||||
static FIXTURE: OnceLock<Fixture> = OnceLock::new();
|
||||
FIXTURE.get_or_init(|| {
|
||||
let pairs = colliding_pairs(6);
|
||||
for (a, b) in &pairs {
|
||||
assert_ne!(a, b);
|
||||
assert_eq!(jenkins_lookup3(a.as_bytes()), jenkins_lookup3(b.as_bytes()));
|
||||
}
|
||||
// Pairs 0-2 both present (either can be the one libhdf5 orders
|
||||
// first), pairs 3-5 only the first: its partner must not be found.
|
||||
// "k69209"/"k155448" is the pair the writer once misordered.
|
||||
let mut present: Vec<String> = vec!["k69209".into(), "k155448".into()];
|
||||
let mut missing: Vec<String> = Vec::new();
|
||||
for (i, (a, b)) in pairs.into_iter().enumerate() {
|
||||
present.push(a);
|
||||
if i < 3 {
|
||||
present.push(b);
|
||||
} else {
|
||||
missing.push(b);
|
||||
}
|
||||
}
|
||||
missing.extend(["", "nope", "n35001x", "N1"].map(String::from));
|
||||
let mut links = present.clone();
|
||||
let mut i = 0;
|
||||
while links.len() < LINKS {
|
||||
links.push(format!("n{i}"));
|
||||
i += 1;
|
||||
}
|
||||
let mut attrs = present.clone();
|
||||
attrs.extend((0..ATTRS - present.len()).map(|i| format!("a{i}")));
|
||||
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let path = dir.path().join("big.h5").display().to_string();
|
||||
// The names go through a file: 35 001 of them overflow an argument.
|
||||
let names = dir.path().join("names.json");
|
||||
std::fs::write(
|
||||
&names,
|
||||
serde_json::to_string(&(&links, &attrs, &missing)).unwrap(),
|
||||
)
|
||||
.unwrap();
|
||||
let names = names.display();
|
||||
let out = run_python(&format!(
|
||||
"import h5py, json, numpy as np\n\
|
||||
links, attrs, missing = json.load(open(r'{names}'))\n\
|
||||
with h5py.File(r'{path}', 'w', libver='latest') as f:\n\
|
||||
\x20 g = f.create_group('g')\n\
|
||||
\x20 for i, n in enumerate(links):\n\
|
||||
\x20 g.create_dataset(n, data=np.int64(i))\n\
|
||||
\x20 x = f.create_dataset('x', data=np.int64(0))\n\
|
||||
\x20 for i, n in enumerate(attrs):\n\
|
||||
\x20 x.attrs[n] = np.int64(1000 + i)\n\
|
||||
with h5py.File(r'{path}', 'r') as f:\n\
|
||||
\x20 g, a = f['g'], f['x'].attrs\n\
|
||||
\x20 print(json.dumps([{{n: int(g[n][()]) for n in g}}, {{n: int(a[n]) for n in a}},\n\
|
||||
\x20 [n for n in missing if n and n in g], [n for n in missing if n and n in a]]))",
|
||||
));
|
||||
let (links, attrs, found_links, found_attrs): H5pyView =
|
||||
serde_json::from_str(&out).unwrap();
|
||||
assert_eq!(links.len(), LINKS);
|
||||
assert_eq!(attrs.len(), ATTRS);
|
||||
assert!(found_links.is_empty() && found_attrs.is_empty());
|
||||
Fixture {
|
||||
_dir: dir,
|
||||
path,
|
||||
links,
|
||||
missing_links: missing.clone(),
|
||||
attrs,
|
||||
missing_attrs: missing,
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
fn is_not_found(e: &clawhdf5::Error) -> bool {
|
||||
matches!(e, clawhdf5::Error::Format(FormatError::PathNotFound(_)))
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn one_link_lookup_reads_the_index_not_every_link() {
|
||||
skip_if_no_python!();
|
||||
let fx = fixture();
|
||||
let f = File::open(&fx.path).unwrap();
|
||||
let g = f.group("g").unwrap();
|
||||
for (name, value) in &fx.links {
|
||||
lookup_stats::reset();
|
||||
let ds = g.dataset(name).unwrap();
|
||||
// One link decoded per lookup, two where hashes collide — not 35 001.
|
||||
let read = lookup_stats::heap_objects_read();
|
||||
assert!(read <= 2, "looking up {name} read {read} heap objects");
|
||||
assert_eq!(ds.read_i64().unwrap(), vec![*value], "{name}");
|
||||
}
|
||||
|
||||
for name in &fx.missing_links {
|
||||
lookup_stats::reset();
|
||||
let err = g.dataset(name).unwrap_err();
|
||||
assert!(is_not_found(&err), "{name:?}: {err:?}");
|
||||
assert!(lookup_stats::heap_objects_read() <= 2, "{name:?}");
|
||||
}
|
||||
|
||||
// A path resolves each component the same way.
|
||||
for name in ["k155448", "n0", "n34000"] {
|
||||
lookup_stats::reset();
|
||||
let ds = f.dataset(&format!("/g/{name}")).unwrap();
|
||||
assert!(lookup_stats::heap_objects_read() <= 2);
|
||||
assert_eq!(ds.read_i64().unwrap(), vec![fx.links[name]]);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn one_attribute_lookup_reads_the_index_not_every_attribute() {
|
||||
skip_if_no_python!();
|
||||
let fx = fixture();
|
||||
let f = File::open(&fx.path).unwrap();
|
||||
let x = f.dataset("x").unwrap();
|
||||
let all = x.attrs().unwrap();
|
||||
assert_eq!(all.len(), ATTRS);
|
||||
for (name, value) in &fx.attrs {
|
||||
lookup_stats::reset();
|
||||
let got = x.attr(name).unwrap();
|
||||
assert!(lookup_stats::heap_objects_read() <= 2, "{name}");
|
||||
assert!(
|
||||
matches!(got, Some(AttrValue::I64(v)) if v == *value),
|
||||
"{name}: {got:?}"
|
||||
);
|
||||
assert!(matches!(all.get(name), Some(AttrValue::I64(v)) if v == value));
|
||||
}
|
||||
for name in &fx.missing_attrs {
|
||||
lookup_stats::reset();
|
||||
assert!(x.attr(name).unwrap().is_none(), "{name:?}");
|
||||
assert!(lookup_stats::heap_objects_read() <= 2, "{name:?}");
|
||||
}
|
||||
// Compact attributes (on the root group: none) and a group's attributes.
|
||||
assert!(f.root().attr("k69209").unwrap().is_none());
|
||||
}
|
||||
|
||||
/// Every child of the big group opened by name through each file type,
|
||||
/// within `limit`: with a scan per lookup this is ~1.2e9 link decodes.
|
||||
#[test]
|
||||
fn opening_every_child_of_a_35001_link_group_by_name_is_quick() {
|
||||
skip_if_no_python!();
|
||||
let fx = fixture();
|
||||
let limit = Duration::from_secs(120);
|
||||
let started = Instant::now();
|
||||
let check_time = |n: usize| {
|
||||
assert!(
|
||||
started.elapsed() < limit,
|
||||
"{n} lookups took {:?}",
|
||||
started.elapsed()
|
||||
);
|
||||
};
|
||||
|
||||
let f = File::open(&fx.path).unwrap();
|
||||
let g = f.group("g").unwrap();
|
||||
for (n, (name, value)) in fx.links.iter().enumerate() {
|
||||
assert_eq!(g.dataset(name).unwrap().read_i64().unwrap(), vec![*value]);
|
||||
check_time(n);
|
||||
}
|
||||
// The listing hands out entries: open each by address.
|
||||
let entries = g.entries().unwrap();
|
||||
assert_eq!(entries.len(), LINKS);
|
||||
for (name, address) in &entries {
|
||||
let ds = f.dataset_at(*address).unwrap();
|
||||
assert_eq!(ds.read_i64().unwrap(), vec![fx.links[name]]);
|
||||
}
|
||||
assert!(f.group_at(g_address(&f)).dataset("n0").is_ok());
|
||||
|
||||
let m = MmapFile::open(&fx.path).unwrap();
|
||||
let mg = m.group("g").unwrap();
|
||||
for (n, (name, value)) in fx.links.iter().enumerate() {
|
||||
assert_eq!(mg.dataset(name).unwrap().read_i64().unwrap(), vec![*value]);
|
||||
check_time(n);
|
||||
}
|
||||
assert!(mg.group("nope").is_err_and(|e| is_not_found(&e)));
|
||||
|
||||
let l = LazyFile::open_mmap(&fx.path).unwrap();
|
||||
let lg = l.group("g").unwrap();
|
||||
for (n, (name, value)) in fx.links.iter().enumerate() {
|
||||
assert_eq!(lg.dataset(name).unwrap().read_i64().unwrap(), vec![*value]);
|
||||
check_time(n);
|
||||
}
|
||||
let lx = l.dataset("x").unwrap();
|
||||
assert!(
|
||||
matches!(lx.attr("k155448").unwrap(), Some(AttrValue::I64(v)) if v == fx.attrs["k155448"])
|
||||
);
|
||||
assert!(
|
||||
lg.dataset(&fx.missing_links[0])
|
||||
.is_err_and(|e| is_not_found(&e))
|
||||
);
|
||||
}
|
||||
|
||||
fn g_address(f: &File) -> u64 {
|
||||
f.root()
|
||||
.entries()
|
||||
.unwrap()
|
||||
.into_iter()
|
||||
.find(|(n, _)| n == "g")
|
||||
.unwrap()
|
||||
.1
|
||||
}
|
||||
|
||||
/// Every kind of link, looked up by name in a dense group (through the name
|
||||
/// index) and in a compact one, opens what h5py opens and nothing it cannot:
|
||||
/// hard links, soft links (absolute, relative, to a group), and not a
|
||||
/// dangling soft link, an external link or a missing name.
|
||||
#[test]
|
||||
fn links_of_every_kind_resolve_by_name_as_in_h5py() {
|
||||
skip_if_no_python!();
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let path = dir.path().join("links.h5").display().to_string();
|
||||
// For each group and name: "dataset <value>", "group", or "none" as
|
||||
// h5py sees it.
|
||||
let out = run_python(&format!(
|
||||
"import h5py, json, numpy as np\n\
|
||||
with h5py.File(r'{path}', 'w', libver='latest') as f:\n\
|
||||
\x20 for gname, n in (('dense', 20), ('compact', 2)):\n\
|
||||
\x20 g = f.create_group(gname)\n\
|
||||
\x20 for i in range(n):\n\
|
||||
\x20 g.create_dataset(f'd{{i}}', data=np.int64(100 + i))\n\
|
||||
\x20 s = g.create_group('sub')\n\
|
||||
\x20 s.create_dataset('x', data=np.int64(7))\n\
|
||||
\x20 g['abs'] = h5py.SoftLink(f'/{{gname}}/d1')\n\
|
||||
\x20 g['rel'] = h5py.SoftLink('sub/x')\n\
|
||||
\x20 g['tosub'] = h5py.SoftLink('sub')\n\
|
||||
\x20 g['dangling'] = h5py.SoftLink('/nowhere')\n\
|
||||
\x20 g['ext'] = h5py.ExternalLink('other.h5', '/y')\n\
|
||||
names = ['d0', 'd1', 'sub', 'abs', 'rel', 'tosub', 'dangling', 'ext', 'nope', '']\n\
|
||||
seen = {{}}\n\
|
||||
with h5py.File(r'{path}', 'r') as f:\n\
|
||||
\x20 for gname in ('dense', 'compact'):\n\
|
||||
\x20 g = f[gname]\n\
|
||||
\x20 for n in names:\n\
|
||||
\x20 try:\n\
|
||||
\x20 o = g[n] if n else None\n\
|
||||
\x20 except (KeyError, OSError):\n\
|
||||
\x20 o = None\n\
|
||||
\x20 if isinstance(o, h5py.Dataset):\n\
|
||||
\x20 seen[f'{{gname}}/{{n}}'] = f'dataset {{int(o[()])}}'\n\
|
||||
\x20 elif isinstance(o, h5py.Group):\n\
|
||||
\x20 seen[f'{{gname}}/{{n}}'] = 'group'\n\
|
||||
\x20 else:\n\
|
||||
\x20 seen[f'{{gname}}/{{n}}'] = 'none'\n\
|
||||
print(json.dumps(seen))",
|
||||
));
|
||||
let seen: BTreeMap<String, String> = serde_json::from_str(&out).unwrap();
|
||||
assert_eq!(seen.len(), 20);
|
||||
|
||||
let f = File::open(&path).unwrap();
|
||||
// The dense group's links are in a heap, the compact group's in its
|
||||
// header.
|
||||
for (gname, dense) in [("dense", true), ("compact", false)] {
|
||||
let g = f.group(gname).unwrap();
|
||||
lookup_stats::reset();
|
||||
g.dataset("d0").unwrap();
|
||||
assert_eq!(lookup_stats::heap_objects_read() > 0, dense, "{gname}");
|
||||
}
|
||||
let m = MmapFile::open(&path).unwrap();
|
||||
let l = LazyFile::open_mmap(&path).unwrap();
|
||||
for (key, want) in &seen {
|
||||
let (gname, name) = key.split_once('/').unwrap();
|
||||
let got = {
|
||||
let g = f.group(gname).unwrap();
|
||||
match (g.dataset(name), g.group(name)) {
|
||||
(Ok(ds), _) => format!("dataset {}", ds.read_i64().unwrap()[0]),
|
||||
(Err(clawhdf5::Error::NotADataset(_)), Ok(sub)) => {
|
||||
// A group: it has the child `x` (checks the address).
|
||||
assert!(sub.dataset("x").is_ok() || name == "sub" || name == "tosub");
|
||||
"group".to_string()
|
||||
}
|
||||
(Err(e), Err(e2)) => {
|
||||
assert!(is_not_found(&e) && is_not_found(&e2), "{key}: {e:?} / {e2:?}");
|
||||
"none".to_string()
|
||||
}
|
||||
(Err(e), Ok(_)) => panic!("{key}: dataset {e:?} but group ok"),
|
||||
}
|
||||
};
|
||||
assert_eq!(&got, want, "{key}");
|
||||
// The other readers agree, and a path through the group resolves the
|
||||
// same way.
|
||||
let mg = m.group(gname).unwrap();
|
||||
let lg = l.group(gname).unwrap();
|
||||
match want.strip_prefix("dataset ") {
|
||||
Some(v) => {
|
||||
let v: i64 = v.parse().unwrap();
|
||||
assert_eq!(mg.dataset(name).unwrap().read_i64().unwrap(), vec![v]);
|
||||
assert_eq!(lg.dataset(name).unwrap().read_i64().unwrap(), vec![v]);
|
||||
let ds = f.dataset(&format!("/{gname}/{name}")).unwrap();
|
||||
assert_eq!(ds.read_i64().unwrap(), vec![v], "{key}");
|
||||
}
|
||||
None if want == "group" => {
|
||||
assert!(mg.group(name).unwrap().dataset("x").is_ok(), "{key}");
|
||||
assert!(lg.group(name).unwrap().dataset("x").is_ok(), "{key}");
|
||||
let ds = f.dataset(&format!("/{gname}/{name}/x")).unwrap();
|
||||
assert_eq!(ds.read_i64().unwrap(), vec![7]);
|
||||
}
|
||||
None => {
|
||||
assert!(mg.dataset(name).is_err_and(|e| is_not_found(&e)), "{key}");
|
||||
assert!(lg.group(name).is_err_and(|e| is_not_found(&e)), "{key}");
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user