read: look names up through the dense name indexes
Finding one link or attribute by name read every entry: Group::dataset and Group::group (File, MmapFile, LazyFile) listed the whole group per call, and path resolution scanned each group's links. Opening every child of a 35 001-link group by name decoded ~1.2e9 links. Now a dense group's v2 B-tree name index (type 5, lookup3 hash of the name) is descended to the records with the name's hash (btree_v2::find_btree_v2_records reads only the nodes whose key interval overlaps), and only those links are read and compared; all hash-equal records are compared, so libhdf5's tie order does not matter. Dense attributes the same through their type 8 index (attribute::find_attribute_in_file, facade attr(name)); huge heap objects through their ID-ordered index. group_v2::resolve_child returns what the listing has under a name (soft links followed, dangling/external ones not found). Group::entries and File::group_at hand out a listing's addresses. The lookup-stats feature counts heap objects read. Tests: one lookup in an h5py-written 35 001-link group with colliding hashes reads at most two links (before: 35 001, failing), attribute lookups likewise (before: 3 000, failing), every child opens through all three readers and matches h5py, every link kind resolves as h5py resolves it in dense and compact groups, 300 huge attributes are found, and a range search matches a full scan at every tree depth. Co-Authored-By: Claude Opus 5.5 (1M context) <[email protected]>
This commit is contained in:
@@ -80,6 +80,9 @@ blosc = ["lz4_flex", "ruzstd", "snap", "deflate", "std"]
|
|||||||
blosc2 = ["blosc"]
|
blosc2 = ["blosc"]
|
||||||
# Every plugin filter above.
|
# Every plugin filter above.
|
||||||
plugin-filters = ["lzf", "bitshuffle", "bzip2", "blosc", "blosc2"]
|
plugin-filters = ["lzf", "bitshuffle", "bzip2", "blosc", "blosc2"]
|
||||||
|
# Test instrumentation: per-thread counts of heap objects read (see
|
||||||
|
# `lookup_stats`), so tests can bound the cost of a name lookup.
|
||||||
|
lookup-stats = ["std"]
|
||||||
|
|
||||||
[[bench]]
|
[[bench]]
|
||||||
name = "parallel_decompress_bench"
|
name = "parallel_decompress_bench"
|
||||||
|
|||||||
@@ -5,8 +5,10 @@ use alloc::{borrow::Cow, string::String, vec::Vec};
|
|||||||
#[cfg(feature = "std")]
|
#[cfg(feature = "std")]
|
||||||
use std::borrow::Cow;
|
use std::borrow::Cow;
|
||||||
|
|
||||||
|
use crate::addr::to_usize;
|
||||||
use crate::attribute_info::AttributeInfoMessage;
|
use crate::attribute_info::AttributeInfoMessage;
|
||||||
use crate::btree_v2::{BTreeV2Header, collect_btree_v2_records};
|
use crate::btree_v2::{BTreeV2Header, collect_btree_v2_records, find_btree_v2_records};
|
||||||
|
use crate::checksum::jenkins_lookup3;
|
||||||
use crate::data_read;
|
use crate::data_read;
|
||||||
use crate::dataspace::Dataspace;
|
use crate::dataspace::Dataspace;
|
||||||
use crate::datatype::Datatype;
|
use crate::datatype::Datatype;
|
||||||
@@ -341,7 +343,8 @@ fn compute_raw_data(
|
|||||||
dataspace: &Dataspace,
|
dataspace: &Dataspace,
|
||||||
datatype: &Datatype,
|
datatype: &Datatype,
|
||||||
) -> Vec<u8> {
|
) -> Vec<u8> {
|
||||||
let num_elements = dataspace.num_elements() as usize;
|
// Saturating, like the product: the size is capped at what is there.
|
||||||
|
let num_elements = usize::try_from(dataspace.num_elements()).unwrap_or(usize::MAX);
|
||||||
let elem_size = datatype.type_size() as usize;
|
let elem_size = datatype.type_size() as usize;
|
||||||
let expected_size = num_elements.saturating_mul(elem_size);
|
let expected_size = num_elements.saturating_mul(elem_size);
|
||||||
let available = data.len().saturating_sub(pos);
|
let available = data.len().saturating_sub(pos);
|
||||||
@@ -453,7 +456,145 @@ fn extract_attributes_with(
|
|||||||
// Each attribute's creation order, where the file records one.
|
// Each attribute's creation order, where the file records one.
|
||||||
let mut orders: Vec<u32> = Vec::new();
|
let mut orders: Vec<u32> = Vec::new();
|
||||||
|
|
||||||
// Collect compact attributes (inline in OH)
|
extract_compact_attributes(
|
||||||
|
file_data,
|
||||||
|
header,
|
||||||
|
offset_size,
|
||||||
|
length_size,
|
||||||
|
&mut attrs,
|
||||||
|
&mut orders,
|
||||||
|
on_error,
|
||||||
|
)?;
|
||||||
|
|
||||||
|
// Check for dense attributes via AttributeInfo message
|
||||||
|
let attr_info = find_attribute_info(header, offset_size)?;
|
||||||
|
if let Some(info) = &attr_info
|
||||||
|
&& let Some(fh_addr) = info.fractal_heap_address
|
||||||
|
{
|
||||||
|
extract_dense_attributes(
|
||||||
|
file_data,
|
||||||
|
info,
|
||||||
|
fh_addr,
|
||||||
|
offset_size,
|
||||||
|
length_size,
|
||||||
|
&mut attrs,
|
||||||
|
&mut orders,
|
||||||
|
on_error,
|
||||||
|
)?;
|
||||||
|
}
|
||||||
|
|
||||||
|
// An object that tracks attribute creation order lists its attributes
|
||||||
|
// in that order (h5py's `track_order=True`), as libhdf5 does; otherwise
|
||||||
|
// they come in storage order.
|
||||||
|
if attr_info.is_some_and(|i| i.max_creation_index.is_some()) {
|
||||||
|
let mut paired: Vec<(u32, AttributeMessage)> = orders.into_iter().zip(attrs).collect();
|
||||||
|
paired.sort_by_key(|(o, _)| *o);
|
||||||
|
attrs = paired.into_iter().map(|(_, a)| a).collect();
|
||||||
|
}
|
||||||
|
|
||||||
|
Ok(attrs)
|
||||||
|
}
|
||||||
|
|
||||||
|
/// B-tree v2 record type of dense attribute storage's name index.
|
||||||
|
const ATTRIBUTE_NAME_INDEX: u8 = 8;
|
||||||
|
|
||||||
|
/// The attribute called `name` on the object with header `header`: the
|
||||||
|
/// first one [`extract_attributes_tolerant`] returns under that name, or
|
||||||
|
/// `None` if it returns none (an attribute that cannot be read is not
|
||||||
|
/// returned there either).
|
||||||
|
///
|
||||||
|
/// Compact attributes are in the header and are scanned. Dense attributes
|
||||||
|
/// are found through the name index (a v2 B-tree of lookup3 name hashes,
|
||||||
|
/// record type 8): only the attributes whose names hash like `name` are read
|
||||||
|
/// from the heap, O(log n) instead of all of them. Errors in the structures
|
||||||
|
/// that index the attributes fail the call, as they fail a listing.
|
||||||
|
pub fn find_attribute_in_file(
|
||||||
|
file_data: &[u8],
|
||||||
|
header: &ObjectHeader,
|
||||||
|
name: &str,
|
||||||
|
offset_size: u8,
|
||||||
|
length_size: u8,
|
||||||
|
) -> Result<Option<AttributeMessage>, FormatError> {
|
||||||
|
let attr_info = find_attribute_info(header, offset_size)?;
|
||||||
|
let dense = attr_info
|
||||||
|
.as_ref()
|
||||||
|
.and_then(|i| Some((i.fractal_heap_address?, i.btree_name_index_address?)));
|
||||||
|
let Some((fh_addr, btree_addr)) = dense else {
|
||||||
|
// Compact only (or dense storage without a name index, which a
|
||||||
|
// listing reports): as a listing finds it.
|
||||||
|
return Ok(
|
||||||
|
extract_attributes_tolerant(file_data, header, offset_size, length_size)?
|
||||||
|
.0
|
||||||
|
.into_iter()
|
||||||
|
.find(|a| a.name == name),
|
||||||
|
);
|
||||||
|
};
|
||||||
|
let btree_hdr =
|
||||||
|
BTreeV2Header::parse(file_data, to_usize(btree_addr)?, offset_size, length_size)?;
|
||||||
|
let fh = FractalHeapHeader::parse(file_data, to_usize(fh_addr)?, offset_size, length_size)?;
|
||||||
|
if btree_hdr.tree_type != ATTRIBUTE_NAME_INDEX || btree_hdr.record_size < 4 {
|
||||||
|
return Ok(
|
||||||
|
extract_attributes_tolerant(file_data, header, offset_size, length_size)?
|
||||||
|
.0
|
||||||
|
.into_iter()
|
||||||
|
.find(|a| a.name == name),
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
// A listing has the compact attributes first.
|
||||||
|
let mut compact = Vec::new();
|
||||||
|
extract_compact_attributes(
|
||||||
|
file_data,
|
||||||
|
header,
|
||||||
|
offset_size,
|
||||||
|
length_size,
|
||||||
|
&mut compact,
|
||||||
|
&mut Vec::new(),
|
||||||
|
&mut |_| Ok(()),
|
||||||
|
)?;
|
||||||
|
if let Some(a) = compact.into_iter().find(|a| a.name == name) {
|
||||||
|
return Ok(Some(a));
|
||||||
|
}
|
||||||
|
|
||||||
|
// Record: heap ID + message flags(1) + creation order(4) + hash(4); the
|
||||||
|
// hash is the last field.
|
||||||
|
let hash = jenkins_lookup3(name.as_bytes());
|
||||||
|
let hash_at = usize::from(btree_hdr.record_size) - 4;
|
||||||
|
let records = find_btree_v2_records(file_data, &btree_hdr, offset_size, &mut |r| match r
|
||||||
|
.get(hash_at..hash_at + 4)
|
||||||
|
{
|
||||||
|
Some(h) => u32::from_le_bytes([h[0], h[1], h[2], h[3]]).cmp(&hash),
|
||||||
|
None => core::cmp::Ordering::Less,
|
||||||
|
})?;
|
||||||
|
let id_len = usize::from(fh.heap_id_length);
|
||||||
|
for record in &records {
|
||||||
|
let Some(id_bytes) = record.data.get(..id_len) else {
|
||||||
|
continue;
|
||||||
|
};
|
||||||
|
let attr = fh
|
||||||
|
.read_managed_object(file_data, id_bytes, offset_size)
|
||||||
|
.and_then(|d| AttributeMessage::parse_in_file(&d, file_data, offset_size, length_size));
|
||||||
|
// One that cannot be read is left out, as from a listing.
|
||||||
|
if let Ok(attr) = attr
|
||||||
|
&& attr.name == name
|
||||||
|
{
|
||||||
|
return Ok(Some(attr));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
Ok(None)
|
||||||
|
}
|
||||||
|
|
||||||
|
/// The attributes stored in the object header itself (compact storage), and
|
||||||
|
/// each one's creation order into `orders`.
|
||||||
|
fn extract_compact_attributes(
|
||||||
|
file_data: &[u8],
|
||||||
|
header: &ObjectHeader,
|
||||||
|
offset_size: u8,
|
||||||
|
length_size: u8,
|
||||||
|
attrs: &mut Vec<AttributeMessage>,
|
||||||
|
orders: &mut Vec<u32>,
|
||||||
|
on_error: &mut dyn FnMut(FormatError) -> Result<(), FormatError>,
|
||||||
|
) -> Result<(), FormatError> {
|
||||||
for msg in &header.messages {
|
for msg in &header.messages {
|
||||||
if msg.msg_type == MessageType::Attribute {
|
if msg.msg_type == MessageType::Attribute {
|
||||||
let attr = if shared_message::is_shared(msg.flags) {
|
let attr = if shared_message::is_shared(msg.flags) {
|
||||||
@@ -489,34 +630,7 @@ fn extract_attributes_with(
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
Ok(())
|
||||||
// Check for dense attributes via AttributeInfo message
|
|
||||||
let attr_info = find_attribute_info(header, offset_size)?;
|
|
||||||
if let Some(info) = &attr_info
|
|
||||||
&& let Some(fh_addr) = info.fractal_heap_address
|
|
||||||
{
|
|
||||||
extract_dense_attributes(
|
|
||||||
file_data,
|
|
||||||
info,
|
|
||||||
fh_addr,
|
|
||||||
offset_size,
|
|
||||||
length_size,
|
|
||||||
&mut attrs,
|
|
||||||
&mut orders,
|
|
||||||
on_error,
|
|
||||||
)?;
|
|
||||||
}
|
|
||||||
|
|
||||||
// An object that tracks attribute creation order lists its attributes
|
|
||||||
// in that order (h5py's `track_order=True`), as libhdf5 does; otherwise
|
|
||||||
// they come in storage order.
|
|
||||||
if attr_info.is_some_and(|i| i.max_creation_index.is_some()) {
|
|
||||||
let mut paired: Vec<(u32, AttributeMessage)> = orders.into_iter().zip(attrs).collect();
|
|
||||||
paired.sort_by_key(|(o, _)| *o);
|
|
||||||
attrs = paired.into_iter().map(|(_, a)| a).collect();
|
|
||||||
}
|
|
||||||
|
|
||||||
Ok(attrs)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Find and parse the Attribute Info message from an object header.
|
/// Find and parse the Attribute Info message from an object header.
|
||||||
@@ -547,7 +661,7 @@ fn extract_dense_attributes(
|
|||||||
on_error: &mut dyn FnMut(FormatError) -> Result<(), FormatError>,
|
on_error: &mut dyn FnMut(FormatError) -> Result<(), FormatError>,
|
||||||
) -> Result<(), FormatError> {
|
) -> Result<(), FormatError> {
|
||||||
// Parse fractal heap
|
// Parse fractal heap
|
||||||
let fh = FractalHeapHeader::parse(file_data, fh_addr as usize, offset_size, length_size)?;
|
let fh = FractalHeapHeader::parse(file_data, to_usize(fh_addr)?, offset_size, length_size)?;
|
||||||
|
|
||||||
// Parse B-tree v2 for name index (type 8)
|
// Parse B-tree v2 for name index (type 8)
|
||||||
let btree_addr = attr_info
|
let btree_addr = attr_info
|
||||||
@@ -556,7 +670,8 @@ fn extract_dense_attributes(
|
|||||||
expected: 1,
|
expected: 1,
|
||||||
available: 0,
|
available: 0,
|
||||||
})?;
|
})?;
|
||||||
let btree_hdr = BTreeV2Header::parse(file_data, btree_addr as usize, offset_size, length_size)?;
|
let btree_hdr =
|
||||||
|
BTreeV2Header::parse(file_data, to_usize(btree_addr)?, offset_size, length_size)?;
|
||||||
let records = collect_btree_v2_records(file_data, &btree_hdr, offset_size, length_size)?;
|
let records = collect_btree_v2_records(file_data, &btree_hdr, offset_size, length_size)?;
|
||||||
|
|
||||||
for record in &records {
|
for record in &records {
|
||||||
|
|||||||
@@ -2,10 +2,12 @@
|
|||||||
|
|
||||||
#[cfg(not(feature = "std"))]
|
#[cfg(not(feature = "std"))]
|
||||||
use alloc::vec::Vec;
|
use alloc::vec::Vec;
|
||||||
|
use core::cmp::Ordering;
|
||||||
|
|
||||||
#[cfg(feature = "checksum")]
|
#[cfg(feature = "checksum")]
|
||||||
use byteorder::{ByteOrder, LittleEndian};
|
use byteorder::{ByteOrder, LittleEndian};
|
||||||
|
|
||||||
|
use crate::addr::to_usize;
|
||||||
use crate::error::FormatError;
|
use crate::error::FormatError;
|
||||||
|
|
||||||
/// Parsed B-tree v2 header (signature "BTHD").
|
/// Parsed B-tree v2 header (signature "BTHD").
|
||||||
@@ -216,7 +218,7 @@ pub fn collect_btree_v2_records(
|
|||||||
// Root is a leaf
|
// Root is a leaf
|
||||||
parse_leaf_records(
|
parse_leaf_records(
|
||||||
file_data,
|
file_data,
|
||||||
header.root_node_address as usize,
|
to_usize(header.root_node_address)?,
|
||||||
header.num_records_in_root,
|
header.num_records_in_root,
|
||||||
header.record_size,
|
header.record_size,
|
||||||
)
|
)
|
||||||
@@ -225,7 +227,7 @@ pub fn collect_btree_v2_records(
|
|||||||
let mut records = Vec::new();
|
let mut records = Vec::new();
|
||||||
collect_internal_records(
|
collect_internal_records(
|
||||||
file_data,
|
file_data,
|
||||||
header.root_node_address as usize,
|
to_usize(header.root_node_address)?,
|
||||||
header.num_records_in_root,
|
header.num_records_in_root,
|
||||||
header.depth,
|
header.depth,
|
||||||
header.record_size,
|
header.record_size,
|
||||||
@@ -289,9 +291,10 @@ fn parse_leaf_records(
|
|||||||
Ok(records)
|
Ok(records)
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Recursively collect records from an internal node.
|
/// An internal node's layout: where its records start, and its children as
|
||||||
#[allow(clippy::too_many_arguments, clippy::only_used_in_recursion)]
|
/// `(address, record count)`.
|
||||||
fn collect_internal_records(
|
#[allow(clippy::too_many_arguments)]
|
||||||
|
fn read_internal_node(
|
||||||
file_data: &[u8],
|
file_data: &[u8],
|
||||||
offset: usize,
|
offset: usize,
|
||||||
num_records: u16,
|
num_records: u16,
|
||||||
@@ -299,11 +302,8 @@ fn collect_internal_records(
|
|||||||
record_size: u16,
|
record_size: u16,
|
||||||
node_size: u32,
|
node_size: u32,
|
||||||
offset_size: u8,
|
offset_size: u8,
|
||||||
length_size: u8,
|
|
||||||
max_leaf_nrec: u64,
|
max_leaf_nrec: u64,
|
||||||
budget: &mut usize,
|
) -> Result<(usize, Vec<(u64, u16)>), FormatError> {
|
||||||
out: &mut Vec<BTreeV2Record>,
|
|
||||||
) -> Result<(), FormatError> {
|
|
||||||
// signature(4) + version(1) + type(1) = 6
|
// signature(4) + version(1) + type(1) = 6
|
||||||
ensure_len(file_data, offset, 6)?;
|
ensure_len(file_data, offset, 6)?;
|
||||||
if &file_data[offset..offset + 4] != b"BTIN" {
|
if &file_data[offset..offset + 4] != b"BTIN" {
|
||||||
@@ -314,7 +314,7 @@ fn collect_internal_records(
|
|||||||
let rs = record_size as usize;
|
let rs = record_size as usize;
|
||||||
let mut pos = offset + 6;
|
let mut pos = offset + 6;
|
||||||
|
|
||||||
// Read all records first
|
// Records first
|
||||||
let records_total = nr.checked_mul(rs).ok_or(FormatError::UnexpectedEof {
|
let records_total = nr.checked_mul(rs).ok_or(FormatError::UnexpectedEof {
|
||||||
expected: usize::MAX,
|
expected: usize::MAX,
|
||||||
available: file_data.len(),
|
available: file_data.len(),
|
||||||
@@ -346,7 +346,6 @@ fn collect_internal_records(
|
|||||||
let child_ptr_size = offset_size as usize + nrec_width + total_nrec_width;
|
let child_ptr_size = offset_size as usize + nrec_width + total_nrec_width;
|
||||||
ensure_len(file_data, pos, num_children * child_ptr_size)?;
|
ensure_len(file_data, pos, num_children * child_ptr_size)?;
|
||||||
|
|
||||||
// Read child pointers
|
|
||||||
let mut children = Vec::with_capacity(num_children);
|
let mut children = Vec::with_capacity(num_children);
|
||||||
for _ in 0..num_children {
|
for _ in 0..num_children {
|
||||||
let addr = read_offset(file_data, pos, offset_size)?;
|
let addr = read_offset(file_data, pos, offset_size)?;
|
||||||
@@ -356,6 +355,61 @@ fn collect_internal_records(
|
|||||||
pos += total_nrec_width; // skip total records in subtree
|
pos += total_nrec_width; // skip total records in subtree
|
||||||
children.push((addr, child_nrec));
|
children.push((addr, child_nrec));
|
||||||
}
|
}
|
||||||
|
Ok((records_start, children))
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Record `i` of an internal node whose records start at `records_start`.
|
||||||
|
fn internal_record(
|
||||||
|
file_data: &[u8],
|
||||||
|
records_start: usize,
|
||||||
|
i: usize,
|
||||||
|
rs: usize,
|
||||||
|
) -> Result<&[u8], FormatError> {
|
||||||
|
let overflow = || FormatError::UnexpectedEof {
|
||||||
|
expected: usize::MAX,
|
||||||
|
available: file_data.len(),
|
||||||
|
};
|
||||||
|
let rec_start = i
|
||||||
|
.checked_mul(rs)
|
||||||
|
.and_then(|o| records_start.checked_add(o))
|
||||||
|
.ok_or_else(overflow)?;
|
||||||
|
let rec_end = rec_start.checked_add(rs).ok_or_else(overflow)?;
|
||||||
|
file_data
|
||||||
|
.get(rec_start..rec_end)
|
||||||
|
.ok_or(FormatError::UnexpectedEof {
|
||||||
|
expected: rec_end,
|
||||||
|
available: file_data.len(),
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Recursively collect records from an internal node.
|
||||||
|
#[allow(clippy::too_many_arguments, clippy::only_used_in_recursion)]
|
||||||
|
fn collect_internal_records(
|
||||||
|
file_data: &[u8],
|
||||||
|
offset: usize,
|
||||||
|
num_records: u16,
|
||||||
|
depth: u16,
|
||||||
|
record_size: u16,
|
||||||
|
node_size: u32,
|
||||||
|
offset_size: u8,
|
||||||
|
length_size: u8,
|
||||||
|
max_leaf_nrec: u64,
|
||||||
|
budget: &mut usize,
|
||||||
|
out: &mut Vec<BTreeV2Record>,
|
||||||
|
) -> Result<(), FormatError> {
|
||||||
|
let nr = num_records as usize;
|
||||||
|
let rs = record_size as usize;
|
||||||
|
let (records_start, children) = read_internal_node(
|
||||||
|
file_data,
|
||||||
|
offset,
|
||||||
|
num_records,
|
||||||
|
depth,
|
||||||
|
record_size,
|
||||||
|
node_size,
|
||||||
|
offset_size,
|
||||||
|
max_leaf_nrec,
|
||||||
|
)?;
|
||||||
|
let child_depth = depth - 1;
|
||||||
|
|
||||||
// Interleave: child[0], record[0], child[1], record[1], ..., child[nr]
|
// Interleave: child[0], record[0], child[1], record[1], ..., child[nr]
|
||||||
// We collect child[0] records, then record[0], then child[1], etc.
|
// We collect child[0] records, then record[0], then child[1], etc.
|
||||||
@@ -364,12 +418,12 @@ fn collect_internal_records(
|
|||||||
// Before parsing, so a refused tree is not also a large allocation.
|
// Before parsing, so a refused tree is not also a large allocation.
|
||||||
spend(budget, usize::from(child_nrec))?;
|
spend(budget, usize::from(child_nrec))?;
|
||||||
let leaf_recs =
|
let leaf_recs =
|
||||||
parse_leaf_records(file_data, child_addr as usize, child_nrec, record_size)?;
|
parse_leaf_records(file_data, to_usize(child_addr)?, child_nrec, record_size)?;
|
||||||
out.extend(leaf_recs);
|
out.extend(leaf_recs);
|
||||||
} else {
|
} else {
|
||||||
collect_internal_records(
|
collect_internal_records(
|
||||||
file_data,
|
file_data,
|
||||||
child_addr as usize,
|
to_usize(child_addr)?,
|
||||||
child_nrec,
|
child_nrec,
|
||||||
child_depth,
|
child_depth,
|
||||||
record_size,
|
record_size,
|
||||||
@@ -384,32 +438,10 @@ fn collect_internal_records(
|
|||||||
|
|
||||||
// Add record[i] (except after the last child)
|
// Add record[i] (except after the last child)
|
||||||
if i < nr {
|
if i < nr {
|
||||||
let rec_offset = i.checked_mul(rs).ok_or(FormatError::UnexpectedEof {
|
let data = internal_record(file_data, records_start, i, rs)?;
|
||||||
expected: usize::MAX,
|
|
||||||
available: file_data.len(),
|
|
||||||
})?;
|
|
||||||
let rec_start =
|
|
||||||
records_start
|
|
||||||
.checked_add(rec_offset)
|
|
||||||
.ok_or(FormatError::UnexpectedEof {
|
|
||||||
expected: usize::MAX,
|
|
||||||
available: file_data.len(),
|
|
||||||
})?;
|
|
||||||
let rec_end = rec_start
|
|
||||||
.checked_add(rs)
|
|
||||||
.ok_or(FormatError::UnexpectedEof {
|
|
||||||
expected: usize::MAX,
|
|
||||||
available: file_data.len(),
|
|
||||||
})?;
|
|
||||||
if rec_end > file_data.len() {
|
|
||||||
return Err(FormatError::UnexpectedEof {
|
|
||||||
expected: rec_end,
|
|
||||||
available: file_data.len(),
|
|
||||||
});
|
|
||||||
}
|
|
||||||
spend(budget, 1)?;
|
spend(budget, 1)?;
|
||||||
out.push(BTreeV2Record {
|
out.push(BTreeV2Record {
|
||||||
data: file_data[rec_start..rec_end].to_vec(),
|
data: data.to_vec(),
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -417,6 +449,116 @@ fn collect_internal_records(
|
|||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// The records of a B-tree v2 that fall in one key range, found by
|
||||||
|
/// descending the tree instead of reading all of it.
|
||||||
|
///
|
||||||
|
/// `cmp` places a record relative to the range: `Less` if the record sorts
|
||||||
|
/// before it, `Greater` if after, `Equal` if the record is in it. The tree
|
||||||
|
/// must be ordered consistently with `cmp`, as libhdf5 orders it (a link or
|
||||||
|
/// attribute name index by name hash, so all records with one hash form a
|
||||||
|
/// range whatever order their names are in). Only the nodes whose key
|
||||||
|
/// interval overlaps the range are read: O(depth) nodes plus those holding
|
||||||
|
/// the matches. Matches come in tree order.
|
||||||
|
pub fn find_btree_v2_records(
|
||||||
|
file_data: &[u8],
|
||||||
|
header: &BTreeV2Header,
|
||||||
|
offset_size: u8,
|
||||||
|
cmp: &mut dyn FnMut(&[u8]) -> Ordering,
|
||||||
|
) -> Result<Vec<BTreeV2Record>, FormatError> {
|
||||||
|
if header.total_records == 0 || header.num_records_in_root == 0 {
|
||||||
|
return Ok(Vec::new());
|
||||||
|
}
|
||||||
|
if header.depth > MAX_DEPTH {
|
||||||
|
return Err(FormatError::NestingDepthExceeded);
|
||||||
|
}
|
||||||
|
// As in `collect_btree_v2_records`: a valid tree cannot hold more
|
||||||
|
// records than the file has room for, however its children are shared.
|
||||||
|
let mut budget = file_data.len() / usize::from(header.record_size.max(1));
|
||||||
|
let max_leaf_nrec = max_records_leaf(header.node_size, header.record_size);
|
||||||
|
let mut out = Vec::new();
|
||||||
|
find_in_node(
|
||||||
|
file_data,
|
||||||
|
header,
|
||||||
|
to_usize(header.root_node_address)?,
|
||||||
|
header.num_records_in_root,
|
||||||
|
header.depth,
|
||||||
|
offset_size,
|
||||||
|
max_leaf_nrec,
|
||||||
|
cmp,
|
||||||
|
&mut budget,
|
||||||
|
&mut out,
|
||||||
|
)?;
|
||||||
|
Ok(out)
|
||||||
|
}
|
||||||
|
|
||||||
|
#[allow(clippy::too_many_arguments)]
|
||||||
|
fn find_in_node(
|
||||||
|
file_data: &[u8],
|
||||||
|
header: &BTreeV2Header,
|
||||||
|
offset: usize,
|
||||||
|
num_records: u16,
|
||||||
|
depth: u16,
|
||||||
|
offset_size: u8,
|
||||||
|
max_leaf_nrec: u64,
|
||||||
|
cmp: &mut dyn FnMut(&[u8]) -> Ordering,
|
||||||
|
budget: &mut usize,
|
||||||
|
out: &mut Vec<BTreeV2Record>,
|
||||||
|
) -> Result<(), FormatError> {
|
||||||
|
spend(budget, usize::from(num_records))?;
|
||||||
|
if depth == 0 {
|
||||||
|
let records = parse_leaf_records(file_data, offset, num_records, header.record_size)?;
|
||||||
|
out.extend(
|
||||||
|
records
|
||||||
|
.into_iter()
|
||||||
|
.filter(|r| cmp(&r.data) == Ordering::Equal),
|
||||||
|
);
|
||||||
|
return Ok(());
|
||||||
|
}
|
||||||
|
let rs = usize::from(header.record_size);
|
||||||
|
let (records_start, children) = read_internal_node(
|
||||||
|
file_data,
|
||||||
|
offset,
|
||||||
|
num_records,
|
||||||
|
depth,
|
||||||
|
header.record_size,
|
||||||
|
header.node_size,
|
||||||
|
offset_size,
|
||||||
|
max_leaf_nrec,
|
||||||
|
)?;
|
||||||
|
let nr = usize::from(num_records);
|
||||||
|
let mut order = Vec::with_capacity(nr);
|
||||||
|
for i in 0..nr {
|
||||||
|
order.push(cmp(internal_record(file_data, records_start, i, rs)?));
|
||||||
|
}
|
||||||
|
// Child `i` holds the keys between record `i - 1` and record `i`: it can
|
||||||
|
// hold a match unless the record before it is already past the range or
|
||||||
|
// the record after it is still before it.
|
||||||
|
for (i, &(child_addr, child_nrec)) in children.iter().enumerate() {
|
||||||
|
let after_left = i == 0 || order[i - 1] != Ordering::Greater;
|
||||||
|
let before_right = i == nr || order[i] != Ordering::Less;
|
||||||
|
if after_left && before_right {
|
||||||
|
find_in_node(
|
||||||
|
file_data,
|
||||||
|
header,
|
||||||
|
to_usize(child_addr)?,
|
||||||
|
child_nrec,
|
||||||
|
depth - 1,
|
||||||
|
offset_size,
|
||||||
|
max_leaf_nrec,
|
||||||
|
cmp,
|
||||||
|
budget,
|
||||||
|
out,
|
||||||
|
)?;
|
||||||
|
}
|
||||||
|
if i < nr && order[i] == Ordering::Equal {
|
||||||
|
out.push(BTreeV2Record {
|
||||||
|
data: internal_record(file_data, records_start, i, rs)?.to_vec(),
|
||||||
|
});
|
||||||
|
}
|
||||||
|
}
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
/// Most records a subtree whose root is at `depth` can hold (libhdf5's
|
/// Most records a subtree whose root is at `depth` can hold (libhdf5's
|
||||||
/// `cum_max_nrec`). See [`node_info`].
|
/// `cum_max_nrec`). See [`node_info`].
|
||||||
fn cum_max_records(
|
fn cum_max_records(
|
||||||
|
|||||||
@@ -385,6 +385,59 @@ mod tests {
|
|||||||
assert!(nodes > 0);
|
assert!(nodes > 0);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Descending to a key range finds exactly the records a full read
|
||||||
|
/// holds in it — runs of equal keys that straddle node boundaries
|
||||||
|
/// included — at every depth, and nothing for keys not in the tree.
|
||||||
|
#[test]
|
||||||
|
fn a_key_range_search_matches_a_full_scan() {
|
||||||
|
use crate::btree_v2::find_btree_v2_records;
|
||||||
|
use core::cmp::Ordering;
|
||||||
|
let rs = 11usize;
|
||||||
|
// Keys 0, 0, 0, 2, 2, 2, 4, ...: runs of three, odd keys missing.
|
||||||
|
for n in [1usize, 45, 46, 1150, 30_000] {
|
||||||
|
let mut recs = Vec::with_capacity(n * rs);
|
||||||
|
for i in 0..n {
|
||||||
|
let mut r = vec![0u8; rs];
|
||||||
|
r[..8].copy_from_slice(&((i / 3 * 2) as u64).to_be_bytes());
|
||||||
|
r[8..].copy_from_slice(&[(i % 3) as u8, 0, 0]);
|
||||||
|
recs.extend_from_slice(&r);
|
||||||
|
}
|
||||||
|
let base = 4096u64;
|
||||||
|
let tree = build_btree_v2(params(512, 11), &recs, base, 8, 8).unwrap();
|
||||||
|
let mut file = vec![0u8; base as usize];
|
||||||
|
file.extend_from_slice(&tree);
|
||||||
|
let hdr = BTreeV2Header::parse(&file, base as usize, 8, 8).unwrap();
|
||||||
|
let all = collect_btree_v2_records(&file, &hdr, 8, 8).unwrap();
|
||||||
|
let key = |r: &[u8]| u64::from_be_bytes(r[..8].try_into().unwrap());
|
||||||
|
let last = key(&all[n - 1].data);
|
||||||
|
let probes = (0..=last + 1).step_by(if n > 1000 { 37 } else { 1 });
|
||||||
|
for k in probes.chain([last, last + 1, u64::MAX]) {
|
||||||
|
let found =
|
||||||
|
find_btree_v2_records(&file, &hdr, 8, &mut |r: &[u8]| key(r).cmp(&k)).unwrap();
|
||||||
|
let want: Vec<&[u8]> = all
|
||||||
|
.iter()
|
||||||
|
.map(|r| r.data.as_slice())
|
||||||
|
.filter(|r| key(r) == k)
|
||||||
|
.collect();
|
||||||
|
let got: Vec<&[u8]> = found.iter().map(|r| r.data.as_slice()).collect();
|
||||||
|
assert_eq!(got, want, "n {n} key {k}");
|
||||||
|
assert_eq!(
|
||||||
|
got.len(),
|
||||||
|
if k % 2 == 0 && k <= last {
|
||||||
|
want.len()
|
||||||
|
} else {
|
||||||
|
0
|
||||||
|
}
|
||||||
|
);
|
||||||
|
}
|
||||||
|
// Every record, or none, when the whole tree is in or out of range.
|
||||||
|
let every = find_btree_v2_records(&file, &hdr, 8, &mut |_| Ordering::Equal).unwrap();
|
||||||
|
assert_eq!(every.len(), n);
|
||||||
|
let none = find_btree_v2_records(&file, &hdr, 8, &mut |_| Ordering::Less).unwrap();
|
||||||
|
assert!(none.is_empty());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn a_node_too_small_or_too_big_is_an_error() {
|
fn a_node_too_small_or_too_big_is_an_error() {
|
||||||
assert!(build_btree_v2(params(16, 11), &records(1, 11), 0, 8, 8).is_err());
|
assert!(build_btree_v2(params(16, 11), &records(1, 11), 0, 8, 8).is_err());
|
||||||
|
|||||||
@@ -6,7 +6,8 @@ use alloc::{format, vec::Vec};
|
|||||||
#[cfg(feature = "checksum")]
|
#[cfg(feature = "checksum")]
|
||||||
use byteorder::{ByteOrder, LittleEndian};
|
use byteorder::{ByteOrder, LittleEndian};
|
||||||
|
|
||||||
use crate::btree_v2::{BTreeV2Header, collect_btree_v2_records};
|
use crate::addr::to_usize;
|
||||||
|
use crate::btree_v2::{BTreeV2Header, find_btree_v2_records};
|
||||||
use crate::error::FormatError;
|
use crate::error::FormatError;
|
||||||
use crate::filter_pipeline::FilterPipeline;
|
use crate::filter_pipeline::FilterPipeline;
|
||||||
|
|
||||||
@@ -354,6 +355,7 @@ impl FractalHeapHeader {
|
|||||||
id_bytes: &[u8],
|
id_bytes: &[u8],
|
||||||
offset_size: u8,
|
offset_size: u8,
|
||||||
) -> Result<Vec<u8>, FormatError> {
|
) -> Result<Vec<u8>, FormatError> {
|
||||||
|
crate::lookup_stats::heap_object_read();
|
||||||
let Some(&first) = id_bytes.first() else {
|
let Some(&first) = id_bytes.first() else {
|
||||||
return Err(FormatError::UnexpectedEof {
|
return Err(FormatError::UnexpectedEof {
|
||||||
expected: 1,
|
expected: 1,
|
||||||
@@ -448,7 +450,7 @@ impl FractalHeapHeader {
|
|||||||
}
|
}
|
||||||
let hdr = BTreeV2Header::parse(
|
let hdr = BTreeV2Header::parse(
|
||||||
file_data,
|
file_data,
|
||||||
self.huge_btree_address as usize,
|
to_usize(self.huge_btree_address)?,
|
||||||
self.offset_size,
|
self.offset_size,
|
||||||
self.length_size,
|
self.length_size,
|
||||||
)?;
|
)?;
|
||||||
@@ -463,8 +465,12 @@ impl FractalHeapHeader {
|
|||||||
if hdr.tree_type != expected_type || usize::from(hdr.record_size) < rec_len {
|
if hdr.tree_type != expected_type || usize::from(hdr.record_size) < rec_len {
|
||||||
return Err(heap_error("unexpected huge-object B-tree record type"));
|
return Err(heap_error("unexpected huge-object B-tree record type"));
|
||||||
}
|
}
|
||||||
let records =
|
// Records are ordered by ID (the last field): descend to the ones
|
||||||
collect_btree_v2_records(file_data, &hdr, self.offset_size, self.length_size)?;
|
// equal to `key` instead of reading the whole index.
|
||||||
|
let id_at = rec_len - ls;
|
||||||
|
let records = find_btree_v2_records(file_data, &hdr, self.offset_size, &mut |r| {
|
||||||
|
le_uint(&r[id_at..id_at + ls]).cmp(&key)
|
||||||
|
})?;
|
||||||
for rec in &records {
|
for rec in &records {
|
||||||
let d = &rec.data;
|
let d = &rec.data;
|
||||||
if d.len() < rec_len {
|
if d.len() < rec_len {
|
||||||
@@ -533,24 +539,24 @@ impl FractalHeapHeader {
|
|||||||
self.read_from_direct_block(
|
self.read_from_direct_block(
|
||||||
file_data,
|
file_data,
|
||||||
DirectBlock {
|
DirectBlock {
|
||||||
addr: self.root_block_address as usize,
|
addr: to_usize(self.root_block_address)?,
|
||||||
size: self.starting_block_size,
|
size: self.starting_block_size,
|
||||||
heap_offset: 0,
|
heap_offset: 0,
|
||||||
filtered_size: self.root_direct_block_filtered_size,
|
filtered_size: self.root_direct_block_filtered_size,
|
||||||
filter_mask: self.root_direct_block_filter_mask,
|
filter_mask: self.root_direct_block_filter_mask,
|
||||||
},
|
},
|
||||||
heap_offset,
|
heap_offset,
|
||||||
obj_len as usize,
|
to_usize(obj_len)?,
|
||||||
)
|
)
|
||||||
} else {
|
} else {
|
||||||
// Root is an indirect block — limit recursion to 64 levels
|
// Root is an indirect block — limit recursion to 64 levels
|
||||||
self.read_from_indirect_block(
|
self.read_from_indirect_block(
|
||||||
file_data,
|
file_data,
|
||||||
self.root_block_address as usize,
|
to_usize(self.root_block_address)?,
|
||||||
self.current_rows_in_root_indirect_block,
|
self.current_rows_in_root_indirect_block,
|
||||||
0, // block offset
|
0, // block offset
|
||||||
heap_offset,
|
heap_offset,
|
||||||
obj_len as usize,
|
to_usize(obj_len)?,
|
||||||
offset_size,
|
offset_size,
|
||||||
64, // max recursion depth
|
64, // max recursion depth
|
||||||
)
|
)
|
||||||
@@ -572,11 +578,11 @@ impl FractalHeapHeader {
|
|||||||
) -> Result<Vec<u8>, FormatError> {
|
) -> Result<Vec<u8>, FormatError> {
|
||||||
if target_offset < block.heap_offset {
|
if target_offset < block.heap_offset {
|
||||||
return Err(FormatError::UnexpectedEof {
|
return Err(FormatError::UnexpectedEof {
|
||||||
expected: block.heap_offset as usize,
|
expected: to_usize(block.heap_offset)?,
|
||||||
available: target_offset as usize,
|
available: to_usize(target_offset)?,
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
let local_offset = (target_offset - block.heap_offset) as usize;
|
let local_offset = to_usize(target_offset - block.heap_offset)?;
|
||||||
if let Some(pipeline) = &self.filter_pipeline {
|
if let Some(pipeline) = &self.filter_pipeline {
|
||||||
let stored_len = usize::try_from(block.filtered_size)
|
let stored_len = usize::try_from(block.filtered_size)
|
||||||
.map_err(|_| heap_error("direct block size"))?;
|
.map_err(|_| heap_error("direct block size"))?;
|
||||||
@@ -673,7 +679,7 @@ impl FractalHeapHeader {
|
|||||||
return self.read_from_direct_block(
|
return self.read_from_direct_block(
|
||||||
file_data,
|
file_data,
|
||||||
DirectBlock {
|
DirectBlock {
|
||||||
addr: child_addr as usize,
|
addr: to_usize(child_addr)?,
|
||||||
size: block_size,
|
size: block_size,
|
||||||
heap_offset: current_heap_offset,
|
heap_offset: current_heap_offset,
|
||||||
filtered_size,
|
filtered_size,
|
||||||
@@ -705,7 +711,7 @@ impl FractalHeapHeader {
|
|||||||
{
|
{
|
||||||
return self.read_from_indirect_block(
|
return self.read_from_indirect_block(
|
||||||
file_data,
|
file_data,
|
||||||
child_addr as usize,
|
to_usize(child_addr)?,
|
||||||
child_nrows,
|
child_nrows,
|
||||||
current_heap_offset,
|
current_heap_offset,
|
||||||
target_offset,
|
target_offset,
|
||||||
@@ -719,7 +725,7 @@ impl FractalHeapHeader {
|
|||||||
}
|
}
|
||||||
|
|
||||||
Err(FormatError::UnexpectedEof {
|
Err(FormatError::UnexpectedEof {
|
||||||
expected: target_offset as usize + length,
|
expected: to_usize(target_offset)?.saturating_add(length),
|
||||||
available: file_data.len(),
|
available: file_data.len(),
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -6,7 +6,9 @@
|
|||||||
#[cfg(not(feature = "std"))]
|
#[cfg(not(feature = "std"))]
|
||||||
use alloc::{string::String, vec::Vec};
|
use alloc::{string::String, vec::Vec};
|
||||||
|
|
||||||
use crate::btree_v2::{BTreeV2Header, collect_btree_v2_records};
|
use crate::addr::to_usize;
|
||||||
|
use crate::btree_v2::{BTreeV2Header, collect_btree_v2_records, find_btree_v2_records};
|
||||||
|
use crate::checksum::jenkins_lookup3;
|
||||||
use crate::error::FormatError;
|
use crate::error::FormatError;
|
||||||
use crate::fractal_heap::FractalHeapHeader;
|
use crate::fractal_heap::FractalHeapHeader;
|
||||||
use crate::group_v1::{self, GroupEntry};
|
use crate::group_v1::{self, GroupEntry};
|
||||||
@@ -93,13 +95,14 @@ fn for_each_dense_link(
|
|||||||
mut visit: impl FnMut(LinkMessage),
|
mut visit: impl FnMut(LinkMessage),
|
||||||
) -> Result<(), FormatError> {
|
) -> Result<(), FormatError> {
|
||||||
// Parse fractal heap
|
// Parse fractal heap
|
||||||
let fh = FractalHeapHeader::parse(file_data, fh_addr as usize, offset_size, length_size)?;
|
let fh = FractalHeapHeader::parse(file_data, to_usize(fh_addr)?, offset_size, length_size)?;
|
||||||
|
|
||||||
// Parse B-tree v2 for name index
|
// Parse B-tree v2 for name index
|
||||||
let btree_addr = link_info
|
let btree_addr = link_info
|
||||||
.btree_name_index_address
|
.btree_name_index_address
|
||||||
.ok_or_else(|| FormatError::PathNotFound(String::from("no B-tree v2 name index")))?;
|
.ok_or_else(|| FormatError::PathNotFound(String::from("no B-tree v2 name index")))?;
|
||||||
let btree_hdr = BTreeV2Header::parse(file_data, btree_addr as usize, offset_size, length_size)?;
|
let btree_hdr =
|
||||||
|
BTreeV2Header::parse(file_data, to_usize(btree_addr)?, offset_size, length_size)?;
|
||||||
let records = collect_btree_v2_records(file_data, &btree_hdr, offset_size, length_size)?;
|
let records = collect_btree_v2_records(file_data, &btree_hdr, offset_size, length_size)?;
|
||||||
|
|
||||||
for record in &records {
|
for record in &records {
|
||||||
@@ -181,10 +184,55 @@ fn find_symbolic_link(
|
|||||||
if !is_v2_group(object_header) {
|
if !is_v2_group(object_header) {
|
||||||
return Ok(None);
|
return Ok(None);
|
||||||
}
|
}
|
||||||
let is_symbolic = |t: &LinkTarget| !matches!(t, LinkTarget::Hard { .. });
|
// As when all links were scanned: the last symbolic link of that name.
|
||||||
|
Ok(
|
||||||
|
links_named(file_data, object_header, name, offset_size, length_size)?
|
||||||
|
.into_iter()
|
||||||
|
.rev()
|
||||||
|
.map(|link| link.link_target)
|
||||||
|
.find(|t| !matches!(t, LinkTarget::Hard { .. })),
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
/// B-tree v2 record type of a dense group's link name index.
|
||||||
|
const LINK_NAME_INDEX: u8 = 5;
|
||||||
|
|
||||||
|
/// The links called `name` in a v2 group (a valid group has at most one).
|
||||||
|
///
|
||||||
|
/// In dense storage the link name index (a v2 B-tree of lookup3 name
|
||||||
|
/// hashes, record type 5) is descended to the records with the name's hash,
|
||||||
|
/// and only their links are read from the heap — O(log n) instead of every
|
||||||
|
/// link. libhdf5 orders records with equal hashes by name; all of them are
|
||||||
|
/// read and compared here, so that order does not matter. An index of
|
||||||
|
/// another type is scanned in full.
|
||||||
|
fn links_named(
|
||||||
|
file_data: &[u8],
|
||||||
|
object_header: &ObjectHeader,
|
||||||
|
name: &str,
|
||||||
|
offset_size: u8,
|
||||||
|
length_size: u8,
|
||||||
|
) -> Result<Vec<LinkMessage>, FormatError> {
|
||||||
|
let mut found = Vec::new();
|
||||||
let link_info = find_link_info(object_header, offset_size)?;
|
let link_info = find_link_info(object_header, offset_size)?;
|
||||||
let mut found = None;
|
let Some(fh_addr) = link_info.fractal_heap_address else {
|
||||||
if let Some(fh_addr) = link_info.fractal_heap_address {
|
for msg in &object_header.messages {
|
||||||
|
if msg.msg_type == MessageType::Link
|
||||||
|
&& let Some(link) = parse_link(&msg.data, offset_size)?
|
||||||
|
&& link.name == name
|
||||||
|
{
|
||||||
|
found.push(link);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return Ok(found);
|
||||||
|
};
|
||||||
|
|
||||||
|
let fh = FractalHeapHeader::parse(file_data, to_usize(fh_addr)?, offset_size, length_size)?;
|
||||||
|
let btree_addr = link_info
|
||||||
|
.btree_name_index_address
|
||||||
|
.ok_or_else(|| FormatError::PathNotFound(String::from("no B-tree v2 name index")))?;
|
||||||
|
let btree_hdr =
|
||||||
|
BTreeV2Header::parse(file_data, to_usize(btree_addr)?, offset_size, length_size)?;
|
||||||
|
if btree_hdr.tree_type != LINK_NAME_INDEX {
|
||||||
for_each_dense_link(
|
for_each_dense_link(
|
||||||
file_data,
|
file_data,
|
||||||
&link_info,
|
&link_info,
|
||||||
@@ -192,26 +240,134 @@ fn find_symbolic_link(
|
|||||||
offset_size,
|
offset_size,
|
||||||
length_size,
|
length_size,
|
||||||
|link| {
|
|link| {
|
||||||
if link.name == name && is_symbolic(&link.link_target) {
|
if link.name == name {
|
||||||
found = Some(link.link_target);
|
found.push(link);
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
)?;
|
)?;
|
||||||
} else {
|
return Ok(found);
|
||||||
for msg in &object_header.messages {
|
}
|
||||||
if msg.msg_type == MessageType::Link {
|
|
||||||
let Some(link) = parse_link(&msg.data, offset_size)? else {
|
// Record: hash(4) + heap ID.
|
||||||
continue;
|
let hash = jenkins_lookup3(name.as_bytes());
|
||||||
};
|
let records = find_btree_v2_records(file_data, &btree_hdr, offset_size, &mut |r| {
|
||||||
if link.name == name && is_symbolic(&link.link_target) {
|
match r.get(..4) {
|
||||||
found = Some(link.link_target);
|
Some(h) => u32::from_le_bytes([h[0], h[1], h[2], h[3]]).cmp(&hash),
|
||||||
}
|
// Too short to hold a hash (a corrupt record size): never a match.
|
||||||
}
|
None => core::cmp::Ordering::Less,
|
||||||
|
}
|
||||||
|
})?;
|
||||||
|
let id_len = usize::from(fh.heap_id_length);
|
||||||
|
for record in &records {
|
||||||
|
let Some(id_bytes) = record.data.get(4..4 + id_len) else {
|
||||||
|
continue;
|
||||||
|
};
|
||||||
|
let link_data = fh.read_managed_object(file_data, id_bytes, offset_size)?;
|
||||||
|
if let Some(link) = parse_link(&link_data, offset_size)?
|
||||||
|
&& link.name == name
|
||||||
|
{
|
||||||
|
found.push(link);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
Ok(found)
|
Ok(found)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// The link [`resolve_path_any`] follows for one path component `name` of
|
||||||
|
/// the group with header `object_header`: a hard link (as `Hard`), else a
|
||||||
|
/// soft or external link of that name, else `None`. Fails with
|
||||||
|
/// `PathNotFound` if the object is not a group.
|
||||||
|
fn lookup_link(
|
||||||
|
file_data: &[u8],
|
||||||
|
object_header: &ObjectHeader,
|
||||||
|
name: &str,
|
||||||
|
offset_size: u8,
|
||||||
|
length_size: u8,
|
||||||
|
) -> Result<Option<LinkTarget>, FormatError> {
|
||||||
|
if is_v1_group(object_header) {
|
||||||
|
let entries = resolve_group_entries(file_data, object_header, offset_size, length_size)?;
|
||||||
|
if let Some(e) = entries
|
||||||
|
.iter()
|
||||||
|
.find(|e| e.name == name && e.object_header_address != u64::MAX)
|
||||||
|
{
|
||||||
|
return Ok(Some(LinkTarget::Hard {
|
||||||
|
object_header_address: e.object_header_address,
|
||||||
|
}));
|
||||||
|
}
|
||||||
|
return find_symbolic_link(file_data, object_header, name, offset_size, length_size);
|
||||||
|
}
|
||||||
|
if !is_v2_group(object_header) {
|
||||||
|
return Err(FormatError::PathNotFound(String::from(
|
||||||
|
"object header is not a group",
|
||||||
|
)));
|
||||||
|
}
|
||||||
|
let links = links_named(file_data, object_header, name, offset_size, length_size)?;
|
||||||
|
if let Some(addr) = links.iter().find_map(|l| match l.link_target {
|
||||||
|
LinkTarget::Hard {
|
||||||
|
object_header_address,
|
||||||
|
} if object_header_address != u64::MAX => Some(object_header_address),
|
||||||
|
_ => None,
|
||||||
|
}) {
|
||||||
|
return Ok(Some(LinkTarget::Hard {
|
||||||
|
object_header_address: addr,
|
||||||
|
}));
|
||||||
|
}
|
||||||
|
Ok(links
|
||||||
|
.into_iter()
|
||||||
|
.rev()
|
||||||
|
.map(|link| link.link_target)
|
||||||
|
.find(|t| !matches!(t, LinkTarget::Hard { .. })))
|
||||||
|
}
|
||||||
|
|
||||||
|
/// The object header address of the child called `name` of the group at
|
||||||
|
/// `group_address`: the address [`resolve_group_children`] lists under that
|
||||||
|
/// name, or `PathNotFound` if it lists none.
|
||||||
|
///
|
||||||
|
/// A dense group's child is found through its link name index (see
|
||||||
|
/// [`links_named`]) and only the named link is read and, if it is a soft
|
||||||
|
/// link, followed — not every link in the group. A v1 group is listed.
|
||||||
|
pub fn resolve_child(
|
||||||
|
file_data: &[u8],
|
||||||
|
superblock: &Superblock,
|
||||||
|
group_address: u64,
|
||||||
|
name: &str,
|
||||||
|
) -> Result<u64, FormatError> {
|
||||||
|
let os = superblock.offset_size;
|
||||||
|
let ls = superblock.length_size;
|
||||||
|
let not_found = || FormatError::PathNotFound(String::from(name));
|
||||||
|
let header = ObjectHeader::parse(file_data, to_usize(group_address)?, os, ls)?;
|
||||||
|
if !is_v2_group(&header) || is_v1_group(&header) {
|
||||||
|
return resolve_group_children(file_data, superblock, group_address)?
|
||||||
|
.into_iter()
|
||||||
|
.find(|e| e.name == name)
|
||||||
|
.map(|e| e.object_header_address)
|
||||||
|
.ok_or_else(not_found);
|
||||||
|
}
|
||||||
|
let links = links_named(file_data, &header, name, os, ls)?;
|
||||||
|
// The listing puts hard links before resolved soft links.
|
||||||
|
if let Some(addr) = links.iter().find_map(|l| match l.link_target {
|
||||||
|
LinkTarget::Hard {
|
||||||
|
object_header_address,
|
||||||
|
} => Some(object_header_address),
|
||||||
|
_ => None,
|
||||||
|
}) {
|
||||||
|
return Ok(addr);
|
||||||
|
}
|
||||||
|
for link in links {
|
||||||
|
if let LinkTarget::Soft { target_path } = link.link_target {
|
||||||
|
return match resolve_path_from(file_data, superblock, group_address, &target_path) {
|
||||||
|
// Left out of the listing: dangling, cyclic, or in another file.
|
||||||
|
Err(
|
||||||
|
FormatError::PathNotFound(_)
|
||||||
|
| FormatError::NestingDepthExceeded
|
||||||
|
| FormatError::ExternalLinkUnsupported { .. },
|
||||||
|
) => Err(not_found()),
|
||||||
|
other => other,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
}
|
||||||
|
Err(not_found())
|
||||||
|
}
|
||||||
|
|
||||||
/// Find and parse the Link Info message from an object header.
|
/// Find and parse the Link Info message from an object header.
|
||||||
fn find_link_info(
|
fn find_link_info(
|
||||||
object_header: &ObjectHeader,
|
object_header: &ObjectHeader,
|
||||||
@@ -298,7 +454,7 @@ pub fn resolve_group_children(
|
|||||||
) -> Result<Vec<GroupEntry>, FormatError> {
|
) -> Result<Vec<GroupEntry>, FormatError> {
|
||||||
let os = superblock.offset_size;
|
let os = superblock.offset_size;
|
||||||
let ls = superblock.length_size;
|
let ls = superblock.length_size;
|
||||||
let header = ObjectHeader::parse(file_data, group_address as usize, os, ls)?;
|
let header = ObjectHeader::parse(file_data, to_usize(group_address)?, os, ls)?;
|
||||||
|
|
||||||
let mut entries = Vec::new();
|
let mut entries = Vec::new();
|
||||||
let mut soft = Vec::new();
|
let mut soft = Vec::new();
|
||||||
@@ -383,24 +539,21 @@ fn resolve_path_following_links(
|
|||||||
let ls = superblock.length_size;
|
let ls = superblock.length_size;
|
||||||
|
|
||||||
let mut current_addr = start;
|
let mut current_addr = start;
|
||||||
let mut current_header = ObjectHeader::parse(file_data, start as usize, os, ls)?;
|
let mut current_header = ObjectHeader::parse(file_data, to_usize(start)?, os, ls)?;
|
||||||
|
|
||||||
for (i, component) in components.iter().enumerate() {
|
for (i, component) in components.iter().enumerate() {
|
||||||
let entries = resolve_group_entries(file_data, ¤t_header, os, ls)?;
|
match lookup_link(file_data, ¤t_header, component, os, ls)? {
|
||||||
|
Some(LinkTarget::Hard {
|
||||||
let found = entries
|
object_header_address,
|
||||||
.iter()
|
}) => {
|
||||||
.find(|e| e.name == *component && e.object_header_address != u64::MAX);
|
|
||||||
match found {
|
|
||||||
Some(entry) => {
|
|
||||||
if i == components.len() - 1 {
|
if i == components.len() - 1 {
|
||||||
return Ok(entry.object_header_address);
|
return Ok(object_header_address);
|
||||||
}
|
}
|
||||||
current_addr = entry.object_header_address;
|
current_addr = object_header_address;
|
||||||
current_header = ObjectHeader::parse(file_data, current_addr as usize, os, ls)?;
|
current_header = ObjectHeader::parse(file_data, to_usize(current_addr)?, os, ls)?;
|
||||||
}
|
}
|
||||||
None => {
|
found => {
|
||||||
return match find_symbolic_link(file_data, ¤t_header, component, os, ls)? {
|
return match found {
|
||||||
Some(LinkTarget::Soft { target_path }) => {
|
Some(LinkTarget::Soft { target_path }) => {
|
||||||
if depth >= MAX_SOFT_LINK_DEPTH {
|
if depth >= MAX_SOFT_LINK_DEPTH {
|
||||||
return Err(FormatError::NestingDepthExceeded);
|
return Err(FormatError::NestingDepthExceeded);
|
||||||
|
|||||||
@@ -108,6 +108,7 @@ pub mod lane_partition;
|
|||||||
pub mod link_info;
|
pub mod link_info;
|
||||||
pub mod link_message;
|
pub mod link_message;
|
||||||
pub mod local_heap;
|
pub mod local_heap;
|
||||||
|
pub mod lookup_stats;
|
||||||
pub mod message_type;
|
pub mod message_type;
|
||||||
pub mod metadata_cache;
|
pub mod metadata_cache;
|
||||||
pub mod metadata_index;
|
pub mod metadata_index;
|
||||||
|
|||||||
@@ -0,0 +1,31 @@
|
|||||||
|
//! Work counters for tests of lookup cost (feature `lookup-stats`).
|
||||||
|
//!
|
||||||
|
//! Counts fractal-heap objects read — each is one link or attribute message
|
||||||
|
//! decoded out of a dense group or dense attribute storage — so a test can
|
||||||
|
//! check that finding one name reads a handful of them, not the whole group.
|
||||||
|
//! Per thread, so tests running in parallel do not see each other's reads.
|
||||||
|
//! Without the feature the counting compiles to nothing.
|
||||||
|
|
||||||
|
#[cfg(feature = "lookup-stats")]
|
||||||
|
std::thread_local! {
|
||||||
|
static HEAP_OBJECTS: core::cell::Cell<u64> = const { core::cell::Cell::new(0) };
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Record one heap object read.
|
||||||
|
#[inline(always)]
|
||||||
|
pub(crate) fn heap_object_read() {
|
||||||
|
#[cfg(feature = "lookup-stats")]
|
||||||
|
HEAP_OBJECTS.with(|c| c.set(c.get() + 1));
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Heap objects read on this thread since the last [`reset`].
|
||||||
|
#[cfg(feature = "lookup-stats")]
|
||||||
|
pub fn heap_objects_read() -> u64 {
|
||||||
|
HEAP_OBJECTS.with(core::cell::Cell::get)
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Zero this thread's counters.
|
||||||
|
#[cfg(feature = "lookup-stats")]
|
||||||
|
pub fn reset() {
|
||||||
|
HEAP_OBJECTS.with(|c| c.set(0));
|
||||||
|
}
|
||||||
@@ -19,7 +19,8 @@ rayon = { version = "1", optional = true }
|
|||||||
tempfile = { workspace = true }
|
tempfile = { workspace = true }
|
||||||
criterion = { workspace = true }
|
criterion = { workspace = true }
|
||||||
clawhdf5-io = { path = "../clawhdf5-io", version = "2.7.0", features = ["mmap"] }
|
clawhdf5-io = { path = "../clawhdf5-io", version = "2.7.0", features = ["mmap"] }
|
||||||
clawhdf5-format = { path = "../clawhdf5-format", version = "2.7.0", features = ["parallel", "fast-checksum"] }
|
clawhdf5-format = { path = "../clawhdf5-format", version = "2.7.0", features = ["parallel", "fast-checksum", "lookup-stats"] }
|
||||||
|
serde_json = "1"
|
||||||
clawhdf5-filters = { path = "../clawhdf5-filters", version = "2.7.0" }
|
clawhdf5-filters = { path = "../clawhdf5-filters", version = "2.7.0" }
|
||||||
|
|
||||||
[[bench]]
|
[[bench]]
|
||||||
|
|||||||
+44
-13
@@ -28,7 +28,7 @@ use clawhdf5_format::superblock::Superblock;
|
|||||||
use clawhdf5_io::HDF5Read;
|
use clawhdf5_io::HDF5Read;
|
||||||
|
|
||||||
use crate::error::Error;
|
use crate::error::Error;
|
||||||
use crate::types::{AttrValue, DType, classify_datatype, read_attrs};
|
use crate::types::{AttrValue, DType, classify_datatype, read_attr, read_attrs};
|
||||||
|
|
||||||
/// A lazy HDF5 file handle that parses metadata on demand.
|
/// A lazy HDF5 file handle that parses metadata on demand.
|
||||||
///
|
///
|
||||||
@@ -304,12 +304,7 @@ impl<'f, R: HDF5Read> LazyGroup<'f, R> {
|
|||||||
|
|
||||||
/// Get a dataset within this group by name.
|
/// Get a dataset within this group by name.
|
||||||
pub fn dataset(&self, name: &str) -> Result<LazyDataset<'f, R>, Error> {
|
pub fn dataset(&self, name: &str) -> Result<LazyDataset<'f, R>, Error> {
|
||||||
let entries = self.children()?;
|
let hdr = self.file.get_or_parse_header(self.child_address(name)?)?;
|
||||||
let entry = entries
|
|
||||||
.iter()
|
|
||||||
.find(|e| e.name == name)
|
|
||||||
.ok_or_else(|| Error::Format(FormatError::PathNotFound(name.to_string())))?;
|
|
||||||
let hdr = self.file.get_or_parse_header(entry.object_header_address)?;
|
|
||||||
if !has_message(&hdr, MessageType::DataLayout) {
|
if !has_message(&hdr, MessageType::DataLayout) {
|
||||||
return Err(Error::NotADataset(name.to_string()));
|
return Err(Error::NotADataset(name.to_string()));
|
||||||
}
|
}
|
||||||
@@ -322,17 +317,38 @@ impl<'f, R: HDF5Read> LazyGroup<'f, R> {
|
|||||||
|
|
||||||
/// Get a subgroup within this group by name.
|
/// Get a subgroup within this group by name.
|
||||||
pub fn group(&self, name: &str) -> Result<LazyGroup<'f, R>, Error> {
|
pub fn group(&self, name: &str) -> Result<LazyGroup<'f, R>, Error> {
|
||||||
let entries = self.children()?;
|
|
||||||
let entry = entries
|
|
||||||
.iter()
|
|
||||||
.find(|e| e.name == name)
|
|
||||||
.ok_or_else(|| Error::Format(FormatError::PathNotFound(name.to_string())))?;
|
|
||||||
Ok(LazyGroup {
|
Ok(LazyGroup {
|
||||||
file: self.file,
|
file: self.file,
|
||||||
address: entry.object_header_address,
|
address: self.child_address(name)?,
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// The attribute called `name`, or `None` if it has none by that name
|
||||||
|
/// (or it cannot be read) — the value [`attrs`](Self::attrs) has under
|
||||||
|
/// that name, found without reading the other attributes when they are
|
||||||
|
/// stored densely.
|
||||||
|
pub fn attr(&self, name: &str) -> Result<Option<AttrValue>, Error> {
|
||||||
|
let hdr = self.file.get_or_parse_header(self.address)?;
|
||||||
|
let data = self.file.hdf5_bytes();
|
||||||
|
read_attr(
|
||||||
|
data,
|
||||||
|
&hdr,
|
||||||
|
name,
|
||||||
|
self.file.offset_size(),
|
||||||
|
self.file.length_size(),
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
/// The object header address of the child called `name`: the entry of
|
||||||
|
/// the group's listing with that name, looked up through the group's
|
||||||
|
/// name index rather than by listing the group (see
|
||||||
|
/// [`group_v2::resolve_child`]).
|
||||||
|
fn child_address(&self, name: &str) -> Result<u64, Error> {
|
||||||
|
let data = self.file.hdf5_bytes();
|
||||||
|
group_v2::resolve_child(data, &self.file.superblock, self.address, name)
|
||||||
|
.map_err(Error::Format)
|
||||||
|
}
|
||||||
|
|
||||||
/// This group's links that can be opened: hard links, and soft links
|
/// This group's links that can be opened: hard links, and soft links
|
||||||
/// resolved to their targets (see
|
/// resolved to their targets (see
|
||||||
/// [`group_v2::resolve_group_children`]); dangling, external and
|
/// [`group_v2::resolve_group_children`]); dangling, external and
|
||||||
@@ -555,6 +571,21 @@ impl<'f, R: HDF5Read> LazyDataset<'f, R> {
|
|||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// The attribute called `name`, or `None` if it has none by that name
|
||||||
|
/// (or it cannot be read) — the value [`attrs`](Self::attrs) has under
|
||||||
|
/// that name, found without reading the other attributes when they are
|
||||||
|
/// stored densely.
|
||||||
|
pub fn attr(&self, name: &str) -> Result<Option<AttrValue>, Error> {
|
||||||
|
let data = self.file.hdf5_bytes();
|
||||||
|
read_attr(
|
||||||
|
data,
|
||||||
|
&self.header,
|
||||||
|
name,
|
||||||
|
self.file.offset_size(),
|
||||||
|
self.file.length_size(),
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
/// A header message's payload, resolved through the shared-message
|
/// A header message's payload, resolved through the shared-message
|
||||||
/// indirection when needed (e.g. a committed datatype). See
|
/// indirection when needed (e.g. a committed datatype). See
|
||||||
/// [`clawhdf5_format::shared_message::message_data`].
|
/// [`clawhdf5_format::shared_message::message_data`].
|
||||||
|
|||||||
@@ -23,7 +23,7 @@ use clawhdf5_format::superblock::Superblock;
|
|||||||
use clawhdf5_io::MmapReader;
|
use clawhdf5_io::MmapReader;
|
||||||
|
|
||||||
use crate::error::Error;
|
use crate::error::Error;
|
||||||
use crate::types::{AttrValue, DType, classify_datatype, read_attrs};
|
use crate::types::{AttrValue, DType, classify_datatype, read_attr, read_attrs};
|
||||||
|
|
||||||
/// An HDF5 file opened via memory mapping.
|
/// An HDF5 file opened via memory mapping.
|
||||||
///
|
///
|
||||||
@@ -242,12 +242,7 @@ impl<'f> MmapGroup<'f> {
|
|||||||
|
|
||||||
/// Get a dataset within this group by name.
|
/// Get a dataset within this group by name.
|
||||||
pub fn dataset(&self, name: &str) -> Result<MmapDataset<'f>, Error> {
|
pub fn dataset(&self, name: &str) -> Result<MmapDataset<'f>, Error> {
|
||||||
let entries = self.children()?;
|
let hdr = self.file.parse_header(self.child_address(name)?)?;
|
||||||
let entry = entries
|
|
||||||
.iter()
|
|
||||||
.find(|e| e.name == name)
|
|
||||||
.ok_or_else(|| Error::Format(FormatError::PathNotFound(name.to_string())))?;
|
|
||||||
let hdr = self.file.parse_header(entry.object_header_address)?;
|
|
||||||
if !has_message(&hdr, MessageType::DataLayout) {
|
if !has_message(&hdr, MessageType::DataLayout) {
|
||||||
return Err(Error::NotADataset(name.to_string()));
|
return Err(Error::NotADataset(name.to_string()));
|
||||||
}
|
}
|
||||||
@@ -260,17 +255,38 @@ impl<'f> MmapGroup<'f> {
|
|||||||
|
|
||||||
/// Get a subgroup within this group by name.
|
/// Get a subgroup within this group by name.
|
||||||
pub fn group(&self, name: &str) -> Result<MmapGroup<'f>, Error> {
|
pub fn group(&self, name: &str) -> Result<MmapGroup<'f>, Error> {
|
||||||
let entries = self.children()?;
|
|
||||||
let entry = entries
|
|
||||||
.iter()
|
|
||||||
.find(|e| e.name == name)
|
|
||||||
.ok_or_else(|| Error::Format(FormatError::PathNotFound(name.to_string())))?;
|
|
||||||
Ok(MmapGroup {
|
Ok(MmapGroup {
|
||||||
file: self.file,
|
file: self.file,
|
||||||
address: entry.object_header_address,
|
address: self.child_address(name)?,
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// The attribute called `name`, or `None` if it has none by that name
|
||||||
|
/// (or it cannot be read) — the value [`attrs`](Self::attrs) has under
|
||||||
|
/// that name, found without reading the other attributes when they are
|
||||||
|
/// stored densely.
|
||||||
|
pub fn attr(&self, name: &str) -> Result<Option<AttrValue>, Error> {
|
||||||
|
let hdr = self.file.parse_header(self.address)?;
|
||||||
|
let data = self.file.hdf5_bytes();
|
||||||
|
read_attr(
|
||||||
|
data,
|
||||||
|
&hdr,
|
||||||
|
name,
|
||||||
|
self.file.offset_size(),
|
||||||
|
self.file.length_size(),
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
/// The object header address of the child called `name`: the entry of
|
||||||
|
/// the group's listing with that name, looked up through the group's
|
||||||
|
/// name index rather than by listing the group (see
|
||||||
|
/// [`group_v2::resolve_child`]).
|
||||||
|
fn child_address(&self, name: &str) -> Result<u64, Error> {
|
||||||
|
let data = self.file.meta()?;
|
||||||
|
group_v2::resolve_child(data, &self.file.superblock, self.address, name)
|
||||||
|
.map_err(Error::Format)
|
||||||
|
}
|
||||||
|
|
||||||
/// This group's links that can be opened: hard links, and soft links
|
/// This group's links that can be opened: hard links, and soft links
|
||||||
/// resolved to their targets (see
|
/// resolved to their targets (see
|
||||||
/// [`group_v2::resolve_group_children`]); dangling, external and
|
/// [`group_v2::resolve_group_children`]); dangling, external and
|
||||||
@@ -506,6 +522,21 @@ impl<'f> MmapDataset<'f> {
|
|||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// The attribute called `name`, or `None` if it has none by that name
|
||||||
|
/// (or it cannot be read) — the value [`attrs`](Self::attrs) has under
|
||||||
|
/// that name, found without reading the other attributes when they are
|
||||||
|
/// stored densely.
|
||||||
|
pub fn attr(&self, name: &str) -> Result<Option<AttrValue>, Error> {
|
||||||
|
let data = self.file.hdf5_bytes();
|
||||||
|
read_attr(
|
||||||
|
data,
|
||||||
|
&self.header,
|
||||||
|
name,
|
||||||
|
self.file.offset_size(),
|
||||||
|
self.file.length_size(),
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
/// A header message's payload, resolved through the shared-message
|
/// A header message's payload, resolved through the shared-message
|
||||||
/// indirection when needed (e.g. a committed datatype). See
|
/// indirection when needed (e.g. a committed datatype). See
|
||||||
/// [`clawhdf5_format::shared_message::message_data`].
|
/// [`clawhdf5_format::shared_message::message_data`].
|
||||||
|
|||||||
@@ -23,7 +23,7 @@ use clawhdf5_format::superblock::Superblock;
|
|||||||
|
|
||||||
use crate::cache_image::{self, ImageView};
|
use crate::cache_image::{self, ImageView};
|
||||||
use crate::error::Error;
|
use crate::error::Error;
|
||||||
use crate::types::{AttrValue, DType, classify_datatype, read_attrs};
|
use crate::types::{AttrValue, DType, classify_datatype, read_attr, read_attrs};
|
||||||
|
|
||||||
// ---------------------------------------------------------------------------
|
// ---------------------------------------------------------------------------
|
||||||
// FileData — internal storage for either owned bytes or an mmap
|
// FileData — internal storage for either owned bytes or an mmap
|
||||||
@@ -230,9 +230,10 @@ impl File {
|
|||||||
|
|
||||||
/// A `Dataset` handle for the object header at `address` (an address
|
/// A `Dataset` handle for the object header at `address` (an address
|
||||||
/// from a group listing, or one kept from an earlier lookup), without
|
/// from a group listing, or one kept from an earlier lookup), without
|
||||||
/// resolving a path. Resolving a path walks every group on it, which in
|
/// resolving a path. Resolving a path looks each component up in its
|
||||||
/// a large group costs a scan of its links; keep the address instead to
|
/// group (through the name index of a dense group; a v1 group's entries
|
||||||
/// open the same dataset repeatedly.
|
/// are scanned); keep the address instead to open the same dataset
|
||||||
|
/// repeatedly.
|
||||||
pub fn dataset_at(&self, address: u64) -> Result<Dataset<'_>, Error> {
|
pub fn dataset_at(&self, address: u64) -> Result<Dataset<'_>, Error> {
|
||||||
let hdr = self.parse_header(address)?;
|
let hdr = self.parse_header(address)?;
|
||||||
if !has_message(&hdr, MessageType::DataLayout) {
|
if !has_message(&hdr, MessageType::DataLayout) {
|
||||||
@@ -245,6 +246,17 @@ impl File {
|
|||||||
.check_open()
|
.check_open()
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// A `Group` handle for the object header at `address` (from
|
||||||
|
/// [`Group::entries`], or kept from an earlier lookup), without
|
||||||
|
/// resolving a path. Like [`group`](Self::group), the object is not
|
||||||
|
/// checked to be a group; a non-group has no children.
|
||||||
|
pub fn group_at(&self, address: u64) -> Group<'_> {
|
||||||
|
Group {
|
||||||
|
file: self,
|
||||||
|
address,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
/// Resolve a path and return a `Group` handle.
|
/// Resolve a path and return a `Group` handle.
|
||||||
///
|
///
|
||||||
/// The path uses `/` separators (e.g., `"sensors"`).
|
/// The path uses `/` separators (e.g., `"sensors"`).
|
||||||
@@ -483,12 +495,7 @@ impl<'f> Group<'f> {
|
|||||||
|
|
||||||
/// Get a dataset within this group by name.
|
/// Get a dataset within this group by name.
|
||||||
pub fn dataset(&self, name: &str) -> Result<Dataset<'f>, Error> {
|
pub fn dataset(&self, name: &str) -> Result<Dataset<'f>, Error> {
|
||||||
let entries = self.children()?;
|
let hdr = self.file.parse_header(self.child_address(name)?)?;
|
||||||
let entry = entries
|
|
||||||
.iter()
|
|
||||||
.find(|e| e.name == name)
|
|
||||||
.ok_or_else(|| Error::Format(FormatError::PathNotFound(name.to_string())))?;
|
|
||||||
let hdr = self.file.parse_header(entry.object_header_address)?;
|
|
||||||
if !has_message(&hdr, MessageType::DataLayout) {
|
if !has_message(&hdr, MessageType::DataLayout) {
|
||||||
return Err(Error::NotADataset(name.to_string()));
|
return Err(Error::NotADataset(name.to_string()));
|
||||||
}
|
}
|
||||||
@@ -501,17 +508,51 @@ impl<'f> Group<'f> {
|
|||||||
|
|
||||||
/// Get a subgroup within this group by name.
|
/// Get a subgroup within this group by name.
|
||||||
pub fn group(&self, name: &str) -> Result<Group<'f>, Error> {
|
pub fn group(&self, name: &str) -> Result<Group<'f>, Error> {
|
||||||
let entries = self.children()?;
|
|
||||||
let entry = entries
|
|
||||||
.iter()
|
|
||||||
.find(|e| e.name == name)
|
|
||||||
.ok_or_else(|| Error::Format(FormatError::PathNotFound(name.to_string())))?;
|
|
||||||
Ok(Group {
|
Ok(Group {
|
||||||
file: self.file,
|
file: self.file,
|
||||||
address: entry.object_header_address,
|
address: self.child_address(name)?,
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// The attribute called `name`, or `None` if it has none by that name
|
||||||
|
/// (or it cannot be read) — the value [`attrs`](Self::attrs) has under
|
||||||
|
/// that name, found without reading the other attributes when they are
|
||||||
|
/// stored densely.
|
||||||
|
pub fn attr(&self, name: &str) -> Result<Option<AttrValue>, Error> {
|
||||||
|
let hdr = self.file.parse_header(self.address)?;
|
||||||
|
let data = self.file.data.as_bytes();
|
||||||
|
read_attr(
|
||||||
|
data,
|
||||||
|
&hdr,
|
||||||
|
name,
|
||||||
|
self.file.offset_size(),
|
||||||
|
self.file.length_size(),
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
/// The object header address of the child called `name`: the entry of
|
||||||
|
/// the group's listing with that name, looked up through the group's
|
||||||
|
/// name index rather than by listing the group (see
|
||||||
|
/// [`group_v2::resolve_child`]).
|
||||||
|
fn child_address(&self, name: &str) -> Result<u64, Error> {
|
||||||
|
let data = self.file.data.meta()?;
|
||||||
|
group_v2::resolve_child(data, &self.file.superblock, self.address, name)
|
||||||
|
.map_err(Error::Format)
|
||||||
|
}
|
||||||
|
|
||||||
|
/// This group's children that can be opened, as `(name, object header
|
||||||
|
/// address)` in listing order — the entries [`datasets`](Self::datasets)
|
||||||
|
/// and [`groups`](Self::groups) are drawn from. Open one with
|
||||||
|
/// [`File::dataset_at`] or [`File::group_at`] to skip looking its name
|
||||||
|
/// up again, or keep the addresses to revisit the objects.
|
||||||
|
pub fn entries(&self) -> Result<Vec<(String, u64)>, Error> {
|
||||||
|
Ok(self
|
||||||
|
.children()?
|
||||||
|
.into_iter()
|
||||||
|
.map(|e| (e.name, e.object_header_address))
|
||||||
|
.collect())
|
||||||
|
}
|
||||||
|
|
||||||
/// This group's links that can be opened: hard links, and soft links
|
/// This group's links that can be opened: hard links, and soft links
|
||||||
/// resolved to their targets (see
|
/// resolved to their targets (see
|
||||||
/// [`group_v2::resolve_group_children`]); dangling, external and
|
/// [`group_v2::resolve_group_children`]); dangling, external and
|
||||||
@@ -1051,6 +1092,21 @@ impl<'f> Dataset<'f> {
|
|||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// The attribute called `name`, or `None` if it has none by that name
|
||||||
|
/// (or it cannot be read) — the value [`attrs`](Self::attrs) has under
|
||||||
|
/// that name, found without reading the other attributes when they are
|
||||||
|
/// stored densely.
|
||||||
|
pub fn attr(&self, name: &str) -> Result<Option<AttrValue>, Error> {
|
||||||
|
let data = self.file.data.as_bytes();
|
||||||
|
read_attr(
|
||||||
|
data,
|
||||||
|
&self.header,
|
||||||
|
name,
|
||||||
|
self.file.offset_size(),
|
||||||
|
self.file.length_size(),
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
/// Verify this dataset's content against its stored provenance hash
|
/// Verify this dataset's content against its stored provenance hash
|
||||||
/// (`_provenance_sha256`, written automatically on save when a
|
/// (`_provenance_sha256`, written automatically on save when a
|
||||||
/// [`Provenance`](clawhdf5_format::provenance::Provenance) is set — see
|
/// [`Provenance`](clawhdf5_format::provenance::Provenance) is set — see
|
||||||
|
|||||||
@@ -182,6 +182,35 @@ pub(crate) fn read_attrs(
|
|||||||
))
|
))
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// The attribute called `name` on the object with header `header`, decoded
|
||||||
|
/// as [`read_attrs`] decodes it, or `None` (see
|
||||||
|
/// [`find_attribute_in_file`](clawhdf5_format::attribute::find_attribute_in_file)).
|
||||||
|
pub(crate) fn read_attr(
|
||||||
|
file_data: &[u8],
|
||||||
|
header: &clawhdf5_format::object_header::ObjectHeader,
|
||||||
|
name: &str,
|
||||||
|
offset_size: u8,
|
||||||
|
length_size: u8,
|
||||||
|
) -> Result<Option<AttrValue>, crate::Error> {
|
||||||
|
let Some(msg) = clawhdf5_format::attribute::find_attribute_in_file(
|
||||||
|
file_data,
|
||||||
|
header,
|
||||||
|
name,
|
||||||
|
offset_size,
|
||||||
|
length_size,
|
||||||
|
)?
|
||||||
|
else {
|
||||||
|
return Ok(None);
|
||||||
|
};
|
||||||
|
Ok(attrs_to_map(
|
||||||
|
std::slice::from_ref(&msg),
|
||||||
|
file_data,
|
||||||
|
offset_size,
|
||||||
|
length_size,
|
||||||
|
)
|
||||||
|
.remove(name))
|
||||||
|
}
|
||||||
|
|
||||||
pub(crate) fn attrs_to_map(
|
pub(crate) fn attrs_to_map(
|
||||||
attrs: &[clawhdf5_format::attribute::AttributeMessage],
|
attrs: &[clawhdf5_format::attribute::AttributeMessage],
|
||||||
file_data: &[u8],
|
file_data: &[u8],
|
||||||
|
|||||||
@@ -211,6 +211,30 @@ fn dense_attribute_stored_as_a_huge_heap_object() {
|
|||||||
assert!(matches!(&attrs["bigger"], AttrValue::I64Array(v) if *v == bigger));
|
assert!(matches!(&attrs["bigger"], AttrValue::I64Array(v) if *v == bigger));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Enough huge attributes that the huge-object B-tree has internal nodes:
|
||||||
|
/// each one is found by descending it by heap ID (libhdf5 orders the
|
||||||
|
/// records of indirectly addressed huge objects by ID), and every value
|
||||||
|
/// matches what was written.
|
||||||
|
#[test]
|
||||||
|
fn many_huge_attributes_are_found_through_their_index() {
|
||||||
|
skip_if_no_python!();
|
||||||
|
let (_dir, path) = h5py_file(
|
||||||
|
"d = f.create_dataset('d', data=[1.0])\n\
|
||||||
|
for i in range(300):\n\
|
||||||
|
\x20 d.attrs['h%03d' % i] = np.arange(600, dtype='i8') + i\n",
|
||||||
|
);
|
||||||
|
let f = File::open(&path).unwrap();
|
||||||
|
let d = f.dataset("d").unwrap();
|
||||||
|
let attrs = d.attrs().unwrap();
|
||||||
|
assert_eq!(attrs.len(), 300);
|
||||||
|
for i in 0..300i64 {
|
||||||
|
let name = format!("h{i:03}");
|
||||||
|
let want: Vec<i64> = (0..600).map(|v| v + i).collect();
|
||||||
|
assert!(matches!(&attrs[&name], AttrValue::I64Array(v) if *v == want), "{name}");
|
||||||
|
assert!(matches!(d.attr(&name).unwrap(), Some(AttrValue::I64Array(v)) if v == want), "{name}");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
/// A group whose link heap has a deflate I/O filter (set on the group
|
/// A group whose link heap has a deflate I/O filter (set on the group
|
||||||
/// creation property list), with 3 000 links and one link whose message is
|
/// creation property list), with 3 000 links and one link whose message is
|
||||||
/// larger than the heap's managed-object limit, so it is a huge object.
|
/// larger than the heap's managed-object limit, so it is a huge object.
|
||||||
|
|||||||
@@ -0,0 +1,410 @@
|
|||||||
|
//! Looking one name up in a dense group (links in a fractal heap, indexed by
|
||||||
|
//! a v2 B-tree of name hashes) or in dense attribute storage reads the name
|
||||||
|
//! index, not every link: O(log n) index nodes and only the links whose
|
||||||
|
//! lookup3 hash equals the name's. Before, every lookup decoded all n links,
|
||||||
|
//! so opening each child of a 35 001-link group by name decoded ~1.2e9.
|
||||||
|
//!
|
||||||
|
//! The file is written by h5py (libhdf5 orders the index), with names whose
|
||||||
|
//! hashes collide, and every result is compared with what h5py reads.
|
||||||
|
//!
|
||||||
|
//! Skipped when python3 with h5py is unavailable, unless
|
||||||
|
//! `CLAWHDF5_REQUIRE_INTEROP=1`.
|
||||||
|
|
||||||
|
use std::collections::{BTreeMap, HashMap};
|
||||||
|
use std::process::Command;
|
||||||
|
use std::sync::OnceLock;
|
||||||
|
use std::time::{Duration, Instant};
|
||||||
|
|
||||||
|
use clawhdf5::{AttrValue, File, LazyFile, MmapFile};
|
||||||
|
use clawhdf5_format::checksum::jenkins_lookup3;
|
||||||
|
use clawhdf5_format::error::FormatError;
|
||||||
|
use clawhdf5_format::lookup_stats;
|
||||||
|
|
||||||
|
fn python() -> String {
|
||||||
|
std::env::var("CLAWHDF5_PYTHON").unwrap_or_else(|_| "python3".to_string())
|
||||||
|
}
|
||||||
|
|
||||||
|
fn interop_required() -> bool {
|
||||||
|
std::env::var("CLAWHDF5_REQUIRE_INTEROP").is_ok_and(|v| v == "1")
|
||||||
|
}
|
||||||
|
|
||||||
|
fn python_available() -> bool {
|
||||||
|
Command::new(python())
|
||||||
|
.args(["-c", "import h5py"])
|
||||||
|
.output()
|
||||||
|
.map(|o| o.status.success())
|
||||||
|
.unwrap_or(false)
|
||||||
|
}
|
||||||
|
|
||||||
|
macro_rules! skip_if_no_python {
|
||||||
|
() => {
|
||||||
|
if !python_available() {
|
||||||
|
assert!(
|
||||||
|
!interop_required(),
|
||||||
|
"CLAWHDF5_REQUIRE_INTEROP=1 but python3 with h5py is not available"
|
||||||
|
);
|
||||||
|
eprintln!("SKIP: python3 with h5py not available");
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
fn run_python(script: &str) -> String {
|
||||||
|
let output = Command::new(python())
|
||||||
|
.args(["-c", script])
|
||||||
|
.output()
|
||||||
|
.expect("failed to run python");
|
||||||
|
assert!(
|
||||||
|
output.status.success(),
|
||||||
|
"Python script failed:\nSTDOUT: {}\nSTDERR: {}",
|
||||||
|
String::from_utf8_lossy(&output.stdout),
|
||||||
|
String::from_utf8_lossy(&output.stderr)
|
||||||
|
);
|
||||||
|
String::from_utf8_lossy(&output.stdout).trim().to_string()
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Links in the big group, as libhdf5's `h5stat_newgrat.h5` has.
|
||||||
|
const LINKS: usize = 35_001;
|
||||||
|
/// Attributes on the dense-attribute dataset.
|
||||||
|
const ATTRS: usize = 3_000;
|
||||||
|
|
||||||
|
/// Pairs of distinct names with equal lookup3 hashes, found by search (the
|
||||||
|
/// hash is fixed, so the pairs are too).
|
||||||
|
fn colliding_pairs(count: usize) -> Vec<(String, String)> {
|
||||||
|
let mut seen: HashMap<u32, String> = HashMap::new();
|
||||||
|
let mut pairs = Vec::new();
|
||||||
|
for i in 0.. {
|
||||||
|
let name = format!("c{i}");
|
||||||
|
let h = jenkins_lookup3(name.as_bytes());
|
||||||
|
if let Some(first) = seen.insert(h, name.clone()) {
|
||||||
|
pairs.push((first, name));
|
||||||
|
if pairs.len() == count {
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
pairs
|
||||||
|
}
|
||||||
|
|
||||||
|
/// What h5py reads: link values, attribute values, and which of the
|
||||||
|
/// missing names it finds as links and as attributes (none).
|
||||||
|
type H5pyView = (
|
||||||
|
BTreeMap<String, i64>,
|
||||||
|
BTreeMap<String, i64>,
|
||||||
|
Vec<String>,
|
||||||
|
Vec<String>,
|
||||||
|
);
|
||||||
|
|
||||||
|
struct Fixture {
|
||||||
|
_dir: tempfile::TempDir,
|
||||||
|
path: String,
|
||||||
|
/// Names in the big group, with the value of the scalar dataset each
|
||||||
|
/// links to, as h5py reads them.
|
||||||
|
links: BTreeMap<String, i64>,
|
||||||
|
/// Names that are not links but hash like one that is.
|
||||||
|
missing_links: Vec<String>,
|
||||||
|
/// Attributes of `/x`, as h5py reads them.
|
||||||
|
attrs: BTreeMap<String, i64>,
|
||||||
|
missing_attrs: Vec<String>,
|
||||||
|
}
|
||||||
|
|
||||||
|
fn fixture() -> &'static Fixture {
|
||||||
|
static FIXTURE: OnceLock<Fixture> = OnceLock::new();
|
||||||
|
FIXTURE.get_or_init(|| {
|
||||||
|
let pairs = colliding_pairs(6);
|
||||||
|
for (a, b) in &pairs {
|
||||||
|
assert_ne!(a, b);
|
||||||
|
assert_eq!(jenkins_lookup3(a.as_bytes()), jenkins_lookup3(b.as_bytes()));
|
||||||
|
}
|
||||||
|
// Pairs 0-2 both present (either can be the one libhdf5 orders
|
||||||
|
// first), pairs 3-5 only the first: its partner must not be found.
|
||||||
|
// "k69209"/"k155448" is the pair the writer once misordered.
|
||||||
|
let mut present: Vec<String> = vec!["k69209".into(), "k155448".into()];
|
||||||
|
let mut missing: Vec<String> = Vec::new();
|
||||||
|
for (i, (a, b)) in pairs.into_iter().enumerate() {
|
||||||
|
present.push(a);
|
||||||
|
if i < 3 {
|
||||||
|
present.push(b);
|
||||||
|
} else {
|
||||||
|
missing.push(b);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
missing.extend(["", "nope", "n35001x", "N1"].map(String::from));
|
||||||
|
let mut links = present.clone();
|
||||||
|
let mut i = 0;
|
||||||
|
while links.len() < LINKS {
|
||||||
|
links.push(format!("n{i}"));
|
||||||
|
i += 1;
|
||||||
|
}
|
||||||
|
let mut attrs = present.clone();
|
||||||
|
attrs.extend((0..ATTRS - present.len()).map(|i| format!("a{i}")));
|
||||||
|
|
||||||
|
let dir = tempfile::tempdir().unwrap();
|
||||||
|
let path = dir.path().join("big.h5").display().to_string();
|
||||||
|
// The names go through a file: 35 001 of them overflow an argument.
|
||||||
|
let names = dir.path().join("names.json");
|
||||||
|
std::fs::write(
|
||||||
|
&names,
|
||||||
|
serde_json::to_string(&(&links, &attrs, &missing)).unwrap(),
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
|
let names = names.display();
|
||||||
|
let out = run_python(&format!(
|
||||||
|
"import h5py, json, numpy as np\n\
|
||||||
|
links, attrs, missing = json.load(open(r'{names}'))\n\
|
||||||
|
with h5py.File(r'{path}', 'w', libver='latest') as f:\n\
|
||||||
|
\x20 g = f.create_group('g')\n\
|
||||||
|
\x20 for i, n in enumerate(links):\n\
|
||||||
|
\x20 g.create_dataset(n, data=np.int64(i))\n\
|
||||||
|
\x20 x = f.create_dataset('x', data=np.int64(0))\n\
|
||||||
|
\x20 for i, n in enumerate(attrs):\n\
|
||||||
|
\x20 x.attrs[n] = np.int64(1000 + i)\n\
|
||||||
|
with h5py.File(r'{path}', 'r') as f:\n\
|
||||||
|
\x20 g, a = f['g'], f['x'].attrs\n\
|
||||||
|
\x20 print(json.dumps([{{n: int(g[n][()]) for n in g}}, {{n: int(a[n]) for n in a}},\n\
|
||||||
|
\x20 [n for n in missing if n and n in g], [n for n in missing if n and n in a]]))",
|
||||||
|
));
|
||||||
|
let (links, attrs, found_links, found_attrs): H5pyView =
|
||||||
|
serde_json::from_str(&out).unwrap();
|
||||||
|
assert_eq!(links.len(), LINKS);
|
||||||
|
assert_eq!(attrs.len(), ATTRS);
|
||||||
|
assert!(found_links.is_empty() && found_attrs.is_empty());
|
||||||
|
Fixture {
|
||||||
|
_dir: dir,
|
||||||
|
path,
|
||||||
|
links,
|
||||||
|
missing_links: missing.clone(),
|
||||||
|
attrs,
|
||||||
|
missing_attrs: missing,
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
fn is_not_found(e: &clawhdf5::Error) -> bool {
|
||||||
|
matches!(e, clawhdf5::Error::Format(FormatError::PathNotFound(_)))
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn one_link_lookup_reads_the_index_not_every_link() {
|
||||||
|
skip_if_no_python!();
|
||||||
|
let fx = fixture();
|
||||||
|
let f = File::open(&fx.path).unwrap();
|
||||||
|
let g = f.group("g").unwrap();
|
||||||
|
for (name, value) in &fx.links {
|
||||||
|
lookup_stats::reset();
|
||||||
|
let ds = g.dataset(name).unwrap();
|
||||||
|
// One link decoded per lookup, two where hashes collide — not 35 001.
|
||||||
|
let read = lookup_stats::heap_objects_read();
|
||||||
|
assert!(read <= 2, "looking up {name} read {read} heap objects");
|
||||||
|
assert_eq!(ds.read_i64().unwrap(), vec![*value], "{name}");
|
||||||
|
}
|
||||||
|
|
||||||
|
for name in &fx.missing_links {
|
||||||
|
lookup_stats::reset();
|
||||||
|
let err = g.dataset(name).unwrap_err();
|
||||||
|
assert!(is_not_found(&err), "{name:?}: {err:?}");
|
||||||
|
assert!(lookup_stats::heap_objects_read() <= 2, "{name:?}");
|
||||||
|
}
|
||||||
|
|
||||||
|
// A path resolves each component the same way.
|
||||||
|
for name in ["k155448", "n0", "n34000"] {
|
||||||
|
lookup_stats::reset();
|
||||||
|
let ds = f.dataset(&format!("/g/{name}")).unwrap();
|
||||||
|
assert!(lookup_stats::heap_objects_read() <= 2);
|
||||||
|
assert_eq!(ds.read_i64().unwrap(), vec![fx.links[name]]);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn one_attribute_lookup_reads_the_index_not_every_attribute() {
|
||||||
|
skip_if_no_python!();
|
||||||
|
let fx = fixture();
|
||||||
|
let f = File::open(&fx.path).unwrap();
|
||||||
|
let x = f.dataset("x").unwrap();
|
||||||
|
let all = x.attrs().unwrap();
|
||||||
|
assert_eq!(all.len(), ATTRS);
|
||||||
|
for (name, value) in &fx.attrs {
|
||||||
|
lookup_stats::reset();
|
||||||
|
let got = x.attr(name).unwrap();
|
||||||
|
assert!(lookup_stats::heap_objects_read() <= 2, "{name}");
|
||||||
|
assert!(
|
||||||
|
matches!(got, Some(AttrValue::I64(v)) if v == *value),
|
||||||
|
"{name}: {got:?}"
|
||||||
|
);
|
||||||
|
assert!(matches!(all.get(name), Some(AttrValue::I64(v)) if v == value));
|
||||||
|
}
|
||||||
|
for name in &fx.missing_attrs {
|
||||||
|
lookup_stats::reset();
|
||||||
|
assert!(x.attr(name).unwrap().is_none(), "{name:?}");
|
||||||
|
assert!(lookup_stats::heap_objects_read() <= 2, "{name:?}");
|
||||||
|
}
|
||||||
|
// Compact attributes (on the root group: none) and a group's attributes.
|
||||||
|
assert!(f.root().attr("k69209").unwrap().is_none());
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Every child of the big group opened by name through each file type,
|
||||||
|
/// within `limit`: with a scan per lookup this is ~1.2e9 link decodes.
|
||||||
|
#[test]
|
||||||
|
fn opening_every_child_of_a_35001_link_group_by_name_is_quick() {
|
||||||
|
skip_if_no_python!();
|
||||||
|
let fx = fixture();
|
||||||
|
let limit = Duration::from_secs(120);
|
||||||
|
let started = Instant::now();
|
||||||
|
let check_time = |n: usize| {
|
||||||
|
assert!(
|
||||||
|
started.elapsed() < limit,
|
||||||
|
"{n} lookups took {:?}",
|
||||||
|
started.elapsed()
|
||||||
|
);
|
||||||
|
};
|
||||||
|
|
||||||
|
let f = File::open(&fx.path).unwrap();
|
||||||
|
let g = f.group("g").unwrap();
|
||||||
|
for (n, (name, value)) in fx.links.iter().enumerate() {
|
||||||
|
assert_eq!(g.dataset(name).unwrap().read_i64().unwrap(), vec![*value]);
|
||||||
|
check_time(n);
|
||||||
|
}
|
||||||
|
// The listing hands out entries: open each by address.
|
||||||
|
let entries = g.entries().unwrap();
|
||||||
|
assert_eq!(entries.len(), LINKS);
|
||||||
|
for (name, address) in &entries {
|
||||||
|
let ds = f.dataset_at(*address).unwrap();
|
||||||
|
assert_eq!(ds.read_i64().unwrap(), vec![fx.links[name]]);
|
||||||
|
}
|
||||||
|
assert!(f.group_at(g_address(&f)).dataset("n0").is_ok());
|
||||||
|
|
||||||
|
let m = MmapFile::open(&fx.path).unwrap();
|
||||||
|
let mg = m.group("g").unwrap();
|
||||||
|
for (n, (name, value)) in fx.links.iter().enumerate() {
|
||||||
|
assert_eq!(mg.dataset(name).unwrap().read_i64().unwrap(), vec![*value]);
|
||||||
|
check_time(n);
|
||||||
|
}
|
||||||
|
assert!(mg.group("nope").is_err_and(|e| is_not_found(&e)));
|
||||||
|
|
||||||
|
let l = LazyFile::open_mmap(&fx.path).unwrap();
|
||||||
|
let lg = l.group("g").unwrap();
|
||||||
|
for (n, (name, value)) in fx.links.iter().enumerate() {
|
||||||
|
assert_eq!(lg.dataset(name).unwrap().read_i64().unwrap(), vec![*value]);
|
||||||
|
check_time(n);
|
||||||
|
}
|
||||||
|
let lx = l.dataset("x").unwrap();
|
||||||
|
assert!(
|
||||||
|
matches!(lx.attr("k155448").unwrap(), Some(AttrValue::I64(v)) if v == fx.attrs["k155448"])
|
||||||
|
);
|
||||||
|
assert!(
|
||||||
|
lg.dataset(&fx.missing_links[0])
|
||||||
|
.is_err_and(|e| is_not_found(&e))
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
fn g_address(f: &File) -> u64 {
|
||||||
|
f.root()
|
||||||
|
.entries()
|
||||||
|
.unwrap()
|
||||||
|
.into_iter()
|
||||||
|
.find(|(n, _)| n == "g")
|
||||||
|
.unwrap()
|
||||||
|
.1
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Every kind of link, looked up by name in a dense group (through the name
|
||||||
|
/// index) and in a compact one, opens what h5py opens and nothing it cannot:
|
||||||
|
/// hard links, soft links (absolute, relative, to a group), and not a
|
||||||
|
/// dangling soft link, an external link or a missing name.
|
||||||
|
#[test]
|
||||||
|
fn links_of_every_kind_resolve_by_name_as_in_h5py() {
|
||||||
|
skip_if_no_python!();
|
||||||
|
let dir = tempfile::tempdir().unwrap();
|
||||||
|
let path = dir.path().join("links.h5").display().to_string();
|
||||||
|
// For each group and name: "dataset <value>", "group", or "none" as
|
||||||
|
// h5py sees it.
|
||||||
|
let out = run_python(&format!(
|
||||||
|
"import h5py, json, numpy as np\n\
|
||||||
|
with h5py.File(r'{path}', 'w', libver='latest') as f:\n\
|
||||||
|
\x20 for gname, n in (('dense', 20), ('compact', 2)):\n\
|
||||||
|
\x20 g = f.create_group(gname)\n\
|
||||||
|
\x20 for i in range(n):\n\
|
||||||
|
\x20 g.create_dataset(f'd{{i}}', data=np.int64(100 + i))\n\
|
||||||
|
\x20 s = g.create_group('sub')\n\
|
||||||
|
\x20 s.create_dataset('x', data=np.int64(7))\n\
|
||||||
|
\x20 g['abs'] = h5py.SoftLink(f'/{{gname}}/d1')\n\
|
||||||
|
\x20 g['rel'] = h5py.SoftLink('sub/x')\n\
|
||||||
|
\x20 g['tosub'] = h5py.SoftLink('sub')\n\
|
||||||
|
\x20 g['dangling'] = h5py.SoftLink('/nowhere')\n\
|
||||||
|
\x20 g['ext'] = h5py.ExternalLink('other.h5', '/y')\n\
|
||||||
|
names = ['d0', 'd1', 'sub', 'abs', 'rel', 'tosub', 'dangling', 'ext', 'nope', '']\n\
|
||||||
|
seen = {{}}\n\
|
||||||
|
with h5py.File(r'{path}', 'r') as f:\n\
|
||||||
|
\x20 for gname in ('dense', 'compact'):\n\
|
||||||
|
\x20 g = f[gname]\n\
|
||||||
|
\x20 for n in names:\n\
|
||||||
|
\x20 try:\n\
|
||||||
|
\x20 o = g[n] if n else None\n\
|
||||||
|
\x20 except (KeyError, OSError):\n\
|
||||||
|
\x20 o = None\n\
|
||||||
|
\x20 if isinstance(o, h5py.Dataset):\n\
|
||||||
|
\x20 seen[f'{{gname}}/{{n}}'] = f'dataset {{int(o[()])}}'\n\
|
||||||
|
\x20 elif isinstance(o, h5py.Group):\n\
|
||||||
|
\x20 seen[f'{{gname}}/{{n}}'] = 'group'\n\
|
||||||
|
\x20 else:\n\
|
||||||
|
\x20 seen[f'{{gname}}/{{n}}'] = 'none'\n\
|
||||||
|
print(json.dumps(seen))",
|
||||||
|
));
|
||||||
|
let seen: BTreeMap<String, String> = serde_json::from_str(&out).unwrap();
|
||||||
|
assert_eq!(seen.len(), 20);
|
||||||
|
|
||||||
|
let f = File::open(&path).unwrap();
|
||||||
|
// The dense group's links are in a heap, the compact group's in its
|
||||||
|
// header.
|
||||||
|
for (gname, dense) in [("dense", true), ("compact", false)] {
|
||||||
|
let g = f.group(gname).unwrap();
|
||||||
|
lookup_stats::reset();
|
||||||
|
g.dataset("d0").unwrap();
|
||||||
|
assert_eq!(lookup_stats::heap_objects_read() > 0, dense, "{gname}");
|
||||||
|
}
|
||||||
|
let m = MmapFile::open(&path).unwrap();
|
||||||
|
let l = LazyFile::open_mmap(&path).unwrap();
|
||||||
|
for (key, want) in &seen {
|
||||||
|
let (gname, name) = key.split_once('/').unwrap();
|
||||||
|
let got = {
|
||||||
|
let g = f.group(gname).unwrap();
|
||||||
|
match (g.dataset(name), g.group(name)) {
|
||||||
|
(Ok(ds), _) => format!("dataset {}", ds.read_i64().unwrap()[0]),
|
||||||
|
(Err(clawhdf5::Error::NotADataset(_)), Ok(sub)) => {
|
||||||
|
// A group: it has the child `x` (checks the address).
|
||||||
|
assert!(sub.dataset("x").is_ok() || name == "sub" || name == "tosub");
|
||||||
|
"group".to_string()
|
||||||
|
}
|
||||||
|
(Err(e), Err(e2)) => {
|
||||||
|
assert!(is_not_found(&e) && is_not_found(&e2), "{key}: {e:?} / {e2:?}");
|
||||||
|
"none".to_string()
|
||||||
|
}
|
||||||
|
(Err(e), Ok(_)) => panic!("{key}: dataset {e:?} but group ok"),
|
||||||
|
}
|
||||||
|
};
|
||||||
|
assert_eq!(&got, want, "{key}");
|
||||||
|
// The other readers agree, and a path through the group resolves the
|
||||||
|
// same way.
|
||||||
|
let mg = m.group(gname).unwrap();
|
||||||
|
let lg = l.group(gname).unwrap();
|
||||||
|
match want.strip_prefix("dataset ") {
|
||||||
|
Some(v) => {
|
||||||
|
let v: i64 = v.parse().unwrap();
|
||||||
|
assert_eq!(mg.dataset(name).unwrap().read_i64().unwrap(), vec![v]);
|
||||||
|
assert_eq!(lg.dataset(name).unwrap().read_i64().unwrap(), vec![v]);
|
||||||
|
let ds = f.dataset(&format!("/{gname}/{name}")).unwrap();
|
||||||
|
assert_eq!(ds.read_i64().unwrap(), vec![v], "{key}");
|
||||||
|
}
|
||||||
|
None if want == "group" => {
|
||||||
|
assert!(mg.group(name).unwrap().dataset("x").is_ok(), "{key}");
|
||||||
|
assert!(lg.group(name).unwrap().dataset("x").is_ok(), "{key}");
|
||||||
|
let ds = f.dataset(&format!("/{gname}/{name}/x")).unwrap();
|
||||||
|
assert_eq!(ds.read_i64().unwrap(), vec![7]);
|
||||||
|
}
|
||||||
|
None => {
|
||||||
|
assert!(mg.dataset(name).is_err_and(|e| is_not_found(&e)), "{key}");
|
||||||
|
assert!(lg.group(name).is_err_and(|e| is_not_found(&e)), "{key}");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
Reference in New Issue
Block a user