quantumclaw/clawhdf5
3
1
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 1 Wiki Activity

Merge branch 'feat/p2b-writer-btree-internal-nodes' into feat/p2b-scale

Browse Source 
# Conflicts:
#	CHANGELOG.md
This commit is contained in:
osobh
2026-09-26 11:57:11 -05:00
parent bdb2c0e36b 193a5f8a82
commit d7f07fa5c1
15 changed files with 1587 additions and 292 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/clawhdf5-format/src/attribute.rs
+30 -5
View File
@@ -450,6 +450,8 @@ fn extract_attributes_with(
on_error: &mut dyn FnMut(FormatError) -> Result<(), FormatError>,
) -> Result<Vec<AttributeMessage>, FormatError> {
let mut attrs = Vec::new();
// Each attribute's creation order, where the file records one.
let mut orders: Vec<u32> = Vec::new();
// Collect compact attributes (inline in OH)
for msg in &header.messages {
@@ -479,7 +481,10 @@ fn extract_attributes_with(
};
let attr = attr.and_then(|a| check_in_header(a, header));
match attr {
Ok(attr) => attrs.push(attr),
Ok(attr) => {
attrs.push(attr);
orders.push(msg.creation_order.map_or(0, u32::from));
}
Err(e) => on_error(e)?,
}
}
@@ -487,20 +492,30 @@ fn extract_attributes_with(
// Check for dense attributes via AttributeInfo message
let attr_info = find_attribute_info(header, offset_size)?;
if let Some(info) = attr_info
if let Some(info) = &attr_info
&& let Some(fh_addr) = info.fractal_heap_address
{
extract_dense_attributes(
file_data,
&info,
info,
fh_addr,
offset_size,
length_size,
&mut attrs,
&mut orders,
on_error,
)?;
}
// An object that tracks attribute creation order lists its attributes
// in that order (h5py's `track_order=True`), as libhdf5 does; otherwise
// they come in storage order.
if attr_info.is_some_and(|i| i.max_creation_index.is_some()) {
let mut paired: Vec<(u32, AttributeMessage)> = orders.into_iter().zip(attrs).collect();
paired.sort_by_key(|(o, _)| *o);
attrs = paired.into_iter().map(|(_, a)| a).collect();
}
Ok(attrs)
}
@@ -518,7 +533,9 @@ fn find_attribute_info(
Ok(None)
}
/// Extract attributes from dense storage (fractal heap + B-tree v2).
/// Extract attributes from dense storage (fractal heap + B-tree v2), and
/// each one's creation order into `orders`.
#[allow(clippy::too_many_arguments)]
fn extract_dense_attributes(
file_data: &[u8],
attr_info: &AttributeInfoMessage,
@@ -526,6 +543,7 @@ fn extract_dense_attributes(
offset_size: u8,
length_size: u8,
attrs: &mut Vec<AttributeMessage>,
orders: &mut Vec<u32>,
on_error: &mut dyn FnMut(FormatError) -> Result<(), FormatError>,
) -> Result<(), FormatError> {
// Parse fractal heap
@@ -561,7 +579,14 @@ fn extract_dense_attributes(
AttributeMessage::parse_in_file(&attr_data, file_data, offset_size, length_size)
});
match attr {
Ok(attr) => attrs.push(attr),
Ok(attr) => {
attrs.push(attr);
let order = record
.data
.get(id_len + 1..id_len + 5)
.map_or(0, |b| u32::from_le_bytes([b[0], b[1], b[2], b[3]]));
orders.push(order);
}
Err(e) => on_error(e)?,
}
}
crates/clawhdf5-format/src/btree_v2.rs
+68 -13
View File
@@ -71,7 +71,7 @@ fn ensure_len(data: &[u8], pos: usize, needed: usize) -> Result<(), FormatError>
/// Compute the number of bytes needed to represent a count, using variable-width encoding.
/// B-tree v2 uses this for the number of records fields in internal nodes.
fn bytes_for_max_records(max_nrec: u64) -> usize {
pub(crate) fn bytes_for_max_records(max_nrec: u64) -> usize {
if max_nrec == 0 {
return 1;
}
@@ -163,7 +163,7 @@ impl BTreeV2Header {
/// Compute maximum records per node for a given depth level.
/// leaf: (node_size - overhead) / record_size
/// internal: depends on pointers
fn max_records_leaf(node_size: u32, record_size: u16) -> u64 {
pub(crate) fn max_records_leaf(node_size: u32, record_size: u16) -> u64 {
// Leaf overhead: signature(4) + version(1) + type(1) + checksum(4) = 10
let overhead = 10u32;
if node_size <= overhead || record_size == 0 {
@@ -418,10 +418,7 @@ fn collect_internal_records(
}
/// Most records a subtree whose root is at `depth` can hold (libhdf5's
/// `cum_max_nrec`): a leaf holds `max_leaf_nrec`; an internal node at depth
/// `d` holds `max_nrec(d)` records and `max_nrec(d) + 1` subtrees of depth
/// `d - 1`, where `max_nrec(d)` is what fits in a node once each record is
/// paired with a child pointer of the width depth `d` needs.
/// `cum_max_nrec`). See [`node_info`].
fn cum_max_records(
node_size: u32,
record_size: u16,
@@ -429,24 +426,82 @@ fn cum_max_records(
max_leaf_nrec: u64,
depth: u16,
) -> u64 {
node_info_from_leaf(node_size, record_size, offset_size, max_leaf_nrec, depth)
.last()
.map_or(max_leaf_nrec, |n| n.cum_max_nrec)
}
/// Capacity of a B-tree v2 node at one depth, as libhdf5 computes it
/// (`H5B2__hdr_init`'s `node_info`).
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub(crate) struct NodeInfo {
/// Most records one node at this depth holds.
pub(crate) max_nrec: u64,
/// Most records a subtree rooted at this depth holds.
pub(crate) cum_max_nrec: u64,
/// Bytes a subtree's total record count takes in a pointer to a node
/// at this depth (0 for a leaf, whose count is its own).
pub(crate) cum_max_nrec_size: usize,
}
/// Node capacities for depths `0..=depth` (entry `d` for depth `d`): a leaf
/// holds `max_nrec(0)` records; an internal node at depth `d` holds
/// `max_nrec(d)` records and `max_nrec(d) + 1` subtrees of depth `d - 1`,
/// where `max_nrec(d)` is what fits in a node once each record is paired
/// with a child pointer of the width depth `d` needs (address, the child's
/// record count in the width a *leaf's* maximum needs, and below the first
/// internal level the child subtree's total in the width its maximum
/// needs), with one pointer more than records.
pub(crate) fn node_info(
node_size: u32,
record_size: u16,
offset_size: u8,
depth: u16,
) -> Vec<NodeInfo> {
let max_leaf = max_records_leaf(node_size, record_size);
node_info_from_leaf(node_size, record_size, offset_size, max_leaf, depth)
}
fn node_info_from_leaf(
node_size: u32,
record_size: u16,
offset_size: u8,
max_leaf_nrec: u64,
depth: u16,
) -> Vec<NodeInfo> {
// Internal node overhead: signature(4) + version(1) + type(1) + checksum(4).
const PREFIX: u64 = 10;
let nrec_width = bytes_for_max_records(max_leaf_nrec) as u64;
let mut cum = max_leaf_nrec;
let mut cum_width = 0u64;
let mut info = Vec::with_capacity(usize::from(depth) + 1);
info.push(NodeInfo {
max_nrec: max_leaf_nrec,
cum_max_nrec: max_leaf_nrec,
cum_max_nrec_size: 0,
});
for d in 1..=depth {
let ptr = u64::from(offset_size) + nrec_width + if d > 1 { cum_width } else { 0 };
let below = info[usize::from(d) - 1];
let ptr = u64::from(offset_size)
+ nrec_width
+ if d > 1 {
below.cum_max_nrec_size as u64
} else {
0
};
let max_nrec = u64::from(node_size)
.saturating_sub(PREFIX)
.saturating_sub(ptr)
/ (u64::from(record_size) + ptr).max(1);
cum = max_nrec
let cum = max_nrec
.saturating_add(1)
.saturating_mul(cum)
.saturating_mul(below.cum_max_nrec)
.saturating_add(max_nrec);
cum_width = bytes_for_max_records(cum) as u64;
info.push(NodeInfo {
max_nrec,
cum_max_nrec: cum,
cum_max_nrec_size: bytes_for_max_records(cum),
});
}
cum
info
}
#[cfg(test)]
crates/clawhdf5-format/src/btree_v2_write.rs
+396
View File
@@ -0,0 +1,396 @@
//! Writing version-2 B-trees: a header (`BTHD`) and its nodes, leaves
//! (`BTLF`) and, for more records than one leaf holds, internal nodes
//! (`BTIN`) to any depth.
//!
//! Node capacities come from [`crate::btree_v2::node_info`], the arithmetic
//! libhdf5 uses (`H5B2__hdr_init`) and the reader decodes pointers with, so
//! the pointer widths the writer encodes are the ones every reader expects.
#[cfg(not(feature = "std"))]
use alloc::{format, vec, vec::Vec};
use crate::btree_v2::{NodeInfo, bytes_for_max_records, node_info};
use crate::checksum::jenkins_lookup3;
use crate::error::FormatError;
/// How a B-tree is laid out: its record type and node geometry, as the
/// header records them.
#[derive(Debug, Clone, Copy)]
pub(crate) struct BTreeV2Params {
/// Record type (5: link names, 6: link creation order, 8: attribute
/// names, 9: attribute creation order, 10/11: chunks).
pub(crate) tree_type: u8,
/// Bytes per node.
pub(crate) node_size: u32,
/// Bytes per record.
pub(crate) record_size: u16,
/// Split and merge percentages. The writer fills nodes itself; these
/// only tell libhdf5 when to split and merge as it modifies the tree.
pub(crate) split_percent: u8,
pub(crate) merge_percent: u8,
}
/// Size of a B-tree v2 header.
pub(crate) fn header_size(offset_size: u8, length_size: u8) -> usize {
4 + 1 + 1 + 4 + 2 + 2 + 1 + 1 + offset_size as usize + 2 + length_size as usize + 4
}
/// Deepest tree the writer builds. Even at the smallest fan-out libhdf5's
/// arithmetic allows, a few levels hold more records than any file could.
const MAX_WRITE_DEPTH: u16 = 32;
/// Write a B-tree v2 holding `records` (`record_size` bytes each,
/// concatenated, already in the tree's key order) at `addr`: the header,
/// then its nodes, each `node_size` bytes. No records gives a header with
/// an undefined root.
///
/// The tree is as shallow as the node size allows: a single leaf when the
/// records fit one, otherwise internal nodes above leaves. Records are
/// spread evenly over each node's children, so every node but the root is
/// at least about half full (above libhdf5's merge threshold, which is below
/// half), and each node holds at most its depth's maximum.
pub(crate) fn build_btree_v2(
p: BTreeV2Params,
records: &[u8],
addr: u64,
offset_size: u8,
length_size: u8,
) -> Result<Vec<u8>, FormatError> {
let rs = usize::from(p.record_size);
if rs == 0 || !records.len().is_multiple_of(rs) {
return Err(FormatError::SerializationError(format!(
"B-tree v2 records are {} bytes, not a multiple of the record size {rs}",
records.len()
)));
}
let n = (records.len() / rs) as u64;
let hdr_len = header_size(offset_size, length_size);
// The shallowest depth whose subtree can hold every record.
let mut info = node_info(p.node_size, p.record_size, offset_size, 0);
let max_leaf = info[0].max_nrec;
if max_leaf == 0 || max_leaf > u64::from(u16::MAX) {
return Err(FormatError::SerializationError(format!(
"a {}-byte B-tree v2 node holds {max_leaf} {}-byte records; \
a node holds 1 to 65535",
p.node_size, p.record_size
)));
}
let mut depth = 0u16;
while info[usize::from(depth)].cum_max_nrec < n {
depth += 1;
if depth > MAX_WRITE_DEPTH {
return Err(FormatError::SerializationError(format!(
"{n} records do not fit a B-tree v2 of {}-byte nodes",
p.node_size
)));
}
info = node_info(p.node_size, p.record_size, offset_size, depth);
let max = info[usize::from(depth)].max_nrec;
if max == 0 || max > u64::from(u16::MAX) {
return Err(FormatError::SerializationError(format!(
"a {}-byte B-tree v2 internal node holds {max} records; \
a node holds 1 to 65535",
p.node_size
)));
}
}
let mut w = TreeWriter {
p,
records,
info: &info,
nrec_width: bytes_for_max_records(max_leaf),
offset_size,
first_node: addr + hdr_len as u64,
nodes: Vec::new(),
};
let root = (n > 0).then(|| w.node(depth, 0, n as usize)).transpose()?;
let mut out = Vec::with_capacity(hdr_len + w.nodes.len() * p.node_size as usize);
out.extend_from_slice(b"BTHD");
out.push(0); // version
out.push(p.tree_type);
out.extend_from_slice(&p.node_size.to_le_bytes());
out.extend_from_slice(&p.record_size.to_le_bytes());
out.extend_from_slice(&depth.to_le_bytes());
out.push(p.split_percent);
out.push(p.merge_percent);
match root {
Some(r) => push_uint(&mut out, r.addr, offset_size as usize),
None => out.extend(core::iter::repeat_n(0xFF, offset_size as usize)),
}
let root_nrec = root.map_or(0, |r| r.nrec);
out.extend_from_slice(&(root_nrec as u16).to_le_bytes());
push_uint(&mut out, n, length_size as usize);
let sum = jenkins_lookup3(&out);
out.extend_from_slice(&sum.to_le_bytes());
debug_assert_eq!(out.len(), hdr_len);
for node in &w.nodes {
out.extend_from_slice(node);
}
Ok(out)
}
/// A written node, as its parent points at it.
#[derive(Debug, Clone, Copy)]
struct NodeRef {
addr: u64,
/// Records in the node itself.
nrec: u64,
/// Records in the subtree it roots.
all_nrec: u64,
}
struct TreeWriter<'a> {
p: BTreeV2Params,
records: &'a [u8],
info: &'a [NodeInfo],
/// Width of a child's record count: what a leaf's maximum needs.
nrec_width: usize,
offset_size: u8,
/// Address of the first node (right after the header).
first_node: u64,
/// Nodes in file order (children before their parent).
nodes: Vec<Vec<u8>>,
}
impl TreeWriter<'_> {
fn record(&self, i: usize) -> &[u8] {
let rs = usize::from(self.p.record_size);
&self.records[i * rs..(i + 1) * rs]
}
fn push_node(&mut self, mut node: Vec<u8>) -> u64 {
// The checksum covers the node up to it, not the padding after.
let sum = jenkins_lookup3(&node);
node.extend_from_slice(&sum.to_le_bytes());
debug_assert!(node.len() <= self.p.node_size as usize);
node.resize(self.p.node_size as usize, 0);
let addr = self.first_node + self.nodes.len() as u64 * u64::from(self.p.node_size);
self.nodes.push(node);
addr
}
/// Write the subtree of `depth` holding records `first..first + n`.
fn node(&mut self, depth: u16, first: usize, n: usize) -> Result<NodeRef, FormatError> {
let rs = usize::from(self.p.record_size);
let mut node = Vec::with_capacity(self.p.node_size as usize);
if depth == 0 {
debug_assert!(n as u64 <= self.info[0].max_nrec);
node.extend_from_slice(b"BTLF");
node.push(0); // version
node.push(self.p.tree_type);
node.extend_from_slice(&self.records[first * rs..(first + n) * rs]);
let addr = self.push_node(node);
return Ok(NodeRef {
addr,
nrec: n as u64,
all_nrec: n as u64,
});
}
// As few children as hold the records, at least two, with the
// records spread evenly: `k` children and `k - 1` records between
// them.
let below = self.info[usize::from(depth) - 1].cum_max_nrec;
let k = (n as u64 + 1).div_ceil(below + 1).max(2);
let max = self.info[usize::from(depth)].max_nrec;
if k - 1 > max || (n as u64) < k - 1 + k {
return Err(FormatError::SerializationError(format!(
"cannot spread {n} B-tree v2 records over {k} children at depth {depth}"
)));
}
let k = k as usize;
let in_children = n - (k - 1);
let (base, extra) = (in_children / k, in_children % k);
let mut children = Vec::with_capacity(k);
let mut separators = Vec::with_capacity(k - 1);
let mut next = first;
for c in 0..k {
let m = base + usize::from(c < extra);
children.push(self.node(depth - 1, next, m)?);
next += m;
if c + 1 < k {
separators.push(next);
next += 1;
}
}
debug_assert_eq!(next, first + n);
node.extend_from_slice(b"BTIN");
node.push(0); // version
node.push(self.p.tree_type);
for &s in &separators {
node.extend_from_slice(self.record(s));
}
let total_width = if depth > 1 {
self.info[usize::from(depth) - 1].cum_max_nrec_size
} else {
0
};
for c in &children {
push_uint(&mut node, c.addr, self.offset_size as usize);
push_uint(&mut node, c.nrec, self.nrec_width);
if depth > 1 {
push_uint(&mut node, c.all_nrec, total_width);
}
}
let addr = self.push_node(node);
Ok(NodeRef {
addr,
nrec: (k - 1) as u64,
all_nrec: n as u64,
})
}
}
/// Append `v` as a `width`-byte little-endian integer.
fn push_uint(buf: &mut Vec<u8>, v: u64, width: usize) {
let bytes = v.to_le_bytes();
buf.extend_from_slice(&bytes[..width.min(8)]);
buf.extend(vec![0u8; width.saturating_sub(8)]);
}
#[cfg(test)]
mod tests {
use super::*;
use crate::btree_v2::{BTreeV2Header, collect_btree_v2_records};
fn params(node_size: u32, record_size: u16) -> BTreeV2Params {
BTreeV2Params {
tree_type: 5,
node_size,
record_size,
split_percent: 100,
merge_percent: 40,
}
}
/// `n` 11-byte records: a big-endian counter, so byte order is key order.
fn records(n: usize, rs: usize) -> Vec<u8> {
let mut out = Vec::with_capacity(n * rs);
for i in 0..n {
let mut r = vec![0u8; rs];
r[..8].copy_from_slice(&(i as u64).to_be_bytes());
out.extend_from_slice(&r);
}
out
}
fn roundtrip(node_size: u32, rs: u16, n: usize, os: u8, ls: u8) -> BTreeV2Header {
let recs = records(n, usize::from(rs));
let base = 4096u64;
let tree = build_btree_v2(params(node_size, rs), &recs, base, os, ls).unwrap();
let mut file = vec![0u8; base as usize];
file.extend_from_slice(&tree);
let hdr = BTreeV2Header::parse(&file, base as usize, os, ls).unwrap();
assert_eq!(hdr.total_records, n as u64);
let got = collect_btree_v2_records(&file, &hdr, os, ls).unwrap();
assert_eq!(got.len(), n);
let flat: Vec<u8> = got.into_iter().flat_map(|r| r.data).collect();
assert_eq!(flat, recs, "node {node_size} rs {rs} n {n}");
hdr
}
#[test]
fn one_leaf_then_deeper_trees_read_back_in_order() {
// 512-byte nodes of 11-byte records: 45 per leaf, 1149 at depth 1,
// 26 449 at depth 2.
let info = node_info(512, 11, 8, 3);
assert_eq!(
info.iter().map(|i| i.cum_max_nrec).collect::<Vec<_>>(),
[45, 1149, 26_449, 608_349]
);
for (n, depth) in [
(0, 0),
(1, 0),
(45, 0),
(46, 1),
(1149, 1),
(1150, 2),
(26_449, 2),
(26_450, 3),
(100_000, 3),
] {
let hdr = roundtrip(512, 11, n, 8, 8);
assert_eq!(hdr.depth, depth, "{n} records");
}
}
#[test]
fn pointer_widths_follow_the_offset_and_length_sizes() {
for (os, ls) in [(4, 4), (8, 4), (4, 8), (2, 2)] {
roundtrip(512, 11, 5000, os, ls);
}
// Wide counts: a leaf of 2048 bytes / 9-byte records (226, one byte)
// and deeper subtree totals of three bytes.
roundtrip(2048, 9, 300_000, 8, 8);
}
#[test]
fn every_node_is_within_its_capacity_and_above_the_merge_threshold() {
let rs = 17u16;
let n = 70_000usize;
let info = node_info(512, rs, 8, 3);
let recs = records(n, usize::from(rs));
let tree = build_btree_v2(params(512, rs), &recs, 0, 8, 8).unwrap();
let hdr_len = header_size(8, 8);
let nodes = (tree.len() - hdr_len) / 512;
for i in 0..nodes {
let node = &tree[hdr_len + i * 512..hdr_len + (i + 1) * 512];
let sig = &node[..4];
if sig == b"BTLF" {
continue; // counts checked through the parents below
}
assert_eq!(sig, b"BTIN");
}
// Walk from the header: each child's count within [40%, 100%].
let hdr = BTreeV2Header::parse(&tree, 0, 8, 8).unwrap();
assert_eq!(hdr.depth, 3);
assert!(u64::from(hdr.num_records_in_root) <= info[3].max_nrec);
fn walk(tree: &[u8], addr: usize, nrec: usize, depth: usize, info: &[NodeInfo], rs: usize) {
if depth == 0 {
return;
}
let nrec_w = bytes_for_max_records(info[0].max_nrec);
let tot_w = if depth > 1 {
info[depth - 1].cum_max_nrec_size
} else {
0
};
let mut pos = addr + 6 + nrec * rs;
for _ in 0..=nrec {
let a = u64::from_le_bytes(tree[pos..pos + 8].try_into().unwrap()) as usize;
pos += 8;
let mut c = 0usize;
for b in 0..nrec_w {
c |= usize::from(tree[pos + b]) << (8 * b);
}
pos += nrec_w + tot_w;
let max = info[depth - 1].max_nrec as usize;
assert!(c <= max && c * 100 > max * 40, "{c} of {max}");
walk(tree, a, c, depth - 1, info, rs);
}
}
walk(
&tree,
hdr.root_node_address as usize,
usize::from(hdr.num_records_in_root),
3,
&info,
usize::from(rs),
);
assert!(nodes > 0);
}
#[test]
fn a_node_too_small_or_too_big_is_an_error() {
assert!(build_btree_v2(params(16, 11), &records(1, 11), 0, 8, 8).is_err());
// A leaf with room for more than 65 535 records.
assert!(build_btree_v2(params(1 << 20, 11), &records(1, 11), 0, 8, 8).is_err());
// Records that are not whole.
assert!(build_btree_v2(params(512, 11), &[0u8; 12], 0, 8, 8).is_err());
}
}
crates/clawhdf5-format/src/chunked_write.rs
+34 -56
View File
@@ -6,6 +6,7 @@ extern crate alloc;
#[cfg(not(feature = "std"))]
use alloc::{format, vec, vec::Vec};
use crate::btree_v2_write::{BTreeV2Params, build_btree_v2};
use crate::checksum::jenkins_lookup3;
use crate::chunk_cache::{CACHE_LINE_SIZE, align_to_cache_line};
use crate::chunk_grid::ChunkGrid;
@@ -1105,11 +1106,12 @@ const BT2_CHUNK_FILTERED: u8 = 11;
///
/// `records` are `(scaled coordinates, chunk)` in lexicographic order of the
/// coordinates, which is the order the library's comparator
/// (`H5VM_vector_cmp_u`) keeps them in. The tree is a single leaf: the
/// library's 2048-byte node when the records fit, otherwise a leaf node
/// sized to hold them all (the root's record count is 16-bit, so at most
/// 65535 chunks). Returns the bytes and the node size the layout message
/// must record.
/// (`H5VM_vector_cmp_u`) keeps them in. Up to 65 535 chunks go in a single
/// leaf: the library's 2048-byte node when the records fit, otherwise a leaf
/// node sized to hold them all (the layout the writer has always used, kept
/// so those files do not change). More chunks get the library's 2048-byte
/// nodes with internal nodes above the leaves. Returns the bytes and the
/// node size the layout message must record.
fn build_btree_v2_chunk_index_at(
rank: usize,
records: &[(Vec<u64>, &WrittenChunk)],
@@ -1119,73 +1121,49 @@ fn build_btree_v2_chunk_index_at(
base_address: u64,
) -> Result<(Vec<u8>, u32), FormatError> {
let os = offset_size as usize;
let nrec = u16::try_from(records.len()).map_err(|_| {
FormatError::ChunkedReadError(
"more than 65535 chunks with more than one unlimited dimension: \
use larger chunks"
.into(),
)
})?;
let chunk_size_bytes = has_filters.then(|| {
let slots: Vec<Option<WrittenChunk>> =
records.iter().map(|(_, c)| Some((*c).clone())).collect();
filtered_chunk_size_len(&slots)
});
let record_size = os + chunk_size_bytes.map_or(0, |n| n + 4) + 8 * rank;
// Leaf: signature, version, type, records, checksum.
let leaf_len = 4 + 1 + 1 + records.len() * record_size + 4;
let node_size = u32::try_from(leaf_len)
.map_err(|_| FormatError::Overflow("B-tree v2 leaf size".into()))?
.max(BT2_NODE_SIZE);
let record_size_u16 = u16::try_from(record_size)
.map_err(|_| FormatError::Overflow("B-tree v2 record size".into()))?;
let node_size = if records.len() <= usize::from(u16::MAX) {
// Leaf: signature, version, type, records, checksum.
let leaf_len = 4 + 1 + 1 + records.len() * record_size + 4;
u32::try_from(leaf_len)
.map_err(|_| FormatError::Overflow("B-tree v2 leaf size".into()))?
.max(BT2_NODE_SIZE)
} else {
BT2_NODE_SIZE
};
let tree_type = if has_filters {
BT2_CHUNK_FILTERED
} else {
BT2_CHUNK_UNFILTERED
};
let hdr_len = 4 + 1 + 1 + 4 + 2 + 2 + 1 + 1 + os + 2 + length_size as usize + 4;
let leaf_address = base_address + hdr_len as u64;
let mut out = Vec::with_capacity(hdr_len + node_size as usize);
out.extend_from_slice(b"BTHD");
out.push(0); // version
out.push(tree_type);
out.extend_from_slice(&node_size.to_le_bytes());
out.extend_from_slice(&(record_size as u16).to_le_bytes());
out.extend_from_slice(&0u16.to_le_bytes()); // depth
out.push(BT2_SPLIT_PERCENT);
out.push(BT2_MERGE_PERCENT);
if records.is_empty() {
out.extend(core::iter::repeat_n(0xFF, os));
} else {
push_addr(&mut out, leaf_address, offset_size);
}
out.extend_from_slice(&nrec.to_le_bytes());
match length_size {
4 => out.extend_from_slice(&(records.len() as u32).to_le_bytes()),
_ => out.extend_from_slice(&(records.len() as u64).to_le_bytes()),
}
let sum = jenkins_lookup3(&out);
out.extend_from_slice(&sum.to_le_bytes());
debug_assert_eq!(out.len(), hdr_len);
if records.is_empty() {
return Ok((out, node_size));
}
let leaf_start = out.len();
out.extend_from_slice(b"BTLF");
out.push(0); // version
out.push(tree_type);
let mut flat = Vec::with_capacity(records.len() * record_size);
for (scaled, chunk) in records {
push_index_element(&mut out, Some(chunk), offset_size, chunk_size_bytes);
push_index_element(&mut flat, Some(chunk), offset_size, chunk_size_bytes);
for &c in scaled {
out.extend_from_slice(&c.to_le_bytes());
flat.extend_from_slice(&c.to_le_bytes());
}
}
let sum = jenkins_lookup3(&out[leaf_start..]);
out.extend_from_slice(&sum.to_le_bytes());
// The library reads whole nodes; pad the leaf out to the node size.
out.resize(leaf_start + node_size as usize, 0);
let out = build_btree_v2(
BTreeV2Params {
tree_type,
node_size,
record_size: record_size_u16,
split_percent: BT2_SPLIT_PERCENT,
merge_percent: BT2_MERGE_PERCENT,
},
&flat,
base_address,
offset_size,
length_size,
)?;
Ok((out, node_size))
}
crates/clawhdf5-format/src/file_writer.rs
+358 -164
View File
@@ -7,6 +7,7 @@
use alloc::{format, vec, vec::Vec};
use crate::attribute::AttributeMessage;
use crate::btree_v2_write::{BTreeV2Params, build_btree_v2};
use crate::chunked_write::{
ChunkOptions, PrecompressedChunks, build_chunked_data_from_precompressed, precompress_chunks,
};
@@ -74,14 +75,57 @@ const DENSE_LINK_THRESHOLD: usize = 8;
// ---- OH builders ----
/// An object's attributes as its header stores them: inline Attribute
/// messages, or (`dense`) the Attribute Info message of dense storage; with
/// `track_order`, their creation order tracked and indexed.
#[derive(Clone, Copy)]
pub(crate) struct AttrStorage<'a> {
pub(crate) attrs: &'a [AttributeMessage],
pub(crate) dense: Option<&'a DenseAttrBlob>,
pub(crate) track_order: bool,
}
impl AttrStorage<'_> {
/// Add the attribute messages to the header being built. Tracking
/// creation order, as libhdf5 does it: the header's flags say so, an
/// Attribute Info message is written even for inline attributes (it
/// holds the next creation order), and each inline attribute's message
/// carries its creation order.
fn add_to(&self, w: &mut ObjectHeaderWriter) {
if self.track_order {
w.track_attr_order();
}
if let Some(blob) = self.dense {
w.add_message(MessageType::AttributeInfo, blob.attr_info_message.clone());
return;
}
if self.track_order {
w.add_message(
MessageType::AttributeInfo,
serialize_attribute_info(
u64::MAX,
u64::MAX,
Some((self.attrs.len() as u16, u64::MAX)),
),
);
}
for (i, attr) in self.attrs.iter().enumerate() {
w.add_message_with_order(
MessageType::Attribute,
attr.serialize(LENGTH_SIZE),
i as u16,
);
}
}
}
#[allow(clippy::too_many_arguments)]
pub(crate) fn build_chunked_dataset_oh(
dt: &Datatype,
ds: &Dataspace,
layout_message: &[u8],
pipeline_message: Option<&[u8]>,
attrs: &[AttributeMessage],
dense_blob: Option<&DenseAttrBlob>,
attrs: AttrStorage<'_>,
fill_message: &[u8],
refcount: u32,
) -> Result<Vec<u8>, FormatError> {
@@ -93,13 +137,7 @@ pub(crate) fn build_chunked_dataset_oh(
if let Some(pm) = pipeline_message {
w.add_message(MessageType::FilterPipeline, pm.to_vec());
}
if let Some(blob) = dense_blob {
w.add_message(MessageType::AttributeInfo, blob.attr_info_message.clone());
} else {
for attr in attrs {
w.add_message(MessageType::Attribute, attr.serialize(LENGTH_SIZE));
}
}
attrs.add_to(&mut w);
add_refcount(&mut w, refcount);
w.serialize()
}
@@ -110,8 +148,7 @@ pub(crate) fn build_dataset_oh(
ds: &Dataspace,
data_addr: u64,
data_size: u64,
attrs: &[AttributeMessage],
dense_blob: Option<&DenseAttrBlob>,
attrs: AttrStorage<'_>,
fill_message: &[u8],
refcount: u32,
) -> Result<Vec<u8>, FormatError> {
@@ -131,13 +168,7 @@ pub(crate) fn build_dataset_oh(
dl.extend_from_slice(&data_addr.to_le_bytes());
dl.extend_from_slice(&data_size.to_le_bytes());
w.add_message(MessageType::DataLayout, dl);
if let Some(blob) = dense_blob {
w.add_message(MessageType::AttributeInfo, blob.attr_info_message.clone());
} else {
for attr in attrs {
w.add_message(MessageType::Attribute, attr.serialize(LENGTH_SIZE));
}
}
attrs.add_to(&mut w);
add_refcount(&mut w, refcount);
w.serialize()
}
@@ -147,8 +178,7 @@ pub(crate) fn build_compact_dataset_oh(
dt: &Datatype,
ds: &Dataspace,
data: &[u8],
attrs: &[AttributeMessage],
dense_blob: Option<&DenseAttrBlob>,
attrs: AttrStorage<'_>,
fill_message: &[u8],
refcount: u32,
) -> Result<Vec<u8>, FormatError> {
@@ -163,13 +193,7 @@ pub(crate) fn build_compact_dataset_oh(
dl.extend_from_slice(&(data.len() as u16).to_le_bytes());
dl.extend_from_slice(data);
w.add_message(MessageType::DataLayout, dl);
if let Some(blob) = dense_blob {
w.add_message(MessageType::AttributeInfo, blob.attr_info_message.clone());
} else {
for attr in attrs {
w.add_message(MessageType::Attribute, attr.serialize(LENGTH_SIZE));
}
}
attrs.add_to(&mut w);
add_refcount(&mut w, refcount);
w.serialize()
}
@@ -181,8 +205,7 @@ pub(crate) fn build_group_oh(
links: &[LinkMessage],
link_info: &[u8],
dense_links: bool,
attrs: &[AttributeMessage],
dense_blob: Option<&DenseAttrBlob>,
attrs: AttrStorage<'_>,
refcount: u32,
) -> Result<Vec<u8>, FormatError> {
let mut w = ObjectHeaderWriter::new();
@@ -197,13 +220,7 @@ pub(crate) fn build_group_oh(
w.add_message(MessageType::Link, link.serialize(OFFSET_SIZE));
}
}
if let Some(blob) = dense_blob {
w.add_message(MessageType::AttributeInfo, blob.attr_info_message.clone());
} else {
for attr in attrs {
w.add_message(MessageType::Attribute, attr.serialize(LENGTH_SIZE));
}
}
attrs.add_to(&mut w);
add_refcount(&mut w, refcount);
w.serialize()
}
@@ -890,11 +907,31 @@ fn write_frhp(p: WriteFrhp) -> Vec<u8> {
frhp
}
/// libhdf5 numbers the attributes of an object that tracks their creation
/// order with a 2-byte counter.
fn check_tracked_attr_count(track_order: bool, n: usize) -> Result<(), FormatError> {
if track_order && n > usize::from(u16::MAX) {
return Err(FormatError::SerializationError(format!(
"{n} attributes on one object with creation order tracked: libhdf5 \
numbers at most {} (set fewer, or turn off track_order)",
u16::MAX
)));
}
Ok(())
}
/// Build dense attribute storage for a set of attributes.
///
/// With `track_order` the Attribute Info message tracks creation order (an
/// attribute's creation order is its position in `attrs`) and a type-9
/// creation-order index follows the name index, as libhdf5 writes for h5py's
/// `track_order=True`. libhdf5 numbers at most 65 535 attributes.
pub(crate) fn build_dense_attrs(
attrs: &[AttributeMessage],
base_address: u64,
track_order: bool,
) -> Result<DenseAttrBlob, FormatError> {
check_tracked_attr_count(track_order, attrs.len())?;
// Dense attrs use v3 attribute messages (adds character set encoding byte).
let serialized: Vec<Vec<u8>> = attrs.iter().map(|a| a.serialize_v3(LENGTH_SIZE)).collect();
@@ -910,31 +947,55 @@ pub(crate) fn build_dense_attrs(
let heap_id_length = heap.heap_id_length;
let heap_ids = &heap.heap_ids;
// Build B-tree v2 type 8 records (17 bytes each)
// Build B-tree v2 type 8 records (17 bytes each), in the index's key
// order: libhdf5 compares the name hash, then — for names whose hashes
// collide — the names themselves (`strcmp`).
let record_size: u16 = heap_id_length + 1 + 4 + 4;
let mut records: Vec<(u32, u32, Vec<u8>)> = Vec::with_capacity(attrs.len());
for (i, heap_id) in heap_ids.iter().enumerate() {
let mut rec = Vec::with_capacity(record_size as usize);
rec.extend_from_slice(heap_id);
rec.push(0); // msg_flags
rec.extend_from_slice(&(i as u32).to_le_bytes()); // creation_order
rec.extend_from_slice(&name_hashes[i].to_le_bytes()); // hash
records.push((name_hashes[i], i as u32, rec));
}
records.sort_by(|a, b| a.0.cmp(&b.0).then(a.1.cmp(&b.1)));
let records: Vec<Vec<u8>> = records.into_iter().map(|(_, _, rec)| rec).collect();
let mut order: Vec<usize> = (0..attrs.len()).collect();
order.sort_by(|&a, &b| {
name_hashes[a]
.cmp(&name_hashes[b])
.then_with(|| attrs[a].name.as_bytes().cmp(attrs[b].name.as_bytes()))
});
let records: Vec<Vec<u8>> = order
.into_iter()
.map(|i| {
let mut rec = Vec::with_capacity(record_size as usize);
rec.extend_from_slice(&heap_ids[i]);
rec.push(0); // msg_flags
rec.extend_from_slice(&(i as u32).to_le_bytes()); // creation_order
rec.extend_from_slice(&name_hashes[i].to_le_bytes()); // hash
rec
})
.collect();
let bthd_addr = btree_addr;
let mut blob = heap.blob;
blob.extend_from_slice(&single_leaf_v2_btree(
8,
record_size,
&records,
bthd_addr,
"attributes on one object",
)?);
blob.extend_from_slice(&dense_v2_btree(8, record_size, &records, bthd_addr)?);
let attr_info = serialize_attribute_info(frhp_addr, bthd_addr);
let order = if track_order {
// Type 9 records: heap ID, message flags, creation order (the key).
let records: Vec<Vec<u8>> = heap_ids
.iter()
.enumerate()
.map(|(i, heap_id)| {
let mut rec = heap_id.clone();
rec.push(0); // msg_flags
rec.extend_from_slice(&(i as u32).to_le_bytes());
rec
})
.collect();
let corder_addr = base_address + blob.len() as u64;
blob.extend_from_slice(&dense_v2_btree(
9,
heap_id_length + 1 + 4,
&records,
corder_addr,
)?);
Some((attrs.len() as u16, corder_addr))
} else {
None
};
let attr_info = serialize_attribute_info(frhp_addr, bthd_addr, order);
Ok(DenseAttrBlob {
attr_info_message: attr_info,
@@ -953,69 +1014,57 @@ pub(crate) struct DenseLinkBlob {
pub(crate) blob: Vec<u8>,
}
/// A v2 B-tree of `btree_type` holding `records` (already in key order) in a
/// single leaf, laid out at `addr`: the header, then the leaf. `what` names
/// the records in the error for too many ("links in one group").
fn single_leaf_v2_btree(
/// libhdf5's node size for the dense link and attribute indexes it creates
/// (`H5G_NAME_BT2_NODE_SIZE`, `H5A_NAME_BT2_NODE_SIZE`, and the
/// creation-order indexes'), with their split and merge percentages.
const DENSE_BT2_NODE_SIZE: u32 = 512;
const DENSE_BT2_SPLIT_PERCENT: u8 = 100;
const DENSE_BT2_MERGE_PERCENT: u8 = 40;
/// A dense-storage v2 B-tree of `btree_type` holding `records` (already in
/// key order), laid out at `addr`: the header, then its nodes.
///
/// Up to 65 535 records go in one leaf node sized to hold them (the layout
/// the writer has always used, kept so those files do not change). A leaf's
/// record count is a 2-byte field, and libhdf5 sizes a leaf's capacity from
/// the node size: a node with room for more than 65 535 records makes it
/// overflow that count when it adds one, so the node is capped at a full
/// leaf. More records get libhdf5's own 512-byte nodes, with internal nodes
/// above the leaves.
fn dense_v2_btree(
btree_type: u8,
record_size: u16,
records: &[Vec<u8>],
addr: u64,
what: &str,
) -> Result<Vec<u8>, FormatError> {
let os = OFFSET_SIZE as usize;
let ls = LENGTH_SIZE as usize;
// The root node's record count is a 2-byte field; more records need
// internal nodes, which the writer does not build.
let num_records = u16::try_from(records.len()).map_err(|_| {
FormatError::SerializationError(format!(
"{} {what}: at most {} can be written \
(a deeper B-tree index is not implemented)",
records.len(),
u16::MAX
))
})?;
let bthd_size = 4 + 1 + 1 + 4 + 2 + 2 + 1 + 1 + os + 2 + ls + 4;
let btlf_size = 4 + 1 + 1 + (records.len() * record_size as usize) + 4;
// libhdf5 sizes a leaf's capacity from the node size, and a leaf's
// record count is a 2-byte field: a node with room for more than
// 65 535 records makes it overflow that count when it adds one (the
// group can then no longer be listed). Cap the node at a full leaf.
let max_node = btlf_size - records.len() * record_size as usize
+ usize::from(u16::MAX) * record_size as usize;
let node_size = btlf_size.next_power_of_two().max(512).min(max_node) as u32;
let btlf_addr = addr + bthd_size as u64;
let mut out = Vec::with_capacity(bthd_size + node_size as usize);
out.extend_from_slice(b"BTHD");
out.push(0); // version
out.push(btree_type);
out.extend_from_slice(&node_size.to_le_bytes());
out.extend_from_slice(&record_size.to_le_bytes());
out.extend_from_slice(&0u16.to_le_bytes()); // depth = 0 (single leaf)
out.push(100); // split_percent
out.push(40); // merge_percent
write_offset(&mut out, btlf_addr, OFFSET_SIZE);
out.extend_from_slice(&num_records.to_le_bytes());
write_length(&mut out, records.len() as u64, LENGTH_SIZE);
let checksum = crate::checksum::jenkins_lookup3(&out);
out.extend_from_slice(&checksum.to_le_bytes());
debug_assert_eq!(out.len(), bthd_size);
let mut btlf = Vec::with_capacity(node_size as usize);
btlf.extend_from_slice(b"BTLF");
btlf.push(0); // version
btlf.push(btree_type);
for rec in records {
debug_assert_eq!(rec.len(), record_size as usize);
btlf.extend_from_slice(rec);
}
// The checksum follows the records, not the end of the node.
let checksum = crate::checksum::jenkins_lookup3(&btlf);
btlf.extend_from_slice(&checksum.to_le_bytes());
btlf.resize(node_size as usize, 0);
out.extend_from_slice(&btlf);
Ok(out)
let rs = usize::from(record_size);
let n = records.len();
let node_size = if n <= usize::from(u16::MAX) {
let btlf_size = 4 + 1 + 1 + n * rs + 4;
let max_node = 4 + 1 + 1 + usize::from(u16::MAX) * rs + 4;
u32::try_from(btlf_size.next_power_of_two().max(512).min(max_node))
.map_err(|_| FormatError::Overflow("B-tree v2 node size".into()))?
} else {
DENSE_BT2_NODE_SIZE
};
let flat: Vec<u8> = records
.iter()
.inspect(|r| debug_assert_eq!(r.len(), rs))
.flat_map(|r| r.iter().copied())
.collect();
build_btree_v2(
BTreeV2Params {
tree_type: btree_type,
node_size,
record_size,
split_percent: DENSE_BT2_SPLIT_PERCENT,
merge_percent: DENSE_BT2_MERGE_PERCENT,
},
&flat,
addr,
OFFSET_SIZE,
LENGTH_SIZE,
)
}
/// Build dense link storage for a group's links, laid out at `base_address`.
@@ -1039,14 +1088,18 @@ pub(crate) fn build_dense_links(
let heap = build_single_block_fractal_heap(&serialized, base_address, 32, 7)?;
let heap_id_length = heap.heap_id_length;
// Type 5 records: hash(4) + heap_id. The B-tree's key is the name hash,
// so records are sorted by (hash, order).
// Type 5 records: hash(4) + heap_id. The B-tree's key is the name hash
// and, for names whose hashes collide, the name (libhdf5 compares them
// with `strcmp`): records out of that order are not found by name.
let mut by_name: Vec<(u32, usize)> = links
.iter()
.enumerate()
.map(|(i, l)| (crate::checksum::jenkins_lookup3(l.name.as_bytes()), i))
.collect();
by_name.sort_unstable();
by_name.sort_unstable_by(|&(ha, a), &(hb, b)| {
ha.cmp(&hb)
.then_with(|| links[a].name.as_bytes().cmp(links[b].name.as_bytes()))
});
let name_records: Vec<Vec<u8>> = by_name
.iter()
.map(|&(hash, i)| {
@@ -1057,12 +1110,11 @@ pub(crate) fn build_dense_links(
.collect();
let name_bt_addr = heap.btree_addr;
let mut blob = heap.blob;
blob.extend_from_slice(&single_leaf_v2_btree(
blob.extend_from_slice(&dense_v2_btree(
5,
4 + heap_id_length,
&name_records,
name_bt_addr,
"links in one group",
)?);
let link_info_message = if track_order {
@@ -1082,12 +1134,11 @@ pub(crate) fn build_dense_links(
})
.collect();
let order_bt_addr = base_address + blob.len() as u64;
blob.extend_from_slice(&single_leaf_v2_btree(
blob.extend_from_slice(&dense_v2_btree(
6,
8 + heap_id_length,
&order_records,
order_bt_addr,
"links in one group",
)?);
let next_order = by_order.last().map_or(0, |&(o, _)| o + 1);
serialize_link_info(
@@ -1147,12 +1198,25 @@ fn encode_managed_id(offset: u64, length: u64, max_heap_size: u16, id_length: u1
id
}
fn serialize_attribute_info(fh_addr: u64, btree_name_addr: u64) -> Vec<u8> {
/// Serialize an Attribute Info message (version 0). `order` — the next
/// creation order to assign and the creation-order index's address — is
/// present when creation order is tracked and indexed.
fn serialize_attribute_info(
fh_addr: u64,
btree_name_addr: u64,
order: Option<(u16, u64)>,
) -> Vec<u8> {
let mut data = Vec::new();
data.push(0); // version
data.push(0x00); // flags
data.push(if order.is_some() { 0x03 } else { 0x00 }); // flags: tracked, indexed
if let Some((next, _)) = order {
data.extend_from_slice(&next.to_le_bytes());
}
data.extend_from_slice(&fh_addr.to_le_bytes());
data.extend_from_slice(&btree_name_addr.to_le_bytes());
if let Some((_, corder_addr)) = order {
data.extend_from_slice(&corder_addr.to_le_bytes());
}
data
}
@@ -1227,8 +1291,7 @@ pub(crate) fn build_vds_dataset_oh(
dt: &Datatype,
ds: &Dataspace,
global_heap_addr: u64,
attrs: &[AttributeMessage],
dense_blob: Option<&DenseAttrBlob>,
attrs: AttrStorage<'_>,
fill_message: &[u8],
refcount: u32,
) -> Result<Vec<u8>, FormatError> {
@@ -1243,13 +1306,7 @@ pub(crate) fn build_vds_dataset_oh(
dl.extend_from_slice(&global_heap_addr.to_le_bytes());
dl.extend_from_slice(&1u32.to_le_bytes()); // object index 1 in the collection
w.add_message(MessageType::DataLayout, dl);
if let Some(blob) = dense_blob {
w.add_message(MessageType::AttributeInfo, blob.attr_info_message.clone());
} else {
for attr in attrs {
w.add_message(MessageType::Attribute, attr.serialize(LENGTH_SIZE));
}
}
attrs.add_to(&mut w);
add_refcount(&mut w, refcount);
w.serialize()
}
@@ -1284,7 +1341,8 @@ fn write_undef_offset(buf: &mut Vec<u8>, offset_size: u8) {
pub struct FileWriter {
/// The root group's contents (its name is unused).
root: GroupBuilder,
/// Default for groups that do not call [`GroupBuilder::track_order`].
/// Default for groups and datasets that do not set their own
/// `track_order`.
track_order: bool,
/// Global alignment threshold: datasets with raw data >= this many bytes
/// will have their data aligned to `alignment_bytes`.
@@ -1319,11 +1377,18 @@ struct DsFlat {
virtual_sources: Option<Vec<VdsMapping>>,
/// Number of hard links to the dataset.
refcount: u32,
/// Track (and index) attribute creation order.
track_order: bool,
}
/// Convert a DatasetBuilder into a DsFlat, handling VDS (which does not
/// require a `data` field).
fn flatten_ds(db: DatasetBuilder, refcount: u32) -> Result<DsFlat, FormatError> {
fn flatten_ds(
db: DatasetBuilder,
refcount: u32,
default_track_order: bool,
) -> Result<DsFlat, FormatError> {
let track_order = db.track_order.unwrap_or(default_track_order);
let dt = db.datatype.ok_or(FormatError::DatasetMissingData)?;
let shape = db.shape.ok_or(FormatError::DatasetMissingShape)?;
let is_vds = db.virtual_sources.is_some();
@@ -1373,6 +1438,7 @@ fn flatten_ds(db: DatasetBuilder, refcount: u32) -> Result<DsFlat, FormatError>
alignment: db.alignment,
virtual_sources: db.virtual_sources,
refcount,
track_order,
})
}
@@ -1429,10 +1495,12 @@ impl FileWriter {
self
}
/// Track (and index) link creation order in every group that does not
/// set its own [`GroupBuilder::track_order`], the root included — as
/// h5py's `track_order=True`: libhdf5 then lists members in the order
/// they were added. Off by default (members are listed by name).
/// Track (and index) creation order — of links and attributes in every
/// group that does not set its own [`GroupBuilder::track_order`], the
/// root included, and of attributes on every dataset that does not set
/// its own [`DatasetBuilder::track_order`] — as h5py's
/// `track_order=True`: libhdf5 then lists members and attributes in the
/// order they were added. Off by default (they are listed by name).
pub fn track_order(&mut self, track: bool) -> &mut Self {
self.track_order = track;
self
@@ -1503,7 +1571,7 @@ impl FileWriter {
let all_ds: Vec<DsFlat> = tree
.datasets
.into_iter()
.map(|(db, refcount)| flatten_ds(db, refcount))
.map(|(db, refcount)| flatten_ds(db, refcount, self.track_order))
.collect::<Result<_, _>>()?;
let groups: Vec<GrpFlat> = tree
.groups
@@ -1520,6 +1588,15 @@ impl FileWriter {
})
.collect();
// Refuse up front what dense storage would refuse after the work.
let tracked = groups
.iter()
.map(|g| (g.track_order, g.attrs.len()))
.chain(all_ds.iter().map(|d| (d.track_order, d.attrs.len())));
for (track, n) in tracked {
check_tracked_attr_count(track, n)?;
}
// Every datatype must have an on-disk encoding before anything is laid
// out: `Datatype::serialize` itself cannot report a failure.
let group_attrs = groups.iter().flat_map(|g| &g.attrs);
@@ -1574,7 +1651,7 @@ impl FileWriter {
.map(|(gi, g)| {
let dummy_links = g.link_messages(&[], &[]);
let attr_blob = group_dense[gi]
.then(|| build_dense_attrs(&g.attrs, 0))
.then(|| build_dense_attrs(&g.attrs, 0, g.track_order))
.transpose()?;
let li = if group_links_dense[gi] {
serialize_link_info(
@@ -1590,8 +1667,11 @@ impl FileWriter {
&dummy_links,
&li,
group_links_dense[gi],
&g.attrs,
attr_blob.as_ref(),
AttrStorage {
attrs: &g.attrs,
dense: attr_blob.as_ref(),
track_order: g.track_order,
},
g.refcount,
)
.map(|oh| oh.len())
@@ -1610,7 +1690,7 @@ impl FileWriter {
let mut dummy_cursor = 0u64;
for (i, d) in all_ds.iter().enumerate() {
let dense_blob = ds_dense[i]
.then(|| build_dense_attrs(&d.attrs, 0))
.then(|| build_dense_attrs(&d.attrs, 0, d.track_order))
.transpose()?;
if is_vds[i] {
// VDS: dummy OH with address 0 to get the OH size. The global
@@ -1619,8 +1699,11 @@ impl FileWriter {
&d.dt,
&d.ds,
0, // dummy address
&d.attrs,
dense_blob.as_ref(),
AttrStorage {
attrs: &d.attrs,
dense: dense_blob.as_ref(),
track_order: d.track_order,
},
&d.fill_message,
d.refcount,
)?;
@@ -1659,8 +1742,11 @@ impl FileWriter {
&d.ds,
&result.layout_message,
result.pipeline_message.as_deref(),
&d.attrs,
dense_blob.as_ref(),
AttrStorage {
attrs: &d.attrs,
dense: dense_blob.as_ref(),
track_order: d.track_order,
},
&d.fill_message,
d.refcount,
)?;
@@ -1674,8 +1760,11 @@ impl FileWriter {
&d.dt,
&d.ds,
&d.raw,
&d.attrs,
dense_blob.as_ref(),
AttrStorage {
attrs: &d.attrs,
dense: dense_blob.as_ref(),
track_order: d.track_order,
},
&d.fill_message,
d.refcount,
)?;
@@ -1690,8 +1779,11 @@ impl FileWriter {
&d.ds,
0,
d.raw.len() as u64,
&d.attrs,
dense_blob.as_ref(),
AttrStorage {
attrs: &d.attrs,
dense: dense_blob.as_ref(),
track_order: d.track_order,
},
&d.fill_message,
d.refcount,
)?;
@@ -1735,7 +1827,7 @@ impl FileWriter {
group_link_blob_addrs.push(None);
}
if group_dense[gi] {
let blob = build_dense_attrs(&g.attrs, cursor2 as u64)?;
let blob = build_dense_attrs(&g.attrs, cursor2 as u64, g.track_order)?;
cursor2 += blob.blob.len();
group_dense_blobs.push(Some(blob));
} else {
@@ -1752,7 +1844,8 @@ impl FileWriter {
let addr = cursor2 as u64;
cursor2 += sz;
if ds_dense[i] {
let blob = build_dense_attrs(&all_ds[i].attrs, cursor2 as u64)?;
let blob =
build_dense_attrs(&all_ds[i].attrs, cursor2 as u64, all_ds[i].track_order)?;
cursor2 += blob.blob.len();
ds_dense_blobs.push(Some(blob));
} else {
@@ -1776,8 +1869,11 @@ impl FileWriter {
&d.dt,
&d.ds,
heap_addr,
&d.attrs,
ds_dense_blobs[i].as_ref(),
AttrStorage {
attrs: &d.attrs,
dense: ds_dense_blobs[i].as_ref(),
track_order: d.track_order,
},
&d.fill_message,
d.refcount,
)?;
@@ -1804,8 +1900,11 @@ impl FileWriter {
&d.ds,
&result.layout_message,
result.pipeline_message.as_deref(),
&d.attrs,
ds_dense_blobs[i].as_ref(),
AttrStorage {
attrs: &d.attrs,
dense: ds_dense_blobs[i].as_ref(),
track_order: d.track_order,
},
&d.fill_message,
d.refcount,
)?;
@@ -1820,8 +1919,11 @@ impl FileWriter {
&d.dt,
&d.ds,
&d.raw,
&d.attrs,
ds_dense_blobs[i].as_ref(),
AttrStorage {
attrs: &d.attrs,
dense: ds_dense_blobs[i].as_ref(),
track_order: d.track_order,
},
&d.fill_message,
d.refcount,
)?;
@@ -1846,8 +1948,11 @@ impl FileWriter {
&d.ds,
cursor2 as u64,
d.raw.len() as u64,
&d.attrs,
ds_dense_blobs[i].as_ref(),
AttrStorage {
attrs: &d.attrs,
dense: ds_dense_blobs[i].as_ref(),
track_order: d.track_order,
},
&d.fill_message,
d.refcount,
)?;
@@ -1915,8 +2020,11 @@ impl FileWriter {
&links,
&li,
link_blob.is_some(),
&g.attrs,
group_dense_blobs[gi].as_ref(),
AttrStorage {
attrs: &g.attrs,
dense: group_dense_blobs[gi].as_ref(),
track_order: g.track_order,
},
g.refcount,
)?;
debug_assert_eq!(oh.len(), group_oh_sizes[gi]);
@@ -2147,6 +2255,92 @@ mod tests {
assert_eq!(read_dataset_f64(&bytes, "data"), vec![1.0, 2.0, 3.0]);
}
/// Attribute names of the object at `path`, in the order the reader
/// lists them.
fn attr_names(bytes: &[u8], path: &str) -> Vec<String> {
let sig = signature::find_signature(bytes).unwrap();
let sb = Superblock::parse(bytes, sig).unwrap();
let addr = if path == "/" {
sb.root_group_address
} else {
resolve_path_any(bytes, &sb, path).unwrap()
};
let hdr =
ObjectHeader::parse(bytes, addr as usize, sb.offset_size, sb.length_size).unwrap();
crate::attribute::extract_attributes_full(bytes, &hdr, sb.offset_size, sb.length_size)
.unwrap()
.into_iter()
.map(|a| a.name)
.collect()
}
#[test]
fn tracked_attributes_are_read_in_creation_order() {
let set = |names: &[String]| -> Vec<(String, AttrValue)> {
names
.iter()
.enumerate()
.map(|(i, n)| (n.clone(), AttrValue::I64(i as i64)))
.collect()
};
let compact: Vec<String> = ["zeta", "alpha", "mid"].map(String::from).to_vec();
let dense: Vec<String> = (0..30).rev().map(|i| format!("a{i:02}")).collect();
let mut fw = FileWriter::new();
fw.track_order(true);
for (n, v) in set(&compact) {
fw.set_root_attr(&n, v);
}
let ds = fw.create_dataset("dense");
ds.with_i32_data(&[1]);
for (n, v) in set(&dense) {
ds.set_attr(&n, v);
}
let ds = fw.create_dataset("untracked");
ds.with_i32_data(&[1]).track_order(false);
for (n, v) in set(&dense) {
ds.set_attr(&n, v);
}
let mut g = fw.create_group("g");
g.track_order(false);
for (n, v) in set(&compact) {
g.set_attr(&n, v);
}
fw.add_group(g.finish());
let bytes = fw.finish().unwrap();
assert_eq!(attr_names(&bytes, "/"), compact);
assert_eq!(attr_names(&bytes, "dense"), dense);
// Without tracking: storage order (inline: as added; dense: hash).
assert_eq!(attr_names(&bytes, "g"), compact);
let mut by_hash = dense.clone();
by_hash.sort_by_key(|n| crate::checksum::jenkins_lookup3(n.as_bytes()));
assert_eq!(attr_names(&bytes, "untracked"), by_hash);
}
#[test]
fn too_many_tracked_attributes_is_an_error() {
// libhdf5 numbers at most 65 535 attributes on an object that
// tracks their creation order (a 2-byte field). (`set_attr` looks
// for an earlier value, so 65 536 of them through the builder take
// a while; build the messages directly.)
let attrs: Vec<AttributeMessage> = (0..65_536)
.map(|i| build_attr_message(&format!("a{i}"), &AttrValue::I64(i)))
.collect();
let err = build_dense_attrs(&attrs, 0, true)
.err()
.unwrap()
.to_string();
assert!(err.contains("65536 attributes on one object"), "{err}");
assert!(build_dense_attrs(&attrs[1..], 0, true).is_ok());
assert!(build_dense_attrs(&attrs, 0, false).is_ok());
let mut fw = FileWriter::new();
let ds = fw.create_dataset("x");
ds.with_i32_data(&[1]).track_order(true);
for i in 0..20 {
ds.set_attr(&format!("a{i}"), AttrValue::I64(i));
}
assert!(fw.finish().is_ok());
}
#[test]
fn dense_attrs_root_group_self_roundtrip() {
let mut fw = FileWriter::new();
crates/clawhdf5-format/src/lib.rs
+1
View File
@@ -61,6 +61,7 @@ pub mod attribute;
pub mod attribute_info;
pub mod btree_v1;
pub mod btree_v2;
mod btree_v2_write;
mod bulk_alloc;
pub mod checksum;
pub mod chunk_cache;
crates/clawhdf5-format/src/object_header_writer.rs
+57 -8
View File
@@ -12,9 +12,16 @@ use crate::message_type::MessageType;
/// its size truncated to 16 bits produced files libhdf5 refuses.
pub const MAX_MESSAGE_SIZE: usize = u16::MAX as usize;
/// Object header flags: attribute creation order tracked (each message
/// then carries a 2-byte creation order) and indexed.
const OHDR_ATTR_CRT_ORDER_TRACKED: u8 = 0x04;
const OHDR_ATTR_CRT_ORDER_INDEXED: u8 = 0x08;
/// Writer for v2 object headers with proper checksums.
pub struct ObjectHeaderWriter {
messages: Vec<(MessageType, Vec<u8>, u8)>, // (type, data, msg_flags)
messages: Vec<(MessageType, Vec<u8>, u8, u16)>, // (type, data, msg_flags, creation order)
/// Attribute creation order tracked and indexed.
attr_order: bool,
}
impl ObjectHeaderWriter {
@@ -22,17 +29,33 @@ impl ObjectHeaderWriter {
pub fn new() -> Self {
Self {
messages: Vec::new(),
attr_order: false,
}
}
/// Track and index attribute creation order, as libhdf5 does for an
/// object created with `H5P_CRT_ORDER_TRACKED | H5P_CRT_ORDER_INDEXED`
/// (h5py's `track_order=True`): the header's flags say so, and every
/// message carries a creation order (an attribute's own; 0 for the
/// others). libhdf5 reads the setting back from these flags.
pub fn track_attr_order(&mut self) {
self.attr_order = true;
}
/// Add a message to the header with default flags (0).
pub fn add_message(&mut self, msg_type: MessageType, data: Vec<u8>) {
self.messages.push((msg_type, data, 0));
self.messages.push((msg_type, data, 0, 0));
}
/// Add a message with specific flags.
pub fn add_message_with_flags(&mut self, msg_type: MessageType, data: Vec<u8>, flags: u8) {
self.messages.push((msg_type, data, flags));
self.messages.push((msg_type, data, flags, 0));
}
/// Add a message with its creation order, which is written only when
/// attribute creation order is tracked ([`Self::track_attr_order`]).
pub fn add_message_with_order(&mut self, msg_type: MessageType, data: Vec<u8>, order: u16) {
self.messages.push((msg_type, data, 0, order));
}
/// Serialize the complete v2 object header (OHDR + messages + checksum).
@@ -41,10 +64,10 @@ impl ObjectHeaderWriter {
/// than [`MAX_MESSAGE_SIZE`] (e.g. an attribute over ~64 KiB, which would
/// need dense attribute storage), rather than writing a corrupt header.
pub fn serialize(&self) -> Result<Vec<u8>, FormatError> {
if let Some((msg_type, data, _)) = self
if let Some((msg_type, data, _, _)) = self
.messages
.iter()
.find(|(_, data, _)| data.len() > MAX_MESSAGE_SIZE)
.find(|(_, data, _, _)| data.len() > MAX_MESSAGE_SIZE)
{
return Err(FormatError::SerializationError(format!(
"{msg_type:?} message is {} bytes; an object header message holds at most \
@@ -52,11 +75,13 @@ impl ObjectHeaderWriter {
data.len()
)));
}
// Calculate total message bytes: each message has type(1) + size(2) + flags(1) + data
// Calculate total message bytes: each message has type(1) + size(2) +
// flags(1) [+ creation order(2)] + data
let msg_header = if self.attr_order { 6 } else { 4 };
let msg_bytes_total: usize = self
.messages
.iter()
.map(|(_, data, _)| 4 + data.len())
.map(|(_, data, _, _)| msg_header + data.len())
.sum();
// Determine chunk size field width based on msg_bytes_total
@@ -68,6 +93,12 @@ impl ObjectHeaderWriter {
(0x02u8, 4)
};
let flags = if self.attr_order {
flags | OHDR_ATTR_CRT_ORDER_TRACKED | OHDR_ATTR_CRT_ORDER_INDEXED
} else {
flags
};
let mut buf = Vec::new();
// OHDR signature
@@ -85,7 +116,7 @@ impl ObjectHeaderWriter {
}
// Messages
for (msg_type, data, msg_flags) in &self.messages {
for (msg_type, data, msg_flags, order) in &self.messages {
let type_id = msg_type.to_u16();
assert!(
type_id <= 255,
@@ -94,6 +125,9 @@ impl ObjectHeaderWriter {
buf.push(type_id as u8); // type (1 byte in v2)
buf.extend_from_slice(&(data.len() as u16).to_le_bytes()); // size (2 bytes)
buf.push(*msg_flags); // flags
if self.attr_order {
buf.extend_from_slice(&order.to_le_bytes()); // creation order
}
buf.extend_from_slice(data);
}
@@ -193,6 +227,21 @@ mod tests {
assert_eq!(hdr.messages.len(), 0);
}
#[test]
fn tracked_attribute_order_is_in_the_flags_and_every_message() {
let mut writer = ObjectHeaderWriter::new();
writer.track_attr_order();
writer.add_message(MessageType::Dataspace, vec![1, 2, 3, 4]);
writer.add_message_with_order(MessageType::Attribute, vec![5, 6], 7);
let bytes = writer.serialize().unwrap();
assert_eq!(bytes[5] & 0x0C, 0x0C);
let hdr = ObjectHeader::parse(&bytes, 0, 8, 8).unwrap();
assert_eq!(hdr.messages.len(), 2);
assert_eq!(hdr.messages[0].creation_order, Some(0));
assert_eq!(hdr.messages[1].creation_order, Some(7));
assert_eq!(hdr.messages[1].data, vec![5, 6]);
}
#[test]
fn two_messages_roundtrip() {
let mut writer = ObjectHeaderWriter::new();
crates/clawhdf5-format/src/type_builders.rs
+19 -4
View File
@@ -503,6 +503,9 @@ pub struct DatasetBuilder {
/// `data` field is ignored; instead the global heap blob is built from
/// these mappings and a VDS layout message is emitted.
pub(crate) virtual_sources: Option<Vec<VdsMapping>>,
/// Track (and index) attribute creation order; `None` follows the
/// file's default (`FileWriter::track_order`).
pub(crate) track_order: Option<bool>,
#[cfg(feature = "provenance")]
pub(crate) provenance: Option<ProvenanceConfig>,
}
@@ -522,11 +525,22 @@ impl DatasetBuilder {
compact: false,
alignment: 0,
virtual_sources: None,
track_order: None,
#[cfg(feature = "provenance")]
provenance: None,
}
}
/// Track the creation order of this dataset's attributes, and index it,
/// as h5py's `create_dataset(..., track_order=True)` does: libhdf5 (and
/// h5py) then list the attributes in the order they were set rather
/// than by name. libhdf5 numbers at most 65 535 attributes on an object
/// that tracks their order; more is an error when the file is written.
pub fn track_order(&mut self, track: bool) -> &mut Self {
self.track_order = Some(track);
self
}
pub fn with_f64_data(&mut self, data: &[f64]) -> &mut Self {
self.datatype = Some(make_f64_type());
let mut b = Vec::with_capacity(data.len() * 8);
@@ -986,10 +1000,11 @@ impl GroupBuilder {
self.attrs.push((name.to_string(), value));
}
/// Track the creation order of this group's links, and index it, as
/// h5py's `track_order=True` does: libhdf5 (and h5py) then list the
/// group's members in the order they were added rather than by name.
/// Applies to links only, not to attributes.
/// Track the creation order of this group's links and attributes, and
/// index it, as h5py's `track_order=True` does: libhdf5 (and h5py) then
/// list the group's members, and its attributes, in the order they were
/// added rather than by name. libhdf5 numbers at most 65 535 attributes
/// on an object that tracks their order.
pub fn track_order(&mut self, track: bool) -> &mut Self {
self.track_order = Some(track);
self