quantumclaw/clawhdf5
3
1
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 1 Wiki Activity

writer: v2 B-trees with internal nodes (no 65 535-record limit)

Browse Source 
Dense link and attribute indexes and the chunk index for several
unlimited dimensions were single leaves, capping them at 65 535
records. btree_v2_write builds trees of any depth, with node capacities
and pointer widths from libhdf5's H5B2__hdr_init arithmetic (now shared
with the reader as btree_v2::node_info) and libhdf5's node sizes (512
dense, 2048 chunks). Indexes that fit the old one-leaf layout are
written byte for byte as before (compared for 10..65 535 links, attrs
and chunks, tracked and filtered).

Tests: 100 000 links (short names; long names with creation order),
70 000 attributes, 200 000 chunks (and 80 000 deflated), read by h5py,
h5dump and clawhdf5 and edited by h5py r+; h5rs check on the same
shapes, asserting depths 2-3.

Co-Authored-By: Claude Opus 5.5 (1M context) <[email protected]>
This commit is contained in:
osobh
2026-09-26 10:12:07 -05:00
co-authored by Claude Opus 5.5
parent 7acfb79584
commit d63c76e7ab
11 changed files with 1005 additions and 185 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/clawhdf5-format/src/file_writer.rs
+50 -69
View File
@@ -7,6 +7,7 @@
use alloc::{format, vec, vec::Vec};
use crate::attribute::AttributeMessage;
use crate::btree_v2_write::{BTreeV2Params, build_btree_v2};
use crate::chunked_write::{
ChunkOptions, PrecompressedChunks, build_chunked_data_from_precompressed, precompress_chunks,
};
@@ -933,13 +934,7 @@ pub(crate) fn build_dense_attrs(
.collect();
let bthd_addr = btree_addr;
let mut blob = heap.blob;
blob.extend_from_slice(&single_leaf_v2_btree(
8,
record_size,
&records,
bthd_addr,
"attributes on one object",
)?);
blob.extend_from_slice(&dense_v2_btree(8, record_size, &records, bthd_addr)?);
let attr_info = serialize_attribute_info(frhp_addr, bthd_addr);
@@ -960,69 +955,57 @@ pub(crate) struct DenseLinkBlob {
pub(crate) blob: Vec<u8>,
}
/// A v2 B-tree of `btree_type` holding `records` (already in key order) in a
/// single leaf, laid out at `addr`: the header, then the leaf. `what` names
/// the records in the error for too many ("links in one group").
fn single_leaf_v2_btree(
/// libhdf5's node size for the dense link and attribute indexes it creates
/// (`H5G_NAME_BT2_NODE_SIZE`, `H5A_NAME_BT2_NODE_SIZE`, and the
/// creation-order indexes'), with their split and merge percentages.
const DENSE_BT2_NODE_SIZE: u32 = 512;
const DENSE_BT2_SPLIT_PERCENT: u8 = 100;
const DENSE_BT2_MERGE_PERCENT: u8 = 40;
/// A dense-storage v2 B-tree of `btree_type` holding `records` (already in
/// key order), laid out at `addr`: the header, then its nodes.
///
/// Up to 65 535 records go in one leaf node sized to hold them (the layout
/// the writer has always used, kept so those files do not change). A leaf's
/// record count is a 2-byte field, and libhdf5 sizes a leaf's capacity from
/// the node size: a node with room for more than 65 535 records makes it
/// overflow that count when it adds one, so the node is capped at a full
/// leaf. More records get libhdf5's own 512-byte nodes, with internal nodes
/// above the leaves.
fn dense_v2_btree(
btree_type: u8,
record_size: u16,
records: &[Vec<u8>],
addr: u64,
what: &str,
) -> Result<Vec<u8>, FormatError> {
let os = OFFSET_SIZE as usize;
let ls = LENGTH_SIZE as usize;
// The root node's record count is a 2-byte field; more records need
// internal nodes, which the writer does not build.
let num_records = u16::try_from(records.len()).map_err(|_| {
FormatError::SerializationError(format!(
"{} {what}: at most {} can be written \
(a deeper B-tree index is not implemented)",
records.len(),
u16::MAX
))
})?;
let bthd_size = 4 + 1 + 1 + 4 + 2 + 2 + 1 + 1 + os + 2 + ls + 4;
let btlf_size = 4 + 1 + 1 + (records.len() * record_size as usize) + 4;
// libhdf5 sizes a leaf's capacity from the node size, and a leaf's
// record count is a 2-byte field: a node with room for more than
// 65 535 records makes it overflow that count when it adds one (the
// group can then no longer be listed). Cap the node at a full leaf.
let max_node = btlf_size - records.len() * record_size as usize
+ usize::from(u16::MAX) * record_size as usize;
let node_size = btlf_size.next_power_of_two().max(512).min(max_node) as u32;
let btlf_addr = addr + bthd_size as u64;
let mut out = Vec::with_capacity(bthd_size + node_size as usize);
out.extend_from_slice(b"BTHD");
out.push(0); // version
out.push(btree_type);
out.extend_from_slice(&node_size.to_le_bytes());
out.extend_from_slice(&record_size.to_le_bytes());
out.extend_from_slice(&0u16.to_le_bytes()); // depth = 0 (single leaf)
out.push(100); // split_percent
out.push(40); // merge_percent
write_offset(&mut out, btlf_addr, OFFSET_SIZE);
out.extend_from_slice(&num_records.to_le_bytes());
write_length(&mut out, records.len() as u64, LENGTH_SIZE);
let checksum = crate::checksum::jenkins_lookup3(&out);
out.extend_from_slice(&checksum.to_le_bytes());
debug_assert_eq!(out.len(), bthd_size);
let mut btlf = Vec::with_capacity(node_size as usize);
btlf.extend_from_slice(b"BTLF");
btlf.push(0); // version
btlf.push(btree_type);
for rec in records {
debug_assert_eq!(rec.len(), record_size as usize);
btlf.extend_from_slice(rec);
}
// The checksum follows the records, not the end of the node.
let checksum = crate::checksum::jenkins_lookup3(&btlf);
btlf.extend_from_slice(&checksum.to_le_bytes());
btlf.resize(node_size as usize, 0);
out.extend_from_slice(&btlf);
Ok(out)
let rs = usize::from(record_size);
let n = records.len();
let node_size = if n <= usize::from(u16::MAX) {
let btlf_size = 4 + 1 + 1 + n * rs + 4;
let max_node = 4 + 1 + 1 + usize::from(u16::MAX) * rs + 4;
u32::try_from(btlf_size.next_power_of_two().max(512).min(max_node))
.map_err(|_| FormatError::Overflow("B-tree v2 node size".into()))?
} else {
DENSE_BT2_NODE_SIZE
};
let flat: Vec<u8> = records
.iter()
.inspect(|r| debug_assert_eq!(r.len(), rs))
.flat_map(|r| r.iter().copied())
.collect();
build_btree_v2(
BTreeV2Params {
tree_type: btree_type,
node_size,
record_size,
split_percent: DENSE_BT2_SPLIT_PERCENT,
merge_percent: DENSE_BT2_MERGE_PERCENT,
},
&flat,
addr,
OFFSET_SIZE,
LENGTH_SIZE,
)
}
/// Build dense link storage for a group's links, laid out at `base_address`.
@@ -1068,12 +1051,11 @@ pub(crate) fn build_dense_links(
.collect();
let name_bt_addr = heap.btree_addr;
let mut blob = heap.blob;
blob.extend_from_slice(&single_leaf_v2_btree(
blob.extend_from_slice(&dense_v2_btree(
5,
4 + heap_id_length,
&name_records,
name_bt_addr,
"links in one group",
)?);
let link_info_message = if track_order {
@@ -1093,12 +1075,11 @@ pub(crate) fn build_dense_links(
})
.collect();
let order_bt_addr = base_address + blob.len() as u64;
blob.extend_from_slice(&single_leaf_v2_btree(
blob.extend_from_slice(&dense_v2_btree(
6,
8 + heap_id_length,
&order_records,
order_bt_addr,
"links in one group",
)?);
let next_order = by_order.last().map_or(0, |&(o, _)| o + 1);
serialize_link_info(