osobhandClaude Opus 5.5
193a5f8a82
h5py's track_order=True orders attributes as well as links; the writer tracked links only. A tracking object's header now sets the attribute creation order tracked/indexed flags and carries per-message creation orders, an Attribute Info message holds the next order (inline too), and dense storage gets a type-9 creation-order index. The file default applies to datasets, with DatasetBuilder::track_order per dataset; more than 65 535 attributes on a tracking object is an error (libhdf5's counter is 2 bytes). The reader lists such attributes in creation order. h5py lists them in order (inline, dense, 20 000 on one dataset) and keeps numbering in r+ mode, including its inline-to-dense move. Co-Authored-By: Claude Opus 5.5 (1M context) <[email protected]>
2769 lines
102 KiB
Rust
2769 lines
102 KiB
Rust
//! HDF5 file creation (write pipeline).
|
|
//!
|
|
//! Produces valid HDF5 files with v3 superblock, v2 object headers,
|
|
//! link messages, contiguous datasets, inline and dense attributes.
|
|
|
|
#[cfg(not(feature = "std"))]
|
|
use alloc::{format, vec, vec::Vec};
|
|
|
|
use crate::attribute::AttributeMessage;
|
|
use crate::btree_v2_write::{BTreeV2Params, build_btree_v2};
|
|
use crate::chunked_write::{
|
|
ChunkOptions, PrecompressedChunks, build_chunked_data_from_precompressed, precompress_chunks,
|
|
};
|
|
use crate::data_layout::VdsMapping;
|
|
use crate::dataspace::{Dataspace, DataspaceType};
|
|
use crate::error::FormatError;
|
|
use crate::link_message::{LinkMessage, LinkTarget};
|
|
use crate::message_type::MessageType;
|
|
use crate::metadata_index::{DatasetMetadata, MetadataBlock, MetadataIndex};
|
|
use crate::object_header_writer::ObjectHeaderWriter;
|
|
use crate::superblock::Superblock;
|
|
use crate::type_builders::{
|
|
DatasetBuilder, FinishedGroup, GroupBuilder, build_attr_message, fill_value_message,
|
|
};
|
|
use crate::writer_tree::{self, LinkTo};
|
|
|
|
// Re-export public types that moved to type_builders for API compatibility.
|
|
#[cfg(feature = "provenance")]
|
|
pub use crate::type_builders::ProvenanceConfig;
|
|
pub use crate::type_builders::{AttrValue, CompoundTypeBuilder, EnumTypeBuilder};
|
|
|
|
use crate::datatype::{CharacterSet, Datatype};
|
|
|
|
pub(crate) const OFFSET_SIZE: u8 = 8;
|
|
pub(crate) const LENGTH_SIZE: u8 = 8;
|
|
const SUPERBLOCK_SIZE: usize = 48;
|
|
|
|
/// Largest raw data a compact dataset can hold: the layout message (version,
|
|
/// class, 2-byte size, data) must fit an object header message, whose size
|
|
/// field is 2 bytes. Bigger "compact" requests fall back to contiguous storage.
|
|
const MAX_COMPACT_DATA_SIZE: usize = crate::object_header_writer::MAX_MESSAGE_SIZE - 4;
|
|
|
|
/// libhdf5's bounds on a file space page size (`H5F_FILE_SPACE_PAGE_SIZE_MIN`
|
|
/// and `_MAX`).
|
|
const MIN_FILE_SPACE_PAGE_SIZE: u32 = 512;
|
|
const MAX_FILE_SPACE_PAGE_SIZE: u32 = 1024 * 1024 * 1024;
|
|
|
|
/// Superblock extension object header for a file using the paged file-space
|
|
/// strategy: a single File Space Info message (0x0017), as libhdf5 writes it
|
|
/// for `fs_strategy="page"` without persisted free space.
|
|
fn build_paged_superblock_extension(page_size: u32) -> Result<Vec<u8>, FormatError> {
|
|
let mut fsinfo = Vec::new();
|
|
fsinfo.push(1); // version
|
|
fsinfo.push(1); // strategy: H5F_FSPACE_STRATEGY_PAGE
|
|
fsinfo.push(0); // persisting free space: no
|
|
write_length(&mut fsinfo, 1, LENGTH_SIZE); // free-space section threshold
|
|
write_length(&mut fsinfo, u64::from(page_size), LENGTH_SIZE);
|
|
fsinfo.extend_from_slice(&0u16.to_le_bytes()); // page end metadata threshold
|
|
write_undef_offset(&mut fsinfo, OFFSET_SIZE); // EOA before free-space info
|
|
let mut w = ObjectHeaderWriter::new();
|
|
// Flags as libhdf5 sets them: bit 2 (never share) and bit 4 (mark if
|
|
// unknown). Not constant: libhdf5 rewrites the message when it closes a
|
|
// file it opened for writing.
|
|
w.add_message_with_flags(MessageType::Unknown(0x0017), fsinfo, 0x14);
|
|
w.serialize()
|
|
}
|
|
|
|
/// Threshold for switching from compact (inline) to dense attribute storage.
|
|
const DENSE_ATTR_THRESHOLD: usize = 8;
|
|
|
|
/// Threshold for switching a group from compact (inline Link messages) to dense
|
|
/// link storage (fractal heap + v2 B-tree), matching libhdf5's default
|
|
/// `max_compact` of 8 links.
|
|
const DENSE_LINK_THRESHOLD: usize = 8;
|
|
|
|
// ---- OH builders ----
|
|
|
|
/// An object's attributes as its header stores them: inline Attribute
|
|
/// messages, or (`dense`) the Attribute Info message of dense storage; with
|
|
/// `track_order`, their creation order tracked and indexed.
|
|
#[derive(Clone, Copy)]
|
|
pub(crate) struct AttrStorage<'a> {
|
|
pub(crate) attrs: &'a [AttributeMessage],
|
|
pub(crate) dense: Option<&'a DenseAttrBlob>,
|
|
pub(crate) track_order: bool,
|
|
}
|
|
|
|
impl AttrStorage<'_> {
|
|
/// Add the attribute messages to the header being built. Tracking
|
|
/// creation order, as libhdf5 does it: the header's flags say so, an
|
|
/// Attribute Info message is written even for inline attributes (it
|
|
/// holds the next creation order), and each inline attribute's message
|
|
/// carries its creation order.
|
|
fn add_to(&self, w: &mut ObjectHeaderWriter) {
|
|
if self.track_order {
|
|
w.track_attr_order();
|
|
}
|
|
if let Some(blob) = self.dense {
|
|
w.add_message(MessageType::AttributeInfo, blob.attr_info_message.clone());
|
|
return;
|
|
}
|
|
if self.track_order {
|
|
w.add_message(
|
|
MessageType::AttributeInfo,
|
|
serialize_attribute_info(
|
|
u64::MAX,
|
|
u64::MAX,
|
|
Some((self.attrs.len() as u16, u64::MAX)),
|
|
),
|
|
);
|
|
}
|
|
for (i, attr) in self.attrs.iter().enumerate() {
|
|
w.add_message_with_order(
|
|
MessageType::Attribute,
|
|
attr.serialize(LENGTH_SIZE),
|
|
i as u16,
|
|
);
|
|
}
|
|
}
|
|
}
|
|
|
|
#[allow(clippy::too_many_arguments)]
|
|
pub(crate) fn build_chunked_dataset_oh(
|
|
dt: &Datatype,
|
|
ds: &Dataspace,
|
|
layout_message: &[u8],
|
|
pipeline_message: Option<&[u8]>,
|
|
attrs: AttrStorage<'_>,
|
|
fill_message: &[u8],
|
|
refcount: u32,
|
|
) -> Result<Vec<u8>, FormatError> {
|
|
let mut w = ObjectHeaderWriter::new();
|
|
w.add_message_with_flags(MessageType::Datatype, dt.serialize(), 0x01);
|
|
w.add_message(MessageType::Dataspace, ds.serialize(LENGTH_SIZE));
|
|
w.add_message_with_flags(MessageType::FillValue, fill_message.to_vec(), 0x01);
|
|
w.add_message(MessageType::DataLayout, layout_message.to_vec());
|
|
if let Some(pm) = pipeline_message {
|
|
w.add_message(MessageType::FilterPipeline, pm.to_vec());
|
|
}
|
|
attrs.add_to(&mut w);
|
|
add_refcount(&mut w, refcount);
|
|
w.serialize()
|
|
}
|
|
|
|
#[allow(clippy::too_many_arguments)]
|
|
pub(crate) fn build_dataset_oh(
|
|
dt: &Datatype,
|
|
ds: &Dataspace,
|
|
data_addr: u64,
|
|
data_size: u64,
|
|
attrs: AttrStorage<'_>,
|
|
fill_message: &[u8],
|
|
refcount: u32,
|
|
) -> Result<Vec<u8>, FormatError> {
|
|
let mut w = ObjectHeaderWriter::new();
|
|
w.add_message_with_flags(MessageType::Datatype, dt.serialize(), 0x01);
|
|
w.add_message(MessageType::Dataspace, ds.serialize(LENGTH_SIZE));
|
|
w.add_message_with_flags(MessageType::FillValue, fill_message.to_vec(), 0x01);
|
|
let mut dl = Vec::new();
|
|
dl.push(4); // version
|
|
dl.push(1); // class = contiguous
|
|
// An empty dataset has no storage: its address must be the undefined
|
|
// address, as libhdf5 writes it. A real address with size 0 trips
|
|
// libhdf5's `addr + size <= addr` overflow check, and it refuses the
|
|
// dataset as "invalid dataset size, likely file corruption" — which made
|
|
// every store with no sessions or knowledge graph unreadable by h5py.
|
|
let data_addr = if data_size == 0 { u64::MAX } else { data_addr };
|
|
dl.extend_from_slice(&data_addr.to_le_bytes());
|
|
dl.extend_from_slice(&data_size.to_le_bytes());
|
|
w.add_message(MessageType::DataLayout, dl);
|
|
attrs.add_to(&mut w);
|
|
add_refcount(&mut w, refcount);
|
|
w.serialize()
|
|
}
|
|
|
|
/// Build a compact dataset object header where data is stored inline.
|
|
pub(crate) fn build_compact_dataset_oh(
|
|
dt: &Datatype,
|
|
ds: &Dataspace,
|
|
data: &[u8],
|
|
attrs: AttrStorage<'_>,
|
|
fill_message: &[u8],
|
|
refcount: u32,
|
|
) -> Result<Vec<u8>, FormatError> {
|
|
let mut w = ObjectHeaderWriter::new();
|
|
w.add_message_with_flags(MessageType::Datatype, dt.serialize(), 0x01);
|
|
w.add_message(MessageType::Dataspace, ds.serialize(LENGTH_SIZE));
|
|
w.add_message_with_flags(MessageType::FillValue, fill_message.to_vec(), 0x01);
|
|
// Compact layout message: version=4, class=0, u16 size, inline data
|
|
let mut dl = Vec::new();
|
|
dl.push(4); // version
|
|
dl.push(0); // class = compact
|
|
dl.extend_from_slice(&(data.len() as u16).to_le_bytes());
|
|
dl.extend_from_slice(data);
|
|
w.add_message(MessageType::DataLayout, dl);
|
|
attrs.add_to(&mut w);
|
|
add_refcount(&mut w, refcount);
|
|
w.serialize()
|
|
}
|
|
|
|
/// Build a group's object header. `link_info` is its Link Info message;
|
|
/// with `dense_links` the links live in the fractal heap it points at and are
|
|
/// not written inline.
|
|
pub(crate) fn build_group_oh(
|
|
links: &[LinkMessage],
|
|
link_info: &[u8],
|
|
dense_links: bool,
|
|
attrs: AttrStorage<'_>,
|
|
refcount: u32,
|
|
) -> Result<Vec<u8>, FormatError> {
|
|
let mut w = ObjectHeaderWriter::new();
|
|
w.add_message(MessageType::LinkInfo, link_info.to_vec());
|
|
// Group Info (version 0, default link-phase thresholds, no estimates).
|
|
// Readers don't need it, but libhdf5 reads it before inserting a link:
|
|
// without one, adding a link to a group we wrote (h5py in "r+" mode)
|
|
// failed with "message type not found".
|
|
w.add_message(MessageType::GroupInfo, vec![0, 0]);
|
|
if !dense_links {
|
|
for link in links {
|
|
w.add_message(MessageType::Link, link.serialize(OFFSET_SIZE));
|
|
}
|
|
}
|
|
attrs.add_to(&mut w);
|
|
add_refcount(&mut w, refcount);
|
|
w.serialize()
|
|
}
|
|
|
|
/// An object with more than one hard link records the count in an Object
|
|
/// Reference Count message (libhdf5 omits it for a count of one). Without
|
|
/// it, libhdf5 deleting one of the links would free an object that is still
|
|
/// linked.
|
|
fn add_refcount(w: &mut ObjectHeaderWriter, refcount: u32) {
|
|
if refcount > 1 {
|
|
let mut msg = vec![0u8]; // version
|
|
msg.extend_from_slice(&refcount.to_le_bytes());
|
|
w.add_message(MessageType::ObjectReferenceCount, msg);
|
|
}
|
|
}
|
|
|
|
/// The character set a link name is written with: UTF-8 when it is not
|
|
/// plain ASCII, as h5py writes it.
|
|
fn name_charset(name: &str) -> CharacterSet {
|
|
if name.is_ascii() {
|
|
CharacterSet::Ascii
|
|
} else {
|
|
CharacterSet::Utf8
|
|
}
|
|
}
|
|
|
|
/// The Link message for `link`, whose group and dataset targets are at the
|
|
/// given addresses (indexed as in the writer tree).
|
|
fn link_message(link: &writer_tree::Link, group_addrs: &[u64], ds_addrs: &[u64]) -> LinkMessage {
|
|
let link_target = match &link.to {
|
|
LinkTo::Group(g) => LinkTarget::Hard {
|
|
object_header_address: group_addrs.get(*g).copied().unwrap_or(0),
|
|
},
|
|
LinkTo::Dataset(d) => LinkTarget::Hard {
|
|
object_header_address: ds_addrs.get(*d).copied().unwrap_or(0),
|
|
},
|
|
LinkTo::Soft(target_path) => LinkTarget::Soft {
|
|
target_path: target_path.clone(),
|
|
},
|
|
LinkTo::External { file, path } => LinkTarget::External {
|
|
filename: file.clone(),
|
|
object_path: path.clone(),
|
|
},
|
|
};
|
|
LinkMessage {
|
|
name: link.name.clone(),
|
|
link_target,
|
|
creation_order: link.creation_order,
|
|
charset: name_charset(&link.name),
|
|
}
|
|
}
|
|
|
|
/// Link Info message for a group with compact (inline) links: no heap, no
|
|
/// B-trees. A group tracking creation order records the next order to use.
|
|
fn compact_link_info(track_order: bool, nlinks: usize) -> Vec<u8> {
|
|
let max_corder = track_order.then_some(nlinks as u64);
|
|
serialize_link_info(
|
|
max_corder,
|
|
u64::MAX,
|
|
u64::MAX,
|
|
track_order.then_some(u64::MAX),
|
|
)
|
|
}
|
|
|
|
// ---- Dense attribute blob ----
|
|
|
|
/// Pre-built dense attribute storage (fractal heap + B-tree v2 + attribute info message).
|
|
pub(crate) struct DenseAttrBlob {
|
|
/// Serialized AttributeInfo message data (to embed in the object header).
|
|
pub(crate) attr_info_message: Vec<u8>,
|
|
/// The combined fractal heap header + direct block + B-tree v2 bytes.
|
|
pub(crate) blob: Vec<u8>,
|
|
}
|
|
|
|
/// A fractal heap holding a set of serialized objects, plus the heap IDs that
|
|
/// address them. Shared by dense attribute and dense link storage, which differ
|
|
/// only in their v2 B-tree record layout.
|
|
pub(crate) struct FractalHeapBlock {
|
|
/// The complete heap bytes: FRHP header, then either a single root direct
|
|
/// block, or a root indirect block (FHIB) followed by its direct blocks.
|
|
blob: Vec<u8>,
|
|
/// Address of the fractal heap header.
|
|
frhp_addr: u64,
|
|
/// Address where the v2 B-tree should be placed (right after the heap).
|
|
btree_addr: u64,
|
|
/// Heap ID for each object, in input order.
|
|
heap_ids: Vec<Vec<u8>>,
|
|
/// Heap ID length (bytes).
|
|
heap_id_length: u16,
|
|
}
|
|
|
|
/// Build a fractal heap for `serialized` objects, laid out at `base_address`.
|
|
///
|
|
/// Uses a single root direct block when the data fits in one (≤ the maximum
|
|
/// direct block size), otherwise a root indirect block over multiple direct
|
|
/// blocks following the doubling table. The caller builds the matching v2
|
|
/// B-tree (type 5 for links, type 8 for attributes) at the returned
|
|
/// `btree_addr`.
|
|
pub(crate) fn build_single_block_fractal_heap(
|
|
serialized: &[Vec<u8>],
|
|
base_address: u64,
|
|
max_heap_size: u16,
|
|
heap_id_length: u16,
|
|
) -> Result<FractalHeapBlock, FormatError> {
|
|
let os = OFFSET_SIZE as usize;
|
|
let ls = LENGTH_SIZE as usize;
|
|
let block_offset_bytes = (max_heap_size as usize).div_ceil(8);
|
|
let max_direct_block_size: u64 = 65536;
|
|
|
|
// Direct block layout: sig(4) + ver(1) + heap_addr(os) + block_offset(bo_bytes)
|
|
// + checksum(4) [when flags bit 1 set] + data...
|
|
let dblock_header_size = 4 + 1 + os + block_offset_bytes + 4; // +4 for checksum
|
|
|
|
// An object must fit one direct block: the writer has no huge-object
|
|
// path, and libhdf5 cannot read an object that overruns its block.
|
|
let max_managed = max_direct_block_size as usize - dblock_header_size;
|
|
if let Some(big) = serialized.iter().find(|s| s.len() > max_managed) {
|
|
return Err(FormatError::SerializationError(format!(
|
|
"a {}-byte message cannot go in dense storage: a fractal heap \
|
|
object holds at most {max_managed} bytes (huge heap objects are \
|
|
not written)",
|
|
big.len()
|
|
)));
|
|
}
|
|
let total_data_size: usize = serialized.iter().map(|s| s.len()).sum();
|
|
let dblock_content_size = dblock_header_size + total_data_size;
|
|
let starting_block_size = dblock_content_size.next_power_of_two().max(512) as u64;
|
|
|
|
// When the objects don't fit in a single direct block, fall back to a
|
|
// multi-block heap with a root indirect block.
|
|
if starting_block_size > max_direct_block_size {
|
|
return build_multiblock_fractal_heap(
|
|
serialized,
|
|
base_address,
|
|
max_heap_size,
|
|
heap_id_length,
|
|
);
|
|
}
|
|
|
|
// Fractal heap header size
|
|
let frhp_size = 4
|
|
+ 1
|
|
+ 2
|
|
+ 2
|
|
+ 1
|
|
+ 4
|
|
+ ls
|
|
+ os
|
|
+ ls
|
|
+ os
|
|
+ ls
|
|
+ ls
|
|
+ ls
|
|
+ ls
|
|
+ ls
|
|
+ ls
|
|
+ ls
|
|
+ ls
|
|
+ 2
|
|
+ ls
|
|
+ ls
|
|
+ 2
|
|
+ 2
|
|
+ os
|
|
+ 2
|
|
+ 4;
|
|
|
|
let frhp_addr = base_address;
|
|
let dblock_addr = frhp_addr + frhp_size as u64;
|
|
let btree_addr = dblock_addr + starting_block_size;
|
|
|
|
let data_space = starting_block_size as usize - dblock_header_size;
|
|
let free_space = data_space - total_data_size;
|
|
|
|
// Build fractal heap header
|
|
let mut frhp = Vec::with_capacity(frhp_size);
|
|
frhp.extend_from_slice(b"FRHP");
|
|
frhp.push(0); // version
|
|
frhp.extend_from_slice(&heap_id_length.to_le_bytes());
|
|
frhp.extend_from_slice(&0u16.to_le_bytes()); // io_filter_encoded_length
|
|
frhp.push(0x02); // flags: bit 1 = checksum direct blocks
|
|
frhp.extend_from_slice(&(max_managed as u32).to_le_bytes());
|
|
write_length(&mut frhp, 0, LENGTH_SIZE); // next_huge_object_id
|
|
write_undef_offset(&mut frhp, OFFSET_SIZE); // btree_huge_objects_address
|
|
write_length(&mut frhp, free_space as u64, LENGTH_SIZE); // free_space_managed_blocks
|
|
write_undef_offset(&mut frhp, OFFSET_SIZE); // free_space_mgr_addr
|
|
write_length(&mut frhp, starting_block_size, LENGTH_SIZE); // managed_space_in_heap
|
|
write_length(&mut frhp, starting_block_size, LENGTH_SIZE); // allocated_managed_space
|
|
write_length(&mut frhp, 0, LENGTH_SIZE); // dblock_alloc_iter
|
|
write_length(&mut frhp, serialized.len() as u64, LENGTH_SIZE); // managed_objects_count
|
|
write_length(&mut frhp, 0, LENGTH_SIZE); // huge_objects_size
|
|
write_length(&mut frhp, 0, LENGTH_SIZE); // huge_objects_count
|
|
write_length(&mut frhp, 0, LENGTH_SIZE); // tiny_objects_size
|
|
write_length(&mut frhp, 0, LENGTH_SIZE); // tiny_objects_count
|
|
frhp.extend_from_slice(&4u16.to_le_bytes()); // table_width
|
|
write_length(&mut frhp, starting_block_size, LENGTH_SIZE);
|
|
write_length(&mut frhp, max_direct_block_size, LENGTH_SIZE); // max_direct_block_size
|
|
frhp.extend_from_slice(&max_heap_size.to_le_bytes());
|
|
let sri: u16 = 1;
|
|
frhp.extend_from_slice(&sri.to_le_bytes()); // starting_row_of_indirect_blocks
|
|
write_offset(&mut frhp, dblock_addr, OFFSET_SIZE);
|
|
frhp.extend_from_slice(&0u16.to_le_bytes()); // root is direct block
|
|
let frhp_checksum = crate::checksum::jenkins_lookup3(&frhp);
|
|
frhp.extend_from_slice(&frhp_checksum.to_le_bytes());
|
|
debug_assert_eq!(frhp.len(), frhp_size);
|
|
|
|
// Build direct block: header (with checksum) + data + padding
|
|
let mut dblock = Vec::with_capacity(starting_block_size as usize);
|
|
dblock.extend_from_slice(b"FHDB");
|
|
dblock.push(0); // version
|
|
write_offset(&mut dblock, frhp_addr, OFFSET_SIZE);
|
|
dblock.extend_from_slice(&vec![0u8; block_offset_bytes]); // block_offset = 0 for root
|
|
let cksum_pos = dblock.len();
|
|
dblock.extend_from_slice(&[0u8; 4]); // checksum placeholder
|
|
debug_assert_eq!(dblock.len(), dblock_header_size);
|
|
|
|
// Data area starts after header
|
|
let mut obj_offsets: Vec<(u64, u64)> = Vec::with_capacity(serialized.len());
|
|
for s in serialized {
|
|
let offset_in_heap = dblock.len() as u64;
|
|
obj_offsets.push((offset_in_heap, s.len() as u64));
|
|
dblock.extend_from_slice(s);
|
|
}
|
|
|
|
// Pad to full block size
|
|
dblock.resize(starting_block_size as usize, 0);
|
|
|
|
// Checksum: computed over entire block with checksum field zeroed
|
|
let dblock_checksum = crate::checksum::jenkins_lookup3(&dblock);
|
|
dblock[cksum_pos..cksum_pos + 4].copy_from_slice(&dblock_checksum.to_le_bytes());
|
|
debug_assert_eq!(dblock.len(), starting_block_size as usize);
|
|
|
|
// Build heap IDs
|
|
let heap_ids: Vec<Vec<u8>> = obj_offsets
|
|
.iter()
|
|
.map(|(off, len)| encode_managed_id(*off, *len, max_heap_size, heap_id_length))
|
|
.collect();
|
|
|
|
let mut blob = Vec::with_capacity(frhp.len() + dblock.len());
|
|
blob.extend_from_slice(&frhp);
|
|
blob.extend_from_slice(&dblock);
|
|
|
|
Ok(FractalHeapBlock {
|
|
blob,
|
|
frhp_addr,
|
|
btree_addr,
|
|
heap_ids,
|
|
heap_id_length,
|
|
})
|
|
}
|
|
|
|
/// Build a multi-block fractal heap: a root indirect block (FHIB) over direct
|
|
/// blocks sized by the doubling table. Used when the objects don't fit in a
|
|
/// single direct block.
|
|
///
|
|
/// Rows of the doubling table whose block size exceeds the maximum direct
|
|
/// block size hold child indirect blocks, as the HDF5 spec (and libhdf5)
|
|
/// reads them: a child in row `r` spans that row's block size of heap space
|
|
/// and has `log2(size) - log2(start * width) + 1` rows of its own, which may
|
|
/// in turn hold indirect blocks. Objects are packed into direct blocks in
|
|
/// heap-offset order and never span blocks; a block too small for the next
|
|
/// object is left unallocated (an undefined address), as libhdf5 skips rows
|
|
/// when it needs a bigger block. The caller has checked that every object
|
|
/// fits a maximum-size direct block (there is no huge-object path).
|
|
fn build_multiblock_fractal_heap(
|
|
serialized: &[Vec<u8>],
|
|
base_address: u64,
|
|
max_heap_size: u16,
|
|
heap_id_length: u16,
|
|
) -> Result<FractalHeapBlock, FormatError> {
|
|
let os = OFFSET_SIZE as usize;
|
|
let block_offset_bytes = (max_heap_size as usize).div_ceil(8);
|
|
let geom = HeapGeometry {
|
|
width: 4,
|
|
starting_block_size: 512,
|
|
max_direct_block_size: 65536,
|
|
dblock_header_size: 4 + 1 + os + block_offset_bytes + 4,
|
|
iblock_fixed_size: 5 + os + block_offset_bytes + 4,
|
|
max_heap_size,
|
|
};
|
|
|
|
// ---- Pack objects into the doubling table ----
|
|
let mut packer = HeapPacker {
|
|
geom: &geom,
|
|
objects: serialized,
|
|
next: 0,
|
|
blocks: Vec::new(),
|
|
obj_loc: vec![(0, 0); serialized.len()],
|
|
};
|
|
let root = packer.fill(0, None)?;
|
|
let HeapPacker {
|
|
blocks, obj_loc, ..
|
|
} = packer;
|
|
|
|
// ---- Addresses ----
|
|
let frhp_size = frhp_header_size(os, LENGTH_SIZE as usize);
|
|
let frhp_addr = base_address;
|
|
let fhib_addr = frhp_addr + frhp_size as u64;
|
|
let heap_len = root.subtree_size(&geom, &blocks);
|
|
let btree_addr = fhib_addr + heap_len;
|
|
|
|
// Bookkeeping totals, as libhdf5 keeps them: the managed space is what
|
|
// the root's rows span, the allocated space the direct blocks written,
|
|
// and the allocation iterator the heap offset after the last of them.
|
|
let cur_rows = root.nrows as u16;
|
|
let managed_space: u64 = (0..root.nrows).map(|r| geom.row_size(r) * geom.width).sum();
|
|
let alloc_space: u64 = blocks.iter().map(|b| b.size).sum();
|
|
let used: u64 = blocks
|
|
.iter()
|
|
.map(|b| geom.dblock_header_size as u64 + b.data.len() as u64)
|
|
.sum();
|
|
let free_space = alloc_space.saturating_sub(used);
|
|
let alloc_iter = blocks.last().map_or(0, |b| b.heap_offset + b.size);
|
|
|
|
// ---- FRHP header ----
|
|
let max_managed = geom.max_managed();
|
|
let frhp = write_frhp(WriteFrhp {
|
|
heap_id_length,
|
|
max_managed,
|
|
free_space,
|
|
managed_space,
|
|
alloc_space,
|
|
alloc_iter,
|
|
nobjects: serialized.len() as u64,
|
|
table_width: geom.width as u16,
|
|
starting_block_size: geom.starting_block_size,
|
|
max_direct_block_size: geom.max_direct_block_size,
|
|
max_heap_size,
|
|
root_addr: fhib_addr,
|
|
cur_rows,
|
|
});
|
|
debug_assert_eq!(frhp.len(), frhp_size);
|
|
|
|
// ---- Indirect and direct blocks, depth first after the root ----
|
|
let mut blob = frhp;
|
|
root.emit(&geom, &blocks, frhp_addr, fhib_addr, &mut blob);
|
|
debug_assert_eq!(blob.len() as u64, frhp_size as u64 + heap_len);
|
|
|
|
let heap_ids: Vec<Vec<u8>> = obj_loc
|
|
.iter()
|
|
.map(|(off, len)| encode_managed_id(*off, *len, max_heap_size, heap_id_length))
|
|
.collect();
|
|
|
|
Ok(FractalHeapBlock {
|
|
blob,
|
|
frhp_addr,
|
|
btree_addr,
|
|
heap_ids,
|
|
heap_id_length,
|
|
})
|
|
}
|
|
|
|
/// The doubling table of a heap the writer builds.
|
|
struct HeapGeometry {
|
|
width: u64,
|
|
starting_block_size: u64,
|
|
max_direct_block_size: u64,
|
|
dblock_header_size: usize,
|
|
/// An indirect block's size without its child entries.
|
|
iblock_fixed_size: usize,
|
|
max_heap_size: u16,
|
|
}
|
|
|
|
impl HeapGeometry {
|
|
fn row_size(&self, row: usize) -> u64 {
|
|
block_size_for_row(self.starting_block_size, row)
|
|
}
|
|
|
|
/// Rows holding direct blocks: `log2(max_direct / start) + 2`.
|
|
fn max_direct_rows(&self) -> usize {
|
|
(self.max_direct_block_size / self.starting_block_size).ilog2() as usize + 2
|
|
}
|
|
|
|
/// `log2(start * width)`, libhdf5's `first_row_bits`.
|
|
fn first_row_bits(&self) -> u32 {
|
|
(self.starting_block_size * self.width).ilog2()
|
|
}
|
|
|
|
/// Rows of an indirect block spanning `size` bytes of heap space
|
|
/// (libhdf5's `H5HF__dtable_size_to_rows`).
|
|
fn rows_for_size(&self, size: u64) -> usize {
|
|
(size.ilog2() - self.first_row_bits() + 1) as usize
|
|
}
|
|
|
|
/// Rows the root indirect block can have: enough to span the heap's
|
|
/// whole `2^max_heap_size` address space.
|
|
fn max_root_rows(&self) -> usize {
|
|
(u32::from(self.max_heap_size) - self.first_row_bits() + 1) as usize
|
|
}
|
|
|
|
/// The largest object a direct block holds.
|
|
fn max_managed(&self) -> u32 {
|
|
(self.max_direct_block_size - self.dblock_header_size as u64) as u32
|
|
}
|
|
}
|
|
|
|
/// A direct block the packer filled.
|
|
struct HeapDirectBlock {
|
|
size: u64,
|
|
heap_offset: u64,
|
|
data: Vec<u8>,
|
|
}
|
|
|
|
/// One entry of an indirect block.
|
|
enum HeapSlot {
|
|
/// Not allocated (undefined address).
|
|
Empty,
|
|
/// Index into the packer's direct blocks.
|
|
Direct(usize),
|
|
Indirect(HeapIndirectBlock),
|
|
}
|
|
|
|
struct HeapIndirectBlock {
|
|
heap_offset: u64,
|
|
nrows: usize,
|
|
/// `nrows * width` entries, row-major.
|
|
slots: Vec<HeapSlot>,
|
|
}
|
|
|
|
impl HeapIndirectBlock {
|
|
fn own_size(&self, geom: &HeapGeometry) -> u64 {
|
|
(geom.iblock_fixed_size + self.slots.len() * OFFSET_SIZE as usize) as u64
|
|
}
|
|
|
|
/// Bytes of this block and everything below it.
|
|
fn subtree_size(&self, geom: &HeapGeometry, blocks: &[HeapDirectBlock]) -> u64 {
|
|
self.own_size(geom)
|
|
+ self
|
|
.slots
|
|
.iter()
|
|
.map(|s| match s {
|
|
HeapSlot::Empty => 0,
|
|
HeapSlot::Direct(i) => blocks[*i].size,
|
|
HeapSlot::Indirect(ib) => ib.subtree_size(geom, blocks),
|
|
})
|
|
.sum::<u64>()
|
|
}
|
|
|
|
/// Append this block at `addr` (= `out`'s current end, relative to the
|
|
/// same base as `frhp_addr`), then its children in entry order.
|
|
fn emit(
|
|
&self,
|
|
geom: &HeapGeometry,
|
|
blocks: &[HeapDirectBlock],
|
|
frhp_addr: u64,
|
|
addr: u64,
|
|
out: &mut Vec<u8>,
|
|
) {
|
|
let block_offset_bytes = (geom.max_heap_size as usize).div_ceil(8);
|
|
let start = out.len();
|
|
out.extend_from_slice(b"FHIB");
|
|
out.push(0); // version
|
|
write_offset(out, frhp_addr, OFFSET_SIZE);
|
|
out.extend_from_slice(&self.heap_offset.to_le_bytes()[..block_offset_bytes]);
|
|
let mut child = addr + self.own_size(geom);
|
|
for s in &self.slots {
|
|
match s {
|
|
HeapSlot::Empty => write_undef_offset(out, OFFSET_SIZE),
|
|
HeapSlot::Direct(i) => {
|
|
write_offset(out, child, OFFSET_SIZE);
|
|
child += blocks[*i].size;
|
|
}
|
|
HeapSlot::Indirect(ib) => {
|
|
write_offset(out, child, OFFSET_SIZE);
|
|
child += ib.subtree_size(geom, blocks);
|
|
}
|
|
}
|
|
}
|
|
let checksum = crate::checksum::jenkins_lookup3(&out[start..]);
|
|
out.extend_from_slice(&checksum.to_le_bytes());
|
|
|
|
let mut child = addr + self.own_size(geom);
|
|
for s in &self.slots {
|
|
match s {
|
|
HeapSlot::Empty => {}
|
|
HeapSlot::Direct(i) => {
|
|
let b = &blocks[*i];
|
|
let d = out.len();
|
|
out.extend_from_slice(b"FHDB");
|
|
out.push(0); // version
|
|
write_offset(out, frhp_addr, OFFSET_SIZE);
|
|
out.extend_from_slice(&b.heap_offset.to_le_bytes()[..block_offset_bytes]);
|
|
let cksum_pos = out.len();
|
|
out.extend_from_slice(&[0u8; 4]); // checksum placeholder
|
|
out.extend_from_slice(&b.data);
|
|
out.resize(d + b.size as usize, 0);
|
|
let cksum = crate::checksum::jenkins_lookup3(&out[d..]);
|
|
out[cksum_pos..cksum_pos + 4].copy_from_slice(&cksum.to_le_bytes());
|
|
child += b.size;
|
|
}
|
|
HeapSlot::Indirect(ib) => {
|
|
ib.emit(geom, blocks, frhp_addr, child, out);
|
|
child += ib.subtree_size(geom, blocks);
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
/// Packs objects into a heap's doubling table in heap-offset order.
|
|
struct HeapPacker<'a> {
|
|
geom: &'a HeapGeometry,
|
|
objects: &'a [Vec<u8>],
|
|
/// The next object to place.
|
|
next: usize,
|
|
blocks: Vec<HeapDirectBlock>,
|
|
/// Each object's (heap offset, length).
|
|
obj_loc: Vec<(u64, u64)>,
|
|
}
|
|
|
|
impl HeapPacker<'_> {
|
|
/// Fill an indirect block at `heap_offset` with `nrows` rows, or, for the
|
|
/// root (`None`), with as many rows as the objects need.
|
|
fn fill(
|
|
&mut self,
|
|
heap_offset: u64,
|
|
nrows: Option<usize>,
|
|
) -> Result<HeapIndirectBlock, FormatError> {
|
|
let geom = self.geom;
|
|
let width = geom.width as usize;
|
|
let mut slots = Vec::new();
|
|
let mut off = heap_offset;
|
|
let mut row = 0usize;
|
|
while self.next < self.objects.len() && nrows.is_none_or(|n| row < n) {
|
|
if nrows.is_none() && row >= geom.max_root_rows() {
|
|
return Err(FormatError::SerializationError(format!(
|
|
"fractal heap: {} objects do not fit its {}-bit address space",
|
|
self.objects.len(),
|
|
geom.max_heap_size
|
|
)));
|
|
}
|
|
let size = geom.row_size(row);
|
|
for _ in 0..width {
|
|
if self.next == self.objects.len() {
|
|
slots.push(HeapSlot::Empty);
|
|
} else if row < geom.max_direct_rows() {
|
|
slots.push(self.fill_direct(off, size));
|
|
} else {
|
|
let child_rows = geom.rows_for_size(size);
|
|
// A child whose biggest direct block cannot hold the
|
|
// next object is skipped whole, not walked.
|
|
let biggest = geom.row_size(child_rows.min(geom.max_direct_rows()) - 1);
|
|
if self.objects[self.next].len() > (biggest as usize - geom.dblock_header_size)
|
|
{
|
|
slots.push(HeapSlot::Empty);
|
|
off += size;
|
|
continue;
|
|
}
|
|
let child = self.fill(off, Some(child_rows))?;
|
|
let used = child.slots.iter().any(|s| !matches!(s, HeapSlot::Empty));
|
|
slots.push(if used {
|
|
HeapSlot::Indirect(child)
|
|
} else {
|
|
HeapSlot::Empty
|
|
});
|
|
}
|
|
off += size;
|
|
}
|
|
row += 1;
|
|
}
|
|
let nrows = nrows.unwrap_or(row);
|
|
slots.resize_with(nrows * width, || HeapSlot::Empty);
|
|
Ok(HeapIndirectBlock {
|
|
heap_offset,
|
|
nrows,
|
|
slots,
|
|
})
|
|
}
|
|
|
|
/// Fill the direct block at `heap_offset` with as many of the next
|
|
/// objects as fit; leave it unallocated if not even the next one does.
|
|
fn fill_direct(&mut self, heap_offset: u64, size: u64) -> HeapSlot {
|
|
let header = self.geom.dblock_header_size;
|
|
let capacity = size as usize - header;
|
|
let mut data = Vec::new();
|
|
while let Some(obj) = self.objects.get(self.next) {
|
|
if data.len() + obj.len() > capacity {
|
|
break;
|
|
}
|
|
self.obj_loc[self.next] =
|
|
(heap_offset + (header + data.len()) as u64, obj.len() as u64);
|
|
data.extend_from_slice(obj);
|
|
self.next += 1;
|
|
}
|
|
if data.is_empty() && self.objects.get(self.next).is_some_and(|o| !o.is_empty()) {
|
|
return HeapSlot::Empty;
|
|
}
|
|
self.blocks.push(HeapDirectBlock {
|
|
size,
|
|
heap_offset,
|
|
data,
|
|
});
|
|
HeapSlot::Direct(self.blocks.len() - 1)
|
|
}
|
|
}
|
|
|
|
/// Doubling-table block size for `row`: rows 0 and 1 share the starting size;
|
|
/// row r (r ≥ 1) is `start * 2^(r-1)`.
|
|
fn block_size_for_row(starting_block_size: u64, row: usize) -> u64 {
|
|
if row <= 1 {
|
|
starting_block_size
|
|
} else {
|
|
starting_block_size << (row - 1)
|
|
}
|
|
}
|
|
|
|
/// Size in bytes of the FRHP header for the given offset/length sizes.
|
|
fn frhp_header_size(os: usize, ls: usize) -> usize {
|
|
4 + 1
|
|
+ 2
|
|
+ 2
|
|
+ 1
|
|
+ 4
|
|
+ ls
|
|
+ os
|
|
+ ls
|
|
+ os
|
|
+ ls
|
|
+ ls
|
|
+ ls
|
|
+ ls
|
|
+ ls
|
|
+ ls
|
|
+ ls
|
|
+ ls
|
|
+ 2
|
|
+ ls
|
|
+ ls
|
|
+ 2
|
|
+ 2
|
|
+ os
|
|
+ 2
|
|
+ 4
|
|
}
|
|
|
|
/// Parameters for [`write_frhp`].
|
|
struct WriteFrhp {
|
|
heap_id_length: u16,
|
|
max_managed: u32,
|
|
free_space: u64,
|
|
managed_space: u64,
|
|
alloc_space: u64,
|
|
/// Heap offset of the next direct block to allocate.
|
|
alloc_iter: u64,
|
|
nobjects: u64,
|
|
table_width: u16,
|
|
starting_block_size: u64,
|
|
max_direct_block_size: u64,
|
|
max_heap_size: u16,
|
|
root_addr: u64,
|
|
cur_rows: u16,
|
|
}
|
|
|
|
/// Serialize a fractal heap header (FRHP).
|
|
fn write_frhp(p: WriteFrhp) -> Vec<u8> {
|
|
let mut frhp = Vec::with_capacity(frhp_header_size(OFFSET_SIZE as usize, LENGTH_SIZE as usize));
|
|
frhp.extend_from_slice(b"FRHP");
|
|
frhp.push(0); // version
|
|
frhp.extend_from_slice(&p.heap_id_length.to_le_bytes());
|
|
frhp.extend_from_slice(&0u16.to_le_bytes()); // io_filter_encoded_length
|
|
frhp.push(0x02); // flags: bit 1 = checksum direct blocks
|
|
frhp.extend_from_slice(&p.max_managed.to_le_bytes());
|
|
write_length(&mut frhp, 0, LENGTH_SIZE); // next_huge_object_id
|
|
write_undef_offset(&mut frhp, OFFSET_SIZE); // btree_huge_objects_address
|
|
write_length(&mut frhp, p.free_space, LENGTH_SIZE); // free_space_managed_blocks
|
|
write_undef_offset(&mut frhp, OFFSET_SIZE); // free_space_mgr_addr
|
|
write_length(&mut frhp, p.managed_space, LENGTH_SIZE); // managed_space_in_heap
|
|
write_length(&mut frhp, p.alloc_space, LENGTH_SIZE); // allocated_managed_space
|
|
write_length(&mut frhp, p.alloc_iter, LENGTH_SIZE); // dblock_alloc_iter
|
|
write_length(&mut frhp, p.nobjects, LENGTH_SIZE); // managed_objects_count
|
|
write_length(&mut frhp, 0, LENGTH_SIZE); // huge_objects_size
|
|
write_length(&mut frhp, 0, LENGTH_SIZE); // huge_objects_count
|
|
write_length(&mut frhp, 0, LENGTH_SIZE); // tiny_objects_size
|
|
write_length(&mut frhp, 0, LENGTH_SIZE); // tiny_objects_count
|
|
frhp.extend_from_slice(&p.table_width.to_le_bytes());
|
|
write_length(&mut frhp, p.starting_block_size, LENGTH_SIZE);
|
|
write_length(&mut frhp, p.max_direct_block_size, LENGTH_SIZE);
|
|
frhp.extend_from_slice(&p.max_heap_size.to_le_bytes());
|
|
frhp.extend_from_slice(&1u16.to_le_bytes()); // starting # rows in root indirect block
|
|
write_offset(&mut frhp, p.root_addr, OFFSET_SIZE);
|
|
frhp.extend_from_slice(&p.cur_rows.to_le_bytes());
|
|
let checksum = crate::checksum::jenkins_lookup3(&frhp);
|
|
frhp.extend_from_slice(&checksum.to_le_bytes());
|
|
frhp
|
|
}
|
|
|
|
/// libhdf5 numbers the attributes of an object that tracks their creation
|
|
/// order with a 2-byte counter.
|
|
fn check_tracked_attr_count(track_order: bool, n: usize) -> Result<(), FormatError> {
|
|
if track_order && n > usize::from(u16::MAX) {
|
|
return Err(FormatError::SerializationError(format!(
|
|
"{n} attributes on one object with creation order tracked: libhdf5 \
|
|
numbers at most {} (set fewer, or turn off track_order)",
|
|
u16::MAX
|
|
)));
|
|
}
|
|
Ok(())
|
|
}
|
|
|
|
/// Build dense attribute storage for a set of attributes.
|
|
///
|
|
/// With `track_order` the Attribute Info message tracks creation order (an
|
|
/// attribute's creation order is its position in `attrs`) and a type-9
|
|
/// creation-order index follows the name index, as libhdf5 writes for h5py's
|
|
/// `track_order=True`. libhdf5 numbers at most 65 535 attributes.
|
|
pub(crate) fn build_dense_attrs(
|
|
attrs: &[AttributeMessage],
|
|
base_address: u64,
|
|
track_order: bool,
|
|
) -> Result<DenseAttrBlob, FormatError> {
|
|
check_tracked_attr_count(track_order, attrs.len())?;
|
|
// Dense attrs use v3 attribute messages (adds character set encoding byte).
|
|
let serialized: Vec<Vec<u8>> = attrs.iter().map(|a| a.serialize_v3(LENGTH_SIZE)).collect();
|
|
|
|
let name_hashes: Vec<u32> = attrs
|
|
.iter()
|
|
.map(|a| crate::checksum::jenkins_lookup3(a.name.as_bytes()))
|
|
.collect();
|
|
|
|
// Attribute heaps use max_heap_size 40 / heap ID length 8 (matching libhdf5).
|
|
let heap = build_single_block_fractal_heap(&serialized, base_address, 40, 8)?;
|
|
let frhp_addr = heap.frhp_addr;
|
|
let btree_addr = heap.btree_addr;
|
|
let heap_id_length = heap.heap_id_length;
|
|
let heap_ids = &heap.heap_ids;
|
|
|
|
// Build B-tree v2 type 8 records (17 bytes each), in the index's key
|
|
// order: libhdf5 compares the name hash, then — for names whose hashes
|
|
// collide — the names themselves (`strcmp`).
|
|
let record_size: u16 = heap_id_length + 1 + 4 + 4;
|
|
let mut order: Vec<usize> = (0..attrs.len()).collect();
|
|
order.sort_by(|&a, &b| {
|
|
name_hashes[a]
|
|
.cmp(&name_hashes[b])
|
|
.then_with(|| attrs[a].name.as_bytes().cmp(attrs[b].name.as_bytes()))
|
|
});
|
|
let records: Vec<Vec<u8>> = order
|
|
.into_iter()
|
|
.map(|i| {
|
|
let mut rec = Vec::with_capacity(record_size as usize);
|
|
rec.extend_from_slice(&heap_ids[i]);
|
|
rec.push(0); // msg_flags
|
|
rec.extend_from_slice(&(i as u32).to_le_bytes()); // creation_order
|
|
rec.extend_from_slice(&name_hashes[i].to_le_bytes()); // hash
|
|
rec
|
|
})
|
|
.collect();
|
|
let bthd_addr = btree_addr;
|
|
let mut blob = heap.blob;
|
|
blob.extend_from_slice(&dense_v2_btree(8, record_size, &records, bthd_addr)?);
|
|
|
|
let order = if track_order {
|
|
// Type 9 records: heap ID, message flags, creation order (the key).
|
|
let records: Vec<Vec<u8>> = heap_ids
|
|
.iter()
|
|
.enumerate()
|
|
.map(|(i, heap_id)| {
|
|
let mut rec = heap_id.clone();
|
|
rec.push(0); // msg_flags
|
|
rec.extend_from_slice(&(i as u32).to_le_bytes());
|
|
rec
|
|
})
|
|
.collect();
|
|
let corder_addr = base_address + blob.len() as u64;
|
|
blob.extend_from_slice(&dense_v2_btree(
|
|
9,
|
|
heap_id_length + 1 + 4,
|
|
&records,
|
|
corder_addr,
|
|
)?);
|
|
Some((attrs.len() as u16, corder_addr))
|
|
} else {
|
|
None
|
|
};
|
|
let attr_info = serialize_attribute_info(frhp_addr, bthd_addr, order);
|
|
|
|
Ok(DenseAttrBlob {
|
|
attr_info_message: attr_info,
|
|
blob,
|
|
})
|
|
}
|
|
|
|
// ---- Dense link blob ----
|
|
|
|
/// Pre-built dense link storage (fractal heap + B-tree v2 + link-info message).
|
|
pub(crate) struct DenseLinkBlob {
|
|
/// Serialized LinkInfo message (to embed in the group's object header).
|
|
pub(crate) link_info_message: Vec<u8>,
|
|
/// The combined fractal heap, name-index B-tree and (when creation order
|
|
/// is tracked) creation-order-index B-tree bytes.
|
|
pub(crate) blob: Vec<u8>,
|
|
}
|
|
|
|
/// libhdf5's node size for the dense link and attribute indexes it creates
|
|
/// (`H5G_NAME_BT2_NODE_SIZE`, `H5A_NAME_BT2_NODE_SIZE`, and the
|
|
/// creation-order indexes'), with their split and merge percentages.
|
|
const DENSE_BT2_NODE_SIZE: u32 = 512;
|
|
const DENSE_BT2_SPLIT_PERCENT: u8 = 100;
|
|
const DENSE_BT2_MERGE_PERCENT: u8 = 40;
|
|
|
|
/// A dense-storage v2 B-tree of `btree_type` holding `records` (already in
|
|
/// key order), laid out at `addr`: the header, then its nodes.
|
|
///
|
|
/// Up to 65 535 records go in one leaf node sized to hold them (the layout
|
|
/// the writer has always used, kept so those files do not change). A leaf's
|
|
/// record count is a 2-byte field, and libhdf5 sizes a leaf's capacity from
|
|
/// the node size: a node with room for more than 65 535 records makes it
|
|
/// overflow that count when it adds one, so the node is capped at a full
|
|
/// leaf. More records get libhdf5's own 512-byte nodes, with internal nodes
|
|
/// above the leaves.
|
|
fn dense_v2_btree(
|
|
btree_type: u8,
|
|
record_size: u16,
|
|
records: &[Vec<u8>],
|
|
addr: u64,
|
|
) -> Result<Vec<u8>, FormatError> {
|
|
let rs = usize::from(record_size);
|
|
let n = records.len();
|
|
let node_size = if n <= usize::from(u16::MAX) {
|
|
let btlf_size = 4 + 1 + 1 + n * rs + 4;
|
|
let max_node = 4 + 1 + 1 + usize::from(u16::MAX) * rs + 4;
|
|
u32::try_from(btlf_size.next_power_of_two().max(512).min(max_node))
|
|
.map_err(|_| FormatError::Overflow("B-tree v2 node size".into()))?
|
|
} else {
|
|
DENSE_BT2_NODE_SIZE
|
|
};
|
|
let flat: Vec<u8> = records
|
|
.iter()
|
|
.inspect(|r| debug_assert_eq!(r.len(), rs))
|
|
.flat_map(|r| r.iter().copied())
|
|
.collect();
|
|
build_btree_v2(
|
|
BTreeV2Params {
|
|
tree_type: btree_type,
|
|
node_size,
|
|
record_size,
|
|
split_percent: DENSE_BT2_SPLIT_PERCENT,
|
|
merge_percent: DENSE_BT2_MERGE_PERCENT,
|
|
},
|
|
&flat,
|
|
addr,
|
|
OFFSET_SIZE,
|
|
LENGTH_SIZE,
|
|
)
|
|
}
|
|
|
|
/// Build dense link storage for a group's links, laid out at `base_address`.
|
|
///
|
|
/// Mirrors [`build_dense_attrs`]: each link is stored as a serialized Link
|
|
/// message in a fractal heap, indexed by a v2 B-tree of **type 5** (link-name
|
|
/// index, record = name hash + heap ID). With `track_order` (every link then
|
|
/// carries its creation order) a **type 6** B-tree (creation-order index,
|
|
/// record = creation order + heap ID) follows, as libhdf5 writes for a group
|
|
/// created with an indexed creation order. The returned LinkInfo message
|
|
/// points at the heap and the B-trees.
|
|
pub(crate) fn build_dense_links(
|
|
links: &[LinkMessage],
|
|
base_address: u64,
|
|
track_order: bool,
|
|
) -> Result<DenseLinkBlob, FormatError> {
|
|
let serialized: Vec<Vec<u8>> = links.iter().map(|l| l.serialize(OFFSET_SIZE)).collect();
|
|
|
|
// libhdf5's link heap uses max_heap_size 32 / heap ID length 7 (vs 40/8 for
|
|
// attributes), giving a 7-byte heap ID and an 11-byte type-5 record.
|
|
let heap = build_single_block_fractal_heap(&serialized, base_address, 32, 7)?;
|
|
let heap_id_length = heap.heap_id_length;
|
|
|
|
// Type 5 records: hash(4) + heap_id. The B-tree's key is the name hash
|
|
// and, for names whose hashes collide, the name (libhdf5 compares them
|
|
// with `strcmp`): records out of that order are not found by name.
|
|
let mut by_name: Vec<(u32, usize)> = links
|
|
.iter()
|
|
.enumerate()
|
|
.map(|(i, l)| (crate::checksum::jenkins_lookup3(l.name.as_bytes()), i))
|
|
.collect();
|
|
by_name.sort_unstable_by(|&(ha, a), &(hb, b)| {
|
|
ha.cmp(&hb)
|
|
.then_with(|| links[a].name.as_bytes().cmp(links[b].name.as_bytes()))
|
|
});
|
|
let name_records: Vec<Vec<u8>> = by_name
|
|
.iter()
|
|
.map(|&(hash, i)| {
|
|
let mut rec = hash.to_le_bytes().to_vec();
|
|
rec.extend_from_slice(&heap.heap_ids[i]);
|
|
rec
|
|
})
|
|
.collect();
|
|
let name_bt_addr = heap.btree_addr;
|
|
let mut blob = heap.blob;
|
|
blob.extend_from_slice(&dense_v2_btree(
|
|
5,
|
|
4 + heap_id_length,
|
|
&name_records,
|
|
name_bt_addr,
|
|
)?);
|
|
|
|
let link_info_message = if track_order {
|
|
// Type 6 records: creation order(8) + heap_id, sorted by order.
|
|
let mut by_order: Vec<(u64, usize)> = links
|
|
.iter()
|
|
.enumerate()
|
|
.map(|(i, l)| (l.creation_order.unwrap_or(i as u64), i))
|
|
.collect();
|
|
by_order.sort_unstable();
|
|
let order_records: Vec<Vec<u8>> = by_order
|
|
.iter()
|
|
.map(|&(order, i)| {
|
|
let mut rec = order.to_le_bytes().to_vec();
|
|
rec.extend_from_slice(&heap.heap_ids[i]);
|
|
rec
|
|
})
|
|
.collect();
|
|
let order_bt_addr = base_address + blob.len() as u64;
|
|
blob.extend_from_slice(&dense_v2_btree(
|
|
6,
|
|
8 + heap_id_length,
|
|
&order_records,
|
|
order_bt_addr,
|
|
)?);
|
|
let next_order = by_order.last().map_or(0, |&(o, _)| o + 1);
|
|
serialize_link_info(
|
|
Some(next_order),
|
|
heap.frhp_addr,
|
|
name_bt_addr,
|
|
Some(order_bt_addr),
|
|
)
|
|
} else {
|
|
serialize_link_info(None, heap.frhp_addr, name_bt_addr, None)
|
|
};
|
|
|
|
Ok(DenseLinkBlob {
|
|
link_info_message,
|
|
blob,
|
|
})
|
|
}
|
|
|
|
/// Serialize a LinkInfo message (version 0). `max_creation_order` (the next
|
|
/// creation order to assign) is present when creation order is tracked, and
|
|
/// `btree_corder_addr` when it is indexed; both set flag bits.
|
|
fn serialize_link_info(
|
|
max_creation_order: Option<u64>,
|
|
fh_addr: u64,
|
|
btree_name_addr: u64,
|
|
btree_corder_addr: Option<u64>,
|
|
) -> Vec<u8> {
|
|
let mut data = Vec::new();
|
|
data.push(0); // version
|
|
let mut flags = 0u8;
|
|
if max_creation_order.is_some() {
|
|
flags |= 0x01; // creation order tracked
|
|
}
|
|
if btree_corder_addr.is_some() {
|
|
flags |= 0x02; // creation order indexed
|
|
}
|
|
data.push(flags);
|
|
if let Some(m) = max_creation_order {
|
|
data.extend_from_slice(&m.to_le_bytes());
|
|
}
|
|
write_offset(&mut data, fh_addr, OFFSET_SIZE);
|
|
write_offset(&mut data, btree_name_addr, OFFSET_SIZE);
|
|
if let Some(a) = btree_corder_addr {
|
|
write_offset(&mut data, a, OFFSET_SIZE);
|
|
}
|
|
data
|
|
}
|
|
|
|
fn encode_managed_id(offset: u64, length: u64, max_heap_size: u16, id_length: u16) -> Vec<u8> {
|
|
let mut id = vec![0u8; id_length as usize];
|
|
id[0] = 0x00; // type = 0 (managed)
|
|
let combined = offset | (length << max_heap_size);
|
|
let payload_len = (id_length as usize) - 1;
|
|
for i in 0..payload_len.min(8) {
|
|
id[1 + i] = ((combined >> (i * 8)) & 0xFF) as u8;
|
|
}
|
|
id
|
|
}
|
|
|
|
/// Serialize an Attribute Info message (version 0). `order` — the next
|
|
/// creation order to assign and the creation-order index's address — is
|
|
/// present when creation order is tracked and indexed.
|
|
fn serialize_attribute_info(
|
|
fh_addr: u64,
|
|
btree_name_addr: u64,
|
|
order: Option<(u16, u64)>,
|
|
) -> Vec<u8> {
|
|
let mut data = Vec::new();
|
|
data.push(0); // version
|
|
data.push(if order.is_some() { 0x03 } else { 0x00 }); // flags: tracked, indexed
|
|
if let Some((next, _)) = order {
|
|
data.extend_from_slice(&next.to_le_bytes());
|
|
}
|
|
data.extend_from_slice(&fh_addr.to_le_bytes());
|
|
data.extend_from_slice(&btree_name_addr.to_le_bytes());
|
|
if let Some((_, corder_addr)) = order {
|
|
data.extend_from_slice(&corder_addr.to_le_bytes());
|
|
}
|
|
data
|
|
}
|
|
|
|
// ---- VDS helpers ----
|
|
|
|
/// Serialize VDS mappings for storage in a global heap object.
|
|
///
|
|
/// Delegates to `data_layout_write::serialize_vds_mappings` (the canonical
|
|
/// implementation with full version/external-file handling), then appends a
|
|
/// trailing 4-byte Jenkins lookup3 checksum that parsers skip after consuming
|
|
/// all `nused` entries.
|
|
pub(crate) fn serialize_vds_mappings(mappings: &[VdsMapping]) -> Vec<u8> {
|
|
let mut buf = crate::data_layout_write::serialize_vds_mappings(mappings, 8);
|
|
let cksum = crate::checksum::jenkins_lookup3(&buf);
|
|
buf.extend_from_slice(&cksum.to_le_bytes());
|
|
buf
|
|
}
|
|
|
|
/// Build a minimal global heap collection containing a single object.
|
|
///
|
|
/// Returns the serialized collection bytes. The object index is always 1.
|
|
///
|
|
/// Global heap collection layout:
|
|
/// ```text
|
|
/// "GCOL"(4) · version(1) · reserved(3) · collection_size(8)
|
|
/// · [index(2) · ref_count(2) · reserved(4) · object_size(8) · data · padding]
|
|
/// · free-space-marker(2)
|
|
/// ```
|
|
pub(crate) fn build_global_heap_collection(object_data: &[u8]) -> Vec<u8> {
|
|
let ls = LENGTH_SIZE as usize;
|
|
let header_size = 8 + ls; // sig(4)+ver(1)+rsv(3)+coll_size(ls)
|
|
let obj_header_size = 8 + ls; // idx(2)+rc(2)+rsv(4)+obj_size(ls)
|
|
let padded_data_len = pad8(object_data.len());
|
|
let free_marker_size = 2;
|
|
let collection_size = header_size + obj_header_size + padded_data_len + free_marker_size;
|
|
|
|
let mut buf = Vec::with_capacity(collection_size);
|
|
buf.extend_from_slice(b"GCOL");
|
|
buf.push(1); // version
|
|
buf.extend_from_slice(&[0u8; 3]); // reserved
|
|
buf.extend_from_slice(&(collection_size as u64).to_le_bytes()); // collection_size
|
|
|
|
// Object 1
|
|
buf.extend_from_slice(&1u16.to_le_bytes()); // index
|
|
buf.extend_from_slice(&1u16.to_le_bytes()); // reference count
|
|
buf.extend_from_slice(&[0u8; 4]); // reserved
|
|
buf.extend_from_slice(&(object_data.len() as u64).to_le_bytes()); // object size
|
|
buf.extend_from_slice(object_data);
|
|
// Pad object data to 8-byte boundary
|
|
let pad = padded_data_len - object_data.len();
|
|
buf.extend_from_slice(&vec![0u8; pad]);
|
|
|
|
// Free space marker
|
|
buf.extend_from_slice(&0u16.to_le_bytes());
|
|
|
|
debug_assert_eq!(buf.len(), collection_size);
|
|
buf
|
|
}
|
|
|
|
/// Round up to the next multiple of 8.
|
|
fn pad8(x: usize) -> usize {
|
|
(x + 7) & !7
|
|
}
|
|
|
|
/// Build a Virtual Dataset object header.
|
|
///
|
|
/// The layout message for a VDS dataset is:
|
|
/// ```text
|
|
/// version(1=4) · class(1=3) · global_heap_address(8) · global_heap_index(4)
|
|
/// ```
|
|
pub(crate) fn build_vds_dataset_oh(
|
|
dt: &Datatype,
|
|
ds: &Dataspace,
|
|
global_heap_addr: u64,
|
|
attrs: AttrStorage<'_>,
|
|
fill_message: &[u8],
|
|
refcount: u32,
|
|
) -> Result<Vec<u8>, FormatError> {
|
|
let mut w = ObjectHeaderWriter::new();
|
|
w.add_message_with_flags(MessageType::Datatype, dt.serialize(), 0x01);
|
|
w.add_message(MessageType::Dataspace, ds.serialize(LENGTH_SIZE));
|
|
w.add_message_with_flags(MessageType::FillValue, fill_message.to_vec(), 0x01);
|
|
// VDS layout message: version=4, class=3, global_heap_address(8), global_heap_index=1(4)
|
|
let mut dl = Vec::new();
|
|
dl.push(4u8); // version
|
|
dl.push(3u8); // class = virtual
|
|
dl.extend_from_slice(&global_heap_addr.to_le_bytes());
|
|
dl.extend_from_slice(&1u32.to_le_bytes()); // object index 1 in the collection
|
|
w.add_message(MessageType::DataLayout, dl);
|
|
attrs.add_to(&mut w);
|
|
add_refcount(&mut w, refcount);
|
|
w.serialize()
|
|
}
|
|
|
|
fn write_offset(buf: &mut Vec<u8>, val: u64, offset_size: u8) {
|
|
match offset_size {
|
|
2 => buf.extend_from_slice(&(val as u16).to_le_bytes()),
|
|
4 => buf.extend_from_slice(&(val as u32).to_le_bytes()),
|
|
8 => buf.extend_from_slice(&val.to_le_bytes()),
|
|
_ => {}
|
|
}
|
|
}
|
|
|
|
fn write_length(buf: &mut Vec<u8>, val: u64, length_size: u8) {
|
|
write_offset(buf, val, length_size);
|
|
}
|
|
|
|
fn write_undef_offset(buf: &mut Vec<u8>, offset_size: u8) {
|
|
for _ in 0..offset_size {
|
|
buf.push(0xFF);
|
|
}
|
|
}
|
|
|
|
// ---- FileWriter ----
|
|
|
|
/// The main file creation API.
|
|
///
|
|
/// Groups nest to any depth: a name may be a path (`"a/b/x"`), and missing
|
|
/// intermediate groups are created, as h5py does; groups also nest through
|
|
/// [`GroupBuilder::add_group`]. See [`GroupBuilder`] for how names and
|
|
/// repeated groups are handled, and for soft, hard and external links.
|
|
pub struct FileWriter {
|
|
/// The root group's contents (its name is unused).
|
|
root: GroupBuilder,
|
|
/// Default for groups and datasets that do not set their own
|
|
/// `track_order`.
|
|
track_order: bool,
|
|
/// Global alignment threshold: datasets with raw data >= this many bytes
|
|
/// will have their data aligned to `alignment_bytes`.
|
|
alignment_threshold: usize,
|
|
/// Global alignment boundary in bytes (0 = disabled).
|
|
alignment_bytes: usize,
|
|
/// File space page size. When set, the file uses libhdf5's paged
|
|
/// file-space strategy (a File Space Info message in the superblock
|
|
/// extension).
|
|
page_size: Option<u32>,
|
|
}
|
|
|
|
impl Default for FileWriter {
|
|
fn default() -> Self {
|
|
Self::new()
|
|
}
|
|
}
|
|
|
|
/// A dataset ready for layout.
|
|
struct DsFlat {
|
|
dt: Datatype,
|
|
ds: Dataspace,
|
|
raw: Vec<u8>,
|
|
attrs: Vec<AttributeMessage>,
|
|
chunk_options: ChunkOptions,
|
|
maxshape: Option<Vec<u64>>,
|
|
/// Serialized Fill Value message.
|
|
fill_message: Vec<u8>,
|
|
compact: bool,
|
|
alignment: usize,
|
|
/// VDS source mappings (set for Virtual datasets).
|
|
virtual_sources: Option<Vec<VdsMapping>>,
|
|
/// Number of hard links to the dataset.
|
|
refcount: u32,
|
|
/// Track (and index) attribute creation order.
|
|
track_order: bool,
|
|
}
|
|
|
|
/// Convert a DatasetBuilder into a DsFlat, handling VDS (which does not
|
|
/// require a `data` field).
|
|
fn flatten_ds(
|
|
db: DatasetBuilder,
|
|
refcount: u32,
|
|
default_track_order: bool,
|
|
) -> Result<DsFlat, FormatError> {
|
|
let track_order = db.track_order.unwrap_or(default_track_order);
|
|
let dt = db.datatype.ok_or(FormatError::DatasetMissingData)?;
|
|
let shape = db.shape.ok_or(FormatError::DatasetMissingShape)?;
|
|
let is_vds = db.virtual_sources.is_some();
|
|
let raw = if is_vds {
|
|
// VDS datasets have no raw data stored in this file.
|
|
db.data.unwrap_or_default()
|
|
} else {
|
|
db.data.ok_or(FormatError::DatasetMissingData)?
|
|
};
|
|
let max_dimensions = db.maxshape.clone();
|
|
let dspace = Dataspace {
|
|
space_type: if shape.is_empty() {
|
|
DataspaceType::Scalar
|
|
} else {
|
|
DataspaceType::Simple
|
|
},
|
|
rank: shape.len() as u8,
|
|
dimensions: shape,
|
|
max_dimensions,
|
|
};
|
|
let mut attrs = Vec::new();
|
|
for (n, v) in &db.attrs {
|
|
attrs.push(build_attr_message(n, v));
|
|
}
|
|
#[cfg(feature = "provenance")]
|
|
if let Some(ref prov) = db.provenance {
|
|
let p = crate::provenance::Provenance {
|
|
creator: prov.creator.clone(),
|
|
timestamp: prov.timestamp.clone(),
|
|
source: prov.source.clone(),
|
|
};
|
|
// The provenance attributes replace any the caller set by hand.
|
|
let prov = p.build_attrs(&raw);
|
|
attrs.retain(|a| prov.iter().all(|b| b.name != a.name));
|
|
attrs.extend(prov);
|
|
}
|
|
let fill_message = fill_value_message(db.fill_time, db.fill_value.as_deref(), &dt)?;
|
|
Ok(DsFlat {
|
|
dt,
|
|
ds: dspace,
|
|
raw,
|
|
attrs,
|
|
chunk_options: db.chunk_options,
|
|
maxshape: db.maxshape,
|
|
fill_message,
|
|
compact: db.compact,
|
|
alignment: db.alignment,
|
|
virtual_sources: db.virtual_sources,
|
|
refcount,
|
|
track_order,
|
|
})
|
|
}
|
|
|
|
/// A group ready for layout.
|
|
struct GrpFlat {
|
|
attrs: Vec<AttributeMessage>,
|
|
links: Vec<writer_tree::Link>,
|
|
track_order: bool,
|
|
refcount: u32,
|
|
}
|
|
|
|
impl GrpFlat {
|
|
fn link_messages(&self, group_addrs: &[u64], ds_addrs: &[u64]) -> Vec<LinkMessage> {
|
|
self.links
|
|
.iter()
|
|
.map(|l| link_message(l, group_addrs, ds_addrs))
|
|
.collect()
|
|
}
|
|
}
|
|
|
|
impl FileWriter {
|
|
pub fn new() -> Self {
|
|
Self {
|
|
root: GroupBuilder::new("/"),
|
|
track_order: false,
|
|
alignment_threshold: 0,
|
|
alignment_bytes: 0,
|
|
page_size: None,
|
|
}
|
|
}
|
|
|
|
/// Set global file alignment: datasets with raw data >= `threshold` bytes
|
|
/// will have their data aligned to `bytes` boundary.
|
|
///
|
|
/// For example, `.alignment(1, 4096)` aligns all datasets to 4KB pages.
|
|
pub fn alignment(&mut self, threshold: usize, bytes: usize) -> &mut Self {
|
|
self.alignment_threshold = threshold;
|
|
self.alignment_bytes = bytes;
|
|
self
|
|
}
|
|
|
|
/// Write the file with libhdf5's *paged* file-space strategy and the given
|
|
/// page size, as `H5Pset_file_space_strategy(H5F_FSPACE_STRATEGY_PAGE)` +
|
|
/// `H5Pset_file_space_page_size` (h5py: `fs_strategy="page"`,
|
|
/// `fs_page_size=...`) do: a v3 superblock with an extension holding a
|
|
/// File Space Info message, and the file padded to a whole number of
|
|
/// pages. Readers with a page buffer can then fetch metadata page by page.
|
|
///
|
|
/// `page_size` must be between 512 bytes and 1 GiB (libhdf5's limits);
|
|
/// [`Self::finish`] fails otherwise. This used to write a "version 4"
|
|
/// superblock, which does not exist and no HDF5 library can open.
|
|
pub fn with_page_size(&mut self, page_size: u32) -> &mut Self {
|
|
self.page_size = Some(page_size);
|
|
self
|
|
}
|
|
|
|
/// Track (and index) creation order — of links and attributes in every
|
|
/// group that does not set its own [`GroupBuilder::track_order`], the
|
|
/// root included, and of attributes on every dataset that does not set
|
|
/// its own [`DatasetBuilder::track_order`] — as h5py's
|
|
/// `track_order=True`: libhdf5 then lists members and attributes in the
|
|
/// order they were added. Off by default (they are listed by name).
|
|
pub fn track_order(&mut self, track: bool) -> &mut Self {
|
|
self.track_order = track;
|
|
self
|
|
}
|
|
|
|
/// Start a group. The builder is detached: fill it, then pass
|
|
/// `finish()`'s result to [`Self::add_group`]. `name` may be a path
|
|
/// (`"a/b"`); missing intermediate groups are created.
|
|
pub fn create_group(&mut self, name: &str) -> GroupBuilder {
|
|
GroupBuilder::new(name)
|
|
}
|
|
|
|
/// Add a finished group to the root group.
|
|
pub fn add_group(&mut self, group: FinishedGroup) {
|
|
self.root.add_group(group);
|
|
}
|
|
|
|
/// Create a dataset. `name` may be a path (`"a/b/x"`, or `"/a/b/x"`);
|
|
/// missing intermediate groups are created.
|
|
pub fn create_dataset(&mut self, name: &str) -> &mut DatasetBuilder {
|
|
self.root.create_dataset(name)
|
|
}
|
|
|
|
pub fn set_root_attr(&mut self, name: &str, value: AttrValue) {
|
|
self.root.set_attr(name, value);
|
|
}
|
|
|
|
/// Add a soft link `name` (a path from the root) to `target`. See
|
|
/// [`GroupBuilder::add_soft_link`].
|
|
pub fn add_soft_link(&mut self, name: &str, target: &str) -> &mut Self {
|
|
self.root.add_soft_link(name, target);
|
|
self
|
|
}
|
|
|
|
/// Add another hard link `name` (a path from the root) to the object at
|
|
/// `target`. See [`GroupBuilder::add_hard_link`].
|
|
pub fn add_hard_link(&mut self, name: &str, target: &str) -> &mut Self {
|
|
self.root.add_hard_link(name, target);
|
|
self
|
|
}
|
|
|
|
/// Add an external link `name` (a path from the root) to `target_path`
|
|
/// in the file `target_file`.
|
|
pub fn add_external_link(
|
|
&mut self,
|
|
name: &str,
|
|
target_file: &str,
|
|
target_path: &str,
|
|
) -> &mut Self {
|
|
self.root.add_external_link(name, target_file, target_path);
|
|
self
|
|
}
|
|
|
|
pub fn finish(self) -> Result<Vec<u8>, FormatError> {
|
|
let page_size = self.page_size;
|
|
if let Some(ps) = page_size
|
|
&& !(MIN_FILE_SPACE_PAGE_SIZE..=MAX_FILE_SPACE_PAGE_SIZE).contains(&ps)
|
|
{
|
|
return Err(FormatError::SerializationError(format!(
|
|
"file space page size {ps} is outside libhdf5's \
|
|
{MIN_FILE_SPACE_PAGE_SIZE}..={MAX_FILE_SPACE_PAGE_SIZE} bytes"
|
|
)));
|
|
}
|
|
|
|
// The group tree, in layout order: groups depth-first from the root,
|
|
// then every group's datasets in the same order.
|
|
let tree = writer_tree::build(self.root, self.track_order)?;
|
|
let all_ds: Vec<DsFlat> = tree
|
|
.datasets
|
|
.into_iter()
|
|
.map(|(db, refcount)| flatten_ds(db, refcount, self.track_order))
|
|
.collect::<Result<_, _>>()?;
|
|
let groups: Vec<GrpFlat> = tree
|
|
.groups
|
|
.into_iter()
|
|
.map(|g| GrpFlat {
|
|
attrs: g
|
|
.attrs
|
|
.iter()
|
|
.map(|(n, v)| build_attr_message(n, v))
|
|
.collect(),
|
|
links: g.links,
|
|
track_order: g.track_order,
|
|
refcount: g.refcount,
|
|
})
|
|
.collect();
|
|
|
|
// Refuse up front what dense storage would refuse after the work.
|
|
let tracked = groups
|
|
.iter()
|
|
.map(|g| (g.track_order, g.attrs.len()))
|
|
.chain(all_ds.iter().map(|d| (d.track_order, d.attrs.len())));
|
|
for (track, n) in tracked {
|
|
check_tracked_attr_count(track, n)?;
|
|
}
|
|
|
|
// Every datatype must have an on-disk encoding before anything is laid
|
|
// out: `Datatype::serialize` itself cannot report a failure.
|
|
let group_attrs = groups.iter().flat_map(|g| &g.attrs);
|
|
let ds_attrs = all_ds.iter().flat_map(|d| &d.attrs);
|
|
for a in group_attrs.chain(ds_attrs) {
|
|
a.datatype.check_encodable()?;
|
|
}
|
|
for d in &all_ds {
|
|
d.dt.check_encodable()?;
|
|
}
|
|
|
|
let is_vds: Vec<bool> = all_ds.iter().map(|d| d.virtual_sources.is_some()).collect();
|
|
let is_chunked: Vec<bool> = all_ds
|
|
.iter()
|
|
.enumerate()
|
|
.map(|(i, d)| {
|
|
// Only a dataset that can grow needs chunks; a maxshape equal
|
|
// to the shape is as fixed as no maxshape at all.
|
|
let resizable = d.maxshape.as_ref().is_some_and(|m| *m != d.ds.dimensions);
|
|
!is_vds[i] && (d.chunk_options.is_chunked() || resizable)
|
|
})
|
|
.collect();
|
|
// Determine which datasets use compact storage
|
|
let is_compact: Vec<bool> = all_ds
|
|
.iter()
|
|
.enumerate()
|
|
.map(|(i, d)| {
|
|
!is_vds[i] && !is_chunked[i] && d.compact && d.raw.len() <= MAX_COMPACT_DATA_SIZE
|
|
})
|
|
.collect();
|
|
let group_dense: Vec<bool> = groups
|
|
.iter()
|
|
.map(|g| g.attrs.len() > DENSE_ATTR_THRESHOLD)
|
|
.collect();
|
|
let ds_dense: Vec<bool> = all_ds
|
|
.iter()
|
|
.map(|d| d.attrs.len() > DENSE_ATTR_THRESHOLD)
|
|
.collect();
|
|
|
|
// Dense link decision: a group with more than the compact threshold of
|
|
// links stores them in a fractal heap + v2 B-tree instead of inline.
|
|
let group_links_dense: Vec<bool> = groups
|
|
.iter()
|
|
.map(|g| g.links.len() > DENSE_LINK_THRESHOLD)
|
|
.collect();
|
|
|
|
// Pass 1: compute OH sizes with dummy addresses. Link messages and
|
|
// the Link Info message are the same size whatever the addresses.
|
|
let group_oh_sizes: Vec<usize> = groups
|
|
.iter()
|
|
.enumerate()
|
|
.map(|(gi, g)| {
|
|
let dummy_links = g.link_messages(&[], &[]);
|
|
let attr_blob = group_dense[gi]
|
|
.then(|| build_dense_attrs(&g.attrs, 0, g.track_order))
|
|
.transpose()?;
|
|
let li = if group_links_dense[gi] {
|
|
serialize_link_info(
|
|
g.track_order.then_some(0),
|
|
0,
|
|
0,
|
|
g.track_order.then_some(0),
|
|
)
|
|
} else {
|
|
compact_link_info(g.track_order, g.links.len())
|
|
};
|
|
build_group_oh(
|
|
&dummy_links,
|
|
&li,
|
|
group_links_dense[gi],
|
|
AttrStorage {
|
|
attrs: &g.attrs,
|
|
dense: attr_blob.as_ref(),
|
|
track_order: g.track_order,
|
|
},
|
|
g.refcount,
|
|
)
|
|
.map(|oh| oh.len())
|
|
})
|
|
.collect::<Result<_, _>>()?;
|
|
|
|
struct DataBlob {
|
|
data: Vec<u8>,
|
|
oh_bytes: Vec<u8>,
|
|
/// Cached compressed chunks for chunked datasets; reused in Pass 2
|
|
/// to avoid re-compressing the same data.
|
|
precompressed: Option<PrecompressedChunks>,
|
|
}
|
|
|
|
let mut dummy_blobs: Vec<DataBlob> = Vec::new();
|
|
let mut dummy_cursor = 0u64;
|
|
for (i, d) in all_ds.iter().enumerate() {
|
|
let dense_blob = ds_dense[i]
|
|
.then(|| build_dense_attrs(&d.attrs, 0, d.track_order))
|
|
.transpose()?;
|
|
if is_vds[i] {
|
|
// VDS: dummy OH with address 0 to get the OH size. The global
|
|
// heap blob will be placed after the OHs in pass 2.
|
|
let oh = build_vds_dataset_oh(
|
|
&d.dt,
|
|
&d.ds,
|
|
0, // dummy address
|
|
AttrStorage {
|
|
attrs: &d.attrs,
|
|
dense: dense_blob.as_ref(),
|
|
track_order: d.track_order,
|
|
},
|
|
&d.fill_message,
|
|
d.refcount,
|
|
)?;
|
|
// Global heap blob size is address-independent; compute it now
|
|
// so pass 2 can place it correctly.
|
|
let vds_mappings = d.virtual_sources.as_deref().unwrap_or(&[]);
|
|
let gcol_bytes =
|
|
build_global_heap_collection(&serialize_vds_mappings(vds_mappings));
|
|
dummy_blobs.push(DataBlob {
|
|
data: gcol_bytes, // store heap blob here temporarily
|
|
oh_bytes: oh,
|
|
precompressed: None,
|
|
});
|
|
} else if is_chunked[i] {
|
|
let elem_size = d.dt.type_size() as usize;
|
|
let chunk_dims = d
|
|
.chunk_options
|
|
.resolve_chunk_dims_for(&d.ds.dimensions, elem_size);
|
|
// Compress once in Pass 1; cache the result so Pass 2 can skip
|
|
// re-compression and just rebuild the index with real addresses.
|
|
let pre = precompress_chunks(
|
|
&d.raw,
|
|
&d.ds.dimensions,
|
|
&chunk_dims,
|
|
elem_size,
|
|
&d.chunk_options,
|
|
)?;
|
|
let result = build_chunked_data_from_precompressed(
|
|
&pre,
|
|
dummy_cursor,
|
|
d.maxshape.as_deref(),
|
|
)?;
|
|
dummy_cursor += result.data_bytes.len() as u64;
|
|
let oh = build_chunked_dataset_oh(
|
|
&d.dt,
|
|
&d.ds,
|
|
&result.layout_message,
|
|
result.pipeline_message.as_deref(),
|
|
AttrStorage {
|
|
attrs: &d.attrs,
|
|
dense: dense_blob.as_ref(),
|
|
track_order: d.track_order,
|
|
},
|
|
&d.fill_message,
|
|
d.refcount,
|
|
)?;
|
|
dummy_blobs.push(DataBlob {
|
|
data: result.data_bytes,
|
|
oh_bytes: oh,
|
|
precompressed: Some(pre),
|
|
});
|
|
} else if is_compact[i] {
|
|
let oh = build_compact_dataset_oh(
|
|
&d.dt,
|
|
&d.ds,
|
|
&d.raw,
|
|
AttrStorage {
|
|
attrs: &d.attrs,
|
|
dense: dense_blob.as_ref(),
|
|
track_order: d.track_order,
|
|
},
|
|
&d.fill_message,
|
|
d.refcount,
|
|
)?;
|
|
dummy_blobs.push(DataBlob {
|
|
data: vec![],
|
|
oh_bytes: oh,
|
|
precompressed: None,
|
|
});
|
|
} else {
|
|
let oh = build_dataset_oh(
|
|
&d.dt,
|
|
&d.ds,
|
|
0,
|
|
d.raw.len() as u64,
|
|
AttrStorage {
|
|
attrs: &d.attrs,
|
|
dense: dense_blob.as_ref(),
|
|
track_order: d.track_order,
|
|
},
|
|
&d.fill_message,
|
|
d.refcount,
|
|
)?;
|
|
dummy_blobs.push(DataBlob {
|
|
data: vec![],
|
|
oh_bytes: oh,
|
|
precompressed: None,
|
|
});
|
|
}
|
|
}
|
|
|
|
let actual_ds_oh_sizes: Vec<usize> = dummy_blobs.iter().map(|b| b.oh_bytes.len()).collect();
|
|
|
|
// Pass 2: compute real addresses
|
|
// A paged file carries its File Space Info in a superblock extension
|
|
// object header, placed right after the superblock.
|
|
let sb_ext = page_size
|
|
.map(build_paged_superblock_extension)
|
|
.transpose()?;
|
|
let superblock_size = SUPERBLOCK_SIZE + sb_ext.as_ref().map_or(0, Vec::len);
|
|
let mut cursor2 = superblock_size;
|
|
|
|
// Each group (the root first) is laid out as: object header, then (if
|
|
// dense) its link blob, then (if dense) its attribute blob. Link blobs
|
|
// are sized with dummy target addresses here — link message size is
|
|
// address-independent — and rebuilt with real addresses when written.
|
|
let mut group_link_blob_addrs: Vec<Option<u64>> = Vec::new();
|
|
let mut group_dense_blobs: Vec<Option<DenseAttrBlob>> = Vec::new();
|
|
let mut group_addrs2: Vec<u64> = Vec::with_capacity(groups.len());
|
|
for (gi, g) in groups.iter().enumerate() {
|
|
group_addrs2.push(cursor2 as u64);
|
|
cursor2 += group_oh_sizes[gi];
|
|
if group_links_dense[gi] {
|
|
let blob_addr = cursor2 as u64;
|
|
let dummy = g.link_messages(&[], &[]);
|
|
cursor2 += build_dense_links(&dummy, blob_addr, g.track_order)?
|
|
.blob
|
|
.len();
|
|
group_link_blob_addrs.push(Some(blob_addr));
|
|
} else {
|
|
group_link_blob_addrs.push(None);
|
|
}
|
|
if group_dense[gi] {
|
|
let blob = build_dense_attrs(&g.attrs, cursor2 as u64, g.track_order)?;
|
|
cursor2 += blob.blob.len();
|
|
group_dense_blobs.push(Some(blob));
|
|
} else {
|
|
group_dense_blobs.push(None);
|
|
}
|
|
}
|
|
let root_group_addr = group_addrs2[0];
|
|
|
|
let mut ds_dense_blobs: Vec<Option<DenseAttrBlob>> = Vec::new();
|
|
let ds_oh_addrs2: Vec<u64> = actual_ds_oh_sizes
|
|
.iter()
|
|
.enumerate()
|
|
.map(|(i, &sz)| {
|
|
let addr = cursor2 as u64;
|
|
cursor2 += sz;
|
|
if ds_dense[i] {
|
|
let blob =
|
|
build_dense_attrs(&all_ds[i].attrs, cursor2 as u64, all_ds[i].track_order)?;
|
|
cursor2 += blob.blob.len();
|
|
ds_dense_blobs.push(Some(blob));
|
|
} else {
|
|
ds_dense_blobs.push(None);
|
|
}
|
|
Ok(addr)
|
|
})
|
|
.collect::<Result<_, FormatError>>()?;
|
|
|
|
let mut ds_blobs2: Vec<DataBlob> = Vec::new();
|
|
let global_align_threshold = self.alignment_threshold;
|
|
let global_align_bytes = self.alignment_bytes;
|
|
for (i, d) in all_ds.iter().enumerate() {
|
|
if is_vds[i] {
|
|
// VDS: place the global heap collection right after the OHs,
|
|
// then rebuild the OH with the real heap address.
|
|
let gcol_bytes = &dummy_blobs[i].data; // pre-computed in pass 1
|
|
let heap_addr = cursor2 as u64;
|
|
cursor2 += gcol_bytes.len();
|
|
let oh = build_vds_dataset_oh(
|
|
&d.dt,
|
|
&d.ds,
|
|
heap_addr,
|
|
AttrStorage {
|
|
attrs: &d.attrs,
|
|
dense: ds_dense_blobs[i].as_ref(),
|
|
track_order: d.track_order,
|
|
},
|
|
&d.fill_message,
|
|
d.refcount,
|
|
)?;
|
|
ds_blobs2.push(DataBlob {
|
|
data: gcol_bytes.clone(),
|
|
oh_bytes: oh,
|
|
precompressed: None,
|
|
});
|
|
} else if is_chunked[i] {
|
|
let base_address = cursor2 as u64;
|
|
// Reuse precompressed chunks from Pass 1 — avoids re-compressing
|
|
// the same data a second time.
|
|
let result = build_chunked_data_from_precompressed(
|
|
dummy_blobs[i]
|
|
.precompressed
|
|
.as_ref()
|
|
.expect("chunked dataset missing precompressed cache"),
|
|
base_address,
|
|
d.maxshape.as_deref(),
|
|
)?;
|
|
cursor2 += result.data_bytes.len();
|
|
let oh = build_chunked_dataset_oh(
|
|
&d.dt,
|
|
&d.ds,
|
|
&result.layout_message,
|
|
result.pipeline_message.as_deref(),
|
|
AttrStorage {
|
|
attrs: &d.attrs,
|
|
dense: ds_dense_blobs[i].as_ref(),
|
|
track_order: d.track_order,
|
|
},
|
|
&d.fill_message,
|
|
d.refcount,
|
|
)?;
|
|
ds_blobs2.push(DataBlob {
|
|
data: result.data_bytes,
|
|
oh_bytes: oh,
|
|
precompressed: None,
|
|
});
|
|
} else if is_compact[i] {
|
|
// Compact: data is inline in the object header, no external blob
|
|
let oh = build_compact_dataset_oh(
|
|
&d.dt,
|
|
&d.ds,
|
|
&d.raw,
|
|
AttrStorage {
|
|
attrs: &d.attrs,
|
|
dense: ds_dense_blobs[i].as_ref(),
|
|
track_order: d.track_order,
|
|
},
|
|
&d.fill_message,
|
|
d.refcount,
|
|
)?;
|
|
ds_blobs2.push(DataBlob {
|
|
data: vec![],
|
|
oh_bytes: oh,
|
|
precompressed: None,
|
|
});
|
|
} else {
|
|
// Determine alignment: per-dataset overrides global
|
|
let align = if d.alignment > 0 {
|
|
d.alignment
|
|
} else if global_align_bytes > 0 && d.raw.len() >= global_align_threshold {
|
|
global_align_bytes
|
|
} else {
|
|
8 // default: 8-byte alignment for zero-copy read support
|
|
};
|
|
let padding = (align - (cursor2 % align)) % align;
|
|
cursor2 += padding;
|
|
let oh = build_dataset_oh(
|
|
&d.dt,
|
|
&d.ds,
|
|
cursor2 as u64,
|
|
d.raw.len() as u64,
|
|
AttrStorage {
|
|
attrs: &d.attrs,
|
|
dense: ds_dense_blobs[i].as_ref(),
|
|
track_order: d.track_order,
|
|
},
|
|
&d.fill_message,
|
|
d.refcount,
|
|
)?;
|
|
let mut data = vec![0u8; padding];
|
|
data.extend_from_slice(&d.raw);
|
|
cursor2 += d.raw.len();
|
|
ds_blobs2.push(DataBlob {
|
|
data,
|
|
oh_bytes: oh,
|
|
precompressed: None,
|
|
});
|
|
}
|
|
}
|
|
|
|
let actual_ds_oh_sizes2: Vec<usize> = ds_blobs2.iter().map(|b| b.oh_bytes.len()).collect();
|
|
debug_assert_eq!(actual_ds_oh_sizes, actual_ds_oh_sizes2);
|
|
|
|
// libhdf5 ends a paged file on a page boundary.
|
|
let data_end = cursor2;
|
|
if let Some(ps) = page_size {
|
|
cursor2 = cursor2.next_multiple_of(ps as usize);
|
|
}
|
|
let eof_addr2 = cursor2 as u64;
|
|
let mut buf = Vec::with_capacity(cursor2);
|
|
|
|
let sb = Superblock {
|
|
version: 3,
|
|
offset_size: OFFSET_SIZE,
|
|
length_size: LENGTH_SIZE,
|
|
base_address: 0,
|
|
eof_address: eof_addr2,
|
|
root_group_address: root_group_addr,
|
|
group_leaf_node_k: None,
|
|
group_internal_node_k: None,
|
|
indexed_storage_internal_node_k: None,
|
|
free_space_address: None,
|
|
driver_info_address: None,
|
|
consistency_flags: 0,
|
|
superblock_extension_address: Some(if sb_ext.is_some() {
|
|
SUPERBLOCK_SIZE as u64
|
|
} else {
|
|
u64::MAX
|
|
}),
|
|
checksum: None,
|
|
page_size: None,
|
|
};
|
|
buf.extend_from_slice(&sb.serialize());
|
|
if let Some(ref ext) = sb_ext {
|
|
buf.extend_from_slice(ext);
|
|
}
|
|
|
|
// Group OHs + dense blobs (link blob, then attr blob, matching pass 2)
|
|
for (gi, g) in groups.iter().enumerate() {
|
|
let links = g.link_messages(&group_addrs2, &ds_oh_addrs2);
|
|
// Rebuild the link blob with real target addresses (same size as
|
|
// the dummy used for layout); its LinkInfo goes in the OH.
|
|
let link_blob = group_link_blob_addrs[gi]
|
|
.map(|addr| build_dense_links(&links, addr, g.track_order))
|
|
.transpose()?;
|
|
let li = match &link_blob {
|
|
Some(b) => b.link_info_message.clone(),
|
|
None => compact_link_info(g.track_order, links.len()),
|
|
};
|
|
let oh = build_group_oh(
|
|
&links,
|
|
&li,
|
|
link_blob.is_some(),
|
|
AttrStorage {
|
|
attrs: &g.attrs,
|
|
dense: group_dense_blobs[gi].as_ref(),
|
|
track_order: g.track_order,
|
|
},
|
|
g.refcount,
|
|
)?;
|
|
debug_assert_eq!(oh.len(), group_oh_sizes[gi]);
|
|
debug_assert_eq!(buf.len() as u64, group_addrs2[gi]);
|
|
buf.extend_from_slice(&oh);
|
|
if let Some(ref b) = link_blob {
|
|
buf.extend_from_slice(&b.blob);
|
|
}
|
|
if let Some(ref blob) = group_dense_blobs[gi] {
|
|
buf.extend_from_slice(&blob.blob);
|
|
}
|
|
}
|
|
|
|
// Dataset OHs + dense blobs
|
|
for (i, blob) in ds_blobs2.iter().enumerate() {
|
|
buf.extend_from_slice(&blob.oh_bytes);
|
|
if let Some(ref dense) = ds_dense_blobs[i] {
|
|
buf.extend_from_slice(&dense.blob);
|
|
}
|
|
}
|
|
|
|
// Data
|
|
for blob in &ds_blobs2 {
|
|
buf.extend_from_slice(&blob.data);
|
|
}
|
|
|
|
debug_assert_eq!(buf.len(), data_end);
|
|
buf.resize(cursor2, 0);
|
|
Ok(buf)
|
|
}
|
|
}
|
|
|
|
// ---- Independent parallel dataset creation ----
|
|
|
|
/// Builder that creates datasets without locking the file header.
|
|
///
|
|
/// Each `IndependentDatasetBuilder` accumulates its own [`MetadataBlock`]
|
|
/// independently. On [`IndependentDatasetBuilder::finish`], the block is
|
|
/// returned for later merging.
|
|
///
|
|
/// Thread-safety: each thread should own its own builder instance.
|
|
pub struct IndependentDatasetBuilder {
|
|
block: MetadataBlock,
|
|
}
|
|
|
|
impl IndependentDatasetBuilder {
|
|
/// Create a new independent builder with the given creator id.
|
|
pub fn new(creator_id: u32) -> Self {
|
|
Self {
|
|
block: MetadataBlock::new(creator_id),
|
|
}
|
|
}
|
|
|
|
/// Add a dataset specification to this builder.
|
|
pub fn add_dataset(&mut self, meta: DatasetMetadata) {
|
|
self.block.add_dataset(meta);
|
|
}
|
|
|
|
/// Consume the builder and return the metadata block.
|
|
pub fn finish(self) -> MetadataBlock {
|
|
self.block
|
|
}
|
|
}
|
|
|
|
/// Finalize multiple independently-created metadata blocks into a complete HDF5 file.
|
|
///
|
|
/// This implements the write-ahead approach: each block's data is laid out
|
|
/// sequentially, then the index table (root group with links) is written last
|
|
/// to point at all the dataset object headers.
|
|
pub fn finalize_parallel(blocks: Vec<MetadataBlock>) -> Result<Vec<u8>, FormatError> {
|
|
let index = MetadataIndex::merge_blocks(&blocks)?;
|
|
finalize_from_index(index)
|
|
}
|
|
|
|
/// Build a complete HDF5 file from a merged MetadataIndex.
|
|
fn finalize_from_index(index: MetadataIndex) -> Result<Vec<u8>, FormatError> {
|
|
// Convert DatasetMetadata into the internal DsFlat representation and
|
|
// delegate to the same two-pass algorithm used by FileWriter.
|
|
let mut fw = FileWriter::new();
|
|
for ds_meta in &index.datasets {
|
|
let db = fw.create_dataset(&ds_meta.name);
|
|
// Set the datatype and raw data directly via internal fields
|
|
db.datatype = Some(ds_meta.datatype.clone());
|
|
db.shape = Some(ds_meta.dataspace.dimensions.clone());
|
|
db.maxshape = ds_meta.maxshape.clone();
|
|
db.data = Some(ds_meta.raw_data.clone());
|
|
db.chunk_options = ds_meta.chunk_options.clone();
|
|
for (name, val) in &ds_meta.attrs {
|
|
db.set_attr(name, val.clone());
|
|
}
|
|
}
|
|
fw.finish()
|
|
}
|
|
|
|
#[cfg(test)]
|
|
mod tests {
|
|
use super::*;
|
|
use crate::group_v2::resolve_path_any;
|
|
use crate::object_header::ObjectHeader;
|
|
use crate::signature;
|
|
|
|
fn parse_file(bytes: &[u8]) -> (Superblock, ObjectHeader) {
|
|
let sig = signature::find_signature(bytes).unwrap();
|
|
let sb = Superblock::parse(bytes, sig).unwrap();
|
|
let oh = ObjectHeader::parse(
|
|
bytes,
|
|
sb.root_group_address as usize,
|
|
sb.offset_size,
|
|
sb.length_size,
|
|
)
|
|
.unwrap();
|
|
(sb, oh)
|
|
}
|
|
|
|
fn read_dataset_f64(bytes: &[u8], path: &str) -> Vec<f64> {
|
|
let sig = signature::find_signature(bytes).unwrap();
|
|
let sb = Superblock::parse(bytes, sig).unwrap();
|
|
let addr = resolve_path_any(bytes, &sb, path).unwrap();
|
|
let hdr =
|
|
ObjectHeader::parse(bytes, addr as usize, sb.offset_size, sb.length_size).unwrap();
|
|
let dt_data = &hdr
|
|
.messages
|
|
.iter()
|
|
.find(|m| m.msg_type == MessageType::Datatype)
|
|
.unwrap()
|
|
.data;
|
|
let ds_data = &hdr
|
|
.messages
|
|
.iter()
|
|
.find(|m| m.msg_type == MessageType::Dataspace)
|
|
.unwrap()
|
|
.data;
|
|
let dl_data = &hdr
|
|
.messages
|
|
.iter()
|
|
.find(|m| m.msg_type == MessageType::DataLayout)
|
|
.unwrap()
|
|
.data;
|
|
let (dt, _) = Datatype::parse(dt_data).unwrap();
|
|
let ds = Dataspace::parse(ds_data, sb.length_size).unwrap();
|
|
let dl =
|
|
crate::data_layout::DataLayout::parse(dl_data, sb.offset_size, sb.length_size).unwrap();
|
|
let raw = crate::data_read::read_raw_data(bytes, &dl, &ds, &dt).unwrap();
|
|
crate::data_read::read_as_f64(&raw, &dt).unwrap()
|
|
}
|
|
|
|
#[test]
|
|
fn empty_file_root_group_only() {
|
|
let fw = FileWriter::new();
|
|
let bytes = fw.finish().unwrap();
|
|
let (sb, oh) = parse_file(&bytes);
|
|
assert_eq!(sb.version, 3);
|
|
assert_eq!(oh.version, 2);
|
|
}
|
|
|
|
#[test]
|
|
fn file_with_f64_dataset() {
|
|
let mut fw = FileWriter::new();
|
|
fw.create_dataset("data").with_f64_data(&[1.0, 2.0, 3.0]);
|
|
let bytes = fw.finish().unwrap();
|
|
assert_eq!(read_dataset_f64(&bytes, "data"), vec![1.0, 2.0, 3.0]);
|
|
}
|
|
|
|
#[test]
|
|
fn file_with_dataset_attrs() {
|
|
let mut fw = FileWriter::new();
|
|
fw.create_dataset("data")
|
|
.with_f64_data(&[1.0, 2.0])
|
|
.set_attr("scale", AttrValue::F64(0.5));
|
|
let bytes = fw.finish().unwrap();
|
|
assert_eq!(read_dataset_f64(&bytes, "data"), vec![1.0, 2.0]);
|
|
let sig = signature::find_signature(&bytes).unwrap();
|
|
let sb = Superblock::parse(&bytes, sig).unwrap();
|
|
let addr = resolve_path_any(&bytes, &sb, "data").unwrap();
|
|
let hdr =
|
|
ObjectHeader::parse(&bytes, addr as usize, sb.offset_size, sb.length_size).unwrap();
|
|
let attrs = crate::attribute::extract_attributes(&hdr, sb.length_size).unwrap();
|
|
assert_eq!(attrs.len(), 1);
|
|
assert_eq!(attrs[0].name, "scale");
|
|
}
|
|
|
|
#[test]
|
|
fn file_with_group_and_dataset() {
|
|
let mut fw = FileWriter::new();
|
|
let mut gb = fw.create_group("grp");
|
|
gb.create_dataset("vals").with_f64_data(&[10.0, 20.0]);
|
|
fw.add_group(gb.finish());
|
|
let bytes = fw.finish().unwrap();
|
|
assert_eq!(read_dataset_f64(&bytes, "grp/vals"), vec![10.0, 20.0]);
|
|
}
|
|
|
|
#[test]
|
|
fn file_with_root_attr() {
|
|
let mut fw = FileWriter::new();
|
|
fw.set_root_attr("version", AttrValue::I64(42));
|
|
let bytes = fw.finish().unwrap();
|
|
let (sb, oh) = parse_file(&bytes);
|
|
let attrs = crate::attribute::extract_attributes(&oh, sb.length_size).unwrap();
|
|
assert_eq!(attrs[0].name, "version");
|
|
}
|
|
|
|
#[test]
|
|
fn dense_attrs_self_roundtrip() {
|
|
let mut fw = FileWriter::new();
|
|
let ds = fw.create_dataset("data");
|
|
ds.with_f64_data(&[1.0, 2.0, 3.0]);
|
|
for i in 0..20 {
|
|
ds.set_attr(&format!("attr_{i:03}"), AttrValue::F64(i as f64 * 1.5));
|
|
}
|
|
let bytes = fw.finish().unwrap();
|
|
let sig = signature::find_signature(&bytes).unwrap();
|
|
let sb = Superblock::parse(&bytes, sig).unwrap();
|
|
let addr = resolve_path_any(&bytes, &sb, "data").unwrap();
|
|
let hdr =
|
|
ObjectHeader::parse(&bytes, addr as usize, sb.offset_size, sb.length_size).unwrap();
|
|
let attrs =
|
|
crate::attribute::extract_attributes_full(&bytes, &hdr, sb.offset_size, sb.length_size)
|
|
.unwrap();
|
|
assert_eq!(attrs.len(), 20);
|
|
for i in 0..20 {
|
|
let attr = attrs
|
|
.iter()
|
|
.find(|a| a.name == format!("attr_{i:03}"))
|
|
.unwrap();
|
|
let v = attr.read_as_f64().unwrap();
|
|
assert!((v[0] - i as f64 * 1.5).abs() < 1e-10);
|
|
}
|
|
assert_eq!(read_dataset_f64(&bytes, "data"), vec![1.0, 2.0, 3.0]);
|
|
}
|
|
|
|
/// Attribute names of the object at `path`, in the order the reader
|
|
/// lists them.
|
|
fn attr_names(bytes: &[u8], path: &str) -> Vec<String> {
|
|
let sig = signature::find_signature(bytes).unwrap();
|
|
let sb = Superblock::parse(bytes, sig).unwrap();
|
|
let addr = if path == "/" {
|
|
sb.root_group_address
|
|
} else {
|
|
resolve_path_any(bytes, &sb, path).unwrap()
|
|
};
|
|
let hdr =
|
|
ObjectHeader::parse(bytes, addr as usize, sb.offset_size, sb.length_size).unwrap();
|
|
crate::attribute::extract_attributes_full(bytes, &hdr, sb.offset_size, sb.length_size)
|
|
.unwrap()
|
|
.into_iter()
|
|
.map(|a| a.name)
|
|
.collect()
|
|
}
|
|
|
|
#[test]
|
|
fn tracked_attributes_are_read_in_creation_order() {
|
|
let set = |names: &[String]| -> Vec<(String, AttrValue)> {
|
|
names
|
|
.iter()
|
|
.enumerate()
|
|
.map(|(i, n)| (n.clone(), AttrValue::I64(i as i64)))
|
|
.collect()
|
|
};
|
|
let compact: Vec<String> = ["zeta", "alpha", "mid"].map(String::from).to_vec();
|
|
let dense: Vec<String> = (0..30).rev().map(|i| format!("a{i:02}")).collect();
|
|
let mut fw = FileWriter::new();
|
|
fw.track_order(true);
|
|
for (n, v) in set(&compact) {
|
|
fw.set_root_attr(&n, v);
|
|
}
|
|
let ds = fw.create_dataset("dense");
|
|
ds.with_i32_data(&[1]);
|
|
for (n, v) in set(&dense) {
|
|
ds.set_attr(&n, v);
|
|
}
|
|
let ds = fw.create_dataset("untracked");
|
|
ds.with_i32_data(&[1]).track_order(false);
|
|
for (n, v) in set(&dense) {
|
|
ds.set_attr(&n, v);
|
|
}
|
|
let mut g = fw.create_group("g");
|
|
g.track_order(false);
|
|
for (n, v) in set(&compact) {
|
|
g.set_attr(&n, v);
|
|
}
|
|
fw.add_group(g.finish());
|
|
let bytes = fw.finish().unwrap();
|
|
assert_eq!(attr_names(&bytes, "/"), compact);
|
|
assert_eq!(attr_names(&bytes, "dense"), dense);
|
|
// Without tracking: storage order (inline: as added; dense: hash).
|
|
assert_eq!(attr_names(&bytes, "g"), compact);
|
|
let mut by_hash = dense.clone();
|
|
by_hash.sort_by_key(|n| crate::checksum::jenkins_lookup3(n.as_bytes()));
|
|
assert_eq!(attr_names(&bytes, "untracked"), by_hash);
|
|
}
|
|
|
|
#[test]
|
|
fn too_many_tracked_attributes_is_an_error() {
|
|
// libhdf5 numbers at most 65 535 attributes on an object that
|
|
// tracks their creation order (a 2-byte field). (`set_attr` looks
|
|
// for an earlier value, so 65 536 of them through the builder take
|
|
// a while; build the messages directly.)
|
|
let attrs: Vec<AttributeMessage> = (0..65_536)
|
|
.map(|i| build_attr_message(&format!("a{i}"), &AttrValue::I64(i)))
|
|
.collect();
|
|
let err = build_dense_attrs(&attrs, 0, true)
|
|
.err()
|
|
.unwrap()
|
|
.to_string();
|
|
assert!(err.contains("65536 attributes on one object"), "{err}");
|
|
assert!(build_dense_attrs(&attrs[1..], 0, true).is_ok());
|
|
assert!(build_dense_attrs(&attrs, 0, false).is_ok());
|
|
let mut fw = FileWriter::new();
|
|
let ds = fw.create_dataset("x");
|
|
ds.with_i32_data(&[1]).track_order(true);
|
|
for i in 0..20 {
|
|
ds.set_attr(&format!("a{i}"), AttrValue::I64(i));
|
|
}
|
|
assert!(fw.finish().is_ok());
|
|
}
|
|
|
|
#[test]
|
|
fn dense_attrs_root_group_self_roundtrip() {
|
|
let mut fw = FileWriter::new();
|
|
fw.create_dataset("dummy").with_f64_data(&[0.0]);
|
|
for i in 0..15 {
|
|
fw.set_root_attr(&format!("root_{i:02}"), AttrValue::F64(i as f64 * 2.0));
|
|
}
|
|
let bytes = fw.finish().unwrap();
|
|
let sig = signature::find_signature(&bytes).unwrap();
|
|
let sb = Superblock::parse(&bytes, sig).unwrap();
|
|
let oh = ObjectHeader::parse(
|
|
&bytes,
|
|
sb.root_group_address as usize,
|
|
sb.offset_size,
|
|
sb.length_size,
|
|
)
|
|
.unwrap();
|
|
let attrs =
|
|
crate::attribute::extract_attributes_full(&bytes, &oh, sb.offset_size, sb.length_size)
|
|
.unwrap();
|
|
assert_eq!(attrs.len(), 15);
|
|
}
|
|
|
|
#[test]
|
|
fn inline_attrs_below_threshold() {
|
|
let mut fw = FileWriter::new();
|
|
let ds = fw.create_dataset("data");
|
|
ds.with_f64_data(&[1.0]);
|
|
for i in 0..5 {
|
|
ds.set_attr(&format!("a{i}"), AttrValue::F64(i as f64));
|
|
}
|
|
let bytes = fw.finish().unwrap();
|
|
let sig = signature::find_signature(&bytes).unwrap();
|
|
let sb = Superblock::parse(&bytes, sig).unwrap();
|
|
let addr = resolve_path_any(&bytes, &sb, "data").unwrap();
|
|
let hdr =
|
|
ObjectHeader::parse(&bytes, addr as usize, sb.offset_size, sb.length_size).unwrap();
|
|
assert!(
|
|
!hdr.messages
|
|
.iter()
|
|
.any(|m| m.msg_type == MessageType::AttributeInfo)
|
|
);
|
|
let attrs = crate::attribute::extract_attributes(&hdr, sb.length_size).unwrap();
|
|
assert_eq!(attrs.len(), 5);
|
|
}
|
|
|
|
#[test]
|
|
fn encode_decode_managed_id_roundtrip() {
|
|
let id = encode_managed_id(100, 42, 40, 8);
|
|
let fh = crate::fractal_heap::FractalHeapHeader {
|
|
heap_id_length: 8,
|
|
io_filter_encoded_length: 0,
|
|
max_managed_object_size: 1024,
|
|
table_width: 4,
|
|
starting_block_size: 4096,
|
|
max_direct_block_size: 65536,
|
|
max_heap_size: 40,
|
|
starting_row_of_indirect_blocks: 1,
|
|
root_block_address: 0,
|
|
current_rows_in_root_indirect_block: 0,
|
|
managed_objects_count: 0,
|
|
huge_btree_address: u64::MAX,
|
|
filter_pipeline: None,
|
|
root_direct_block_filtered_size: 0,
|
|
root_direct_block_filter_mask: 0,
|
|
offset_size: 8,
|
|
length_size: 8,
|
|
};
|
|
let (off, len) = fh.decode_managed_id(&id).unwrap();
|
|
assert_eq!(off, 100);
|
|
assert_eq!(len, 42);
|
|
}
|
|
|
|
#[test]
|
|
fn finalize_parallel_basic() {
|
|
use crate::chunked_write::ChunkOptions;
|
|
use crate::metadata_index::{MetadataBlock, build_dataset_metadata};
|
|
use crate::type_builders::make_f64_type;
|
|
|
|
let mut b0 = MetadataBlock::new(0);
|
|
let data_a: Vec<u8> = [1.0f64, 2.0, 3.0]
|
|
.iter()
|
|
.flat_map(|v| v.to_le_bytes())
|
|
.collect();
|
|
b0.add_dataset(build_dataset_metadata(
|
|
"alpha",
|
|
make_f64_type(),
|
|
vec![3],
|
|
data_a,
|
|
ChunkOptions::default(),
|
|
None,
|
|
vec![],
|
|
));
|
|
|
|
let mut b1 = MetadataBlock::new(1);
|
|
let data_b: Vec<u8> = [10.0f64, 20.0]
|
|
.iter()
|
|
.flat_map(|v| v.to_le_bytes())
|
|
.collect();
|
|
b1.add_dataset(build_dataset_metadata(
|
|
"beta",
|
|
make_f64_type(),
|
|
vec![2],
|
|
data_b,
|
|
ChunkOptions::default(),
|
|
None,
|
|
vec![],
|
|
));
|
|
|
|
let bytes = finalize_parallel(vec![b0, b1]).unwrap();
|
|
assert_eq!(read_dataset_f64(&bytes, "alpha"), vec![1.0, 2.0, 3.0]);
|
|
assert_eq!(read_dataset_f64(&bytes, "beta"), vec![10.0, 20.0]);
|
|
}
|
|
|
|
#[test]
|
|
fn finalize_parallel_duplicate_error() {
|
|
use crate::chunked_write::ChunkOptions;
|
|
use crate::metadata_index::{MetadataBlock, build_dataset_metadata};
|
|
use crate::type_builders::make_f64_type;
|
|
|
|
let mut b0 = MetadataBlock::new(0);
|
|
b0.add_dataset(build_dataset_metadata(
|
|
"dup",
|
|
make_f64_type(),
|
|
vec![1],
|
|
vec![0u8; 8],
|
|
ChunkOptions::default(),
|
|
None,
|
|
vec![],
|
|
));
|
|
let mut b1 = MetadataBlock::new(1);
|
|
b1.add_dataset(build_dataset_metadata(
|
|
"dup",
|
|
make_f64_type(),
|
|
vec![1],
|
|
vec![0u8; 8],
|
|
ChunkOptions::default(),
|
|
None,
|
|
vec![],
|
|
));
|
|
let err = finalize_parallel(vec![b0, b1]).unwrap_err();
|
|
assert!(matches!(err, FormatError::DuplicateDatasetName(_)));
|
|
}
|
|
|
|
// ---- Virtual Dataset (VDS) round-trip tests ----
|
|
|
|
/// Serialize an H5S ALL selection (type=3, version=1, 16 bytes).
|
|
fn sel_all() -> Vec<u8> {
|
|
vec![
|
|
3, 0, 0, 0, // type = ALL
|
|
1, 0, 0, 0, // version
|
|
0, 0, 0, 0, // reserved
|
|
0, 0, 0, 0, // length (unused for ALL)
|
|
]
|
|
}
|
|
|
|
/// Serialize an H5S HYPER selection (version 3, rank 1, enc_size 2).
|
|
/// Encodes start=`start`, stride=1, count=1, block=`block`.
|
|
fn sel_hyper_1d(start: u16, block: u16) -> Vec<u8> {
|
|
let mut v = vec![
|
|
2, 0, 0, 0, // type = HYPER
|
|
3, 0, 0, 0, // version 3
|
|
0x01, // flags = regular
|
|
0x02, // enc_size = 2 (u16 per coordinate)
|
|
1, 0, 0, 0, // rank = 1
|
|
];
|
|
v.extend_from_slice(&start.to_le_bytes()); // start
|
|
v.extend_from_slice(&1u16.to_le_bytes()); // stride
|
|
v.extend_from_slice(&1u16.to_le_bytes()); // count
|
|
v.extend_from_slice(&block.to_le_bytes()); // block
|
|
v
|
|
}
|
|
|
|
#[test]
|
|
fn vds_write_read_virtual_layout() {
|
|
use crate::data_layout::DataLayout;
|
|
|
|
// A virtual dataset /vds of shape [8] backed by two same-file sources:
|
|
// /src_a maps to virtual[0:4] and /src_b maps to virtual[4:8].
|
|
let mapping_a = VdsMapping {
|
|
source_file: ".".into(),
|
|
source_dataset: "src_a".into(),
|
|
source_selection: sel_all(),
|
|
virtual_selection: sel_hyper_1d(0, 4),
|
|
};
|
|
let mapping_b = VdsMapping {
|
|
source_file: ".".into(),
|
|
source_dataset: "src_b".into(),
|
|
source_selection: sel_all(),
|
|
virtual_selection: sel_hyper_1d(4, 4),
|
|
};
|
|
|
|
let mut fw = FileWriter::new();
|
|
// Source datasets (real data in this file)
|
|
fw.create_dataset("src_a")
|
|
.with_f64_data(&[1.0, 2.0, 3.0, 4.0]);
|
|
fw.create_dataset("src_b")
|
|
.with_f64_data(&[5.0, 6.0, 7.0, 8.0]);
|
|
// Virtual dataset
|
|
fw.create_dataset("vds")
|
|
.with_shape(&[8])
|
|
.with_f64_data(&[]) // shape hint; raw data is ignored for VDS
|
|
.with_virtual_sources(vec![mapping_a, mapping_b]);
|
|
|
|
let bytes = fw.finish().unwrap();
|
|
|
|
// Verify the virtual dataset resolves to DataLayout::Virtual
|
|
let sig = signature::find_signature(&bytes).unwrap();
|
|
let sb = Superblock::parse(&bytes, sig).unwrap();
|
|
let vds_addr = resolve_path_any(&bytes, &sb, "vds").unwrap();
|
|
let hdr =
|
|
ObjectHeader::parse(&bytes, vds_addr as usize, sb.offset_size, sb.length_size).unwrap();
|
|
|
|
let dl_data = &hdr
|
|
.messages
|
|
.iter()
|
|
.find(|m| m.msg_type == MessageType::DataLayout)
|
|
.unwrap()
|
|
.data;
|
|
|
|
let mut layout = DataLayout::parse(dl_data, sb.offset_size, sb.length_size).unwrap();
|
|
|
|
// Before resolution, mappings field is empty.
|
|
assert!(
|
|
matches!(layout, DataLayout::Virtual { .. }),
|
|
"expected Virtual layout, got {layout:?}"
|
|
);
|
|
|
|
// Resolve VDS mappings from the global heap.
|
|
layout.resolve_vds_mappings(&bytes, sb.length_size).unwrap();
|
|
|
|
match &layout {
|
|
DataLayout::Virtual { mappings, .. } => {
|
|
assert_eq!(mappings.len(), 2, "expected 2 VDS mappings");
|
|
assert_eq!(mappings[0].source_file, ".");
|
|
assert_eq!(mappings[0].source_dataset, "src_a");
|
|
assert_eq!(mappings[1].source_file, ".");
|
|
assert_eq!(mappings[1].source_dataset, "src_b");
|
|
|
|
// Verify the virtual selections cover [0:4] and [4:8].
|
|
use crate::selection::Selection;
|
|
let (vsel_a, _) =
|
|
Selection::decode_serialized(&mappings[0].virtual_selection).unwrap();
|
|
let (vsel_b, _) =
|
|
Selection::decode_serialized(&mappings[1].virtual_selection).unwrap();
|
|
assert_eq!(vsel_a.iter_linear_1d(8).unwrap(), vec![0, 1, 2, 3]);
|
|
assert_eq!(vsel_b.iter_linear_1d(8).unwrap(), vec![4, 5, 6, 7]);
|
|
}
|
|
other => panic!("expected Virtual layout after resolution, got {other:?}"),
|
|
}
|
|
|
|
// Source datasets still readable normally.
|
|
assert_eq!(read_dataset_f64(&bytes, "src_a"), vec![1.0, 2.0, 3.0, 4.0]);
|
|
assert_eq!(read_dataset_f64(&bytes, "src_b"), vec![5.0, 6.0, 7.0, 8.0]);
|
|
}
|
|
|
|
#[test]
|
|
fn vds_external_source_file() {
|
|
use crate::data_layout::DataLayout;
|
|
|
|
// A VDS mapping referencing an external file ("other.h5").
|
|
let mapping_ext = VdsMapping {
|
|
source_file: "other.h5".into(),
|
|
source_dataset: "data".into(),
|
|
source_selection: sel_all(),
|
|
virtual_selection: sel_all(),
|
|
};
|
|
|
|
let mut fw = FileWriter::new();
|
|
fw.create_dataset("ext_vds")
|
|
.with_shape(&[10])
|
|
.with_f64_data(&[]) // shape hint only
|
|
.with_virtual_sources(vec![mapping_ext]);
|
|
|
|
let bytes = fw.finish().unwrap();
|
|
|
|
let sig = signature::find_signature(&bytes).unwrap();
|
|
let sb = Superblock::parse(&bytes, sig).unwrap();
|
|
let addr = resolve_path_any(&bytes, &sb, "ext_vds").unwrap();
|
|
let hdr =
|
|
ObjectHeader::parse(&bytes, addr as usize, sb.offset_size, sb.length_size).unwrap();
|
|
let dl_data = &hdr
|
|
.messages
|
|
.iter()
|
|
.find(|m| m.msg_type == MessageType::DataLayout)
|
|
.unwrap()
|
|
.data;
|
|
let mut layout = DataLayout::parse(dl_data, sb.offset_size, sb.length_size).unwrap();
|
|
layout.resolve_vds_mappings(&bytes, sb.length_size).unwrap();
|
|
|
|
match &layout {
|
|
DataLayout::Virtual { mappings, .. } => {
|
|
assert_eq!(mappings.len(), 1);
|
|
assert_eq!(mappings[0].source_file, "other.h5");
|
|
assert_eq!(mappings[0].source_dataset, "data");
|
|
}
|
|
other => panic!("expected Virtual, got {other:?}"),
|
|
}
|
|
}
|
|
|
|
#[test]
|
|
fn vds_empty_mapping_list() {
|
|
// Calling with_virtual_sources([]) is silently ignored — the dataset
|
|
// falls back to a normal contiguous layout rather than writing an empty VDS.
|
|
use crate::data_layout::DataLayout;
|
|
|
|
let mut fw = FileWriter::new();
|
|
fw.create_dataset("empty_vds")
|
|
.with_shape(&[0])
|
|
.with_f64_data(&[])
|
|
.with_virtual_sources(vec![]);
|
|
|
|
let bytes = fw.finish().unwrap();
|
|
|
|
let sig = signature::find_signature(&bytes).unwrap();
|
|
let sb = Superblock::parse(&bytes, sig).unwrap();
|
|
let addr = resolve_path_any(&bytes, &sb, "empty_vds").unwrap();
|
|
let hdr =
|
|
ObjectHeader::parse(&bytes, addr as usize, sb.offset_size, sb.length_size).unwrap();
|
|
let dl_data = &hdr
|
|
.messages
|
|
.iter()
|
|
.find(|m| m.msg_type == MessageType::DataLayout)
|
|
.unwrap()
|
|
.data;
|
|
let layout = DataLayout::parse(dl_data, sb.offset_size, sb.length_size).unwrap();
|
|
|
|
// Empty mapping list → no VDS layout; should be Contiguous or Compact.
|
|
assert!(
|
|
!matches!(layout, DataLayout::Virtual { .. }),
|
|
"empty with_virtual_sources should NOT produce a VDS layout, got {layout:?}"
|
|
);
|
|
}
|
|
|
|
#[test]
|
|
fn external_link_write_roundtrip() {
|
|
let mut fw = FileWriter::new();
|
|
let mut grp = fw.create_group("sensors");
|
|
grp.create_dataset("local_ds").with_f64_data(&[1.0, 2.0]);
|
|
grp.add_external_link("remote_temp", "other_file.h5", "/temperature");
|
|
fw.add_group(grp.finish());
|
|
|
|
let bytes = fw.finish().unwrap();
|
|
|
|
let sig = signature::find_signature(&bytes).unwrap();
|
|
let sb = Superblock::parse(&bytes, sig).unwrap();
|
|
let sensors_addr = resolve_path_any(&bytes, &sb, "sensors").unwrap();
|
|
let hdr = ObjectHeader::parse(
|
|
&bytes,
|
|
sensors_addr as usize,
|
|
sb.offset_size,
|
|
sb.length_size,
|
|
)
|
|
.unwrap();
|
|
|
|
// Find the external LinkMessage directly in the object header.
|
|
let ext_link = hdr
|
|
.messages
|
|
.iter()
|
|
.filter(|m| m.msg_type == MessageType::Link)
|
|
.filter_map(|m| crate::link_message::LinkMessage::parse(&m.data, sb.offset_size).ok())
|
|
.find(|l| l.name == "remote_temp")
|
|
.expect("external link 'remote_temp' not found in group OH");
|
|
|
|
match &ext_link.link_target {
|
|
crate::link_message::LinkTarget::External {
|
|
filename,
|
|
object_path,
|
|
} => {
|
|
assert_eq!(filename, "other_file.h5");
|
|
assert_eq!(object_path, "/temperature");
|
|
}
|
|
other => panic!("expected External link, got {other:?}"),
|
|
}
|
|
}
|
|
|
|
#[test]
|
|
fn file_writer_paged_file_uses_v3_superblock_and_fsinfo_extension() {
|
|
// This used to write superblock "version 4", which does not exist.
|
|
let mut fw = FileWriter::new();
|
|
fw.with_page_size(4096);
|
|
fw.create_dataset("data").with_f64_data(&[1.0, 2.0]);
|
|
let bytes = fw.finish().unwrap();
|
|
|
|
let sig = signature::find_signature(&bytes).unwrap();
|
|
let sb = Superblock::parse(&bytes, sig).unwrap();
|
|
assert_eq!(sb.version, 3);
|
|
assert_eq!(sb.superblock_extension_address, Some(48));
|
|
assert_eq!(bytes.len() % 4096, 0);
|
|
assert_eq!(sb.eof_address, bytes.len() as u64);
|
|
let ext = ObjectHeader::parse(&bytes, 48, 8, 8).unwrap();
|
|
let fsinfo = &ext.messages[0];
|
|
assert_eq!(fsinfo.msg_type, MessageType::Unknown(0x0017));
|
|
// Byte-for-byte what HDF5 2.0 writes for fs_strategy="page",
|
|
// fs_page_size=4096.
|
|
let mut expected = vec![1u8, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0];
|
|
expected.extend_from_slice(&4096u64.to_le_bytes());
|
|
expected.extend_from_slice(&[0, 0]);
|
|
expected.extend_from_slice(&[0xff; 8]);
|
|
assert_eq!(fsinfo.data, expected);
|
|
assert_eq!(fsinfo.flags, 0x14);
|
|
assert_eq!(read_dataset_f64(&bytes, "data"), vec![1.0, 2.0]);
|
|
}
|
|
|
|
#[test]
|
|
fn file_writer_rejects_page_sizes_libhdf5_would() {
|
|
for ps in [0u32, 511, MAX_FILE_SPACE_PAGE_SIZE + 1] {
|
|
let mut fw = FileWriter::new();
|
|
fw.with_page_size(ps);
|
|
assert!(fw.finish().is_err(), "page size {ps}");
|
|
}
|
|
}
|
|
|
|
#[test]
|
|
fn file_writer_default_superblock_is_v3() {
|
|
let mut fw = FileWriter::new();
|
|
fw.create_dataset("data").with_f64_data(&[1.0, 2.0]);
|
|
let bytes = fw.finish().unwrap();
|
|
|
|
let sig = signature::find_signature(&bytes).unwrap();
|
|
let sb = Superblock::parse(&bytes, sig).unwrap();
|
|
assert_eq!(sb.version, 3);
|
|
assert_eq!(sb.page_size, None);
|
|
}
|
|
}
|