Merge pull request 'docs(clawhdf5): document DType variants, fix unresolved doc links' (#17) from sdlc-docs/clawhdf5-types-20260514-165210 into main
This commit is contained in:
redclawsystems
@@ -0,0 +1,377 @@
|
||||
//! Extensible Array writer: serialize EA layout messages and build EA structures.
|
||||
|
||||
#[cfg(not(feature = "std"))]
|
||||
extern crate alloc;
|
||||
|
||||
#[cfg(not(feature = "std"))]
|
||||
use alloc::{vec, vec::Vec};
|
||||
|
||||
use crate::checksum::jenkins_lookup3;
|
||||
use crate::chunked_write::WrittenChunk;
|
||||
|
||||
/// Serialize a v4 Extensible Array layout message.
|
||||
pub(crate) fn serialize_v4_extensible_array(
|
||||
chunk_dims: &[u32],
|
||||
ea_address: u64,
|
||||
offset_size: u8,
|
||||
element_size: u32,
|
||||
) -> Vec<u8> {
|
||||
let mut buf = Vec::new();
|
||||
buf.push(4); // version
|
||||
buf.push(2); // class = chunked
|
||||
buf.push(0x00); // flags
|
||||
|
||||
let ndims = chunk_dims.len() as u8 + 1;
|
||||
buf.push(ndims);
|
||||
|
||||
let max_dim = chunk_dims
|
||||
.iter()
|
||||
.map(|&d| d as u64)
|
||||
.chain(core::iter::once(element_size as u64))
|
||||
.max()
|
||||
.unwrap_or(1);
|
||||
let dim_encoded_len: u8 = if max_dim <= 0xFF {
|
||||
1
|
||||
} else if max_dim <= 0xFFFF {
|
||||
2
|
||||
} else {
|
||||
4
|
||||
};
|
||||
buf.push(dim_encoded_len);
|
||||
|
||||
for &d in chunk_dims {
|
||||
match dim_encoded_len {
|
||||
1 => buf.push(d as u8),
|
||||
2 => buf.extend_from_slice(&(d as u16).to_le_bytes()),
|
||||
4 => buf.extend_from_slice(&d.to_le_bytes()),
|
||||
_ => unreachable!("unexpected dim_encoded_len: {dim_encoded_len}"),
|
||||
}
|
||||
}
|
||||
match dim_encoded_len {
|
||||
1 => buf.push(element_size as u8),
|
||||
2 => buf.extend_from_slice(&(element_size as u16).to_le_bytes()),
|
||||
4 => buf.extend_from_slice(&element_size.to_le_bytes()),
|
||||
_ => unreachable!("unexpected dim_encoded_len: {dim_encoded_len}"),
|
||||
}
|
||||
|
||||
// chunk index type = 4 (Extensible Array)
|
||||
buf.push(4);
|
||||
|
||||
// EA creation parameters (must match AEHD and HDF5 C library defaults)
|
||||
buf.push(32); // max_nelmts_bits
|
||||
buf.push(4); // idx_blk_elmts
|
||||
buf.push(4); // super_blk_min_data_ptrs
|
||||
buf.push(16); // data_blk_min_elmts
|
||||
buf.push(10); // max_dblk_page_nelmts_bits
|
||||
|
||||
// EA header address
|
||||
match offset_size {
|
||||
4 => buf.extend_from_slice(&(ea_address as u32).to_le_bytes()),
|
||||
8 => buf.extend_from_slice(&ea_address.to_le_bytes()),
|
||||
_ => unreachable!("unexpected offset size: {offset_size}"),
|
||||
}
|
||||
|
||||
buf
|
||||
}
|
||||
|
||||
/// Build a complete Extensible Array at a known absolute address.
|
||||
///
|
||||
/// For simplicity, we put all elements inline in the index block when the
|
||||
/// number of chunks is small (up to idx_blk_elmts), otherwise use inline +
|
||||
/// direct data blocks.
|
||||
pub fn build_extensible_array_at(
|
||||
chunks: &[WrittenChunk],
|
||||
offset_size: u8,
|
||||
length_size: u8,
|
||||
has_filters: bool,
|
||||
ea_base_address: u64,
|
||||
) -> Vec<u8> {
|
||||
let os = offset_size as usize;
|
||||
let num_elements = chunks.len();
|
||||
|
||||
// Compute element encoding size (same logic as Fixed Array)
|
||||
let chunk_size_bytes: usize = if has_filters {
|
||||
let max_raw = chunks.iter().map(|c| c.raw_size).max().unwrap_or(1);
|
||||
let log2_val = if max_raw <= 1 {
|
||||
0
|
||||
} else {
|
||||
63 - max_raw.leading_zeros()
|
||||
};
|
||||
let len = 1 + ((log2_val + 8) / 8) as usize;
|
||||
len.min(8)
|
||||
} else {
|
||||
0
|
||||
};
|
||||
|
||||
let elem_size = if has_filters {
|
||||
os + chunk_size_bytes + 4
|
||||
} else {
|
||||
os
|
||||
};
|
||||
|
||||
let client_id: u8 = if has_filters { 1 } else { 0 };
|
||||
|
||||
// EA creation parameters — must match HDF5 C library defaults exactly
|
||||
let max_nelmts_bits: u8 = 32;
|
||||
let idx_blk_elmts: u8 = 4;
|
||||
let min_dblk_nelmts: u8 = 16;
|
||||
let super_blk_min_nelmts: u8 = 4;
|
||||
let max_dblk_nelmts_bits: u8 = 10;
|
||||
|
||||
// EAHD size: fixed(12) + 6 stats(6*length_size) + addr(offset_size) + checksum(4)
|
||||
let aehd_size = 4 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 6 * length_size as usize + os + 4;
|
||||
let aeib_address = ea_base_address + aehd_size as u64;
|
||||
|
||||
// Determine how many elements go inline vs data blocks
|
||||
let n_inline = (idx_blk_elmts as usize).min(num_elements);
|
||||
let remaining_after_inline = num_elements.saturating_sub(n_inline);
|
||||
|
||||
// Compute super block layout per HDF5 spec
|
||||
let sblk_min = super_blk_min_nelmts as usize;
|
||||
let log2_dblk_min = if min_dblk_nelmts <= 1 {
|
||||
0
|
||||
} else {
|
||||
(min_dblk_nelmts as u32).trailing_zeros() as usize
|
||||
};
|
||||
let nsblks = (max_nelmts_bits as usize).saturating_sub(log2_dblk_min) + 1;
|
||||
|
||||
// Direct data block addresses (from super blocks 0..sblk_min-1)
|
||||
let mut dblk_sizes: Vec<usize> = Vec::new();
|
||||
for sblk_idx in 0..sblk_min.min(nsblks) {
|
||||
let ndblks = 1usize << (sblk_idx / 2);
|
||||
let dblk_nelmts = (min_dblk_nelmts as usize) * (1 << sblk_idx.div_ceil(2));
|
||||
for _ in 0..ndblks {
|
||||
dblk_sizes.push(dblk_nelmts);
|
||||
}
|
||||
}
|
||||
let n_direct_dblks = dblk_sizes.len();
|
||||
|
||||
// Super block addresses (for super blocks sblk_min..nsblks-1)
|
||||
let n_sblk_addrs = nsblks.saturating_sub(sblk_min);
|
||||
|
||||
// EAIB size
|
||||
let aeib_size = 4
|
||||
+ 1
|
||||
+ 1
|
||||
+ os
|
||||
+ idx_blk_elmts as usize * elem_size
|
||||
+ n_direct_dblks * os
|
||||
+ n_sblk_addrs * os
|
||||
+ 4;
|
||||
|
||||
// Build AEHD
|
||||
let mut aehd = Vec::with_capacity(aehd_size);
|
||||
aehd.extend_from_slice(b"EAHD");
|
||||
aehd.push(0); // version
|
||||
aehd.push(client_id);
|
||||
aehd.push(elem_size as u8);
|
||||
aehd.push(max_nelmts_bits);
|
||||
aehd.push(idx_blk_elmts);
|
||||
aehd.push(min_dblk_nelmts);
|
||||
aehd.push(super_blk_min_nelmts);
|
||||
aehd.push(max_dblk_nelmts_bits);
|
||||
|
||||
// Count data blocks that will have chunks
|
||||
let n_active_dblks: u64 = if remaining_after_inline > 0 {
|
||||
let mut count = 0u64;
|
||||
let mut ci = n_inline;
|
||||
for &sz in &dblk_sizes {
|
||||
if ci < num_elements {
|
||||
count += 1;
|
||||
ci += sz;
|
||||
}
|
||||
}
|
||||
count
|
||||
} else {
|
||||
0
|
||||
};
|
||||
let blk_off_size = (max_nelmts_bits as usize).div_ceil(8);
|
||||
let aedb_header_overhead = 4 + 1 + 1 + os + blk_off_size + 4;
|
||||
let data_blk_total_size: u64 = if remaining_after_inline > 0 {
|
||||
let mut total = 0u64;
|
||||
let mut ci = n_inline;
|
||||
for &sz in &dblk_sizes {
|
||||
if ci < num_elements {
|
||||
total += (aedb_header_overhead + sz * elem_size) as u64;
|
||||
ci += sz;
|
||||
}
|
||||
}
|
||||
total
|
||||
} else {
|
||||
0
|
||||
};
|
||||
let max_idx_set: u64 = if remaining_after_inline > 0 {
|
||||
let mut max_set = idx_blk_elmts as u64;
|
||||
let mut ci = n_inline;
|
||||
for &sz in &dblk_sizes {
|
||||
if ci < num_elements {
|
||||
max_set += sz as u64;
|
||||
ci += sz;
|
||||
}
|
||||
}
|
||||
max_set
|
||||
} else {
|
||||
idx_blk_elmts as u64
|
||||
};
|
||||
|
||||
let write_length = |buf: &mut Vec<u8>, val: u64| match length_size {
|
||||
4 => buf.extend_from_slice(&(val as u32).to_le_bytes()),
|
||||
_ => buf.extend_from_slice(&val.to_le_bytes()),
|
||||
};
|
||||
let write_addr = |buf: &mut Vec<u8>, val: u64| match offset_size {
|
||||
4 => buf.extend_from_slice(&(val as u32).to_le_bytes()),
|
||||
_ => buf.extend_from_slice(&val.to_le_bytes()),
|
||||
};
|
||||
|
||||
write_length(&mut aehd, 0);
|
||||
write_length(&mut aehd, 0);
|
||||
write_length(&mut aehd, n_active_dblks);
|
||||
write_length(&mut aehd, data_blk_total_size);
|
||||
write_length(&mut aehd, num_elements as u64);
|
||||
write_length(&mut aehd, max_idx_set);
|
||||
|
||||
write_addr(&mut aehd, aeib_address);
|
||||
|
||||
let aehd_checksum = jenkins_lookup3(&aehd);
|
||||
aehd.extend_from_slice(&aehd_checksum.to_le_bytes());
|
||||
debug_assert_eq!(aehd.len(), aehd_size);
|
||||
|
||||
// Build AEIB
|
||||
let mut aeib = Vec::with_capacity(aeib_size);
|
||||
aeib.extend_from_slice(b"EAIB");
|
||||
aeib.push(0);
|
||||
aeib.push(client_id);
|
||||
|
||||
match offset_size {
|
||||
4 => aeib.extend_from_slice(&(ea_base_address as u32).to_le_bytes()),
|
||||
8 => aeib.extend_from_slice(&ea_base_address.to_le_bytes()),
|
||||
_ => aeib.extend_from_slice(&ea_base_address.to_le_bytes()),
|
||||
}
|
||||
|
||||
// Inline elements
|
||||
#[allow(clippy::needless_range_loop)]
|
||||
for i in 0..idx_blk_elmts as usize {
|
||||
if i < n_inline {
|
||||
write_chunk_element(
|
||||
&mut aeib,
|
||||
&chunks[i],
|
||||
offset_size,
|
||||
has_filters,
|
||||
chunk_size_bytes,
|
||||
);
|
||||
} else {
|
||||
write_undefined_element(&mut aeib, offset_size, has_filters, chunk_size_bytes);
|
||||
}
|
||||
}
|
||||
|
||||
// Data block addresses + build data blocks
|
||||
let mut data_blocks_buf = Vec::new();
|
||||
let dblks_base = aeib_address + aeib_size as u64;
|
||||
let mut dblk_cursor = dblks_base;
|
||||
let mut chunk_idx = n_inline;
|
||||
|
||||
for &nelmts in &dblk_sizes {
|
||||
if chunk_idx >= num_elements {
|
||||
match offset_size {
|
||||
4 => aeib.extend_from_slice(&u32::MAX.to_le_bytes()),
|
||||
8 => aeib.extend_from_slice(&u64::MAX.to_le_bytes()),
|
||||
_ => aeib.extend_from_slice(&u64::MAX.to_le_bytes()),
|
||||
}
|
||||
continue;
|
||||
}
|
||||
|
||||
match offset_size {
|
||||
4 => aeib.extend_from_slice(&(dblk_cursor as u32).to_le_bytes()),
|
||||
8 => aeib.extend_from_slice(&dblk_cursor.to_le_bytes()),
|
||||
_ => aeib.extend_from_slice(&dblk_cursor.to_le_bytes()),
|
||||
}
|
||||
|
||||
// Build EADB
|
||||
let mut aedb = Vec::new();
|
||||
aedb.extend_from_slice(b"EADB");
|
||||
aedb.push(0);
|
||||
aedb.push(client_id);
|
||||
match offset_size {
|
||||
4 => aedb.extend_from_slice(&(ea_base_address as u32).to_le_bytes()),
|
||||
8 => aedb.extend_from_slice(&ea_base_address.to_le_bytes()),
|
||||
_ => aedb.extend_from_slice(&ea_base_address.to_le_bytes()),
|
||||
}
|
||||
|
||||
let blk_off_size = (max_nelmts_bits as usize).div_ceil(8);
|
||||
let blk_off_val = (chunk_idx - n_inline) as u64;
|
||||
aedb.extend_from_slice(&blk_off_val.to_le_bytes()[..blk_off_size]);
|
||||
|
||||
for slot in 0..nelmts {
|
||||
if chunk_idx + slot < num_elements {
|
||||
write_chunk_element(
|
||||
&mut aedb,
|
||||
&chunks[chunk_idx + slot],
|
||||
offset_size,
|
||||
has_filters,
|
||||
chunk_size_bytes,
|
||||
);
|
||||
} else {
|
||||
write_undefined_element(&mut aedb, offset_size, has_filters, chunk_size_bytes);
|
||||
}
|
||||
}
|
||||
|
||||
let aedb_checksum = jenkins_lookup3(&aedb);
|
||||
aedb.extend_from_slice(&aedb_checksum.to_le_bytes());
|
||||
|
||||
dblk_cursor += aedb.len() as u64;
|
||||
data_blocks_buf.extend_from_slice(&aedb);
|
||||
chunk_idx += nelmts;
|
||||
}
|
||||
|
||||
// Super block addresses (all undefined)
|
||||
for _ in 0..n_sblk_addrs {
|
||||
match offset_size {
|
||||
4 => aeib.extend_from_slice(&u32::MAX.to_le_bytes()),
|
||||
8 => aeib.extend_from_slice(&u64::MAX.to_le_bytes()),
|
||||
_ => aeib.extend_from_slice(&u64::MAX.to_le_bytes()),
|
||||
}
|
||||
}
|
||||
|
||||
let aeib_checksum = jenkins_lookup3(&aeib);
|
||||
aeib.extend_from_slice(&aeib_checksum.to_le_bytes());
|
||||
debug_assert_eq!(aeib.len(), aeib_size);
|
||||
|
||||
let mut combined = aehd;
|
||||
combined.extend_from_slice(&aeib);
|
||||
combined.extend_from_slice(&data_blocks_buf);
|
||||
combined
|
||||
}
|
||||
|
||||
fn write_chunk_element(
|
||||
buf: &mut Vec<u8>,
|
||||
chunk: &WrittenChunk,
|
||||
offset_size: u8,
|
||||
has_filters: bool,
|
||||
chunk_size_bytes: usize,
|
||||
) {
|
||||
match offset_size {
|
||||
4 => buf.extend_from_slice(&(chunk.address as u32).to_le_bytes()),
|
||||
8 => buf.extend_from_slice(&chunk.address.to_le_bytes()),
|
||||
_ => buf.extend_from_slice(&chunk.address.to_le_bytes()),
|
||||
}
|
||||
if has_filters {
|
||||
let cs_bytes = chunk.compressed_size.to_le_bytes();
|
||||
buf.extend_from_slice(&cs_bytes[..chunk_size_bytes]);
|
||||
buf.extend_from_slice(&chunk.filter_mask.to_le_bytes());
|
||||
}
|
||||
}
|
||||
|
||||
fn write_undefined_element(
|
||||
buf: &mut Vec<u8>,
|
||||
offset_size: u8,
|
||||
has_filters: bool,
|
||||
chunk_size_bytes: usize,
|
||||
) {
|
||||
let os = offset_size as usize;
|
||||
// Use extend with repeat to avoid heap-allocating a temporary Vec on each call.
|
||||
buf.extend(core::iter::repeat_n(0xFF, os));
|
||||
if has_filters {
|
||||
buf.extend(core::iter::repeat_n(0x00, chunk_size_bytes));
|
||||
buf.extend_from_slice(&0u32.to_le_bytes());
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user