fix(format): index every Extensible Array chunk, not just the first 244
The Extensible Array writer only filled the index block's 4 inline elements and the 6 data blocks it addresses directly (240 elements); its super block addresses were always undefined. Chunks from index 244 on were written to the file but never indexed, so they read back as fill values in our reader and in libhdf5, without an error. The writer now lays out data blocks and super blocks for any element count as H5EA__hdr_init sizes them, pages data blocks larger than 1024 elements (page-init bits in the owning super block), leaves blocks with no defined element unallocated, and records real header statistics (max_idx_set is one past the highest defined index). Co-Authored-By: Claude Opus 5.5 (1M context) <[email protected]>
This commit is contained in:
osobh
@@ -503,7 +503,7 @@ fn serialize_v4_fixed_array(
|
||||
/// default, `H5D_FARRAY_MAX_DBLK_PAGE_NELMTS_BITS`).
|
||||
const FA_PAGE_BITS: u8 = 10;
|
||||
|
||||
fn push_addr(buf: &mut Vec<u8>, addr: u64, offset_size: u8) {
|
||||
pub(crate) fn push_addr(buf: &mut Vec<u8>, addr: u64, offset_size: u8) {
|
||||
match offset_size {
|
||||
4 => buf.extend_from_slice(&(addr as u32).to_le_bytes()),
|
||||
_ => buf.extend_from_slice(&addr.to_le_bytes()),
|
||||
@@ -734,8 +734,9 @@ pub fn build_chunked_data_from_precompressed(
|
||||
|
||||
let layout_message = if use_extensible {
|
||||
let ea_address = base_address + data_buf.len() as u64;
|
||||
let slots: Vec<Option<WrittenChunk>> = written_chunks.iter().cloned().map(Some).collect();
|
||||
let ea_bytes = ea_writer::build_extensible_array_at(
|
||||
&written_chunks,
|
||||
&slots,
|
||||
offset_size,
|
||||
length_size,
|
||||
pre.has_filters,
|
||||
@@ -1463,7 +1464,8 @@ mod tests {
|
||||
filter_mask: 0,
|
||||
},
|
||||
];
|
||||
let ea = ea_writer::build_extensible_array_at(&chunks, 8, 8, false, 0x2000);
|
||||
let slots: Vec<_> = chunks.into_iter().map(Some).collect();
|
||||
let ea = ea_writer::build_extensible_array_at(&slots, 8, 8, false, 0x2000);
|
||||
assert_eq!(&ea[0..4], b"EAHD");
|
||||
// Find EAIB after EAHD: 12 fixed + 6*8 stats + 8 addr + 4 checksum = 72
|
||||
let aehd_size = 4 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 6 * 8 + 8 + 4;
|
||||
|
||||
@@ -7,7 +7,7 @@ extern crate alloc;
|
||||
use alloc::{vec, vec::Vec};
|
||||
|
||||
use crate::checksum::jenkins_lookup3;
|
||||
use crate::chunked_write::WrittenChunk;
|
||||
use crate::chunked_write::{WrittenChunk, filtered_chunk_size_len, push_addr, push_index_element};
|
||||
|
||||
/// Serialize a v4 Extensible Array layout message.
|
||||
pub(crate) fn serialize_v4_extensible_array(
|
||||
@@ -58,11 +58,11 @@ pub(crate) fn serialize_v4_extensible_array(
|
||||
buf.push(4);
|
||||
|
||||
// EA creation parameters (must match AEHD and HDF5 C library defaults)
|
||||
buf.push(32); // max_nelmts_bits
|
||||
buf.push(4); // idx_blk_elmts
|
||||
buf.push(4); // super_blk_min_data_ptrs
|
||||
buf.push(16); // data_blk_min_elmts
|
||||
buf.push(10); // max_dblk_page_nelmts_bits
|
||||
buf.push(MAX_NELMTS_BITS);
|
||||
buf.push(IDX_BLK_ELMTS);
|
||||
buf.push(SUP_BLK_MIN_DATA_PTRS);
|
||||
buf.push(DATA_BLK_MIN_ELMTS);
|
||||
buf.push(MAX_DBLK_PAGE_NELMTS_BITS);
|
||||
|
||||
// EA header address
|
||||
match offset_size {
|
||||
@@ -74,304 +74,281 @@ pub(crate) fn serialize_v4_extensible_array(
|
||||
buf
|
||||
}
|
||||
|
||||
// EA creation parameters — the HDF5 library's defaults for chunk indexes
|
||||
// (`H5D_EARRAY_*`); the layout message above and the header must agree.
|
||||
const MAX_NELMTS_BITS: u8 = 32;
|
||||
const IDX_BLK_ELMTS: u8 = 4;
|
||||
const SUP_BLK_MIN_DATA_PTRS: u8 = 4;
|
||||
const DATA_BLK_MIN_ELMTS: u8 = 16;
|
||||
const MAX_DBLK_PAGE_NELMTS_BITS: u8 = 10;
|
||||
|
||||
/// One data block of the array: its first element (relative to the end of
|
||||
/// the index block's own elements), element count, and address when it is
|
||||
/// allocated.
|
||||
struct DataBlock {
|
||||
start: usize,
|
||||
nelmts: usize,
|
||||
addr: Option<u64>,
|
||||
}
|
||||
|
||||
/// Build a complete Extensible Array at a known absolute address.
|
||||
///
|
||||
/// For simplicity, we put all elements inline in the index block when the
|
||||
/// number of chunks is small (up to idx_blk_elmts), otherwise use inline +
|
||||
/// direct data blocks.
|
||||
/// `slots[i]` is the element at linear index `i` (see `chunk_grid`); `None`
|
||||
/// marks an unallocated chunk. The first `IDX_BLK_ELMTS` elements live in
|
||||
/// the index block, the rest in data blocks grouped by super block level
|
||||
/// exactly as `H5EA__hdr_init` sizes them: level `u` has `2^(u/2)` data
|
||||
/// blocks of `DATA_BLK_MIN_ELMTS * 2^ceil(u/2)` elements. The data blocks of
|
||||
/// the first levels are addressed straight from the index block; later
|
||||
/// levels go through a super block (EASB). Data blocks larger than a page
|
||||
/// (`2^MAX_DBLK_PAGE_NELMTS_BITS` elements) are paged, with their page-init
|
||||
/// bits kept in the owning super block. Only blocks holding a defined element
|
||||
/// are allocated; the rest keep the undefined address, as in a file the
|
||||
/// library wrote.
|
||||
pub fn build_extensible_array_at(
|
||||
chunks: &[WrittenChunk],
|
||||
slots: &[Option<WrittenChunk>],
|
||||
offset_size: u8,
|
||||
length_size: u8,
|
||||
has_filters: bool,
|
||||
ea_base_address: u64,
|
||||
) -> Vec<u8> {
|
||||
let os = offset_size as usize;
|
||||
let num_elements = chunks.len();
|
||||
|
||||
// Compute element encoding size (same logic as Fixed Array)
|
||||
let chunk_size_bytes: usize = if has_filters {
|
||||
let max_raw = chunks.iter().map(|c| c.raw_size).max().unwrap_or(1);
|
||||
let log2_val = if max_raw <= 1 {
|
||||
0
|
||||
} else {
|
||||
63 - max_raw.leading_zeros()
|
||||
};
|
||||
let len = 1 + ((log2_val + 8) / 8) as usize;
|
||||
len.min(8)
|
||||
} else {
|
||||
0
|
||||
};
|
||||
|
||||
let elem_size = if has_filters {
|
||||
os + chunk_size_bytes + 4
|
||||
} else {
|
||||
os
|
||||
};
|
||||
|
||||
let chunk_size_bytes = has_filters.then(|| filtered_chunk_size_len(slots));
|
||||
let elem_size = os + chunk_size_bytes.map_or(0, |n| n + 4);
|
||||
let client_id: u8 = if has_filters { 1 } else { 0 };
|
||||
let arr_off_size = (MAX_NELMTS_BITS as usize).div_ceil(8);
|
||||
let page_nelmts = 1usize << MAX_DBLK_PAGE_NELMTS_BITS;
|
||||
let idx_blk = IDX_BLK_ELMTS as usize;
|
||||
|
||||
// EA creation parameters — must match HDF5 C library defaults exactly
|
||||
let max_nelmts_bits: u8 = 32;
|
||||
let idx_blk_elmts: u8 = 4;
|
||||
let min_dblk_nelmts: u8 = 16;
|
||||
let super_blk_min_nelmts: u8 = 4;
|
||||
let max_dblk_nelmts_bits: u8 = 10;
|
||||
// Elements past the last defined one are never realised
|
||||
// (`max_idx_set` is one past the highest index ever set).
|
||||
let max_idx_set = slots.iter().rposition(Option::is_some).map_or(0, |i| i + 1);
|
||||
let slots = &slots[..max_idx_set];
|
||||
let defined_in = |start: usize, n: usize| -> bool {
|
||||
let lo = idx_blk.saturating_add(start).min(slots.len());
|
||||
let hi = idx_blk
|
||||
.saturating_add(start)
|
||||
.saturating_add(n)
|
||||
.min(slots.len());
|
||||
slots[lo..hi].iter().any(Option::is_some)
|
||||
};
|
||||
|
||||
// EAHD size: fixed(12) + 6 stats(6*length_size) + addr(offset_size) + checksum(4)
|
||||
// Super block levels: (ndblks, dblk_nelmts, first element).
|
||||
let log2_dmin = (DATA_BLK_MIN_ELMTS as u32).trailing_zeros() as usize;
|
||||
let nsblks = 1 + MAX_NELMTS_BITS as usize - log2_dmin;
|
||||
let ndblk_addrs = 2 * (SUP_BLK_MIN_DATA_PTRS as usize - 1);
|
||||
let mut levels: Vec<(usize, usize, usize)> = Vec::with_capacity(nsblks);
|
||||
let mut start = 0usize;
|
||||
for u in 0..nsblks {
|
||||
let ndblks = 1usize << (u / 2);
|
||||
let nelmts = (DATA_BLK_MIN_ELMTS as usize) << u.div_ceil(2);
|
||||
levels.push((ndblks, nelmts, start));
|
||||
// Saturate: on 32-bit targets the last levels only need to compare
|
||||
// as "beyond the end".
|
||||
start = start.saturating_add(ndblks.saturating_mul(nelmts));
|
||||
}
|
||||
// Levels whose data blocks the index block addresses directly.
|
||||
let mut direct_levels = 0;
|
||||
let mut n = 0;
|
||||
while n < ndblk_addrs {
|
||||
n += levels[direct_levels].0;
|
||||
direct_levels += 1;
|
||||
}
|
||||
let nsblk_addrs = nsblks - direct_levels;
|
||||
|
||||
let dblk_size = |nelmts: usize| -> usize {
|
||||
let prefix = 4 + 1 + 1 + os + arr_off_size + 4;
|
||||
if nelmts > page_nelmts {
|
||||
prefix + (nelmts / page_nelmts) * (page_nelmts * elem_size + 4)
|
||||
} else {
|
||||
prefix + nelmts * elem_size
|
||||
}
|
||||
};
|
||||
let sblk_bitmap_len = |ndblks: usize, nelmts: usize| -> usize {
|
||||
if nelmts > page_nelmts {
|
||||
ndblks * (nelmts / page_nelmts).div_ceil(8)
|
||||
} else {
|
||||
0
|
||||
}
|
||||
};
|
||||
|
||||
// Plan addresses: header, index block, the direct data blocks, then each
|
||||
// allocated super block followed by its allocated data blocks.
|
||||
let aehd_size = 4 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 6 * length_size as usize + os + 4;
|
||||
let aeib_address = ea_base_address + aehd_size as u64;
|
||||
let aeib_size = 4 + 1 + 1 + os + idx_blk * elem_size + ndblk_addrs * os + nsblk_addrs * os + 4;
|
||||
let mut cursor = aeib_address + aeib_size as u64;
|
||||
|
||||
// Determine how many elements go inline vs data blocks
|
||||
let n_inline = (idx_blk_elmts as usize).min(num_elements);
|
||||
let remaining_after_inline = num_elements.saturating_sub(n_inline);
|
||||
let mut ndata_blks = 0u64;
|
||||
let mut data_blk_size = 0u64;
|
||||
let mut nsuper_blks = 0u64;
|
||||
let mut super_blk_size = 0u64;
|
||||
let mut realized = idx_blk as u64;
|
||||
|
||||
// Compute super block layout per HDF5 spec
|
||||
let sblk_min = super_blk_min_nelmts as usize;
|
||||
let log2_dblk_min = if min_dblk_nelmts <= 1 {
|
||||
0
|
||||
} else {
|
||||
(min_dblk_nelmts as u32).trailing_zeros() as usize
|
||||
};
|
||||
let nsblks = (max_nelmts_bits as usize).saturating_sub(log2_dblk_min) + 1;
|
||||
|
||||
// Direct data block addresses (from super blocks 0..sblk_min-1)
|
||||
let mut dblk_sizes: Vec<usize> = Vec::new();
|
||||
for sblk_idx in 0..sblk_min.min(nsblks) {
|
||||
let ndblks = 1usize << (sblk_idx / 2);
|
||||
let dblk_nelmts = (min_dblk_nelmts as usize) * (1 << sblk_idx.div_ceil(2));
|
||||
for _ in 0..ndblks {
|
||||
dblk_sizes.push(dblk_nelmts);
|
||||
let mut plan_dblk = |cursor: &mut u64, start: usize, nelmts: usize| -> DataBlock {
|
||||
let addr = defined_in(start, nelmts).then(|| {
|
||||
let a = *cursor;
|
||||
let size = dblk_size(nelmts) as u64;
|
||||
*cursor += size;
|
||||
ndata_blks += 1;
|
||||
data_blk_size += size;
|
||||
realized += nelmts as u64;
|
||||
a
|
||||
});
|
||||
DataBlock {
|
||||
start,
|
||||
nelmts,
|
||||
addr,
|
||||
}
|
||||
}
|
||||
let n_direct_dblks = dblk_sizes.len();
|
||||
|
||||
// Super block addresses (for super blocks sblk_min..nsblks-1)
|
||||
let n_sblk_addrs = nsblks.saturating_sub(sblk_min);
|
||||
|
||||
// EAIB size
|
||||
let aeib_size = 4
|
||||
+ 1
|
||||
+ 1
|
||||
+ os
|
||||
+ idx_blk_elmts as usize * elem_size
|
||||
+ n_direct_dblks * os
|
||||
+ n_sblk_addrs * os
|
||||
+ 4;
|
||||
|
||||
// Build AEHD
|
||||
let mut aehd = Vec::with_capacity(aehd_size);
|
||||
aehd.extend_from_slice(b"EAHD");
|
||||
aehd.push(0); // version
|
||||
aehd.push(client_id);
|
||||
aehd.push(elem_size as u8);
|
||||
aehd.push(max_nelmts_bits);
|
||||
aehd.push(idx_blk_elmts);
|
||||
aehd.push(min_dblk_nelmts);
|
||||
aehd.push(super_blk_min_nelmts);
|
||||
aehd.push(max_dblk_nelmts_bits);
|
||||
|
||||
// Count data blocks that will have chunks
|
||||
let n_active_dblks: u64 = if remaining_after_inline > 0 {
|
||||
let mut count = 0u64;
|
||||
let mut ci = n_inline;
|
||||
for &sz in &dblk_sizes {
|
||||
if ci < num_elements {
|
||||
count += 1;
|
||||
ci += sz;
|
||||
}
|
||||
}
|
||||
count
|
||||
} else {
|
||||
0
|
||||
};
|
||||
let blk_off_size = (max_nelmts_bits as usize).div_ceil(8);
|
||||
let aedb_header_overhead = 4 + 1 + 1 + os + blk_off_size + 4;
|
||||
let data_blk_total_size: u64 = if remaining_after_inline > 0 {
|
||||
let mut total = 0u64;
|
||||
let mut ci = n_inline;
|
||||
for &sz in &dblk_sizes {
|
||||
if ci < num_elements {
|
||||
total += (aedb_header_overhead + sz * elem_size) as u64;
|
||||
ci += sz;
|
||||
}
|
||||
}
|
||||
total
|
||||
} else {
|
||||
0
|
||||
};
|
||||
let max_idx_set: u64 = if remaining_after_inline > 0 {
|
||||
let mut max_set = idx_blk_elmts as u64;
|
||||
let mut ci = n_inline;
|
||||
for &sz in &dblk_sizes {
|
||||
if ci < num_elements {
|
||||
max_set += sz as u64;
|
||||
ci += sz;
|
||||
}
|
||||
}
|
||||
max_set
|
||||
} else {
|
||||
idx_blk_elmts as u64
|
||||
};
|
||||
|
||||
let mut direct: Vec<DataBlock> = Vec::with_capacity(ndblk_addrs);
|
||||
for &(ndblks, nelmts, first) in &levels[..direct_levels] {
|
||||
for k in 0..ndblks {
|
||||
direct.push(plan_dblk(&mut cursor, first + k * nelmts, nelmts));
|
||||
}
|
||||
}
|
||||
// (super block address, level, its data blocks)
|
||||
let mut supers: Vec<(Option<u64>, usize, Vec<DataBlock>)> = Vec::with_capacity(nsblk_addrs);
|
||||
for (u, &(ndblks, nelmts, first)) in levels.iter().enumerate().skip(direct_levels) {
|
||||
if !defined_in(first, ndblks.saturating_mul(nelmts)) {
|
||||
supers.push((None, u, Vec::new()));
|
||||
continue;
|
||||
}
|
||||
let sb_size =
|
||||
4 + 1 + 1 + os + arr_off_size + sblk_bitmap_len(ndblks, nelmts) + ndblks * os + 4;
|
||||
let sb_addr = cursor;
|
||||
cursor += sb_size as u64;
|
||||
nsuper_blks += 1;
|
||||
super_blk_size += sb_size as u64;
|
||||
let dblks = (0..ndblks)
|
||||
.map(|k| plan_dblk(&mut cursor, first + k * nelmts, nelmts))
|
||||
.collect();
|
||||
supers.push((Some(sb_addr), u, dblks));
|
||||
}
|
||||
|
||||
let slot = |i: usize| slots.get(i).and_then(Option::as_ref);
|
||||
let write_length = |buf: &mut Vec<u8>, val: u64| match length_size {
|
||||
4 => buf.extend_from_slice(&(val as u32).to_le_bytes()),
|
||||
_ => buf.extend_from_slice(&val.to_le_bytes()),
|
||||
};
|
||||
let write_addr = |buf: &mut Vec<u8>, val: u64| match offset_size {
|
||||
4 => buf.extend_from_slice(&(val as u32).to_le_bytes()),
|
||||
_ => buf.extend_from_slice(&val.to_le_bytes()),
|
||||
let write_addr_opt = |buf: &mut Vec<u8>, addr: Option<u64>| match addr {
|
||||
Some(a) => push_addr(buf, a, offset_size),
|
||||
None => buf.extend(core::iter::repeat_n(0xFF, os)),
|
||||
};
|
||||
let block_prefix = |buf: &mut Vec<u8>, sig: &[u8; 4], block_off: usize| {
|
||||
buf.extend_from_slice(sig);
|
||||
buf.push(0); // version
|
||||
buf.push(client_id);
|
||||
push_addr(buf, ea_base_address, offset_size);
|
||||
buf.extend_from_slice(&(block_off as u64).to_le_bytes()[..arr_off_size]);
|
||||
};
|
||||
// Serialise one data block (paged or not) onto `out`.
|
||||
let write_dblk = |out: &mut Vec<u8>, db: &DataBlock| {
|
||||
let at = out.len();
|
||||
block_prefix(out, b"EADB", db.start);
|
||||
let first = idx_blk + db.start;
|
||||
if db.nelmts > page_nelmts {
|
||||
// Paged: the prefix carries only its own checksum; each page
|
||||
// follows with one of its own.
|
||||
let sum = jenkins_lookup3(&out[at..]);
|
||||
out.extend_from_slice(&sum.to_le_bytes());
|
||||
for p in 0..db.nelmts / page_nelmts {
|
||||
let page_at = out.len();
|
||||
for e in 0..page_nelmts {
|
||||
let i = first + p * page_nelmts + e;
|
||||
push_index_element(out, slot(i), offset_size, chunk_size_bytes);
|
||||
}
|
||||
let sum = jenkins_lookup3(&out[page_at..]);
|
||||
out.extend_from_slice(&sum.to_le_bytes());
|
||||
}
|
||||
} else {
|
||||
for i in first..first + db.nelmts {
|
||||
push_index_element(out, slot(i), offset_size, chunk_size_bytes);
|
||||
}
|
||||
let sum = jenkins_lookup3(&out[at..]);
|
||||
out.extend_from_slice(&sum.to_le_bytes());
|
||||
}
|
||||
debug_assert_eq!(out.len() - at, dblk_size(db.nelmts));
|
||||
};
|
||||
|
||||
write_length(&mut aehd, 0);
|
||||
write_length(&mut aehd, 0);
|
||||
write_length(&mut aehd, n_active_dblks);
|
||||
write_length(&mut aehd, data_blk_total_size);
|
||||
write_length(&mut aehd, num_elements as u64);
|
||||
write_length(&mut aehd, max_idx_set);
|
||||
// Header (EAHD). The six statistics are, in order: super blocks, their
|
||||
// bytes, data blocks, their bytes, max index set, elements realised.
|
||||
let mut out = Vec::with_capacity((cursor - ea_base_address) as usize);
|
||||
out.extend_from_slice(b"EAHD");
|
||||
out.push(0); // version
|
||||
out.push(client_id);
|
||||
out.push(elem_size as u8);
|
||||
out.push(MAX_NELMTS_BITS);
|
||||
out.push(IDX_BLK_ELMTS);
|
||||
out.push(DATA_BLK_MIN_ELMTS);
|
||||
out.push(SUP_BLK_MIN_DATA_PTRS);
|
||||
out.push(MAX_DBLK_PAGE_NELMTS_BITS);
|
||||
write_length(&mut out, nsuper_blks);
|
||||
write_length(&mut out, super_blk_size);
|
||||
write_length(&mut out, ndata_blks);
|
||||
write_length(&mut out, data_blk_size);
|
||||
write_length(&mut out, max_idx_set as u64);
|
||||
write_length(&mut out, realized);
|
||||
push_addr(&mut out, aeib_address, offset_size);
|
||||
let sum = jenkins_lookup3(&out);
|
||||
out.extend_from_slice(&sum.to_le_bytes());
|
||||
debug_assert_eq!(out.len(), aehd_size);
|
||||
|
||||
write_addr(&mut aehd, aeib_address);
|
||||
|
||||
let aehd_checksum = jenkins_lookup3(&aehd);
|
||||
aehd.extend_from_slice(&aehd_checksum.to_le_bytes());
|
||||
debug_assert_eq!(aehd.len(), aehd_size);
|
||||
|
||||
// Build AEIB
|
||||
let mut aeib = Vec::with_capacity(aeib_size);
|
||||
aeib.extend_from_slice(b"EAIB");
|
||||
aeib.push(0);
|
||||
aeib.push(client_id);
|
||||
|
||||
match offset_size {
|
||||
4 => aeib.extend_from_slice(&(ea_base_address as u32).to_le_bytes()),
|
||||
8 => aeib.extend_from_slice(&ea_base_address.to_le_bytes()),
|
||||
_ => aeib.extend_from_slice(&ea_base_address.to_le_bytes()),
|
||||
// Index block (EAIB): inline elements, data block and super block
|
||||
// addresses.
|
||||
let ib_start = out.len();
|
||||
out.extend_from_slice(b"EAIB");
|
||||
out.push(0);
|
||||
out.push(client_id);
|
||||
push_addr(&mut out, ea_base_address, offset_size);
|
||||
for i in 0..idx_blk {
|
||||
push_index_element(&mut out, slot(i), offset_size, chunk_size_bytes);
|
||||
}
|
||||
|
||||
// Inline elements
|
||||
#[allow(clippy::needless_range_loop)]
|
||||
for i in 0..idx_blk_elmts as usize {
|
||||
if i < n_inline {
|
||||
write_chunk_element(
|
||||
&mut aeib,
|
||||
&chunks[i],
|
||||
offset_size,
|
||||
has_filters,
|
||||
chunk_size_bytes,
|
||||
);
|
||||
} else {
|
||||
write_undefined_element(&mut aeib, offset_size, has_filters, chunk_size_bytes);
|
||||
for db in &direct {
|
||||
write_addr_opt(&mut out, db.addr);
|
||||
}
|
||||
for (sb_addr, _, _) in &supers {
|
||||
write_addr_opt(&mut out, *sb_addr);
|
||||
}
|
||||
let sum = jenkins_lookup3(&out[ib_start..]);
|
||||
out.extend_from_slice(&sum.to_le_bytes());
|
||||
debug_assert_eq!(out.len() - ib_start, aeib_size);
|
||||
|
||||
// Data block addresses + build data blocks
|
||||
let mut data_blocks_buf = Vec::new();
|
||||
let dblks_base = aeib_address + aeib_size as u64;
|
||||
let mut dblk_cursor = dblks_base;
|
||||
let mut chunk_idx = n_inline;
|
||||
|
||||
for &nelmts in &dblk_sizes {
|
||||
if chunk_idx >= num_elements {
|
||||
match offset_size {
|
||||
4 => aeib.extend_from_slice(&u32::MAX.to_le_bytes()),
|
||||
8 => aeib.extend_from_slice(&u64::MAX.to_le_bytes()),
|
||||
_ => aeib.extend_from_slice(&u64::MAX.to_le_bytes()),
|
||||
for db in direct.iter().filter(|d| d.addr.is_some()) {
|
||||
write_dblk(&mut out, db);
|
||||
}
|
||||
for (sb_addr, u, dblks) in &supers {
|
||||
if sb_addr.is_none() {
|
||||
continue;
|
||||
}
|
||||
|
||||
match offset_size {
|
||||
4 => aeib.extend_from_slice(&(dblk_cursor as u32).to_le_bytes()),
|
||||
8 => aeib.extend_from_slice(&dblk_cursor.to_le_bytes()),
|
||||
_ => aeib.extend_from_slice(&dblk_cursor.to_le_bytes()),
|
||||
}
|
||||
|
||||
// Build EADB
|
||||
let mut aedb = Vec::new();
|
||||
aedb.extend_from_slice(b"EADB");
|
||||
aedb.push(0);
|
||||
aedb.push(client_id);
|
||||
match offset_size {
|
||||
4 => aedb.extend_from_slice(&(ea_base_address as u32).to_le_bytes()),
|
||||
8 => aedb.extend_from_slice(&ea_base_address.to_le_bytes()),
|
||||
_ => aedb.extend_from_slice(&ea_base_address.to_le_bytes()),
|
||||
}
|
||||
|
||||
let blk_off_size = (max_nelmts_bits as usize).div_ceil(8);
|
||||
let blk_off_val = (chunk_idx - n_inline) as u64;
|
||||
aedb.extend_from_slice(&blk_off_val.to_le_bytes()[..blk_off_size]);
|
||||
|
||||
for slot in 0..nelmts {
|
||||
if chunk_idx + slot < num_elements {
|
||||
write_chunk_element(
|
||||
&mut aedb,
|
||||
&chunks[chunk_idx + slot],
|
||||
offset_size,
|
||||
has_filters,
|
||||
chunk_size_bytes,
|
||||
);
|
||||
} else {
|
||||
write_undefined_element(&mut aedb, offset_size, has_filters, chunk_size_bytes);
|
||||
let (ndblks, nelmts, first) = levels[*u];
|
||||
let sb_start = out.len();
|
||||
block_prefix(&mut out, b"EASB", first);
|
||||
if nelmts > page_nelmts {
|
||||
// Page-init bits, `npages` per data block, packed MSB-first
|
||||
// (`H5VM_bit_set`): every page of an allocated data block is
|
||||
// written.
|
||||
let npages = nelmts / page_nelmts;
|
||||
let mut bitmap = vec![0u8; sblk_bitmap_len(ndblks, nelmts)];
|
||||
for (k, db) in dblks.iter().enumerate() {
|
||||
if db.addr.is_some() {
|
||||
for p in 0..npages {
|
||||
let bit = k * npages + p;
|
||||
bitmap[bit / 8] |= 0x80 >> (bit % 8);
|
||||
}
|
||||
}
|
||||
|
||||
let aedb_checksum = jenkins_lookup3(&aedb);
|
||||
aedb.extend_from_slice(&aedb_checksum.to_le_bytes());
|
||||
|
||||
dblk_cursor += aedb.len() as u64;
|
||||
data_blocks_buf.extend_from_slice(&aedb);
|
||||
chunk_idx += nelmts;
|
||||
}
|
||||
|
||||
// Super block addresses (all undefined)
|
||||
for _ in 0..n_sblk_addrs {
|
||||
match offset_size {
|
||||
4 => aeib.extend_from_slice(&u32::MAX.to_le_bytes()),
|
||||
8 => aeib.extend_from_slice(&u64::MAX.to_le_bytes()),
|
||||
_ => aeib.extend_from_slice(&u64::MAX.to_le_bytes()),
|
||||
out.extend_from_slice(&bitmap);
|
||||
}
|
||||
for db in dblks {
|
||||
write_addr_opt(&mut out, db.addr);
|
||||
}
|
||||
let sum = jenkins_lookup3(&out[sb_start..]);
|
||||
out.extend_from_slice(&sum.to_le_bytes());
|
||||
for db in dblks.iter().filter(|d| d.addr.is_some()) {
|
||||
write_dblk(&mut out, db);
|
||||
}
|
||||
}
|
||||
|
||||
let aeib_checksum = jenkins_lookup3(&aeib);
|
||||
aeib.extend_from_slice(&aeib_checksum.to_le_bytes());
|
||||
debug_assert_eq!(aeib.len(), aeib_size);
|
||||
|
||||
let mut combined = aehd;
|
||||
combined.extend_from_slice(&aeib);
|
||||
combined.extend_from_slice(&data_blocks_buf);
|
||||
combined
|
||||
}
|
||||
|
||||
fn write_chunk_element(
|
||||
buf: &mut Vec<u8>,
|
||||
chunk: &WrittenChunk,
|
||||
offset_size: u8,
|
||||
has_filters: bool,
|
||||
chunk_size_bytes: usize,
|
||||
) {
|
||||
match offset_size {
|
||||
4 => buf.extend_from_slice(&(chunk.address as u32).to_le_bytes()),
|
||||
8 => buf.extend_from_slice(&chunk.address.to_le_bytes()),
|
||||
_ => buf.extend_from_slice(&chunk.address.to_le_bytes()),
|
||||
}
|
||||
if has_filters {
|
||||
let cs_bytes = chunk.compressed_size.to_le_bytes();
|
||||
buf.extend_from_slice(&cs_bytes[..chunk_size_bytes]);
|
||||
buf.extend_from_slice(&chunk.filter_mask.to_le_bytes());
|
||||
}
|
||||
}
|
||||
|
||||
fn write_undefined_element(
|
||||
buf: &mut Vec<u8>,
|
||||
offset_size: u8,
|
||||
has_filters: bool,
|
||||
chunk_size_bytes: usize,
|
||||
) {
|
||||
let os = offset_size as usize;
|
||||
// Use extend with repeat to avoid heap-allocating a temporary Vec on each call.
|
||||
buf.extend(core::iter::repeat_n(0xFF, os));
|
||||
if has_filters {
|
||||
buf.extend(core::iter::repeat_n(0x00, chunk_size_bytes));
|
||||
buf.extend_from_slice(&0u32.to_le_bytes());
|
||||
}
|
||||
debug_assert_eq!(out.len() as u64, cursor - ea_base_address);
|
||||
out
|
||||
}
|
||||
|
||||
@@ -390,3 +390,24 @@ fn we_write_paged_fixed_array() {
|
||||
cases.push(wcase("fa_2d_1100", &[110, 40], &[1, 4], None));
|
||||
check_we_write(&cases);
|
||||
}
|
||||
|
||||
/// An Extensible Array holds 4 elements in its index block and 240 in the
|
||||
/// data blocks the index block addresses; everything after that lives under
|
||||
/// super blocks, and from ~131K elements on in paged data blocks. Chunks past
|
||||
/// index 243 used to be written but never indexed (read back as fill by us
|
||||
/// and by libhdf5).
|
||||
#[test]
|
||||
fn we_write_extensible_array_past_index_block() {
|
||||
let unl: &[u64] = &[u64::MAX];
|
||||
let mut cases: Vec<WriteCase> = [1u64, 4, 5, 243, 244, 245, 300, 1000, 5000]
|
||||
.iter()
|
||||
.map(|&n| wcase(&format!("ea_{n}"), &[n * 4], &[4], Some(unl)))
|
||||
.collect();
|
||||
let mut filtered = wcase("ea_300_deflate", &[300 * 4], &[4], Some(unl));
|
||||
filtered.deflate = true;
|
||||
cases.push(filtered);
|
||||
// Several super blocks and paged data blocks (level 13, the first with
|
||||
// data blocks over 1024 elements, starts at element 4 + 131056).
|
||||
cases.push(wcase("ea_140000", &[140_000], &[1], Some(unl)));
|
||||
check_we_write(&cases);
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user