quantumclaw/clawhdf5
3
1
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 1 Wiki Activity

fix(format): write layout v4 chunk dimensions in the fewest bytes

Browse Source 
libhdf5 encodes a version-4 layout's chunk dimensions in (log2(max) +
8) / 8 bytes, and HDF5 2.0.0 (h5py 3.16) refuses any other width:
"stored chunk dimension encoding length does not match value calculated
from chunk dimensions". The writer rounded 3 bytes up to 4, so h5py
could not open a dataset we wrote with a chunk dimension from 65 536 to
16 777 215, for every chunk index (single chunk, fixed and extensible
array, v2 B-tree). The three encoders now share push_v4_chunk_dims,
which writes the exact width.

Co-Authored-By: Claude Opus 5.5 (1M context) <[email protected]>
This commit is contained in:
osobh
2026-09-26 01:27:22 -05:00
co-authored by Claude Opus 5.5
parent f713847e65
commit afae86f3ea
4 changed files with 57 additions and 92 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/clawhdf5-format/src/ea_writer.rs
+4 -30
View File
@@ -7,7 +7,9 @@ extern crate alloc;
use alloc::{vec, vec::Vec};
use crate::checksum::jenkins_lookup3;
use crate::chunked_write::{WrittenChunk, filtered_chunk_size_len, push_addr, push_index_element};
use crate::chunked_write::{
WrittenChunk, filtered_chunk_size_len, push_addr, push_index_element, push_v4_chunk_dims,
};
/// Serialize a v4 Extensible Array layout message.
pub(crate) fn serialize_v4_extensible_array(
@@ -24,35 +26,7 @@ pub(crate) fn serialize_v4_extensible_array(
let ndims = chunk_dims.len() as u8 + 1;
buf.push(ndims);
let max_dim = chunk_dims
.iter()
.map(|&d| d as u64)
.chain(core::iter::once(element_size as u64))
.max()
.unwrap_or(1);
let dim_encoded_len: u8 = if max_dim <= 0xFF {
1
} else if max_dim <= 0xFFFF {
2
} else {
4
};
buf.push(dim_encoded_len);
for &d in chunk_dims {
match dim_encoded_len {
1 => buf.push(d as u8),
2 => buf.extend_from_slice(&(d as u16).to_le_bytes()),
4 => buf.extend_from_slice(&d.to_le_bytes()),
_ => unreachable!("unexpected dim_encoded_len: {dim_encoded_len}"),
}
}
match dim_encoded_len {
1 => buf.push(element_size as u8),
2 => buf.extend_from_slice(&(element_size as u16).to_le_bytes()),
4 => buf.extend_from_slice(&element_size.to_le_bytes()),
_ => unreachable!("unexpected dim_encoded_len: {dim_encoded_len}"),
}
push_v4_chunk_dims(&mut buf, chunk_dims, element_size);
// chunk index type = 4 (Extensible Array)
buf.push(4);