quantumclaw/clawhdf5
3
1
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 1 Wiki Activity

fix(format): write layout v4 chunk dimensions in the fewest bytes

Browse Source 
libhdf5 encodes a version-4 layout's chunk dimensions in (log2(max) +
8) / 8 bytes, and HDF5 2.0.0 (h5py 3.16) refuses any other width:
"stored chunk dimension encoding length does not match value calculated
from chunk dimensions". The writer rounded 3 bytes up to 4, so h5py
could not open a dataset we wrote with a chunk dimension from 65 536 to
16 777 215, for every chunk index (single chunk, fixed and extensible
array, v2 B-tree). The three encoders now share push_v4_chunk_dims,
which writes the exact width.

Co-Authored-By: Claude Opus 5.5 (1M context) <[email protected]>
This commit is contained in:
osobh
2026-09-26 01:27:22 -05:00
co-authored by Claude Opus 5.5
parent f713847e65
commit afae86f3ea
4 changed files with 57 additions and 92 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/clawhdf5-format/src/chunked_write.rs
+21 -62
View File
@@ -383,39 +383,7 @@ fn serialize_v4_single_chunk(
let ndims = chunk_dims.len() as u8 + 1;
buf.push(ndims);
// dim_size_encoded_length: how many bytes per dimension
// We need to figure out the minimum encoding width
let max_dim = chunk_dims
.iter()
.map(|&d| d as u64)
.chain(core::iter::once(element_size as u64))
.max()
.unwrap_or(1);
let dim_encoded_len: u8 = if max_dim <= 0xFF {
1
} else if max_dim <= 0xFFFF {
2
} else {
4
};
buf.push(dim_encoded_len);
// dimension sizes (chunk dims + element size)
for &d in chunk_dims {
match dim_encoded_len {
1 => buf.push(d as u8),
2 => buf.extend_from_slice(&(d as u16).to_le_bytes()),
4 => buf.extend_from_slice(&d.to_le_bytes()),
_ => {}
}
}
// Element size dimension
match dim_encoded_len {
1 => buf.push(element_size as u8),
2 => buf.extend_from_slice(&(element_size as u16).to_le_bytes()),
4 => buf.extend_from_slice(&element_size.to_le_bytes()),
_ => {}
}
push_v4_chunk_dims(&mut buf, chunk_dims, element_size);
// chunk index type = 1 (single chunk)
buf.push(1);
@@ -465,6 +433,25 @@ fn serialize_v4_fixed_array(
/// The part of a v4 chunked layout message before the chunk index type:
/// version, class, flags and the chunk dimensions (plus the element size).
/// Append a v4 layout's dimension width and its dimensions (the chunk
/// dimensions, then the element size). Each takes the fewest bytes that hold
/// the largest, as libhdf5 computes it (`H5D__chunk_set_sizes`:
/// `(log2(dim) + 8) / 8`); HDF5 2.0.0 refuses any other width.
pub(crate) fn push_v4_chunk_dims(buf: &mut Vec<u8>, chunk_dims: &[u32], element_size: u32) {
let max_dim = chunk_dims
.iter()
.copied()
.chain(core::iter::once(element_size))
.max()
.unwrap_or(1)
.max(1);
let width = (32 - max_dim.leading_zeros()).div_ceil(8) as usize;
buf.push(width as u8);
for &d in chunk_dims.iter().chain(core::iter::once(&element_size)) {
buf.extend_from_slice(&d.to_le_bytes()[..width]);
}
}
fn layout_v4_chunked_prefix(chunk_dims: &[u32], element_size: u32) -> Vec<u8> {
let mut buf = Vec::new();
buf.push(4); // version
@@ -476,35 +463,7 @@ fn layout_v4_chunked_prefix(chunk_dims: &[u32], element_size: u32) -> Vec<u8> {
let ndims = chunk_dims.len() as u8 + 1;
buf.push(ndims);
let max_dim = chunk_dims
.iter()
.map(|&d| d as u64)
.chain(core::iter::once(element_size as u64))
.max()
.unwrap_or(1);
let dim_encoded_len: u8 = if max_dim <= 0xFF {
1
} else if max_dim <= 0xFFFF {
2
} else {
4
};
buf.push(dim_encoded_len);
for &d in chunk_dims {
match dim_encoded_len {
1 => buf.push(d as u8),
2 => buf.extend_from_slice(&(d as u16).to_le_bytes()),
4 => buf.extend_from_slice(&d.to_le_bytes()),
_ => {}
}
}
match dim_encoded_len {
1 => buf.push(element_size as u8),
2 => buf.extend_from_slice(&(element_size as u16).to_le_bytes()),
4 => buf.extend_from_slice(&element_size.to_le_bytes()),
_ => {}
}
push_v4_chunk_dims(&mut buf, chunk_dims, element_size);
buf
}