From afae86f3ea6fb3a7fea7fca26daced0cd5d651a6 Mon Sep 17 00:00:00 2001 From: osobh Date: Sat, 26 Sep 2026 01:27:22 -0500 Subject: [PATCH] fix(format): write layout v4 chunk dimensions in the fewest bytes libhdf5 encodes a version-4 layout's chunk dimensions in (log2(max) + 8) / 8 bytes, and HDF5 2.0.0 (h5py 3.16) refuses any other width: "stored chunk dimension encoding length does not match value calculated from chunk dimensions". The writer rounded 3 bytes up to 4, so h5py could not open a dataset we wrote with a chunk dimension from 65 536 to 16 777 215, for every chunk index (single chunk, fixed and extensible array, v2 B-tree). The three encoders now share push_v4_chunk_dims, which writes the exact width. Co-Authored-By: Claude Opus 5.5 (1M context) --- CHANGELOG.md | 7 ++ crates/clawhdf5-format/src/chunked_write.rs | 83 +++++--------------- crates/clawhdf5-format/src/ea_writer.rs | 34 +------- crates/clawhdf5/tests/chunk_index_interop.rs | 25 ++++++ 4 files changed, 57 insertions(+), 92 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b3afc14..c20f408 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -197,6 +197,13 @@ takes `--f32`; it had kept printing "f32" after the default changed. ### Interop +- **h5py could not open chunked datasets we wrote with a chunk dimension + from 65 536 to 16 777 215.** A version-4 layout must store its chunk + dimensions in the fewest bytes that hold the largest (3 for 70 000); + the writer rounded 3 up to 4, and HDF5 2.0.0 (h5py 3.16) refuses that + ("stored chunk dimension encoding length does not match value calculated + from chunk dimensions"). Newer libhdf5 and clawhdf5 read those files; new + files use the exact width. Test: `we_write_chunk_dimensions_in_the_fewest_bytes`. - **Conformance sweep in the repo** (`conformance/`, report in `CONFORMANCE.md`). `conformance/run.sh` fetches eight public HDF5 corpora pinned by commit (libhdf5's test files, the HDF Group's CVE reproducers, diff --git a/crates/clawhdf5-format/src/chunked_write.rs b/crates/clawhdf5-format/src/chunked_write.rs index ff5081e..dcd0e43 100644 --- a/crates/clawhdf5-format/src/chunked_write.rs +++ b/crates/clawhdf5-format/src/chunked_write.rs @@ -383,39 +383,7 @@ fn serialize_v4_single_chunk( let ndims = chunk_dims.len() as u8 + 1; buf.push(ndims); - // dim_size_encoded_length: how many bytes per dimension - // We need to figure out the minimum encoding width - let max_dim = chunk_dims - .iter() - .map(|&d| d as u64) - .chain(core::iter::once(element_size as u64)) - .max() - .unwrap_or(1); - let dim_encoded_len: u8 = if max_dim <= 0xFF { - 1 - } else if max_dim <= 0xFFFF { - 2 - } else { - 4 - }; - buf.push(dim_encoded_len); - - // dimension sizes (chunk dims + element size) - for &d in chunk_dims { - match dim_encoded_len { - 1 => buf.push(d as u8), - 2 => buf.extend_from_slice(&(d as u16).to_le_bytes()), - 4 => buf.extend_from_slice(&d.to_le_bytes()), - _ => {} - } - } - // Element size dimension - match dim_encoded_len { - 1 => buf.push(element_size as u8), - 2 => buf.extend_from_slice(&(element_size as u16).to_le_bytes()), - 4 => buf.extend_from_slice(&element_size.to_le_bytes()), - _ => {} - } + push_v4_chunk_dims(&mut buf, chunk_dims, element_size); // chunk index type = 1 (single chunk) buf.push(1); @@ -465,6 +433,25 @@ fn serialize_v4_fixed_array( /// The part of a v4 chunked layout message before the chunk index type: /// version, class, flags and the chunk dimensions (plus the element size). +/// Append a v4 layout's dimension width and its dimensions (the chunk +/// dimensions, then the element size). Each takes the fewest bytes that hold +/// the largest, as libhdf5 computes it (`H5D__chunk_set_sizes`: +/// `(log2(dim) + 8) / 8`); HDF5 2.0.0 refuses any other width. +pub(crate) fn push_v4_chunk_dims(buf: &mut Vec, chunk_dims: &[u32], element_size: u32) { + let max_dim = chunk_dims + .iter() + .copied() + .chain(core::iter::once(element_size)) + .max() + .unwrap_or(1) + .max(1); + let width = (32 - max_dim.leading_zeros()).div_ceil(8) as usize; + buf.push(width as u8); + for &d in chunk_dims.iter().chain(core::iter::once(&element_size)) { + buf.extend_from_slice(&d.to_le_bytes()[..width]); + } +} + fn layout_v4_chunked_prefix(chunk_dims: &[u32], element_size: u32) -> Vec { let mut buf = Vec::new(); buf.push(4); // version @@ -476,35 +463,7 @@ fn layout_v4_chunked_prefix(chunk_dims: &[u32], element_size: u32) -> Vec { let ndims = chunk_dims.len() as u8 + 1; buf.push(ndims); - let max_dim = chunk_dims - .iter() - .map(|&d| d as u64) - .chain(core::iter::once(element_size as u64)) - .max() - .unwrap_or(1); - let dim_encoded_len: u8 = if max_dim <= 0xFF { - 1 - } else if max_dim <= 0xFFFF { - 2 - } else { - 4 - }; - buf.push(dim_encoded_len); - - for &d in chunk_dims { - match dim_encoded_len { - 1 => buf.push(d as u8), - 2 => buf.extend_from_slice(&(d as u16).to_le_bytes()), - 4 => buf.extend_from_slice(&d.to_le_bytes()), - _ => {} - } - } - match dim_encoded_len { - 1 => buf.push(element_size as u8), - 2 => buf.extend_from_slice(&(element_size as u16).to_le_bytes()), - 4 => buf.extend_from_slice(&element_size.to_le_bytes()), - _ => {} - } + push_v4_chunk_dims(&mut buf, chunk_dims, element_size); buf } diff --git a/crates/clawhdf5-format/src/ea_writer.rs b/crates/clawhdf5-format/src/ea_writer.rs index e0f0375..f669534 100644 --- a/crates/clawhdf5-format/src/ea_writer.rs +++ b/crates/clawhdf5-format/src/ea_writer.rs @@ -7,7 +7,9 @@ extern crate alloc; use alloc::{vec, vec::Vec}; use crate::checksum::jenkins_lookup3; -use crate::chunked_write::{WrittenChunk, filtered_chunk_size_len, push_addr, push_index_element}; +use crate::chunked_write::{ + WrittenChunk, filtered_chunk_size_len, push_addr, push_index_element, push_v4_chunk_dims, +}; /// Serialize a v4 Extensible Array layout message. pub(crate) fn serialize_v4_extensible_array( @@ -24,35 +26,7 @@ pub(crate) fn serialize_v4_extensible_array( let ndims = chunk_dims.len() as u8 + 1; buf.push(ndims); - let max_dim = chunk_dims - .iter() - .map(|&d| d as u64) - .chain(core::iter::once(element_size as u64)) - .max() - .unwrap_or(1); - let dim_encoded_len: u8 = if max_dim <= 0xFF { - 1 - } else if max_dim <= 0xFFFF { - 2 - } else { - 4 - }; - buf.push(dim_encoded_len); - - for &d in chunk_dims { - match dim_encoded_len { - 1 => buf.push(d as u8), - 2 => buf.extend_from_slice(&(d as u16).to_le_bytes()), - 4 => buf.extend_from_slice(&d.to_le_bytes()), - _ => unreachable!("unexpected dim_encoded_len: {dim_encoded_len}"), - } - } - match dim_encoded_len { - 1 => buf.push(element_size as u8), - 2 => buf.extend_from_slice(&(element_size as u16).to_le_bytes()), - 4 => buf.extend_from_slice(&element_size.to_le_bytes()), - _ => unreachable!("unexpected dim_encoded_len: {dim_encoded_len}"), - } + push_v4_chunk_dims(&mut buf, chunk_dims, element_size); // chunk index type = 4 (Extensible Array) buf.push(4); diff --git a/crates/clawhdf5/tests/chunk_index_interop.rs b/crates/clawhdf5/tests/chunk_index_interop.rs index b72b7e2..3ff0420 100644 --- a/crates/clawhdf5/tests/chunk_index_interop.rs +++ b/crates/clawhdf5/tests/chunk_index_interop.rs @@ -515,6 +515,31 @@ fn we_write_maxshape_larger_than_shape() { check_we_write(&cases); } +/// A version-4 layout must encode every chunk dimension in the fewest bytes +/// that hold the largest one, as libhdf5 does: HDF5 2.0.0 (h5py 3.16) +/// refuses a wider encoding ("stored chunk dimension encoding length does +/// not match value calculated from chunk dimensions"). We rounded 3 bytes +/// up to 4, so h5py could not open any dataset we wrote with a chunk +/// dimension from 65 536 to 16 777 215. +#[test] +fn we_write_chunk_dimensions_in_the_fewest_bytes() { + const U: u64 = u64::MAX; + let mut cases = vec![ + // Single chunk, Fixed Array, Extensible Array, v2 B-tree. + wcase("single_70000", &[70_000], &[70_000], None), + wcase("fa_70000", &[140_000], &[70_000], None), + wcase("ea_70000", &[140_000], &[70_000], Some(&[U])), + wcase("bt2_70000", &[2, 70_000], &[1, 70_000], Some(&[U, U])), + // 2 bytes and 1 byte still, with the element size (4) the largest. + wcase("fa_300", &[600], &[300], None), + wcase("fa_3", &[6], &[3], None), + ]; + let mut filtered = wcase("single_70000_deflate", &[70_000], &[70_000], None); + filtered.deflate = true; + cases.push(filtered); + check_we_write(&cases); +} + /// More than one unlimited dimension needs a version-2 B-tree chunk index, /// as the library uses; an Extensible Array for `(None, None)` made libhdf5 /// refuse the whole file ("already found unlimited dimension").