CHANGELOG.md

@@ -197,6 +197,13 @@
   takes `--f32`; it had kept printing "f32" after the default changed.
 
 ### Interop
+- **h5py could not open chunked datasets we wrote with a chunk dimension
+  from 65 536 to 16 777 215.** A version-4 layout must store its chunk
+  dimensions in the fewest bytes that hold the largest (3 for 70 000);
+  the writer rounded 3 up to 4, and HDF5 2.0.0 (h5py 3.16) refuses that
+  ("stored chunk dimension encoding length does not match value calculated
+  from chunk dimensions"). Newer libhdf5 and clawhdf5 read those files; new
+  files use the exact width. Test: `we_write_chunk_dimensions_in_the_fewest_bytes`.
 - **Conformance sweep in the repo** (`conformance/`, report in
   `CONFORMANCE.md`). `conformance/run.sh` fetches eight public HDF5 corpora
   pinned by commit (libhdf5's test files, the HDF Group's CVE reproducers,

crates/clawhdf5-format/src/chunked_write.rs

@@ -383,39 +383,7 @@ fn serialize_v4_single_chunk(
     let ndims = chunk_dims.len() as u8 + 1;
     buf.push(ndims);
 
-    // dim_size_encoded_length: how many bytes per dimension
+    push_v4_chunk_dims(&mut buf, chunk_dims, element_size);
-    // We need to figure out the minimum encoding width
-    let max_dim = chunk_dims
-        .iter()
-        .map(|&d| d as u64)
-        .chain(core::iter::once(element_size as u64))
-        .max()
-        .unwrap_or(1);
-    let dim_encoded_len: u8 = if max_dim <= 0xFF {
-        1
-    } else if max_dim <= 0xFFFF {
-        2
-    } else {
-        4
-    };
-    buf.push(dim_encoded_len);
-
-    // dimension sizes (chunk dims + element size)
-    for &d in chunk_dims {
-        match dim_encoded_len {
-            1 => buf.push(d as u8),
-            2 => buf.extend_from_slice(&(d as u16).to_le_bytes()),
-            4 => buf.extend_from_slice(&d.to_le_bytes()),
-            _ => {}
-        }
-    }
-    // Element size dimension
-    match dim_encoded_len {
-        1 => buf.push(element_size as u8),
-        2 => buf.extend_from_slice(&(element_size as u16).to_le_bytes()),
-        4 => buf.extend_from_slice(&element_size.to_le_bytes()),
-        _ => {}
-    }
 
     // chunk index type = 1 (single chunk)
     buf.push(1);
@@ -465,6 +433,25 @@ fn serialize_v4_fixed_array(
 
 /// The part of a v4 chunked layout message before the chunk index type:
 /// version, class, flags and the chunk dimensions (plus the element size).
+/// Append a v4 layout's dimension width and its dimensions (the chunk
+/// dimensions, then the element size). Each takes the fewest bytes that hold
+/// the largest, as libhdf5 computes it (`H5D__chunk_set_sizes`:
+/// `(log2(dim) + 8) / 8`); HDF5 2.0.0 refuses any other width.
+pub(crate) fn push_v4_chunk_dims(buf: &mut Vec<u8>, chunk_dims: &[u32], element_size: u32) {
+    let max_dim = chunk_dims
+        .iter()
+        .copied()
+        .chain(core::iter::once(element_size))
+        .max()
+        .unwrap_or(1)
+        .max(1);
+    let width = (32 - max_dim.leading_zeros()).div_ceil(8) as usize;
+    buf.push(width as u8);
+    for &d in chunk_dims.iter().chain(core::iter::once(&element_size)) {
+        buf.extend_from_slice(&d.to_le_bytes()[..width]);
+    }
+}
+
 fn layout_v4_chunked_prefix(chunk_dims: &[u32], element_size: u32) -> Vec<u8> {
     let mut buf = Vec::new();
     buf.push(4); // version
@@ -476,35 +463,7 @@ fn layout_v4_chunked_prefix(chunk_dims: &[u32], element_size: u32) -> Vec<u8> {
     let ndims = chunk_dims.len() as u8 + 1;
     buf.push(ndims);
 
-    let max_dim = chunk_dims
+    push_v4_chunk_dims(&mut buf, chunk_dims, element_size);
-        .iter()
-        .map(|&d| d as u64)
-        .chain(core::iter::once(element_size as u64))
-        .max()
-        .unwrap_or(1);
-    let dim_encoded_len: u8 = if max_dim <= 0xFF {
-        1
-    } else if max_dim <= 0xFFFF {
-        2
-    } else {
-        4
-    };
-    buf.push(dim_encoded_len);
-
-    for &d in chunk_dims {
-        match dim_encoded_len {
-            1 => buf.push(d as u8),
-            2 => buf.extend_from_slice(&(d as u16).to_le_bytes()),
-            4 => buf.extend_from_slice(&d.to_le_bytes()),
-            _ => {}
-        }
-    }
-    match dim_encoded_len {
-        1 => buf.push(element_size as u8),
-        2 => buf.extend_from_slice(&(element_size as u16).to_le_bytes()),
-        4 => buf.extend_from_slice(&element_size.to_le_bytes()),
-        _ => {}
-    }
     buf
 }
 

crates/clawhdf5-format/src/ea_writer.rs

@@ -7,7 +7,9 @@ extern crate alloc;
 use alloc::{vec, vec::Vec};
 
 use crate::checksum::jenkins_lookup3;
-use crate::chunked_write::{WrittenChunk, filtered_chunk_size_len, push_addr, push_index_element};
+use crate::chunked_write::{
+    WrittenChunk, filtered_chunk_size_len, push_addr, push_index_element, push_v4_chunk_dims,
+};
 
 /// Serialize a v4 Extensible Array layout message.
 pub(crate) fn serialize_v4_extensible_array(
@@ -24,35 +26,7 @@ pub(crate) fn serialize_v4_extensible_array(
     let ndims = chunk_dims.len() as u8 + 1;
     buf.push(ndims);
 
-    let max_dim = chunk_dims
+    push_v4_chunk_dims(&mut buf, chunk_dims, element_size);
-        .iter()
-        .map(|&d| d as u64)
-        .chain(core::iter::once(element_size as u64))
-        .max()
-        .unwrap_or(1);
-    let dim_encoded_len: u8 = if max_dim <= 0xFF {
-        1
-    } else if max_dim <= 0xFFFF {
-        2
-    } else {
-        4
-    };
-    buf.push(dim_encoded_len);
-
-    for &d in chunk_dims {
-        match dim_encoded_len {
-            1 => buf.push(d as u8),
-            2 => buf.extend_from_slice(&(d as u16).to_le_bytes()),
-            4 => buf.extend_from_slice(&d.to_le_bytes()),
-            _ => unreachable!("unexpected dim_encoded_len: {dim_encoded_len}"),
-        }
-    }
-    match dim_encoded_len {
-        1 => buf.push(element_size as u8),
-        2 => buf.extend_from_slice(&(element_size as u16).to_le_bytes()),
-        4 => buf.extend_from_slice(&element_size.to_le_bytes()),
-        _ => unreachable!("unexpected dim_encoded_len: {dim_encoded_len}"),
-    }
 
     // chunk index type = 4 (Extensible Array)
     buf.push(4);

crates/clawhdf5/tests/chunk_index_interop.rs

@@ -515,6 +515,31 @@ fn we_write_maxshape_larger_than_shape() {
     check_we_write(&cases);
 }
 
+/// A version-4 layout must encode every chunk dimension in the fewest bytes
+/// that hold the largest one, as libhdf5 does: HDF5 2.0.0 (h5py 3.16)
+/// refuses a wider encoding ("stored chunk dimension encoding length does
+/// not match value calculated from chunk dimensions"). We rounded 3 bytes
+/// up to 4, so h5py could not open any dataset we wrote with a chunk
+/// dimension from 65 536 to 16 777 215.
+#[test]
+fn we_write_chunk_dimensions_in_the_fewest_bytes() {
+    const U: u64 = u64::MAX;
+    let mut cases = vec![
+        // Single chunk, Fixed Array, Extensible Array, v2 B-tree.
+        wcase("single_70000", &[70_000], &[70_000], None),
+        wcase("fa_70000", &[140_000], &[70_000], None),
+        wcase("ea_70000", &[140_000], &[70_000], Some(&[U])),
+        wcase("bt2_70000", &[2, 70_000], &[1, 70_000], Some(&[U, U])),
+        // 2 bytes and 1 byte still, with the element size (4) the largest.
+        wcase("fa_300", &[600], &[300], None),
+        wcase("fa_3", &[6], &[3], None),
+    ];
+    let mut filtered = wcase("single_70000_deflate", &[70_000], &[70_000], None);
+    filtered.deflate = true;
+    cases.push(filtered);
+    check_we_write(&cases);
+}
+
 /// More than one unlimited dimension needs a version-2 B-tree chunk index,
 /// as the library uses; an Extensible Array for `(None, None)` made libhdf5
 /// refuse the whole file ("already found unlimited dimension").