Huge chunks: selection tests with fill values, LZ4 over 256 MiB, wasm32 check

A selection of a chunked dataset with a fill value is compared with the
full read in every index, through a map and through positioned reads. An
LZ4 chunk larger than 256 MiB is bounded by the chunk size, not refused.
The wasm package test reads the 4 GiB-chunk fixture and gets a clean
error in every index.

Co-Authored-By: Claude Opus 5.5 (1M context) <[email protected]>

crates/clawhdf5-format/src/chunked_read.rs

@@ -1491,7 +1491,7 @@ pub fn list_chunks_for_read_in<S: Storage + ?Sized>(
         let chunk_bytes = checked_chunk_byte_len(&chunk_dims, elem_size)?;
         if let Some(c) = chunks
             .iter()
-            .find(|c| c.address != u64::MAX && c.chunk_size as usize != chunk_bytes)
+            .find(|c| c.address != u64::MAX && c.chunk_size != chunk_bytes as u64)
         {
             return Err(FormatError::ChunkedReadError(format!(
                 "incorrect chunk size returned from index for unfiltered chunk at {:?}: \
@@ -2470,7 +2470,7 @@ mod tests {
         // Entries: key[i], child[i] pairs, then final key
         for chunk in chunks {
             // Key: chunk_size(4) + filter_mask(4) + ndims offsets
-            buf.extend_from_slice(&chunk.chunk_size.to_le_bytes());
+            buf.extend_from_slice(&(chunk.chunk_size as u32).to_le_bytes());
             buf.extend_from_slice(&chunk.filter_mask.to_le_bytes());
             for d in 0..ndims {
                 let off = if d < chunk.offsets.len() {

crates/clawhdf5-format/src/chunked_write.rs

@@ -2263,7 +2263,7 @@ mod tests {
         for i in 0..n {
             let (offsets, chunk) = extract_chunk(&data, &shape, &chunks, 2, i);
             assert_eq!(chunk.len(), 2 * 3 * 2 * 2);
-            for (e, pair) in chunk.chunks_exact(2).enumerate() {
+            for (e, pair) in chunk.as_chunks::<2>().0.iter().enumerate() {
                 let c = [e / 6, (e / 2) % 3, e % 2];
                 let g: Vec<u64> = (0..3).map(|d| offsets[d] + c[d] as u64).collect();
                 let expect = if (0..3).all(|d| g[d] < shape[d]) {
@@ -2272,7 +2272,7 @@ mod tests {
                 } else {
                     [0, 0]
                 };
-                assert_eq!(pair, expect, "chunk {i} element {e}");
+                assert_eq!(*pair, expect, "chunk {i} element {e}");
             }
         }
         // Data shorter than the shape: the missing elements stay zero.

crates/clawhdf5-format/src/extensible_array.rs

@@ -946,7 +946,7 @@ mod tests {
         assert_eq!(chunks.len(), 2);
         assert_eq!(chunks[0].address, base_addr);
         assert_eq!(chunks[0].offsets, vec![0]);
-        assert_eq!(chunks[0].chunk_size, chunk_byte_size as u64);
+        assert_eq!(chunks[0].chunk_size, chunk_byte_size);
         assert_eq!(chunks[1].address, base_addr + chunk_byte_size);
         assert_eq!(chunks[1].offsets, vec![20]);
     }

crates/clawhdf5-format/src/filters.rs

@@ -2764,6 +2764,39 @@ mod tests {
         assert_eq!(decompressed, data);
     }
 
+    /// A chunk's size bounds an LZ4 chunk, not the 256 MiB ceiling for an
+    /// unknown size: a 300 MiB chunk was refused ("declared size exceeds
+    /// limit").
+    #[test]
+    #[cfg(feature = "lz4")]
+    fn lz4_chunks_over_256_mib_decode() {
+        let mut data = vec![0u8; 300 << 20];
+        data[12345] = 7;
+        let compressed = lz4_compress(&data, &[]).unwrap();
+        let decompressed = lz4_decompress(&compressed, data.len()).unwrap();
+        assert!(decompressed == data);
+        // Without a chunk size the ceiling still applies.
+        assert!(lz4_decompress(&compressed, 0).is_err());
+    }
+
+    /// A chunk of 4 GiB or more is always in the registered framing, whose
+    /// big-endian size then does not start with four zero bytes: its size
+    /// is read whole (here larger than the chunk, so refused before any
+    /// allocation), not taken for a legacy 4-byte size.
+    #[test]
+    #[cfg(all(feature = "lz4", target_pointer_width = "64"))]
+    fn lz4_chunks_of_4_gib_use_the_registered_framing() {
+        let chunk = (1usize << 32) + 8;
+        let mut data = ((chunk + 8) as u64).to_be_bytes().to_vec();
+        data.extend_from_slice(&(1u32 << 30).to_be_bytes());
+        data.extend_from_slice(&[0; 8]);
+        let err = lz4_decompress(&data, chunk).unwrap_err();
+        assert!(
+            matches!(&err, FormatError::DecompressionError(m) if m.contains("exceeds chunk size")),
+            "{err:?}"
+        );
+    }
+
     #[test]
     #[cfg(feature = "lz4")]
     fn pipeline_lz4_only() {

crates/clawhdf5-format/src/parallel_read.rs

@@ -255,7 +255,7 @@ pub fn decompress_chunks_lane_partitioned_in<S: Storage + ?Sized>(
                     for &local in &indices {
                         let index = batch.start + local;
                         let chunk_info = &chunks[index];
-                        let size = chunk_info.chunk_size as usize;
+                        let size = crate::addr::saturating_usize(chunk_info.chunk_size);
                         let raw_chunk = raw_bytes.get(index, &reqs[index])?;
 
                         let decompressed = decompress_chunk_exact(

crates/clawhdf5-format/src/partial_read.rs

@@ -368,7 +368,10 @@ pub fn read_selection_filled_in<S: Storage + ?Sized>(
     fill: Option<&[u8]>,
 ) -> Result<Option<Vec<u8>>, FormatError> {
     let dims = &dataspace.dimensions;
-    if dims.is_empty() || elem_size == 0 {
+    // A fill value that is not one element's bytes is the full path's to
+    // interpret.
+    let odd_fill = fill.is_some_and(|f| !f.is_empty() && f.len() != elem_size);
+    if dims.is_empty() || elem_size == 0 || odd_fill {
         return Ok(None);
     }
     let total = dataspace.checked_num_elements()?;

crates/clawhdf5-format/tests/raw_fetch_bounds.rs

@@ -151,7 +151,7 @@ fn crafted() -> (Vec<u8>, Chunked, Vec<ChunkInfo>) {
         // v1 B-tree key (size, filter mask, offsets + 0) then the child
         // address.
         let mut pat = Vec::new();
-        pat.extend_from_slice(&c.chunk_size.to_le_bytes());
+        pat.extend_from_slice(&(c.chunk_size as u32).to_le_bytes());
         pat.extend_from_slice(&c.filter_mask.to_le_bytes());
         // The key holds one offset per dimension plus the element offset
         // (0); `offsets` may or may not list that last one.