fix(format): a chunk that decodes short is an error, not zero-filled

HDF5 stores every chunk at the full chunk size (edge chunks are padded
before filtering, and with "don't filter partial edge chunks" they are
stored raw at full size), so a filter pipeline that decodes to fewer bytes
means a corrupt chunk. Every chunk reader padded it with zeros and
returned it as data. libhdf5 returns the rest uninitialised, or fails when
the filter checks (Blosc with nbytes = 0).

New filters::decompress_chunk_exact decodes and then requires exactly the
chunk size, with the chunk's coordinates in the error
(ChunkedReadError "chunk at [16] decoded to 16 bytes, expected 32"). It
replaces decompress_chunk_masked at every chunk read path: the full read
(sequential and lane-partitioned), the cached read, the sweep read, the
planned-selection read, parallel_read's three decoders and partial_read's
box read. decompress_chunk_masked is unchanged (fractal-heap huge objects
already checked their own size). Blosc also rejects a frame declaring no
data where the chunk size is known.

Tests, each failing with the check disabled: filters and parallel_read
unit tests; h5py_short_decoded_chunk_is_an_error (gzip chunks rewritten
short with write_direct_chunk: 1-D, a 2-D edge chunk, and 40 chunks with
shuffle, read through File full/cached/selection reads, a selection that
avoids the chunk still reads, MmapFile and LazyFile, with and without the
parallel feature); plugin_filters_interop short_decoding_chunks_are_errors
(Blosc nbytes=0 and short, LZF and bzip2 short; the Blosc nbytes=0 case
read as 16 zeros before). The existing don't-filter-partial-edge-chunks
tests still pass. Conformance (tank, 2026-09-26): 573 of 697 ok, and no
file changed class, reader result or first issue against the pre-fix run.

Co-Authored-By: Claude Opus 5.5 (1M context) <[email protected]>

crates/clawhdf5-format/src/chunked_read.rs

@@ -15,7 +15,7 @@ use crate::datatype::Datatype;
 use crate::error::FormatError;
 use crate::extensible_array::{ExtensibleArrayHeader, read_extensible_array_chunks};
 use crate::filter_pipeline::FilterPipeline;
-use crate::filters::{all_filters_skipped, decompress_chunk_masked};
+use crate::filters::{all_filters_skipped, decompress_chunk_exact};
 use crate::fixed_array::{FixedArrayHeader, read_fixed_array_chunks};
 #[cfg(feature = "std")]
 use std::sync::Arc;
@@ -65,12 +65,13 @@ fn decompress_all_chunks(
         let raw_chunk = &file_data[c_addr..c_addr + size];
 
         let decompressed = if let Some(pl) = pipeline {
-            decompress_chunk_masked(
+            decompress_chunk_exact(
                 raw_chunk,
                 pl,
                 chunk_total_bytes,
                 element_size,
                 chunk_info.filter_mask,
+                &chunk_info.offsets,
             )?
         } else {
             raw_chunk.to_vec()
@@ -932,12 +933,13 @@ pub fn read_chunked_data_cached(
     let cache_them = total_bytes <= cache.max_bytes();
     if let Some(pl) = pipeline {
         let decode = |c: &&ChunkInfo| -> Result<Vec<u8>, FormatError> {
-            decompress_chunk_masked(
+            decompress_chunk_exact(
                 raw_bytes(c)?,
                 pl,
                 chunk_total_bytes,
                 elem_size as u32,
                 c.filter_mask,
+                &c.offsets,
             )
         };
         for batch in misses.chunks(DECODE_BATCH) {
@@ -1217,12 +1219,13 @@ pub fn read_chunked_data_sweep(
             ensure_len(file_data, c_addr, size)?;
             let raw_chunk = &file_data[c_addr..c_addr + size];
             let dec = if let Some(pl) = pipeline {
-                decompress_chunk_masked(
+                decompress_chunk_exact(
                     raw_chunk,
                     pl,
                     chunk_total_bytes,
                     elem_size as u32,
                     chunk_info.filter_mask,
+                    &coord,
                 )?
             } else {
                 raw_chunk.to_vec()
@@ -1346,12 +1349,13 @@ pub fn read_chunked_data_indexed(
             ensure_len(file_data, c_addr, size)?;
             let raw_chunk = &file_data[c_addr..c_addr + size];
             let decompressed = if let Some(pl) = pipeline {
-                decompress_chunk_masked(
+                decompress_chunk_exact(
                     raw_chunk,
                     pl,
                     chunk_total_bytes,
                     elem_size as u32,
                     *filter_mask,
+                    coord,
                 )?
             } else {
                 raw_chunk.to_vec()

crates/clawhdf5-format/src/filters.rs

@@ -4,7 +4,7 @@
 extern crate alloc;
 
 #[cfg(not(feature = "std"))]
-use alloc::{boxed::Box, vec, vec::Vec};
+use alloc::{boxed::Box, format, vec, vec::Vec};
 
 use crate::error::FormatError;
 #[cfg(feature = "deflate")]
@@ -120,6 +120,34 @@ pub fn decompress_chunk_masked(
     Ok(data)
 }
 
+/// Decode one stored chunk of a chunked dataset: [`decompress_chunk_masked`],
+/// then require exactly `chunk_size` bytes (when `chunk_size` is known).
+///
+/// HDF5 stores every chunk at the full chunk size — edge chunks are padded
+/// before they are filtered, and an edge chunk left unfiltered is written
+/// full-size too — so a pipeline that decodes to fewer bytes means a
+/// corrupt chunk. libhdf5 fails such a read (or returns uninitialised
+/// memory); it must never read back as zeros. `coords` (the chunk's offset
+/// in the dataset) is named in the error.
+pub fn decompress_chunk_exact(
+    compressed: &[u8],
+    pipeline: &FilterPipeline,
+    chunk_size: usize,
+    element_size: u32,
+    filter_mask: u32,
+    coords: &[u64],
+) -> Result<Vec<u8>, FormatError> {
+    let data =
+        decompress_chunk_masked(compressed, pipeline, chunk_size, element_size, filter_mask)?;
+    if chunk_size != 0 && data.len() != chunk_size {
+        return Err(FormatError::ChunkedReadError(format!(
+            "chunk at {coords:?} decoded to {} bytes, expected {chunk_size}",
+            data.len()
+        )));
+    }
+    Ok(data)
+}
+
 /// Apply a filter pipeline to compress a chunk.
 /// Filters are applied in FORWARD order for compression.
 pub fn compress_chunk(
@@ -1516,6 +1544,39 @@ fn pcodec_decompress(
 
 #[cfg(test)]
 mod tests {
+
+    /// A chunk whose pipeline decodes to fewer bytes than the chunk holds is
+    /// an error naming the chunk, never a short buffer the reader pads.
+    #[test]
+    fn short_decoded_chunk_is_an_error() {
+        let pipeline = FilterPipeline {
+            version: 2,
+            filters: vec![FilterDescription {
+                filter_id: FILTER_SHUFFLE,
+                name: None,
+                flags: 0,
+                client_data: vec![4],
+            }],
+        };
+        let full = [7u8; 32];
+        assert_eq!(
+            decompress_chunk_exact(&full, &pipeline, 32, 4, 0, &[8]).unwrap(),
+            full
+        );
+        let err = decompress_chunk_exact(&full[..16], &pipeline, 32, 4, 0, &[8, 0]).unwrap_err();
+        let msg = err.to_string();
+        assert!(
+            msg.contains("[8, 0]") && msg.contains("16") && msg.contains("32"),
+            "{msg}"
+        );
+        // Every filter skipped: the stored bytes are the chunk, still checked.
+        assert!(decompress_chunk_exact(&full[..16], &pipeline, 32, 4, 1, &[0]).is_err());
+        // Unknown chunk size: not checked.
+        assert_eq!(
+            decompress_chunk_exact(&full[..16], &pipeline, 0, 4, 0, &[0]).unwrap(),
+            &full[..16]
+        );
+    }
     use super::*;
     use crate::filter_pipeline::FilterDescription;
 

crates/clawhdf5-format/src/filters_blosc.rs

@@ -113,8 +113,15 @@ fn decode_stream(
 }
 
 /// Decode a Blosc-filtered chunk: one Blosc 1 frame.
+///
+/// An HDF5 chunk is never empty, so a frame that decodes to nothing where
+/// the chunk size is known is corrupt (libhdf5's filter fails it too).
 pub(crate) fn blosc_decode(input: &[u8], ctx: &FilterContext<'_>) -> Result<Vec<u8>, FormatError> {
-    blosc_decompress(input, ctx.output_limit())
+    let out = blosc_decompress(input, ctx.output_limit())?;
+    if out.is_empty() && ctx.max_output != 0 {
+        return Err(err("empty frame for a non-empty chunk"));
+    }
+    Ok(out)
 }
 
 /// Decompress a Blosc 1 frame, refusing more than `limit` bytes of output.
@@ -606,6 +613,23 @@ mod tests {
         assert!(blosc_encode(&data, &ctx0).is_err());
     }
 
+    /// A frame that declares no data, for a chunk that has some.
+    #[test]
+    fn empty_frame_for_a_non_empty_chunk_is_an_error() {
+        let mut frame = vec![2u8, 1, 0x20, 4];
+        for v in [0u32, 64, 16] {
+            frame.extend_from_slice(&v.to_le_bytes());
+        }
+        assert_eq!(blosc_decompress(&frame, 64).unwrap(), b"");
+        let f = desc(vec![2, 2, 4, 64, 5, 1, 1]);
+        let ctx = FilterContext {
+            filter: &f,
+            element_size: 4,
+            max_output: 64,
+        };
+        assert!(blosc_decode(&frame, &ctx).is_err());
+    }
+
     /// A frame whose header claims a compressed size smaller than the
     /// header itself, not stored raw: an error, not an arithmetic overflow
     /// (it panicked in debug builds).

crates/clawhdf5-format/src/parallel_read.rs

@@ -10,7 +10,7 @@
 use crate::chunked_read::ChunkInfo;
 use crate::error::FormatError;
 use crate::filter_pipeline::FilterPipeline;
-use crate::filters::decompress_chunk_masked;
+use crate::filters::decompress_chunk_exact;
 use crate::lane_partition::{self, LaneStats, PartitionStats};
 
 /// Threshold: only use parallel decompression when chunk count exceeds this.
@@ -84,12 +84,13 @@ pub fn decompress_chunks_lane_partitioned(
                 }
                 let raw_chunk = &file_data[c_addr..c_addr + size];
 
-                let decompressed = decompress_chunk_masked(
+                let decompressed = decompress_chunk_exact(
                     raw_chunk,
                     pipeline,
                     chunk_total_bytes,
                     element_size,
                     chunk_info.filter_mask,
+                    &chunk_info.offsets,
                 )?;
 
                 stats.chunks_processed += 1;
@@ -160,12 +161,13 @@ pub fn decompress_chunks_parallel(
             }
             let raw_chunk = &file_data[c_addr..c_addr + size];
 
-            let decompressed = decompress_chunk_masked(
+            let decompressed = decompress_chunk_exact(
                 raw_chunk,
                 pipeline,
                 chunk_total_bytes,
                 element_size,
                 chunk_info.filter_mask,
+                &chunk_info.offsets,
             )?;
 
             Ok(DecompressedChunk {
@@ -204,12 +206,13 @@ pub fn decompress_chunks_sequential(
         let raw_chunk = &file_data[c_addr..c_addr + size];
 
         let decompressed = if let Some(pl) = pipeline {
-            decompress_chunk_masked(
+            decompress_chunk_exact(
                 raw_chunk,
                 pl,
                 chunk_total_bytes,
                 element_size,
                 chunk_info.filter_mask,
+                &chunk_info.offsets,
             )?
         } else {
             raw_chunk.to_vec()
@@ -218,3 +221,60 @@ pub fn decompress_chunks_sequential(
     }
     Ok(result)
 }
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use crate::filter_pipeline::{FILTER_SHUFFLE, FilterDescription};
+
+    /// Eight shuffled 32-byte chunks; chunk 5 is stored short when `short`.
+    fn chunks(short: bool) -> (Vec<u8>, Vec<ChunkInfo>) {
+        let mut file = Vec::new();
+        let mut infos = Vec::new();
+        for i in 0..8u64 {
+            let len = if short && i == 5 { 16 } else { 32 };
+            infos.push(ChunkInfo {
+                chunk_size: len as u32,
+                filter_mask: 0,
+                offsets: vec![i * 8],
+                address: file.len() as u64,
+            });
+            file.extend(core::iter::repeat_n(i as u8, len));
+        }
+        (file, infos)
+    }
+
+    /// Every parallel decoder refuses a chunk that decodes short, naming it.
+    #[test]
+    fn short_decoded_chunk_is_an_error() {
+        let pipeline = FilterPipeline {
+            version: 2,
+            filters: vec![FilterDescription {
+                filter_id: FILTER_SHUFFLE,
+                name: None,
+                flags: 0,
+                client_data: vec![4],
+            }],
+        };
+        let (file, good) = chunks(false);
+        assert_eq!(
+            decompress_chunks_parallel(&file, &good, &pipeline, 32, 4).unwrap()[5],
+            [5u8; 32]
+        );
+        let (file, bad) = chunks(true);
+        let errs = [
+            decompress_chunks_lane_partitioned(&file, &bad, &pipeline, 32, 4, 1, Some(3))
+                .map(|_| ())
+                .unwrap_err(),
+            decompress_chunks_parallel(&file, &bad, &pipeline, 32, 4)
+                .map(|_| ())
+                .unwrap_err(),
+            decompress_chunks_sequential(&file, &bad, Some(&pipeline), 32, 4)
+                .map(|_| ())
+                .unwrap_err(),
+        ];
+        for e in errs {
+            assert!(e.to_string().contains("[40]"), "{e}");
+        }
+    }
+}

crates/clawhdf5-format/src/partial_read.rs

@@ -22,7 +22,7 @@ use crate::data_read::extract_selection_from_buffer;
 use crate::dataspace::Dataspace;
 use crate::error::FormatError;
 use crate::filter_pipeline::FilterPipeline;
-use crate::filters::{all_filters_skipped, decompress_chunk_masked};
+use crate::filters::{all_filters_skipped, decompress_chunk_exact};
 use crate::selection::Selection;
 
 /// The smallest axis-aligned box containing every selected element, as
@@ -330,12 +330,13 @@ pub fn read_selection(
                 let decoded;
                 let data: &[u8] = match pipeline {
                     Some(pl) if !all_filters_skipped(pl, chunk.filter_mask) => {
-                        decoded = decompress_chunk_masked(
+                        decoded = decompress_chunk_exact(
                             raw,
                             pl,
                             chunk_bytes,
                             elem_size as u32,
                             chunk.filter_mask,
+                            &chunk.offsets[..rank],
                         )?;
                         &decoded
                     }

crates/clawhdf5/tests/h5py_chunked_read_tests.rs

@@ -343,3 +343,77 @@ print("OK")
     let want: Vec<u8> = line[990..].iter().flat_map(|v| v.to_le_bytes()).collect();
     assert_eq!(tail, want);
 }
+
+// ---------------------------------------------------------------------------
+// Chunks that decode short
+// ---------------------------------------------------------------------------
+
+/// HDF5 stores every chunk at the full chunk size, so a chunk whose filters
+/// decode to fewer bytes is corrupt. libhdf5 returns the rest of such a
+/// chunk uninitialised (or, for filters that check, fails); clawhdf5 padded
+/// it with zeros and returned it as data. Every read path must fail,
+/// naming the chunk, and chunks that decode fully must still read.
+#[test]
+fn h5py_short_decoded_chunk_is_an_error() {
+    skip_if_no_python!();
+    let dir = tempfile::tempdir().unwrap();
+    let path = dir.path().join("short.h5");
+    let p = path.display().to_string();
+    run_python(&format!(
+        r#"
+import h5py, numpy as np, zlib
+with h5py.File("{p}", "w") as f:
+    # 1-D, 4 gzip chunks of 8 i4; chunk 2 (offset 16) inflates to 16 bytes.
+    ds = f.create_dataset("line", shape=(32,), chunks=(8,), dtype="<i4", compression="gzip")
+    ds[...] = np.arange(32, dtype="<i4") + 1
+    ds.id.write_direct_chunk((16,), zlib.compress(np.arange(4, dtype="<i4").tobytes()))
+    # 2-D with a partial edge chunk; the short chunk is the edge one (8, 4).
+    g = f.create_dataset("grid", shape=(10, 7), chunks=(4, 4), dtype="<f8", compression="gzip")
+    g[...] = np.arange(70, dtype="<f8").reshape(10, 7) + 1
+    g.id.write_direct_chunk((8, 4), zlib.compress(np.ones(3, dtype="<f8").tobytes()))
+    # 40 chunks (enough for the parallel decoder), one short, shuffle + gzip.
+    m = f.create_dataset("many", shape=(320,), chunks=(8,), dtype="<i4",
+                         compression="gzip", shuffle=True)
+    m[...] = np.arange(320, dtype="<i4") + 1
+    m.id.write_direct_chunk((200,), zlib.compress(bytes(31)))
+"#
+    ));
+
+    let hyperslab = |start: u64, count: u64| Selection::Hyperslab {
+        start: vec![start],
+        stride: vec![1],
+        count: vec![1],
+        block: vec![count],
+    };
+    let file = File::open(&path).unwrap();
+    for (name, coords) in [("line", "[16"), ("grid", "[8, 4"), ("many", "[200")] {
+        let ds = file.dataset(name).unwrap();
+        let full = || {
+            if name == "grid" {
+                ds.read_f64().map(|_| ())
+            } else {
+                ds.read_i32().map(|_| ())
+            }
+        };
+        let err = full().expect_err(name).to_string();
+        assert!(err.contains(coords), "{name}: {err}");
+        // Cached reads decode the same way; a second read fails too.
+        assert!(full().is_err(), "{name}");
+        assert!(ds.read_selection(&Selection::All).is_err(), "{name}");
+    }
+    let line = file.dataset("line").unwrap();
+    assert!(line.read_selection(&hyperslab(14, 4)).is_err());
+    // A selection that avoids the short chunk still reads.
+    let want: Vec<u8> = (1..=16i32).flat_map(i32::to_le_bytes).collect();
+    assert_eq!(line.read_selection(&hyperslab(0, 16)).unwrap(), want);
+    let many = file.dataset("many").unwrap();
+    assert!(many.read_selection(&hyperslab(190, 20)).is_err());
+
+    // The memory-mapped and lazy readers.
+    let mm = clawhdf5::MmapFile::open(&path).unwrap();
+    assert!(mm.dataset("line").unwrap().read_i32().is_err());
+    assert!(mm.dataset("many").unwrap().read_i32().is_err());
+    let lazy = clawhdf5::LazyFile::open_mmap(&path).unwrap();
+    assert!(lazy.dataset("line").unwrap().read_i32().is_err());
+    assert!(lazy.dataset("grid").unwrap().read_f64().is_err());
+}

crates/clawhdf5/tests/plugin_filters_interop.rs

@@ -415,3 +415,55 @@ with h5py.File(sys.argv[1], 'w') as f:
         );
     }
 }
+
+/// A corrupt chunk must never read as zeros: a Blosc frame that declares no
+/// data (libhdf5's filter fails it), and Blosc, LZF and bzip2 chunks that
+/// decode short (libhdf5 returns the rest of the chunk uninitialised).
+#[test]
+fn short_decoding_chunks_are_errors() {
+    if !have_python("h5py, hdf5plugin") {
+        return;
+    }
+    let dir = tempfile::tempdir().unwrap();
+    let path = dir.path().join("short.h5");
+    run_python(
+        r#"
+import sys, struct, bz2
+import numpy as np, h5py, hdf5plugin
+def lzf(b):
+    # One literal run per 32 bytes: a valid LZF stream.
+    return b"".join(bytes([len(b[i:i+32]) - 1]) + b[i:i+32] for i in range(0, len(b), 32))
+with h5py.File(sys.argv[1], 'w') as f:
+    kw = dict(shape=(16,), dtype='<i4', chunks=(16,))
+    empty = f.create_dataset('blosc_empty', **kw, **hdf5plugin.Blosc(cname='lz4'))
+    empty.id.write_direct_chunk((0,), bytes([2, 1, 0x20, 4]) + struct.pack('<III', 0, 64, 16))
+    short = f.create_dataset('blosc_short', **kw, **hdf5plugin.Blosc(cname='lz4'))
+    short.id.write_direct_chunk((0,), bytes([2, 1, 0x22, 4]) + struct.pack('<III', 32, 32, 48) + bytes(32))
+    try:
+        f['blosc_empty'][...]
+    except OSError:
+        pass
+    else:
+        raise SystemExit('blosc_empty: libhdf5 read it')
+    lz = f.create_dataset('lzf_short', **kw, compression='lzf')
+    lz.id.write_direct_chunk((0,), lzf(np.arange(8, dtype='<i4').tobytes()))
+    bz = f.create_dataset('bzip2_short', **kw, **hdf5plugin.BZip2())
+    bz.id.write_direct_chunk((0,), bz2.compress(np.arange(8, dtype='<i4').tobytes()))
+    ok = f.create_dataset('lzf_ok', **kw, compression='lzf')
+    ok.id.write_direct_chunk((0,), lzf(np.arange(16, dtype='<i4').tobytes()))
+"#,
+        &[path.to_str().unwrap()],
+    );
+    let file = File::open(&path).unwrap();
+    for name in ["blosc_empty", "blosc_short", "lzf_short", "bzip2_short"] {
+        let ds = file.dataset(name).unwrap();
+        assert!(
+            ds.read_i32().is_err(),
+            "{name}: {:?}",
+            ds.read_i32().unwrap()
+        );
+        assert!(ds.read_selection(&Selection::All).is_err(), "{name}");
+    }
+    let want: Vec<i32> = (0..16).collect();
+    assert_eq!(file.dataset("lzf_ok").unwrap().read_i32().unwrap(), want);
+}