Write chunks of 4 GiB or more; FileEditor refuses to rewrite them

The writer gives a chunk of more than u32::MAX bytes layout message
version 5 and, filtered, index elements whose stored size takes the file's
size of lengths, as libhdf5 2.x does (the Fixed and Extensible Array
structures match libhdf5's byte for byte). Chunk dimensions of 2^32 or more
and filters that cannot take such a chunk (LZF, bitshuffle, bzip2, Blosc,
pcodec) are refused instead of truncated. Chunks are extracted row by row
and one at a time; deflate no longer cuts input at 4 GiB - 1 bytes, nor
holds the worst-case bound of a large chunk; an LZ4 chunk of 4 GiB or more
is read as the registered framing.

FileEditor refuses writing values into, or pruning/allocating, chunks of
4 GiB or more before anything is written; growing the extent and
attributes still work.

Co-Authored-By: Claude Opus 5.5 (1M context) <[email protected]>

crates/clawhdf5-filters/src/fast_deflate.rs

@@ -327,24 +327,53 @@ fn inflate_bounded(data: &[u8], size_hint: usize, limit: usize) -> Result<Vec<u8
     }
 }
 
+/// Largest compression output reserved at its worst-case size up front.
+const DEFLATE_EXACT_BOUND: usize = 64 << 20;
+
 /// Compress data using flate2 (zlib-ng, zlib-rs or miniz_oxide; see module docs).
 pub(crate) fn flate2_compress(data: &[u8], level: u32) -> Result<Vec<u8>, String> {
     use flate2::{Compress, Compression, FlushCompress, Status};
 
     // zlib's compressBound, plus the zlib header and trailer.
     let bound = data.len() + (data.len() >> 12) + (data.len() >> 14) + (data.len() >> 25) + 13 + 6;
+    // flate2's Rust backends (zlib-rs, miniz_oxide) zero the whole spare
+    // capacity on each call, so a large input's worst-case bound would be
+    // memory held for nothing (4 GiB for a 4 GiB chunk that deflates to a
+    // few MiB): past 64 MiB the output starts at 1/16 of the bound and
+    // doubles as needed.
+    let first = if bound <= DEFLATE_EXACT_BOUND {
+        bound
+    } else {
+        bound / 16
+    };
     let mut out = Vec::new();
-    out.try_reserve_exact(bound)
+    out.try_reserve_exact(first)
         .map_err(|e| format!("deflate: cannot allocate output: {e}"))?;
 
     let mut deflater = Compress::new(Compression::new(level), true);
     loop {
         let (in_before, out_before) = (deflater.total_in(), deflater.total_out());
+        let rest = &data[in_before as usize..];
+        // zlib takes at most u32::MAX input bytes per call, and `Finish`
+        // ends the stream after the bytes it took: input of 4 GiB or more
+        // was cut at 4 GiB - 1. Finish only once the rest fits one call.
+        let flush = if rest.len() > u32::MAX as usize {
+            FlushCompress::None
+        } else {
+            FlushCompress::Finish
+        };
         let status = deflater
-            .compress_vec(&data[in_before as usize..], &mut out, FlushCompress::Finish)
+            .compress_vec(rest, &mut out, flush)
             .map_err(|e| format!("deflate: {e}"))?;
         match status {
-            Status::StreamEnd => return Ok(out),
+            Status::StreamEnd => {
+                if out.capacity() - out.len() > DEFLATE_EXACT_BOUND {
+                    out.shrink_to_fit();
+                }
+                return Ok(out);
+            }
+            // Out of room (the bound makes it unreachable below
+            // `DEFLATE_EXACT_BOUND`): grow rather than fail.
             Status::Ok | Status::BufError if out.len() == out.capacity() => out
                 .try_reserve(out.capacity().max(4096))
                 .map_err(|e| format!("deflate: cannot allocate output: {e}"))?,