fix(format): write and read the registered HDF5 LZ4 filter format

Filter 32004 chunks were framed as a 4-byte little-endian size plus one
LZ4 block. That is not the registered HDF5 LZ4 format (H5Zlz4.c: 8-byte
big-endian total size, 4-byte big-endian block size, then per block a
4-byte big-endian compressed length and the block, stored raw when the
length equals the block size), so libhdf5 + hdf5plugin could not read
our LZ4 datasets and we could not read theirs (h5ex_d_lz4.h5:
"lz4: 0 is not a valid match offset").

Write the registered format (cd_values[0] is honoured as the block size,
default 1 GiB like the plugin) and read it, multi-block and raw blocks
included. Chunks in the old framing stay readable: an HDF5 chunk is under
4 GiB, so a registered chunk always starts with four zero bytes and is at
least 12 bytes long, while an old one starts with four zero bytes only
when empty (5 bytes).

Tests: lz4_reads_registered_hdf5_format (chunk of the HDF Group's
h5ex_d_lz4.h5, block size 3), lz4_writes_registered_hdf5_format,
lz4_reads_legacy_clawhdf5_format, and hdf5plugin_reads_our_lz4 (ignored
interop test; failed before with "filter returned failure during read").

Co-Authored-By: Claude Opus 5.5 (1M context) <[email protected]>

crates/clawhdf5-format/src/filters.rs

@@ -69,7 +69,7 @@ pub fn compress_chunk(
                 let level = filter.client_data.first().copied().unwrap_or(6);
                 deflate_compress(&result, level)?
             }
-            FILTER_LZ4 => lz4_compress(&result)?,
+            FILTER_LZ4 => lz4_compress(&result, &filter.client_data)?,
             FILTER_ZSTD => {
                 let level = filter.client_data.first().copied().unwrap_or(3);
                 zstd_compress(&result, level)?
@@ -813,12 +813,32 @@ fn deflate_compress(_data: &[u8], _level: u32) -> Result<Vec<u8>, FormatError> {
     Err(FormatError::UnsupportedFilter(FILTER_DEFLATE))
 }
 
-/// Decompress LZ4 data. Format: 4 bytes LE original size + LZ4 block data.
+/// Default LZ4 block size of the registered HDF5 LZ4 filter (`H5Zlz4.c`,
+/// `DEFAULT_BLOCK_SIZE`): 1 GiB, so an HDF5 chunk is normally one block.
+#[cfg(feature = "lz4")]
+const LZ4_DEFAULT_BLOCK_SIZE: usize = 1 << 30;
+
+/// Decompress an LZ4 (filter 32004) chunk.
 ///
-/// The 4-byte "original size" header is part of the attacker-controlled
+/// Two framings are read:
-/// compressed payload itself, so it is bounded against `expected_bytes` (the
+///
-/// pipeline's declared chunk size) before being used to size the output
+/// * The registered HDF5 LZ4 filter format (`H5Zlz4.c`, what libhdf5 +
-/// allocation — otherwise a crafted 4-byte value can request up to ~4 GiB.
+///   hdf5plugin write, and what clawhdf5 writes after 2.7.0): an 8-byte
+///   big-endian total decompressed size, a 4-byte big-endian block size, then
+///   per block a 4-byte big-endian compressed length followed by the block. A
+///   block whose compressed length equals its decompressed length is stored
+///   raw.
+/// * The legacy clawhdf5 framing (up to 2.7.0): a 4-byte little-endian size
+///   followed by one raw LZ4 block. libhdf5 cannot read it.
+///
+/// They are told apart unambiguously: an HDF5 chunk is smaller than 4 GiB, so
+/// the registered format's big-endian `u64` size always starts with four zero
+/// bytes and the whole chunk is at least 12 bytes; a legacy chunk starts with
+/// four zero bytes only when it is empty, and is then 5 bytes long.
+///
+/// Every size read from the payload is bounded against `expected_bytes` (the
+/// pipeline's declared chunk size) before it sizes an allocation, so a crafted
+/// header cannot request gigabytes.
 #[cfg(feature = "lz4")]
 fn lz4_decompress(data: &[u8], expected_bytes: usize) -> Result<Vec<u8>, FormatError> {
     if data.len() < 4 {
@@ -826,38 +846,112 @@ fn lz4_decompress(data: &[u8], expected_bytes: usize) -> Result<Vec<u8>, FormatE
             "lz4: data too short".into(),
         ));
     }
+    let check_size = |size: usize| -> Result<(), FormatError> {
+        if expected_bytes != 0 && size > expected_bytes {
+            return Err(FormatError::DecompressionError(
+                "lz4: declared size exceeds chunk size".into(),
+            ));
+        }
+        if size > MAX_DECOMPRESS_SIZE {
+            return Err(FormatError::DecompressionError(
+                "lz4: declared size exceeds limit".into(),
+            ));
+        }
+        Ok(())
+    };
+    if data.len() >= 12 && data[..4] == [0, 0, 0, 0] {
+        return lz4_decompress_hdf5(data, check_size);
+    }
+    // Legacy clawhdf5 framing: 4-byte LE size + one LZ4 block.
     let orig_size = u32::from_le_bytes([data[0], data[1], data[2], data[3]]) as usize;
-    if expected_bytes != 0 && orig_size > expected_bytes {
+    check_size(orig_size)?;
-        return Err(FormatError::DecompressionError(
-            "lz4: declared size exceeds chunk size".into(),
-        ));
-    }
-    if orig_size > MAX_DECOMPRESS_SIZE {
-        return Err(FormatError::DecompressionError(
-            "lz4: declared size exceeds limit".into(),
-        ));
-    }
     lz4_flex::block::decompress(&data[4..], orig_size)
         .map_err(|e| FormatError::DecompressionError(format!("lz4: {e}")))
 }
 
+/// Decode the registered HDF5 LZ4 framing (see [`lz4_decompress`]).
+#[cfg(feature = "lz4")]
+fn lz4_decompress_hdf5(
+    data: &[u8],
+    check_size: impl Fn(usize) -> Result<(), FormatError>,
+) -> Result<Vec<u8>, FormatError> {
+    let err = |m: &str| FormatError::DecompressionError(format!("lz4: {m}"));
+    let be32 = |b: &[u8]| u32::from_be_bytes([b[0], b[1], b[2], b[3]]) as usize;
+    // The first four bytes are zero (checked by the caller), so the size is
+    // the low 32 bits of the big-endian u64.
+    let orig_size = be32(&data[4..8]);
+    check_size(orig_size)?;
+    let block_size = be32(&data[8..12]).min(orig_size);
+    if block_size == 0 && orig_size != 0 {
+        return Err(err("zero block size"));
+    }
+    let mut out = vec![0u8; orig_size];
+    let mut pos = 12usize;
+    let mut done = 0usize;
+    while done < orig_size {
+        let this_block = block_size.min(orig_size - done);
+        let comp_len = be32(
+            data.get(pos..pos + 4)
+                .ok_or_else(|| err("truncated block header"))?,
+        );
+        pos += 4;
+        let block = data
+            .get(pos..pos.saturating_add(comp_len))
+            .ok_or_else(|| err("truncated block"))?;
+        let dst = &mut out[done..done + this_block];
+        if comp_len == this_block {
+            dst.copy_from_slice(block);
+        } else {
+            let n = lz4_flex::block::decompress_into(block, dst)
+                .map_err(|e| FormatError::DecompressionError(format!("lz4: {e}")))?;
+            if n != this_block {
+                return Err(err("block decompressed to the wrong size"));
+            }
+        }
+        pos += comp_len;
+        done += this_block;
+    }
+    Ok(out)
+}
+
 #[cfg(not(feature = "lz4"))]
 fn lz4_decompress(_data: &[u8], _expected_bytes: usize) -> Result<Vec<u8>, FormatError> {
     Err(FormatError::UnsupportedFilter(FILTER_LZ4))
 }
 
-/// Compress data with LZ4 block format. Format: 4 bytes LE original size + LZ4 block data.
+/// Compress data in the registered HDF5 LZ4 filter format (see
+/// [`lz4_decompress`]), so libhdf5 with the LZ4 plugin (e.g. hdf5plugin) can
+/// read it. `cd[0]`, when present and non-zero, is the block size in bytes,
+/// as in `H5Zlz4.c`; otherwise the 1 GiB default applies.
 #[cfg(feature = "lz4")]
-fn lz4_compress(data: &[u8]) -> Result<Vec<u8>, FormatError> {
+fn lz4_compress(data: &[u8], cd: &[u32]) -> Result<Vec<u8>, FormatError> {
-    let compressed = lz4_flex::block::compress(data);
+    let block_size = match cd.first() {
-    let mut result = Vec::with_capacity(4 + compressed.len());
+        Some(&b) if b != 0 => b as usize,
-    result.extend_from_slice(&(data.len() as u32).to_le_bytes());
+        _ => LZ4_DEFAULT_BLOCK_SIZE,
-    result.extend_from_slice(&compressed);
+    }
+    .min(data.len());
+    let mut result = Vec::with_capacity(16 + data.len() / 2);
+    result.extend_from_slice(&(data.len() as u64).to_be_bytes());
+    result.extend_from_slice(&(block_size as u32).to_be_bytes());
+    if block_size == 0 {
+        return Ok(result);
+    }
+    for block in data.chunks(block_size) {
+        let compressed = lz4_flex::block::compress(block);
+        if compressed.len() >= block.len() {
+            // Incompressible: stored raw, marked by length == block length.
+            result.extend_from_slice(&(block.len() as u32).to_be_bytes());
+            result.extend_from_slice(block);
+        } else {
+            result.extend_from_slice(&(compressed.len() as u32).to_be_bytes());
+            result.extend_from_slice(&compressed);
+        }
+    }
     Ok(result)
 }
 
 #[cfg(not(feature = "lz4"))]
-fn lz4_compress(_data: &[u8]) -> Result<Vec<u8>, FormatError> {
+fn lz4_compress(_data: &[u8], _cd: &[u32]) -> Result<Vec<u8>, FormatError> {
     Err(FormatError::UnsupportedFilter(FILTER_LZ4))
 }
 
@@ -1480,11 +1574,92 @@ mod tests {
 
     // --- LZ4 tests ---
 
+    /// Chunk (0,0) of `/DS1` in the HDF Group's `h5ex_d_lz4.h5` example,
+    /// written by libhdf5's registered LZ4 plugin with a 3-byte block size
+    /// (so it has many blocks, some stored raw). Byte range from h5py's
+    /// `get_chunk_info`; values are `i*j - j` (i32 LE), as h5py reads them.
+    #[test]
+    #[cfg(feature = "lz4")]
+    fn lz4_reads_registered_hdf5_format() {
+        let file: &[u8] = include_bytes!("../tests/fixtures/filters/h5ex_d_lz4.h5");
+        let chunk = &file[4016..4016 + 312];
+        let pipeline = FilterPipeline {
+            version: 2,
+            filters: vec![FilterDescription {
+                filter_id: FILTER_LZ4,
+                name: None,
+                flags: 1,
+                client_data: vec![3],
+            }],
+        };
+        let out = decompress_chunk(chunk, &pipeline, 4 * 8 * 4, 4).unwrap();
+        let expected: Vec<u8> = (0..4i32)
+            .flat_map(|i| (0..8i32).map(move |j| i * j - j))
+            .flat_map(i32::to_le_bytes)
+            .collect();
+        assert_eq!(out, expected);
+    }
+
+    #[test]
+    #[cfg(feature = "lz4")]
+    fn lz4_writes_registered_hdf5_format() {
+        // Compressible data, one block: 8-byte BE size, 4-byte BE block size,
+        // 4-byte BE compressed length, block.
+        let data = vec![7u8; 1000];
+        let c = lz4_compress(&data, &[]).unwrap();
+        assert_eq!(&c[0..8], &1000u64.to_be_bytes());
+        assert_eq!(&c[8..12], &1000u32.to_be_bytes());
+        let len = u32::from_be_bytes(c[12..16].try_into().unwrap()) as usize;
+        assert_eq!(c.len(), 16 + len);
+        assert!(len < 1000);
+        assert_eq!(lz4_decompress(&c, 1000).unwrap(), data);
+
+        // Several blocks, incompressible ones stored raw (length == block).
+        let data: Vec<u8> = (0..10u8).collect();
+        let c = lz4_compress(&data, &[3]).unwrap();
+        assert_eq!(&c[8..12], &3u32.to_be_bytes());
+        assert_eq!(&c[12..16], &3u32.to_be_bytes());
+        assert_eq!(&c[16..19], &[0, 1, 2]);
+        assert_eq!(c.len(), 12 + 3 * (4 + 3) + (4 + 1));
+        assert_eq!(lz4_decompress(&c, 10).unwrap(), data);
+
+        let c = lz4_compress(&[], &[]).unwrap();
+        assert_eq!(lz4_decompress(&c, 0).unwrap(), Vec::<u8>::new());
+    }
+
+    /// Chunks written by clawhdf5 up to 2.7.0 (4-byte LE size + one LZ4
+    /// block) must stay readable.
+    #[test]
+    #[cfg(feature = "lz4")]
+    fn lz4_reads_legacy_clawhdf5_format() {
+        for data in [vec![], vec![5u8; 300], (0..=255u8).collect::<Vec<u8>>()] {
+            let mut legacy = (data.len() as u32).to_le_bytes().to_vec();
+            legacy.extend_from_slice(&lz4_flex::block::compress(&data));
+            assert_eq!(lz4_decompress(&legacy, data.len()).unwrap(), data);
+        }
+    }
+
+    #[test]
+    #[cfg(feature = "lz4")]
+    fn lz4_registered_format_rejects_hostile_sizes() {
+        // Declared total larger than the chunk.
+        let mut c = 1000u64.to_be_bytes().to_vec();
+        c.extend_from_slice(&1000u32.to_be_bytes());
+        c.extend_from_slice(&[0u8; 8]);
+        assert!(lz4_decompress(&c, 64).is_err());
+        // Truncated block.
+        let mut c = 16u64.to_be_bytes().to_vec();
+        c.extend_from_slice(&16u32.to_be_bytes());
+        c.extend_from_slice(&16u32.to_be_bytes());
+        c.extend_from_slice(&[1u8; 4]);
+        assert!(lz4_decompress(&c, 16).is_err());
+    }
+
     #[test]
     #[cfg(feature = "lz4")]
     fn lz4_compress_decompress_roundtrip() {
         let data: Vec<u8> = (0..256).map(|i| (i % 256) as u8).collect();
-        let compressed = lz4_compress(&data).unwrap();
+        let compressed = lz4_compress(&data, &[]).unwrap();
         let decompressed = lz4_decompress(&compressed, data.len()).unwrap();
         assert_eq!(decompressed, data);
     }

crates/clawhdf5-format/tests/fixtures/filters/README.md

@@ -0,0 +1,10 @@
+# Filter conformance fixtures
+
+Files written by libhdf5 (and its registered filter plugins), used by the
+filter regression tests in `src/filters.rs` to compare our decoders against
+the values h5py/libhdf5 read from the same bytes. Chunk byte ranges quoted in
+the tests come from h5py's `DatasetID.get_chunk_info`.
+
+| File | Origin | Licence |
+|------|--------|---------|
+| `h5ex_d_lz4.h5` | HDF Group `HDF5Examples/C/H5FLT/tfiles/h5ex_d_lz4.h5` (hdf5 repository) | HDF5 licence (BSD-3-Clause style) |

crates/clawhdf5-format/tests/writer_h5py_tests.rs

@@ -940,3 +940,48 @@ fn provenance_verify_written_file() {
             .unwrap();
     assert_eq!(result, clawhdf5_format::provenance::VerifyResult::Ok);
 }
+
+// ---- hdf5plugin interop: registered third-party compression filters ----
+
+/// Write `data` (f64, 1-D, chunked) with `configure` applied, then read it
+/// back with h5py + hdf5plugin (libhdf5's registered filter plugins) and
+/// return the values it decodes.
+#[cfg(any(feature = "lz4", feature = "zstd"))]
+fn hdf5plugin_roundtrip(
+    tag: &str,
+    data: &[f64],
+    configure: impl FnOnce(&mut clawhdf5_format::type_builders::DatasetBuilder),
+) -> Vec<f64> {
+    let mut fw = FileWriter::new();
+    let ds = fw.create_dataset("data");
+    ds.with_f64_data(data)
+        .with_shape(&[data.len() as u64])
+        .with_chunks(&[250]);
+    configure(ds);
+    let bytes = fw.finish().unwrap();
+    let path = std::env::temp_dir().join(format!("clawhdf5_hdf5plugin_{tag}.h5"));
+    std::fs::write(&path, &bytes).unwrap();
+    let script = format!(
+        "import h5py,hdf5plugin,json; f=h5py.File('{}','r'); print(json.dumps(f['data'][:].tolist()))",
+        path.display()
+    );
+    let stdout = h5py_read(&path, &script);
+    serde_json::from_str(&stdout).unwrap()
+}
+
+/// libhdf5's LZ4 plugin must decode what we write (it could not while we
+/// wrote a private 4-byte-LE-size framing).
+#[cfg(feature = "lz4")]
+#[test]
+#[ignore = "requires Python h5py + hdf5plugin"]
+fn hdf5plugin_reads_our_lz4() {
+    let data: Vec<f64> = (0..1000).map(|i| (i % 37) as f64 * 0.5).collect();
+    let got = hdf5plugin_roundtrip("lz4", &data, |ds| {
+        ds.with_lz4();
+    });
+    assert_eq!(got, data);
+    let got = hdf5plugin_roundtrip("lz4_noshuffle", &data, |ds| {
+        ds.with_lz4().without_shuffle();
+    });
+    assert_eq!(got, data);
+}