From 183d96ee265008b686416fb6ef0f1f78381098de Mon Sep 17 00:00:00 2001
From: osobh <om.sobh@gmail.com>
Date: Fri, 25 Sep 2026 21:04:29 -0500
Subject: [PATCH] fix(format): record the content size in zstd frames

Filter 32015 chunks were written with the streaming encoder
(zstd::encode_all), whose frames carry no content size. The registered
HDF5 Zstandard filter (H5Zzstd.c, libhdf5 + hdf5plugin) sizes its output
from ZSTD_getFrameContentSize and fails on such frames, so h5py could not
read our zstd datasets ("filter returned failure during read"). Compress
with the one-shot API, which records the size.

Tests: zstd_frames_record_content_size (content size was None before),
hdf5plugin_reads_our_zstd (ignored interop test; failed before).

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
---
 crates/clawhdf5-format/src/filters.rs         | 25 +++++++++++++++++--
 .../tests/writer_h5py_tests.rs                | 13 ++++++++++
 2 files changed, 36 insertions(+), 2 deletions(-)

diff --git a/crates/clawhdf5-format/src/filters.rs b/crates/clawhdf5-format/src/filters.rs
index d12302b..a28d4be 100644
--- a/crates/clawhdf5-format/src/filters.rs
+++ b/crates/clawhdf5-format/src/filters.rs
@@ -988,10 +988,14 @@ fn zstd_decompress(_data: &[u8], _expected_bytes: usize) -> Result<Vec<u8>, Form
     Err(FormatError::UnsupportedFilter(FILTER_ZSTD))
 }
 
-/// Compress data with zstd.
+/// Compress data with zstd as one frame whose header records the content
+/// size. The registered HDF5 Zstandard filter (`H5Zzstd.c`, used by
+/// libhdf5 + hdf5plugin) sizes its output buffer from
+/// `ZSTD_getFrameContentSize` and fails on a frame without it, which is what
+/// the streaming encoder (`zstd::encode_all`) produced.
 #[cfg(feature = "zstd")]
 fn zstd_compress(data: &[u8], level: u32) -> Result<Vec<u8>, FormatError> {
-    zstd::encode_all(data, level as i32)
+    zstd::bulk::compress(data, level as i32)
         .map_err(|e| FormatError::CompressionError(format!("zstd: {e}")))
 }
 
@@ -1710,6 +1714,23 @@ mod tests {
 
     // --- Zstd tests ---
 
+    /// libhdf5's zstd plugin needs the frame content size to size its
+    /// output; frames without it fail to decode there.
+    #[test]
+    #[cfg(feature = "zstd")]
+    fn zstd_frames_record_content_size() {
+        for n in [0usize, 1, 200, 100_000] {
+            let data: Vec<u8> = (0..n).map(|i| (i % 7) as u8).collect();
+            let c = zstd_compress(&data, 3).unwrap();
+            assert_eq!(
+                zstd::zstd_safe::get_frame_content_size(&c).unwrap(),
+                Some(n as u64),
+                "{n} bytes"
+            );
+            assert_eq!(zstd_decompress(&c, n).unwrap(), data);
+        }
+    }
+
     #[test]
     #[cfg(feature = "zstd")]
     fn zstd_compress_decompress_roundtrip() {
diff --git a/crates/clawhdf5-format/tests/writer_h5py_tests.rs b/crates/clawhdf5-format/tests/writer_h5py_tests.rs
index 44c42f6..cfbaaa8 100644
--- a/crates/clawhdf5-format/tests/writer_h5py_tests.rs
+++ b/crates/clawhdf5-format/tests/writer_h5py_tests.rs
@@ -985,3 +985,16 @@ fn hdf5plugin_reads_our_lz4() {
     });
     assert_eq!(got, data);
 }
+
+/// libhdf5's Zstandard plugin must decode what we write (it could not while
+/// our frames lacked the content size).
+#[cfg(feature = "zstd")]
+#[test]
+#[ignore = "requires Python h5py + hdf5plugin"]
+fn hdf5plugin_reads_our_zstd() {
+    let data: Vec<f64> = (0..1000).map(|i| (i % 37) as f64 * 0.5).collect();
+    let got = hdf5plugin_roundtrip("zstd", &data, |ds| {
+        ds.with_zstd(3);
+    });
+    assert_eq!(got, data);
+}