fix(format): record the content size in zstd frames
Filter 32015 chunks were written with the streaming encoder
(zstd::encode_all), whose frames carry no content size. The registered
HDF5 Zstandard filter (H5Zzstd.c, libhdf5 + hdf5plugin) sizes its output
from ZSTD_getFrameContentSize and fails on such frames, so h5py could not
read our zstd datasets ("filter returned failure during read"). Compress
with the one-shot API, which records the size.
Tests: zstd_frames_record_content_size (content size was None before),
hdf5plugin_reads_our_zstd (ignored interop test; failed before).
Co-Authored-By: Claude Opus 5.5 (1M context) <[email protected]>
This commit is contained in:
@@ -985,3 +985,16 @@ fn hdf5plugin_reads_our_lz4() {
|
||||
});
|
||||
assert_eq!(got, data);
|
||||
}
|
||||
|
||||
/// libhdf5's Zstandard plugin must decode what we write (it could not while
|
||||
/// our frames lacked the content size).
|
||||
#[cfg(feature = "zstd")]
|
||||
#[test]
|
||||
#[ignore = "requires Python h5py + hdf5plugin"]
|
||||
fn hdf5plugin_reads_our_zstd() {
|
||||
let data: Vec<f64> = (0..1000).map(|i| (i % 37) as f64 * 0.5).collect();
|
||||
let got = hdf5plugin_roundtrip("zstd", &data, |ds| {
|
||||
ds.with_zstd(3);
|
||||
});
|
||||
assert_eq!(got, data);
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user