feat(format): LZF filter (32000), read and write, pure Rust

h5py's built-in compression="lzf" failed with UnsupportedFilter(32000). The
new `lzf` feature (no dependencies, on by default in clawhdf5-format and
the facade) decodes the raw liblzf stream h5py's filter stores, bounded by
the chunk size, and encodes it: DatasetBuilder::with_lzf() (or
with_plugin_filter(PluginFilter::Lzf)) writes the filter with h5py's
cd_values (filter version 4, liblzf 0x0105, chunk size in bytes), flagged
optional as h5py does. ChunkOptions gains a `plugin` field for the plugin
filters; build_pipeline_for_chunk passes the chunk size to filters that
record it.

tests/plugin_filters_interop.rs: h5py writes LZF (alone, with shuffle, with
shuffle+fletcher32) over 12 dtype/shape/chunk/data cases with partial edge
chunks and incompressible data, and every dataset reads byte for byte equal
to its unfiltered twin; our LZF output (1-D and 2-D, edge chunks, with and
without shuffle) reads back in h5py. Both fail with the decoder removed.

Co-Authored-By: Claude Opus 5.5 (1M context) <[email protected]>

crates/clawhdf5-format/src/chunked_write.rs

@@ -12,7 +12,7 @@ use crate::chunk_grid::ChunkGrid;
 use crate::ea_writer;
 use crate::error::FormatError;
 use crate::filter_pipeline::{
-    FILTER_DEFLATE, FILTER_FLETCHER32, FILTER_LZ4, FILTER_PCODEC, FILTER_PCODEC_NAME,
+    FILTER_DEFLATE, FILTER_FLETCHER32, FILTER_LZ4, FILTER_LZF, FILTER_PCODEC, FILTER_PCODEC_NAME,
     FILTER_SHUFFLE, FILTER_ZSTD, FilterDescription, FilterPipeline,
 };
 use crate::filters::compress_chunk;
@@ -48,6 +48,45 @@ pub struct ChunkOptions {
     /// Pcodec lossless numerical compression. Private, unregistered filter
     /// ID [`FILTER_PCODEC`] (480): only clawhdf5 can read it.
     pub pcodec: bool,
+    /// A plugin compression filter (LZF, ...). Takes priority over the
+    /// codecs above. Each needs its cargo feature to be written.
+    pub plugin: Option<PluginFilter>,
+}
+
+/// A compression filter from the common HDF5 plugin set, written in the
+/// format the libhdf5 plugin (h5py / hdf5plugin) reads.
+#[derive(Debug, Clone, PartialEq, Eq)]
+#[non_exhaustive]
+pub enum PluginFilter {
+    /// LZF (filter 32000), h5py's built-in `compression="lzf"`. Needs the
+    /// `lzf` feature.
+    Lzf,
+}
+
+impl PluginFilter {
+    /// Whether the filter reorders bytes itself, so the automatic shuffle
+    /// pre-filter would only get in its way.
+    fn shuffles_itself(&self) -> bool {
+        match self {
+            PluginFilter::Lzf => false,
+        }
+    }
+
+    /// The pipeline entry for this filter. `chunk_bytes` is one chunk's
+    /// uncompressed size (0 if unknown).
+    fn description(&self, _element_size: u32, chunk_bytes: u32) -> FilterDescription {
+        match self {
+            // h5py's lzf_set_local: filter version, liblzf version, chunk
+            // size in bytes. Optional, as h5py flags it: a chunk the filter
+            // cannot shrink may then be stored unfiltered.
+            PluginFilter::Lzf => FilterDescription {
+                filter_id: FILTER_LZF,
+                name: Some("lzf".into()),
+                flags: 1,
+                client_data: vec![4, 0x0105, chunk_bytes],
+            },
+        }
+    }
 }
 
 /// Largest chunk the automatic choice produces, in bytes.
@@ -92,14 +131,33 @@ impl ChunkOptions {
             || self.lz4
             || self.zstd_level.is_some()
             || self.pcodec
+            || self.plugin.is_some()
     }
 
     /// Build a FilterPipeline from the options.
     pub fn build_pipeline(&self, element_size: u32) -> Option<FilterPipeline> {
+        self.build_pipeline_for_chunk(element_size, 0)
+    }
+
+    /// Build a FilterPipeline for chunks of `chunk_bytes` uncompressed bytes
+    /// (0 if unknown). Some plugin filters record the chunk size in their
+    /// client data.
+    pub fn build_pipeline_for_chunk(
+        &self,
+        element_size: u32,
+        chunk_bytes: u32,
+    ) -> Option<FilterPipeline> {
         let mut filters = Vec::new();
 
-        let has_compression =
-            self.deflate_level.is_some() || self.zstd_level.is_some() || self.lz4 || self.pcodec;
+        let plugin_shuffles = self
+            .plugin
+            .as_ref()
+            .is_some_and(PluginFilter::shuffles_itself);
+        let has_compression = self.deflate_level.is_some()
+            || self.zstd_level.is_some()
+            || self.lz4
+            || self.pcodec
+            || (self.plugin.is_some() && !plugin_shuffles);
 
         // Shuffle before compression. Applied if explicitly requested OR if compression
         // is active and the caller hasn't disabled it — matches h5py default behavior
@@ -113,8 +171,11 @@ impl ChunkOptions {
             });
         }
 
-        // Compression filters (mutually exclusive, priority: pcodec > zstd > lz4 > deflate)
-        if self.pcodec {
+        // Compression filters (mutually exclusive, priority: plugin > pcodec >
+        // zstd > lz4 > deflate)
+        if let Some(plugin) = &self.plugin {
+            filters.push(plugin.description(element_size, chunk_bytes));
+        } else if self.pcodec {
             filters.push(FilterDescription {
                 filter_id: FILTER_PCODEC,
                 name: Some(FILTER_PCODEC_NAME.into()),
@@ -675,7 +736,12 @@ pub fn precompress_chunks(
     element_size: usize,
     options: &ChunkOptions,
 ) -> Result<PrecompressedChunks, FormatError> {
-    let pipeline = options.build_pipeline(element_size as u32);
+    let chunk_bytes = chunk_dims
+        .iter()
+        .try_fold(element_size as u64, |acc, &d| acc.checked_mul(d))
+        .and_then(|b| u32::try_from(b).ok())
+        .unwrap_or(0);
+    let pipeline = options.build_pipeline_for_chunk(element_size as u32, chunk_bytes);
     let has_filters = pipeline.is_some();
     let pipeline_message = pipeline.as_ref().map(|pl| pl.serialize());
 
@@ -1569,6 +1635,20 @@ mod tests {
         assert_eq!(pl.filters[1].client_data, vec![3]);
     }
 
+    #[test]
+    fn chunk_options_pipeline_lzf() {
+        let options = ChunkOptions {
+            plugin: Some(PluginFilter::Lzf),
+            ..Default::default()
+        };
+        assert!(options.is_chunked());
+        let pl = options.build_pipeline_for_chunk(8, 800).unwrap();
+        assert_eq!(pl.filters.len(), 2);
+        assert_eq!(pl.filters[0].filter_id, FILTER_SHUFFLE);
+        assert_eq!(pl.filters[1].filter_id, FILTER_LZF);
+        assert_eq!(pl.filters[1].client_data, vec![4, 0x0105, 800]);
+    }
+
     #[test]
     fn chunk_options_zstd_priority_over_deflate() {
         let options = ChunkOptions {