feat(format): bitshuffle filter (32008), with its LZ4 and Zstandard modes

hdf5plugin's Bitshuffle failed with UnsupportedFilter(32008). The new
`bitshuffle` feature (pure Rust: lz4_flex, and ruzstd for Zstandard — the
`zstd` feature's libzstd is not needed) decodes all three modes of
bshuf_h5filter.c — transpose only, LZ4 and Zstandard blocks behind the
12-byte header — including the default and explicit block sizes, the
shorter last block rounded down to a multiple of 8 elements, and the
untransposed trailing elements. Sizes read from the chunk are bounded by
the chunk size.

It also encodes: DatasetBuilder::with_bitshuffle(BitshuffleCompression)
or PluginFilter::Bitshuffle { block_size, compression } writes the filter
with hdf5plugin's cd_values and no automatic byte shuffle. ruzstd has one
compression level (about zstd's 1); the requested level is recorded.

The bit transpose is checked bit for bit against a one-bit-at-a-time model
(which matched hdf5plugin's output) and is shared with blosc next.
Interop: hdf5plugin writes none/LZ4/Zstandard at default and explicit
block sizes and levels over the 12-case matrix, read byte for byte; our
three modes at two block sizes read back through hdf5plugin. Both fail
with the decoder removed.

Co-Authored-By: Claude Opus 5.5 (1M context) <[email protected]>

crates/clawhdf5-format/src/chunked_write.rs

@@ -12,8 +12,8 @@ use crate::chunk_grid::ChunkGrid;
 use crate::ea_writer;
 use crate::error::FormatError;
 use crate::filter_pipeline::{
-    FILTER_DEFLATE, FILTER_FLETCHER32, FILTER_LZ4, FILTER_LZF, FILTER_PCODEC, FILTER_PCODEC_NAME,
-    FILTER_SHUFFLE, FILTER_ZSTD, FilterDescription, FilterPipeline,
+    FILTER_BITSHUFFLE, FILTER_DEFLATE, FILTER_FLETCHER32, FILTER_LZ4, FILTER_LZF, FILTER_PCODEC,
+    FILTER_PCODEC_NAME, FILTER_SHUFFLE, FILTER_ZSTD, FilterDescription, FilterPipeline,
 };
 use crate::filters::compress_chunk;
 /// Round a file offset up to the next cache-line boundary.
@@ -61,6 +61,30 @@ pub enum PluginFilter {
     /// LZF (filter 32000), h5py's built-in `compression="lzf"`. Needs the
     /// `lzf` feature.
     Lzf,
+    /// Bitshuffle (filter 32008): a bit transpose of each block of
+    /// `block_size` elements (0 = bitshuffle's default, else a multiple of
+    /// 8), optionally compressed. Needs the `bitshuffle` feature.
+    Bitshuffle {
+        /// Block size in elements; 0 for the default.
+        block_size: u32,
+        /// Compression after the transpose.
+        compression: BitshuffleCompression,
+    },
+}
+
+/// What bitshuffle compresses its blocks with.
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub enum BitshuffleCompression {
+    /// Transpose only.
+    None,
+    /// LZ4 (bitshuffle's `cname="lz4"`, the common choice).
+    Lz4,
+    /// Zstandard. clawhdf5's pure-Rust encoder has a single level (about
+    /// zstd's level 1); `level` is recorded in the file for other writers.
+    Zstd {
+        /// Level recorded in `cd_values[5]`.
+        level: u32,
+    },
 }
 
 impl PluginFilter {
@@ -69,12 +93,13 @@ impl PluginFilter {
     fn shuffles_itself(&self) -> bool {
         match self {
             PluginFilter::Lzf => false,
+            PluginFilter::Bitshuffle { .. } => true,
         }
     }
 
     /// The pipeline entry for this filter. `chunk_bytes` is one chunk's
     /// uncompressed size (0 if unknown).
-    fn description(&self, _element_size: u32, chunk_bytes: u32) -> FilterDescription {
+    fn description(&self, element_size: u32, chunk_bytes: u32) -> FilterDescription {
         match self {
             // h5py's lzf_set_local: filter version, liblzf version, chunk
             // size in bytes. Optional, as h5py flags it: a chunk the filter
@@ -85,6 +110,25 @@ impl PluginFilter {
                 flags: 1,
                 client_data: vec![4, 0x0105, chunk_bytes],
             },
+            // bshuf_h5_set_local: version 0.4, element size, block size,
+            // compression (0 none, 2 LZ4, 3 Zstandard), Zstandard level.
+            PluginFilter::Bitshuffle {
+                block_size,
+                compression,
+            } => {
+                let mut cd = vec![0, 4, element_size, *block_size];
+                match compression {
+                    BitshuffleCompression::None => cd.push(0),
+                    BitshuffleCompression::Lz4 => cd.push(2),
+                    BitshuffleCompression::Zstd { level } => cd.extend([3, *level]),
+                }
+                FilterDescription {
+                    filter_id: FILTER_BITSHUFFLE,
+                    name: Some("bitshuffle; see https://github.com/kiyo-masui/bitshuffle".into()),
+                    flags: 1,
+                    client_data: cd,
+                }
+            }
         }
     }
 }
@@ -1649,6 +1693,21 @@ mod tests {
         assert_eq!(pl.filters[1].client_data, vec![4, 0x0105, 800]);
     }
 
+    #[test]
+    fn chunk_options_pipeline_bitshuffle_has_no_auto_shuffle() {
+        let options = ChunkOptions {
+            plugin: Some(PluginFilter::Bitshuffle {
+                block_size: 0,
+                compression: BitshuffleCompression::Zstd { level: 5 },
+            }),
+            ..Default::default()
+        };
+        let pl = options.build_pipeline(4).unwrap();
+        assert_eq!(pl.filters.len(), 1);
+        assert_eq!(pl.filters[0].filter_id, FILTER_BITSHUFFLE);
+        assert_eq!(pl.filters[0].client_data, vec![0, 4, 4, 0, 3, 5]);
+    }
+
     #[test]
     fn chunk_options_zstd_priority_over_deflate() {
         let options = ChunkOptions {